Google Git Google Git
Sign in
llvm/llvm-project/792bf8ef030d7c2aa12abbab7662ffbcbe2d8b21/./mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
blob: c0019a278b4a6dc7ba968506ac65c538abf6e453 [file]
//===- ModuleTranslation.cpp - MLIR to LLVM conversion --------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the translation between an MLIR LLVM dialect module and
// the corresponding LLVMIR module. It only handles core LLVM IR operations.
//
//===----------------------------------------------------------------------===//
#include "mlir/Target/LLVMIR/ModuleTranslation.h"
#include "AttrKindDetail.h"
#include "DebugTranslation.h"
#include "LoopAnnotationTranslation.h"
#include "mlir/Analysis/TopologicalSortUtils.h"
#include "mlir/Dialect/DLTI/DLTI.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/LLVMIR/LLVMInterfaces.h"
#include "mlir/Dialect/LLVMIR/Transforms/DIExpressionLegalization.h"
#include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h"
#include "mlir/IR/AttrTypeSubElements.h"
#include "mlir/IR/Attributes.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/DialectResourceBlobManager.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Target/LLVMIR/LLVMTranslationInterface.h"
#include "mlir/Target/LLVMIR/TypeToLLVM.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Analysis/TargetFolder.h"
#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <numeric>
#include <optional>
#define DEBUG_TYPE "llvm-dialect-to-llvm-ir"
using namespace mlir;
using namespace mlir::LLVM;
using namespace mlir::LLVM::detail;
#include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc"
namespace {
/// A customized inserter for LLVM's IRBuilder that captures all LLVM IR
/// instructions that are created for future reference.
///
/// This is intended to be used with the `CollectionScope` RAII object:
///
/// llvm::IRBuilder<..., InstructionCapturingInserter> builder;
/// {
/// InstructionCapturingInserter::CollectionScope scope(builder);
/// // Call IRBuilder methods as usual.
///
/// // This will return a list of all instructions created by the builder,
/// // in order of creation.
/// builder.getInserter().getCapturedInstructions();
/// }
/// // This will return an empty list.
/// builder.getInserter().getCapturedInstructions();
///
/// The capturing functionality is _disabled_ by default for performance
/// consideration. It needs to be explicitly enabled, which is achieved by
/// creating a `CollectionScope`.
class InstructionCapturingInserter : public llvm::IRBuilderCallbackInserter {
public:
/// Constructs the inserter.
InstructionCapturingInserter()
: llvm::IRBuilderCallbackInserter([this](llvm::Instruction *instruction) {
if (LLVM_LIKELY(enabled))
capturedInstructions.push_back(instruction);
}) {}
/// Returns the list of LLVM IR instructions captured since the last cleanup.
ArrayRef<llvm::Instruction *> getCapturedInstructions() const {
return capturedInstructions;
}
/// Clears the list of captured LLVM IR instructions.
void clearCapturedInstructions() { capturedInstructions.clear(); }
/// RAII object enabling the capture of created LLVM IR instructions.
class CollectionScope {
public:
/// Creates the scope for the given inserter.
CollectionScope(llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing);
/// Ends the scope.
~CollectionScope();
ArrayRef<llvm::Instruction *> getCapturedInstructions() {
if (!inserter)
return {};
return inserter->getCapturedInstructions();
}
private:
/// Back reference to the inserter.
InstructionCapturingInserter *inserter = nullptr;
/// List of instructions in the inserter prior to this scope.
SmallVector<llvm::Instruction *> previouslyCollectedInstructions;
/// Whether the inserter was enabled prior to this scope.
bool wasEnabled;
};
/// Enable or disable the capturing mechanism.
void setEnabled(bool enabled = true) { this->enabled = enabled; }
private:
/// List of captured instructions.
SmallVector<llvm::Instruction *> capturedInstructions;
/// Whether the collection is enabled.
bool enabled = false;
};
using CapturingIRBuilder =
llvm::IRBuilder<llvm::TargetFolder, InstructionCapturingInserter>;
} // namespace
InstructionCapturingInserter::CollectionScope::CollectionScope(
llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing) {
if (!isBuilderCapturing)
return;
auto &capturingIRBuilder = static_cast<CapturingIRBuilder &>(irBuilder);
inserter = &capturingIRBuilder.getInserter();
wasEnabled = inserter->enabled;
if (wasEnabled)
previouslyCollectedInstructions.swap(inserter->capturedInstructions);
inserter->setEnabled(true);
}
InstructionCapturingInserter::CollectionScope::~CollectionScope() {
if (!inserter)
return;
previouslyCollectedInstructions.swap(inserter->capturedInstructions);
// If collection was enabled (likely in another, surrounding scope), keep
// the instructions collected in this scope.
if (wasEnabled) {
llvm::append_range(inserter->capturedInstructions,
previouslyCollectedInstructions);
}
inserter->setEnabled(wasEnabled);
}
/// Translates the given data layout spec attribute to the LLVM IR data layout.
/// Only integer, float, pointer and endianness entries are currently supported.
static FailureOr<llvm::DataLayout>
translateDataLayout(DataLayoutSpecInterface attribute,
const DataLayout &dataLayout,
std::optional<Location> loc = std::nullopt) {
if (!loc)
loc = UnknownLoc::get(attribute.getContext());
// Translate the endianness attribute.
std::string llvmDataLayout;
llvm::raw_string_ostream layoutStream(llvmDataLayout);
for (DataLayoutEntryInterface entry : attribute.getEntries()) {
auto key = llvm::dyn_cast_if_present<StringAttr>(entry.getKey());
if (!key)
continue;
if (key.getValue() == DLTIDialect::kDataLayoutEndiannessKey) {
auto value = cast<StringAttr>(entry.getValue());
bool isLittleEndian =
value.getValue() == DLTIDialect::kDataLayoutEndiannessLittle;
layoutStream << "-" << (isLittleEndian ? "e" : "E");
continue;
}
if (key.getValue() == DLTIDialect::kDataLayoutManglingModeKey) {
auto value = cast<StringAttr>(entry.getValue());
layoutStream << "-m:" << value.getValue();
continue;
}
if (key.getValue() == DLTIDialect::kDataLayoutProgramMemorySpaceKey) {
auto value = cast<IntegerAttr>(entry.getValue());
uint64_t space = value.getValue().getZExtValue();
// Skip the default address space.
if (space == 0)
continue;
layoutStream << "-P" << space;
continue;
}
if (key.getValue() == DLTIDialect::kDataLayoutGlobalMemorySpaceKey) {
auto value = cast<IntegerAttr>(entry.getValue());
uint64_t space = value.getValue().getZExtValue();
// Skip the default address space.
if (space == 0)
continue;
layoutStream << "-G" << space;
continue;
}
if (key.getValue() == DLTIDialect::kDataLayoutAllocaMemorySpaceKey) {
auto value = cast<IntegerAttr>(entry.getValue());
uint64_t space = value.getValue().getZExtValue();
// Skip the default address space.
if (space == 0)
continue;
layoutStream << "-A" << space;
continue;
}
if (key.getValue() == DLTIDialect::kDataLayoutStackAlignmentKey) {
auto value = cast<IntegerAttr>(entry.getValue());
uint64_t alignment = value.getValue().getZExtValue();
// Skip the default stack alignment.
if (alignment == 0)
continue;
layoutStream << "-S" << alignment;
continue;
}
if (key.getValue() == DLTIDialect::kDataLayoutFunctionPointerAlignmentKey) {
auto value = cast<FunctionPointerAlignmentAttr>(entry.getValue());
uint64_t alignment = value.getAlignment();
// Skip the default function pointer alignment.
if (alignment == 0)
continue;
layoutStream << "-F" << (value.getFunctionDependent() ? "n" : "i")
<< alignment;
continue;
}
if (key.getValue() == DLTIDialect::kDataLayoutLegalIntWidthsKey) {
layoutStream << "-n";
llvm::interleave(
cast<DenseI32ArrayAttr>(entry.getValue()).asArrayRef(), layoutStream,
[&](int32_t val) { layoutStream << val; }, ":");
continue;
}
emitError(*loc) << "unsupported data layout key " << key;
return failure();
}
// Go through the list of entries to check which types are explicitly
// specified in entries. Where possible, data layout queries are used instead
// of directly inspecting the entries.
for (DataLayoutEntryInterface entry : attribute.getEntries()) {
auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());
if (!type)
continue;
// Data layout for the index type is irrelevant at this point.
if (isa<IndexType>(type))
continue;
layoutStream << "-";
LogicalResult result =
llvm::TypeSwitch<Type, LogicalResult>(type)
.Case<IntegerType, Float16Type, Float32Type, Float64Type,
Float80Type, Float128Type>([&](Type type) -> LogicalResult {
if (auto intType = dyn_cast<IntegerType>(type)) {
if (intType.getSignedness() != IntegerType::Signless)
return emitError(*loc)
<< "unsupported data layout for non-signless integer "
<< intType;
layoutStream << "i";
} else {
layoutStream << "f";
}
uint64_t size = dataLayout.getTypeSizeInBits(type);
uint64_t abi = dataLayout.getTypeABIAlignment(type) * 8u;
uint64_t preferred =
dataLayout.getTypePreferredAlignment(type) * 8u;
layoutStream << size << ":" << abi;
if (abi != preferred)
layoutStream << ":" << preferred;
return success();
})
.Case([&](LLVMPointerType type) {
layoutStream << "p" << type.getAddressSpace() << ":";
uint64_t size = dataLayout.getTypeSizeInBits(type);
uint64_t abi = dataLayout.getTypeABIAlignment(type) * 8u;
uint64_t preferred =
dataLayout.getTypePreferredAlignment(type) * 8u;
uint64_t index = *dataLayout.getTypeIndexBitwidth(type);
layoutStream << size << ":" << abi << ":" << preferred << ":"
<< index;
return success();
})
.Default([loc](Type type) {
return emitError(*loc)
<< "unsupported type in data layout: " << type;
});
if (failed(result))
return failure();
}
StringRef layoutSpec(llvmDataLayout);
layoutSpec.consume_front("-");
return llvm::DataLayout(layoutSpec);
}
/// Builds a constant of a sequential LLVM type `type`, potentially containing
/// other sequential types recursively, from the individual constant values
/// provided in `constants`. `shape` contains the number of elements in nested
/// sequential types. Reports errors at `loc` and returns nullptr on error.
static llvm::Constant *
buildSequentialConstant(ArrayRef<llvm::Constant *> &constants,
ArrayRef<int64_t> shape, llvm::Type *type,
Location loc) {
if (shape.empty()) {
llvm::Constant *result = constants.front();
constants = constants.drop_front();
return result;
}
llvm::Type *elementType;
if (auto *arrayTy = dyn_cast<llvm::ArrayType>(type)) {
elementType = arrayTy->getElementType();
} else if (auto *vectorTy = dyn_cast<llvm::VectorType>(type)) {
elementType = vectorTy->getElementType();
} else {
emitError(loc) << "expected sequential LLVM types wrapping a scalar";
return nullptr;
}
SmallVector<llvm::Constant *, 8> nested;
nested.reserve(shape.front());
for (int64_t i = 0; i < shape.front(); ++i) {
nested.push_back(buildSequentialConstant(constants, shape.drop_front(),
elementType, loc));
if (!nested.back())
return nullptr;
}
if (shape.size() == 1 && type->isVectorTy())
return llvm::ConstantVector::get(nested);
return llvm::ConstantArray::get(
llvm::ArrayType::get(elementType, shape.front()), nested);
}
/// Returns the first non-sequential type nested in sequential types.
static llvm::Type *getInnermostElementType(llvm::Type *type) {
do {
if (auto *arrayTy = dyn_cast<llvm::ArrayType>(type)) {
type = arrayTy->getElementType();
} else if (auto *vectorTy = dyn_cast<llvm::VectorType>(type)) {
type = vectorTy->getElementType();
} else {
return type;
}
} while (true);
}
/// Convert a dense elements attribute to an LLVM IR constant using its raw data
/// storage if possible. This supports elements attributes of tensor or vector
/// type and avoids constructing separate objects for individual values of the
/// innermost dimension. Constants for other dimensions are still constructed
/// recursively. Returns null if constructing from raw data is not supported for
/// this type, e.g., element type is not a power-of-two-sized primitive. Reports
/// other errors at `loc`.
static llvm::Constant *
convertDenseElementsAttr(Location loc, DenseElementsAttr denseElementsAttr,
llvm::Type *llvmType,
const ModuleTranslation &moduleTranslation) {
if (!denseElementsAttr)
return nullptr;
llvm::Type *innermostLLVMType = getInnermostElementType(llvmType);
if (!llvm::ConstantDataSequential::isElementTypeCompatible(innermostLLVMType))
return nullptr;
ShapedType type = denseElementsAttr.getType();
if (type.getNumElements() == 0)
return nullptr;
// Check that the raw data size matches what is expected for the scalar size.
// TODO: in theory, we could repack the data here to keep constructing from
// raw data.
// TODO: we may also need to consider endianness when cross-compiling to an
// architecture where it is different.
int64_t elementByteSize = denseElementsAttr.getRawData().size() /
denseElementsAttr.getNumElements();
if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits())
return nullptr;
// Compute the shape of all dimensions but the innermost. Note that the
// innermost dimension may be that of the vector element type.
bool hasVectorElementType = isa<VectorType>(type.getElementType());
int64_t numAggregates =
denseElementsAttr.getNumElements() /
(hasVectorElementType ? 1
: denseElementsAttr.getType().getShape().back());
ArrayRef<int64_t> outerShape = type.getShape();
if (!hasVectorElementType)
outerShape = outerShape.drop_back();
// Handle the case of vector splat, LLVM has special support for it.
if (denseElementsAttr.isSplat() &&
(isa<VectorType>(type) || hasVectorElementType)) {
llvm::Constant *splatValue = LLVM::detail::getLLVMConstant(
innermostLLVMType, denseElementsAttr.getSplatValue<Attribute>(), loc,
moduleTranslation);
llvm::Constant *splatVector =
llvm::ConstantDataVector::getSplat(0, splatValue);
SmallVector<llvm::Constant *> constants(numAggregates, splatVector);
ArrayRef<llvm::Constant *> constantsRef = constants;
return buildSequentialConstant(constantsRef, outerShape, llvmType, loc);
}
if (denseElementsAttr.isSplat())
return nullptr;
// In case of non-splat, create a constructor for the innermost constant from
// a piece of raw data.
std::function<llvm::Constant *(StringRef)> buildCstData;
if (isa<TensorType>(type)) {
auto vectorElementType = dyn_cast<VectorType>(type.getElementType());
if (vectorElementType && vectorElementType.getRank() == 1) {
buildCstData = [&](StringRef data) {
return llvm::ConstantDataVector::getRaw(
data, vectorElementType.getShape().back(), innermostLLVMType);
};
} else if (!vectorElementType) {
buildCstData = [&](StringRef data) {
return llvm::ConstantDataArray::getRaw(data, type.getShape().back(),
innermostLLVMType);
};
}
} else if (isa<VectorType>(type)) {
buildCstData = [&](StringRef data) {
return llvm::ConstantDataVector::getRaw(data, type.getShape().back(),
innermostLLVMType);
};
}
if (!buildCstData)
return nullptr;
// Create innermost constants and defer to the default constant creation
// mechanism for other dimensions.
SmallVector<llvm::Constant *> constants;
int64_t aggregateSize = denseElementsAttr.getType().getShape().back() *
(innermostLLVMType->getScalarSizeInBits() / 8);
constants.reserve(numAggregates);
for (unsigned i = 0; i < numAggregates; ++i) {
StringRef data(denseElementsAttr.getRawData().data() + i * aggregateSize,
aggregateSize);
constants.push_back(buildCstData(data));
}
ArrayRef<llvm::Constant *> constantsRef = constants;
return buildSequentialConstant(constantsRef, outerShape, llvmType, loc);
}
/// Convert a dense resource elements attribute to an LLVM IR constant using its
/// raw data storage if possible. This supports elements attributes of tensor or
/// vector type and avoids constructing separate objects for individual values
/// of the innermost dimension. Constants for other dimensions are still
/// constructed recursively. Returns nullptr on failure and emits errors at
/// `loc`.
static llvm::Constant *convertDenseResourceElementsAttr(
Location loc, DenseResourceElementsAttr denseResourceAttr,
llvm::Type *llvmType, const ModuleTranslation &moduleTranslation) {
assert(denseResourceAttr && "expected non-null attribute");
llvm::Type *innermostLLVMType = getInnermostElementType(llvmType);
if (!llvm::ConstantDataSequential::isElementTypeCompatible(
innermostLLVMType)) {
emitError(loc, "no known conversion for innermost element type");
return nullptr;
}
ShapedType type = denseResourceAttr.getType();
assert(type.getNumElements() > 0 && "Expected non-empty elements attribute");
AsmResourceBlob *blob = denseResourceAttr.getRawHandle().getBlob();
if (!blob) {
emitError(loc, "resource does not exist");
return nullptr;
}
ArrayRef<char> rawData = blob->getData();
// Check that the raw data size matches what is expected for the scalar size.
// TODO: in theory, we could repack the data here to keep constructing from
// raw data.
// TODO: we may also need to consider endianness when cross-compiling to an
// architecture where it is different.
int64_t numElements = denseResourceAttr.getType().getNumElements();
int64_t elementByteSize = rawData.size() / numElements;
if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) {
emitError(loc, "raw data size does not match element type size");
return nullptr;
}
// Compute the shape of all dimensions but the innermost. Note that the
// innermost dimension may be that of the vector element type.
bool hasVectorElementType = isa<VectorType>(type.getElementType());
int64_t numAggregates =
numElements / (hasVectorElementType
? 1
: denseResourceAttr.getType().getShape().back());
ArrayRef<int64_t> outerShape = type.getShape();
if (!hasVectorElementType)
outerShape = outerShape.drop_back();
// Create a constructor for the innermost constant from a piece of raw data.
std::function<llvm::Constant *(StringRef)> buildCstData;
if (isa<TensorType>(type)) {
auto vectorElementType = dyn_cast<VectorType>(type.getElementType());
if (vectorElementType && vectorElementType.getRank() == 1) {
buildCstData = [&](StringRef data) {
return llvm::ConstantDataVector::getRaw(
data, vectorElementType.getShape().back(), innermostLLVMType);
};
} else if (!vectorElementType) {
buildCstData = [&](StringRef data) {
return llvm::ConstantDataArray::getRaw(data, type.getShape().back(),
innermostLLVMType);
};
}
} else if (isa<VectorType>(type)) {
buildCstData = [&](StringRef data) {
return llvm::ConstantDataVector::getRaw(data, type.getShape().back(),
innermostLLVMType);
};
}
if (!buildCstData) {
emitError(loc, "unsupported dense_resource type");
return nullptr;
}
// Create innermost constants and defer to the default constant creation
// mechanism for other dimensions.
SmallVector<llvm::Constant *> constants;
int64_t aggregateSize = denseResourceAttr.getType().getShape().back() *
(innermostLLVMType->getScalarSizeInBits() / 8);
constants.reserve(numAggregates);
for (unsigned i = 0; i < numAggregates; ++i) {
StringRef data(rawData.data() + i * aggregateSize, aggregateSize);
constants.push_back(buildCstData(data));
}
ArrayRef<llvm::Constant *> constantsRef = constants;
return buildSequentialConstant(constantsRef, outerShape, llvmType, loc);
}
/// Create an LLVM IR constant of `llvmType` from the MLIR attribute `attr`.
/// This currently supports integer, floating point, splat and dense element
/// attributes and combinations thereof. Also, an array attribute with two
/// elements is supported to represent a complex constant. In case of error,
/// report it to `loc` and return nullptr.
llvm::Constant *mlir::LLVM::detail::getLLVMConstant(
llvm::Type *llvmType, Attribute attr, Location loc,
const ModuleTranslation &moduleTranslation) {
if (!attr || isa<UndefAttr>(attr))
return llvm::UndefValue::get(llvmType);
if (isa<ZeroAttr>(attr))
return llvm::Constant::getNullValue(llvmType);
if (auto *structType = dyn_cast<::llvm::StructType>(llvmType)) {
auto arrayAttr = dyn_cast<ArrayAttr>(attr);
if (!arrayAttr) {
emitError(loc, "expected an array attribute for a struct constant");
return nullptr;
}
SmallVector<llvm::Constant *> structElements;
structElements.reserve(structType->getNumElements());
for (auto [elemType, elemAttr] :
zip_equal(structType->elements(), arrayAttr)) {
llvm::Constant *element =
getLLVMConstant(elemType, elemAttr, loc, moduleTranslation);
if (!element)
return nullptr;
structElements.push_back(element);
}
return llvm::ConstantStruct::get(structType, structElements);
}
// For integer types, we allow a mismatch in sizes as the index type in
// MLIR might have a different size than the index type in the LLVM module.
if (auto intAttr = dyn_cast<IntegerAttr>(attr)) {
// If the attribute is an unsigned integer or a 1-bit integer, zero-extend
// the value to the bit width of the LLVM type. Otherwise, sign-extend.
auto intTy = dyn_cast<IntegerType>(intAttr.getType());
APInt value;
if (intTy && (intTy.isUnsigned() || intTy.getWidth() == 1))
value = intAttr.getValue().zextOrTrunc(llvmType->getIntegerBitWidth());
else
value = intAttr.getValue().sextOrTrunc(llvmType->getIntegerBitWidth());
return llvm::ConstantInt::get(llvmType, value);
}
if (auto floatAttr = dyn_cast<FloatAttr>(attr)) {
const llvm::fltSemantics &sem = floatAttr.getValue().getSemantics();
// Special case for 8-bit floats, which are represented by integers due to
// the lack of native fp8 types in LLVM at the moment. Additionally, handle
// targets (like AMDGPU) that don't implement bfloat and convert all bfloats
// to i16.
unsigned floatWidth = APFloat::getSizeInBits(sem);
if (llvmType->isIntegerTy(floatWidth))
return llvm::ConstantInt::get(llvmType,
floatAttr.getValue().bitcastToAPInt());
if (llvmType !=
llvm::Type::getFloatingPointTy(llvmType->getContext(),
floatAttr.getValue().getSemantics())) {
emitError(loc, "FloatAttr does not match expected type of the constant");
return nullptr;
}
return llvm::ConstantFP::get(llvmType, floatAttr.getValue());
}
if (auto symAttr = dyn_cast<FlatSymbolRefAttr>(attr)) {
StringRef name = symAttr.getValue();
if (llvm::Function *func = moduleTranslation.lookupFunction(name))
return llvm::ConstantExpr::getBitCast(func, llvmType);
if (llvm::GlobalValue *global = moduleTranslation.lookupGlobal(name))
return llvm::ConstantExpr::getBitCast(global, llvmType);
emitError(loc, "unknown symbol reference '") << name << "' in constant";
return nullptr;
}
if (auto splatAttr = dyn_cast<SplatElementsAttr>(attr)) {
llvm::Type *elementType;
uint64_t numElements;
bool isScalable = false;
if (auto *arrayTy = dyn_cast<llvm::ArrayType>(llvmType)) {
elementType = arrayTy->getElementType();
numElements = arrayTy->getNumElements();
} else if (auto *fVectorTy = dyn_cast<llvm::FixedVectorType>(llvmType)) {
elementType = fVectorTy->getElementType();
numElements = fVectorTy->getNumElements();
} else if (auto *sVectorTy = dyn_cast<llvm::ScalableVectorType>(llvmType)) {
elementType = sVectorTy->getElementType();
numElements = sVectorTy->getMinNumElements();
isScalable = true;
} else {
llvm_unreachable("unrecognized constant vector type");
}
// Splat value is a scalar. Extract it only if the element type is not
// another sequence type. The recursion terminates because each step removes
// one outer sequential type.
bool elementTypeSequential =
isa<llvm::ArrayType, llvm::VectorType>(elementType);
llvm::Constant *child = getLLVMConstant(
elementType,
elementTypeSequential ? splatAttr
: splatAttr.getSplatValue<Attribute>(),
loc, moduleTranslation);
if (!child)
return nullptr;
if (llvmType->isVectorTy())
return llvm::ConstantVector::getSplat(
llvm::ElementCount::get(numElements, /*Scalable=*/isScalable), child);
if (llvmType->isArrayTy()) {
auto *arrayType = llvm::ArrayType::get(elementType, numElements);
if (child->isNullValue() && !elementType->isFPOrFPVectorTy()) {
return llvm::ConstantAggregateZero::get(arrayType);
}
if (llvm::ConstantDataSequential::isElementTypeCompatible(elementType)) {
if (isa<llvm::IntegerType>(elementType)) {
if (llvm::ConstantInt *ci = dyn_cast<llvm::ConstantInt>(child)) {
if (ci->getBitWidth() == 8) {
SmallVector<int8_t> constants(numElements, ci->getZExtValue());
return llvm::ConstantDataArray::get(elementType->getContext(),
constants);
}
if (ci->getBitWidth() == 16) {
SmallVector<int16_t> constants(numElements, ci->getZExtValue());
return llvm::ConstantDataArray::get(elementType->getContext(),
constants);
}
if (ci->getBitWidth() == 32) {
SmallVector<int32_t> constants(numElements, ci->getZExtValue());
return llvm::ConstantDataArray::get(elementType->getContext(),
constants);
}
if (ci->getBitWidth() == 64) {
SmallVector<int64_t> constants(numElements, ci->getZExtValue());
return llvm::ConstantDataArray::get(elementType->getContext(),
constants);
}
}
}
if (elementType->isFloatingPointTy()) {
if (llvm::ConstantFP *cfp = dyn_cast<llvm::ConstantFP>(child)) {
APInt bitPattern = cfp->getValueAPF().bitcastToAPInt();
uint64_t value = bitPattern.getZExtValue();
// TODO: This code only handles 16, 32, and 64 bit floats. Handle
// all compatible types, fp8, fp4, etc.
if (bitPattern.getBitWidth() == 16) {
SmallVector<uint16_t> constants(numElements, value);
return llvm::ConstantDataArray::getFP(elementType, constants);
}
if (bitPattern.getBitWidth() == 32) {
SmallVector<uint32_t> constants(numElements, value);
return llvm::ConstantDataArray::getFP(elementType, constants);
}
if (bitPattern.getBitWidth() == 64) {
SmallVector<uint64_t> constants(numElements, value);
return llvm::ConstantDataArray::getFP(elementType, constants);
}
}
}
}
// std::vector is used here to accomodate large number of elements that
// exceed SmallVector capacity.
std::vector<llvm::Constant *> constants(numElements, child);
return llvm::ConstantArray::get(arrayType, constants);
}
}
// Try using raw elements data if possible.
if (llvm::Constant *result =
convertDenseElementsAttr(loc, dyn_cast<DenseElementsAttr>(attr),
llvmType, moduleTranslation)) {
return result;
}
if (auto denseResourceAttr = dyn_cast<DenseResourceElementsAttr>(attr)) {
return convertDenseResourceElementsAttr(loc, denseResourceAttr, llvmType,
moduleTranslation);
}
// Fall back to element-by-element construction otherwise.
if (auto elementsAttr = dyn_cast<ElementsAttr>(attr)) {
assert(elementsAttr.getShapedType().hasStaticShape());
assert(!elementsAttr.getShapedType().getShape().empty() &&
"unexpected empty elements attribute shape");
SmallVector<llvm::Constant *, 8> constants;
constants.reserve(elementsAttr.getNumElements());
llvm::Type *innermostType = getInnermostElementType(llvmType);
for (auto n : elementsAttr.getValues<Attribute>()) {
constants.push_back(
getLLVMConstant(innermostType, n, loc, moduleTranslation));
if (!constants.back())
return nullptr;
}
ArrayRef<llvm::Constant *> constantsRef = constants;
llvm::Constant *result = buildSequentialConstant(
constantsRef, elementsAttr.getShapedType().getShape(), llvmType, loc);
assert(constantsRef.empty() && "did not consume all elemental constants");
return result;
}
if (auto stringAttr = dyn_cast<StringAttr>(attr)) {
return llvm::ConstantDataArray::get(moduleTranslation.getLLVMContext(),
ArrayRef<char>{stringAttr.getValue()});
}
// Handle arrays of structs that cannot be represented as DenseElementsAttr
// in MLIR.
if (auto arrayAttr = dyn_cast<ArrayAttr>(attr)) {
if (auto *arrayTy = dyn_cast<llvm::ArrayType>(llvmType)) {
llvm::Type *elementType = arrayTy->getElementType();
Attribute previousElementAttr;
llvm::Constant *elementCst = nullptr;
SmallVector<llvm::Constant *> constants;
constants.reserve(arrayTy->getNumElements());
for (Attribute elementAttr : arrayAttr) {
// Arrays with a single value or with repeating values are quite common.
// Short-circuit the translation when the element value is the same as
// the previous one.
if (!previousElementAttr || previousElementAttr != elementAttr) {
previousElementAttr = elementAttr;
elementCst =
getLLVMConstant(elementType, elementAttr, loc, moduleTranslation);
if (!elementCst)
return nullptr;
}
constants.push_back(elementCst);
}
return llvm::ConstantArray::get(arrayTy, constants);
}
}
emitError(loc, "unsupported constant value");
return nullptr;
}
ModuleTranslation::ModuleTranslation(Operation *module,
std::unique_ptr<llvm::Module> llvmModule,
llvm::vfs::FileSystem *fs)
: mlirModule(module), llvmModule(std::move(llvmModule)),
debugTranslation(
std::make_unique<DebugTranslation>(module, *this->llvmModule)),
loopAnnotationTranslation(std::make_unique<LoopAnnotationTranslation>(
*this, *this->llvmModule)),
fileSystem(fs), typeTranslator(this->llvmModule->getContext()),
iface(module->getContext()) {
assert(satisfiesLLVMModule(mlirModule) &&
"mlirModule should honor LLVM's module semantics.");
}
ModuleTranslation::~ModuleTranslation() {
if (ompBuilder && !ompBuilder->isFinalized())
ompBuilder->finalize();
}
void ModuleTranslation::forgetMapping(Region &region) {
SmallVector<Region *> toProcess;
toProcess.push_back(&region);
while (!toProcess.empty()) {
Region *current = toProcess.pop_back_val();
for (Block &block : *current) {
blockMapping.erase(&block);
for (Value arg : block.getArguments())
valueMapping.erase(arg);
for (Operation &op : block) {
for (Value value : op.getResults())
valueMapping.erase(value);
if (op.hasSuccessors())
branchMapping.erase(&op);
if (isa<LLVM::GlobalOp>(op))
globalsMapping.erase(&op);
if (isa<LLVM::AliasOp>(op))
aliasesMapping.erase(&op);
if (isa<LLVM::IFuncOp>(op))
ifuncMapping.erase(&op);
if (isa<LLVM::CallOp>(op))
callMapping.erase(&op);
llvm::append_range(
toProcess,
llvm::map_range(op.getRegions(), [](Region &r) { return &r; }));
}
}
}
}
/// Get the SSA value passed to the current block from the terminator operation
/// of its predecessor.
static Value getPHISourceValue(Block *current, Block *pred,
unsigned numArguments, unsigned index) {
Operation &terminator = *pred->getTerminator();
if (isa<LLVM::BrOp>(terminator))
return terminator.getOperand(index);
#ifndef NDEBUG
llvm::SmallPtrSet<Block *, 4> seenSuccessors;
for (unsigned i = 0, e = terminator.getNumSuccessors(); i < e; ++i) {
Block *successor = terminator.getSuccessor(i);
auto branch = cast<BranchOpInterface>(terminator);
SuccessorOperands successorOperands = branch.getSuccessorOperands(i);
assert(
(!seenSuccessors.contains(successor) || successorOperands.empty()) &&
"successors with arguments in LLVM branches must be different blocks");
seenSuccessors.insert(successor);
}
#endif
// For instructions that branch based on a condition value, we need to take
// the operands for the branch that was taken.
if (auto condBranchOp = dyn_cast<LLVM::CondBrOp>(terminator)) {
// For conditional branches, we take the operands from either the "true" or
// the "false" branch.
return condBranchOp.getSuccessor(0) == current
? condBranchOp.getTrueDestOperands()[index]
: condBranchOp.getFalseDestOperands()[index];
}
if (auto switchOp = dyn_cast<LLVM::SwitchOp>(terminator)) {
// For switches, we take the operands from either the default case, or from
// the case branch that was taken.
if (switchOp.getDefaultDestination() == current)
return switchOp.getDefaultOperands()[index];
for (const auto &i : llvm::enumerate(switchOp.getCaseDestinations()))
if (i.value() == current)
return switchOp.getCaseOperands(i.index())[index];
}
if (auto indBrOp = dyn_cast<LLVM::IndirectBrOp>(terminator)) {
// For indirect branches we take operands for each successor.
for (const auto &i : llvm::enumerate(indBrOp->getSuccessors())) {
if (indBrOp->getSuccessor(i.index()) == current)
return indBrOp.getSuccessorOperands(i.index())[index];
}
}
if (auto invokeOp = dyn_cast<LLVM::InvokeOp>(terminator)) {
return invokeOp.getNormalDest() == current
? invokeOp.getNormalDestOperands()[index]
: invokeOp.getUnwindDestOperands()[index];
}
llvm_unreachable(
"only branch, switch or invoke operations can be terminators "
"of a block that has successors");
}
/// Connect the PHI nodes to the results of preceding blocks.
void mlir::LLVM::detail::connectPHINodes(Region &region,
const ModuleTranslation &state) {
// Skip the first block, it cannot be branched to and its arguments correspond
// to the arguments of the LLVM function.
for (Block &bb : llvm::drop_begin(region)) {
llvm::BasicBlock *llvmBB = state.lookupBlock(&bb);
auto phis = llvmBB->phis();
auto numArguments = bb.getNumArguments();
assert(numArguments == std::distance(phis.begin(), phis.end()));
for (auto [index, phiNode] : llvm::enumerate(phis)) {
for (auto *pred : bb.getPredecessors()) {
// Find the LLVM IR block that contains the converted terminator
// instruction and use it in the PHI node. Note that this block is not
// necessarily the same as state.lookupBlock(pred), some operations
// (in particular, OpenMP operations using OpenMPIRBuilder) may have
// split the blocks.
llvm::Instruction *terminator =
state.lookupBranch(pred->getTerminator());
assert(terminator && "missing the mapping for a terminator");
phiNode.addIncoming(state.lookupValue(getPHISourceValue(
&bb, pred, numArguments, index)),
terminator->getParent());
}
}
}
}
llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,
ArrayRef<llvm::Value *> args, ArrayRef<llvm::Type *> tys) {
return builder.CreateIntrinsicWithoutFolding(intrinsic, tys, args);
}
llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,
llvm::Type *retTy, ArrayRef<llvm::Value *> args) {
return builder.CreateIntrinsicWithoutFolding(retTy, intrinsic, args);
}
llvm::CallInst *mlir::LLVM::detail::createIntrinsicCall(
llvm::IRBuilderBase &builder, ModuleTranslation &moduleTranslation,
Operation *intrOp, llvm::Intrinsic::ID intrinsic, unsigned numResults,
ArrayRef<unsigned> overloadedResults, ArrayRef<unsigned> overloadedOperands,
ArrayRef<unsigned> immArgPositions,
ArrayRef<StringLiteral> immArgAttrNames) {
assert(immArgPositions.size() == immArgAttrNames.size() &&
"LLVM `immArgPositions` and MLIR `immArgAttrNames` should have equal "
"length");
SmallVector<llvm::OperandBundleDef> opBundles;
size_t numOpBundleOperands = 0;
auto opBundleSizesAttr = cast_if_present<DenseI32ArrayAttr>(
intrOp->getInherentAttr(LLVMDialect::getOpBundleSizesAttrName())
.value_or(Attribute{}));
auto opBundleTagsAttr = cast_if_present<ArrayAttr>(
intrOp->getInherentAttr(LLVMDialect::getOpBundleTagsAttrName())
.value_or(Attribute{}));
if (opBundleSizesAttr && opBundleTagsAttr) {
ArrayRef<int> opBundleSizes = opBundleSizesAttr.asArrayRef();
assert(opBundleSizes.size() == opBundleTagsAttr.size() &&
"operand bundles and tags do not match");
numOpBundleOperands = llvm::sum_of(opBundleSizes);
assert(numOpBundleOperands <= intrOp->getNumOperands() &&
"operand bundle operands is more than the number of operands");
ValueRange operands = intrOp->getOperands().take_back(numOpBundleOperands);
size_t nextOperandIdx = 0;
opBundles.reserve(opBundleSizesAttr.size());
for (auto [opBundleTagAttr, bundleSize] :
llvm::zip(opBundleTagsAttr, opBundleSizes)) {
auto bundleTag = cast<StringAttr>(opBundleTagAttr).str();
auto bundleOperands = moduleTranslation.lookupValues(
operands.slice(nextOperandIdx, bundleSize));
opBundles.emplace_back(std::move(bundleTag), std::move(bundleOperands));
nextOperandIdx += bundleSize;
}
}
// Map operands and attributes to LLVM values.
auto opOperands = intrOp->getOperands().drop_back(numOpBundleOperands);
auto operands = moduleTranslation.lookupValues(opOperands);
SmallVector<llvm::Value *> args(immArgPositions.size() + operands.size());
for (auto [immArgPos, immArgName] :
llvm::zip(immArgPositions, immArgAttrNames)) {
Attribute attr = intrOp->getInherentAttr(immArgName).value_or(Attribute{});
if (auto intrinsicIntegerAttr =
dyn_cast<LLVM::IntrinsicIntegerAttrInterface>(attr))
attr = intrinsicIntegerAttr.getIntegerAttr();
auto typedAttr = llvm::cast<TypedAttr>(attr);
assert(typedAttr.getType().isIntOrFloat() &&
"expected int or float immarg");
auto *type = moduleTranslation.convertType(typedAttr.getType());
args[immArgPos] = LLVM::detail::getLLVMConstant(
type, typedAttr, intrOp->getLoc(), moduleTranslation);
}
unsigned opArg = 0;
for (auto &arg : args) {
if (!arg)
arg = operands[opArg++];
}
// Resolve overloaded intrinsic declaration.
SmallVector<llvm::Type *> overloadedTypes;
for (unsigned overloadedResultIdx : overloadedResults) {
if (numResults > 1) {
// More than one result is mapped to an LLVM struct.
overloadedTypes.push_back(moduleTranslation.convertType(
llvm::cast<LLVM::LLVMStructType>(intrOp->getResult(0).getType())
.getBody()[overloadedResultIdx]));
} else {
overloadedTypes.push_back(
moduleTranslation.convertType(intrOp->getResult(0).getType()));
}
}
for (unsigned overloadedOperandIdx : overloadedOperands)
overloadedTypes.push_back(args[overloadedOperandIdx]->getType());
llvm::Module *module = builder.getModule();
llvm::Function *llvmIntr = llvm::Intrinsic::getOrInsertDeclaration(
module, intrinsic, overloadedTypes);
return builder.CreateCall(llvmIntr, args, opBundles);
}
/// Given a single MLIR operation, create the corresponding LLVM IR operation
/// using the `builder`.
LogicalResult ModuleTranslation::convertOperationImpl(
Operation &op, llvm::IRBuilderBase &builder, bool recordInsertions) {
const LLVMTranslationDialectInterface *opIface = iface.getInterfaceFor(&op);
if (!opIface)
return op.emitError("cannot be converted to LLVM IR: missing "
"`LLVMTranslationDialectInterface` registration for "
"dialect for op: ")
<< op.getName();
InstructionCapturingInserter::CollectionScope scope(builder,
recordInsertions);
if (failed(opIface->convertOperation(&op, builder, *this)))
return op.emitError("LLVM Translation failed for operation: ")
<< op.getName();
return convertDialectAttributes(&op, scope.getCapturedInstructions());
}
/// Convert block to LLVM IR. Unless `ignoreArguments` is set, emit PHI nodes
/// to define values corresponding to the MLIR block arguments. These nodes
/// are not connected to the source basic blocks, which may not exist yet. Uses
/// `builder` to construct the LLVM IR. Expects the LLVM IR basic block to have
/// been created for `bb` and included in the block mapping. Inserts new
/// instructions at the end of the block and leaves `builder` in a state
/// suitable for further insertion into the end of the block.
LogicalResult ModuleTranslation::convertBlockImpl(Block &bb,
bool ignoreArguments,
llvm::IRBuilderBase &builder,
bool recordInsertions) {
builder.SetInsertPoint(lookupBlock(&bb));
auto *subprogram = builder.GetInsertBlock()->getParent()->getSubprogram();
// Before traversing operations, make block arguments available through
// value remapping and PHI nodes, but do not add incoming edges for the PHI
// nodes just yet: those values may be defined by this or following blocks.
// This step is omitted if "ignoreArguments" is set. The arguments of the
// first block have been already made available through the remapping of
// LLVM function arguments.
if (!ignoreArguments) {
auto predecessors = bb.getPredecessors();
unsigned numPredecessors =
std::distance(predecessors.begin(), predecessors.end());
for (auto arg : bb.getArguments()) {
auto wrappedType = arg.getType();
if (!isCompatibleType(wrappedType))
return emitError(bb.front().getLoc(),
"block argument does not have an LLVM type");
builder.SetCurrentDebugLocation(
debugTranslation->translateLoc(arg.getLoc(), subprogram));
llvm::Type *type = convertType(wrappedType);
llvm::PHINode *phi = builder.CreatePHI(type, numPredecessors);
mapValue(arg, phi);
}
}
// Traverse operations.
for (auto &op : bb) {
// Set the current debug location within the builder.
builder.SetCurrentDebugLocation(
debugTranslation->translateLoc(op.getLoc(), subprogram));
if (failed(convertOperationImpl(op, builder, recordInsertions)))
return failure();
// Set the branch weight metadata on the translated instruction.
if (auto iface = dyn_cast<WeightedBranchOpInterface>(op))
setBranchWeightsMetadata(iface);
}
return success();
}
/// A helper method to get the single Block in an operation honoring LLVM's
/// module requirements.
static Block &getModuleBody(Operation *module) {
return module->getRegion(0).front();
}
/// A helper method to decide if a constant must not be set as a global variable
/// initializer. For an external linkage variable, the variable with an
/// initializer is considered externally visible and defined in this module, the
/// variable without an initializer is externally available and is defined
/// elsewhere.
static bool shouldDropGlobalInitializer(llvm::GlobalValue::LinkageTypes linkage,
llvm::Constant *cst) {
return (linkage == llvm::GlobalVariable::ExternalLinkage && !cst) ||
linkage == llvm::GlobalVariable::ExternalWeakLinkage;
}
/// Sets the runtime preemption specifier of `gv` to dso_local if
/// `dsoLocalRequested` is true, otherwise it is left unchanged.
static void addRuntimePreemptionSpecifier(bool dsoLocalRequested,
llvm::GlobalValue *gv) {
if (dsoLocalRequested)
gv->setDSOLocal(true);
}
/// Attempts to translate an MLIR attribute identified by `key`, optionally with
/// the given `value`, into an LLVM IR attribute. Reports errors at `loc` if
/// any. If the attribute name corresponds to a known LLVM IR attribute kind,
/// creates the LLVM attribute of that kind; otherwise, keeps it as a string
/// attribute. Performs additional checks for attributes known to have or not
/// have a value in order to avoid assertions inside LLVM upon construction.
static FailureOr<llvm::Attribute>
convertMLIRAttributeToLLVM(Location loc, llvm::LLVMContext &ctx, StringRef key,
StringRef value = StringRef()) {
auto kind = llvm::Attribute::getAttrKindFromName(key);
if (kind == llvm::Attribute::None)
return llvm::Attribute::get(ctx, key, value);
if (llvm::Attribute::isIntAttrKind(kind)) {
if (value.empty())
return emitError(loc) << "LLVM attribute '" << key << "' expects a value";
int64_t result;
if (!value.getAsInteger(/*Radix=*/0, result))
return llvm::Attribute::get(ctx, kind, result);
return llvm::Attribute::get(ctx, key, value);
}
if (!value.empty())
return emitError(loc) << "LLVM attribute '" << key
<< "' does not expect a value, found '" << value
<< "'";
return llvm::Attribute::get(ctx, kind);
}
/// Converts the MLIR attributes listed in the given array attribute into LLVM
/// attributes. Returns an `AttrBuilder` containing the converted attributes.
/// Reports error to `loc` if any and returns immediately. Expects `arrayAttr`
/// to contain either string attributes, treated as value-less LLVM attributes,
/// or array attributes containing two string attributes, with the first string
/// being the name of the corresponding LLVM attribute and the second string
/// beings its value. Note that even integer attributes are expected to have
/// their values expressed as strings.
static FailureOr<llvm::AttrBuilder>
convertMLIRAttributesToLLVM(Location loc, llvm::LLVMContext &ctx,
ArrayAttr arrayAttr, StringRef arrayAttrName) {
llvm::AttrBuilder attrBuilder(ctx);
if (!arrayAttr)
return attrBuilder;
for (Attribute attr : arrayAttr) {
if (auto stringAttr = dyn_cast<StringAttr>(attr)) {
FailureOr<llvm::Attribute> llvmAttr =
convertMLIRAttributeToLLVM(loc, ctx, stringAttr.getValue());
if (failed(llvmAttr))
return failure();
attrBuilder.addAttribute(*llvmAttr);
continue;
}
auto arrayAttr = dyn_cast<ArrayAttr>(attr);
if (!arrayAttr || arrayAttr.size() != 2)
return emitError(loc) << "expected '" << arrayAttrName
<< "' to contain string or array attributes";
auto keyAttr = dyn_cast<StringAttr>(arrayAttr[0]);
auto valueAttr = dyn_cast<StringAttr>(arrayAttr[1]);
if (!keyAttr || !valueAttr)
return emitError(loc) << "expected arrays within '" << arrayAttrName
<< "' to contain two strings";
FailureOr<llvm::Attribute> llvmAttr = convertMLIRAttributeToLLVM(
loc, ctx, keyAttr.getValue(), valueAttr.getValue());
if (failed(llvmAttr))
return failure();
attrBuilder.addAttribute(*llvmAttr);
}
return attrBuilder;
}
LogicalResult ModuleTranslation::convertGlobalsAndAliases() {
// Mapping from compile unit to its respective set of global variables.
DenseMap<llvm::DICompileUnit *, SmallVector<llvm::Metadata *>> globalGVars;
// Mapping from subprogram to its respective set of static local variables.
DenseMap<llvm::DISubprogram *, SmallVector<llvm::Metadata *>> staticLocals;
// First, create all global variables and global aliases in LLVM IR. A global
// or alias body may refer to another global/alias or itself, so all the
// mapping needs to happen prior to body conversion.
// Create all llvm::GlobalVariable
for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
llvm::Type *type = convertType(op.getType());
llvm::Constant *cst = nullptr;
const bool deferValueAttrToPass2 = op.getValueOrNull() &&
!op.getInitializerBlock() &&
!isa<StringAttr>(op.getValueOrNull());
if (op.getValueOrNull() && !deferValueAttrToPass2) {
// String attributes are treated separately because they cannot appear as
// in-function constants and are thus not supported by getLLVMConstant.
if (auto strAttr = dyn_cast_or_null<StringAttr>(op.getValueOrNull())) {
cst = llvm::ConstantDataArray::getString(
llvmModule->getContext(), strAttr.getValue(), /*AddNull=*/false);
type = cst->getType();
}
}
auto linkage = convertLinkageToLLVM(op.getLinkage());
// LLVM IR requires constant with linkage other than external or weak
// external to have initializers. If MLIR does not provide an initializer,
// default to undef.
bool dropInitializer = shouldDropGlobalInitializer(linkage, cst);
if (!deferValueAttrToPass2) {
if (!dropInitializer && !cst)
cst = llvm::UndefValue::get(type);
else if (dropInitializer && cst)
cst = nullptr;
} else {
cst = nullptr;
}
auto *var = new llvm::GlobalVariable(
*llvmModule, type, op.getConstant(), linkage, cst, op.getSymName(),
/*InsertBefore=*/nullptr, convertThreadLocalModeToLLVM(op.getTlsMode()),
op.getAddrSpace(), op.getExternallyInitialized());
if (std::optional<mlir::SymbolRefAttr> comdat = op.getComdat()) {
auto selectorOp = cast<ComdatSelectorOp>(
symbolTable().lookupNearestSymbolFrom(op, *comdat));
var->setComdat(comdatMapping.lookup(selectorOp));
}
if (op.getUnnamedAddr().has_value())
var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr()));
if (op.getSection().has_value())
var->setSection(*op.getSection());
addRuntimePreemptionSpecifier(op.getDsoLocal(), var);
std::optional<uint64_t> alignment = op.getAlignment();
if (alignment.has_value())
var->setAlignment(llvm::MaybeAlign(alignment.value()));
var->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));
globalsMapping.try_emplace(op, var);
globalsByNameMapping.try_emplace(op.getSymName(), var);
// Add debug information if present.
if (op.getDbgExprs()) {
for (auto exprAttr :
op.getDbgExprs()->getAsRange<DIGlobalVariableExpressionAttr>()) {
llvm::DIGlobalVariableExpression *diGlobalExpr =
debugTranslation->translateGlobalVariableExpression(exprAttr);
llvm::DIGlobalVariable *diGlobalVar = diGlobalExpr->getVariable();
var->addDebugInfo(diGlobalExpr);
// There is no `globals` field in DICompileUnitAttr which can be
// directly assigned to DICompileUnit. We have to build the list by
// looking at the dbgExpr of all the GlobalOps. The scope of the
// variable is used to get the DICompileUnit in which to add it. But
// there are cases where the scope of a global does not directly point
// to the DICompileUnit and we have to do a bit more work to get to
// it. Some of those cases are:
//
// 1. For the languages that support modules, the scope hierarchy can
// be variable -> DIModule -> DICompileUnit
//
// 2. For the Fortran common block variable, the scope hierarchy can
// be variable -> DICommonBlock -> DISubprogram -> DICompileUnit
//
// 3. For entities like static local variables in C or variable with
// SAVE attribute in Fortran, the scope hierarchy can be
// variable (-> DILocalScope)* -> DISubprogram
llvm::DIScope *scope = diGlobalVar->getScope();
if (auto *mod = dyn_cast_if_present<llvm::DIModule>(scope))
scope = mod->getScope();
else if (auto *cb = dyn_cast_if_present<llvm::DICommonBlock>(scope)) {
if (auto *sp =
dyn_cast_if_present<llvm::DISubprogram>(cb->getScope()))
scope = sp->getUnit();
} else if (auto *lbb =
dyn_cast_if_present<llvm::DILexicalBlockBase>(scope)) {
scope = lbb->getSubprogram();
}
// Get the compile unit (scope) of the the global variable, or the
// subprogram of the static local variable.
if (llvm::DICompileUnit *compileUnit =
dyn_cast_if_present<llvm::DICompileUnit>(scope)) {
// Update the compile unit with this incoming global variable
// expression during the finalizing step later.
globalGVars[compileUnit].push_back(diGlobalExpr);
} else if (llvm::DISubprogram *sp =
dyn_cast_if_present<llvm::DISubprogram>(scope)) {
// Update the subprogram with this incoming static local variable
// expression during the finalizing step later.
staticLocals[sp].push_back(diGlobalExpr);
}
}
}
// Forward the target-specific attributes to LLVM.
FailureOr<llvm::AttrBuilder> convertedTargetSpecificAttrs =
convertMLIRAttributesToLLVM(op.getLoc(), var->getContext(),
op.getTargetSpecificAttrsAttr(),
op.getTargetSpecificAttrsAttrName());
if (failed(convertedTargetSpecificAttrs))
return failure();
var->addAttributes(*convertedTargetSpecificAttrs);
}
// Value-attribute initializers may reference other globals by symbol name.
// Register every global above before materializing those constants.
for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
if (!op.getValueOrNull() || op.getInitializerBlock() ||
isa<StringAttr>(op.getValueOrNull()))
continue;
llvm::Type *type = convertType(op.getType());
llvm::Constant *cst =
getLLVMConstant(type, op.getValueOrNull(), op.getLoc(), *this);
if (!cst)
return failure();
auto linkage = convertLinkageToLLVM(op.getLinkage());
bool dropInitializer = shouldDropGlobalInitializer(linkage, cst);
auto *var = cast<llvm::GlobalVariable>(lookupGlobal(op));
if (dropInitializer)
var->setInitializer(nullptr);
else
var->setInitializer(cst);
}
// Create all llvm::GlobalAlias
for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) {
llvm::Type *type = convertType(op.getType());
llvm::Constant *cst = nullptr;
llvm::GlobalValue::LinkageTypes linkage =
convertLinkageToLLVM(op.getLinkage());
llvm::Module &llvmMod = *llvmModule;
// Note address space and aliasee info isn't set just yet.
llvm::GlobalAlias *var = llvm::GlobalAlias::create(
type, op.getAddrSpace(), linkage, op.getSymName(), /*placeholder*/ cst,
&llvmMod);
var->setThreadLocalMode(convertThreadLocalModeToLLVM(op.getTlsMode()));
// Note there is no need to setup the comdat because GlobalAlias calls into
// the aliasee comdat information automatically.
if (op.getUnnamedAddr().has_value())
var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr()));
var->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));
aliasesMapping.try_emplace(op, var);
}
// Convert global variable bodies.
for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
if (Block *initializer = op.getInitializerBlock()) {
llvm::IRBuilder<llvm::TargetFolder> builder(
*llvmModule, llvm::TargetFolder(llvmModule->getDataLayout()));
[[maybe_unused]] int numConstantsHit = 0;
[[maybe_unused]] int numConstantsErased = 0;
DenseMap<llvm::ConstantAggregate *, int> constantAggregateUseMap;
for (auto &op : initializer->without_terminator()) {
if (failed(convertOperation(op, builder)))
return emitError(op.getLoc(), "fail to convert global initializer");
auto *cst = dyn_cast<llvm::Constant>(lookupValue(op.getResult(0)));
if (!cst)
return emitError(op.getLoc(), "unemittable constant value");
// When emitting an LLVM constant, a new constant is created and the old
// constant may become dangling and take space. We should remove the
// dangling constants to avoid memory explosion especially for constant
// arrays whose number of elements is large.
// Because multiple operations may refer to the same constant, we need
// to count the number of uses of each constant array and remove it only
// when the count becomes zero.
if (auto *agg = dyn_cast<llvm::ConstantAggregate>(cst)) {
numConstantsHit++;
Value result = op.getResult(0);
int numUsers = std::distance(result.use_begin(), result.use_end());
auto [iterator, inserted] =
constantAggregateUseMap.try_emplace(agg, numUsers);
if (!inserted) {
// Key already exists, update the value
iterator->second += numUsers;
}
}
// Scan the operands of the operation to decrement the use count of
// constants. Erase the constant if the use count becomes zero.
for (Value v : op.getOperands()) {
auto *cst = dyn_cast<llvm::ConstantAggregate>(lookupValue(v));
if (!cst)
continue;
auto iter = constantAggregateUseMap.find(cst);
assert(iter != constantAggregateUseMap.end() && "constant not found");
iter->second--;
if (iter->second == 0) {
// NOTE: cannot call removeDeadConstantUsers() here because it
// may remove the constant which has uses not be converted yet.
if (cst->user_empty()) {
cst->destroyConstant();
numConstantsErased++;
}
constantAggregateUseMap.erase(iter);
}
}
}
ReturnOp ret = cast<ReturnOp>(initializer->getTerminator());
llvm::Constant *cst =
cast<llvm::Constant>(lookupValue(ret.getOperand(0)));
auto *global = cast<llvm::GlobalVariable>(lookupGlobal(op));
if (!shouldDropGlobalInitializer(global->getLinkage(), cst))
global->setInitializer(cst);
// Try to remove the dangling constants again after all operations are
// converted.
for (auto it : constantAggregateUseMap) {
auto *cst = it.first;
cst->removeDeadConstantUsers();
if (cst->user_empty()) {
cst->destroyConstant();
numConstantsErased++;
}
}
LLVM_DEBUG(llvm::dbgs()
<< "Convert initializer for " << op.getName() << "\n";
llvm::dbgs() << numConstantsHit << " new constants hit\n";
llvm::dbgs()
<< numConstantsErased << " dangling constants erased\n";);
}
}
// Convert llvm.mlir.global_ctors and dtors.
for (Operation &op : getModuleBody(mlirModule)) {
auto ctorOp = dyn_cast<GlobalCtorsOp>(op);
auto dtorOp = dyn_cast<GlobalDtorsOp>(op);
if (!ctorOp && !dtorOp)
continue;
// The empty / zero initialized version of llvm.global_(c|d)tors cannot be
// handled by appendGlobalFn logic below, which just ignores empty (c|d)tor
// lists. Make sure it gets emitted.
if ((ctorOp && ctorOp.getCtors().empty()) ||
(dtorOp && dtorOp.getDtors().empty())) {
llvm::IRBuilder<llvm::TargetFolder> builder(
*llvmModule, llvm::TargetFolder(llvmModule->getDataLayout()));
llvm::Type *eltTy = llvm::StructType::get(
builder.getInt32Ty(), builder.getPtrTy(), builder.getPtrTy());
llvm::ArrayType *at = llvm::ArrayType::get(eltTy, 0);
llvm::Constant *zeroInit = llvm::Constant::getNullValue(at);
(void)new llvm::GlobalVariable(
*llvmModule, zeroInit->getType(), false,
llvm::GlobalValue::AppendingLinkage, zeroInit,
ctorOp ? "llvm.global_ctors" : "llvm.global_dtors");
} else {
auto range = ctorOp
? llvm::zip(ctorOp.getCtors(), ctorOp.getPriorities())
: llvm::zip(dtorOp.getDtors(), dtorOp.getPriorities());
auto appendGlobalFn =
ctorOp ? llvm::appendToGlobalCtors : llvm::appendToGlobalDtors;
for (const auto &[sym, prio] : range) {
llvm::Function *f =
lookupFunction(cast<FlatSymbolRefAttr>(sym).getValue());
appendGlobalFn(*llvmModule, f, cast<IntegerAttr>(prio).getInt(),
/*Data=*/nullptr);
}
}
}
for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>())
if (failed(convertDialectAttributes(op, {})))
return failure();
// Finally, update the compile units their respective sets of global variables
// created earlier.
for (const auto &[compileUnit, globals] : globalGVars)
compileUnit->replaceGlobalVariables(
llvm::MDTuple::get(getLLVMContext(), globals));
// And update the subprograms with their respective sets of static local
// variables.
for (const auto &[sp, globals] : staticLocals)
sp->retainNodes(globals.begin(), globals.end());
// Convert global alias bodies.
for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>()) {
Block &initializer = op.getInitializerBlock();
llvm::IRBuilder<llvm::TargetFolder> builder(
*llvmModule, llvm::TargetFolder(llvmModule->getDataLayout()));
for (mlir::Operation &op : initializer.without_terminator()) {
if (failed(convertOperation(op, builder)))
return emitError(op.getLoc(), "fail to convert alias initializer");
if (!isa<llvm::Constant>(lookupValue(op.getResult(0))))
return emitError(op.getLoc(), "unemittable constant value");
}
auto ret = cast<ReturnOp>(initializer.getTerminator());
auto *cst = cast<llvm::Constant>(lookupValue(ret.getOperand(0)));
assert(aliasesMapping.count(op));
auto *alias = cast<llvm::GlobalAlias>(aliasesMapping[op]);
alias->setAliasee(cst);
}
for (auto op : getModuleBody(mlirModule).getOps<LLVM::AliasOp>())
if (failed(convertDialectAttributes(op, {})))
return failure();
return success();
}
/// Return a representation of `value` as metadata.
static llvm::Metadata *convertIntegerToMetadata(llvm::LLVMContext &context,
const llvm::APInt &value) {
llvm::Constant *constant = llvm::ConstantInt::get(context, value);
return llvm::ConstantAsMetadata::get(constant);
}
/// Return a representation of `value` as an MDNode.
static llvm::MDNode *convertIntegerToMDNode(llvm::LLVMContext &context,
const llvm::APInt &value) {
return llvm::MDNode::get(context, convertIntegerToMetadata(context, value));
}
/// Return an MDNode encoding `vec_type_hint` metadata.
static llvm::MDNode *convertVecTypeHintToMDNode(llvm::LLVMContext &context,
llvm::Type *type,
bool isSigned) {
llvm::Metadata *typeMD =
llvm::ConstantAsMetadata::get(llvm::UndefValue::get(type));
llvm::Metadata *isSignedMD =
convertIntegerToMetadata(context, llvm::APInt(32, isSigned ? 1 : 0));
return llvm::MDNode::get(context, {typeMD, isSignedMD});
}
/// Return an MDNode with a tuple given by the values in `values`.
static llvm::MDNode *convertIntegerArrayToMDNode(llvm::LLVMContext &context,
ArrayRef<int32_t> values) {
SmallVector<llvm::Metadata *> mdValues;
llvm::transform(
values, std::back_inserter(mdValues), [&context](int32_t value) {
return convertIntegerToMetadata(context, llvm::APInt(32, value));
});
return llvm::MDNode::get(context, mdValues);
}
FailureOr<llvm::Metadata *> ModuleTranslation::convertMetadataAttr(
Attribute attr, function_ref<InFlightDiagnostic()> emitError) {
llvm::LLVMContext &llvmContext = getLLVMContext();
return llvm::TypeSwitch<Attribute, FailureOr<llvm::Metadata *>>(attr)
.Case([&](MDStringAttr a) -> FailureOr<llvm::Metadata *> {
return llvm::MDString::get(llvmContext, a.getValue().getValue());
})
.Case([&](MDConstantAttr a) -> FailureOr<llvm::Metadata *> {
IntegerAttr intAttr = llvm::dyn_cast<IntegerAttr>(a.getValue());
if (!intAttr) {
return emitError()
<< "expected integer attribute in metadata constant";
}
return llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
llvm::Type::getIntNTy(llvmContext,
intAttr.getType().getIntOrFloatBitWidth()),
intAttr.getValue()));
})
.Case([&](MDGlobalValueAttr a) -> FailureOr<llvm::Metadata *> {
return convertSymbolRefToMetadata(a.getName(), emitError);
})
.Case([&](MDNullAttr a) -> FailureOr<llvm::Metadata *> {
return llvm::ConstantAsMetadata::get(llvm::ConstantPointerNull::get(
llvm::PointerType::get(llvmContext, a.getAddressSpace())));
})
.Case([&](MDAddrSpaceCastAttr a) -> FailureOr<llvm::Metadata *> {
FailureOr<llvm::Metadata *> arg =
convertMetadataAttr(a.getArg(), emitError);
if (failed(arg))
return failure();
// The verifier restricts the operand to pointer-valued metadata
// attributes, all of which translate to a ConstantAsMetadata.
auto *argAsMD = cast<llvm::ConstantAsMetadata>(*arg);
return llvm::ConstantAsMetadata::get(
llvm::ConstantExpr::getAddrSpaceCast(
argAsMD->getValue(),
llvm::PointerType::get(llvmContext, a.getAddressSpace())));
})
.Case([&](MDNodeAttr a) -> FailureOr<llvm::Metadata *> {
SmallVector<llvm::Metadata *> operands;
for (Attribute operand : a.getOperands()) {
FailureOr<llvm::Metadata *> md =
convertMetadataAttr(operand, emitError);
if (failed(md))
return failure();
operands.push_back(*md);
}
return llvm::MDNode::get(llvmContext, operands);
})
.Default([&](Attribute attr) -> FailureOr<llvm::Metadata *> {
return emitError() << "unsupported LLVM metadata attribute " << attr;
});
}
FailureOr<llvm::Metadata *> ModuleTranslation::convertSymbolRefToMetadata(
FlatSymbolRefAttr name, function_ref<InFlightDiagnostic()> emitError) {
if (llvm::Function *fn = lookupFunction(name.getValue()))
return llvm::ValueAsMetadata::get(fn);
if (llvm::GlobalValue *global = lookupGlobal(name.getValue()))
return llvm::ValueAsMetadata::get(global);
Operation *symbol = symbolTable().lookupSymbolIn(mlirModule, name);
if (auto alias = dyn_cast_if_present<LLVM::AliasOp>(symbol)) {
if (llvm::GlobalValue *global = lookupAlias(alias))
return llvm::ValueAsMetadata::get(global);
}
if (auto ifunc = dyn_cast_if_present<LLVM::IFuncOp>(symbol)) {
if (llvm::GlobalValue *global = lookupIFunc(ifunc))
return llvm::ValueAsMetadata::get(global);
}
return emitError() << "could not resolve metadata reference '" << name << "'";
}
LogicalResult ModuleTranslation::convertFunctionMetadata() {
for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {
ArrayAttr metadata = function.getFunctionMetadataAttr();
if (!metadata)
continue;
llvm::Function *llvmFunc = lookupFunction(function.getName());
for (auto entry : metadata.getAsRange<LLVM::FunctionMetadataAttr>()) {
StringRef metadataName = entry.getMetadataName().getValue();
FailureOr<llvm::Metadata *> md =
convertMetadataAttr(entry.getNode(), [&]() {
return function.emitError()
<< "failed to convert function_metadata entry '"
<< metadataName << "': ";
});
if (failed(md))
return failure();
llvm::MDNode *node = llvm::dyn_cast_if_present<llvm::MDNode>(*md);
if (!node) {
return function.emitError()
<< "failed to convert function_metadata entry '" << metadataName
<< "'";
}
llvmFunc->addMetadata(metadataName, *node);
}
}
return success();
}
LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
// Clear the block, branch value mappings, they are only relevant within one
// function.
blockMapping.clear();
valueMapping.clear();
branchMapping.clear();
llvm::Function *llvmFunc = lookupFunction(func.getName());
llvm::LLVMContext &llvmContext = llvmFunc->getContext();
// Add function arguments to the value remapping table.
for (auto [mlirArg, llvmArg] :
llvm::zip(func.getArguments(), llvmFunc->args()))
mapValue(mlirArg, &llvmArg);
// Check the personality and set it.
if (func.getPersonality()) {
llvm::Type *ty = llvm::PointerType::get(
llvmContext, llvmModule->getDataLayout().getProgramAddressSpace());
if (llvm::Constant *pfunc = getLLVMConstant(ty, func.getPersonalityAttr(),
func.getLoc(), *this))
llvmFunc->setPersonalityFn(pfunc);
}
if (std::optional<StringRef> section = func.getSection())
llvmFunc->setSection(*section);
if (func.getArmStreaming())
llvmFunc->addFnAttr("aarch64_pstate_sm_enabled");
else if (func.getArmLocallyStreaming())
llvmFunc->addFnAttr("aarch64_pstate_sm_body");
else if (func.getArmStreamingCompatible())
llvmFunc->addFnAttr("aarch64_pstate_sm_compatible");
if (func.getArmNewZa())
llvmFunc->addFnAttr("aarch64_new_za");
else if (func.getArmInZa())
llvmFunc->addFnAttr("aarch64_in_za");
else if (func.getArmOutZa())
llvmFunc->addFnAttr("aarch64_out_za");
else if (func.getArmInoutZa())
llvmFunc->addFnAttr("aarch64_inout_za");
else if (func.getArmPreservesZa())
llvmFunc->addFnAttr("aarch64_preserves_za");
if (auto targetCpu = func.getTargetCpu())
llvmFunc->addFnAttr("target-cpu", *targetCpu);
if (auto tuneCpu = func.getTuneCpu())
llvmFunc->addFnAttr("tune-cpu", *tuneCpu);
if (auto reciprocalEstimates = func.getReciprocalEstimates())
llvmFunc->addFnAttr("reciprocal-estimates", *reciprocalEstimates);
if (auto preferVectorWidth = func.getPreferVectorWidth())
llvmFunc->addFnAttr("prefer-vector-width", *preferVectorWidth);
if (func.getUseSampleProfile())
llvmFunc->addFnAttr("use-sample-profile");
if (auto disableTailCalls = func.getDisableTailCalls())
llvmFunc->addFnAttr("disable-tail-calls",
llvm::toStringRef(*disableTailCalls));
if (auto sampleProfileSuffixElisionPolicy =
func.getSampleProfileSuffixElisionPolicy())
llvmFunc->addFnAttr("sample-profile-suffix-elision-policy",
*sampleProfileSuffixElisionPolicy);
if (auto attr = func.getVscaleRange())
llvmFunc->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(
getLLVMContext(), attr->getMinRange().getInt(),
attr->getMaxRange().getInt()));
if (auto noSignedZerosFpMath = func.getNoSignedZerosFpMath())
llvmFunc->addFnAttr("no-signed-zeros-fp-math",
llvm::toStringRef(*noSignedZerosFpMath));
if (auto fpContract = func.getFpContract())
llvmFunc->addFnAttr("fp-contract", *fpContract);
if (auto instrumentFunctionEntry = func.getInstrumentFunctionEntry())
llvmFunc->addFnAttr("instrument-function-entry", *instrumentFunctionEntry);
if (auto instrumentFunctionExit = func.getInstrumentFunctionExit())
llvmFunc->addFnAttr("instrument-function-exit", *instrumentFunctionExit);
// First, create all blocks so we can jump to them.
for (auto &bb : func) {
auto *llvmBB = llvm::BasicBlock::Create(llvmContext);
llvmBB->insertInto(llvmFunc);
mapBlock(&bb, llvmBB);
}
// Then, convert blocks one by one in topological order to ensure defs are
// converted before uses.
auto blocks = getBlocksSortedByDominance(func.getBody());
for (Block *bb : blocks) {
CapturingIRBuilder builder(*llvmModule,
llvm::TargetFolder(llvmModule->getDataLayout()));
if (failed(convertBlockImpl(*bb, bb->isEntryBlock(), builder,
/*recordInsertions=*/true)))
return failure();
}
// After all blocks have been traversed and values mapped, connect the PHI
// nodes to the results of preceding blocks.
detail::connectPHINodes(func.getBody(), *this);
// Finally, convert dialect attributes attached to the function.
return convertDialectAttributes(func, {});
}
LogicalResult ModuleTranslation::convertDialectAttributes(
Operation *op, ArrayRef<llvm::Instruction *> instructions) {
for (NamedAttribute attribute : op->getDialectAttrs())
if (failed(iface.amendOperation(op, instructions, attribute, *this)))
return failure();
return success();
}
/// Converts memory effect attributes from `func` and attaches them to
/// `llvmFunc`.
static void convertFunctionMemoryAttributes(LLVMFuncOp func,
llvm::Function *llvmFunc) {
if (!func.getMemoryEffects())
return;
MemoryEffectsAttr memEffects = func.getMemoryEffectsAttr();
// Add memory effects incrementally.
llvm::MemoryEffects newMemEffects =
llvm::MemoryEffects(llvm::MemoryEffects::Location::ArgMem,
convertModRefInfoToLLVM(memEffects.getArgMem()));
newMemEffects |= llvm::MemoryEffects(
llvm::MemoryEffects::Location::InaccessibleMem,
convertModRefInfoToLLVM(memEffects.getInaccessibleMem()));
newMemEffects |=
llvm::MemoryEffects(llvm::MemoryEffects::Location::Other,
convertModRefInfoToLLVM(memEffects.getOther()));
newMemEffects |=
llvm::MemoryEffects(llvm::MemoryEffects::Location::ErrnoMem,
convertModRefInfoToLLVM(memEffects.getErrnoMem()));
newMemEffects |=
llvm::MemoryEffects(llvm::MemoryEffects::Location::TargetMem0,
convertModRefInfoToLLVM(memEffects.getTargetMem0()));
newMemEffects |=
llvm::MemoryEffects(llvm::MemoryEffects::Location::TargetMem1,
convertModRefInfoToLLVM(memEffects.getTargetMem1()));
llvmFunc->setMemoryEffects(newMemEffects);
}
llvm::Attribute
ModuleTranslation::convertAllocsizeAttr(DenseI32ArrayAttr allocSizeAttr) {
if (!allocSizeAttr || allocSizeAttr.empty())
return llvm::Attribute{};
unsigned elemSize = static_cast<unsigned>(allocSizeAttr[0]);
std::optional<unsigned> numElems;
if (allocSizeAttr.size() > 1)
numElems = static_cast<unsigned>(allocSizeAttr[1]);
return llvm::Attribute::getWithAllocSizeArgs(getLLVMContext(), elemSize,
numElems);
}
static void convertDenormalFPEnvAttribute(LLVMFuncOp func,
llvm::AttrBuilder &Attrs) {
std::optional<DenormalFPEnvAttr> denormalFpEnv = func.getDenormalFpenv();
if (!denormalFpEnv)
return;
llvm::DenormalMode DefaultMode(
convertDenormalModeKindToLLVM(denormalFpEnv->getDefaultOutputMode()),
convertDenormalModeKindToLLVM(denormalFpEnv->getDefaultInputMode()));
llvm::DenormalMode FloatMode(
convertDenormalModeKindToLLVM(denormalFpEnv->getFloatOutputMode()),
convertDenormalModeKindToLLVM(denormalFpEnv->getFloatInputMode()));
llvm::DenormalFPEnv FPEnv(DefaultMode, FloatMode);
Attrs.addDenormalFPEnvAttr(FPEnv);
}
/// Converts function attributes from `func` and attaches them to `llvmFunc`.
static void convertFunctionAttributes(ModuleTranslation &mod, LLVMFuncOp func,
llvm::Function *llvmFunc) {
// FIXME: Use AttrBuilder far all cases
llvm::AttrBuilder AttrBuilder(llvmFunc->getContext());
if (func.getNoInlineAttr())
llvmFunc->addFnAttr(llvm::Attribute::NoInline);
if (func.getAlwaysInlineAttr())
llvmFunc->addFnAttr(llvm::Attribute::AlwaysInline);
if (func.getInlineHintAttr())
llvmFunc->addFnAttr(llvm::Attribute::InlineHint);
if (func.getOptimizeNoneAttr())
llvmFunc->addFnAttr(llvm::Attribute::OptimizeNone);
if (func.getReturnsTwiceAttr())
llvmFunc->addFnAttr(llvm::Attribute::ReturnsTwice);
if (func.getColdAttr())
llvmFunc->addFnAttr(llvm::Attribute::Cold);
if (func.getHotAttr())
llvmFunc->addFnAttr(llvm::Attribute::Hot);
if (func.getNoduplicateAttr())
llvmFunc->addFnAttr(llvm::Attribute::NoDuplicate);
if (func.getConvergentAttr())
llvmFunc->addFnAttr(llvm::Attribute::Convergent);
if (func.getNoUnwindAttr())
llvmFunc->addFnAttr(llvm::Attribute::NoUnwind);
if (func.getWillReturnAttr())
llvmFunc->addFnAttr(llvm::Attribute::WillReturn);
if (func.getNoreturnAttr())
llvmFunc->addFnAttr(llvm::Attribute::NoReturn);
if (func.getOptsizeAttr())
llvmFunc->addFnAttr(llvm::Attribute::OptimizeForSize);
if (func.getMinsizeAttr())
llvmFunc->addFnAttr(llvm::Attribute::MinSize);
if (func.getSaveRegParamsAttr())
llvmFunc->addFnAttr("save-reg-params");
if (func.getNoCallerSavedRegistersAttr())
llvmFunc->addFnAttr("no_caller_saved_registers");
if (func.getNocallbackAttr())
llvmFunc->addFnAttr(llvm::Attribute::NoCallback);
if (StringAttr modFormat = func.getModularFormatAttr())
llvmFunc->addFnAttr("modular-format", modFormat.getValue());
if (TargetFeaturesAttr targetFeatAttr = func.getTargetFeaturesAttr())
llvmFunc->addFnAttr("target-features", targetFeatAttr.getFeaturesString());
if (FramePointerKindAttr fpAttr = func.getFramePointerAttr())
llvmFunc->addFnAttr("frame-pointer", stringifyFramePointerKind(
fpAttr.getFramePointerKind()));
if (UWTableKindAttr uwTableKindAttr = func.getUwtableKindAttr())
llvmFunc->setUWTableKind(
convertUWTableKindToLLVM(uwTableKindAttr.getUwtableKind()));
if (StringAttr zcsr = func.getZeroCallUsedRegsAttr())
llvmFunc->addFnAttr("zero-call-used-regs", zcsr.getValue());
if (func.getUniformWorkGroupSizeAttr())
llvmFunc->addFnAttr("uniform-work-group-size");
if (ArrayAttr noBuiltins = func.getNobuiltinsAttr()) {
if (noBuiltins.empty())
llvmFunc->addFnAttr("no-builtins");
mod.convertFunctionAttrCollection(noBuiltins, llvmFunc,
ModuleTranslation::convertNoBuiltin);
}
mod.convertFunctionAttrCollection(func.getDefaultFuncAttrsAttr(), llvmFunc,
ModuleTranslation::convertDefaultFuncAttr);
if (llvm::Attribute attr = mod.convertAllocsizeAttr(func.getAllocsizeAttr());
attr.isValid())
llvmFunc->addFnAttr(attr);
convertFunctionMemoryAttributes(func, llvmFunc);
convertDenormalFPEnvAttribute(func, AttrBuilder);
llvmFunc->addFnAttrs(AttrBuilder);
}
/// Converts function attributes from `func` and attaches them to `llvmFunc`.
static void convertFunctionKernelAttributes(LLVMFuncOp func,
llvm::Function *llvmFunc,
ModuleTranslation &translation) {
llvm::LLVMContext &llvmContext = llvmFunc->getContext();
if (VecTypeHintAttr vecTypeHint = func.getVecTypeHintAttr()) {
Type type = vecTypeHint.getHint().getValue();
llvm::Type *llvmType = translation.convertType(type);
bool isSigned = vecTypeHint.getIsSigned();
llvmFunc->setMetadata(
func.getVecTypeHintAttrName(),
convertVecTypeHintToMDNode(llvmContext, llvmType, isSigned));
}
if (std::optional<ArrayRef<int32_t>> workGroupSizeHint =
func.getWorkGroupSizeHint()) {
llvmFunc->setMetadata(
func.getWorkGroupSizeHintAttrName(),
convertIntegerArrayToMDNode(llvmContext, *workGroupSizeHint));
}
if (std::optional<ArrayRef<int32_t>> reqdWorkGroupSize =
func.getReqdWorkGroupSize()) {
llvmFunc->setMetadata(
func.getReqdWorkGroupSizeAttrName(),
convertIntegerArrayToMDNode(llvmContext, *reqdWorkGroupSize));
}
if (std::optional<uint32_t> intelReqdSubGroupSize =
func.getIntelReqdSubGroupSize()) {
llvmFunc->setMetadata(
func.getIntelReqdSubGroupSizeAttrName(),
convertIntegerToMDNode(llvmContext,
llvm::APInt(32, *intelReqdSubGroupSize)));
}
}
static LogicalResult convertParameterAttr(llvm::AttrBuilder &attrBuilder,
llvm::Attribute::AttrKind llvmKind,
NamedAttribute namedAttr,
ModuleTranslation &moduleTranslation,
Location loc) {
return llvm::TypeSwitch<Attribute, LogicalResult>(namedAttr.getValue())
.Case([&](TypeAttr typeAttr) {
attrBuilder.addTypeAttr(
llvmKind, moduleTranslation.convertType(typeAttr.getValue()));
return success();
})
.Case([&](IntegerAttr intAttr) {
attrBuilder.addRawIntAttr(llvmKind, intAttr.getInt());
return success();
})
.Case([&](UnitAttr) {
attrBuilder.addAttribute(llvmKind);
return success();
})
.Case([&](LLVM::ConstantRangeAttr rangeAttr) {
attrBuilder.addConstantRangeAttr(
llvmKind,
llvm::ConstantRange(rangeAttr.getLower(), rangeAttr.getUpper()));
return success();
})
.Default([loc](auto) {
return emitError(loc, "unsupported parameter attribute type");
});
}
FailureOr<llvm::AttrBuilder>
ModuleTranslation::convertParameterAttrs(LLVMFuncOp func, int argIdx,
DictionaryAttr paramAttrs) {
llvm::AttrBuilder attrBuilder(llvmModule->getContext());
auto attrNameToKindMapping = getAttrNameToKindMapping();
Location loc = func.getLoc();
for (auto namedAttr : paramAttrs) {
auto it = attrNameToKindMapping.find(namedAttr.getName());
if (it != attrNameToKindMapping.end()) {
llvm::Attribute::AttrKind llvmKind = it->second;
if (failed(convertParameterAttr(attrBuilder, llvmKind, namedAttr, *this,
loc)))
return failure();
} else if (namedAttr.getNameDialect()) {
if (failed(iface.convertParameterAttr(func, argIdx, namedAttr, *this)))
return failure();
}
}
return attrBuilder;
}
LogicalResult ModuleTranslation::convertArgAndResultAttrs(
ArgAndResultAttrsOpInterface attrsOp, llvm::CallBase *call,
ArrayRef<unsigned> immArgPositions) {
// Convert the argument attributes.
if (ArrayAttr argAttrsArray = attrsOp.getArgAttrsAttr()) {
unsigned argAttrIdx = 0;
llvm::SmallDenseSet<unsigned> immArgPositionsSet(immArgPositions.begin(),
immArgPositions.end());
for (unsigned argIdx : llvm::seq<unsigned>(call->arg_size())) {
if (argAttrIdx >= argAttrsArray.size())
break;
// Skip immediate arguments (they have no entries in argAttrsArray).
if (immArgPositionsSet.contains(argIdx))
continue;
// Skip empty argument attributes.
auto argAttrs = cast<DictionaryAttr>(argAttrsArray[argAttrIdx++]);
if (argAttrs.empty())
continue;
// Convert and add attributes to the call instruction.
FailureOr<llvm::AttrBuilder> attrBuilder =
convertParameterAttrs(attrsOp->getLoc(), argAttrs);
if (failed(attrBuilder))
return failure();
call->addParamAttrs(argIdx, *attrBuilder);
}
}
// Convert the result attributes.
if (ArrayAttr resAttrsArray = attrsOp.getResAttrsAttr()) {
if (!resAttrsArray.empty()) {
auto resAttrs = cast<DictionaryAttr>(resAttrsArray[0]);
FailureOr<llvm::AttrBuilder> attrBuilder =
convertParameterAttrs(attrsOp->getLoc(), resAttrs);
if (failed(attrBuilder))
return failure();
call->addRetAttrs(*attrBuilder);
}
}
return success();
}
std::optional<llvm::Attribute>
ModuleTranslation::convertNoBuiltin(llvm::LLVMContext &ctx, mlir::Attribute a) {
if (auto str = dyn_cast<StringAttr>(a))
return llvm::Attribute::get(ctx, ("no-builtin-" + str.getValue()).str());
return std::nullopt;
}
std::optional<llvm::Attribute>
ModuleTranslation::convertDefaultFuncAttr(llvm::LLVMContext &ctx,
mlir::NamedAttribute namedAttr) {
StringAttr name = namedAttr.getName();
Attribute value = namedAttr.getValue();
if (auto strVal = dyn_cast<StringAttr>(value))
return llvm::Attribute::get(ctx, name.getValue(), strVal.getValue());
if (mlir::isa<UnitAttr>(value))
return llvm::Attribute::get(ctx, name.getValue());
return std::nullopt;
}
FailureOr<llvm::AttrBuilder>
ModuleTranslation::convertParameterAttrs(Location loc,
DictionaryAttr paramAttrs) {
llvm::AttrBuilder attrBuilder(llvmModule->getContext());
auto attrNameToKindMapping = getAttrNameToKindMapping();
for (auto namedAttr : paramAttrs) {
auto it = attrNameToKindMapping.find(namedAttr.getName());
if (it != attrNameToKindMapping.end()) {
llvm::Attribute::AttrKind llvmKind = it->second;
if (failed(convertParameterAttr(attrBuilder, llvmKind, namedAttr, *this,
loc)))
return failure();
}
}
return attrBuilder;
}
LogicalResult ModuleTranslation::convertFunctionSignatures() {
// Declare all functions first because there may be function calls that form a
// call graph with cycles, global initializers that reference functions, or
// metadata that references functions declared later in the module.
for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {
llvm::FunctionCallee llvmFuncCst = llvmModule->getOrInsertFunction(
function.getName(),
cast<llvm::FunctionType>(convertType(function.getFunctionType())));
llvm::Function *llvmFunc = cast<llvm::Function>(llvmFuncCst.getCallee());
mapFunction(function.getName(), llvmFunc);
}
for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {
llvm::Function *llvmFunc = lookupFunction(function.getName());
llvmFunc->setLinkage(convertLinkageToLLVM(function.getLinkage()));
llvmFunc->setCallingConv(convertCConvToLLVM(function.getCConv()));
addRuntimePreemptionSpecifier(function.getDsoLocal(), llvmFunc);
// Convert function attributes.
convertFunctionAttributes(*this, function, llvmFunc);
// Convert function kernel attributes to metadata.
convertFunctionKernelAttributes(function, llvmFunc, *this);
// Convert function_entry_count attribute to metadata.
if (auto entryCount = function.getFunctionEntryCountAttr()) {
ArrayRef<uint64_t> imports = entryCount.getImports();
llvm::DenseSet<llvm::GlobalValue::GUID> importGUIDs;
if (!imports.empty())
importGUIDs.insert(imports.begin(), imports.end());
llvmFunc->setEntryCount(entryCount.getEntryCount(),
imports.empty() ? nullptr : &importGUIDs);
}
// Convert result attributes.
if (ArrayAttr allResultAttrs = function.getAllResultAttrs()) {
DictionaryAttr resultAttrs = cast<DictionaryAttr>(allResultAttrs[0]);
FailureOr<llvm::AttrBuilder> attrBuilder =
convertParameterAttrs(function, -1, resultAttrs);
if (failed(attrBuilder))
return failure();
llvmFunc->addRetAttrs(*attrBuilder);
}
// Convert argument attributes.
for (auto [argIdx, llvmArg] : llvm::enumerate(llvmFunc->args())) {
if (DictionaryAttr argAttrs = function.getArgAttrDict(argIdx)) {
FailureOr<llvm::AttrBuilder> attrBuilder =
convertParameterAttrs(function, argIdx, argAttrs);
if (failed(attrBuilder))
return failure();
llvmArg.addAttrs(*attrBuilder);
}
}
// Forward the pass-through attributes to LLVM.
FailureOr<llvm::AttrBuilder> convertedPassthroughAttrs =
convertMLIRAttributesToLLVM(function.getLoc(), llvmFunc->getContext(),
function.getPassthroughAttr(),
function.getPassthroughAttrName());
if (failed(convertedPassthroughAttrs))
return failure();
llvmFunc->addFnAttrs(*convertedPassthroughAttrs);
// Convert visibility attribute.
llvmFunc->setVisibility(convertVisibilityToLLVM(function.getVisibility_()));
// Convert the comdat attribute.
if (std::optional<mlir::SymbolRefAttr> comdat = function.getComdat()) {
auto selectorOp = cast<ComdatSelectorOp>(
symbolTable().lookupNearestSymbolFrom(function, *comdat));
llvmFunc->setComdat(comdatMapping.lookup(selectorOp));
}
if (auto gc = function.getGarbageCollector())
llvmFunc->setGC(gc->str());
if (auto unnamedAddr = function.getUnnamedAddr())
llvmFunc->setUnnamedAddr(convertUnnamedAddrToLLVM(*unnamedAddr));
if (auto alignment = function.getAlignment())
llvmFunc->setAlignment(llvm::MaybeAlign(*alignment));
// Translate the debug information for this function.
debugTranslation->translate(function, *llvmFunc);
}
return success();
}
LogicalResult ModuleTranslation::convertFunctions() {
// Convert functions.
for (auto function : getModuleBody(mlirModule).getOps<LLVMFuncOp>()) {
// Do not convert external functions, but do process dialect attributes
// attached to them.
if (function.isExternal()) {
if (failed(convertDialectAttributes(function, {})))
return failure();
continue;
}
if (failed(convertOneFunction(function)))
return failure();
}
return success();
}
LogicalResult ModuleTranslation::convertIFuncs() {
for (auto op : getModuleBody(mlirModule).getOps<IFuncOp>()) {
llvm::Type *type = convertType(op.getIFuncType());
llvm::GlobalValue::LinkageTypes linkage =
convertLinkageToLLVM(op.getLinkage());
llvm::Constant *resolver;
if (auto *resolverFn = lookupFunction(op.getResolver())) {
resolver = cast<llvm::Constant>(resolverFn);
} else {
Operation *aliasOp = symbolTable().lookupSymbolIn(parentLLVMModule(op),
op.getResolverAttr());
resolver = cast<llvm::Constant>(lookupAlias(aliasOp));
}
auto *ifunc =
llvm::GlobalIFunc::create(type, op.getAddressSpace(), linkage,
op.getSymName(), resolver, llvmModule.get());
addRuntimePreemptionSpecifier(op.getDsoLocal(), ifunc);
ifunc->setUnnamedAddr(convertUnnamedAddrToLLVM(op.getUnnamedAddr()));
ifunc->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));
ifuncMapping.try_emplace(op, ifunc);
}
return success();
}
// Attach global metadata after all globals, aliases, ifuncs, and function
// signatures exist so symbol references can be resolved.
LogicalResult ModuleTranslation::convertGlobalMetadata() {
for (auto op : getModuleBody(mlirModule).getOps<LLVM::GlobalOp>()) {
auto *var = cast<llvm::GlobalVariable>(lookupGlobal(op));
if (FlatSymbolRefAttr associated = op.getAssociatedAttr()) {
FailureOr<llvm::Metadata *> md =
convertSymbolRefToMetadata(associated, [&]() {
return op.emitError("failed to convert associated metadata");
});
if (failed(md))
return failure();
var->setMetadata(llvm::LLVMContext::MD_associated,
llvm::MDNode::get(var->getContext(), *md));
}
if (ArrayAttr absSym = op.getAbsoluteSymbolAttr()) {
SmallVector<llvm::Metadata *> mdOps;
llvm::LLVMContext &ctx = var->getContext();
mdOps.reserve(absSym.size());
for (Attribute attr : absSym) {
auto intAttr = cast<IntegerAttr>(attr);
llvm::IntegerType *ty = llvm::IntegerType::get(
ctx, intAttr.getType().getIntOrFloatBitWidth());
mdOps.push_back(llvm::ConstantAsMetadata::get(
llvm::ConstantInt::get(ty, intAttr.getValue())));
}
var->setMetadata(llvm::LLVMContext::MD_absolute_symbol,
llvm::MDNode::get(ctx, mdOps));
}
}
return success();
}
LogicalResult ModuleTranslation::convertComdats() {
for (auto comdatOp : getModuleBody(mlirModule).getOps<ComdatOp>()) {
for (auto selectorOp : comdatOp.getOps<ComdatSelectorOp>()) {
llvm::Module *module = getLLVMModule();
if (module->getComdatSymbolTable().contains(selectorOp.getSymName()))
return emitError(selectorOp.getLoc())
<< "comdat selection symbols must be unique even in different "
"comdat regions";
llvm::Comdat *comdat = module->getOrInsertComdat(selectorOp.getSymName());
comdat->setSelectionKind(convertComdatToLLVM(selectorOp.getComdat()));
comdatMapping.try_emplace(selectorOp, comdat);
}
}
return success();
}
LogicalResult ModuleTranslation::convertUnresolvedBlockAddress() {
for (auto &[blockAddressOp, llvmCst] : unresolvedBlockAddressMapping) {
BlockAddressAttr blockAddressAttr = blockAddressOp.getBlockAddr();
llvm::BasicBlock *llvmBlock = lookupBlockAddress(blockAddressAttr);
assert(llvmBlock && "expected LLVM blocks to be already translated");
// Update mapping with new block address constant.
auto *llvmBlockAddr = llvm::BlockAddress::get(
lookupFunction(blockAddressAttr.getFunction().getValue()), llvmBlock);
llvmCst->replaceAllUsesWith(llvmBlockAddr);
assert(llvmCst->use_empty() && "expected all uses to be replaced");
cast<llvm::GlobalVariable>(llvmCst)->eraseFromParent();
}
unresolvedBlockAddressMapping.clear();
return success();
}
void ModuleTranslation::setAccessGroupsMetadata(AccessGroupOpInterface op,
llvm::Instruction *inst) {
if (llvm::MDNode *node = loopAnnotationTranslation->getAccessGroups(op))
inst->setMetadata(llvm::LLVMContext::MD_access_group, node);
}
llvm::MDNode *
ModuleTranslation::getOrCreateAliasScope(AliasScopeAttr aliasScopeAttr) {
auto [scopeIt, scopeInserted] =
aliasScopeMetadataMapping.try_emplace(aliasScopeAttr, nullptr);
if (!scopeInserted)
return scopeIt->second;
llvm::LLVMContext &ctx = llvmModule->getContext();
auto dummy = llvm::MDNode::getTemporary(ctx, {});
// Convert the domain metadata node if necessary.
auto [domainIt, insertedDomain] = aliasDomainMetadataMapping.try_emplace(
aliasScopeAttr.getDomain(), nullptr);
if (insertedDomain) {
llvm::SmallVector<llvm::Metadata *, 3> operands;
// Placeholder for potential self-reference.
operands.push_back(dummy.get());
operands.push_back(
llvm::ConstantAsMetadata::get(llvm::ConstantInt::getFalse(ctx)));
if (StringAttr description = aliasScopeAttr.getDomain().getDescription())
operands.push_back(llvm::MDString::get(ctx, description));
domainIt->second = llvm::MDNode::get(ctx, operands);
// Self-reference for uniqueness.
llvm::Metadata *replacement;
if (auto stringAttr =
dyn_cast<StringAttr>(aliasScopeAttr.getDomain().getId()))
replacement = llvm::MDString::get(ctx, stringAttr.getValue());
else
replacement = domainIt->second;
domainIt->second->replaceOperandWith(0, replacement);
}
// Convert the scope metadata node.
assert(domainIt->second && "Scope's domain should already be valid");
llvm::SmallVector<llvm::Metadata *, 3> operands;
// Placeholder for potential self-reference.
operands.push_back(dummy.get());
operands.push_back(domainIt->second);
if (StringAttr description = aliasScopeAttr.getDescription())
operands.push_back(llvm::MDString::get(ctx, description));
scopeIt->second = llvm::MDNode::get(ctx, operands);
// Self-reference for uniqueness.
llvm::Metadata *replacement;
if (auto stringAttr = dyn_cast<StringAttr>(aliasScopeAttr.getId()))
replacement = llvm::MDString::get(ctx, stringAttr.getValue());
else
replacement = scopeIt->second;
scopeIt->second->replaceOperandWith(0, replacement);
return scopeIt->second;
}
llvm::MDNode *ModuleTranslation::getOrCreateAliasScopes(
ArrayRef<AliasScopeAttr> aliasScopeAttrs) {
SmallVector<llvm::Metadata *> nodes;
nodes.reserve(aliasScopeAttrs.size());
for (AliasScopeAttr aliasScopeAttr : aliasScopeAttrs)
nodes.push_back(getOrCreateAliasScope(aliasScopeAttr));
return llvm::MDNode::get(getLLVMContext(), nodes);
}
void ModuleTranslation::setAliasScopeMetadata(AliasAnalysisOpInterface op,
llvm::Instruction *inst) {
auto populateScopeMetadata = [&](ArrayAttr aliasScopeAttrs, unsigned kind) {
if (!aliasScopeAttrs || aliasScopeAttrs.empty())
return;
llvm::MDNode *node = getOrCreateAliasScopes(
llvm::to_vector(aliasScopeAttrs.getAsRange<AliasScopeAttr>()));
inst->setMetadata(kind, node);
};
populateScopeMetadata(op.getAliasScopesOrNull(),
llvm::LLVMContext::MD_alias_scope);
populateScopeMetadata(op.getNoAliasScopesOrNull(),
llvm::LLVMContext::MD_noalias);
}
llvm::MDNode *ModuleTranslation::getTBAANode(TBAATagAttr tbaaAttr) const {
return tbaaMetadataMapping.lookup(tbaaAttr);
}
void ModuleTranslation::setTBAAMetadata(AliasAnalysisOpInterface op,
llvm::Instruction *inst) {
ArrayAttr tagRefs = op.getTBAATagsOrNull();
if (!tagRefs || tagRefs.empty())
return;
// LLVM IR currently does not support attaching more than one TBAA access tag
// to a memory accessing instruction. It may be useful to support this in
// future, but for the time being just ignore the metadata if MLIR operation
// has multiple access tags.
if (tagRefs.size() > 1) {
op.emitWarning() << "TBAA access tags were not translated, because LLVM "
"IR only supports a single tag per instruction";
return;
}
llvm::MDNode *node = getTBAANode(cast<TBAATagAttr>(tagRefs[0]));
inst->setMetadata(llvm::LLVMContext::MD_tbaa, node);
}
void ModuleTranslation::setDereferenceableMetadata(
DereferenceableOpInterface op, llvm::Instruction *inst) {
DereferenceableAttr derefAttr = op.getDereferenceableOrNull();
if (!derefAttr)
return;
llvm::MDNode *derefSizeNode = llvm::MDNode::get(
getLLVMContext(),
llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
llvm::IntegerType::get(getLLVMContext(), 64), derefAttr.getBytes())));
unsigned kindId = derefAttr.getMayBeNull()
? llvm::LLVMContext::MD_dereferenceable_or_null
: llvm::LLVMContext::MD_dereferenceable;
inst->setMetadata(kindId, derefSizeNode);
}
void ModuleTranslation::setBranchWeightsMetadata(WeightedBranchOpInterface op) {
SmallVector<uint32_t> weights;
llvm::transform(op.getWeights(), std::back_inserter(weights),
[](int32_t value) { return static_cast<uint32_t>(value); });
if (weights.empty())
return;
llvm::Instruction *inst = isa<CallOp>(op) ? lookupCall(op) : lookupBranch(op);
assert(inst && "expected the operation to have a mapping to an instruction");
inst->setMetadata(
llvm::LLVMContext::MD_prof,
llvm::MDBuilder(getLLVMContext()).createBranchWeights(weights));
}
LogicalResult ModuleTranslation::createTBAAMetadata() {
llvm::LLVMContext &ctx = llvmModule->getContext();
llvm::IntegerType *offsetTy = llvm::IntegerType::get(ctx, 64);
// Walk the entire module and create all metadata nodes for the TBAA
// attributes. The code below relies on two invariants of the
// `AttrTypeWalker`:
// 1. Attributes are visited in post-order: Since the attributes create a DAG,
// this ensures that any lookups into `tbaaMetadataMapping` for child
// attributes succeed.
// 2. Attributes are only ever visited once: This way we don't leak any
// LLVM metadata instances.
AttrTypeWalker walker;
walker.addWalk([&](TBAARootAttr root) {
llvm::MDNode *node;
if (StringAttr id = root.getId()) {
node = llvm::MDNode::get(ctx, llvm::MDString::get(ctx, id));
} else {
// Anonymous root nodes are self-referencing.
auto selfRef = llvm::MDNode::getTemporary(ctx, {});
node = llvm::MDNode::get(ctx, {selfRef.get()});
node->replaceOperandWith(0, node);
}
tbaaMetadataMapping.insert({root, node});
});
walker.addWalk([&](TBAATypeDescriptorAttr descriptor) {
SmallVector<llvm::Metadata *> operands;
operands.push_back(llvm::MDString::get(ctx, descriptor.getId()));
for (TBAAMemberAttr member : descriptor.getMembers()) {
operands.push_back(tbaaMetadataMapping.lookup(member.getTypeDesc()));
operands.push_back(llvm::ConstantAsMetadata::get(
llvm::ConstantInt::get(offsetTy, member.getOffset())));
}
tbaaMetadataMapping.insert({descriptor, llvm::MDNode::get(ctx, operands)});
});
walker.addWalk([&](TBAATagAttr tag) {
SmallVector<llvm::Metadata *> operands;
operands.push_back(tbaaMetadataMapping.lookup(tag.getBaseType()));
operands.push_back(tbaaMetadataMapping.lookup(tag.getAccessType()));
operands.push_back(llvm::ConstantAsMetadata::get(
llvm::ConstantInt::get(offsetTy, tag.getOffset())));
if (tag.getConstant())
operands.push_back(
llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(offsetTy, 1)));
tbaaMetadataMapping.insert({tag, llvm::MDNode::get(ctx, operands)});
});
mlirModule->walk([&](AliasAnalysisOpInterface analysisOpInterface) {
if (auto attr = analysisOpInterface.getTBAATagsOrNull())
walker.walk(attr);
});
return success();
}
LogicalResult ModuleTranslation::createIdentMetadata() {
if (auto attr = mlirModule->getDiscardableAttrOfType<StringAttr>(
LLVMDialect::getIdentAttrName())) {
StringRef ident = attr;
llvm::LLVMContext &ctx = llvmModule->getContext();
llvm::NamedMDNode *namedMd =
llvmModule->getOrInsertNamedMetadata(LLVMDialect::getIdentAttrName());
llvm::MDNode *md = llvm::MDNode::get(ctx, llvm::MDString::get(ctx, ident));
namedMd->addOperand(md);
}
return success();
}
LogicalResult ModuleTranslation::createCommandlineMetadata() {
if (auto attr = mlirModule->getDiscardableAttrOfType<StringAttr>(
LLVMDialect::getCommandlineAttrName())) {
StringRef cmdLine = attr;
llvm::LLVMContext &ctx = llvmModule->getContext();
llvm::NamedMDNode *nmd = llvmModule->getOrInsertNamedMetadata(
LLVMDialect::getCommandlineAttrName());
llvm::MDNode *md =
llvm::MDNode::get(ctx, llvm::MDString::get(ctx, cmdLine));
nmd->addOperand(md);
}
return success();
}
LogicalResult ModuleTranslation::createDependentLibrariesMetadata() {
if (auto dependentLibrariesAttr = mlirModule->getDiscardableAttr(
LLVM::LLVMDialect::getDependentLibrariesAttrName())) {
auto *nmd =
llvmModule->getOrInsertNamedMetadata("llvm.dependent-libraries");
llvm::LLVMContext &ctx = llvmModule->getContext();
for (auto libAttr :
cast<ArrayAttr>(dependentLibrariesAttr).getAsRange<StringAttr>()) {
auto *md =
llvm::MDNode::get(ctx, llvm::MDString::get(ctx, libAttr.getValue()));
nmd->addOperand(md);
}
}
return success();
}
void ModuleTranslation::setLoopMetadata(Operation *op,
llvm::Instruction *inst) {
LoopAnnotationAttr attr =
TypeSwitch<Operation *, LoopAnnotationAttr>(op)
.Case<LLVM::BrOp, LLVM::CondBrOp>(
[](auto branchOp) { return branchOp.getLoopAnnotationAttr(); });
if (!attr)
return;
llvm::MDNode *loopMD =
loopAnnotationTranslation->translateLoopAnnotation(attr, op);
inst->setMetadata(llvm::LLVMContext::MD_loop, loopMD);
}
void ModuleTranslation::setDisjointFlag(Operation *op, llvm::Value *value) {
auto iface = cast<DisjointFlagInterface>(op);
// We do a dyn_cast here in case the value got folded into a constant.
if (auto *disjointInst = dyn_cast<llvm::PossiblyDisjointInst>(value))
disjointInst->setIsDisjoint(iface.getIsDisjoint());
}
llvm::Type *ModuleTranslation::convertType(Type type) {
return typeTranslator.translateType(type);
}
/// A helper to look up remapped operands in the value remapping table.
SmallVector<llvm::Value *> ModuleTranslation::lookupValues(ValueRange values) {
SmallVector<llvm::Value *> remapped;
remapped.reserve(values.size());
for (Value v : values)
remapped.push_back(lookupValue(v));
return remapped;
}
void ModuleTranslation::remapAllValuesWith(llvm::Value *oldValue,
llvm::Value *newValue) {
if (oldValue == newValue)
return;
for (auto &entry : valueMapping)
if (entry.second == oldValue)
entry.second = newValue;
}
llvm::OpenMPIRBuilder *ModuleTranslation::getOpenMPBuilder() {
if (!ompBuilder) {
ompBuilder = std::make_unique<llvm::OpenMPIRBuilder>(*llvmModule);
// Flags represented as top-level OpenMP dialect attributes are set in
// `OpenMPDialectLLVMIRTranslationInterface::amendOperation()`. Here we set
// the default configuration.
llvm::OpenMPIRBuilderConfig config(
/* IsTargetDevice = */ false, /* IsGPU = */ false,
/* OpenMPOffloadMandatory = */ false,
/* HasRequiresReverseOffload = */ false,
/* HasRequiresUnifiedAddress = */ false,
/* HasRequiresUnifiedSharedMemory = */ false,
/* HasRequiresDynamicAllocators = */ false);
unsigned int defaultAS =
llvmModule->getDataLayout().getProgramAddressSpace();
config.setDefaultTargetAS(defaultAS);
config.setRuntimeCC(llvmModule->getTargetTriple().isSPIRV()
? llvm::CallingConv::SPIR_FUNC
: llvm::CallingConv::C);
ompBuilder->setConfig(std::move(config));
ompBuilder->initialize();
}
return ompBuilder.get();
}
llvm::vfs::FileSystem &ModuleTranslation::getFileSystem() {
if (fileSystem)
return *fileSystem;
return *llvm::vfs::getRealFileSystem();
}
llvm::DILocation *ModuleTranslation::translateLoc(Location loc,
llvm::DILocalScope *scope) {
return debugTranslation->translateLoc(loc, scope);
}
llvm::DIExpression *
ModuleTranslation::translateExpression(LLVM::DIExpressionAttr attr) {
return debugTranslation->translateExpression(attr);
}
llvm::DIGlobalVariableExpression *
ModuleTranslation::translateGlobalVariableExpression(
LLVM::DIGlobalVariableExpressionAttr attr) {
return debugTranslation->translateGlobalVariableExpression(attr);
}
llvm::Metadata *ModuleTranslation::translateDebugInfo(LLVM::DINodeAttr attr) {
return debugTranslation->translate(attr);
}
llvm::RoundingMode
ModuleTranslation::translateRoundingMode(LLVM::RoundingMode rounding) {
return convertRoundingModeToLLVM(rounding);
}
llvm::fp::ExceptionBehavior ModuleTranslation::translateFPExceptionBehavior(
LLVM::FPExceptionBehavior exceptionBehavior) {
return convertFPExceptionBehaviorToLLVM(exceptionBehavior);
}
llvm::NamedMDNode *
ModuleTranslation::getOrInsertNamedModuleMetadata(StringRef name) {
return llvmModule->getOrInsertNamedMetadata(name);
}
static std::unique_ptr<llvm::Module>
prepareLLVMModule(Operation *m, llvm::LLVMContext &llvmContext,
StringRef name) {
m->getContext()->getOrLoadDialect<LLVM::LLVMDialect>();
auto llvmModule = std::make_unique<llvm::Module>(name, llvmContext);
if (auto dataLayoutAttr =
m->getDiscardableAttr(LLVM::LLVMDialect::getDataLayoutAttrName())) {
llvmModule->setDataLayout(cast<StringAttr>(dataLayoutAttr).getValue());
} else {
FailureOr<llvm::DataLayout> llvmDataLayout(llvm::DataLayout(""));
if (auto iface = dyn_cast<DataLayoutOpInterface>(m)) {
if (DataLayoutSpecInterface spec = iface.getDataLayoutSpec()) {
llvmDataLayout =
translateDataLayout(spec, DataLayout(iface), m->getLoc());
}
} else if (auto mod = dyn_cast<ModuleOp>(m)) {
if (DataLayoutSpecInterface spec = mod.getDataLayoutSpec()) {
llvmDataLayout =
translateDataLayout(spec, DataLayout(mod), m->getLoc());
}
}
if (failed(llvmDataLayout))
return nullptr;
llvmModule->setDataLayout(*llvmDataLayout);
}
if (auto targetTripleAttr =
m->getDiscardableAttr(LLVM::LLVMDialect::getTargetTripleAttrName()))
llvmModule->setTargetTriple(
llvm::Triple(cast<StringAttr>(targetTripleAttr).getValue()));
if (auto asmAttr = m->getDiscardableAttr(
LLVM::LLVMDialect::getModuleLevelAsmAttrName())) {
auto asmArrayAttr = dyn_cast<ArrayAttr>(asmAttr);
if (!asmArrayAttr) {
m->emitError("expected an array attribute for a module level asm");
return nullptr;
}
for (Attribute elt : asmArrayAttr) {
auto asmStrAttr = dyn_cast<StringAttr>(elt);
if (!asmStrAttr) {
m->emitError(
"expected a string attribute for each entry of a module level asm");
return nullptr;
}
llvmModule->appendModuleInlineAsm(asmStrAttr.getValue());
}
}
return llvmModule;
}
std::unique_ptr<llvm::Module>
mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
StringRef name, bool disableVerification,
llvm::vfs::FileSystem *fs) {
if (!satisfiesLLVMModule(module)) {
module->emitOpError("can not be translated to an LLVMIR module");
return nullptr;
}
std::unique_ptr<llvm::Module> llvmModule =
prepareLLVMModule(module, llvmContext, name);
if (!llvmModule)
return nullptr;
LLVM::ensureDistinctSuccessors(module);
LLVM::legalizeDIExpressionsRecursively(module);
ModuleTranslation translator(module, std::move(llvmModule), fs);
llvm::IRBuilder<llvm::TargetFolder> llvmBuilder(
*translator.getLLVMModule(),
llvm::TargetFolder(translator.getLLVMModule()->getDataLayout()));
// Convert module before functions and operations inside, so dialect
// attributes can be used to change dialect-specific global configurations via
// `amendOperation()`. These configurations can then influence the translation
// of operations afterwards.
if (failed(translator.convertOperation(*module, llvmBuilder)))
return nullptr;
if (failed(translator.convertComdats()))
return nullptr;
if (failed(translator.convertFunctionSignatures()))
return nullptr;
if (failed(translator.convertGlobalsAndAliases()))
return nullptr;
if (failed(translator.convertIFuncs()))
return nullptr;
if (failed(translator.convertGlobalMetadata()))
return nullptr;
if (failed(translator.convertFunctionMetadata()))
return nullptr;
if (failed(translator.createTBAAMetadata()))
return nullptr;
if (failed(translator.createIdentMetadata()))
return nullptr;
if (failed(translator.createCommandlineMetadata()))
return nullptr;
if (failed(translator.createDependentLibrariesMetadata()))
return nullptr;
// Convert other top-level operations if possible.
for (Operation &o : getModuleBody(module).getOperations()) {
if (!isa<LLVM::LLVMFuncOp, LLVM::AliasOp, LLVM::GlobalOp,
LLVM::GlobalCtorsOp, LLVM::GlobalDtorsOp, LLVM::ComdatOp,
LLVM::IFuncOp>(&o) &&
!o.hasTrait<OpTrait::IsTerminator>() &&
failed(translator.convertOperation(o, llvmBuilder))) {
return nullptr;
}
}
// Operations in function bodies with symbolic references must be converted
// after the top-level operations they refer to are declared, so we do it
// last.
if (failed(translator.convertFunctions()))
return nullptr;
// Now that all MLIR blocks are resolved into LLVM ones, patch block address
// constants to point to the correct blocks.
if (failed(translator.convertUnresolvedBlockAddress()))
return nullptr;
// Add the necessary debug info module flags, if they were not encoded in MLIR
// beforehand.
translator.debugTranslation->addModuleFlagsIfNotPresent();
// Call the OpenMP IR Builder callbacks prior to verifying the module
if (auto *ompBuilder = translator.getOpenMPBuilder())
ompBuilder->finalize();
if (!disableVerification &&
llvm::verifyModule(*translator.llvmModule, &llvm::errs()))
return nullptr;
return std::move(translator.llvmModule);
}