blob: 2b7ff5ef605836988a5bfcb64ab2ed2e96558765 [file] [log] [blame]
//===- DataLayoutInterfaces.cpp - Data Layout Interface Implementation ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "mlir/Interfaces/DataLayoutInterfaces.h"
#include "mlir/IR/BuiltinDialect.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Operation.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/MathExtras.h"
using namespace mlir;
//===----------------------------------------------------------------------===//
// Default implementations
//===----------------------------------------------------------------------===//
/// Reports that the given type is missing the data layout information and
/// exits.
[[noreturn]] static void reportMissingDataLayout(Type type) {
std::string message;
llvm::raw_string_ostream os(message);
os << "neither the scoping op nor the type class provide data layout "
"information for "
<< type;
llvm::report_fatal_error(Twine(os.str()));
}
/// Returns the bitwidth of the index type if specified in the param list.
/// Assumes 64-bit index otherwise.
static unsigned getIndexBitwidth(DataLayoutEntryListRef params) {
if (params.empty())
return 64;
auto attr = params.front().getValue().cast<IntegerAttr>();
return attr.getValue().getZExtValue();
}
unsigned
mlir::detail::getDefaultTypeSize(Type type, const DataLayout &dataLayout,
ArrayRef<DataLayoutEntryInterface> params) {
unsigned bits = getDefaultTypeSizeInBits(type, dataLayout, params);
return llvm::divideCeil(bits, 8);
}
unsigned mlir::detail::getDefaultTypeSizeInBits(Type type,
const DataLayout &dataLayout,
DataLayoutEntryListRef params) {
if (type.isa<IntegerType, FloatType>())
return type.getIntOrFloatBitWidth();
if (auto ctype = type.dyn_cast<ComplexType>()) {
auto et = ctype.getElementType();
auto innerAlignment =
getDefaultPreferredAlignment(et, dataLayout, params) * 8;
auto innerSize = getDefaultTypeSizeInBits(et, dataLayout, params);
// Include padding required to align the imaginary value in the complex
// type.
return llvm::alignTo(innerSize, innerAlignment) + innerSize;
}
// Index is an integer of some bitwidth.
if (type.isa<IndexType>())
return dataLayout.getTypeSizeInBits(
IntegerType::get(type.getContext(), getIndexBitwidth(params)));
// Sizes of vector types are rounded up to those of types with closest
// power-of-two number of elements in the innermost dimension. We also assume
// there is no bit-packing at the moment element sizes are taken in bytes and
// multiplied with 8 bits.
// TODO: make this extensible.
if (auto vecType = type.dyn_cast<VectorType>())
return vecType.getNumElements() / vecType.getShape().back() *
llvm::PowerOf2Ceil(vecType.getShape().back()) *
dataLayout.getTypeSize(vecType.getElementType()) * 8;
if (auto typeInterface = type.dyn_cast<DataLayoutTypeInterface>())
return typeInterface.getTypeSizeInBits(dataLayout, params);
reportMissingDataLayout(type);
}
unsigned mlir::detail::getDefaultABIAlignment(
Type type, const DataLayout &dataLayout,
ArrayRef<DataLayoutEntryInterface> params) {
// Natural alignment is the closest power-of-two number above.
if (type.isa<FloatType, VectorType>())
return llvm::PowerOf2Ceil(dataLayout.getTypeSize(type));
// Index is an integer of some bitwidth.
if (type.isa<IndexType>())
return dataLayout.getTypeABIAlignment(
IntegerType::get(type.getContext(), getIndexBitwidth(params)));
if (auto intType = type.dyn_cast<IntegerType>()) {
return intType.getWidth() < 64
? llvm::PowerOf2Ceil(llvm::divideCeil(intType.getWidth(), 8))
: 4;
}
if (auto ctype = type.dyn_cast<ComplexType>())
return getDefaultABIAlignment(ctype.getElementType(), dataLayout, params);
if (auto typeInterface = type.dyn_cast<DataLayoutTypeInterface>())
return typeInterface.getABIAlignment(dataLayout, params);
reportMissingDataLayout(type);
}
unsigned mlir::detail::getDefaultPreferredAlignment(
Type type, const DataLayout &dataLayout,
ArrayRef<DataLayoutEntryInterface> params) {
// Preferred alignment is same as natural for floats and vectors.
if (type.isa<FloatType, VectorType>())
return dataLayout.getTypeABIAlignment(type);
// Preferred alignment is the cloest power-of-two number above for integers
// (ABI alignment may be smaller).
if (type.isa<IntegerType, IndexType>())
return llvm::PowerOf2Ceil(dataLayout.getTypeSize(type));
if (auto ctype = type.dyn_cast<ComplexType>())
return getDefaultPreferredAlignment(ctype.getElementType(), dataLayout,
params);
if (auto typeInterface = type.dyn_cast<DataLayoutTypeInterface>())
return typeInterface.getPreferredAlignment(dataLayout, params);
reportMissingDataLayout(type);
}
DataLayoutEntryList
mlir::detail::filterEntriesForType(DataLayoutEntryListRef entries,
TypeID typeID) {
return llvm::to_vector<4>(llvm::make_filter_range(
entries, [typeID](DataLayoutEntryInterface entry) {
auto type = entry.getKey().dyn_cast<Type>();
return type && type.getTypeID() == typeID;
}));
}
DataLayoutEntryInterface
mlir::detail::filterEntryForIdentifier(DataLayoutEntryListRef entries,
StringAttr id) {
const auto *it = llvm::find_if(entries, [id](DataLayoutEntryInterface entry) {
if (!entry.getKey().is<StringAttr>())
return false;
return entry.getKey().get<StringAttr>() == id;
});
return it == entries.end() ? DataLayoutEntryInterface() : *it;
}
static DataLayoutSpecInterface getSpec(Operation *operation) {
return llvm::TypeSwitch<Operation *, DataLayoutSpecInterface>(operation)
.Case<ModuleOp, DataLayoutOpInterface>(
[&](auto op) { return op.getDataLayoutSpec(); })
.Default([](Operation *) {
llvm_unreachable("expected an op with data layout spec");
return DataLayoutSpecInterface();
});
}
/// Populates `opsWithLayout` with the list of proper ancestors of `leaf` that
/// are either modules or implement the `DataLayoutOpInterface`.
static void
collectParentLayouts(Operation *leaf,
SmallVectorImpl<DataLayoutSpecInterface> &specs,
SmallVectorImpl<Location> *opLocations = nullptr) {
if (!leaf)
return;
for (Operation *parent = leaf->getParentOp(); parent != nullptr;
parent = parent->getParentOp()) {
llvm::TypeSwitch<Operation *>(parent)
.Case<ModuleOp>([&](ModuleOp op) {
// Skip top-level module op unless it has a layout. Top-level module
// without layout is most likely the one implicitly added by the
// parser and it doesn't have location. Top-level null specification
// would have had the same effect as not having a specification at all
// (using type defaults).
if (!op->getParentOp() && !op.getDataLayoutSpec())
return;
specs.push_back(op.getDataLayoutSpec());
if (opLocations)
opLocations->push_back(op.getLoc());
})
.Case<DataLayoutOpInterface>([&](DataLayoutOpInterface op) {
specs.push_back(op.getDataLayoutSpec());
if (opLocations)
opLocations->push_back(op.getLoc());
});
}
}
/// Returns a layout spec that is a combination of the layout specs attached
/// to the given operation and all its ancestors.
static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf) {
if (!leaf)
return {};
assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
"expected an op with data layout spec");
SmallVector<DataLayoutOpInterface> opsWithLayout;
SmallVector<DataLayoutSpecInterface> specs;
collectParentLayouts(leaf, specs);
// Fast track if there are no ancestors.
if (specs.empty())
return getSpec(leaf);
// Create the list of non-null specs (null/missing specs can be safely
// ignored) from the outermost to the innermost.
auto nonNullSpecs = llvm::to_vector<2>(llvm::make_filter_range(
llvm::reverse(specs),
[](DataLayoutSpecInterface iface) { return iface != nullptr; }));
// Combine the specs using the innermost as anchor.
if (DataLayoutSpecInterface current = getSpec(leaf))
return current.combineWith(nonNullSpecs);
if (nonNullSpecs.empty())
return {};
return nonNullSpecs.back().combineWith(
llvm::makeArrayRef(nonNullSpecs).drop_back());
}
LogicalResult mlir::detail::verifyDataLayoutOp(Operation *op) {
DataLayoutSpecInterface spec = getSpec(op);
// The layout specification may be missing and it's fine.
if (!spec)
return success();
if (failed(spec.verifySpec(op->getLoc())))
return failure();
if (!getCombinedDataLayout(op)) {
InFlightDiagnostic diag =
op->emitError()
<< "data layout does not combine with layouts of enclosing ops";
SmallVector<DataLayoutSpecInterface> specs;
SmallVector<Location> opLocations;
collectParentLayouts(op, specs, &opLocations);
for (Location loc : opLocations)
diag.attachNote(loc) << "enclosing op with data layout";
return diag;
}
return success();
}
//===----------------------------------------------------------------------===//
// DataLayout
//===----------------------------------------------------------------------===//
template <typename OpTy>
void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op) {
if (!originalLayout) {
assert((!op || !op.getDataLayoutSpec()) &&
"could not compute layout information for an op (failed to "
"combine attributes?)");
}
}
mlir::DataLayout::DataLayout() : DataLayout(ModuleOp()) {}
mlir::DataLayout::DataLayout(DataLayoutOpInterface op)
: originalLayout(getCombinedDataLayout(op)), scope(op) {
#if LLVM_ENABLE_ABI_BREAKING_CHECKS
checkMissingLayout(originalLayout, op);
collectParentLayouts(op, layoutStack);
#endif
}
mlir::DataLayout::DataLayout(ModuleOp op)
: originalLayout(getCombinedDataLayout(op)), scope(op) {
#if LLVM_ENABLE_ABI_BREAKING_CHECKS
checkMissingLayout(originalLayout, op);
collectParentLayouts(op, layoutStack);
#endif
}
mlir::DataLayout mlir::DataLayout::closest(Operation *op) {
// Search the closest parent either being a module operation or implementing
// the data layout interface.
while (op) {
if (auto module = dyn_cast<ModuleOp>(op))
return DataLayout(module);
if (auto iface = dyn_cast<DataLayoutOpInterface>(op))
return DataLayout(iface);
op = op->getParentOp();
}
return DataLayout();
}
void mlir::DataLayout::checkValid() const {
#if LLVM_ENABLE_ABI_BREAKING_CHECKS
SmallVector<DataLayoutSpecInterface> specs;
collectParentLayouts(scope, specs);
assert(specs.size() == layoutStack.size() &&
"data layout object used, but no longer valid due to the change in "
"number of nested layouts");
for (auto pair : llvm::zip(specs, layoutStack)) {
Attribute newLayout = std::get<0>(pair);
Attribute origLayout = std::get<1>(pair);
assert(newLayout == origLayout &&
"data layout object used, but no longer valid "
"due to the change in layout attributes");
}
#endif
assert(((!scope && !this->originalLayout) ||
(scope && this->originalLayout == getCombinedDataLayout(scope))) &&
"data layout object used, but no longer valid due to the change in "
"layout spec");
}
/// Looks up the value for the given type key in the given cache. If there is no
/// such value in the cache, compute it using the given callback and put it in
/// the cache before returning.
static unsigned cachedLookup(Type t, DenseMap<Type, unsigned> &cache,
function_ref<unsigned(Type)> compute) {
auto it = cache.find(t);
if (it != cache.end())
return it->second;
auto result = cache.try_emplace(t, compute(t));
return result.first->second;
}
unsigned mlir::DataLayout::getTypeSize(Type t) const {
checkValid();
return cachedLookup(t, sizes, [&](Type ty) {
DataLayoutEntryList list;
if (originalLayout)
list = originalLayout.getSpecForType(ty.getTypeID());
if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
return iface.getTypeSize(ty, *this, list);
return detail::getDefaultTypeSize(ty, *this, list);
});
}
unsigned mlir::DataLayout::getTypeSizeInBits(Type t) const {
checkValid();
return cachedLookup(t, bitsizes, [&](Type ty) {
DataLayoutEntryList list;
if (originalLayout)
list = originalLayout.getSpecForType(ty.getTypeID());
if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
return iface.getTypeSizeInBits(ty, *this, list);
return detail::getDefaultTypeSizeInBits(ty, *this, list);
});
}
unsigned mlir::DataLayout::getTypeABIAlignment(Type t) const {
checkValid();
return cachedLookup(t, abiAlignments, [&](Type ty) {
DataLayoutEntryList list;
if (originalLayout)
list = originalLayout.getSpecForType(ty.getTypeID());
if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
return iface.getTypeABIAlignment(ty, *this, list);
return detail::getDefaultABIAlignment(ty, *this, list);
});
}
unsigned mlir::DataLayout::getTypePreferredAlignment(Type t) const {
checkValid();
return cachedLookup(t, preferredAlignments, [&](Type ty) {
DataLayoutEntryList list;
if (originalLayout)
list = originalLayout.getSpecForType(ty.getTypeID());
if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
return iface.getTypePreferredAlignment(ty, *this, list);
return detail::getDefaultPreferredAlignment(ty, *this, list);
});
}
//===----------------------------------------------------------------------===//
// DataLayoutSpecInterface
//===----------------------------------------------------------------------===//
void DataLayoutSpecInterface::bucketEntriesByType(
DenseMap<TypeID, DataLayoutEntryList> &types,
DenseMap<StringAttr, DataLayoutEntryInterface> &ids) {
for (DataLayoutEntryInterface entry : getEntries()) {
if (auto type = entry.getKey().dyn_cast<Type>())
types[type.getTypeID()].push_back(entry);
else
ids[entry.getKey().get<StringAttr>()] = entry;
}
}
LogicalResult mlir::detail::verifyDataLayoutSpec(DataLayoutSpecInterface spec,
Location loc) {
// First, verify individual entries.
for (DataLayoutEntryInterface entry : spec.getEntries())
if (failed(entry.verifyEntry(loc)))
return failure();
// Second, dispatch verifications of entry groups to types or dialects they
// are are associated with.
DenseMap<TypeID, DataLayoutEntryList> types;
DenseMap<StringAttr, DataLayoutEntryInterface> ids;
spec.bucketEntriesByType(types, ids);
for (const auto &kvp : types) {
auto sampleType = kvp.second.front().getKey().get<Type>();
if (sampleType.isa<IndexType>()) {
assert(kvp.second.size() == 1 &&
"expected one data layout entry for non-parametric 'index' type");
if (!kvp.second.front().getValue().isa<IntegerAttr>())
return emitError(loc)
<< "expected integer attribute in the data layout entry for "
<< sampleType;
continue;
}
if (isa<BuiltinDialect>(&sampleType.getDialect()))
return emitError(loc) << "unexpected data layout for a built-in type";
auto dlType = sampleType.dyn_cast<DataLayoutTypeInterface>();
if (!dlType)
return emitError(loc)
<< "data layout specified for a type that does not support it";
if (failed(dlType.verifyEntries(kvp.second, loc)))
return failure();
}
for (const auto &kvp : ids) {
StringAttr identifier = kvp.second.getKey().get<StringAttr>();
Dialect *dialect = identifier.getReferencedDialect();
// Ignore attributes that belong to an unknown dialect, the dialect may
// actually implement the relevant interface but we don't know about that.
if (!dialect)
continue;
const auto *iface =
dialect->getRegisteredInterface<DataLayoutDialectInterface>();
if (!iface) {
return emitError(loc)
<< "the '" << dialect->getNamespace()
<< "' dialect does not support identifier data layout entries";
}
if (failed(iface->verifyEntry(kvp.second, loc)))
return failure();
}
return success();
}
#include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"
#include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"
#include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"