Google Git Google Git
Sign in
llvm/llvm-project/refs/heads/users/cdevadas/enhance-createFrom-function/./flang/lib/Optimizer/Transforms/FIRToMemRef.cpp
blob: 1542047724e1d891df660660c230e7b48916f095 [file] [edit]
//===-- FIRToMemRef.cpp - Convert FIR loads and stores to MemRef ---------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass lowers FIR dialect memory operations to the MemRef dialect.
// In particular it:
//
// - Rewrites `fir.alloca` to `memref.alloca`.
//
// - Rewrites `fir.load` / `fir.store` to `memref.load` / `memref.store`.
//
// - Allows FIR and MemRef to coexist by introducing `fir.convert` at
// memory-use sites. Memory operations (`memref.load`, `memref.store`,
// `memref.reinterpret_cast`, etc.) see MemRef-typed values, while the
// original FIR-typed values remain available for non-memory uses. For
// example:
//
// %fir_ref = ... : !fir.ref<!fir.array<...>>
// %memref = fir.convert %fir_ref
// : !fir.ref<!fir.array<...>> -> memref<...>
// %val = memref.load %memref[...] : memref<...>
// fir.call @callee(%fir_ref) : (!fir.ref<!fir.array<...>>) -> ()
//
// Here the MemRef-typed value is used for `memref.load`, while the
// original FIR-typed value is preserved for `fir.call`.
//
// - Computes shapes, strides, and indices as needed for slices and shifts
// and emits `memref.reinterpret_cast` when dynamic layout is required
// (TODO: use memref.cast instead).
//
// - Leaves volatile `fir.load` / `fir.store` alone. Memref dialect has no way
// to express volatility currently. These accesses are lowered straight
// to LLVM by the FIR code generation path.
//
//===----------------------------------------------------------------------===//
#include "flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h"
#include "flang/Optimizer/Dialect/FIROps.h"
#include "flang/Optimizer/Dialect/FIROpsSupport.h"
#include "flang/Optimizer/Dialect/FIRType.h"
#include "flang/Optimizer/Dialect/Support/FIRContext.h"
#include "flang/Optimizer/Dialect/Support/KindMapping.h"
#include "flang/Optimizer/Transforms/FIRToMemRefTypeConverter.h"
#include "flang/Optimizer/Transforms/Passes.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/OpenACC/OpenACC.h"
#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/IR/Block.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/Dominance.h"
#include "mlir/IR/Location.h"
#include "mlir/IR/MLIRContext.h"
#include "mlir/IR/Operation.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/IR/Region.h"
#include "mlir/IR/Value.h"
#include "mlir/IR/ValueRange.h"
#include "mlir/IR/Verifier.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Transforms/DialectConversion.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#define DEBUG_TYPE "fir-to-memref"
using namespace mlir;
namespace fir {
#define GEN_PASS_DEF_FIRTOMEMREF
#include "flang/Optimizer/Transforms/Passes.h.inc"
static bool isMarshalLike(Operation *op) {
auto convert = dyn_cast_if_present<fir::ConvertOp>(op);
if (!convert)
return false;
bool resIsMemRef = isa<MemRefType>(convert.getType());
bool argIsMemRef = isa<MemRefType>(convert.getValue().getType());
assert(!(resIsMemRef && argIsMemRef) &&
"unexpected fir.convert memref -> memref in isMarshalLike");
return resIsMemRef || argIsMemRef;
}
/// True when \p a and \p b access the same pointee (or boxed) element type.
/// Used to distinguish a transparent address wrapper from storage association.
static bool sameAccessedElementType(Type a, Type b) {
return fir::unwrapPassByRefType(a) == fir::unwrapPassByRefType(b);
}
/// Peel zero-offset views (e.g. fir.declare) down to the underlying address
/// producer, e.g. fir.array_coor. Stops at a view that changes the accessed
/// element type (e.g. !fir.ref<complex<f32>> to !fir.ref<f32>).
static Value peelZeroOffsetViews(Value memref) {
while (Operation *defOp = memref.getDefiningOp()) {
auto result = cast<OpResult>(memref);
auto view = dyn_cast<fir::FortranObjectViewOpInterface>(defOp);
// Marshal-like fir.convert has its own dedicated path.
// fir.volatile_cast may change volatility, don't peel past it.
if (!view || isMarshalLike(defOp) || isa<fir::VolatileCastOp>(defOp) ||
view.getViewOffset(result) != 0)
break;
Value source = view.getViewSource(result);
if (!sameAccessedElementType(memref.getType(), source.getType()))
break;
memref = source;
}
return memref;
}
using MemRefInfo = FailureOr<std::pair<Value, SmallVector<Value>>>;
static llvm::cl::opt<bool> enableFIRConvertOptimizations(
"enable-fir-convert-opts",
llvm::cl::desc("enable emilinating redundant fir.convert in FIR-to-MemRef"),
llvm::cl::init(false), llvm::cl::Hidden);
class FIRToMemRef : public fir::impl::FIRToMemRefBase<FIRToMemRef> {
public:
void runOnOperation() override;
private:
llvm::SmallSetVector<Operation *, 32> eraseOps;
DominanceInfo *domInfo = nullptr;
void rewriteAlloca(fir::AllocaOp, PatternRewriter &,
FIRToMemRefTypeConverter &);
void rewriteLoadOp(fir::LoadOp, PatternRewriter &,
FIRToMemRefTypeConverter &);
void rewriteStoreOp(fir::StoreOp, PatternRewriter &,
FIRToMemRefTypeConverter &);
MemRefInfo getMemRefInfo(Value, PatternRewriter &, FIRToMemRefTypeConverter &,
Operation *);
MemRefInfo convertArrayCoorOp(Operation *memOp, fir::ArrayCoorOp,
PatternRewriter &, FIRToMemRefTypeConverter &);
/// Returns true if \p coordinateOp can be lowered to an indexed memref access
/// by convertCoordinateArrayOp. This is true when the base is a reference to
/// a statically-shaped scalar array.
bool isArrayIndexingCoordinateOp(fir::CoordinateOp coordinateOp,
FIRToMemRefTypeConverter &) const;
/// Lower a fir.coordinate_of that indexes into a static-extent scalar array
/// (e.g. a struct component like `A%v(i)`) to a memref + index pair.
MemRefInfo convertCoordinateArrayOp(Operation *memOp, fir::CoordinateOp,
PatternRewriter &,
FIRToMemRefTypeConverter &);
void replaceFIRMemrefs(Value, Value, ArrayRef<Value> indices,
PatternRewriter &) const;
FailureOr<Value> getFIRConvert(Operation *memOp, Operation *memref,
PatternRewriter &, FIRToMemRefTypeConverter &);
/// Marshal \p memrefOp itself (no bounds-aware indexing) via getFIRConvert,
/// logging \p debugContext on failure.
MemRefInfo marshalView(Operation *memOp, Operation *memrefOp, Value firMemref,
PatternRewriter &, FIRToMemRefTypeConverter &,
llvm::StringRef debugContext);
FailureOr<SmallVector<Value>> getMemrefIndices(fir::ArrayCoorOp, Operation *,
PatternRewriter &,
Value) const;
bool memrefIsOptional(Operation *) const;
Value canonicalizeIndex(Value, PatternRewriter &) const;
// Logical section information used by FIRToMemRef. For projected slices, the
// descriptor still owns the physical layout, so `sliceVec` intentionally
// stays empty while `shapeVec`/`shiftVec` remain available for index math.
struct SliceInfo {
SmallVector<Value> shapeVec;
SmallVector<Value> shiftVec;
SmallVector<Value> sliceVec;
bool hasProjectedSlice = false;
// Constant value of the first projected-slice field, if any.
std::optional<std::int64_t> projectedSliceStart;
};
template <typename OpTy>
void collectSliceInfoFrom(OpTy op, SliceInfo &info) const;
void foldSliceLbIntoIndices(SmallVectorImpl<Value> &indices,
fir::EmboxOp embox, PatternRewriter &rewriter,
Location loc) const;
void populateShapeAndShift(SmallVectorImpl<Value> &shapeVec,
SmallVectorImpl<Value> &shiftVec,
fir::ShapeShiftOp shift) const;
void populateShift(SmallVectorImpl<Value> &vec, fir::ShiftOp shift) const;
void populateShape(SmallVectorImpl<Value> &vec, fir::ShapeOp shape) const;
/// Recover per-dimension extent SSA values from a shape operand. Inserts
/// `fir.shape_extents` when the defining `fir.shape` or `fir.shapeshift` is
/// not visible (e.g. block argument from control-flow merge).
bool materializeShapeExtents(Value shapeVal, PatternRewriter &rewriter,
Location loc,
SmallVectorImpl<Value> &shapeVec) const;
static fir::SliceOp getSliceOp(Value sliceVal) {
return sliceVal ? sliceVal.getDefiningOp<fir::SliceOp>() : fir::SliceOp{};
}
static bool hasProjectedSlice(fir::SliceOp sliceOp) {
return sliceOp && !sliceOp.getFields().empty();
}
// Returns the constant first projected-slice field, if available.
static std::optional<std::int64_t>
getProjectedSliceStartIfConstant(fir::SliceOp sliceOp) {
auto fields = sliceOp.getFields();
if (fields.empty())
return std::nullopt;
if (std::optional<llvm::APInt> constant =
fir::getIntIfConstant(fields.front()))
return constant->trySExtValue();
return std::nullopt;
}
unsigned getRankFromEmbox(fir::EmboxOp embox) const {
auto memrefType = embox.getMemref().getType();
Type unwrappedType = fir::unwrapRefType(memrefType);
if (auto seqType = dyn_cast<fir::SequenceType>(unwrappedType))
return seqType.getDimension();
return 0;
}
bool isCompilerGeneratedAlloca(Operation *op) const;
void copyAttribute(Operation *from, Operation *to,
llvm::StringRef name) const;
Type getBaseType(Type type, bool complexBaseTypes = false) const;
bool memrefIsDeviceData(Operation *memref) const;
mlir::Attribute findCudaDataAttr(Value val) const;
Value materializeBoxAddressIfNeeded(Value basePtr, PatternRewriter &rewriter,
Location loc) const;
};
void FIRToMemRef::populateShapeAndShift(SmallVectorImpl<Value> &shapeVec,
SmallVectorImpl<Value> &shiftVec,
fir::ShapeShiftOp shift) const {
for (mlir::OperandRange::iterator i = shift.getPairs().begin(),
endIter = shift.getPairs().end();
i != endIter;) {
shiftVec.push_back(*i++);
shapeVec.push_back(*i++);
}
}
bool FIRToMemRef::isCompilerGeneratedAlloca(Operation *op) const {
if (!isa<fir::AllocaOp, memref::AllocaOp>(op))
llvm_unreachable("expected alloca op");
return !op->getAttr("bindc_name") && !op->getAttr("uniq_name");
}
void FIRToMemRef::copyAttribute(Operation *from, Operation *to,
llvm::StringRef name) const {
if (Attribute value = from->getAttr(name))
to->setAttr(name, value);
}
Type FIRToMemRef::getBaseType(Type type, bool complexBaseTypes) const {
if (fir::isa_fir_type(type)) {
type = fir::getFortranElementType(type);
} else if (auto memrefTy = dyn_cast<MemRefType>(type)) {
type = memrefTy.getElementType();
}
if (!complexBaseTypes)
if (auto complexTy = dyn_cast<ComplexType>(type))
type = complexTy.getElementType();
return type;
}
bool FIRToMemRef::memrefIsDeviceData(Operation *memref) const {
if (isa<ACC_DATA_ENTRY_OPS>(memref))
return true;
return cuf::hasDeviceDataAttr(memref);
}
mlir::Attribute FIRToMemRef::findCudaDataAttr(Value val) const {
Value currentVal = val;
llvm::SmallPtrSet<Operation *, 8> visited;
while (currentVal) {
Operation *defOp = currentVal.getDefiningOp();
if (!defOp || !visited.insert(defOp).second)
break;
if (cuf::DataAttributeAttr cudaAttr = cuf::getDataAttr(defOp))
return cudaAttr;
// TODO: This is a best-effort backward walk; it is easy to miss attributes
// as FIR evolves. Long term, it would be preferable if the necessary
// information was carried in the type system (or otherwise made available
// without relying on a walk-back through defining ops).
if (auto reboxOp = dyn_cast<fir::ReboxOp>(defOp)) {
currentVal = reboxOp.getBox();
} else if (auto convertOp = dyn_cast<fir::ConvertOp>(defOp)) {
currentVal = convertOp->getOperand(0);
} else if (auto emboxOp = dyn_cast<fir::EmboxOp>(defOp)) {
currentVal = emboxOp.getMemref();
} else if (auto boxAddrOp = dyn_cast<fir::BoxAddrOp>(defOp)) {
currentVal = boxAddrOp.getVal();
} else if (auto declareOp = dyn_cast<fir::DeclareOp>(defOp)) {
currentVal = declareOp.getMemref();
} else {
break;
}
}
return nullptr;
}
Value FIRToMemRef::materializeBoxAddressIfNeeded(Value basePtr,
PatternRewriter &rewriter,
Location loc) const {
if (!isa<fir::BoxType>(basePtr.getType()))
return basePtr;
auto boxAddrOp = fir::BoxAddrOp::create(rewriter, loc, basePtr);
if (auto cudaAttr = findCudaDataAttr(basePtr))
boxAddrOp->setAttr(cuf::getDataAttrName(), cudaAttr);
return boxAddrOp.getResult();
}
void FIRToMemRef::populateShift(SmallVectorImpl<Value> &vec,
fir::ShiftOp shift) const {
vec.append(shift.getOrigins().begin(), shift.getOrigins().end());
}
void FIRToMemRef::populateShape(SmallVectorImpl<Value> &vec,
fir::ShapeOp shape) const {
vec.append(shape.getExtents().begin(), shape.getExtents().end());
}
bool FIRToMemRef::materializeShapeExtents(
Value shapeVal, PatternRewriter &rewriter, Location loc,
SmallVectorImpl<Value> &shapeVec) const {
if (!shapeVal)
return false;
if (auto shapeOp = shapeVal.getDefiningOp<fir::ShapeOp>()) {
shapeVec.append(shapeOp.getExtents().begin(), shapeOp.getExtents().end());
return true;
}
if (auto ssOp = shapeVal.getDefiningOp<fir::ShapeShiftOp>()) {
shapeVec.append(ssOp.getExtents().begin(), ssOp.getExtents().end());
return true;
}
if (mlir::isa<fir::ShapeType, fir::ShapeShiftType>(shapeVal.getType())) {
auto extentsOp = fir::ShapeExtentsOp::create(rewriter, loc, shapeVal);
shapeVec.append(extentsOp.getExtents().begin(),
extentsOp.getExtents().end());
return true;
}
return false;
}
/// Accumulate the shape/shift/slice operands carried by `op` into `info`.
///
/// Accepts `fir::ArrayCoorOp`, `fir::EmboxOp`, and `fir::ReboxOp`; on any
/// other op type this is a no-op (`if constexpr` guard in the body). Fields
/// of `info` are **appended** to, never cleared -- callers routinely invoke
/// this twice (once for the array_coor, once for the underlying embox/rebox)
/// so ordering matters.
///
/// What each shape operand contributes:
/// - `fir.shape %e0, %e1, ...` -> appends extents to `shapeVec`.
/// - `fir.shape_shift %lb0, %e0, ...` -> appends extents to `shapeVec`
/// and lower bounds to `shiftVec`.
/// - `fir.shift %lb0, %lb1, ...` -> appends lower bounds only,
/// into `shiftVec`.
/// - no shape operand (nullptr) -> `shapeVec`/`shiftVec` unchanged.
///
/// What the slice operand contributes:
/// - `fir.slice %lb0, %ub0, %step0, ...` -> appends *all* triple SSA values
/// to `sliceVec` (per-dim, in
/// Fortran order).
/// - projected slice (extra `%fields`) -> also sets `hasProjectedSlice`
/// and, when the first field is
/// a compile-time constant,
/// `projectedSliceStart`.
/// - no slice operand (nullptr) -> `sliceVec` unchanged.
///
/// Examples (IR shown Fortran-first; sliceVec entries listed in append
/// order):
///
/// 1) `fir.array_coor %arr(%shape) [%slice] %i, %j` where
/// %shape = fir.shape %c4, %c2 ; parent 4x2
/// %slice = fir.slice %c1, %c2, %c1, %c1, %c2, %c1
///
/// Before: info = { shapeVec=[], shiftVec=[], sliceVec=[] }
/// After : info = { shapeVec=[%c4, %c2],
/// shiftVec=[],
/// sliceVec=[%c1, %c2, %c1, // dim 0 triple
/// %c1, %c2, %c1] } // dim 1 triple
///
/// 2) `fir.embox %arr(%shape_shift) [%slice]` where
/// %shape_shift = fir.shape_shift %clb0, %e0, %clb1, %e1
/// %slice = fir.slice %c2, %c2, %c1, %c1, %c1, %c1
///
/// After: info = { shapeVec=[%e0, %e1],
/// shiftVec=[%clb0, %clb1],
/// sliceVec=[%c2, %c2, %c1, // dim 0 triple
/// %c1, %c1, %c1] } // dim 1 triple
///
/// 3) Two-slice stack -- first the array_coor, then the underlying embox:
/// call sequence:
/// collectSliceInfoFrom(arrayCoorOp, info);
/// collectSliceInfoFrom(emboxOp, info);
///
/// Result: `sliceVec` holds the array_coor's triples first, followed by
/// the embox's triples. Downstream helpers (e.g. `getMemrefIndices`) only
/// consume the leading `rank` triples, so the two contributions are
/// positionally distinguishable but not intrinsically labelled.
///
/// 4) Rank-reducing embox slice via scalar subscript
/// (`fir.slice %c1, %c3, %c1, %c2, %undef, %undef, %c1, %c2, %c1`):
/// All 9 triple SSAs are appended to `sliceVec`. Callers detect the
/// scalar-subscript form by testing `isa<fir::UndefOp>` on the ub/step
/// entries.
template <typename OpTy>
void FIRToMemRef::collectSliceInfoFrom(OpTy op, SliceInfo &info) const {
if constexpr (std::is_same_v<OpTy, fir::ArrayCoorOp> ||
std::is_same_v<OpTy, fir::ReboxOp> ||
std::is_same_v<OpTy, fir::EmboxOp>) {
Value shapeVal = op.getShape();
if (shapeVal) {
if (Operation *shapeValOp = shapeVal.getDefiningOp()) {
if (auto shapeOp = dyn_cast<fir::ShapeOp>(shapeValOp)) {
populateShape(info.shapeVec, shapeOp);
} else if (auto shapeShiftOp =
dyn_cast<fir::ShapeShiftOp>(shapeValOp)) {
populateShapeAndShift(info.shapeVec, info.shiftVec, shapeShiftOp);
} else if (auto shiftOp = dyn_cast<fir::ShiftOp>(shapeValOp)) {
populateShift(info.shiftVec, shiftOp);
}
}
}
if (auto sliceOp = getSliceOp(op.getSlice())) {
if (hasProjectedSlice(sliceOp)) {
info.hasProjectedSlice = true;
info.projectedSliceStart = getProjectedSliceStartIfConstant(sliceOp);
}
auto triples = sliceOp.getTriples();
info.sliceVec.append(triples.begin(), triples.end());
}
}
}
void FIRToMemRef::rewriteAlloca(fir::AllocaOp firAlloca,
PatternRewriter &rewriter,
FIRToMemRefTypeConverter &typeConverter) {
if (!typeConverter.convertibleType(firAlloca.getInType()))
return;
if (typeConverter.isEmptyArray(firAlloca.getType()))
return;
rewriter.setInsertionPointAfter(firAlloca);
Type type = firAlloca.getType();
MemRefType memrefTy = typeConverter.convertMemrefType(type);
Location loc = firAlloca.getLoc();
SmallVector<Value> sizes = firAlloca.getOperands();
std::reverse(sizes.begin(), sizes.end());
auto alloca = memref::AllocaOp::create(rewriter, loc, memrefTy, sizes);
copyAttribute(firAlloca, alloca, firAlloca.getBindcNameAttrName());
copyAttribute(firAlloca, alloca, firAlloca.getUniqNameAttrName());
copyAttribute(firAlloca, alloca, cuf::getDataAttrName());
copyAttribute(firAlloca, alloca, acc::getVarNameAttrName());
auto convert = fir::ConvertOp::create(rewriter, loc, type, alloca);
rewriter.replaceOp(firAlloca, convert);
if (isCompilerGeneratedAlloca(alloca)) {
for (Operation *userOp : convert->getUsers()) {
if (auto declareOp = dyn_cast<fir::DeclareOp>(userOp)) {
LLVM_DEBUG(llvm::dbgs()
<< "FIRToMemRef: removing declare for compiler temp:\n";
declareOp->dump());
declareOp->replaceAllUsesWith(convert);
eraseOps.insert(userOp);
}
}
}
}
bool FIRToMemRef::memrefIsOptional(Operation *op) const {
if (auto declare = dyn_cast<fir::DeclareOp>(op)) {
if (fir::FortranVariableOpInterface(declare).isOptional())
return true;
Value operand = declare.getMemref();
Operation *operandOp = operand.getDefiningOp();
if (operandOp && isa<fir::AbsentOp>(operandOp))
return true;
}
for (mlir::Value result : op->getResults())
for (mlir::Operation *userOp : result.getUsers())
if (isa<fir::IsPresentOp>(userOp))
return true;
// TODO: If `op` is not a `fir.declare`, OPTIONAL information may still be
// present on a related `fir.declare` reached by tracing the address/box
// through common forwarding ops (e.g. `fir.convert`, `fir.rebox`,
// `fir.embox`, `fir.box_addr`), then checking `declare.isOptional()`. Add the
// search after FIR improves on it.
return false;
}
static Value castTypeToIndexType(Value originalValue,
PatternRewriter &rewriter) {
if (originalValue.getType().isIndex())
return originalValue;
Type indexType = rewriter.getIndexType();
return arith::IndexCastOp::create(rewriter, originalValue.getLoc(), indexType,
originalValue);
}
/// Fold the embox's slice per-dim (lb - 1) shift and stride into `indices`
/// after `getMemrefIndices` has produced them. Handles both range and
/// rank-reducing (scalar-subscript) slice triples uniformly.
///
/// Necessary because `getMemrefIndices` only consumes the first `rank*3`
/// entries of `sliceInfo.sliceVec` (the array_coor's own slice); the embox's
/// triples sit at `sliceVec[rank*3 .. 2*rank*3-1]` and are otherwise dropped.
///
/// Behaviour, per Fortran dim `d` of the embox slice (memref position
/// `m = parentRank - 1 - d`):
/// - Range triple `(lb, ub, stride)`:
/// indices[m] = indices[m] * stride + (lb - 1)
/// `indices[m]` on entry is a *0-based* box index produced by
/// `getMemrefIndices`. Converting a 0-based box index j to a 0-based
/// parent index via the embox range (lb, stride) gives:
/// parent_0based = lb + j*stride - 1 = j*stride + (lb - 1)
/// For unit stride this reduces to `indices[m] += (lb - 1)`.
/// - Scalar-subscript triple `(lb, undef, undef)`:
/// indices[m] = (lb - 1)
/// Overwrites the memref position outright. The dim is collapsed in the
/// box's rank, but the reinterpret_cast still keeps a memref index slot
/// for it (with the correct parent stride). Placing `(lb - 1)` there is
/// mathematically equivalent to shifting the flat offset by
/// `(lb - 1) * parentStride[d]`, without needing any parent-shape
/// lookups from the caller.
///
/// Example A -- non-rank-reducing (2-D), unit strides:
///
/// `fir.array_coor %box(...)[%innerSlice] %i, %j` with
/// - array_coor slice: identity in both dims
/// - embox slice : `fir.slice %c2,%c2,%c1, %c1,%c1,%c1`
/// (Fortran dim 0 lb = 2, stride = 1)
///
/// Incoming `indices` (memref order): [ %j', %i' ]
/// Fold:
/// - dim 0 (range, stride=1, lb=2): indices[1] = %i'*1 + 1
/// - dim 1 (range, stride=1, lb=1): indices[0] = %j'*1 + 0
/// Result: [ %j', %i' + 1 ]
///
/// Example B -- non-rank-reducing (1-D), non-unit stride (e.g. a(1:10:2)):
///
/// `fir.array_coor %box(...)[%innerSlice] %i` with
/// - array_coor slice: identity
/// - embox slice : `fir.slice %c1,%c10,%c2`
/// (Fortran dim 0 lb = 1, stride = 2)
///
/// Incoming `indices`: [ %i' ] where %i' = i - 1 (from getMemrefIndices)
/// Fold:
/// - dim 0 (range, stride=2, lb=1): indices[0] = %i'*2 + 0
/// Result: [ (i-1)*2 ]
///
/// Example C -- rank-reducing (3-D parent, dim 1 collapsed):
///
/// %parent : !fir.ref<!fir.array<3x2x2xi32>>
/// %shape = fir.shape %c3, %c2, %c2
/// %eslc = fir.slice %c1, %c3, %c1, %c2, %undef, %undef, %c1, %c2, %c1
/// %box = fir.embox %parent(%shape) [%eslc]
/// : !fir.box<!fir.array<3x2xi32>>
/// %bshape = fir.shape %c3, %c2 ; box's rank-2 shape
/// %addr = fir.array_coor %box(%bshape) %ii, %jj
/// : (!fir.box<!fir.array<3x2xi32>>, !fir.shape<2>, index, index)
/// -> !fir.ref<i32>
///
/// parentRank = 3. `getRankFromEmbox` returns the parent's rank, so
/// `getMemrefIndices` builds a rank-3 index vector (memref order):
/// indices = [ %jj', <ac scalar sub for collapsed dim>, %ii' ]
/// ^ dim 2 ^ dim 1 (collapsed) ^ dim 0
///
/// Fold:
/// - dim 0 (range, stride=1, lb=1): indices[2] = %ii'*1 + 0
/// - dim 1 (scalar, lb=2): indices[1] = 1 (overwrite)
/// - dim 2 (range, stride=1, lb=1): indices[0] = %jj'*1 + 0
///
/// Final: `memref.load %reinterpret_cast[%jj', 1, %ii'] : memref<?x?x?xi32,
/// strided<[?, ?, ?], offset: ?>>` -- stride 3 on the middle position
/// lands on the correct plane, `offset` stays at 0.
void FIRToMemRef::foldSliceLbIntoIndices(SmallVectorImpl<Value> &indices,
fir::EmboxOp embox,
PatternRewriter &rewriter,
Location loc) const {
auto sliceOp = getSliceOp(embox.getSlice());
if (!sliceOp)
return;
auto triples = sliceOp.getTriples();
unsigned parentRank = triples.size() / 3;
if (parentRank == 0)
return;
auto isUndef = [](Value v) {
return v && v.getDefiningOp<fir::UndefOp>() != nullptr;
};
Value cOne = arith::ConstantIndexOp::create(rewriter, loc, 1);
for (unsigned d = 0; d < parentRank; ++d) {
unsigned m = parentRank - 1 - d;
if (m >= indices.size())
continue;
Value lb = castTypeToIndexType(triples[d * 3], rewriter);
Value delta = arith::SubIOp::create(rewriter, loc, lb, cOne);
bool isScalar = isUndef(triples[d * 3 + 1]) || isUndef(triples[d * 3 + 2]);
if (isScalar) {
indices[m] = delta;
} else {
// Scale the box-relative index by the embox stride before adding the lb
// offset. For non-unit strides (e.g. a(1:10:2)) omitting this would give
// the wrong physical element.
Value strideVal = castTypeToIndexType(triples[d * 3 + 2], rewriter);
Value scaled =
arith::MulIOp::create(rewriter, loc, indices[m], strideVal);
indices[m] = arith::AddIOp::create(rewriter, loc, scaled, delta);
}
}
}
static bool shouldUseBoundaryBitcast(mlir::Type fromTy, mlir::Type toTy) {
auto isBitcastCompatibleScalarType = [](mlir::Type ty) {
return mlir::isa<mlir::IntegerType, mlir::FloatType, fir::LogicalType>(
ty) ||
(mlir::isa<fir::CharacterType>(ty) &&
mlir::cast<fir::CharacterType>(ty).getLen() ==
fir::CharacterType::singleton());
};
auto getKnownScalarBitWidth = [](mlir::Type ty) -> std::optional<unsigned> {
if (auto intTy = mlir::dyn_cast<mlir::IntegerType>(ty))
return intTy.getWidth();
if (auto floatTy = mlir::dyn_cast<mlir::FloatType>(ty))
return floatTy.getWidth();
return std::nullopt;
};
if (fromTy == toTy)
return false;
const bool fromStd = fir::isa_std_type(fromTy);
const bool toStd = fir::isa_std_type(toTy);
if (fromStd == toStd)
return false;
if (!isBitcastCompatibleScalarType(fromTy) ||
!isBitcastCompatibleScalarType(toTy))
return false;
auto fromBits = getKnownScalarBitWidth(fromTy);
auto toBits = getKnownScalarBitWidth(toTy);
if (fromBits && toBits && *fromBits != *toBits)
return false;
return true;
}
static mlir::Value createTypeConversion(PatternRewriter &rewriter,
mlir::Location loc, mlir::Type toTy,
mlir::Value value) {
if (shouldUseBoundaryBitcast(value.getType(), toTy))
return fir::BitcastOp::create(rewriter, loc, toTy, value);
return fir::ConvertOp::create(rewriter, loc, toTy, value);
}
/// Build the 0-based memref indices for the `memref.load`/`memref.store`
/// that will replace `arrayCoorOp`. Returns the indices in **memref order**
/// (the reverse of Fortran col-major dim order).
///
/// Parameters:
/// - `arrayCoorOp` : the source `fir.array_coor`.
/// - `memref` : the defining op of `arrayCoorOp.getMemref()`. Only
/// used to detect a `fir.embox` base (from which extra
/// shape/slice info is pulled -- rebox is deliberately
/// *not* queried for shape/slice info).
/// - `one` : a shared `arith.constant 1 : index` used as the
/// default step/shift when the array_coor has neither
/// slice nor shape_shift.
///
/// Returns `failure()` when the array_coor's index count matches neither
/// the full parent rank nor the non-scalar-subscript rank -- the pass then
/// declines the rewrite.
///
/// Effective rank and slice collection:
/// - Starts with `rank = arrayCoorOp.getIndices().size()`.
/// - If the base is a `fir.embox`, `rank` is overridden to the parent
/// rank (via `getRankFromEmbox`), and `collectSliceInfoFrom(embox, ...)`
/// appends the embox's shape/slice triples after the array_coor's.
/// - `sliceLbs` / `sliceStrides` are then extracted from *the full*
/// `sliceVec`, but the two loops below only read positions
/// `[0..rank-1]` of `sliceLbs`/`sliceStrides` (i.e. `sliceVec` entries
/// `[0..rank*3-1]`). When both the array_coor and the embox have a
/// slice, the embox's triples sit at `sliceVec[rank*3..2*rank*3-1]`
/// and are *not* consumed here.
///
/// Two-pass index assembly (Fortran dim order):
/// Pass 1 -- scalar subscripts:
/// For each Fortran dim `i`, if the array_coor's slice's step-triple
/// is `fir.undef` (scalar-subscript form `lb, undef, undef`), push
/// `sliceLb - shift` and mark `filledPositions[i] = true`. This "eats"
/// the sourced-index count for that dim.
/// Pass 2 -- range dims:
/// For each non-filled dim, take one input from `arrayCoorOp.getIndices()`
/// and lower to
/// delta = index - (1 or shift) ; 0-based within the slice
/// scaled = delta * slice_step
/// slice_off = sliceLb - shift
/// finalIndex = scaled + slice_off
///
/// The vector is then reversed to memref (row-major) order.
///
/// Example A -- plain `fir.array_coor` over a raw ref, no slice:
///
/// %shape = fir.shape %c4, %c2 : (index, index) -> !fir.shape<2>
/// %addr = fir.array_coor %arr(%shape) %i, %j
/// : (!fir.ref<!fir.array<4x2xi32>>, !fir.shape<2>, index, index)
/// -> !fir.ref<i32>
///
/// Emitted (Fortran order, before reverse):
/// [ %i - 1, %j - 1 ]
/// Returned (memref order):
/// [ %j - 1, %i - 1 ] ; memref<2x4> dim 0 = Fortran dim 1
///
/// Example B -- explicit slice on the array_coor (no embox):
///
/// %slice = fir.slice %c2, %c8, %c2, %c1, %c7, %c3
/// %addr = fir.array_coor %arr(%shape) [%slice] %i, %j
///
/// Fortran-dim-0 finalIndex = (%i - 1) * 2 + (2 - 1) = 2*%i - 1
/// Fortran-dim-1 finalIndex = (%j - 1) * 3 + (1 - 1) = 3*(%j - 1)
/// Returned (memref order): [ Fortran-dim-1, Fortran-dim-0 ]
///
/// Example C -- rank-reducing scalar subscript on the array_coor's slice:
///
/// %slice = fir.slice %c1, %c3, %c1, %k, %undef, %undef ; dim1 scalar@k
/// %addr = fir.array_coor %box(%boxShape) [%slice] %i
/// : (!fir.box<...>, !fir.shape<2>, !fir.slice<2>, index)
/// -> !fir.ref<i32>
///
/// Pass 1 marks Fortran dim 1 filled with (%k - 1).
/// Pass 2 computes Fortran dim 0 from %i.
/// Returned (memref order): [ %k - 1, %i - 1 ]
/// Note the array_coor has only *one* input index -- the `rank ==
/// nonScalarRank` branch (`hasReducedRankIndices`) is what makes this
/// validate.
FailureOr<SmallVector<Value>>
FIRToMemRef::getMemrefIndices(fir::ArrayCoorOp arrayCoorOp, Operation *memref,
PatternRewriter &rewriter, Value one) const {
IndexType indexTy = rewriter.getIndexType();
SmallVector<Value> indices;
Location loc = arrayCoorOp->getLoc();
SliceInfo sliceInfo;
int rank = arrayCoorOp.getIndices().size();
collectSliceInfoFrom(arrayCoorOp, sliceInfo);
// Only collect shape/shift from fir.embox, never from fir.rebox.
//
// The distinction is a Fortran descriptor invariant:
//
// | fir.embox | fir.rebox
// --------------|------------------------|---------------------------
// base_addr | raw pointer | pre-adjusted by (lb-1)*stride
// Index formula | index - lb | index - 1
// Collect shift | yes | no
//
// fir.embox creates a box from a raw pointer so base_addr is not adjusted;
// the shift must be collected so index arithmetic subtracts lb correctly.
// fir.rebox re-boxes a live descriptor whose base_addr the runtime has
// already adjusted downward by (lb-1)*stride; collecting the shift here
// would subtract the lower bound a second time, giving the wrong element.
if (auto embox = dyn_cast_or_null<fir::EmboxOp>(memref)) {
collectSliceInfoFrom(embox, sliceInfo);
rank = getRankFromEmbox(embox);
}
SmallVector<Value> &shiftVec = sliceInfo.shiftVec;
SmallVector<Value> &sliceVec = sliceInfo.sliceVec;
SmallVector<Value> sliceLbs, sliceStrides;
for (size_t i = 0; i < sliceVec.size(); i += 3) {
sliceLbs.push_back(castTypeToIndexType(sliceVec[i], rewriter));
sliceStrides.push_back(castTypeToIndexType(sliceVec[i + 2], rewriter));
}
const bool isShifted = !shiftVec.empty();
const bool isSliced = !sliceVec.empty();
ValueRange idxs = arrayCoorOp.getIndices();
Value zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
SmallVector<bool> filledPositions(rank, false);
for (int i = 0; i < rank; ++i) {
Value step = isSliced ? sliceStrides[i] : one;
Operation *stepOp = step.getDefiningOp();
if (stepOp && mlir::isa_and_nonnull<fir::UndefOp>(stepOp)) {
Value shift = isShifted ? shiftVec[i] : one;
Value sliceLb = isSliced ? sliceLbs[i] : shift;
Value offset = arith::SubIOp::create(rewriter, loc, sliceLb, shift);
indices.push_back(offset);
filledPositions[i] = true;
} else {
indices.push_back(zero);
}
}
const int nonScalarRank = llvm::count(filledPositions, false);
const bool hasReducedRankIndices =
static_cast<int>(idxs.size()) == nonScalarRank;
const bool hasFullRankIndices = static_cast<int>(idxs.size()) == rank;
if (!hasReducedRankIndices && !hasFullRankIndices)
return failure();
int arrayCoorIdx = 0;
for (int i = 0; i < rank; ++i) {
if (filledPositions[i])
continue;
Value sourceIndex;
if (hasFullRankIndices) {
// Canonicalized rank-reducing array_coor may carry full-rank indices
// (including scalar-sliced dimensions).
sourceIndex = idxs[i];
} else {
assert((unsigned int)arrayCoorIdx < idxs.size() &&
"empty dimension should be eliminated\n");
sourceIndex = idxs[arrayCoorIdx];
}
Value index = canonicalizeIndex(sourceIndex, rewriter);
Type cTy = index.getType();
if (!llvm::isa<IndexType>(cTy)) {
assert(cTy.isSignlessInteger() && "expected signless integer type");
index = arith::IndexCastOp::create(rewriter, loc, indexTy, index);
}
Value shift = isShifted ? shiftVec[i] : one;
Value stride = isSliced ? sliceStrides[i] : one;
Value sliceLb = isSliced ? sliceLbs[i] : shift;
// When the array_coor has an explicit slice with a shape_shift (i.e.
// non-default lower bounds), the indices are in the shape_shift
// coordinate space; subtract the lower bound (shift) to get 0-based
// memref indices. Otherwise (the slice comes from an embox, or the
// shape has no shift), the indices are 1-based section indices;
// subtract 1.
bool indicesAreFortran = isShifted && arrayCoorOp.getSlice() != nullptr;
Value indexAdjustment =
(isSliced && !indicesAreFortran)
? arith::ConstantIndexOp::create(rewriter, loc, 1)
: shift;
Value delta = arith::SubIOp::create(rewriter, loc, index, indexAdjustment);
Value scaled = arith::MulIOp::create(rewriter, loc, delta, stride);
Value offset = arith::SubIOp::create(rewriter, loc, sliceLb, shift);
Value finalIndex = arith::AddIOp::create(rewriter, loc, scaled, offset);
indices[i] = finalIndex;
if (hasReducedRankIndices)
arrayCoorIdx++;
}
std::reverse(indices.begin(), indices.end());
return indices;
}
MemRefInfo
FIRToMemRef::convertArrayCoorOp(Operation *memOp, fir::ArrayCoorOp arrayCoorOp,
PatternRewriter &rewriter,
FIRToMemRefTypeConverter &typeConverter) {
IndexType indexTy = rewriter.getIndexType();
Value firMemref = arrayCoorOp.getMemref();
if (!typeConverter.convertibleMemrefType(firMemref.getType()))
return failure();
if (typeConverter.isEmptyArray(firMemref.getType()))
return failure();
Location loc = arrayCoorOp->getLoc();
// Prefer lowering the array-coordinates computation to a memref + indices.
// This allows erasing fir.array_coor when it is only used by load/store even
// if the base address is a block argument (e.g. region arguments).
Operation *memref = nullptr;
FailureOr<Value> converted;
if (auto blockArg = dyn_cast<BlockArgument>(firMemref)) {
rewriter.setInsertionPoint(arrayCoorOp);
Value basePtr = materializeBoxAddressIfNeeded(blockArg, rewriter, loc);
Type memrefTy = typeConverter.convertMemrefType(basePtr.getType());
converted =
fir::ConvertOp::create(rewriter, loc, memrefTy, basePtr).getResult();
rewriter.setInsertionPointAfter(arrayCoorOp);
} else if ((memref = firMemref.getDefiningOp()) &&
enableFIRConvertOptimizations && isMarshalLike(memref) &&
!fir::isa_fir_type(firMemref.getType())) {
converted = firMemref;
rewriter.setInsertionPoint(arrayCoorOp);
} else {
Operation *arrayCoorOperation = arrayCoorOp.getOperation();
rewriter.setInsertionPoint(arrayCoorOp);
if (memrefIsOptional(memref)) {
auto ifOp = arrayCoorOperation->getParentOfType<scf::IfOp>();
if (ifOp) {
Operation *condition = ifOp.getCondition().getDefiningOp();
if (condition && isa<fir::IsPresentOp>(condition))
if (condition->getOperand(0) == firMemref) {
if (arrayCoorOperation->getParentRegion() == &ifOp.getThenRegion())
rewriter.setInsertionPointToStart(
&(ifOp.getThenRegion().front()));
else if (arrayCoorOperation->getParentRegion() ==
&ifOp.getElseRegion())
rewriter.setInsertionPointToStart(
&(ifOp.getElseRegion().front()));
}
}
}
converted = getFIRConvert(memOp, memref, rewriter, typeConverter);
if (failed(converted))
return failure();
rewriter.setInsertionPointAfter(arrayCoorOp);
}
SliceInfo sliceInfo;
collectSliceInfoFrom(arrayCoorOp, sliceInfo);
if (auto embox = firMemref.getDefiningOp<fir::EmboxOp>())
collectSliceInfoFrom(embox, sliceInfo);
else if (auto rebox = firMemref.getDefiningOp<fir::ReboxOp>())
collectSliceInfoFrom(rebox, sliceInfo);
if (!sliceInfo.hasProjectedSlice && sliceInfo.shapeVec.empty()) {
auto shapeVal = arrayCoorOp.getShape();
if (shapeVal &&
mlir::isa<fir::ShapeType, fir::ShapeShiftType>(shapeVal.getType())) {
rewriter.setInsertionPoint(arrayCoorOp);
if (!materializeShapeExtents(shapeVal, rewriter, loc, sliceInfo.shapeVec))
return failure();
rewriter.setInsertionPointAfter(arrayCoorOp);
}
}
Value one = arith::ConstantIndexOp::create(rewriter, loc, 1);
FailureOr<SmallVector<Value>> failureOrIndices =
getMemrefIndices(arrayCoorOp, memref, rewriter, one);
if (failed(failureOrIndices))
return failure();
SmallVector<Value> indices = *failureOrIndices;
if (converted == firMemref)
return std::pair{*converted, indices};
Value convertedVal = *converted;
MemRefType memRefTy = dyn_cast<MemRefType>(convertedVal.getType());
bool isRebox = firMemref.getDefiningOp<fir::ReboxOp>() != nullptr;
bool isDescriptor = mlir::isa<fir::BaseBoxType>(firMemref.getType()) ||
firMemref.getDefiningOp<fir::BoxAddrOp>() != nullptr;
// For complex projections, reinterpret memref<d0×...×complex<T>> as
// memref<d0×...×2×T> and append the component index (0=re, 1=im) so that
// each load/store touches exactly sizeof(T) bytes.
auto srcTy = cast<MemRefType>((*converted).getType());
std::optional<int64_t> complexPartIdx;
if (sliceInfo.hasProjectedSlice) {
if (auto complexTy = dyn_cast<mlir::ComplexType>(srcTy.getElementType())) {
if (!sliceInfo.projectedSliceStart ||
(*sliceInfo.projectedSliceStart != 0 &&
*sliceInfo.projectedSliceStart != 1)) {
LLVM_DEBUG(
llvm::dbgs()
<< "FIRToMemRef: projected complex slice selector must be constant "
"0 (real) or 1 (imaginary), bailing out of conversion\n");
return failure();
}
complexPartIdx = *sliceInfo.projectedSliceStart;
SmallVector<int64_t> shape(srcTy.getShape());
shape.push_back(2);
convertedVal =
fir::ConvertOp::create(
rewriter, loc, MemRefType::get(shape, complexTy.getElementType()),
*converted)
.getResult();
memRefTy = cast<MemRefType>(convertedVal.getType());
indices.push_back(
arith::ConstantIndexOp::create(rewriter, loc, *complexPartIdx));
}
}
// Static shape does not imply contiguous layout for descriptor-backed
// entities (e.g. boxed array sections with non-unit stride). Projected
// complex %re/%im also need reinterpret_cast even when the converted type
// is statically shaped (e.g. memref<Nx2xT>).
if (!complexPartIdx && memRefTy.hasStaticShape() && !isDescriptor)
return std::pair{*converted, indices};
const unsigned acRank = arrayCoorOp.getIndices().size();
unsigned rank = acRank;
if (auto embox = firMemref.getDefiningOp<fir::EmboxOp>())
rank = getRankFromEmbox(embox);
SmallVector<Value> sizes;
sizes.reserve(rank);
SmallVector<Value> strides;
strides.reserve(rank);
SmallVector<Value> &shapeVec = sliceInfo.shapeVec;
const bool firMemrefIsBox = mlir::isa<fir::BaseBoxType>(firMemref.getType());
const bool firMemrefIsEmbox =
firMemref.getDefiningOp<fir::EmboxOp>() != nullptr;
// Pick how to derive sizes/strides for the reinterpret_cast view:
//
// shapeVec path: synthesize row-major strides from fir.shape extents.
// Valid when the converted MemRef describes a contiguous storage block:
// either the array_coor base is a raw ref/heap/ptr (no descriptor at
// all), or it is a fir.box produced by fir.embox -- getFIRConvert
// rewinds to embox.getMemref()/box_addr(embox) in that case, so the
// reinterpret_cast operates on the underlying contiguous ref. This
// matches CodeGen XArrayCoorOp's non-boxed branch.
//
// box_dims path: query the descriptor at runtime. Required when:
// (a) we have no shape information at all; or
// (b) the array_coor base is a fir.box that is NOT a fir.embox result.
const bool descriptorOwnsLayout =
shapeVec.empty() || (firMemrefIsBox && !firMemrefIsEmbox);
if (descriptorOwnsLayout) {
// Plain `!fir.ref` without recoverable shape extents cannot use fir.box_*.
if (shapeVec.empty() && !sliceInfo.hasProjectedSlice && !isDescriptor &&
!isRebox)
return failure();
// Complex %re/%im: memref_stride = box_dims_byte_stride / sizeof(T),
Value boxElementSize =
complexPartIdx
? arith::ConstantIndexOp::create(
rewriter, loc,
memRefTy.getElementType().getIntOrFloatBitWidth() / 8)
: fir::BoxEleSizeOp::create(rewriter, loc, indexTy, firMemref)
.getResult();
for (unsigned i = 0; i < rank; ++i) {
Value dim = arith::ConstantIndexOp::create(rewriter, loc, rank - i - 1);
auto boxDims = fir::BoxDimsOp::create(rewriter, loc, indexTy, indexTy,
indexTy, firMemref, dim);
// TODO: when an explicit fir.shape/fir.shape_shift is available
// (shapeVec non-empty), prefer its extents over the descriptor's
// box_dims extent result. For boxed array_coor the shape extents must
// agree with the descriptor's runtime extents, so either source is
// correct; using the shape would let constant extents reach the
// reinterpret_cast and improve downstream analysis.
Value extent = boxDims->getResult(1);
sizes.push_back(castTypeToIndexType(extent, rewriter));
Value byteStride = boxDims->getResult(2);
Value div =
arith::DivSIOp::create(rewriter, loc, byteStride, boxElementSize);
strides.push_back(castTypeToIndexType(div, rewriter));
}
} else {
Value oneIdx =
arith::ConstantIndexOp::create(rewriter, arrayCoorOp->getLoc(), 1);
// shapeVec is populated by collectSliceInfoFrom in the order:
// [<arrayCoor's shape (acRank entries)>, <embox's shape (rank entries)>]
// When both contribute (firMemrefIsEmbox && arrayCoorOp has a slice),
// shapeVec[0..acRank-1] are the box's visible extents and
// shapeVec[acRank..acRank+rank-1] are the parent's extents. Strides for
// the reinterpret_cast must come from the parent's contiguous element
// strides, not the box's extents (using the latter yields an outer stride
// that walks by the slice's size instead of the parent's leading dim).
//
// acRank == rank in the non-rank-reducing case; acRank < rank when the
// embox slice collapses one or more dims via scalar subscripts.
//
// Example -- 4x2 parent, slice a(1:2, :) keeping 2 elements per dim:
// %parent : !fir.ref<!fir.array<4x2xi32>>
// %shape = fir.shape %c4, %c2
// %eslc = fir.slice %c1, %c2, %c1, %c1, %c2, %c1 ; a(1:2, :)
// %box = fir.embox %parent(%shape) [%eslc]
// :
// !fir.box<!fir.array<2x2xi32>>
// ... %addr = fir.array_coor %box(...) [%innerSlc] %i, %j
//
// After collectSliceInfoFrom is called on both the array_coor and the
// embox, shapeVec becomes (four entries, Fortran order):
// [ box_dim0_extent(=2), box_dim1_extent(=2), ; array_coor shape
// parent_dim0_extent(=4), parent_dim1_extent(=2) ] ; embox shape
//
// For rank = 2 the loop below emits (memref order, outer -> inner):
// size[outer] = shapeVec[1] = 2 (slice's dim1)
// stride[outer] = shapeVec[parentShapeStartIdx + 0] = 4 (parent's
// dim0) size[inner] = shapeVec[0] = 2 (slice's
// dim0) stride[inner] = 1
//
// Without the `parentShapeStartIdx` shift, `stride[outer]` would be
// `shapeVec[0] = 2` -- the slice's own extent -- and successive outer
// steps would walk by 2 elements instead of the parent's 4-element row,
// clobbering the wrong columns.
const bool hasParentShape = firMemrefIsEmbox && arrayCoorOp.getSlice() &&
shapeVec.size() >= acRank + rank;
const unsigned parentShapeStartIdx = hasParentShape ? acRank : 0;
for (unsigned i = rank - 1; i > 0; --i) {
// Sizes are always the box/slice's visible extents (shapeVec[0..rank-1]).
Value size = shapeVec[i];
sizes.push_back(castTypeToIndexType(size, rewriter));
// Strides use the parent's extents (via `parentShapeStartIdx`).
Value stride = shapeVec[parentShapeStartIdx + 0];
for (unsigned j = 1; j <= i - 1; ++j)
stride = arith::MulIOp::create(
rewriter, loc, shapeVec[parentShapeStartIdx + j], stride);
if (complexPartIdx)
stride = arith::MulIOp::create(
rewriter, loc, stride,
arith::ConstantIndexOp::create(rewriter, loc, 2));
strides.push_back(castTypeToIndexType(stride, rewriter));
}
sizes.push_back(castTypeToIndexType(shapeVec[0], rewriter));
// shapeVec strides count array elements (complexes). After fir.convert to
// memref<...x2xT>, each step along an array dim must skip two scalars (re
// then im), so multiply by 2. (Box path uses byte_stride / sizeof(T) for
// the same spacing; no /8 here because extents are already index units.)
if (complexPartIdx)
strides.push_back(arith::ConstantIndexOp::create(rewriter, loc, 2));
else
strides.push_back(oneIdx);
}
// fir.convert above already made memref<...x2xT>; sizes/strides built so far
// cover only the array section (rank from array_coor). Finish the
// reinterpret_cast layout with the pair dim that view already has: extent 2
// (re and im), stride 1 (contiguous scalars — index 0/1 from array_coor).
if (complexPartIdx) {
sizes.push_back(arith::ConstantIndexOp::create(rewriter, loc, 2));
strides.push_back(arith::ConstantIndexOp::create(rewriter, loc, 1));
++rank;
}
int64_t dynamicOffset = ShapedType::kDynamic;
SmallVector<int64_t> dynamicStrides(rank, ShapedType::kDynamic);
auto stridedLayout = StridedLayoutAttr::get(convertedVal.getContext(),
dynamicOffset, dynamicStrides);
SmallVector<int64_t> dynamicShape(rank, ShapedType::kDynamic);
memRefTy =
MemRefType::get(dynamicShape, memRefTy.getElementType(), stridedLayout);
Value offset = arith::ConstantIndexOp::create(rewriter, loc, 0);
// Fold the embox's slice contribution into the memref indices.
// getMemrefIndices only consumed the array_coor's own slice; the embox's
// triples are still unconsumed at this point. Handles both range and
// rank-reducing (scalar-subscript) dims uniformly.
if (firMemrefIsEmbox && !complexPartIdx && arrayCoorOp.getSlice()) {
auto embox = firMemref.getDefiningOp<fir::EmboxOp>();
foldSliceLbIntoIndices(indices, embox, rewriter, loc);
}
auto reinterpret = memref::ReinterpretCastOp::create(
rewriter, loc, memRefTy, convertedVal, offset, sizes, strides);
Value result = reinterpret->getResult(0);
return std::pair{result, indices};
}
FailureOr<Value>
FIRToMemRef::getFIRConvert(Operation *memOp, Operation *op,
PatternRewriter &rewriter,
FIRToMemRefTypeConverter &typeConverter) {
if (enableFIRConvertOptimizations && !op->hasOneUse() &&
!memrefIsOptional(op)) {
for (Operation *userOp : op->getUsers()) {
if (auto convertOp = dyn_cast<fir::ConvertOp>(userOp)) {
Value converted = convertOp.getResult();
if (!isa<MemRefType>(converted.getType()))
continue;
if (userOp->getParentOp() == memOp->getParentOp() &&
domInfo->dominates(userOp, memOp))
return converted;
}
}
}
assert(op->getNumResults() == 1 && "expecting one result");
Value basePtr = op->getResult(0);
MemRefType memrefTy = typeConverter.convertMemrefType(basePtr.getType());
Type baseTy = memrefTy.getElementType();
if (fir::isa_std_type(baseTy) && memrefTy.getRank() == 0) {
if (auto convertOp = basePtr.getDefiningOp<fir::ConvertOp>()) {
Value input = convertOp.getOperand();
if (auto alloca = input.getDefiningOp<memref::AllocaOp>()) {
assert(alloca.getType() == memrefTy && "expected same types");
if (isCompilerGeneratedAlloca(alloca))
return alloca.getResult();
}
}
}
const Location loc = op->getLoc();
if (isa<fir::BoxType>(basePtr.getType())) {
Operation *baseOp = basePtr.getDefiningOp();
basePtr = materializeBoxAddressIfNeeded(basePtr, rewriter, loc);
memrefTy = typeConverter.convertMemrefType(basePtr.getType());
if (baseOp) {
auto sameBaseBoxTypes = [&](Type baseType, Type memrefType) -> bool {
Type emboxBaseTy = getBaseType(baseType, true);
Type emboxMemrefTy = getBaseType(memrefType, true);
return emboxBaseTy == emboxMemrefTy;
};
if (auto embox = dyn_cast_or_null<fir::EmboxOp>(baseOp)) {
// A projected slice changes the element type of the boxed view. We
// can only lower it here when the storage element is complex<T> and
// the projection is the real or imaginary part (i.e. %re / %im). For
// such cases sizeof(complex<T>) == 2*sizeof(T), so
// divsi(byte_stride, elesize) is always an exact integer.
//
// Derived-type component projections (e.g. a%x, a%y) may produce a
// non-integer element-unit stride (e.g. sizeof(T)=24,
// sizeof(complex<f64>)=16 -> 24/16 = 1 after truncation, which is
// wrong). For those, the type-restriction check below fires and we
// bail out, leaving the ops for downstream FIR-to-LLVM lowering.
auto isComplexComponentProjection = [&](fir::EmboxOp embox) -> bool {
if (!hasProjectedSlice(getSliceOp(embox.getSlice())))
return false;
Type memTy = fir::unwrapRefType(embox.getMemref().getType());
if (auto seqTy = dyn_cast<fir::SequenceType>(memTy))
memTy = seqTy.getEleTy();
return mlir::isa<mlir::ComplexType>(memTy);
};
bool projectedSlice = isComplexComponentProjection(embox);
if (!projectedSlice &&
!sameBaseBoxTypes(embox.getType(), embox.getMemref().getType())) {
LLVM_DEBUG(llvm::dbgs()
<< "FIRToMemRef: embox base type and memref type are not "
"the same, bailing out of conversion\n");
return failure();
}
if (embox.getSlice() &&
embox.getSlice().getDefiningOp<fir::SliceOp>()) {
Type originalType = embox.getMemref().getType();
basePtr = embox.getMemref();
if (typeConverter.convertibleMemrefType(originalType)) {
auto convertedMemrefTy =
typeConverter.convertMemrefType(originalType);
memrefTy = convertedMemrefTy;
} else {
return failure();
}
}
}
if (auto rebox = dyn_cast<fir::ReboxOp>(baseOp)) {
if (!sameBaseBoxTypes(rebox.getType(), rebox.getBox().getType())) {
LLVM_DEBUG(llvm::dbgs()
<< "FIRToMemRef: rebox base type and box type are not the "
"same, bailing out of conversion\n");
return failure();
}
Type originalType = rebox.getBox().getType();
if (auto boxTy = dyn_cast<fir::BoxType>(originalType))
originalType = boxTy.getElementType();
if (!typeConverter.convertibleMemrefType(originalType)) {
return failure();
} else {
auto convertedMemrefTy =
typeConverter.convertMemrefType(originalType);
memrefTy = convertedMemrefTy;
}
}
}
}
auto convert = fir::ConvertOp::create(rewriter, loc, memrefTy, basePtr);
return convert->getResult(0);
}
MemRefInfo FIRToMemRef::marshalView(Operation *memOp, Operation *memrefOp,
Value firMemref, PatternRewriter &rewriter,
FIRToMemRefTypeConverter &typeConverter,
llvm::StringRef debugContext) {
FailureOr<Value> converted =
getFIRConvert(memOp, memrefOp, rewriter, typeConverter);
if (failed(converted)) {
LLVM_DEBUG(llvm::dbgs() << "FIRToMemRef: unable to create convert for "
<< debugContext << ":\n";
firMemref.dump());
return failure();
}
SmallVector<Value> indices;
return std::pair{*converted, indices};
}
/// Peephole-simplify an index-shaped SSA value before it gets fed into
/// memref index arithmetic. Returns a (possibly newly-created) `Value`;
/// the input is left untouched. Callers must not assume the result is
/// `index`-typed -- they typically follow up with an explicit
/// `arith.index_cast` when needed (see `getMemrefIndices` and the
/// `fir.coordinate_of` rewriter).
///
/// Handled patterns (all other inputs are returned as-is):
///
/// 1. Block argument
/// %arg : any type
/// -> `%arg` unchanged (nothing to inspect).
///
/// 2. `arith.constant` of an integer type that isn't `index`
/// %c5_i64 = arith.constant 5 : i64
/// -> new `%c5 = arith.constant 5 : index` at the same location.
/// (Constants that are already `index` are returned unchanged.)
///
/// 3. `arith.extsi %ic` where the operand of the `extsi` is
/// `arith.index_cast`:
/// %ic = arith.index_cast %x : index to i32
/// %ext = arith.extsi %ic : i32 to i64
/// -> peels back to `%x` (the ext+cast pair is a no-op on the
/// original index-typed value). When the immediate producer isn't an
/// `index_cast`, the `extsi` is stripped and its operand returned
/// (with any further canonicalization the operand itself qualifies
/// for) -- the width extension is dropped either way; callers must
/// `index_cast` if they need an index-typed result.
///
/// 4. `arith.addi %a, %b`
/// -> recursively canonicalize both operands. If neither operand
/// changed, the original op is returned as-is. Otherwise, if the
/// canonicalized operand types match, build a new `arith.addi` at the
/// same location, carrying over the original overflow flags.
/// If the canonicalized operand types diverge, returns the original
/// op untouched (the caller can still `index_cast` externally).
///
/// Only these four patterns fire -- this is intentionally a narrow peephole,
/// not a general folder. Multiplication, sub, cast chains through other ops,
/// etc. all pass through untouched.
///
/// Idempotence: safe to call repeatedly. An already-index constant / a
/// non-recognized op returns immediately.
Value FIRToMemRef::canonicalizeIndex(Value index,
PatternRewriter &rewriter) const {
if (auto blockArg = dyn_cast<BlockArgument>(index))
return index;
Operation *op = index.getDefiningOp();
if (auto constant = dyn_cast<arith::ConstantIntOp>(op)) {
if (!constant.getType().isIndex()) {
Value v = arith::ConstantIndexOp::create(rewriter, op->getLoc(),
constant.value());
return v;
}
return constant;
}
if (auto extsi = dyn_cast<arith::ExtSIOp>(op)) {
Value operand = extsi.getOperand();
if (auto indexCast = operand.getDefiningOp<arith::IndexCastOp>()) {
Value v = indexCast.getOperand();
return v;
}
return canonicalizeIndex(operand, rewriter);
}
if (auto add = dyn_cast<arith::AddIOp>(op)) {
Value lhs = canonicalizeIndex(add.getLhs(), rewriter);
Value rhs = canonicalizeIndex(add.getRhs(), rewriter);
// Neither operand simplified, so a rebuilt op would be an exact duplicate.
// Reuse the original instead.
if (lhs == add.getLhs() && rhs == add.getRhs())
return index;
if (lhs.getType() == rhs.getType()) {
// Carry over overflow flags when width unchanged only.
if (lhs.getType() == add.getType())
return arith::AddIOp::create(rewriter, op->getLoc(), lhs, rhs,
add.getOverflowFlags());
return arith::AddIOp::create(rewriter, op->getLoc(), lhs, rhs);
}
}
return index;
}
bool FIRToMemRef::isArrayIndexingCoordinateOp(
fir::CoordinateOp coordinateOp,
FIRToMemRefTypeConverter &typeConverter) const {
// The base must be a reference/pointer/heap to a sequence/array type.
Type baseType = coordinateOp.getRef().getType();
Type unwrapped = fir::dyn_cast_ptrEleTy(baseType);
if (!unwrapped)
return false;
auto seqTy = dyn_cast<fir::SequenceType>(unwrapped);
if (!seqTy)
return false;
// Restrict to fully static extents — dynamic arrays would need a shape
// operand (which coordinate_of lacks) to build a valid memref descriptor.
if (fir::hasDynamicSize(seqTy))
return false;
// The element type must be a convertible scalar — no derived types.
if (!typeConverter.convertibleMemrefType(baseType))
return false;
return true;
}
MemRefInfo FIRToMemRef::convertCoordinateArrayOp(
Operation *memOp, fir::CoordinateOp coordinateOp, PatternRewriter &rewriter,
FIRToMemRefTypeConverter &typeConverter) {
Value firBase = coordinateOp.getRef();
Location loc = coordinateOp->getLoc();
IndexType indexTy = rewriter.getIndexType();
if (typeConverter.isEmptyArray(firBase.getType()))
return failure();
// Convert the base ref/heap/ptr to a memref. convertMemrefType reverses the
// FIR column-major shape to row-major, keeping it in sync with index reversal
// below.
rewriter.setInsertionPoint(coordinateOp);
FailureOr<Value> converted;
if (isa<BlockArgument>(firBase)) {
Type memrefTy = typeConverter.convertMemrefType(firBase.getType());
if (!memrefTy)
return failure();
converted =
fir::ConvertOp::create(rewriter, loc, memrefTy, firBase).getResult();
} else {
converted =
getFIRConvert(memOp, firBase.getDefiningOp(), rewriter, typeConverter);
if (failed(converted))
return failure();
}
rewriter.setInsertionPointAfter(coordinateOp);
// The converted memref has static shape — no reinterpret_cast needed.
assert(cast<MemRefType>(converted->getType()).hasStaticShape() &&
"expected static shape for coordinate_of array base");
// Collect and normalize the 0-based coor indices
SmallVector<Value> indices;
for (Value v : coordinateOp.getCoor()) {
v = canonicalizeIndex(v, rewriter);
if (!isa<IndexType>(v.getType()))
v = arith::IndexCastOp::create(rewriter, loc, indexTy, v);
indices.push_back(v);
}
std::reverse(indices.begin(), indices.end());
return std::pair{*converted, indices};
}
MemRefInfo FIRToMemRef::getMemRefInfo(Value firMemref,
PatternRewriter &rewriter,
FIRToMemRefTypeConverter &typeConverter,
Operation *memOp) {
Operation *memrefOp = firMemref.getDefiningOp();
if (!memrefOp) {
if (auto blockArg = dyn_cast<BlockArgument>(firMemref)) {
rewriter.setInsertionPoint(memOp);
Type memrefTy = typeConverter.convertMemrefType(blockArg.getType());
if (auto mt = dyn_cast<MemRefType>(memrefTy))
if (auto inner = llvm::dyn_cast<MemRefType>(mt.getElementType()))
memrefTy = inner;
Value converted = fir::ConvertOp::create(rewriter, blockArg.getLoc(),
memrefTy, blockArg);
SmallVector<Value> indices;
return std::pair{converted, indices};
}
llvm_unreachable(
"FIRToMemRef: expected defining op or block argument for FIR memref");
}
// If a zero-offset same-type view wraps an fir.array_coor, dispatch on the
// array_coor so bounds-aware lowering runs. Storage-association converts
// (e.g. a complex element passed as a real dummy) stay as the view so marshal
// keeps the access type.
Value peeled = peelZeroOffsetViews(firMemref);
if (auto arrayCoorOp =
dyn_cast_or_null<fir::ArrayCoorOp>(peeled.getDefiningOp())) {
if (sameAccessedElementType(firMemref.getType(), arrayCoorOp.getType())) {
firMemref = peeled;
memrefOp = arrayCoorOp;
}
}
if (auto arrayCoorOp = dyn_cast<fir::ArrayCoorOp>(memrefOp)) {
MemRefInfo memrefInfo =
convertArrayCoorOp(memOp, arrayCoorOp, rewriter, typeConverter);
if (succeeded(memrefInfo)) {
for (auto user : memrefOp->getUsers()) {
if (!isa<fir::LoadOp, fir::StoreOp>(user)) {
LLVM_DEBUG(
llvm::dbgs()
<< "FIRToMemRef: array memref used by unsupported op:\n";
firMemref.dump(); user->dump());
return memrefInfo;
}
}
eraseOps.insert(memrefOp);
}
return memrefInfo;
}
rewriter.setInsertionPoint(memOp);
if (isMarshalLike(memrefOp))
return marshalView(memOp, memrefOp, firMemref, rewriter, typeConverter,
"marshal-like convert");
if (auto coordinateOp = dyn_cast<fir::CoordinateOp>(memrefOp)) {
// Fast path: coordinate_of used as a plain array indexer on a static-extent
// scalar array (e.g. a struct component `A%v(i)`).
if (isArrayIndexingCoordinateOp(coordinateOp, typeConverter)) {
MemRefInfo memrefInfo = convertCoordinateArrayOp(memOp, coordinateOp,
rewriter, typeConverter);
if (succeeded(memrefInfo)) {
LLVM_DEBUG(llvm::dbgs()
<< "FIRToMemRef: converted coordinate_of array indexer\n");
for (auto user : memrefOp->getUsers()) {
if (!isa<fir::LoadOp, fir::StoreOp>(user)) {
LLVM_DEBUG(
llvm::dbgs()
<< "FIRToMemRef: coordinate_of used by non-load/store, "
"skipping erase\n";
firMemref.dump(); user->dump());
return memrefInfo;
}
}
eraseOps.insert(memrefOp);
return memrefInfo;
}
}
// Fallback: struct field access or dynamic array — produce a rank-0 scalar
// memref from the leaf reference.
return marshalView(memOp, coordinateOp, firMemref, rewriter, typeConverter,
"derived-type memref");
}
if (memrefIsDeviceData(memrefOp))
return marshalView(memOp, memrefOp, firMemref, rewriter, typeConverter,
"device-data memref");
// Remaining zero-offset view (fir.declare, ref-to-ref fir.convert,
// fir.box_addr, fir.volatile_cast, ...): no array_coor underneath, so
// marshal the view itself to keep naming and getFIRConvert dedup.
if (isa<fir::FortranObjectViewOpInterface>(memrefOp))
return marshalView(memOp, memrefOp, firMemref, rewriter, typeConverter,
"view memref");
LLVM_DEBUG(llvm::dbgs()
<< "FIRToMemRef: unable to create convert for memref value:\n";
firMemref.dump());
return failure();
}
void FIRToMemRef::replaceFIRMemrefs(Value firMemref, Value converted,
ArrayRef<Value> indices,
PatternRewriter &rewriter) const {
// converted is only a base memref paired with indices, not a standalone
// address-equivalent for firMemref, so don't redirect other users to it.
if (!indices.empty())
return;
Operation *op = firMemref.getDefiningOp();
if (op && isMarshalLike(op))
return;
SmallPtrSet<Operation *, 4> worklist;
for (auto user : firMemref.getUsers()) {
if (isMarshalLike(user) || isa<fir::LoadOp, fir::StoreOp>(user))
continue;
if (!domInfo->dominates(converted, user))
continue;
if (!(isa<omp::AtomicCaptureOp>(user->getParentOp()) ||
isa<acc::AtomicCaptureOp>(user->getParentOp())))
worklist.insert(user);
}
Type ty = firMemref.getType();
for (auto op : worklist) {
// If op is directly inside a LoopWrapperInterface region, inserting before
// op would violate the single-nested-op invariant. Walk up the wrapper
// chain and insert before the outermost wrapper instead.
Operation *insertBefore = op;
while (mlir::isa<omp::LoopWrapperInterface>(insertBefore->getParentOp()))
insertBefore = insertBefore->getParentOp();
rewriter.setInsertionPoint(insertBefore);
Location loc = op->getLoc();
Value replaceConvert = fir::ConvertOp::create(rewriter, loc, ty, converted);
op->replaceUsesOfWith(firMemref, replaceConvert);
}
worklist.clear();
for (auto user : firMemref.getUsers()) {
if (isMarshalLike(user) || isa<fir::LoadOp, fir::StoreOp>(user))
continue;
if (isa<omp::AtomicCaptureOp>(user->getParentOp()) ||
isa<acc::AtomicCaptureOp>(user->getParentOp()))
if (domInfo->dominates(converted, user))
worklist.insert(user);
}
if (worklist.empty())
return;
while (!worklist.empty()) {
Operation *parentOp = (*worklist.begin())->getParentOp();
Value replaceConvert;
SmallVector<Operation *> erase;
for (auto op : worklist) {
if (op->getParentOp() != parentOp)
continue;
if (!replaceConvert) {
rewriter.setInsertionPoint(parentOp);
replaceConvert =
fir::ConvertOp::create(rewriter, op->getLoc(), ty, converted);
}
op->replaceUsesOfWith(firMemref, replaceConvert);
erase.push_back(op);
}
for (auto op : erase)
worklist.erase(op);
}
}
void FIRToMemRef::rewriteLoadOp(fir::LoadOp load, PatternRewriter &rewriter,
FIRToMemRefTypeConverter &typeConverter) {
if (fir::isa_volatile_type(load.getMemref().getType())) {
LLVM_DEBUG(llvm::dbgs() << "FIRToMemRef: keeping volatile load in FIR:\n";
load.dump());
return;
}
Value firMemref = load.getMemref();
if (!typeConverter.convertibleType(firMemref.getType()))
return;
LLVM_DEBUG(llvm::dbgs() << "FIRToMemRef: attempting to convert FIR load:\n";
load.dump(); firMemref.dump());
MemRefInfo memrefInfo =
getMemRefInfo(firMemref, rewriter, typeConverter, load.getOperation());
if (failed(memrefInfo))
return;
Type originalType = load.getResult().getType();
Value converted = memrefInfo->first;
SmallVector<Value> indices = memrefInfo->second;
LLVM_DEBUG(llvm::dbgs()
<< "FIRToMemRef: convert for FIR load created successfully:\n";
converted.dump());
rewriter.setInsertionPointAfter(load);
Attribute attr = (load.getOperation())->getAttr("tbaa");
memref::LoadOp loadOp =
rewriter.replaceOpWithNewOp<memref::LoadOp>(load, converted, indices);
if (attr)
loadOp.getOperation()->setAttr("tbaa", attr);
LLVM_DEBUG(llvm::dbgs() << "FIRToMemRef: new memref.load op:\n";
loadOp.dump(); assert(succeeded(verify(loadOp))));
if (loadOp.getType() != originalType) {
Value castVal =
createTypeConversion(rewriter, loadOp.getLoc(), originalType, loadOp);
loadOp.getResult().replaceAllUsesExcept(castVal, castVal.getDefiningOp());
}
if (!isa<fir::LogicalType>(originalType))
replaceFIRMemrefs(firMemref, converted, indices, rewriter);
}
void FIRToMemRef::rewriteStoreOp(fir::StoreOp store, PatternRewriter &rewriter,
FIRToMemRefTypeConverter &typeConverter) {
if (fir::isa_volatile_type(store.getMemref().getType()) ||
fir::isa_volatile_type(store.getValue().getType())) {
LLVM_DEBUG(llvm::dbgs() << "FIRToMemRef: keeping volatile store in FIR:\n";
store.dump());
return;
}
Value firMemref = store.getMemref();
if (!typeConverter.convertibleType(firMemref.getType()))
return;
LLVM_DEBUG(llvm::dbgs() << "FIRToMemRef: attempting to convert FIR store:\n";
store.dump(); firMemref.dump());
MemRefInfo memrefInfo =
getMemRefInfo(firMemref, rewriter, typeConverter, store.getOperation());
if (failed(memrefInfo))
return;
Value converted = memrefInfo->first;
SmallVector<Value> indices = memrefInfo->second;
LLVM_DEBUG(
llvm::dbgs()
<< "FIRToMemRef: convert for FIR store created successfully:\n";
converted.dump());
Value value = store.getValue();
rewriter.setInsertionPointAfter(store);
Type convertedType = typeConverter.convertType(value.getType());
if (convertedType != value.getType())
value =
createTypeConversion(rewriter, store.getLoc(), convertedType, value);
Attribute attr = store.getOperation()->getAttr("tbaa");
memref::StoreOp storeOp = rewriter.replaceOpWithNewOp<memref::StoreOp>(
store, value, converted, indices);
if (attr)
storeOp.getOperation()->setAttr("tbaa", attr);
LLVM_DEBUG(llvm::dbgs() << "FIRToMemRef: new memref.store op:\n";
storeOp.dump(); assert(succeeded(verify(storeOp))));
bool isLogicalRef = false;
if (fir::ReferenceType refTy =
llvm::dyn_cast<fir::ReferenceType>(firMemref.getType()))
isLogicalRef = llvm::isa<fir::LogicalType>(refTy.getEleTy());
if (!isLogicalRef)
replaceFIRMemrefs(firMemref, converted, indices, rewriter);
}
// Lower operand and result type of FIR logical operation to get rid
// of bitcast after loads from memref and before store to memref
// storing arrays of logicals as integers.
template <typename Op>
static void rewriteLogicalOperation(Op op, PatternRewriter &rewriter,
FIRToMemRefTypeConverter &typeConverter) {
mlir::Type oldType = op.getResult().getType();
mlir::Type convertedTy = typeConverter.convertType(oldType);
if (convertedTy == oldType)
return;
rewriter.setInsertionPoint(op);
mlir::Location loc = op.getLoc();
// Inserted bitcast from/to logical will be folded with the one created
// around load/store.
mlir::Value lhs =
createTypeConversion(rewriter, loc, convertedTy, op.getLhs());
mlir::Value rhs =
createTypeConversion(rewriter, loc, convertedTy, op.getRhs());
auto newOp = Op::create(rewriter, loc, convertedTy, lhs, rhs);
mlir::Value result = createTypeConversion(rewriter, loc, oldType, newOp);
rewriter.replaceOp(op, result);
}
void FIRToMemRef::runOnOperation() {
LLVM_DEBUG(llvm::dbgs() << "Enter FIRToMemRef()\n");
func::FuncOp op = getOperation();
MLIRContext *context = op.getContext();
ModuleOp mod = op->getParentOfType<ModuleOp>();
FIRToMemRefTypeConverter typeConverter(mod);
typeConverter.setConvertComplexTypes(true);
PatternRewriter rewriter(context);
domInfo = new DominanceInfo(op);
op.walk([&](fir::AllocaOp alloca) {
rewriteAlloca(alloca, rewriter, typeConverter);
});
op.walk([&](Operation *op) {
llvm::TypeSwitch<Operation *>(op)
.Case<fir::LoadOp>([&](auto loadOp) {
rewriteLoadOp(loadOp, rewriter, typeConverter);
})
.Case<fir::StoreOp>([&](auto storeOp) {
rewriteStoreOp(storeOp, rewriter, typeConverter);
})
.Case<fir::LogicalAndOp, fir::LogicalOrOp, fir::EqvOp, fir::NeqvOp>(
[&](auto logicalOp) {
rewriteLogicalOperation(logicalOp, rewriter, typeConverter);
})
.Default([](Operation *) {});
});
for (auto eraseOp : eraseOps)
rewriter.eraseOp(eraseOp);
eraseOps.clear();
if (domInfo)
delete domInfo;
LLVM_DEBUG(llvm::dbgs() << "After FIRToMemRef()\n"; op.dump();
llvm::dbgs() << "Exit FIRToMemRef()\n";);
}
} // namespace fir