| //===-- CUFAllocDelay.cpp -------------------------------------------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // Delay cuf.alloc of descriptor (box) types from function entry to just before | |
| // their first use, possibly in a later block that dominates every use. The | |
| // target block is never on a CFG cycle, so the alloc runs at most once. This | |
| // defers cudaMallocManaged calls so that users can call cudaSetDevice before | |
| // any CUDA context is created. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "flang/Optimizer/Dialect/CUF/CUFOps.h" | |
| #include "flang/Optimizer/Dialect/FIRDialect.h" | |
| #include "flang/Optimizer/Dialect/FIROps.h" | |
| #include "flang/Optimizer/Dialect/FIRType.h" | |
| #include "mlir/IR/Block.h" | |
| #include "mlir/IR/Dominance.h" | |
| #include "mlir/IR/RegionGraphTraits.h" | |
| #include "mlir/Pass/Pass.h" | |
| #include "llvm/ADT/DenseMap.h" | |
| #include "llvm/ADT/SCCIterator.h" | |
| #include "llvm/ADT/SmallPtrSet.h" | |
| #include "llvm/ADT/SmallVector.h" | |
| namespace fir { | |
| #define GEN_PASS_DEF_CUFALLOCDELAY | |
| #include "flang/Optimizer/Transforms/Passes.h.inc" | |
| } // namespace fir | |
| namespace { | |
| /// Return the coordinate_of producing the host association tuple slot that | |
| /// \p storeOp writes \p descriptor into, or null if this is not such a capture. | |
| static fir::CoordinateOp getHostAssocTupleSlot(fir::StoreOp storeOp, | |
| mlir::Value descriptor) { | |
| if (storeOp.getValue() != descriptor || | |
| !mlir::isa<fir::LLVMPointerType>(storeOp.getMemref().getType())) | |
| return nullptr; | |
| auto coord = storeOp.getMemref().getDefiningOp<fir::CoordinateOp>(); | |
| if (!coord || | |
| !mlir::isa<mlir::TupleType>(fir::unwrapRefType(coord.getRef().getType()))) | |
| return nullptr; | |
| return coord; | |
| } | |
| /// Find the point before which the cuf.alloc group should be placed: the | |
| /// earliest use in the block that dominates all uses, or that block's | |
| /// terminator if it holds no use itself. Uses in nested regions resolve to | |
| /// their enclosing top-level op. The block is never one of \p cyclicBlocks, | |
| /// the blocks on a CFG cycle, since an alloc there would run on every trip. | |
| /// | |
| /// Host association stores go to \p hostAssocStores and sink with the group | |
| /// instead of constraining it; the tuple's readers constrain it instead. | |
| static mlir::Operation * | |
| findDelayTarget(fir::DeclareOp declareOp, mlir::Block *entryBlock, | |
| mlir::DominanceInfo &domInfo, | |
| const llvm::SmallPtrSetImpl<mlir::Block *> &cyclicBlocks, | |
| llvm::SmallVectorImpl<fir::StoreOp> &hostAssocStores) { | |
| mlir::Region *funcRegion = entryBlock->getParent(); | |
| // Uses resolved to an op that sits directly in a block of the function. | |
| llvm::SmallVector<mlir::Operation *> uses; | |
| auto recordRealUse = [&](mlir::Operation *user) { | |
| mlir::Operation *op = user; | |
| while (op && op->getBlock() && op->getBlock()->getParent() != funcRegion) | |
| op = op->getParentOp(); | |
| if (op && op->getBlock()) | |
| uses.push_back(op); | |
| }; | |
| for (mlir::Value result : declareOp->getResults()) { | |
| for (mlir::Operation *user : result.getUsers()) { | |
| auto storeOp = mlir::dyn_cast<fir::StoreOp>(user); | |
| fir::CoordinateOp slot = | |
| storeOp ? getHostAssocTupleSlot(storeOp, result) : nullptr; | |
| if (!slot) { | |
| recordRealUse(user); | |
| continue; | |
| } | |
| // Whoever reads the tuple must still see a populated slot. | |
| hostAssocStores.push_back(storeOp); | |
| for (mlir::Operation *tupleUser : slot.getRef().getUsers()) { | |
| auto coord = mlir::dyn_cast<fir::CoordinateOp>(tupleUser); | |
| if (!coord) { | |
| recordRealUse(tupleUser); | |
| continue; | |
| } | |
| // A coordinate_of only computes the slot address. Stores through that | |
| // address populate the tuple; other users actually consume the slot. | |
| for (mlir::Operation *coordUser : coord->getUsers()) { | |
| auto slotStore = mlir::dyn_cast<fir::StoreOp>(coordUser); | |
| if (slotStore && slotStore.getMemref() == coord.getResult()) | |
| continue; | |
| recordRealUse(coordUser); | |
| } | |
| } | |
| } | |
| } | |
| if (uses.empty()) | |
| return nullptr; | |
| mlir::Block *common = uses.front()->getBlock(); | |
| for (mlir::Operation *use : uses) { | |
| common = domInfo.findNearestCommonDominator(common, use->getBlock()); | |
| if (!common) | |
| return nullptr; | |
| } | |
| // A block on a CFG cycle (e.g. the header of a DO loop that lowered to | |
| // branches because it contains a RETURN) executes once per trip, and the | |
| // alloc would replace the descriptor each time. Climb the dominator tree to | |
| // the nearest block that is not on a cycle. The entry block has no | |
| // predecessors, so it is never on a cycle and the climb always ends. | |
| while (cyclicBlocks.contains(common)) { | |
| mlir::DominanceInfoNode *node = domInfo.getNode(common); | |
| if (!node || !node->getIDom()) | |
| return nullptr; | |
| common = node->getIDom()->getBlock(); | |
| } | |
| mlir::Operation *earliest = nullptr; | |
| for (mlir::Operation *use : uses) | |
| if (use->getBlock() == common && | |
| (!earliest || use->isBeforeInBlock(earliest))) | |
| earliest = use; | |
| return earliest ? earliest : common->getTerminator(); | |
| } | |
| struct CUFAllocDelay : public fir::impl::CUFAllocDelayBase<CUFAllocDelay> { | |
| void runOnOperation() override { | |
| mlir::func::FuncOp func = getOperation(); | |
| if (func.empty()) | |
| return; | |
| mlir::Block &entryBlock = func.front(); | |
| mlir::DominanceInfo domInfo(func); | |
| // Collect box-type cuf.alloc ops in the entry block. | |
| llvm::SmallVector<cuf::AllocOp> boxAllocOps; | |
| for (auto &op : entryBlock) | |
| if (auto allocOp = mlir::dyn_cast<cuf::AllocOp>(op)) | |
| if (mlir::isa<fir::BaseBoxType>(allocOp.getInType())) | |
| boxAllocOps.push_back(allocOp); | |
| if (boxAllocOps.empty()) | |
| return; | |
| // Blocks on a CFG cycle, natural loop or not: every block of a strongly | |
| // connected component with more than one block or with a self edge. | |
| llvm::SmallPtrSet<mlir::Block *, 8> cyclicBlocks; | |
| for (auto it = llvm::scc_begin(&func.getBody()); !it.isAtEnd(); ++it) | |
| if (it.hasCycle()) | |
| cyclicBlocks.insert(it->begin(), it->end()); | |
| for (cuf::AllocOp allocOp : boxAllocOps) { | |
| // Find the fir.declare and fir.store using this cuf.alloc; bail on any | |
| // unexpected user. | |
| fir::DeclareOp declareOp = nullptr; | |
| fir::StoreOp storeOp = nullptr; | |
| bool hasUnknownUser = false; | |
| for (mlir::Operation *user : allocOp->getUsers()) { | |
| if (auto decl = mlir::dyn_cast<fir::DeclareOp>(user)) | |
| declareOp = decl; | |
| else if (auto store = mlir::dyn_cast<fir::StoreOp>(user)) | |
| storeOp = store; | |
| else | |
| hasUnknownUser = true; | |
| } | |
| if (!declareOp || hasUnknownUser) | |
| continue; | |
| llvm::SmallVector<fir::StoreOp> hostAssocStores; | |
| mlir::Operation *delayTarget = findDelayTarget( | |
| declareOp, &entryBlock, domInfo, cyclicBlocks, hostAssocStores); | |
| if (!delayTarget) | |
| continue; | |
| // Skip if the target is at or before the alloc, or is the declare. | |
| if (delayTarget->getBlock() == allocOp->getBlock() && | |
| (delayTarget->isBeforeInBlock(allocOp) || delayTarget == allocOp)) | |
| continue; | |
| if (delayTarget == declareOp) | |
| continue; | |
| // Ops that move together, keeping their relative order. | |
| llvm::SmallVector<mlir::Operation *> group; | |
| group.push_back(allocOp); | |
| if (storeOp) | |
| group.push_back(storeOp); | |
| group.push_back(declareOp); | |
| for (fir::StoreOp hostAssocStore : hostAssocStores) | |
| group.push_back(hostAssocStore); | |
| // Whatever the group reads from outside itself stays put, so it must | |
| // already dominate the new position. | |
| llvm::SmallPtrSet<mlir::Operation *, 8> groupSet(group.begin(), | |
| group.end()); | |
| auto readsDominateTarget = [&](mlir::Operation *op) { | |
| return llvm::all_of(op->getOperands(), [&](mlir::Value operand) { | |
| mlir::Operation *def = operand.getDefiningOp(); | |
| return (def && groupSet.contains(def)) || | |
| domInfo.properlyDominates(operand, delayTarget); | |
| }); | |
| }; | |
| if (!llvm::all_of(group, readsDominateTarget)) | |
| continue; | |
| // Sink the group before the target, preserving its relative order. | |
| group.front()->moveBefore(delayTarget); | |
| mlir::Operation *last = group.front(); | |
| for (mlir::Operation *op : llvm::drop_begin(group)) { | |
| op->moveAfter(last); | |
| last = op; | |
| } | |
| } | |
| } | |
| }; | |
| } // end anonymous namespace |