| //===- LLVMMemorySlot.cpp - MemorySlot interfaces ---------------*- C++ -*-===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file implements MemorySlot-related interfaces for LLVM dialect |
| // operations. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "mlir/Dialect/LLVMIR/LLVMDialect.h" |
| #include "mlir/Dialect/LLVMIR/LLVMTypes.h" |
| #include "mlir/IR/IRMapping.h" |
| #include "mlir/IR/Matchers.h" |
| #include "mlir/IR/PatternMatch.h" |
| #include "mlir/IR/ValueRange.h" |
| #include "mlir/Interfaces/DataLayoutInterfaces.h" |
| #include "mlir/Interfaces/MemorySlotInterfaces.h" |
| #include "llvm/ADT/STLExtras.h" |
| #include "llvm/ADT/TypeSwitch.h" |
| |
| using namespace mlir; |
| |
| //===----------------------------------------------------------------------===// |
| // Interfaces for AllocaOp |
| //===----------------------------------------------------------------------===// |
| |
| llvm::SmallVector<MemorySlot> LLVM::AllocaOp::getPromotableSlots() { |
| if (!getOperation()->getBlock()->isEntryBlock()) |
| return {}; |
| |
| return {MemorySlot{getResult(), getElemType()}}; |
| } |
| |
| Value LLVM::AllocaOp::getDefaultValue(const MemorySlot &slot, |
| RewriterBase &rewriter) { |
| return rewriter.create<LLVM::UndefOp>(getLoc(), slot.elemType); |
| } |
| |
| void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot, |
| BlockArgument argument, |
| RewriterBase &rewriter) { |
| for (Operation *user : getOperation()->getUsers()) |
| if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user)) |
| rewriter.create<LLVM::DbgValueOp>(declareOp.getLoc(), argument, |
| declareOp.getVarInfo(), |
| declareOp.getLocationExpr()); |
| } |
| |
| void LLVM::AllocaOp::handlePromotionComplete(const MemorySlot &slot, |
| Value defaultValue, |
| RewriterBase &rewriter) { |
| if (defaultValue && defaultValue.use_empty()) |
| rewriter.eraseOp(defaultValue.getDefiningOp()); |
| rewriter.eraseOp(*this); |
| } |
| |
| SmallVector<DestructurableMemorySlot> LLVM::AllocaOp::getDestructurableSlots() { |
| if (!mlir::matchPattern(getArraySize(), m_One())) |
| return {}; |
| |
| auto destructurable = dyn_cast<DestructurableTypeInterface>(getElemType()); |
| if (!destructurable) |
| return {}; |
| |
| std::optional<DenseMap<Attribute, Type>> destructuredType = |
| destructurable.getSubelementIndexMap(); |
| if (!destructuredType) |
| return {}; |
| |
| DenseMap<Attribute, Type> allocaTypeMap; |
| for (Attribute index : llvm::make_first_range(destructuredType.value())) |
| allocaTypeMap.insert({index, LLVM::LLVMPointerType::get(getContext())}); |
| |
| return { |
| DestructurableMemorySlot{{getResult(), getElemType()}, {allocaTypeMap}}}; |
| } |
| |
| DenseMap<Attribute, MemorySlot> |
| LLVM::AllocaOp::destructure(const DestructurableMemorySlot &slot, |
| const SmallPtrSetImpl<Attribute> &usedIndices, |
| RewriterBase &rewriter) { |
| assert(slot.ptr == getResult()); |
| rewriter.setInsertionPointAfter(*this); |
| |
| auto destructurableType = cast<DestructurableTypeInterface>(getElemType()); |
| DenseMap<Attribute, MemorySlot> slotMap; |
| for (Attribute index : usedIndices) { |
| Type elemType = destructurableType.getTypeAtIndex(index); |
| assert(elemType && "used index must exist"); |
| auto subAlloca = rewriter.create<LLVM::AllocaOp>( |
| getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType, |
| getArraySize()); |
| slotMap.try_emplace<MemorySlot>(index, {subAlloca.getResult(), elemType}); |
| } |
| |
| return slotMap; |
| } |
| |
| void LLVM::AllocaOp::handleDestructuringComplete( |
| const DestructurableMemorySlot &slot, RewriterBase &rewriter) { |
| assert(slot.ptr == getResult()); |
| rewriter.eraseOp(*this); |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Interfaces for LoadOp/StoreOp |
| //===----------------------------------------------------------------------===// |
| |
| bool LLVM::LoadOp::loadsFrom(const MemorySlot &slot) { |
| return getAddr() == slot.ptr; |
| } |
| |
| bool LLVM::LoadOp::storesTo(const MemorySlot &slot) { return false; } |
| |
| Value LLVM::LoadOp::getStored(const MemorySlot &slot, RewriterBase &rewriter) { |
| llvm_unreachable("getStored should not be called on LoadOp"); |
| } |
| |
| bool LLVM::StoreOp::loadsFrom(const MemorySlot &slot) { return false; } |
| |
| bool LLVM::StoreOp::storesTo(const MemorySlot &slot) { |
| return getAddr() == slot.ptr; |
| } |
| |
| Value LLVM::StoreOp::getStored(const MemorySlot &slot, RewriterBase &rewriter) { |
| return getValue(); |
| } |
| |
| bool LLVM::LoadOp::canUsesBeRemoved( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| if (blockingUses.size() != 1) |
| return false; |
| Value blockingUse = (*blockingUses.begin())->get(); |
| // If the blocking use is the slot ptr itself, there will be enough |
| // context to reconstruct the result of the load at removal time, so it can |
| // be removed (provided it loads the exact stored value and is not |
| // volatile). |
| return blockingUse == slot.ptr && getAddr() == slot.ptr && |
| getResult().getType() == slot.elemType && !getVolatile_(); |
| } |
| |
| DeletionKind LLVM::LoadOp::removeBlockingUses( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| RewriterBase &rewriter, Value reachingDefinition) { |
| // `canUsesBeRemoved` checked this blocking use must be the loaded slot |
| // pointer. |
| rewriter.replaceAllUsesWith(getResult(), reachingDefinition); |
| return DeletionKind::Delete; |
| } |
| |
| bool LLVM::StoreOp::canUsesBeRemoved( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| if (blockingUses.size() != 1) |
| return false; |
| Value blockingUse = (*blockingUses.begin())->get(); |
| // If the blocking use is the slot ptr itself, dropping the store is |
| // fine, provided we are currently promoting its target value. Don't allow a |
| // store OF the slot pointer, only INTO the slot pointer. |
| return blockingUse == slot.ptr && getAddr() == slot.ptr && |
| getValue() != slot.ptr && getValue().getType() == slot.elemType && |
| !getVolatile_(); |
| } |
| |
| DeletionKind LLVM::StoreOp::removeBlockingUses( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| RewriterBase &rewriter, Value reachingDefinition) { |
| // `canUsesBeRemoved` checked this blocking use must be the stored slot |
| // pointer. |
| for (Operation *user : slot.ptr.getUsers()) |
| if (auto declareOp = dyn_cast<LLVM::DbgDeclareOp>(user)) |
| rewriter.create<LLVM::DbgValueOp>(declareOp->getLoc(), getValue(), |
| declareOp.getVarInfo(), |
| declareOp.getLocationExpr()); |
| return DeletionKind::Delete; |
| } |
| |
| LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses( |
| const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| return success(getAddr() != slot.ptr || getType() == slot.elemType); |
| } |
| |
| LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses( |
| const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| return success(getAddr() != slot.ptr || |
| getValue().getType() == slot.elemType); |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Interfaces for discardable OPs |
| //===----------------------------------------------------------------------===// |
| |
| /// Conditions the deletion of the operation to the removal of all its uses. |
| static bool forwardToUsers(Operation *op, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| for (Value result : op->getResults()) |
| for (OpOperand &use : result.getUses()) |
| newBlockingUses.push_back(&use); |
| return true; |
| } |
| |
| bool LLVM::BitcastOp::canUsesBeRemoved( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| return forwardToUsers(*this, newBlockingUses); |
| } |
| |
| DeletionKind LLVM::BitcastOp::removeBlockingUses( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) { |
| return DeletionKind::Delete; |
| } |
| |
| bool LLVM::AddrSpaceCastOp::canUsesBeRemoved( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| return forwardToUsers(*this, newBlockingUses); |
| } |
| |
| DeletionKind LLVM::AddrSpaceCastOp::removeBlockingUses( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) { |
| return DeletionKind::Delete; |
| } |
| |
| bool LLVM::LifetimeStartOp::canUsesBeRemoved( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| return true; |
| } |
| |
| DeletionKind LLVM::LifetimeStartOp::removeBlockingUses( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) { |
| return DeletionKind::Delete; |
| } |
| |
| bool LLVM::LifetimeEndOp::canUsesBeRemoved( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| return true; |
| } |
| |
| DeletionKind LLVM::LifetimeEndOp::removeBlockingUses( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) { |
| return DeletionKind::Delete; |
| } |
| |
| bool LLVM::InvariantStartOp::canUsesBeRemoved( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| return true; |
| } |
| |
| DeletionKind LLVM::InvariantStartOp::removeBlockingUses( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) { |
| return DeletionKind::Delete; |
| } |
| |
| bool LLVM::InvariantEndOp::canUsesBeRemoved( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| return true; |
| } |
| |
| DeletionKind LLVM::InvariantEndOp::removeBlockingUses( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) { |
| return DeletionKind::Delete; |
| } |
| |
| bool LLVM::DbgDeclareOp::canUsesBeRemoved( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| return true; |
| } |
| |
| DeletionKind LLVM::DbgDeclareOp::removeBlockingUses( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) { |
| return DeletionKind::Delete; |
| } |
| |
| bool LLVM::DbgValueOp::canUsesBeRemoved( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| // There is only one operand that we can remove the use of. |
| if (blockingUses.size() != 1) |
| return false; |
| |
| return (*blockingUses.begin())->get() == getValue(); |
| } |
| |
| DeletionKind LLVM::DbgValueOp::removeBlockingUses( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) { |
| // Rewriter by default is after '*this', but we need it before '*this'. |
| rewriter.setInsertionPoint(*this); |
| |
| // Rather than dropping the debug value, replace it with undef to preserve the |
| // debug local variable info. This allows the debugger to inform the user that |
| // the variable has been optimized out. |
| auto undef = |
| rewriter.create<UndefOp>(getValue().getLoc(), getValue().getType()); |
| rewriter.modifyOpInPlace(*this, [&] { getValueMutable().assign(undef); }); |
| return DeletionKind::Keep; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Interfaces for GEPOp |
| //===----------------------------------------------------------------------===// |
| |
| static bool hasAllZeroIndices(LLVM::GEPOp gepOp) { |
| return llvm::all_of(gepOp.getIndices(), [](auto index) { |
| auto indexAttr = llvm::dyn_cast_if_present<IntegerAttr>(index); |
| return indexAttr && indexAttr.getValue() == 0; |
| }); |
| } |
| |
| bool LLVM::GEPOp::canUsesBeRemoved( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| // GEP can be removed as long as it is a no-op and its users can be removed. |
| if (!hasAllZeroIndices(*this)) |
| return false; |
| return forwardToUsers(*this, newBlockingUses); |
| } |
| |
| DeletionKind LLVM::GEPOp::removeBlockingUses( |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) { |
| return DeletionKind::Delete; |
| } |
| |
| static bool isFirstIndexZero(LLVM::GEPOp gep) { |
| IntegerAttr index = |
| llvm::dyn_cast_if_present<IntegerAttr>(gep.getIndices()[0]); |
| return index && index.getInt() == 0; |
| } |
| |
| LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses( |
| const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| if (getBase() != slot.ptr) |
| return success(); |
| if (slot.elemType != getElemType()) |
| return failure(); |
| if (!isFirstIndexZero(*this)) |
| return failure(); |
| Type reachedType = getResultPtrElementType(); |
| if (!reachedType) |
| return failure(); |
| mustBeSafelyUsed.emplace_back<MemorySlot>({getResult(), reachedType}); |
| return success(); |
| } |
| |
| bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot, |
| SmallPtrSetImpl<Attribute> &usedIndices, |
| SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| auto basePtrType = llvm::dyn_cast<LLVM::LLVMPointerType>(getBase().getType()); |
| if (!basePtrType) |
| return false; |
| |
| if (getBase() != slot.ptr || slot.elemType != getElemType()) |
| return false; |
| if (!isFirstIndexZero(*this)) |
| return false; |
| Type reachedType = getResultPtrElementType(); |
| if (!reachedType || getIndices().size() < 2) |
| return false; |
| auto firstLevelIndex = dyn_cast<IntegerAttr>(getIndices()[1]); |
| if (!firstLevelIndex) |
| return false; |
| assert(slot.elementPtrs.contains(firstLevelIndex)); |
| if (!llvm::isa<LLVM::LLVMPointerType>(slot.elementPtrs.at(firstLevelIndex))) |
| return false; |
| mustBeSafelyUsed.emplace_back<MemorySlot>({getResult(), reachedType}); |
| usedIndices.insert(firstLevelIndex); |
| return true; |
| } |
| |
| DeletionKind LLVM::GEPOp::rewire(const DestructurableMemorySlot &slot, |
| DenseMap<Attribute, MemorySlot> &subslots, |
| RewriterBase &rewriter) { |
| IntegerAttr firstLevelIndex = |
| llvm::dyn_cast_if_present<IntegerAttr>(getIndices()[1]); |
| const MemorySlot &newSlot = subslots.at(firstLevelIndex); |
| |
| ArrayRef<int32_t> remainingIndices = getRawConstantIndices().slice(2); |
| |
| // If the GEP would become trivial after this transformation, eliminate it. |
| // A GEP should only be eliminated if it has no indices (except the first |
| // pointer index), as simplifying GEPs with all-zero indices would eliminate |
| // structure information useful for further destruction. |
| if (remainingIndices.empty()) { |
| rewriter.replaceAllUsesWith(getResult(), newSlot.ptr); |
| return DeletionKind::Delete; |
| } |
| |
| rewriter.modifyOpInPlace(*this, [&]() { |
| // Rewire the indices by popping off the second index. |
| // Start with a single zero, then add the indices beyond the second. |
| SmallVector<int32_t> newIndices(1); |
| newIndices.append(remainingIndices.begin(), remainingIndices.end()); |
| setRawConstantIndices(newIndices); |
| |
| // Rewire the pointed type. |
| setElemType(newSlot.elemType); |
| |
| // Rewire the pointer. |
| getBaseMutable().assign(newSlot.ptr); |
| }); |
| |
| return DeletionKind::Keep; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Utilities for memory intrinsics |
| //===----------------------------------------------------------------------===// |
| |
| namespace { |
| |
| /// Returns the length of the given memory intrinsic in bytes if it can be known |
| /// at compile-time on a best-effort basis, nothing otherwise. |
| template <class MemIntr> |
| std::optional<uint64_t> getStaticMemIntrLen(MemIntr op) { |
| APInt memIntrLen; |
| if (!matchPattern(op.getLen(), m_ConstantInt(&memIntrLen))) |
| return {}; |
| if (memIntrLen.getBitWidth() > 64) |
| return {}; |
| return memIntrLen.getZExtValue(); |
| } |
| |
| /// Returns the length of the given memory intrinsic in bytes if it can be known |
| /// at compile-time on a best-effort basis, nothing otherwise. |
| /// Because MemcpyInlineOp has its length encoded as an attribute, this requires |
| /// specialized handling. |
| template <> |
| std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemcpyInlineOp op) { |
| APInt memIntrLen = op.getLen(); |
| if (memIntrLen.getBitWidth() > 64) |
| return {}; |
| return memIntrLen.getZExtValue(); |
| } |
| |
| } // namespace |
| |
| /// Returns whether one can be sure the memory intrinsic does not write outside |
| /// of the bounds of the given slot, on a best-effort basis. |
| template <class MemIntr> |
| static bool definitelyWritesOnlyWithinSlot(MemIntr op, const MemorySlot &slot, |
| DataLayout &dataLayout) { |
| if (!isa<LLVM::LLVMPointerType>(slot.ptr.getType()) || |
| op.getDst() != slot.ptr) |
| return false; |
| |
| std::optional<uint64_t> memIntrLen = getStaticMemIntrLen(op); |
| return memIntrLen && *memIntrLen <= dataLayout.getTypeSize(slot.elemType); |
| } |
| |
| /// Checks whether all indices are i32. This is used to check GEPs can index |
| /// into them. |
| static bool areAllIndicesI32(const DestructurableMemorySlot &slot) { |
| Type i32 = IntegerType::get(slot.ptr.getContext(), 32); |
| return llvm::all_of(llvm::make_first_range(slot.elementPtrs), |
| [&](Attribute index) { |
| auto intIndex = dyn_cast<IntegerAttr>(index); |
| return intIndex && intIndex.getType() == i32; |
| }); |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Interfaces for memset |
| //===----------------------------------------------------------------------===// |
| |
| bool LLVM::MemsetOp::loadsFrom(const MemorySlot &slot) { return false; } |
| |
| bool LLVM::MemsetOp::storesTo(const MemorySlot &slot) { |
| return getDst() == slot.ptr; |
| } |
| |
| Value LLVM::MemsetOp::getStored(const MemorySlot &slot, |
| RewriterBase &rewriter) { |
| // TODO: Support non-integer types. |
| return TypeSwitch<Type, Value>(slot.elemType) |
| .Case([&](IntegerType intType) -> Value { |
| if (intType.getWidth() == 8) |
| return getVal(); |
| |
| assert(intType.getWidth() % 8 == 0); |
| |
| // Build the memset integer by repeatedly shifting the value and |
| // or-ing it with the previous value. |
| uint64_t coveredBits = 8; |
| Value currentValue = |
| rewriter.create<LLVM::ZExtOp>(getLoc(), intType, getVal()); |
| while (coveredBits < intType.getWidth()) { |
| Value shiftBy = |
| rewriter.create<LLVM::ConstantOp>(getLoc(), intType, coveredBits); |
| Value shifted = |
| rewriter.create<LLVM::ShlOp>(getLoc(), currentValue, shiftBy); |
| currentValue = |
| rewriter.create<LLVM::OrOp>(getLoc(), currentValue, shifted); |
| coveredBits *= 2; |
| } |
| |
| return currentValue; |
| }) |
| .Default([](Type) -> Value { |
| llvm_unreachable( |
| "getStored should not be called on memset to unsupported type"); |
| }); |
| } |
| |
| bool LLVM::MemsetOp::canUsesBeRemoved( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| // TODO: Support non-integer types. |
| bool canConvertType = |
| TypeSwitch<Type, bool>(slot.elemType) |
| .Case([](IntegerType intType) { |
| return intType.getWidth() % 8 == 0 && intType.getWidth() > 0; |
| }) |
| .Default([](Type) { return false; }); |
| if (!canConvertType) |
| return false; |
| |
| if (getIsVolatile()) |
| return false; |
| |
| DataLayout layout = DataLayout::closest(*this); |
| return getStaticMemIntrLen(*this) == layout.getTypeSize(slot.elemType); |
| } |
| |
| DeletionKind LLVM::MemsetOp::removeBlockingUses( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| RewriterBase &rewriter, Value reachingDefinition) { |
| return DeletionKind::Delete; |
| } |
| |
| LogicalResult LLVM::MemsetOp::ensureOnlySafeAccesses( |
| const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| DataLayout dataLayout = DataLayout::closest(*this); |
| return success(definitelyWritesOnlyWithinSlot(*this, slot, dataLayout)); |
| } |
| |
| bool LLVM::MemsetOp::canRewire(const DestructurableMemorySlot &slot, |
| SmallPtrSetImpl<Attribute> &usedIndices, |
| SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| if (&slot.elemType.getDialect() != getOperation()->getDialect()) |
| return false; |
| |
| if (getIsVolatile()) |
| return false; |
| |
| if (!slot.elemType.cast<DestructurableTypeInterface>() |
| .getSubelementIndexMap()) |
| return false; |
| |
| if (!areAllIndicesI32(slot)) |
| return false; |
| |
| DataLayout dataLayout = DataLayout::closest(*this); |
| return definitelyWritesOnlyWithinSlot(*this, slot, dataLayout); |
| } |
| |
| DeletionKind LLVM::MemsetOp::rewire(const DestructurableMemorySlot &slot, |
| DenseMap<Attribute, MemorySlot> &subslots, |
| RewriterBase &rewriter) { |
| std::optional<DenseMap<Attribute, Type>> types = |
| slot.elemType.cast<DestructurableTypeInterface>().getSubelementIndexMap(); |
| |
| IntegerAttr memsetLenAttr; |
| bool successfulMatch = |
| matchPattern(getLen(), m_Constant<IntegerAttr>(&memsetLenAttr)); |
| (void)successfulMatch; |
| assert(successfulMatch); |
| |
| bool packed = false; |
| if (auto structType = dyn_cast<LLVM::LLVMStructType>(slot.elemType)) |
| packed = structType.isPacked(); |
| |
| Type i32 = IntegerType::get(getContext(), 32); |
| DataLayout dataLayout = DataLayout::closest(*this); |
| uint64_t memsetLen = memsetLenAttr.getValue().getZExtValue(); |
| uint64_t covered = 0; |
| for (size_t i = 0; i < types->size(); i++) { |
| // Create indices on the fly to get elements in the right order. |
| Attribute index = IntegerAttr::get(i32, i); |
| Type elemType = types->at(index); |
| uint64_t typeSize = dataLayout.getTypeSize(elemType); |
| |
| if (!packed) |
| covered = |
| llvm::alignTo(covered, dataLayout.getTypeABIAlignment(elemType)); |
| |
| if (covered >= memsetLen) |
| break; |
| |
| // If this subslot is used, apply a new memset to it. |
| // Otherwise, only compute its offset within the original memset. |
| if (subslots.contains(index)) { |
| uint64_t newMemsetSize = std::min(memsetLen - covered, typeSize); |
| |
| Value newMemsetSizeValue = |
| rewriter |
| .create<LLVM::ConstantOp>( |
| getLen().getLoc(), |
| IntegerAttr::get(memsetLenAttr.getType(), newMemsetSize)) |
| .getResult(); |
| |
| rewriter.create<LLVM::MemsetOp>(getLoc(), subslots.at(index).ptr, |
| getVal(), newMemsetSizeValue, |
| getIsVolatile()); |
| } |
| |
| covered += typeSize; |
| } |
| |
| return DeletionKind::Delete; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Interfaces for memcpy/memmove |
| //===----------------------------------------------------------------------===// |
| |
| template <class MemcpyLike> |
| static bool memcpyLoadsFrom(MemcpyLike op, const MemorySlot &slot) { |
| return op.getSrc() == slot.ptr; |
| } |
| |
| template <class MemcpyLike> |
| static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot) { |
| return op.getDst() == slot.ptr; |
| } |
| |
| template <class MemcpyLike> |
| static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot, |
| RewriterBase &rewriter) { |
| return rewriter.create<LLVM::LoadOp>(op.getLoc(), slot.elemType, op.getSrc()); |
| } |
| |
| template <class MemcpyLike> |
| static bool |
| memcpyCanUsesBeRemoved(MemcpyLike op, const MemorySlot &slot, |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| // If source and destination are the same, memcpy behavior is undefined and |
| // memmove is a no-op. Because there is no memory change happening here, |
| // simplifying such operations is left to canonicalization. |
| if (op.getDst() == op.getSrc()) |
| return false; |
| |
| if (op.getIsVolatile()) |
| return false; |
| |
| DataLayout layout = DataLayout::closest(op); |
| return getStaticMemIntrLen(op) == layout.getTypeSize(slot.elemType); |
| } |
| |
| template <class MemcpyLike> |
| static DeletionKind |
| memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot, |
| const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| RewriterBase &rewriter, Value reachingDefinition) { |
| if (op.loadsFrom(slot)) |
| rewriter.create<LLVM::StoreOp>(op.getLoc(), reachingDefinition, |
| op.getDst()); |
| return DeletionKind::Delete; |
| } |
| |
| template <class MemcpyLike> |
| static LogicalResult |
| memcpyEnsureOnlySafeAccesses(MemcpyLike op, const MemorySlot &slot, |
| SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| DataLayout dataLayout = DataLayout::closest(op); |
| // While rewiring memcpy-like intrinsics only supports full copies, partial |
| // copies are still safe accesses so it is enough to only check for writes |
| // within bounds. |
| return success(definitelyWritesOnlyWithinSlot(op, slot, dataLayout)); |
| } |
| |
| template <class MemcpyLike> |
| static bool memcpyCanRewire(MemcpyLike op, const DestructurableMemorySlot &slot, |
| SmallPtrSetImpl<Attribute> &usedIndices, |
| SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| if (op.getIsVolatile()) |
| return false; |
| |
| if (!slot.elemType.cast<DestructurableTypeInterface>() |
| .getSubelementIndexMap()) |
| return false; |
| |
| if (!areAllIndicesI32(slot)) |
| return false; |
| |
| // Only full copies are supported. |
| DataLayout dataLayout = DataLayout::closest(op); |
| if (getStaticMemIntrLen(op) != dataLayout.getTypeSize(slot.elemType)) |
| return false; |
| |
| if (op.getSrc() == slot.ptr) |
| for (Attribute index : llvm::make_first_range(slot.elementPtrs)) |
| usedIndices.insert(index); |
| |
| return true; |
| } |
| |
| namespace { |
| |
| template <class MemcpyLike> |
| void createMemcpyLikeToReplace(RewriterBase &rewriter, const DataLayout &layout, |
| MemcpyLike toReplace, Value dst, Value src, |
| Type toCpy, bool isVolatile) { |
| Value memcpySize = rewriter.create<LLVM::ConstantOp>( |
| toReplace.getLoc(), IntegerAttr::get(toReplace.getLen().getType(), |
| layout.getTypeSize(toCpy))); |
| rewriter.create<MemcpyLike>(toReplace.getLoc(), dst, src, memcpySize, |
| isVolatile); |
| } |
| |
| template <> |
| void createMemcpyLikeToReplace(RewriterBase &rewriter, const DataLayout &layout, |
| LLVM::MemcpyInlineOp toReplace, Value dst, |
| Value src, Type toCpy, bool isVolatile) { |
| Type lenType = IntegerType::get(toReplace->getContext(), |
| toReplace.getLen().getBitWidth()); |
| rewriter.create<LLVM::MemcpyInlineOp>( |
| toReplace.getLoc(), dst, src, |
| IntegerAttr::get(lenType, layout.getTypeSize(toCpy)), isVolatile); |
| } |
| |
| } // namespace |
| |
| /// Rewires a memcpy-like operation. Only copies to or from the full slot are |
| /// supported. |
| template <class MemcpyLike> |
| static DeletionKind memcpyRewire(MemcpyLike op, |
| const DestructurableMemorySlot &slot, |
| DenseMap<Attribute, MemorySlot> &subslots, |
| RewriterBase &rewriter) { |
| if (subslots.empty()) |
| return DeletionKind::Delete; |
| |
| DataLayout layout = DataLayout::closest(op); |
| |
| assert((slot.ptr == op.getDst()) != (slot.ptr == op.getSrc())); |
| bool isDst = slot.ptr == op.getDst(); |
| |
| #ifndef NDEBUG |
| size_t slotsTreated = 0; |
| #endif |
| |
| // It was previously checked that index types are consistent, so this type can |
| // be fetched now. |
| Type indexType = cast<IntegerAttr>(subslots.begin()->first).getType(); |
| for (size_t i = 0, e = slot.elementPtrs.size(); i != e; i++) { |
| Attribute index = IntegerAttr::get(indexType, i); |
| if (!subslots.contains(index)) |
| continue; |
| const MemorySlot &subslot = subslots.at(index); |
| |
| #ifndef NDEBUG |
| slotsTreated++; |
| #endif |
| |
| // First get a pointer to the equivalent of this subslot from the source |
| // pointer. |
| SmallVector<LLVM::GEPArg> gepIndices{ |
| 0, static_cast<int32_t>( |
| cast<IntegerAttr>(index).getValue().getZExtValue())}; |
| Value subslotPtrInOther = rewriter.create<LLVM::GEPOp>( |
| op.getLoc(), LLVM::LLVMPointerType::get(op.getContext()), slot.elemType, |
| isDst ? op.getSrc() : op.getDst(), gepIndices); |
| |
| // Then create a new memcpy out of this source pointer. |
| createMemcpyLikeToReplace(rewriter, layout, op, |
| isDst ? subslot.ptr : subslotPtrInOther, |
| isDst ? subslotPtrInOther : subslot.ptr, |
| subslot.elemType, op.getIsVolatile()); |
| } |
| |
| assert(subslots.size() == slotsTreated); |
| |
| return DeletionKind::Delete; |
| } |
| |
| bool LLVM::MemcpyOp::loadsFrom(const MemorySlot &slot) { |
| return memcpyLoadsFrom(*this, slot); |
| } |
| |
| bool LLVM::MemcpyOp::storesTo(const MemorySlot &slot) { |
| return memcpyStoresTo(*this, slot); |
| } |
| |
| Value LLVM::MemcpyOp::getStored(const MemorySlot &slot, |
| RewriterBase &rewriter) { |
| return memcpyGetStored(*this, slot, rewriter); |
| } |
| |
| bool LLVM::MemcpyOp::canUsesBeRemoved( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| return memcpyCanUsesBeRemoved(*this, slot, blockingUses, newBlockingUses); |
| } |
| |
| DeletionKind LLVM::MemcpyOp::removeBlockingUses( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| RewriterBase &rewriter, Value reachingDefinition) { |
| return memcpyRemoveBlockingUses(*this, slot, blockingUses, rewriter, |
| reachingDefinition); |
| } |
| |
| LogicalResult LLVM::MemcpyOp::ensureOnlySafeAccesses( |
| const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| return memcpyEnsureOnlySafeAccesses(*this, slot, mustBeSafelyUsed); |
| } |
| |
| bool LLVM::MemcpyOp::canRewire(const DestructurableMemorySlot &slot, |
| SmallPtrSetImpl<Attribute> &usedIndices, |
| SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| return memcpyCanRewire(*this, slot, usedIndices, mustBeSafelyUsed); |
| } |
| |
| DeletionKind LLVM::MemcpyOp::rewire(const DestructurableMemorySlot &slot, |
| DenseMap<Attribute, MemorySlot> &subslots, |
| RewriterBase &rewriter) { |
| return memcpyRewire(*this, slot, subslots, rewriter); |
| } |
| |
| bool LLVM::MemcpyInlineOp::loadsFrom(const MemorySlot &slot) { |
| return memcpyLoadsFrom(*this, slot); |
| } |
| |
| bool LLVM::MemcpyInlineOp::storesTo(const MemorySlot &slot) { |
| return memcpyStoresTo(*this, slot); |
| } |
| |
| Value LLVM::MemcpyInlineOp::getStored(const MemorySlot &slot, |
| RewriterBase &rewriter) { |
| return memcpyGetStored(*this, slot, rewriter); |
| } |
| |
| bool LLVM::MemcpyInlineOp::canUsesBeRemoved( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| return memcpyCanUsesBeRemoved(*this, slot, blockingUses, newBlockingUses); |
| } |
| |
| DeletionKind LLVM::MemcpyInlineOp::removeBlockingUses( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| RewriterBase &rewriter, Value reachingDefinition) { |
| return memcpyRemoveBlockingUses(*this, slot, blockingUses, rewriter, |
| reachingDefinition); |
| } |
| |
| LogicalResult LLVM::MemcpyInlineOp::ensureOnlySafeAccesses( |
| const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| return memcpyEnsureOnlySafeAccesses(*this, slot, mustBeSafelyUsed); |
| } |
| |
| bool LLVM::MemcpyInlineOp::canRewire( |
| const DestructurableMemorySlot &slot, |
| SmallPtrSetImpl<Attribute> &usedIndices, |
| SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| return memcpyCanRewire(*this, slot, usedIndices, mustBeSafelyUsed); |
| } |
| |
| DeletionKind |
| LLVM::MemcpyInlineOp::rewire(const DestructurableMemorySlot &slot, |
| DenseMap<Attribute, MemorySlot> &subslots, |
| RewriterBase &rewriter) { |
| return memcpyRewire(*this, slot, subslots, rewriter); |
| } |
| |
| bool LLVM::MemmoveOp::loadsFrom(const MemorySlot &slot) { |
| return memcpyLoadsFrom(*this, slot); |
| } |
| |
| bool LLVM::MemmoveOp::storesTo(const MemorySlot &slot) { |
| return memcpyStoresTo(*this, slot); |
| } |
| |
| Value LLVM::MemmoveOp::getStored(const MemorySlot &slot, |
| RewriterBase &rewriter) { |
| return memcpyGetStored(*this, slot, rewriter); |
| } |
| |
| bool LLVM::MemmoveOp::canUsesBeRemoved( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| SmallVectorImpl<OpOperand *> &newBlockingUses) { |
| return memcpyCanUsesBeRemoved(*this, slot, blockingUses, newBlockingUses); |
| } |
| |
| DeletionKind LLVM::MemmoveOp::removeBlockingUses( |
| const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| RewriterBase &rewriter, Value reachingDefinition) { |
| return memcpyRemoveBlockingUses(*this, slot, blockingUses, rewriter, |
| reachingDefinition); |
| } |
| |
| LogicalResult LLVM::MemmoveOp::ensureOnlySafeAccesses( |
| const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| return memcpyEnsureOnlySafeAccesses(*this, slot, mustBeSafelyUsed); |
| } |
| |
| bool LLVM::MemmoveOp::canRewire(const DestructurableMemorySlot &slot, |
| SmallPtrSetImpl<Attribute> &usedIndices, |
| SmallVectorImpl<MemorySlot> &mustBeSafelyUsed) { |
| return memcpyCanRewire(*this, slot, usedIndices, mustBeSafelyUsed); |
| } |
| |
| DeletionKind LLVM::MemmoveOp::rewire(const DestructurableMemorySlot &slot, |
| DenseMap<Attribute, MemorySlot> &subslots, |
| RewriterBase &rewriter) { |
| return memcpyRewire(*this, slot, subslots, rewriter); |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Interfaces for destructurable types |
| //===----------------------------------------------------------------------===// |
| |
| std::optional<DenseMap<Attribute, Type>> |
| LLVM::LLVMStructType::getSubelementIndexMap() { |
| Type i32 = IntegerType::get(getContext(), 32); |
| DenseMap<Attribute, Type> destructured; |
| for (const auto &[index, elemType] : llvm::enumerate(getBody())) |
| destructured.insert({IntegerAttr::get(i32, index), elemType}); |
| return destructured; |
| } |
| |
| Type LLVM::LLVMStructType::getTypeAtIndex(Attribute index) { |
| auto indexAttr = llvm::dyn_cast<IntegerAttr>(index); |
| if (!indexAttr || !indexAttr.getType().isInteger(32)) |
| return {}; |
| int32_t indexInt = indexAttr.getInt(); |
| ArrayRef<Type> body = getBody(); |
| if (indexInt < 0 || body.size() <= static_cast<uint32_t>(indexInt)) |
| return {}; |
| return body[indexInt]; |
| } |
| |
| std::optional<DenseMap<Attribute, Type>> |
| LLVM::LLVMArrayType::getSubelementIndexMap() const { |
| constexpr size_t maxArraySizeForDestructuring = 16; |
| if (getNumElements() > maxArraySizeForDestructuring) |
| return {}; |
| int32_t numElements = getNumElements(); |
| |
| Type i32 = IntegerType::get(getContext(), 32); |
| DenseMap<Attribute, Type> destructured; |
| for (int32_t index = 0; index < numElements; ++index) |
| destructured.insert({IntegerAttr::get(i32, index), getElementType()}); |
| return destructured; |
| } |
| |
| Type LLVM::LLVMArrayType::getTypeAtIndex(Attribute index) const { |
| auto indexAttr = llvm::dyn_cast<IntegerAttr>(index); |
| if (!indexAttr || !indexAttr.getType().isInteger(32)) |
| return {}; |
| int32_t indexInt = indexAttr.getInt(); |
| if (indexInt < 0 || getNumElements() <= static_cast<uint32_t>(indexInt)) |
| return {}; |
| return getElementType(); |
| } |
