blob: d93c341928f53c7b2edb490a7ac0eb19eb902a1a [file]
//=== MapperJITLinkMemoryManager.cpp - Memory management with MemoryMapper ===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h"
#include "llvm/ExecutionEngine/JITLink/JITLink.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/Process.h"
using namespace llvm::jitlink;
namespace llvm {
namespace orc {
class MapperJITLinkMemoryManager::InFlightAlloc
: public JITLinkMemoryManager::InFlightAlloc {
public:
InFlightAlloc(MapperJITLinkMemoryManager &Parent, LinkGraph &G,
ExecutorAddr AllocAddr,
std::vector<MemoryMapper::AllocInfo::SegInfo> Segs)
: Parent(Parent), G(G), AllocAddr(AllocAddr), Segs(std::move(Segs)) {}
void finalize(OnFinalizedFunction OnFinalize) override {
MemoryMapper::AllocInfo AI;
AI.MappingBase = AllocAddr;
std::swap(AI.Segments, Segs);
std::swap(AI.Actions, G.allocActions());
Parent.Mapper->initialize(AI, [OnFinalize = std::move(OnFinalize)](
Expected<ExecutorAddr> Result) mutable {
if (!Result) {
OnFinalize(Result.takeError());
return;
}
OnFinalize(FinalizedAlloc(*Result));
});
}
void abandon(OnAbandonedFunction OnFinalize) override {
Parent.Mapper->deinitialize({AllocAddr}, std::move(OnFinalize));
}
private:
MapperJITLinkMemoryManager &Parent;
LinkGraph &G;
ExecutorAddr AllocAddr;
std::vector<MemoryMapper::AllocInfo::SegInfo> Segs;
};
MapperJITLinkMemoryManager::MapperJITLinkMemoryManager(
size_t ReservationGranularity, std::unique_ptr<MemoryMapper> Mapper,
bool ColocatePerJITDylib)
: ReservationUnits(ReservationGranularity),
ColocatePerJITDylib(ColocatePerJITDylib), Mapper(std::move(Mapper)) {}
// Lazily creates and stores Key's pool on first access. Pools are
// heap-allocated behind unique_ptr so the returned reference survives map
// rehashing, and AMAllocator (declared before the pools) outlives them. Also
// registers for a destruction notification the first time a JITDylib is
// seen, so its pool can be freed in notifyDestroying().
MapperJITLinkMemoryManager::AvailableMemoryMap &
MapperJITLinkMemoryManager::getAvailableMemoryLocked(const JITLinkDylib *Key) {
auto [It, Inserted] = Pools.try_emplace(Key);
if (Inserted && Key)
const_cast<JITLinkDylib *>(Key)->notifyOnDestruction(*this);
auto &Pool = It->second.AvailPool;
if (!Pool)
Pool = std::make_unique<AvailableMemoryMap>(AMAllocator);
return *Pool;
}
void MapperJITLinkMemoryManager::notifyDestroying(JITLinkDylib &JD) {
std::lock_guard<std::mutex> Lock(Mutex);
Pools.erase(&JD);
}
void MapperJITLinkMemoryManager::denySlabGrowth() {
std::lock_guard<std::mutex> Lock(Mutex);
OnSlabGrow = [](const JITLinkDylib *JD, size_t RequestedSize) -> Error {
return createStringError(
"JITDylib '%s' slab exhausted (single-slab mode); enable slab "
"growth to allow further reservations (%zu bytes requested)",
JD ? JD->getName().c_str() : "<unnamed>", RequestedSize);
};
}
void MapperJITLinkMemoryManager::allowSlabGrowth() {
std::lock_guard<std::mutex> Lock(Mutex);
OnSlabGrow = [](const JITLinkDylib *, size_t) -> Error {
return Error::success();
};
}
void MapperJITLinkMemoryManager::setSlabPolicy(SlabGrowthPolicy Policy) {
std::lock_guard<std::mutex> Lock(Mutex);
OnSlabGrow = std::move(Policy);
}
Expected<std::unique_ptr<MapperJITLinkMemoryManager>>
createColocatingInProcessMemoryManager(
std::optional<size_t> ReservationGranularity) {
auto Mapper = InProcessMemoryMapper::Create();
if (!Mapper)
return Mapper.takeError();
return std::make_unique<MapperJITLinkMemoryManager>(
ReservationGranularity.value_or(
MapperJITLinkMemoryManager::defaultSlabSize()),
std::move(*Mapper),
/*ColocatePerJITDylib=*/true);
}
void MapperJITLinkMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G,
OnAllocatedFunction OnAllocated) {
BasicLayout BL(G);
// find required address space
auto SegsSizes = BL.getContiguousPageBasedLayoutSizes(Mapper->getPageSize());
if (!SegsSizes) {
OnAllocated(SegsSizes.takeError());
return;
}
auto TotalSize = SegsSizes->total();
// Which pool to draw from: the JITDylib's own pool when colocating
// per-JITDylib, otherwise a single shared (nullptr) pool.
const JITLinkDylib *PoolKey = ColocatePerJITDylib ? JD : nullptr;
auto CompleteAllocation = [this, PoolKey, &G, BL = std::move(BL),
OnAllocated = std::move(OnAllocated)](
Expected<ExecutorAddrRange> Result) mutable {
if (!Result) {
Mutex.unlock();
return OnAllocated(Result.takeError());
}
auto NextSegAddr = Result->Start;
std::vector<MemoryMapper::AllocInfo::SegInfo> SegInfos;
for (auto &KV : BL.segments()) {
auto &AG = KV.first;
auto &Seg = KV.second;
auto TotalSize = Seg.ContentSize + Seg.ZeroFillSize;
Seg.Addr = NextSegAddr;
Seg.WorkingMem = Mapper->prepare(G, NextSegAddr, TotalSize);
NextSegAddr += alignTo(TotalSize, Mapper->getPageSize());
MemoryMapper::AllocInfo::SegInfo SI;
SI.Offset = Seg.Addr - Result->Start;
SI.ContentSize = Seg.ContentSize;
SI.ZeroFillSize = Seg.ZeroFillSize;
SI.AG = AG;
SI.WorkingMem = Seg.WorkingMem;
SegInfos.push_back(SI);
}
UsedMemory.insert({Result->Start, NextSegAddr - Result->Start});
AllocPoolKey[Result->Start] = PoolKey;
if (NextSegAddr < Result->End) {
// Save the remaining memory for reuse by later allocations from the same
// pool.
getAvailableMemoryLocked(PoolKey).insert(NextSegAddr, Result->End - 1,
true);
}
Mutex.unlock();
if (auto Err = BL.apply()) {
OnAllocated(std::move(Err));
return;
}
OnAllocated(std::make_unique<InFlightAlloc>(*this, G, Result->Start,
std::move(SegInfos)));
};
Mutex.lock();
// find an already reserved range in this pool that is large enough
ExecutorAddrRange SelectedRange{};
AvailableMemoryMap &Avail = getAvailableMemoryLocked(PoolKey);
for (AvailableMemoryMap::iterator It = Avail.begin(); It != Avail.end();
It++) {
if (It.stop() - It.start() + 1 >= TotalSize) {
SelectedRange = ExecutorAddrRange(It.start(), It.stop() + 1);
It.erase();
break;
}
}
if (SelectedRange.empty()) { // no already reserved range was found
// A reservation is needed. If this pool already owns one, reserving
// another (possibly out-of-range) slab may place its objects out of range,
// so consult the growth policy first.
bool IsFirstReservation = !Pools[PoolKey].Reserved;
Pools[PoolKey].Reserved = true;
auto TotalAllocation = alignTo(TotalSize, ReservationUnits);
if (!IsFirstReservation) {
if (auto Err = OnSlabGrow(JD, TotalAllocation))
return CompleteAllocation(std::move(Err));
}
Mapper->reserve(TotalAllocation,
[this, PoolKey, IsFirstReservation,
CompleteAllocation = std::move(CompleteAllocation)](
Expected<ExecutorAddrRange> Result) mutable {
// If this was the pool's first reservation attempt and it
// failed, undo the Reserved flag so a later attempt is
// treated as the first one again, rather than
// incorrectly triggering the growth policy for a pool
// that never got a slab.
if (!Result && IsFirstReservation)
Pools[PoolKey].Reserved = false;
CompleteAllocation(std::move(Result));
});
} else {
CompleteAllocation(SelectedRange);
}
}
void MapperJITLinkMemoryManager::deallocate(
std::vector<FinalizedAlloc> Allocs, OnDeallocatedFunction OnDeallocated) {
std::vector<ExecutorAddr> Bases;
Bases.reserve(Allocs.size());
for (auto &FA : Allocs) {
ExecutorAddr Addr = FA.getAddress();
Bases.push_back(Addr);
}
Mapper->deinitialize(Bases, [this, Allocs = std::move(Allocs),
OnDeallocated = std::move(OnDeallocated)](
llvm::Error Err) mutable {
// TODO: How should we treat memory that we fail to deinitialize?
// We're currently bailing out and treating it as "burned" -- should we
// require that a failure to deinitialize still reset the memory so that
// we can reclaim it?
if (Err) {
for (auto &FA : Allocs)
FA.release();
OnDeallocated(std::move(Err));
return;
}
{
std::lock_guard<std::mutex> Lock(Mutex);
for (auto &FA : Allocs) {
ExecutorAddr Addr = FA.getAddress();
ExecutorAddrDiff Size = UsedMemory[Addr];
UsedMemory.erase(Addr);
// Return the range to the pool it was allocated from.
const JITLinkDylib *PoolKey = AllocPoolKey.lookup(Addr);
AllocPoolKey.erase(Addr);
getAvailableMemoryLocked(PoolKey).insert(Addr, Addr + Size - 1, true);
FA.release();
}
}
OnDeallocated(Error::success());
});
}
} // end namespace orc
} // end namespace llvm