blob: 6690dd07d99b2113110dcee8a25f7ec45ec12c4e [file] [log] [blame]
//===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the section-based memory manager used by the MCJIT
// execution engine and RuntimeDyld
//
//===----------------------------------------------------------------------===//
#include "llvm/ExecutionEngine/SectionMemoryManager.h"
#include "llvm/Config/config.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/Process.h"
namespace llvm {
uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size,
unsigned Alignment,
unsigned SectionID,
StringRef SectionName,
bool IsReadOnly) {
if (IsReadOnly)
return allocateSection(SectionMemoryManager::AllocationPurpose::ROData,
Size, Alignment);
return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size,
Alignment);
}
uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size,
unsigned Alignment,
unsigned SectionID,
StringRef SectionName) {
return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size,
Alignment);
}
uint8_t *SectionMemoryManager::allocateSection(
SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size,
unsigned Alignment) {
if (!Alignment)
Alignment = 16;
assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two.");
uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1);
uintptr_t Addr = 0;
MemoryGroup &MemGroup = [&]() -> MemoryGroup & {
switch (Purpose) {
case AllocationPurpose::Code:
return CodeMem;
case AllocationPurpose::ROData:
return RODataMem;
case AllocationPurpose::RWData:
return RWDataMem;
}
llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose");
}();
// Look in the list of free memory regions and use a block there if one
// is available.
for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
if (FreeMB.Free.allocatedSize() >= RequiredSize) {
Addr = (uintptr_t)FreeMB.Free.base();
uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize();
// Align the address.
Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
// The part of the block we're giving out to the user is now pending
MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
// Remember this pending block, such that future allocations can just
// modify it rather than creating a new one
FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
} else {
sys::MemoryBlock &PendingMB =
MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
PendingMB = sys::MemoryBlock(PendingMB.base(),
Addr + Size - (uintptr_t)PendingMB.base());
}
// Remember how much free space is now left in this block
FreeMB.Free =
sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
return (uint8_t *)Addr;
}
}
// No pre-allocated free block was large enough. Allocate a new memory region.
// Note that all sections get allocated as read-write. The permissions will
// be updated later based on memory group.
//
// FIXME: It would be useful to define a default allocation size (or add
// it as a constructor parameter) to minimize the number of allocations.
//
// FIXME: Initialize the Near member for each memory group to avoid
// interleaving.
std::error_code ec;
sys::MemoryBlock MB = MMapper.allocateMappedMemory(
Purpose, RequiredSize, &MemGroup.Near,
sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec);
if (ec) {
// FIXME: Add error propagation to the interface.
return nullptr;
}
// Save this address as the basis for our next request
MemGroup.Near = MB;
// Copy the address to all the other groups, if they have not
// been initialized.
if (CodeMem.Near.base() == 0)
CodeMem.Near = MB;
if (RODataMem.Near.base() == 0)
RODataMem.Near = MB;
if (RWDataMem.Near.base() == 0)
RWDataMem.Near = MB;
// Remember that we allocated this memory
MemGroup.AllocatedMem.push_back(MB);
Addr = (uintptr_t)MB.base();
uintptr_t EndOfBlock = Addr + MB.allocatedSize();
// Align the address.
Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
// The part of the block we're giving out to the user is now pending
MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
// The allocateMappedMemory may allocate much more memory than we need. In
// this case, we store the unused memory as a free memory block.
unsigned FreeSize = EndOfBlock - Addr - Size;
if (FreeSize > 16) {
FreeMemBlock FreeMB;
FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize);
FreeMB.PendingPrefixIndex = (unsigned)-1;
MemGroup.FreeMem.push_back(FreeMB);
}
// Return aligned address
return (uint8_t *)Addr;
}
bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) {
// FIXME: Should in-progress permissions be reverted if an error occurs?
std::error_code ec;
// Make code memory executable.
ec = applyMemoryGroupPermissions(CodeMem,
sys::Memory::MF_READ | sys::Memory::MF_EXEC);
if (ec) {
if (ErrMsg) {
*ErrMsg = ec.message();
}
return true;
}
// Make read-only data memory read-only.
ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ);
if (ec) {
if (ErrMsg) {
*ErrMsg = ec.message();
}
return true;
}
// Read-write data memory already has the correct permissions
// Some platforms with separate data cache and instruction cache require
// explicit cache flush, otherwise JIT code manipulations (like resolved
// relocations) will get to the data cache but not to the instruction cache.
invalidateInstructionCache();
return false;
}
static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {
static const size_t PageSize = sys::Process::getPageSizeEstimate();
size_t StartOverlap =
(PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;
size_t TrimmedSize = M.allocatedSize();
TrimmedSize -= StartOverlap;
TrimmedSize -= TrimmedSize % PageSize;
sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap),
TrimmedSize);
assert(((uintptr_t)Trimmed.base() % PageSize) == 0);
assert((Trimmed.allocatedSize() % PageSize) == 0);
assert(M.base() <= Trimmed.base() &&
Trimmed.allocatedSize() <= M.allocatedSize());
return Trimmed;
}
std::error_code
SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
unsigned Permissions) {
for (sys::MemoryBlock &MB : MemGroup.PendingMem)
if (std::error_code EC = MMapper.protectMappedMemory(MB, Permissions))
return EC;
MemGroup.PendingMem.clear();
// Now go through free blocks and trim any of them that don't span the entire
// page because one of the pending blocks may have overlapped it.
for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
FreeMB.Free = trimBlockToPageSize(FreeMB.Free);
// We cleared the PendingMem list, so all these pointers are now invalid
FreeMB.PendingPrefixIndex = (unsigned)-1;
}
// Remove all blocks which are now empty
erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) {
return FreeMB.Free.allocatedSize() == 0;
});
return std::error_code();
}
void SectionMemoryManager::invalidateInstructionCache() {
for (sys::MemoryBlock &Block : CodeMem.PendingMem)
sys::Memory::InvalidateInstructionCache(Block.base(),
Block.allocatedSize());
}
SectionMemoryManager::~SectionMemoryManager() {
for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
for (sys::MemoryBlock &Block : Group->AllocatedMem)
MMapper.releaseMappedMemory(Block);
}
}
SectionMemoryManager::MemoryMapper::~MemoryMapper() {}
void SectionMemoryManager::anchor() {}
namespace {
// Trivial implementation of SectionMemoryManager::MemoryMapper that just calls
// into sys::Memory.
class DefaultMMapper final : public SectionMemoryManager::MemoryMapper {
public:
sys::MemoryBlock
allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose,
size_t NumBytes, const sys::MemoryBlock *const NearBlock,
unsigned Flags, std::error_code &EC) override {
return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC);
}
std::error_code protectMappedMemory(const sys::MemoryBlock &Block,
unsigned Flags) override {
return sys::Memory::protectMappedMemory(Block, Flags);
}
std::error_code releaseMappedMemory(sys::MemoryBlock &M) override {
return sys::Memory::releaseMappedMemory(M);
}
};
DefaultMMapper DefaultMMapperInstance;
} // namespace
SectionMemoryManager::SectionMemoryManager(MemoryMapper *MM)
: MMapper(MM ? *MM : DefaultMMapperInstance) {}
} // namespace llvm