orc-rt/lib/executor/SimpleNativeMemoryMap.cpp

//===- SimpleNativeMemoryMap.cpp ------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// SimpleNativeMemoryMap and related APIs.
//
// TODO: We don't reset / uncommit pages on deinitialize, or on failure during
//       initialize. We should do that to reduce memory pressure.
//
//===----------------------------------------------------------------------===//

#include "orc-rt/SimpleNativeMemoryMap.h"
#include "orc-rt/Session.h"

#include <optional>
#include <sstream>

#if defined(__APPLE__) || defined(__linux__)
#include "Unix/NativeMemoryAPIs.inc"
#else
#error "Target OS memory APIs unsupported"
#endif

namespace orc_rt {

Expected<std::unique_ptr<SimpleNativeMemoryMap>>
SimpleNativeMemoryMap::Create(Session &S, SimpleSymbolTable &ST,
                              const char *InstanceName,
                              SimpleSymbolTable::MutatorFn AddInterface) {

  std::unique_ptr<SimpleNativeMemoryMap> Instance(new SimpleNativeMemoryMap(S));

  SimpleSymbolTable SNMMST;
  if (auto Err = AddInterface(SNMMST))
    return Err;
  std::pair<const char *, const void *> InstanceSym[] = {
      {InstanceName, static_cast<const void *>(Instance.get())}};
  if (auto Err = SNMMST.addUnique(InstanceSym))
    return std::move(Err);

  if (auto Err = ST.addUnique(SNMMST))
    return std::move(Err);

  return std::move(Instance);
}

void SimpleNativeMemoryMap::reserve(OnReserveCompleteFn &&OnComplete,
                                    size_t Size) {
  if (Size % S.processInfo().pageSize()) {
    return OnComplete(make_error<StringError>(
        (std::ostringstream()
         << "SimpleNativeMemoryMap error: reserved size " << std::hex << Size
         << " is not a page-size multiple")
            .str()));
  }

  auto Addr = hostOSMemoryReserve(Size);
  if (!Addr)
    return OnComplete(Addr.takeError());

  {
    std::scoped_lock<std::mutex> Lock(M);
    assert(!Slabs.count(*Addr) &&
           "hostOSMemoryReserve returned duplicate addresses");
    Slabs.emplace(std::make_pair(*Addr, SlabInfo(Size)));
  }

  OnComplete(*Addr);
}

void SimpleNativeMemoryMap::release(OnReleaseCompleteFn &&OnComplete,
                                    void *Addr) {
  std::optional<SlabInfo> SI;
  {
    std::scoped_lock<std::mutex> Lock(M);
    auto I = Slabs.find(Addr);
    if (I != Slabs.end()) {
      SI = std::move(I->second);
      Slabs.erase(I);
    }
  }

  if (!SI) {
    std::ostringstream ErrMsg;
    ErrMsg << "SimpleNativeMemoryMap error: release called on unrecognized "
              "address "
           << Addr;
    return OnComplete(make_error<StringError>(ErrMsg.str()));
  }

  for (auto &[Addr, DAAs] : SI->DeallocActions)
    runDeallocActions(std::move(DAAs));

  OnComplete(hostOSMemoryRelease(Addr, SI->Size));
}

void SimpleNativeMemoryMap::releaseMultiple(OnReleaseCompleteFn &&OnComplete,
                                            std::vector<void *> Addrs) {
  releaseNext(std::move(OnComplete), std::move(Addrs), false, Error::success());
}

void SimpleNativeMemoryMap::initialize(OnInitializeCompleteFn &&OnComplete,
                                       InitializeRequest IR) {

  void *Base = nullptr;

  // TODO: Record initialize segments for release.
  // std::vector<std::pair<void*, size_t>> InitializeSegments;

  // Check segment validity before proceeding.
  for (auto &S : IR.Segments) {

    if (S.Content.size() > S.Size) {
      return OnComplete(make_error<StringError>(
          (std::ostringstream()
           << "For segment [" << (void *)S.Address << ".."
           << (void *)(S.Address + S.Size) << "), "
           << " content size (" << std::hex << S.Content.size()
           << ") exceeds segment size (" << S.Size << ")")
              .str()));
    }

    // Copy any requested content.
    if (!S.Content.empty())
      memcpy(S.Address, S.Content.data(), S.Content.size());

    // Zero-fill the rest of the section.
    if (size_t ZeroFillSize = S.Size - S.Content.size())
      memset(S.Address + S.Content.size(), 0, ZeroFillSize);

    if (auto Err = hostOSMemoryProtect(S.Address, S.Size, S.AG.getMemProt()))
      return OnComplete(std::move(Err));

    switch (S.AG.getMemLifetime()) {
    case MemLifetime::Standard:
      if (!Base || S.Address < Base)
        Base = S.Address;
      break;
    case MemLifetime::Finalize:
      // TODO: Record finalize segment for release.
      // FinalizeSegments.push_back({S.Address, S.Size});
      break;
    }
  }

  if (!Base)
    return OnComplete(
        make_error<StringError>("SimpleNativeMemoryMap initialize error: "
                                "finalization requires at least "
                                "one standard-lifetime segment"));

  auto DeallocActions = runFinalizeActions(std::move(IR.AAPs));
  if (!DeallocActions)
    return OnComplete(DeallocActions.takeError());

  if (auto Err = recordDeallocActions(Base, std::move(*DeallocActions))) {
    runDeallocActions(std::move(*DeallocActions));
    return OnComplete(std::move(Err));
  }

  OnComplete(Base);
}

void SimpleNativeMemoryMap::deinitialize(OnDeinitializeCompleteFn &&OnComplete,
                                         void *Base) {
  std::vector<AllocAction> DAAs;

  {
    std::unique_lock<std::mutex> Lock(M);
    auto *SI = findSlabInfoFor(Base);
    if (!SI) {
      Lock.unlock();
      return OnComplete(makeBadSlabError(Base, "deinitialize"));
    }

    auto I = SI->DeallocActions.find(Base);
    if (I == SI->DeallocActions.end()) {
      Lock.unlock();
      std::ostringstream ErrMsg;
      ErrMsg
          << "SimpleNativeMemoryMap deinitialize error: no deallocate actions "
             "registered for segment base address "
          << Base;
      return OnComplete(make_error<StringError>(ErrMsg.str()));
    }

    DAAs = std::move(I->second);
    SI->DeallocActions.erase(I);
  }

  runDeallocActions(std::move(DAAs));
  OnComplete(Error::success());
}

void SimpleNativeMemoryMap::deinitializeMultiple(
    OnDeinitializeCompleteFn &&OnComplete, std::vector<void *> Bases) {
  deinitializeNext(std::move(OnComplete), std::move(Bases), false,
                   Error::success());
}

void SimpleNativeMemoryMap::onDetach(Service::OnCompleteFn OnComplete,
                                     bool ShutdownRequested) {
  // Detach is a noop for now: we just retain all actions to run at shutdown
  // time.
  OnComplete();
}

void SimpleNativeMemoryMap::onShutdown(Service::OnCompleteFn OnComplete) {
  // TODO: Establish a clear order to run deallocate actions across slabs,
  // object boundaries.

  // Collect slab base addresses for removal.
  std::vector<void *> Bases;
  {
    std::scoped_lock<std::mutex> Lock(M);
    for (auto &[Base, _] : Slabs)
      Bases.push_back(Base);
  }

  shutdownNext(std::move(OnComplete), std::move(Bases));
}

void SimpleNativeMemoryMap::releaseNext(OnReleaseCompleteFn &&OnComplete,
                                        std::vector<void *> Addrs,
                                        bool AnyError, Error LastErr) {
  if (LastErr) {
    S.reportError(std::move(LastErr));
    AnyError |= true;
  }

  if (Addrs.empty()) {
    if (!AnyError)
      return OnComplete(Error::success());

    return OnComplete(
        make_error<StringError>("Failed to release some addresses"));
  }

  void *NextAddr = Addrs.back();
  Addrs.pop_back();

  release(
      [this, OnComplete = std::move(OnComplete), AnyError = AnyError,
       Addrs = std::move(Addrs)](Error Err) mutable {
        releaseNext(std::move(OnComplete), std::move(Addrs), AnyError,
                    std::move(Err));
      },
      NextAddr);
}

void SimpleNativeMemoryMap::deinitializeNext(
    OnDeinitializeCompleteFn &&OnComplete, std::vector<void *> Addrs,
    bool AnyError, Error LastErr) {
  if (LastErr) {
    S.reportError(std::move(LastErr));
    AnyError |= true;
  }

  if (Addrs.empty()) {
    if (!AnyError)
      return OnComplete(Error::success());

    return OnComplete(
        make_error<StringError>("Failed to deinitialize some addresses"));
  }

  void *NextAddr = Addrs.back();
  Addrs.pop_back();

  deinitialize(
      [this, OnComplete = std::move(OnComplete), AnyError = AnyError,
       Addrs = std::move(Addrs)](Error Err) mutable {
        deinitializeNext(std::move(OnComplete), std::move(Addrs), AnyError,
                         std::move(Err));
      },
      NextAddr);
}

void SimpleNativeMemoryMap::shutdownNext(Service::OnCompleteFn OnComplete,
                                         std::vector<void *> Bases) {
  if (Bases.empty())
    return OnComplete();

  auto *Base = Bases.back();
  Bases.pop_back();

  release(
      [this, Bases = std::move(Bases),
       OnComplete = std::move(OnComplete)](Error Err) mutable {
        if (Err) {
          // TODO: Log release error?
          consumeError(std::move(Err));
        }
        shutdownNext(std::move(OnComplete), std::move(Bases));
      },
      Base);
}

Error SimpleNativeMemoryMap::makeBadSlabError(void *Base, const char *Op) {
  std::ostringstream ErrMsg;
  ErrMsg << "SimpleNativeMemoryMap " << Op << " error: segment base address "
         << Base << " does not fall within an allocated slab";
  return make_error<StringError>(ErrMsg.str());
}

SimpleNativeMemoryMap::SlabInfo *
SimpleNativeMemoryMap::findSlabInfoFor(void *Base) {
  // NOTE: We assume that the caller is holding a lock for M.
  auto I = Slabs.upper_bound(Base);
  if (I == Slabs.begin())
    return nullptr;

  --I;
  if (reinterpret_cast<char *>(I->first) + I->second.Size <=
      reinterpret_cast<char *>(Base))
    return nullptr;

  return &I->second;
}

Error SimpleNativeMemoryMap::recordDeallocActions(
    void *Base, std::vector<AllocAction> DeallocActions) {

  std::unique_lock<std::mutex> Lock(M);
  auto *SI = findSlabInfoFor(Base);
  if (!SI) {
    Lock.unlock();
    return makeBadSlabError(Base, "deinitialize");
  }

  auto I = SI->DeallocActions.find(Base);
  if (I != SI->DeallocActions.end()) {
    Lock.unlock();
    std::ostringstream ErrMsg;
    ErrMsg << "SimpleNativeMemoryMap initialize error: segment base address "
              "reused in subsequent initialize call";
    return make_error<StringError>(ErrMsg.str());
  }

  SI->DeallocActions[Base] = std::move(DeallocActions);
  return Error::success();
}

} // namespace orc_rt