| //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h" |
| |
| #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" |
| #include "llvm/Support/MathExtras.h" |
| |
| #include <future> |
| |
| using namespace llvm; |
| using namespace llvm::orc; |
| |
| namespace llvm { |
| namespace orc { |
| |
| class EPCIndirectionUtilsAccess { |
| public: |
| using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo; |
| using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector; |
| |
| static Expected<IndirectStubInfoVector> |
| getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) { |
| return EPCIU.getIndirectStubs(NumStubs); |
| }; |
| }; |
| |
| } // end namespace orc |
| } // end namespace llvm |
| |
| namespace { |
| |
| class EPCTrampolinePool : public TrampolinePool { |
| public: |
| EPCTrampolinePool(EPCIndirectionUtils &EPCIU); |
| Error deallocatePool(); |
| |
| protected: |
| Error grow() override; |
| |
| using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; |
| |
| EPCIndirectionUtils &EPCIU; |
| unsigned TrampolineSize = 0; |
| unsigned TrampolinesPerPage = 0; |
| std::vector<FinalizedAlloc> TrampolineBlocks; |
| }; |
| |
| class EPCIndirectStubsManager : public IndirectStubsManager, |
| private EPCIndirectionUtilsAccess { |
| public: |
| EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {} |
| |
| Error deallocateStubs(); |
| |
| Error createStub(StringRef StubName, JITTargetAddress StubAddr, |
| JITSymbolFlags StubFlags) override; |
| |
| Error createStubs(const StubInitsMap &StubInits) override; |
| |
| JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; |
| |
| JITEvaluatedSymbol findPointer(StringRef Name) override; |
| |
| Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; |
| |
| private: |
| using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; |
| |
| std::mutex ISMMutex; |
| EPCIndirectionUtils &EPCIU; |
| StringMap<StubInfo> StubInfos; |
| }; |
| |
| EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) |
| : EPCIU(EPCIU) { |
| auto &EPC = EPCIU.getExecutorProcessControl(); |
| auto &ABI = EPCIU.getABISupport(); |
| |
| TrampolineSize = ABI.getTrampolineSize(); |
| TrampolinesPerPage = |
| (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; |
| } |
| |
| Error EPCTrampolinePool::deallocatePool() { |
| Error Err = Error::success(); |
| std::promise<MSVCPError> DeallocResultP; |
| auto DeallocResultF = DeallocResultP.get_future(); |
| |
| EPCIU.getExecutorProcessControl().getMemMgr().deallocate( |
| std::move(TrampolineBlocks), |
| [&](Error Err) { DeallocResultP.set_value(std::move(Err)); }); |
| |
| return DeallocResultF.get(); |
| } |
| |
| Error EPCTrampolinePool::grow() { |
| using namespace jitlink; |
| |
| assert(AvailableTrampolines.empty() && |
| "Grow called with trampolines still available"); |
| |
| auto ResolverAddress = EPCIU.getResolverBlockAddress(); |
| assert(ResolverAddress && "Resolver address can not be null"); |
| |
| auto &EPC = EPCIU.getExecutorProcessControl(); |
| auto PageSize = EPC.getPageSize(); |
| auto Alloc = SimpleSegmentAlloc::Create( |
| EPC.getMemMgr(), nullptr, |
| {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}}); |
| if (!Alloc) |
| return Alloc.takeError(); |
| |
| unsigned NumTrampolines = TrampolinesPerPage; |
| |
| auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); |
| EPCIU.getABISupport().writeTrampolines( |
| SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines); |
| for (unsigned I = 0; I < NumTrampolines; ++I) |
| AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize)); |
| |
| auto FA = Alloc->finalize(); |
| if (!FA) |
| return FA.takeError(); |
| |
| TrampolineBlocks.push_back(std::move(*FA)); |
| |
| return Error::success(); |
| } |
| |
| Error EPCIndirectStubsManager::createStub(StringRef StubName, |
| JITTargetAddress StubAddr, |
| JITSymbolFlags StubFlags) { |
| StubInitsMap SIM; |
| SIM[StubName] = std::make_pair(StubAddr, StubFlags); |
| return createStubs(SIM); |
| } |
| |
| Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { |
| auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size()); |
| if (!AvailableStubInfos) |
| return AvailableStubInfos.takeError(); |
| |
| { |
| std::lock_guard<std::mutex> Lock(ISMMutex); |
| unsigned ASIdx = 0; |
| for (auto &SI : StubInits) { |
| auto &A = (*AvailableStubInfos)[ASIdx++]; |
| StubInfos[SI.first()] = std::make_pair(A, SI.second.second); |
| } |
| } |
| |
| auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); |
| switch (EPCIU.getABISupport().getPointerSize()) { |
| case 4: { |
| unsigned ASIdx = 0; |
| std::vector<tpctypes::UInt32Write> PtrUpdates; |
| for (auto &SI : StubInits) |
| PtrUpdates.push_back( |
| {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), |
| static_cast<uint32_t>(SI.second.first)}); |
| return MemAccess.writeUInt32s(PtrUpdates); |
| } |
| case 8: { |
| unsigned ASIdx = 0; |
| std::vector<tpctypes::UInt64Write> PtrUpdates; |
| for (auto &SI : StubInits) |
| PtrUpdates.push_back( |
| {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), |
| static_cast<uint64_t>(SI.second.first)}); |
| return MemAccess.writeUInt64s(PtrUpdates); |
| } |
| default: |
| return make_error<StringError>("Unsupported pointer size", |
| inconvertibleErrorCode()); |
| } |
| } |
| |
| JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, |
| bool ExportedStubsOnly) { |
| std::lock_guard<std::mutex> Lock(ISMMutex); |
| auto I = StubInfos.find(Name); |
| if (I == StubInfos.end()) |
| return nullptr; |
| return {I->second.first.StubAddress, I->second.second}; |
| } |
| |
| JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { |
| std::lock_guard<std::mutex> Lock(ISMMutex); |
| auto I = StubInfos.find(Name); |
| if (I == StubInfos.end()) |
| return nullptr; |
| return {I->second.first.PointerAddress, I->second.second}; |
| } |
| |
| Error EPCIndirectStubsManager::updatePointer(StringRef Name, |
| JITTargetAddress NewAddr) { |
| |
| JITTargetAddress PtrAddr = 0; |
| { |
| std::lock_guard<std::mutex> Lock(ISMMutex); |
| auto I = StubInfos.find(Name); |
| if (I == StubInfos.end()) |
| return make_error<StringError>("Unknown stub name", |
| inconvertibleErrorCode()); |
| PtrAddr = I->second.first.PointerAddress; |
| } |
| |
| auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); |
| switch (EPCIU.getABISupport().getPointerSize()) { |
| case 4: { |
| tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); |
| return MemAccess.writeUInt32s(PUpdate); |
| } |
| case 8: { |
| tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); |
| return MemAccess.writeUInt64s(PUpdate); |
| } |
| default: |
| return make_error<StringError>("Unsupported pointer size", |
| inconvertibleErrorCode()); |
| } |
| } |
| |
| } // end anonymous namespace. |
| |
| namespace llvm { |
| namespace orc { |
| |
| EPCIndirectionUtils::ABISupport::~ABISupport() {} |
| |
| Expected<std::unique_ptr<EPCIndirectionUtils>> |
| EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { |
| const auto &TT = EPC.getTargetTriple(); |
| switch (TT.getArch()) { |
| default: |
| return make_error<StringError>( |
| std::string("No EPCIndirectionUtils available for ") + TT.str(), |
| inconvertibleErrorCode()); |
| case Triple::aarch64: |
| case Triple::aarch64_32: |
| return CreateWithABI<OrcAArch64>(EPC); |
| |
| case Triple::x86: |
| return CreateWithABI<OrcI386>(EPC); |
| |
| case Triple::mips: |
| return CreateWithABI<OrcMips32Be>(EPC); |
| |
| case Triple::mipsel: |
| return CreateWithABI<OrcMips32Le>(EPC); |
| |
| case Triple::mips64: |
| case Triple::mips64el: |
| return CreateWithABI<OrcMips64>(EPC); |
| |
| case Triple::x86_64: |
| if (TT.getOS() == Triple::OSType::Win32) |
| return CreateWithABI<OrcX86_64_Win32>(EPC); |
| else |
| return CreateWithABI<OrcX86_64_SysV>(EPC); |
| } |
| } |
| |
| Error EPCIndirectionUtils::cleanup() { |
| |
| auto &MemMgr = EPC.getMemMgr(); |
| auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs)); |
| |
| if (TP) |
| Err = joinErrors(std::move(Err), |
| static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); |
| |
| if (ResolverBlock) |
| Err = |
| joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock))); |
| |
| return Err; |
| } |
| |
| Expected<JITTargetAddress> |
| EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, |
| JITTargetAddress ReentryCtxAddr) { |
| using namespace jitlink; |
| |
| assert(ABI && "ABI can not be null"); |
| auto ResolverSize = ABI->getResolverCodeSize(); |
| |
| auto Alloc = |
| SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr, |
| {{MemProt::Read | MemProt::Exec, |
| {ResolverSize, Align(EPC.getPageSize())}}}); |
| |
| if (!Alloc) |
| return Alloc.takeError(); |
| |
| auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); |
| ABI->writeResolverCode(SegInfo.WorkingMem.data(), SegInfo.Addr, ReentryFnAddr, |
| ReentryCtxAddr); |
| |
| auto FA = Alloc->finalize(); |
| if (!FA) |
| return FA.takeError(); |
| |
| ResolverBlock = std::move(*FA); |
| return SegInfo.Addr; |
| } |
| |
| std::unique_ptr<IndirectStubsManager> |
| EPCIndirectionUtils::createIndirectStubsManager() { |
| return std::make_unique<EPCIndirectStubsManager>(*this); |
| } |
| |
| TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { |
| if (!TP) |
| TP = std::make_unique<EPCTrampolinePool>(*this); |
| return *TP; |
| } |
| |
| LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( |
| ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { |
| assert(!LCTM && |
| "createLazyCallThroughManager can not have been called before"); |
| LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, |
| &getTrampolinePool()); |
| return *LCTM; |
| } |
| |
| EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, |
| std::unique_ptr<ABISupport> ABI) |
| : EPC(EPC), ABI(std::move(ABI)) { |
| assert(this->ABI && "ABI can not be null"); |
| |
| assert(EPC.getPageSize() > getABISupport().getStubSize() && |
| "Stubs larger than one page are not supported"); |
| } |
| |
| Expected<EPCIndirectionUtils::IndirectStubInfoVector> |
| EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { |
| using namespace jitlink; |
| |
| std::lock_guard<std::mutex> Lock(EPCUIMutex); |
| |
| // If there aren't enough stubs available then allocate some more. |
| if (NumStubs > AvailableIndirectStubs.size()) { |
| auto NumStubsToAllocate = NumStubs; |
| auto PageSize = EPC.getPageSize(); |
| auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); |
| NumStubsToAllocate = StubBytes / ABI->getStubSize(); |
| auto PtrBytes = |
| alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); |
| |
| auto StubProt = MemProt::Read | MemProt::Exec; |
| auto PtrProt = MemProt::Read | MemProt::Write; |
| |
| auto Alloc = SimpleSegmentAlloc::Create( |
| EPC.getMemMgr(), nullptr, |
| {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}}, |
| {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}}); |
| |
| if (!Alloc) |
| return Alloc.takeError(); |
| |
| auto StubSeg = Alloc->getSegInfo(StubProt); |
| auto PtrSeg = Alloc->getSegInfo(PtrProt); |
| |
| ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr, |
| PtrSeg.Addr, NumStubsToAllocate); |
| |
| auto FA = Alloc->finalize(); |
| if (!FA) |
| return FA.takeError(); |
| |
| IndirectStubAllocs.push_back(std::move(*FA)); |
| |
| auto StubExecutorAddr = StubSeg.Addr; |
| auto PtrExecutorAddr = PtrSeg.Addr; |
| for (unsigned I = 0; I != NumStubsToAllocate; ++I) { |
| AvailableIndirectStubs.push_back( |
| IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr)); |
| StubExecutorAddr += ABI->getStubSize(); |
| PtrExecutorAddr += ABI->getPointerSize(); |
| } |
| } |
| |
| assert(NumStubs <= AvailableIndirectStubs.size() && |
| "Sufficient stubs should have been allocated above"); |
| |
| IndirectStubInfoVector Result; |
| while (NumStubs--) { |
| Result.push_back(AvailableIndirectStubs.back()); |
| AvailableIndirectStubs.pop_back(); |
| } |
| |
| return std::move(Result); |
| } |
| |
| static JITTargetAddress reentry(JITTargetAddress LCTMAddr, |
| JITTargetAddress TrampolineAddr) { |
| auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); |
| std::promise<JITTargetAddress> LandingAddrP; |
| auto LandingAddrF = LandingAddrP.get_future(); |
| LCTM.resolveTrampolineLandingAddress( |
| TrampolineAddr, |
| [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); |
| return LandingAddrF.get(); |
| } |
| |
| Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { |
| auto &LCTM = EPCIU.getLazyCallThroughManager(); |
| return EPCIU |
| .writeResolverBlock(pointerToJITTargetAddress(&reentry), |
| pointerToJITTargetAddress(&LCTM)) |
| .takeError(); |
| } |
| |
| } // end namespace orc |
| } // end namespace llvm |