lib/orc/elfnix_platform.cpp - llvm-project/compiler-rt - Git at Google

 //===- elfnix_platform.cpp ------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains code required to load the rest of the ELF-on-*IX runtime.
 //
 //===----------------------------------------------------------------------===//

 #include "elfnix_platform.h"
 #include "common.h"
 #include "error.h"
 #include "wrapper_function_utils.h"

 #include <map>
 #include <mutex>
 #include <sstream>
 #include <unordered_map>
 #include <vector>

 using namespace __orc_rt;
 using namespace __orc_rt::elfnix;

 // Declare function tags for functions in the JIT process.
 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag)
 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag)
 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)

 // eh-frame registration functions.
 // We expect these to be available for all processes.
 extern "C" void __register_frame(const void *);
 extern "C" void __deregister_frame(const void *);

 namespace {

 Error validatePointerSectionExtent(const char *SectionName,
                                    const ExecutorAddrRange &SE) {
   if (SE.size().getValue() % sizeof(uintptr_t)) {
     std::ostringstream ErrMsg;
     ErrMsg << std::hex << "Size of " << SectionName << " 0x"
            << SE.Start.getValue() << " -- 0x" << SE.End.getValue()
            << " is not a pointer multiple";
     return make_error<StringError>(ErrMsg.str());
   }
   return Error::success();
 }

 Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections,
                    const ELFNixJITDylibInitializers &MOJDIs) {

   for (const auto &ModInits : InitArraySections) {
     if (auto Err = validatePointerSectionExtent(".init_array", ModInits))
       return Err;

     using InitFunc = void (*)();
     for (auto *Init : ModInits.toSpan<InitFunc>())
       (*Init)();
   }

   return Error::success();
 }
 struct TLSInfoEntry {
   unsigned long Key = 0;
   unsigned long DataAddress = 0;
 };

 class ELFNixPlatformRuntimeState {
 private:
   struct AtExitEntry {
     void (*Func)(void *);
     void *Arg;
   };

   using AtExitsVector = std::vector<AtExitEntry>;

   struct PerJITDylibState {
     void *Header = nullptr;
     size_t RefCount = 0;
     bool AllowReinitialization = false;
     AtExitsVector AtExits;
   };

 public:
   static void initialize(void *DSOHandle);
   static ELFNixPlatformRuntimeState &get();
   static void destroy();

   ELFNixPlatformRuntimeState(void *DSOHandle)
       : PlatformJDDSOHandle(DSOHandle) {}

   // Delete copy and move constructors.
   ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
   ELFNixPlatformRuntimeState &
   operator=(const ELFNixPlatformRuntimeState &) = delete;
   ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
   ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;

   Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
   Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);

   const char *dlerror();
   void *dlopen(string_view Name, int Mode);
   int dlclose(void *DSOHandle);
   void *dlsym(void *DSOHandle, string_view Symbol);

   int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
   void runAtExits(void *DSOHandle);

   /// Returns the base address of the section containing ThreadData.
   Expected<std::pair<const char *, size_t>>
   getThreadDataSectionFor(const char *ThreadData);

   void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }

 private:
   PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
   PerJITDylibState *getJITDylibStateByName(string_view Path);
   PerJITDylibState &
   getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);

   Error registerThreadDataSection(span<const char> ThreadDataSection);

   Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
                                                 string_view Symbol);

   Expected<ELFNixJITDylibInitializerSequence>
   getJITDylibInitializersByName(string_view Path);
   Expected<void *> dlopenInitialize(string_view Path, int Mode);
   Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs);

   static ELFNixPlatformRuntimeState *MOPS;

   using InitSectionHandler =
       Error (*)(const std::vector<ExecutorAddrRange> &Sections,
                 const ELFNixJITDylibInitializers &MOJDIs);
   const std::vector<std::pair<const char *, InitSectionHandler>> InitSections =
       {{".init_array", runInitArray}};

   void *PlatformJDDSOHandle;

   // FIXME: Move to thread-state.
   std::string DLFcnError;

   std::recursive_mutex JDStatesMutex;
   std::unordered_map<void *, PerJITDylibState> JDStates;
   std::unordered_map<std::string, void *> JDNameToHeader;

   std::mutex ThreadDataSectionsMutex;
   std::map<const char *, size_t> ThreadDataSections;
 };

 ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;

 void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
   assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
   MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
 }

 ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
   return *MOPS;
 }

 void ELFNixPlatformRuntimeState::destroy() {
   assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
   delete MOPS;
 }

 Error ELFNixPlatformRuntimeState::registerObjectSections(
     ELFNixPerObjectSectionsToRegister POSR) {
   if (POSR.EHFrameSection.Start)
     __register_frame(POSR.EHFrameSection.Start.toPtr<const char *>());

   if (POSR.ThreadDataSection.Start) {
     if (auto Err = registerThreadDataSection(
             POSR.ThreadDataSection.toSpan<const char>()))
       return Err;
   }

   return Error::success();
 }

 Error ELFNixPlatformRuntimeState::deregisterObjectSections(
     ELFNixPerObjectSectionsToRegister POSR) {
   if (POSR.EHFrameSection.Start)
     __deregister_frame(POSR.EHFrameSection.Start.toPtr<const char *>());

   return Error::success();
 }

 const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }

 void *ELFNixPlatformRuntimeState::dlopen(string_view Path, int Mode) {
   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);

   // Use fast path if all JITDylibs are already loaded and don't require
   // re-running initializers.
   if (auto *JDS = getJITDylibStateByName(Path)) {
     if (!JDS->AllowReinitialization) {
       ++JDS->RefCount;
       return JDS->Header;
     }
   }

   auto H = dlopenInitialize(Path, Mode);
   if (!H) {
     DLFcnError = toString(H.takeError());
     return nullptr;
   }

   return *H;
 }

 int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
   runAtExits(DSOHandle);
   return 0;
 }

 void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) {
   auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
   if (!Addr) {
     DLFcnError = toString(Addr.takeError());
     return 0;
   }

   return Addr->toPtr<void *>();
 }

 int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
                                                void *DSOHandle) {
   // FIXME: Handle out-of-memory errors, returning -1 if OOM.
   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
   auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
   assert(JDS && "JITDylib state not initialized");
   JDS->AtExits.push_back({F, Arg});
   return 0;
 }

 void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
   // FIXME: Should atexits be allowed to run concurrently with access to
   // JDState?
   AtExitsVector V;
   {
     std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
     auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
     assert(JDS && "JITDlybi state not initialized");
     std::swap(V, JDS->AtExits);
   }

   while (!V.empty()) {
     auto &AE = V.back();
     AE.Func(AE.Arg);
     V.pop_back();
   }
 }

 Expected<std::pair<const char *, size_t>>
 ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
   auto I = ThreadDataSections.upper_bound(ThreadData);
   // Check that we have a valid entry conovering this address.
   if (I == ThreadDataSections.begin())
     return make_error<StringError>("No thread local data section for key");
   I = std::prev(I);
   if (ThreadData >= I->first + I->second)
     return make_error<StringError>("No thread local data section for key");
   return *I;
 }

 ELFNixPlatformRuntimeState::PerJITDylibState *
 ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
   auto I = JDStates.find(DSOHandle);
   if (I == JDStates.end())
     return nullptr;
   return &I->second;
 }

 ELFNixPlatformRuntimeState::PerJITDylibState *
 ELFNixPlatformRuntimeState::getJITDylibStateByName(string_view Name) {
   // FIXME: Avoid creating string copy here.
   auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
   if (I == JDNameToHeader.end())
     return nullptr;
   void *H = I->second;
   auto J = JDStates.find(H);
   assert(J != JDStates.end() &&
          "JITDylib has name map entry but no header map entry");
   return &J->second;
 }

 ELFNixPlatformRuntimeState::PerJITDylibState &
 ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
     ELFNixJITDylibInitializers &MOJDIs) {
   void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>();

   auto &JDS = JDStates[Header];

   // If this entry hasn't been created yet.
   if (!JDS.Header) {
     assert(!JDNameToHeader.count(MOJDIs.Name) &&
            "JITDylib has header map entry but no name map entry");
     JDNameToHeader[MOJDIs.Name] = Header;
     JDS.Header = Header;
   }

   return JDS;
 }

 Error ELFNixPlatformRuntimeState::registerThreadDataSection(
     span<const char> ThreadDataSection) {
   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
   auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
   if (I != ThreadDataSections.begin()) {
     auto J = std::prev(I);
     if (J->first + J->second > ThreadDataSection.data())
       return make_error<StringError>("Overlapping .tdata sections");
   }
   ThreadDataSections.insert(
       I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
   return Error::success();
 }

 Expected<ExecutorAddr>
 ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
                                                    string_view Sym) {
   Expected<ExecutorAddr> Result((ExecutorAddr()));
   if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
           SPSExecutorAddr, SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag,
                                              Result,
                                              ExecutorAddr::fromPtr(DSOHandle),
                                              Sym))
     return std::move(Err);
   return Result;
 }

 Expected<ELFNixJITDylibInitializerSequence>
 ELFNixPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) {
   Expected<ELFNixJITDylibInitializerSequence> Result(
       (ELFNixJITDylibInitializerSequence()));
   std::string PathStr(Path.data(), Path.size());
   if (auto Err =
           WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>(
               SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result,
                                 Path))
     return std::move(Err);
   return Result;
 }

 Expected<void *> ELFNixPlatformRuntimeState::dlopenInitialize(string_view Path,
                                                               int Mode) {
   // Either our JITDylib wasn't loaded, or it or one of its dependencies allows
   // reinitialization. We need to call in to the JIT to see if there's any new
   // work pending.
   auto InitSeq = getJITDylibInitializersByName(Path);
   if (!InitSeq)
     return InitSeq.takeError();

   // Init sequences should be non-empty.
   if (InitSeq->empty())
     return make_error<StringError>(
         "__orc_rt_elfnix_get_initializers returned an "
         "empty init sequence");

   // Otherwise register and run initializers for each JITDylib.
   for (auto &MOJDIs : *InitSeq)
     if (auto Err = initializeJITDylib(MOJDIs))
       return std::move(Err);

   // Return the header for the last item in the list.
   auto *JDS = getJITDylibStateByHeaderAddr(
       InitSeq->back().DSOHandleAddress.toPtr<void *>());
   assert(JDS && "Missing state entry for JD");
   return JDS->Header;
 }

 Error ELFNixPlatformRuntimeState::initializeJITDylib(
     ELFNixJITDylibInitializers &MOJDIs) {

   auto &JDS = getOrCreateJITDylibState(MOJDIs);
   ++JDS.RefCount;

   for (auto &KV : InitSections) {
     const auto &Name = KV.first;
     const auto &Handler = KV.second;
     auto I = MOJDIs.InitSections.find(Name);
     if (I != MOJDIs.InitSections.end()) {
       if (auto Err = Handler(I->second, MOJDIs))
         return Err;
     }
   }

   return Error::success();
 }
 class ELFNixPlatformRuntimeTLVManager {
 public:
   void *getInstance(const char *ThreadData);

 private:
   std::unordered_map<const char *, char *> Instances;
   std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
 };

 void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
   auto I = Instances.find(ThreadData);
   if (I != Instances.end())
     return I->second;
   auto TDS =
       ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
   if (!TDS) {
     __orc_rt_log_error(toString(TDS.takeError()).c_str());
     return nullptr;
   }

   auto &Allocated = AllocatedSections[TDS->first];
   if (!Allocated) {
     Allocated = std::make_unique<char[]>(TDS->second);
     memcpy(Allocated.get(), TDS->first, TDS->second);
   }
   size_t ThreadDataDelta = ThreadData - TDS->first;
   assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");

   char *Instance = Allocated.get() + ThreadDataDelta;
   Instances[ThreadData] = Instance;
   return Instance;
 }

 void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
   delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
 }

 } // end anonymous namespace

 //------------------------------------------------------------------------------
 //                             JIT entry points
 //------------------------------------------------------------------------------

 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
   return WrapperFunction<void(uint64_t)>::handle(
              ArgData, ArgSize,
              [](uint64_t &DSOHandle) {
                ELFNixPlatformRuntimeState::initialize(
                    reinterpret_cast<void *>(DSOHandle));
              })
       .release();
 }

 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
   ELFNixPlatformRuntimeState::destroy();
   return WrapperFunctionResult().release();
 }

 /// Wrapper function for registering metadata on a per-object basis.
 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
       handle(ArgData, ArgSize,
              [](ELFNixPerObjectSectionsToRegister &POSR) {
                return ELFNixPlatformRuntimeState::get().registerObjectSections(
                    std::move(POSR));
              })
           .release();
 }

 /// Wrapper for releasing per-object metadat.
 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
       handle(ArgData, ArgSize,
              [](ELFNixPerObjectSectionsToRegister &POSR) {
                return ELFNixPlatformRuntimeState::get()
                    .deregisterObjectSections(std::move(POSR));
              })
           .release();
 }

 //------------------------------------------------------------------------------
 //                           TLV support
 //------------------------------------------------------------------------------

 ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
   auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
       pthread_getspecific(D->Key));
   if (!TLVMgr)
     TLVMgr = new ELFNixPlatformRuntimeTLVManager();
   if (pthread_setspecific(D->Key, TLVMgr)) {
     __orc_rt_log_error("Call to pthread_setspecific failed");
     return nullptr;
   }

   return TLVMgr->getInstance(
       reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
 }

 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
 __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
   return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
              ArgData, ArgSize,
              []() -> Expected<uint64_t> {
                pthread_key_t Key;
                if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
                  __orc_rt_log_error("Call to pthread_key_create failed");
                  return make_error<StringError>(strerror(Err));
                }
                return static_cast<uint64_t>(Key);
              })
       .release();
 }

 //------------------------------------------------------------------------------
 //                           cxa_atexit support
 //------------------------------------------------------------------------------

 int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
                                void *dso_handle) {
   return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
                                                           dso_handle);
 }

 int __orc_rt_elfnix_atexit(void (*func)(void *)) {
   auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
   return ELFNixPlatformRuntimeState::get().registerAtExit(
       func, NULL, PlatformRTState.getPlatformJDDSOHandle());
 }

 void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
   ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
 }

 //------------------------------------------------------------------------------
 //                        JIT'd dlfcn alternatives.
 //------------------------------------------------------------------------------

 const char *__orc_rt_elfnix_jit_dlerror() {
   return ELFNixPlatformRuntimeState::get().dlerror();
 }

 void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
   return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
 }

 int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
   return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
 }

 void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
   return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
 }

 //------------------------------------------------------------------------------
 //                             ELFNix Run Program
 //------------------------------------------------------------------------------

 ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
     const char *JITDylibName, const char *EntrySymbolName, int argc,
     char *argv[]) {
   using MainTy = int (*)(int, char *[]);

   void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
                                        __orc_rt::elfnix::ORC_RT_RTLD_LAZY);
   if (!H) {
     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
     return -1;
   }

   auto *Main =
       reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));

   if (!Main) {
     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
     return -1;
   }

   int Result = Main(argc, argv);

   if (__orc_rt_elfnix_jit_dlclose(H) == -1)
     __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());

   return Result;
 }
	//===- elfnix_platform.cpp ------------------------------------------------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	//
	// This file contains code required to load the rest of the ELF-on-*IX runtime.
	//
	//===----------------------------------------------------------------------===//

	#include "elfnix_platform.h"
	#include "common.h"
	#include "error.h"
	#include "wrapper_function_utils.h"

	#include <map>
	#include <mutex>
	#include <sstream>
	#include <unordered_map>
	#include <vector>

	using namespace __orc_rt;
	using namespace __orc_rt::elfnix;

	// Declare function tags for functions in the JIT process.
	ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag)
	ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag)
	ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)

	// eh-frame registration functions.
	// We expect these to be available for all processes.
	extern "C" void __register_frame(const void *);
	extern "C" void __deregister_frame(const void *);

	namespace {

	Error validatePointerSectionExtent(const char *SectionName,
	const ExecutorAddrRange &SE) {
	if (SE.size().getValue() % sizeof(uintptr_t)) {
	std::ostringstream ErrMsg;
	ErrMsg << std::hex << "Size of " << SectionName << " 0x"
	<< SE.Start.getValue() << " -- 0x" << SE.End.getValue()
	<< " is not a pointer multiple";
	return make_error<StringError>(ErrMsg.str());
	}
	return Error::success();
	}

	Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections,
	const ELFNixJITDylibInitializers &MOJDIs) {

	for (const auto &ModInits : InitArraySections) {
	if (auto Err = validatePointerSectionExtent(".init_array", ModInits))
	return Err;

	using InitFunc = void (*)();
	for (auto *Init : ModInits.toSpan<InitFunc>())
	(*Init)();
	}

	return Error::success();
	}
	struct TLSInfoEntry {
	unsigned long Key = 0;
	unsigned long DataAddress = 0;
	};

	class ELFNixPlatformRuntimeState {
	private:
	struct AtExitEntry {
	void (Func)(void );
	void *Arg;
	};

	using AtExitsVector = std::vector<AtExitEntry>;

	struct PerJITDylibState {
	void *Header = nullptr;
	size_t RefCount = 0;
	bool AllowReinitialization = false;
	AtExitsVector AtExits;
	};

	public:
	static void initialize(void *DSOHandle);
	static ELFNixPlatformRuntimeState &get();
	static void destroy();

	ELFNixPlatformRuntimeState(void *DSOHandle)
	: PlatformJDDSOHandle(DSOHandle) {}

	// Delete copy and move constructors.
	ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
	ELFNixPlatformRuntimeState &
	operator=(const ELFNixPlatformRuntimeState &) = delete;
	ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
	ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;

	Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
	Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);

	const char *dlerror();
	void *dlopen(string_view Name, int Mode);
	int dlclose(void *DSOHandle);
	void dlsym(void DSOHandle, string_view Symbol);

	int registerAtExit(void (F)(void ), void Arg, void DSOHandle);
	void runAtExits(void *DSOHandle);

	/// Returns the base address of the section containing ThreadData.
	Expected<std::pair<const char *, size_t>>
	getThreadDataSectionFor(const char *ThreadData);

	void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }

	private:
	PerJITDylibState getJITDylibStateByHeaderAddr(void DSOHandle);
	PerJITDylibState *getJITDylibStateByName(string_view Path);
	PerJITDylibState &
	getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs);

	Error registerThreadDataSection(span<const char> ThreadDataSection);

	Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
	string_view Symbol);

	Expected<ELFNixJITDylibInitializerSequence>
	getJITDylibInitializersByName(string_view Path);
	Expected<void *> dlopenInitialize(string_view Path, int Mode);
	Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs);

	static ELFNixPlatformRuntimeState *MOPS;

	using InitSectionHandler =
	Error (*)(const std::vector<ExecutorAddrRange> &Sections,
	const ELFNixJITDylibInitializers &MOJDIs);
	const std::vector<std::pair<const char *, InitSectionHandler>> InitSections =
	{{".init_array", runInitArray}};

	void *PlatformJDDSOHandle;

	// FIXME: Move to thread-state.
	std::string DLFcnError;

	std::recursive_mutex JDStatesMutex;
	std::unordered_map<void *, PerJITDylibState> JDStates;
	std::unordered_map<std::string, void *> JDNameToHeader;

	std::mutex ThreadDataSectionsMutex;
	std::map<const char *, size_t> ThreadDataSections;
	};

	ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;

	void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
	assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
	MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
	}

	ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
	assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
	return *MOPS;
	}

	void ELFNixPlatformRuntimeState::destroy() {
	assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
	delete MOPS;
	}

	Error ELFNixPlatformRuntimeState::registerObjectSections(
	ELFNixPerObjectSectionsToRegister POSR) {
	if (POSR.EHFrameSection.Start)
	__register_frame(POSR.EHFrameSection.Start.toPtr<const char *>());

	if (POSR.ThreadDataSection.Start) {
	if (auto Err = registerThreadDataSection(
	POSR.ThreadDataSection.toSpan<const char>()))
	return Err;
	}

	return Error::success();
	}

	Error ELFNixPlatformRuntimeState::deregisterObjectSections(
	ELFNixPerObjectSectionsToRegister POSR) {
	if (POSR.EHFrameSection.Start)
	__deregister_frame(POSR.EHFrameSection.Start.toPtr<const char *>());

	return Error::success();
	}

	const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }

	void *ELFNixPlatformRuntimeState::dlopen(string_view Path, int Mode) {
	std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);

	// Use fast path if all JITDylibs are already loaded and don't require
	// re-running initializers.
	if (auto *JDS = getJITDylibStateByName(Path)) {
	if (!JDS->AllowReinitialization) {
	++JDS->RefCount;
	return JDS->Header;
	}
	}

	auto H = dlopenInitialize(Path, Mode);
	if (!H) {
	DLFcnError = toString(H.takeError());
	return nullptr;
	}

	return *H;
	}

	int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
	runAtExits(DSOHandle);
	return 0;
	}

	void ELFNixPlatformRuntimeState::dlsym(void DSOHandle, string_view Symbol) {
	auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
	if (!Addr) {
	DLFcnError = toString(Addr.takeError());
	return 0;
	}

	return Addr->toPtr<void *>();
	}

	int ELFNixPlatformRuntimeState::registerAtExit(void (F)(void ), void *Arg,
	void *DSOHandle) {
	// FIXME: Handle out-of-memory errors, returning -1 if OOM.
	std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
	auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
	assert(JDS && "JITDylib state not initialized");
	JDS->AtExits.push_back({F, Arg});
	return 0;
	}

	void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
	// FIXME: Should atexits be allowed to run concurrently with access to
	// JDState?
	AtExitsVector V;
	{
	std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
	auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
	assert(JDS && "JITDlybi state not initialized");
	std::swap(V, JDS->AtExits);
	}

	while (!V.empty()) {
	auto &AE = V.back();
	AE.Func(AE.Arg);
	V.pop_back();
	}
	}

	Expected<std::pair<const char *, size_t>>
	ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
	std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
	auto I = ThreadDataSections.upper_bound(ThreadData);
	// Check that we have a valid entry conovering this address.
	if (I == ThreadDataSections.begin())
	return make_error<StringError>("No thread local data section for key");
	I = std::prev(I);
	if (ThreadData >= I->first + I->second)
	return make_error<StringError>("No thread local data section for key");
	return *I;
	}

	ELFNixPlatformRuntimeState::PerJITDylibState *
	ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
	auto I = JDStates.find(DSOHandle);
	if (I == JDStates.end())
	return nullptr;
	return &I->second;
	}

	ELFNixPlatformRuntimeState::PerJITDylibState *
	ELFNixPlatformRuntimeState::getJITDylibStateByName(string_view Name) {
	// FIXME: Avoid creating string copy here.
	auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
	if (I == JDNameToHeader.end())
	return nullptr;
	void *H = I->second;
	auto J = JDStates.find(H);
	assert(J != JDStates.end() &&
	"JITDylib has name map entry but no header map entry");
	return &J->second;
	}

	ELFNixPlatformRuntimeState::PerJITDylibState &
	ELFNixPlatformRuntimeState::getOrCreateJITDylibState(
	ELFNixJITDylibInitializers &MOJDIs) {
	void Header = MOJDIs.DSOHandleAddress.toPtr<void >();

	auto &JDS = JDStates[Header];

	// If this entry hasn't been created yet.
	if (!JDS.Header) {
	assert(!JDNameToHeader.count(MOJDIs.Name) &&
	"JITDylib has header map entry but no name map entry");
	JDNameToHeader[MOJDIs.Name] = Header;
	JDS.Header = Header;
	}

	return JDS;
	}

	Error ELFNixPlatformRuntimeState::registerThreadDataSection(
	span<const char> ThreadDataSection) {
	std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
	auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
	if (I != ThreadDataSections.begin()) {
	auto J = std::prev(I);
	if (J->first + J->second > ThreadDataSection.data())
	return make_error<StringError>("Overlapping .tdata sections");
	}
	ThreadDataSections.insert(
	I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
	return Error::success();
	}

	Expected<ExecutorAddr>
	ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
	string_view Sym) {
	Expected<ExecutorAddr> Result((ExecutorAddr()));
	if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
	SPSExecutorAddr, SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag,
	Result,
	ExecutorAddr::fromPtr(DSOHandle),
	Sym))
	return std::move(Err);
	return Result;
	}

	Expected<ELFNixJITDylibInitializerSequence>
	ELFNixPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) {
	Expected<ELFNixJITDylibInitializerSequence> Result(
	(ELFNixJITDylibInitializerSequence()));
	std::string PathStr(Path.data(), Path.size());
	if (auto Err =
	WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>(
	SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result,
	Path))
	return std::move(Err);
	return Result;
	}

	Expected<void *> ELFNixPlatformRuntimeState::dlopenInitialize(string_view Path,
	int Mode) {
	// Either our JITDylib wasn't loaded, or it or one of its dependencies allows
	// reinitialization. We need to call in to the JIT to see if there's any new
	// work pending.
	auto InitSeq = getJITDylibInitializersByName(Path);
	if (!InitSeq)
	return InitSeq.takeError();

	// Init sequences should be non-empty.
	if (InitSeq->empty())
	return make_error<StringError>(
	"__orc_rt_elfnix_get_initializers returned an "
	"empty init sequence");

	// Otherwise register and run initializers for each JITDylib.
	for (auto &MOJDIs : *InitSeq)
	if (auto Err = initializeJITDylib(MOJDIs))
	return std::move(Err);

	// Return the header for the last item in the list.
	auto *JDS = getJITDylibStateByHeaderAddr(
	InitSeq->back().DSOHandleAddress.toPtr<void *>());
	assert(JDS && "Missing state entry for JD");
	return JDS->Header;
	}

	Error ELFNixPlatformRuntimeState::initializeJITDylib(
	ELFNixJITDylibInitializers &MOJDIs) {

	auto &JDS = getOrCreateJITDylibState(MOJDIs);
	++JDS.RefCount;

	for (auto &KV : InitSections) {
	const auto &Name = KV.first;
	const auto &Handler = KV.second;
	auto I = MOJDIs.InitSections.find(Name);
	if (I != MOJDIs.InitSections.end()) {
	if (auto Err = Handler(I->second, MOJDIs))
	return Err;
	}
	}

	return Error::success();
	}
	class ELFNixPlatformRuntimeTLVManager {
	public:
	void getInstance(const char ThreadData);

	private:
	std::unordered_map<const char , char > Instances;
	std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
	};

	void ELFNixPlatformRuntimeTLVManager::getInstance(const char ThreadData) {
	auto I = Instances.find(ThreadData);
	if (I != Instances.end())
	return I->second;
	auto TDS =
	ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
	if (!TDS) {
	__orc_rt_log_error(toString(TDS.takeError()).c_str());
	return nullptr;
	}

	auto &Allocated = AllocatedSections[TDS->first];
	if (!Allocated) {
	Allocated = std::make_unique<char[]>(TDS->second);
	memcpy(Allocated.get(), TDS->first, TDS->second);
	}
	size_t ThreadDataDelta = ThreadData - TDS->first;
	assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");

	char *Instance = Allocated.get() + ThreadDataDelta;
	Instances[ThreadData] = Instance;
	return Instance;
	}

	void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
	delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
	}

	} // end anonymous namespace

	//------------------------------------------------------------------------------
	// JIT entry points
	//------------------------------------------------------------------------------

	ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
	__orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
	return WrapperFunction<void(uint64_t)>::handle(
	ArgData, ArgSize,
	[](uint64_t &DSOHandle) {
	ELFNixPlatformRuntimeState::initialize(
	reinterpret_cast<void *>(DSOHandle));
	})
	.release();
	}

	ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
	__orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
	ELFNixPlatformRuntimeState::destroy();
	return WrapperFunctionResult().release();
	}

	/// Wrapper function for registering metadata on a per-object basis.
	ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
	__orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
	return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
	handle(ArgData, ArgSize,
	[](ELFNixPerObjectSectionsToRegister &POSR) {
	return ELFNixPlatformRuntimeState::get().registerObjectSections(
	std::move(POSR));
	})
	.release();
	}

	/// Wrapper for releasing per-object metadat.
	ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
	__orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
	return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
	handle(ArgData, ArgSize,
	[](ELFNixPerObjectSectionsToRegister &POSR) {
	return ELFNixPlatformRuntimeState::get()
	.deregisterObjectSections(std::move(POSR));
	})
	.release();
	}

	//------------------------------------------------------------------------------
	// TLV support
	//------------------------------------------------------------------------------

	ORC_RT_INTERFACE void __orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry D) {
	auto TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager >(
	pthread_getspecific(D->Key));
	if (!TLVMgr)
	TLVMgr = new ELFNixPlatformRuntimeTLVManager();
	if (pthread_setspecific(D->Key, TLVMgr)) {
	__orc_rt_log_error("Call to pthread_setspecific failed");
	return nullptr;
	}

	return TLVMgr->getInstance(
	reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
	}

	ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
	__orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
	return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
	ArgData, ArgSize,
	[]() -> Expected<uint64_t> {
	pthread_key_t Key;
	if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
	__orc_rt_log_error("Call to pthread_key_create failed");
	return make_error<StringError>(strerror(Err));
	}
	return static_cast<uint64_t>(Key);
	})
	.release();
	}

	//------------------------------------------------------------------------------
	// cxa_atexit support
	//------------------------------------------------------------------------------

	int __orc_rt_elfnix_cxa_atexit(void (func)(void ), void *arg,
	void *dso_handle) {
	return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
	dso_handle);
	}

	int __orc_rt_elfnix_atexit(void (func)(void )) {
	auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
	return ELFNixPlatformRuntimeState::get().registerAtExit(
	func, NULL, PlatformRTState.getPlatformJDDSOHandle());
	}

	void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
	ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
	}

	//------------------------------------------------------------------------------
	// JIT'd dlfcn alternatives.
	//------------------------------------------------------------------------------

	const char *__orc_rt_elfnix_jit_dlerror() {
	return ELFNixPlatformRuntimeState::get().dlerror();
	}

	void __orc_rt_elfnix_jit_dlopen(const char path, int mode) {
	return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
	}

	int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
	return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
	}

	void __orc_rt_elfnix_jit_dlsym(void dso_handle, const char *symbol) {
	return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
	}

	//------------------------------------------------------------------------------
	// ELFNix Run Program
	//------------------------------------------------------------------------------

	ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
	const char JITDylibName, const char EntrySymbolName, int argc,
	char *argv[]) {
	using MainTy = int ()(int, char []);

	void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
	__orc_rt::elfnix::ORC_RT_RTLD_LAZY);
	if (!H) {
	__orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
	return -1;
	}

	auto *Main =
	reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));

	if (!Main) {
	__orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
	return -1;
	}

	int Result = Main(argc, argv);

	if (__orc_rt_elfnix_jit_dlclose(H) == -1)
	__orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());

	return Result;
	}