offload/languages/kernel/src/State.cpp

//===-- State.cpp - Kernel language persistent state ----------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "State.h"
#include "Types.h"

#include "OffloadAPI.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"

#include <atomic>
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <mutex>

using namespace llvm;
using namespace offload;

// Weak so another runtime object can override the default stream mode.
__attribute__((weak)) uint32_t PerThreadQueue = 0;

// Process-wide singleton and thread-state registry.
static std::mutex &getStateLock() {
  static std::mutex StateLock;
  return StateLock;
}
static std::atomic<StateTy *> StatePtr = nullptr;

static thread_local ThreadStateTy *ThreadState = nullptr;

static std::mutex &getThreadStatesLock() {
  static std::mutex ThreadStatesLock;
  return ThreadStatesLock;
}
using ThreadStatesTy = SmallVector<ThreadStateTy *, 64>;
static ThreadStatesTy *ThreadStatesPtr = nullptr;

static void deleteThreadStates() {
  // Detach the registry before deletion because deleteThreadState may be called
  // more than once via atexit and StateTy teardown.
  std::lock_guard<std::mutex> LG(getThreadStatesLock());
  ThreadStatesTy *ThreadStates = ThreadStatesPtr;
  ThreadStatesPtr = nullptr;
  if (!ThreadStates)
    return;

  for (auto *TS : *ThreadStates)
    delete TS;
  delete ThreadStates;
  ThreadState = nullptr;
  ThreadStates = nullptr;
}

static void deleteState() {
  StateTy *ST = StatePtr.load();
  StatePtr.store(nullptr);
  delete ST;
  StatePtr = nullptr;
}

static void destroyQueue(ol_queue_handle_t &Queue) {
  if (!Queue)
    return;

  olSyncQueue(Queue);
  olDestroyQueue(Queue);
  Queue = nullptr;
}

namespace llvm {
namespace offload {

// ThreadStateTy implementation.

ThreadStateTy::ThreadStateTy() {
  if (PerThreadQueue) [[unlikely]]
    createDefaultQueue(getDefaultDevice());
  atexit(deleteThreadStates);
}
ThreadStateTy::~ThreadStateTy() { destroyQueue(DefaultQueue); }

ThreadStateTy &ThreadStateTy::get() {
  auto *&TS = ThreadState;
  if (!TS) {
    TS = new ThreadStateTy();
    std::lock_guard<std::mutex> LG(getThreadStatesLock());
    if (!ThreadStatesPtr)
      ThreadStatesPtr = new ThreadStatesTy;
    ThreadStatesPtr->push_back(TS);
  }
  return *TS;
}

ol_device_handle_t ThreadStateTy::getDefaultDevice() {
  int DD = ThreadStateTy::get().DefaultDevice;
  return StateTy::get().getDevices()[DD];
}

ol_queue_handle_t ThreadStateTy::getDefaultQueue() {
  if (!PerThreadQueue) [[likely]]
    return StateTy::get().DefaultQueue;
  return ThreadStateTy::get().DefaultQueue;
}

CallConfigurationTy &ThreadStateTy::getCallConfiguration() {
  return ThreadStateTy::get().CC;
}

ol_device_handle_t ThreadStateTy::setDefaultDevice(int DeviceNo) {
  ArrayRef<ol_device_handle_t> Devices = StateTy::get().getDevices();
  if (DeviceNo < 0 || DeviceNo >= static_cast<int>(Devices.size()))
    return nullptr;
  ThreadStateTy::get().DefaultDevice = DeviceNo;
  ol_device_handle_t DD = Devices[DeviceNo];
  ThreadStateTy::get().createDefaultQueue(DD);
  return DD;
}

ol_device_handle_t ThreadStateTy::getDevice(int *DeviceNo) {
  *DeviceNo = ThreadStateTy::get().DefaultDevice;
  return ThreadStateTy::getDefaultDevice();
}

uint32_t ThreadStateTy::getLastError() {
  return ThreadStateTy::get().LastError;
}

uint32_t ThreadStateTy::setLastError(uint32_t Error) {
  return ThreadStateTy::get().LastError = Error;
}

void ThreadStateTy::createDefaultQueue(ol_device_handle_t Device) {
  if (DefaultQueue)
    olDestroyQueue(DefaultQueue);
  CHECK_FATAL(olCreateQueue(StateTy::getContext(), Device, &DefaultQueue),
              "Failed to create per-thread default queue");
}

// StateTy implementation.

StateTy &StateTy::get() {
  StateTy *ST = StatePtr.load();
  if (!ST) [[unlikely]] {
    std::lock_guard<std::mutex> LG(getStateLock());
    ST = StatePtr.load();
    if (!ST) {
      ST = new StateTy();
      StatePtr.store(ST);
    }
  }
  return *ST;
}

StateTy *StateTy::tryGet() { return StatePtr.load(); }

ol_device_handle_t StateTy::getHostDevice() { return get().HostDevice; }

ol_context_handle_t StateTy::getContext() { return get().Context; }

int StateTy::getDeviceCount() {
  int DeviceCount = get().getDevices().size();
  return DeviceCount;
}

ArrayRef<ol_device_handle_t> StateTy::getDevices() const { return Devices; }

void StateTy::addDevice(ol_device_handle_t Device) {
  Devices.push_back(Device);
}

void StateTy::setHostDevice(ol_device_handle_t Device) {
  if (!HostDevice)
    HostDevice = Device;
}

void StateTy::addKernel(KernelIDTy KernelID, ol_symbol_handle_t Kernel) {
  KernelMap[KernelID] = Kernel;
}

void StateTy::removeKernel(KernelIDTy KernelID) { KernelMap.erase(KernelID); }

ol_symbol_handle_t StateTy::lookupKernel(KernelIDTy KernelID) {
  return KernelMap[KernelID];
}

void StateTy::registerKernel(const void *ID, ol_symbol_handle_t Kernel) {
  get().addKernel(ID, Kernel);
}

void StateTy::unregisterKernel(const void *ID) {
  if (StateTy *State = tryGet())
    State->removeKernel(ID);
}

ol_symbol_handle_t StateTy::getKernel(const void *ID) {
  return get().lookupKernel(ID);
}

void StateTy::addProgram(const void *Binary, ol_program_handle_t Program) {
  BinaryRegisterMap[Binary] = Program;
}

ol_program_handle_t StateTy::removeProgram(const void *Binary) {
  auto It = BinaryRegisterMap.find(Binary);
  if (It == BinaryRegisterMap.end())
    return nullptr;
  ol_program_handle_t Program = It->second;
  BinaryRegisterMap.erase(It);
  return Program;
}

ol_program_handle_t StateTy::lookupProgram(const void *Binary) {
  assert(BinaryRegisterMap.count(Binary) &&
         "Program not registered for binary");
  return BinaryRegisterMap[Binary];
}

void StateTy::registerProgram(const void *ID, ol_program_handle_t Program) {
  get().addProgram(ID, Program);
}

ol_program_handle_t StateTy::unregisterProgram(const void *ID) {
  if (StateTy *State = tryGet())
    return State->removeProgram(ID);
  return nullptr;
}

ol_program_handle_t StateTy::getProgram(const void *ID) {
  return get().lookupProgram(ID);
}

bool StateTy::addDevices(ol_device_handle_t Device, void *Payload) {
  StateTy &State = *reinterpret_cast<StateTy *>(Payload);
  ol_platform_handle_t Platform;
  ol_result_t Result;

  Result = olGetDeviceInfo(Device, OL_DEVICE_INFO_PLATFORM, sizeof(Platform),
                           &Platform);
  if (Result && Result->Code)
    return true;

  ol_platform_backend_t Backend;
  Result = olGetPlatformInfo(Platform, OL_PLATFORM_INFO_BACKEND,
                             sizeof(Backend), &Backend);
  if (Result && Result->Code)
    return true;

  if (Backend == OL_PLATFORM_BACKEND_HOST)
    State.setHostDevice(Device);
  else
    State.addDevice(Device);
  return true;
}

StateTy::StateTy() {
  CHECK_FATAL(olInit(nullptr), "Failed to initialize the LLVMOffload");
  CHECK_FATAL(olIterateDevices(StateTy::addDevices, this),
              "Failed to identify devices");

  if (!Devices.empty())
    CHECK_FATAL(olCreateContext(Devices.size(), Devices.data(), &Context),
                "Failed to create default context");

  if (!PerThreadQueue) [[likely]]
    if (!Devices.empty()) [[likely]]
      CHECK_FATAL(olCreateQueue(Context, Devices.front(), &DefaultQueue),
                  "Failed to create default queue");

  atexit(deleteState);
}

StateTy::~StateTy() {
  deleteThreadStates();
  destroyQueue(DefaultQueue);
  destroyRegisteredPrograms();
  if (Context)
    olDestroyContext(Context);
  olShutDown();
}

void StateTy::destroyRegisteredPrograms() {
  SmallPtrSet<ol_program_handle_t, 8> Programs;
  for (auto &It : BinaryRegisterMap)
    Programs.insert(It.second);

  KernelMap.clear();
  BinaryRegisterMap.clear();

  for (ol_program_handle_t Program : Programs)
    olDestroyProgram(Program);
}

} // namespace offload
} // namespace llvm