//===----RTLs/cuda/src/rtl.cpp - Target RTLs Implementation ------- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// RTL NextGen for CUDA machine
//
//===----------------------------------------------------------------------===//

#include <cassert>
#include <cstddef>
#include <cuda.h>
#include <string>
#include <unordered_map>

#include "Shared/APITypes.h"
#include "Shared/Debug.h"
#include "Shared/Environment.h"

#include "GlobalHandler.h"
#include "OffloadAPI.h"
#include "PluginInterface.h"
#include "Utils/ELF.h"

#include "llvm/ADT/StringExtras.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
#include "llvm/Frontend/OpenMP/OMPGridValues.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileOutputBuffer.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Program.h"

// This macro should be defined by the build system.
#ifndef OFFLOAD_MIN_CUDA_VERSION
#error "Missing OFFLOAD_MIN_CUDA_VERSION macro"
#endif

using namespace llvm::offload::debug;
using namespace error;

namespace llvm {
namespace omp {
namespace target {
namespace plugin {

/// Forward declarations for all specialized data structures.
struct CUDAKernelTy;
struct CUDADeviceTy;
struct CUDAPluginTy;

/// Class implementing the CUDA device images properties.
struct CUDADeviceImageTy : public DeviceImageTy {
  /// Create the CUDA image with the id and the target image pointer.
  CUDADeviceImageTy(int32_t ImageId, GenericDeviceTy &Device,
                    std::unique_ptr<MemoryBuffer> &&TgtImage)
      : DeviceImageTy(ImageId, Device, std::move(TgtImage)), Module(nullptr) {}

  /// Load the image as a CUDA module.
  Error loadModule() {
    assert(!Module && "Module already loaded");

    CUresult Res = cuModuleLoadDataEx(&Module, getStart(), 0, nullptr, nullptr);
    if (auto Err = Plugin::check(Res, "error in cuModuleLoadDataEx: %s"))
      return Err;

    return Plugin::success();
  }

  /// Unload the CUDA module corresponding to the image.
  Error unloadModule() {
    assert(Module && "Module not loaded");

    CUresult Res = cuModuleUnload(Module);
    if (auto Err = Plugin::check(Res, "error in cuModuleUnload: %s"))
      return Err;

    Module = nullptr;

    return Plugin::success();
  }

  /// Getter of the CUDA module.
  CUmodule getModule() const { return Module; }

private:
  /// The CUDA module that loaded the image.
  CUmodule Module;
};

/// Class implementing the CUDA kernel functionalities which derives from the
/// generic kernel class.
struct CUDAKernelTy : public GenericKernelTy {
  /// Create a CUDA kernel with a name and an execution mode.
  CUDAKernelTy(StringRef Name) : GenericKernelTy(Name), Func(nullptr) {}

  /// Initialize the CUDA kernel.
  Error initImpl(GenericDeviceTy &GenericDevice,
                 DeviceImageTy &Image) override {
    CUresult Res;
    CUDADeviceImageTy &CUDAImage = static_cast<CUDADeviceImageTy &>(Image);

    // Retrieve the function pointer of the kernel.
    Res = cuModuleGetFunction(&Func, CUDAImage.getModule(), getName());
    if (auto Err = Plugin::check(Res, "error in cuModuleGetFunction('%s'): %s",
                                 getName()))
      return Err;

    // Check that the function pointer is valid.
    if (!Func)
      return Plugin::error(ErrorCode::INVALID_BINARY,
                           "invalid function for kernel %s", getName());

    int MaxThreads;
    Res = cuFuncGetAttribute(&MaxThreads,
                             CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, Func);
    if (auto Err = Plugin::check(Res, "error in cuFuncGetAttribute: %s"))
      return Err;

    // The maximum number of threads per block for this kernel's function.
    MaxNumThreads = (uint32_t)MaxThreads;

    int SharedMemSize;
    Res = cuFuncGetAttribute(&SharedMemSize,
                             CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, Func);
    if (auto Err = Plugin::check(Res, "Error in cuFuncGetAttribute: %s"))
      return Err;

    // Set the static block memory size required by the kernel.
    StaticBlockMemSize = SharedMemSize;

    return Plugin::success();
  }

  /// Launch the CUDA kernel function.
  Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],
                   uint32_t NumBlocks[3], uint32_t DynBlockMemSize,
                   KernelLaunchArgsTy &LaunchArgs,
                   AsyncInfoWrapperTy &AsyncInfoWrapper) const override;

  /// Return maximum block size for maximum occupancy
  Expected<uint64_t> maxGroupSize(GenericDeviceTy &,
                                  uint64_t DynamicMemSize) const override {
    int MinGridSize;
    int MaxBlockSize;
    auto Res = cuOccupancyMaxPotentialBlockSize(
        &MinGridSize, &MaxBlockSize, Func, NULL, DynamicMemSize, INT_MAX);
    if (auto Err = Plugin::check(
            Res, "error in cuOccupancyMaxPotentialBlockSize: %s")) {
      return Err;
    }
    return MaxBlockSize;
  }

  /// Get maximum cooperative group count
  Expected<uint32_t>
  getMaxCooperativeGroupCount(GenericDeviceTy &GenericDevice,
                              const uint32_t NumThreads[3],
                              uint32_t DynBlockMemSize) const override;

  /// Return the maximum number of threads per block for this kernel's
  /// function, as reported by the CUDA driver.
  uint32_t getMaxThreads() const override { return MaxNumThreads; }

private:
  /// The CUDA kernel function to execute.
  CUfunction Func;
  /// The maximum amount of dynamic shared memory per thread group. By default,
  /// this is set to 48 KB.
  mutable uint32_t MaxDynBlockMemSize = 49152;
  /// The maximum number of threads per block this kernel's function may use,
  /// as reported by the CUDA driver.
  uint32_t MaxNumThreads = 0;
};

/// Class wrapping a CUDA stream reference. These are the objects handled by the
/// Stream Manager for the CUDA plugin.
struct CUDAStreamRef final : public GenericDeviceResourceRef {
  /// The underlying handle type for streams.
  using HandleTy = CUstream;

  /// Create an empty reference to an invalid stream.
  CUDAStreamRef() : Stream(nullptr) {}

  /// Create a reference to an existing stream.
  CUDAStreamRef(HandleTy Stream) : Stream(Stream) {}

  /// Create a new stream and save the reference. The reference must be empty
  /// before calling to this function.
  Error create(GenericDeviceTy &Device) override {
    if (Stream)
      return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                           "creating an existing stream");

    CUresult Res = cuStreamCreate(&Stream, CU_STREAM_NON_BLOCKING);
    if (auto Err = Plugin::check(Res, "error in cuStreamCreate: %s"))
      return Err;

    return Plugin::success();
  }

  /// Destroy the referenced stream and invalidate the reference. The reference
  /// must be to a valid stream before calling to this function.
  Error destroy(GenericDeviceTy &Device) override {
    if (!Stream)
      return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                           "destroying an invalid stream");

    CUresult Res = cuStreamDestroy(Stream);
    if (auto Err = Plugin::check(Res, "error in cuStreamDestroy: %s"))
      return Err;

    Stream = nullptr;
    return Plugin::success();
  }

  /// Get the underlying CUDA stream.
  operator HandleTy() const { return Stream; }

private:
  /// The reference to the CUDA stream.
  HandleTy Stream;
};

/// Class wrapping a CUDA event reference. These are the objects handled by the
/// Event Manager for the CUDA plugin.
struct CUDAEventRef final : public GenericDeviceResourceRef {
  /// The underlying handle type for events.
  using HandleTy = CUevent;

  /// Create an empty reference to an invalid event.
  CUDAEventRef() : Event(nullptr) {}

  /// Create a reference to an existing event.
  CUDAEventRef(HandleTy Event) : Event(Event) {}

  /// Create a new event and save the reference. The reference must be empty
  /// before calling to this function.
  Error create(GenericDeviceTy &Device) override {
    if (Event)
      return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                           "creating an existing event");

    CUresult Res = cuEventCreate(&Event, CU_EVENT_DISABLE_TIMING);
    if (auto Err = Plugin::check(Res, "error in cuEventCreate: %s"))
      return Err;

    return Plugin::success();
  }

  /// Destroy the referenced event and invalidate the reference. The reference
  /// must be to a valid event before calling to this function.
  Error destroy(GenericDeviceTy &Device) override {
    if (!Event)
      return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                           "destroying an invalid event");

    CUresult Res = cuEventDestroy(Event);
    if (auto Err = Plugin::check(Res, "error in cuEventDestroy: %s"))
      return Err;

    Event = nullptr;
    return Plugin::success();
  }

  /// Get the underlying CUevent.
  operator HandleTy() const { return Event; }

private:
  /// The reference to the CUDA event.
  HandleTy Event;
};

/// Class implementing the CUDA device functionalities which derives from the
/// generic device class.
struct CUDADeviceTy : public GenericDeviceTy {
  // Create a CUDA device with a device id and the default CUDA grid values.
  CUDADeviceTy(GenericPluginTy &Plugin, int32_t DeviceId, int32_t NumDevices)
      : GenericDeviceTy(Plugin, DeviceId, NumDevices, NVPTXGridValues),
        CUDAStreamManager(*this), CUDAEventManager(*this) {}

  ~CUDADeviceTy() {}

  /// Initialize the device, its resources and get its properties.
  Error initImpl(GenericPluginTy &Plugin,
                 GenericProfilerTy *ProfilerPtr) override {
    CUresult Res = cuDeviceGet(&Device, DeviceId);
    if (auto Err = Plugin::check(Res, "error in cuDeviceGet: %s"))
      return Err;

    CUuuid UUID = {0};
    Res = cuDeviceGetUuid(&UUID, Device);
    if (auto Err = Plugin::check(Res, "error in cuDeviceGetUuid: %s"))
      return Err;
    setDeviceUidFromVendorUid(
        toHex(StringRef(UUID.bytes, sizeof(UUID.bytes)), true));

    // Query the current flags of the primary context and set its flags if
    // it is inactive.
    unsigned int FormerPrimaryCtxFlags = 0;
    int FormerPrimaryCtxIsActive = 0;
    Res = cuDevicePrimaryCtxGetState(Device, &FormerPrimaryCtxFlags,
                                     &FormerPrimaryCtxIsActive);
    if (auto Err =
            Plugin::check(Res, "error in cuDevicePrimaryCtxGetState: %s"))
      return Err;

    if (FormerPrimaryCtxIsActive) {
      INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
           "The primary context is active, no change to its flags\n");
      if ((FormerPrimaryCtxFlags & CU_CTX_SCHED_MASK) !=
          CU_CTX_SCHED_BLOCKING_SYNC)
        INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
             "Warning: The current flags are not CU_CTX_SCHED_BLOCKING_SYNC\n");
    } else {
      INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId,
           "The primary context is inactive, set its flags to "
           "CU_CTX_SCHED_BLOCKING_SYNC\n");
      Res = cuDevicePrimaryCtxSetFlags(Device, CU_CTX_SCHED_BLOCKING_SYNC);
      if (auto Err =
              Plugin::check(Res, "error in cuDevicePrimaryCtxSetFlags: %s"))
        return Err;
    }

    // Retain the per device primary context and save it to use whenever this
    // device is selected.
    Res = cuDevicePrimaryCtxRetain(&Context, Device);
    if (auto Err = Plugin::check(Res, "error in cuDevicePrimaryCtxRetain: %s"))
      return Err;

    if (auto Err = setContext())
      return Err;

    // Initialize stream pool.
    if (auto Err = CUDAStreamManager.init(OMPX_InitialNumStreams))
      return Err;

    // Initialize event pool.
    if (auto Err = CUDAEventManager.init(OMPX_InitialNumEvents))
      return Err;

    // Query attributes to determine number of threads/block and blocks/grid.
    if (auto Err = getDeviceAttr(CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X,
                                 GridValues.GV_Max_Teams))
      return Err;

    if (auto Err = getDeviceAttr(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X,
                                 GridValues.GV_Max_WG_Size))
      return Err;

    if (auto Err = getDeviceAttr(CU_DEVICE_ATTRIBUTE_WARP_SIZE,
                                 GridValues.GV_Warp_Size))
      return Err;

    if (auto Err = getDeviceAttr(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR,
                                 ComputeCapability.Major))
      return Err;

    if (auto Err = getDeviceAttr(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR,
                                 ComputeCapability.Minor))
      return Err;

    uint32_t NumMuliprocessors = 0;
    uint32_t MaxThreadsPerSM = 0;
    uint32_t WarpSize = 0;
    if (auto Err = getDeviceAttr(CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT,
                                 NumMuliprocessors))
      return Err;
    if (auto Err =
            getDeviceAttr(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR,
                          MaxThreadsPerSM))
      return Err;
    if (auto Err = getDeviceAttr(CU_DEVICE_ATTRIBUTE_WARP_SIZE, WarpSize))
      return Err;
    HardwareParallelism = NumMuliprocessors * (MaxThreadsPerSM / WarpSize);

    uint32_t MaxSharedMem;
    if (auto Err = getDeviceAttr(
            CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK, MaxSharedMem))
      return Err;
    MaxBlockSharedMemSize = MaxSharedMem;

    CUmemAllocationProp Prop = {};
    Prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
    Prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
    Prop.location.id = DeviceId;

    Res = cuMemGetAllocationGranularity(&Granularity, &Prop,
                                        CU_MEM_ALLOC_GRANULARITY_MINIMUM);
    if (auto Err = Plugin::check(
            Res, "error in cuMemGetAllocationGranularity for the device: %s"))
      return Err;
    if (Granularity == 0)
      return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                           "wrong device page size");

    return Plugin::success();
  }

  Error unloadBinaryImpl(DeviceImageTy *Image) override {
    assert(Context && "Invalid CUDA context");

    // Each image has its own module.
    CUDADeviceImageTy &CUDAImage = static_cast<CUDADeviceImageTy &>(*Image);

    // Unload the module of the image.
    if (auto Err = CUDAImage.unloadModule())
      return Err;

    // Destroy the associated memory and invalidate the object.
    Plugin.free(Image);
    return Plugin::success();
  }

  /// Deinitialize the device and release its resources.
  Error deinitImpl() override {
    if (Context) {
      if (auto Err = setContext())
        return Err;
    }

    // Deinitialize the stream manager.
    if (auto Err = CUDAStreamManager.deinit())
      return Err;

    if (auto Err = CUDAEventManager.deinit())
      return Err;

    if (Context) {
      CUresult Res = cuDevicePrimaryCtxRelease(Device);
      if (auto Err =
              Plugin::check(Res, "error in cuDevicePrimaryCtxRelease: %s"))
        return Err;
    }

    // Invalidate context and device references.
    Context = nullptr;
    Device = CU_DEVICE_INVALID;

    return Plugin::success();
  }

  virtual Error callGlobalConstructors(GenericPluginTy &Plugin,
                                       DeviceImageTy &Image) override {
    return callGlobalCtorDtorCommon(Plugin, Image, /*IsCtor=*/true);
  }

  virtual Error callGlobalDestructors(GenericPluginTy &Plugin,
                                      DeviceImageTy &Image) override {
    return callGlobalCtorDtorCommon(Plugin, Image, /*IsCtor=*/false);
  }

  Expected<std::unique_ptr<MemoryBuffer>>
  doJITPostProcessing(std::unique_ptr<MemoryBuffer> MB) const override {
    // TODO: We should be able to use the 'nvidia-ptxjitcompiler' interface to
    //       avoid the call to 'ptxas'.
    SmallString<128> PTXInputFilePath;
    std::error_code EC = sys::fs::createTemporaryFile("nvptx-pre-link-jit", "s",
                                                      PTXInputFilePath);
    if (EC)
      return Plugin::error(ErrorCode::HOST_IO,
                           "failed to create temporary file for ptxas");

    // Write the file's contents to the output file.
    Expected<std::unique_ptr<FileOutputBuffer>> OutputOrErr =
        FileOutputBuffer::create(PTXInputFilePath, MB->getBuffer().size());
    if (!OutputOrErr)
      return OutputOrErr.takeError();
    std::unique_ptr<FileOutputBuffer> Output = std::move(*OutputOrErr);
    llvm::copy(MB->getBuffer(), Output->getBufferStart());
    if (Error E = Output->commit())
      return std::move(E);

    SmallString<128> PTXOutputFilePath;
    EC = sys::fs::createTemporaryFile("nvptx-post-link-jit", "cubin",
                                      PTXOutputFilePath);
    if (EC)
      return Plugin::error(ErrorCode::HOST_IO,
                           "failed to create temporary file for ptxas");

    // Try to find `ptxas` in the path to compile the PTX to a binary.
    const auto ErrorOrPath = sys::findProgramByName("ptxas");
    if (!ErrorOrPath)
      return Plugin::error(ErrorCode::HOST_TOOL_NOT_FOUND,
                           "failed to find 'ptxas' on the PATH.");

    std::string Arch = getComputeUnitKind();
    StringRef Args[] = {*ErrorOrPath,
                        "-m64",
                        "-O2",
                        "--gpu-name",
                        Arch,
                        "--output-file",
                        PTXOutputFilePath,
                        PTXInputFilePath};

    std::string ErrMsg;
    if (sys::ExecuteAndWait(*ErrorOrPath, Args, std::nullopt, {}, 0, 0,
                            &ErrMsg))
      return Plugin::error(ErrorCode::ASSEMBLE_FAILURE,
                           "running 'ptxas' failed: %s\n", ErrMsg.c_str());

    auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(PTXOutputFilePath.data());
    if (!BufferOrErr)
      return Plugin::error(ErrorCode::HOST_IO,
                           "failed to open temporary file for ptxas");

    // Clean up the temporary files afterwards.
    if (sys::fs::remove(PTXOutputFilePath))
      return Plugin::error(ErrorCode::HOST_IO,
                           "failed to remove temporary file for ptxas");
    if (sys::fs::remove(PTXInputFilePath))
      return Plugin::error(ErrorCode::HOST_IO,
                           "failed to remove temporary file for ptxas");

    return std::move(*BufferOrErr);
  }

  /// Allocate and construct a CUDA kernel.
  Expected<GenericKernelTy &> constructKernel(StringRef Name) override {
    // Allocate and construct the CUDA kernel.
    CUDAKernelTy *CUDAKernel = Plugin.allocate<CUDAKernelTy>();
    if (!CUDAKernel)
      return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
                           "failed to allocate memory for CUDA kernel");

    new (CUDAKernel) CUDAKernelTy(Name);

    return *CUDAKernel;
  }

  /// Set the current context to this device's context.
  Error setContext() override {
    CUresult Res = cuCtxSetCurrent(Context);
    return Plugin::check(Res, "error in cuCtxSetCurrent: %s");
  }

  /// NVIDIA returns the product of the SM count and the number of warps that
  /// fit if the maximum number of threads were scheduled on each SM.
  uint64_t getHardwareParallelism() const override {
    return HardwareParallelism;
  }

  /// We want to set up the RPC server for host services to the GPU if it is
  /// available.
  bool shouldSetupRPCServer() const override { return true; }

  /// The RPC interface should have enough space for all available parallelism.
  uint64_t requestedRPCPortCount() const override {
    return getHardwareParallelism();
  }

  /// Get the stream of the asynchronous info structure or get a new one.
  Error getStream(AsyncInfoWrapperTy &AsyncInfoWrapper, CUstream &Stream) {
    auto WrapperStream =
        AsyncInfoWrapper.getOrInitQueue<CUstream>(CUDAStreamManager);
    if (!WrapperStream)
      return WrapperStream.takeError();
    Stream = *WrapperStream;
    return Plugin::success();
  }

  /// Getters of CUDA references.
  CUcontext getCUDAContext() const { return Context; }
  CUdevice getCUDADevice() const { return Device; }

  /// Load the binary image into the device and allocate an image object.
  Expected<DeviceImageTy *>
  loadBinaryImpl(std::unique_ptr<MemoryBuffer> &&TgtImage, int32_t ImageId,
                 PluginContextTy * /*Context*/) override {
    if (auto Err = setContext())
      return std::move(Err);

    // Allocate and initialize the image object.
    CUDADeviceImageTy *CUDAImage = Plugin.allocate<CUDADeviceImageTy>();
    new (CUDAImage) CUDADeviceImageTy(ImageId, *this, std::move(TgtImage));

    // Load the CUDA module.
    if (auto Err = CUDAImage->loadModule())
      return std::move(Err);

    return CUDAImage;
  }

  /// Allocate memory on the device or related to the device.
  Expected<void *> allocate(size_t Size, void *, TargetAllocTy Kind,
                            size_t Alignment = 0) override {
    if (Size == 0)
      return nullptr;

    if (auto Err = setContext())
      return std::move(Err);

    void *MemAlloc = nullptr;
    CUdeviceptr DevicePtr;
    CUresult Res;

    if (Alignment > Granularity) {
      return Plugin::error(ErrorCode::UNSUPPORTED,
                           "requested alignment (%lu) larger than maximum "
                           "supported alignment (%lu)",
                           Alignment, Granularity);
    }

    switch (Kind) {
    case TARGET_ALLOC_DEFAULT:
    case TARGET_ALLOC_DEVICE:
      Res = cuMemAlloc(&DevicePtr, Size);
      MemAlloc = (void *)DevicePtr;
      break;
    case TARGET_ALLOC_HOST:
      Res = cuMemAllocHost(&MemAlloc, Size);
      break;
    case TARGET_ALLOC_SHARED:
      Res = cuMemAllocManaged(&DevicePtr, Size, CU_MEM_ATTACH_GLOBAL);
      MemAlloc = (void *)DevicePtr;
      break;
    }

    if (auto Err = Plugin::check(Res, "error in cuMemAlloc[Host|Managed]: %s"))
      return std::move(Err);

    if (Alignment > 0 && !isAddrAligned(Align(Alignment), MemAlloc)) {
      if (auto FreeErr = free(MemAlloc, Kind)) {
        return Plugin::error(ErrorCode::UNKNOWN,
                             "Failure in deallcation of the incorrectly "
                             "aligned pointer; requested alignemnt: %lu",
                             Alignment);
      }

      return Plugin::error(ErrorCode::UNSUPPORTED,
                           "unsupported alignment size");
    }

    return MemAlloc;
  }

  /// Deallocate memory on the device or related to the device.
  Error free(void *TgtPtr, TargetAllocTy Kind) override {
    if (TgtPtr == nullptr)
      return Plugin::success();

    if (auto Err = setContext())
      return Err;

    CUresult Res;
    switch (Kind) {
    case TARGET_ALLOC_DEFAULT:
    case TARGET_ALLOC_DEVICE:
    case TARGET_ALLOC_SHARED:
      Res = cuMemFree((CUdeviceptr)TgtPtr);
      break;
    case TARGET_ALLOC_HOST:
      Res = cuMemFreeHost(TgtPtr);
      break;
    }

    return Plugin::check(Res, "error in cuMemFree[Host]: %s");
  }

  /// Synchronize current thread with the pending operations on the async info.
  Error synchronizeImpl(__tgt_async_info &AsyncInfo,
                        bool ReleaseQueue) override {
    CUstream Stream = reinterpret_cast<CUstream>(AsyncInfo.Queue);
    CUresult Res;
    Res = cuStreamSynchronize(Stream);

    // Once the stream is synchronized and we want to release the queue, return
    // it to stream pool and reset AsyncInfo. This is to make sure the
    // synchronization only works for its own tasks.
    if (ReleaseQueue) {
      AsyncInfo.Queue = nullptr;
      if (auto Err = CUDAStreamManager.returnResource(Stream))
        return Err;
    }

    return Plugin::check(Res, "error in cuStreamSynchronize: %s");
  }

  /// Suggest a virtual address for device memory mapping.
  void *getSuggestedVirtualAddress() override {
    return reinterpret_cast<void *>(0x153940000000ULL);
  }

  /// Allocate \p Size bytes on the device and hints the backend to map it to
  /// virtual address \p VAddr. The function returns the allocated virtual
  /// address. The memory must be deallocated through
  /// GenericDeviceTy::deallocateWithVirtualAddress().
  Expected<void *> allocateWithVirtualAddress(uint64_t Size,
                                              void *VAddr) override {
    CUdeviceptr ExpectedVAddr = 0;
    if (VAddr != nullptr)
      ExpectedVAddr = reinterpret_cast<CUdeviceptr>(VAddr);

    // Get the page size in the device.
    size_t Granularity = 0;
    CUmemAllocationProp Prop = {};
    Prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
    Prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
    Prop.location.id = DeviceId;
    CUresult Res = cuMemGetAllocationGranularity(
        &Granularity, &Prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM);
    if (auto Err =
            Plugin::check(Res, "error in cuMemGetAllocationGranularity: %s"))
      return Err;
    if (Granularity == 0)
      return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                           "wrong device page size");

    // Transparently round up to a multiple of the page size.
    Size = llvm::alignTo(Size, Granularity);

    // Reserve the virtual address range.
    CUdeviceptr DevPtr = 0;
    Res = cuMemAddressReserve(&DevPtr, Size, 0, ExpectedVAddr, 0);
    if (auto Err = Plugin::check(Res, "error in cuMemAddressReserve: %s"))
      return Err;

    if (ExpectedVAddr != 0 && ExpectedVAddr != DevPtr)
      ODBG(OLDT_Alloc) << "cuMemAddressReserve reserved device virtual address "
                       << reinterpret_cast<void *>(DevPtr) << " instead of "
                       << reinterpret_cast<void *>(ExpectedVAddr);

    // Create a handle of the allocation.
    CUmemGenericAllocationHandle Handle;
    Res = cuMemCreate(&Handle, Size, &Prop, 0);
    if (auto Err = Plugin::check(Res, "error in cuMemCreate: %s"))
      return Err;

    // Map the virtual address range to the memory allocation.
    Res = cuMemMap(DevPtr, Size, 0, Handle, 0);
    if (auto Err = Plugin::check(Res, "error in cuMemMap: %s"))
      return Err;

    // Set the memory access properties for the allocation.
    CUmemAccessDesc Desc = {};
    Desc.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
    Desc.location.id = DeviceId;
    Desc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
    Res = cuMemSetAccess(DevPtr, Size, &Desc, 1);
    if (auto Err = Plugin::check(Res, "error in cuMemSetAccess: %s"))
      return Err;

    VAddr = reinterpret_cast<void *>(DevPtr);

    // Register the virtual address range in the tracker.
    if (auto Err = VMemTracker.registerReservation(VAddr, Size, Handle))
      return Err;

    return VAddr;
  }

  /// Deallocate device memory \p VAddr, which was allocated through
  /// GenericDeviceTy::allocateWithVirtualAddress(), and unmap the virtual
  /// address range.
  Error deallocateWithVirtualAddress(void *VAddr, uint64_t) override {
    // Unregister the virtual address range and obtain the information about
    // the reservation.
    auto InfoOrErr = VMemTracker.unregisterReservation(VAddr);
    if (!InfoOrErr)
      return InfoOrErr.takeError();

    auto [Size, Handle] = *InfoOrErr;
    CUdeviceptr DevAddr = reinterpret_cast<CUdeviceptr>(VAddr);

    CUresult Res = cuMemUnmap(DevAddr, Size);
    if (auto Err = Plugin::check(Res, "error in cuMemUnmap: %s"))
      return Err;

    Res = cuMemRelease(Handle);
    if (auto Err = Plugin::check(Res, "error in cuMemRelease: %s"))
      return Err;

    Res = cuMemAddressFree(DevAddr, Size);
    if (auto Err = Plugin::check(Res, "error in cuMemAddressFree: %s"))
      return Err;

    return Plugin::success();
  }

  /// Query for the completion of the pending operations on the async info.
  Error queryAsyncImpl(__tgt_async_info &AsyncInfo, bool ReleaseQueue,
                       bool *IsQueueWorkCompleted) override {
    if (IsQueueWorkCompleted)
      *IsQueueWorkCompleted = false;
    CUstream Stream = reinterpret_cast<CUstream>(AsyncInfo.Queue);
    CUresult Res = cuStreamQuery(Stream);

    // Not ready streams must be considered as successful operations.
    if (Res == CUDA_ERROR_NOT_READY)
      return Plugin::success();

    if (IsQueueWorkCompleted)
      *IsQueueWorkCompleted = true;
    // Once the stream is synchronized and the operations completed (or an error
    // occurs), return it to stream pool and reset AsyncInfo. This is to make
    // sure the synchronization only works for its own tasks.
    if (ReleaseQueue) {
      AsyncInfo.Queue = nullptr;
      if (auto Err = CUDAStreamManager.returnResource(Stream))
        return Err;
    }

    return Plugin::check(Res, "error in cuStreamQuery: %s");
  }

  Expected<void *> dataLockImpl(void *HstPtr, int64_t Size) override {
    // TODO: Register the buffer as CUDA host memory.
    return HstPtr;
  }

  Error dataUnlockImpl(void *HstPtr) override { return Plugin::success(); }

  Expected<bool> isPinnedPtrImpl(void *HstPtr, void *&BaseHstPtr,
                                 void *&BaseDevAccessiblePtr,
                                 size_t &BaseSize) const override {
    // TODO: Implement pinning feature for CUDA.
    return false;
  }

  /// Submit data to the device (host to device transfer).
  Error dataSubmitImpl(void *TgtPtr, const void *HstPtr, int64_t Size,
                       AsyncInfoWrapperTy &AsyncInfoWrapper) override {
    if (auto Err = setContext())
      return Err;

    CUstream Stream;
    if (auto Err = getStream(AsyncInfoWrapper, Stream))
      return Err;

    CUresult Res = cuMemcpyHtoDAsync((CUdeviceptr)TgtPtr, HstPtr, Size, Stream);
    return Plugin::check(Res, "error in cuMemcpyHtoDAsync: %s");
  }

  /// Retrieve data from the device (device to host transfer).
  Error dataRetrieveImpl(void *HstPtr, const void *TgtPtr, int64_t Size,
                         AsyncInfoWrapperTy &AsyncInfoWrapper) override {
    if (auto Err = setContext())
      return Err;

    CUstream Stream;
    if (auto Err = getStream(AsyncInfoWrapper, Stream))
      return Err;

    CUresult Res = cuMemcpyDtoHAsync(HstPtr, (CUdeviceptr)TgtPtr, Size, Stream);
    return Plugin::check(Res, "error in cuMemcpyDtoHAsync: %s");
  }

  Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
                       AsyncInfoWrapperTy &AsyncInfoWrapper) override {
    if (auto Err = setContext())
      return Err;

    CUstream Stream;
    if (auto Err = getStream(AsyncInfoWrapper, Stream))
      return Err;

    CUresult Res =
        cuMemcpyAsync((CUdeviceptr)DstPtr, (CUdeviceptr)SrcPtr, Size, Stream);
    return Plugin::check(Res, "error in cuMemcpyAsync: %s");
  }

  /// Exchange data between two devices directly. We may use peer access if
  /// the CUDA devices and driver allow them.
  Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstGenericDevice,
                         void *DstPtr, int64_t Size,
                         AsyncInfoWrapperTy &AsyncInfoWrapper) override;

  Error dataFillImpl(void *TgtPtr, const void *PatternPtr, int64_t PatternSize,
                     int64_t Size,
                     AsyncInfoWrapperTy &AsyncInfoWrapper) override {
    if (auto Err = setContext())
      return Err;

    CUstream Stream;
    if (auto Err = getStream(AsyncInfoWrapper, Stream))
      return Err;

    CUresult Res;
    size_t N = Size / PatternSize;
    if (PatternSize == 1) {
      Res = cuMemsetD8Async((CUdeviceptr)TgtPtr,
                            *(static_cast<const uint8_t *>(PatternPtr)), N,
                            Stream);
    } else if (PatternSize == 2) {
      Res = cuMemsetD16Async((CUdeviceptr)TgtPtr,
                             *(static_cast<const uint16_t *>(PatternPtr)), N,
                             Stream);
    } else if (PatternSize == 4) {
      Res = cuMemsetD32Async((CUdeviceptr)TgtPtr,
                             *(static_cast<const uint32_t *>(PatternPtr)), N,
                             Stream);
    } else {
      // For larger patterns we can do a series of strided fills to copy the
      // pattern efficiently
      int64_t MemsetSize = PatternSize % 4u == 0u   ? 4u
                           : PatternSize % 2u == 0u ? 2u
                                                    : 1u;

      int64_t NumberOfSteps = PatternSize / MemsetSize;
      int64_t Pitch = NumberOfSteps * MemsetSize;
      int64_t Height = Size / PatternSize;

      for (auto Step = 0u; Step < NumberOfSteps; ++Step) {
        if (MemsetSize == 4) {
          Res = cuMemsetD2D32Async(
              (CUdeviceptr)TgtPtr + Step * MemsetSize, Pitch,
              *(static_cast<const uint32_t *>(PatternPtr) + Step), 1u, Height,
              Stream);
        } else if (MemsetSize == 2) {
          Res = cuMemsetD2D16Async(
              (CUdeviceptr)TgtPtr + Step * MemsetSize, Pitch,
              *(static_cast<const uint16_t *>(PatternPtr) + Step), 1u, Height,
              Stream);
        } else {
          Res = cuMemsetD2D8Async(
              (CUdeviceptr)TgtPtr + Step * MemsetSize, Pitch,
              *(static_cast<const uint8_t *>(PatternPtr) + Step), 1u, Height,
              Stream);
        }
      }
    }

    return Plugin::check(Res, "error in cuMemset: %s");
  }

  /// Prefetch managed memory to the device or back to the host.
  // TODO: switch to cuMemPrefetchBatchAsync once the minimum supported CUDA
  // driver is 13 or newer. That entry point takes the (Mems, Sizes, Count)
  // arrays directly and lets the driver batch the migration.
  Error dataPrefetchImpl(size_t Count, const void **Mems, const size_t *Sizes,
                         bool ToHost,
                         AsyncInfoWrapperTy &AsyncInfoWrapper) override {
    if (Count == 0)
      return Plugin::success();
    if (auto Err = setContext())
      return Err;

    // Certain cuda devices and Windows do not have support for some Unified
    // Memory features. cuMemPrefetchAsync requires concurrent memory access
    // for managed memory. Therefore, ignore prefetch hint if concurrent managed
    // memory access is not available.
    int ConcurrentManagedAccess = 0;
    if (getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS,
                         ConcurrentManagedAccess) != CUDA_SUCCESS ||
        !ConcurrentManagedAccess)
      return Plugin::success();

    CUstream Stream;
    if (auto Err = getStream(AsyncInfoWrapper, Stream))
      return Err;

    CUdevice Dst = ToHost ? CU_DEVICE_CPU : Device;
    for (size_t I = 0; I < Count; I++) {
      if (Sizes[I] == 0)
        continue;

      // Prefetch only works with USM (managed) memory; ignore the hint
      // otherwise.
      unsigned int IsManaged = 0;
      if (cuPointerGetAttribute(&IsManaged, CU_POINTER_ATTRIBUTE_IS_MANAGED,
                                (CUdeviceptr)Mems[I]) != CUDA_SUCCESS ||
          !IsManaged)
        continue;

      CUresult Res =
          cuMemPrefetchAsync((CUdeviceptr)Mems[I], Sizes[I], Dst, Stream);
      if (auto Err = Plugin::check(Res, "error in cuMemPrefetchAsync: %s"))
        return Err;
    }
    return Plugin::success();
  }

  /// Insert a data fence between previous data operations and the following
  /// operations. This is a no-op for CUDA devices as operations inserted into
  /// a queue are in-order.
  Error dataFence(__tgt_async_info *Async) override {
    return Plugin::success();
  }

  interop_spec_t selectInteropPreference(int32_t InteropType,
                                         int32_t NumPrefers,
                                         interop_spec_t *Prefers) override {
    return interop_spec_t{tgt_fr_cuda, {true, 0}, 0};
  }

  Expected<omp_interop_val_t *>
  createInterop(int32_t InteropType, interop_spec_t &InteropSpec) override {
    auto *Ret = new omp_interop_val_t(
        DeviceId, static_cast<kmp_interop_type_t>(InteropType));
    Ret->fr_id = tgt_fr_cuda;
    Ret->vendor_id = omp_vendor_nvidia;

    if (InteropType == kmp_interop_type_target ||
        InteropType == kmp_interop_type_targetsync) {
      Ret->device_info.Platform = nullptr;
      Ret->device_info.Device = reinterpret_cast<void *>(Device);
      Ret->device_info.Context = Context;
    }

    if (InteropType == kmp_interop_type_targetsync) {
      Ret->async_info = new __tgt_async_info();
      if (auto Err = setContext())
        return Err;
      CUstream Stream;
      if (auto Err = CUDAStreamManager.getResource(Stream))
        return Err;

      Ret->async_info->Queue = Stream;
    }
    return Ret;
  }

  Error releaseInterop(omp_interop_val_t *Interop) override {
    if (!Interop)
      return Plugin::success();

    if (Interop->async_info)
      delete Interop->async_info;

    delete Interop;
    return Plugin::success();
  }

  Error enqueueHostCallImpl(void (*Callback)(void *), void *UserData,
                            AsyncInfoWrapperTy &AsyncInfo) override {
    if (auto Err = setContext())
      return Err;

    CUstream Stream;
    if (auto Err = getStream(AsyncInfo, Stream))
      return Err;

    CUresult Res = cuLaunchHostFunc(Stream, Callback, UserData);
    return Plugin::check(Res, "error in cuStreamLaunchHostFunc: %s");
  };

  /// Non-profiling events use CU_EVENT_DISABLE_TIMING from the pool.
  /// Profiling events are created directly with timing and not pooled.
  Error createEventImpl(void **EventPtrStorage, bool EnableProfiling) override {
    CUevent *Event = reinterpret_cast<CUevent *>(EventPtrStorage);
    if (EnableProfiling) {
      CUresult Res = cuEventCreate(Event, CU_EVENT_DEFAULT);
      return Plugin::check(Res, "error in cuEventCreate: %s");
    }
    return CUDAEventManager.getResource(*Event);
  }

  /// Profiling events are destroyed directly; non-profiling events return to
  /// the pool.
  Error destroyEventImpl(void *EventPtr, bool EnableProfiling) override {
    CUevent Event = reinterpret_cast<CUevent>(EventPtr);
    if (EnableProfiling) {
      CUresult Res = cuEventDestroy(Event);
      return Plugin::check(Res, "error in cuEventDestroy: %s");
    }
    return CUDAEventManager.returnResource(Event);
  }

  /// Record the event.
  Error recordEventImpl(void *EventPtr, AsyncInfoWrapperTy &AsyncInfoWrapper,
                        bool EnableProfiling) override {
    CUevent Event = reinterpret_cast<CUevent>(EventPtr);

    CUstream Stream;
    if (auto Err = getStream(AsyncInfoWrapper, Stream))
      return Err;

    CUresult Res = cuEventRecord(Event, Stream);
    return Plugin::check(Res, "error in cuEventRecord: %s");
  }

  /// Make the stream wait on the event.
  Error waitEventImpl(void *EventPtr,
                      AsyncInfoWrapperTy &AsyncInfoWrapper) override {
    CUevent Event = reinterpret_cast<CUevent>(EventPtr);

    CUstream Stream;
    if (auto Err = getStream(AsyncInfoWrapper, Stream))
      return Err;

    // Do not use CU_EVENT_WAIT_DEFAULT here as it is only available from
    // specific CUDA version, and defined as 0x0. In previous version, per CUDA
    // API document, that argument has to be 0x0.
    CUresult Res = cuStreamWaitEvent(Stream, Event, 0);
    return Plugin::check(Res, "error in cuStreamWaitEvent: %s");
  }

  Expected<bool> hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfo) override {
    CUstream Stream;
    if (auto Err = getStream(AsyncInfo, Stream))
      return Err;

    CUresult Ret = cuStreamQuery(Stream);
    if (Ret == CUDA_SUCCESS)
      return false;

    if (Ret == CUDA_ERROR_NOT_READY)
      return true;

    return Plugin::check(Ret, "error in cuStreamQuery: %s");
  }

  Expected<bool> isEventCompleteImpl(void *EventPtr,
                                     AsyncInfoWrapperTy &) override {
    CUevent Event = reinterpret_cast<CUevent>(EventPtr);

    CUresult Ret = cuEventQuery(Event);
    if (Ret == CUDA_SUCCESS)
      return true;

    if (Ret == CUDA_ERROR_NOT_READY)
      return false;

    return Plugin::check(Ret, "error in cuEventQuery: %s");
  }

  /// Synchronize the current thread with the event.
  Error syncEventImpl(void *EventPtr) override {
    CUevent Event = reinterpret_cast<CUevent>(EventPtr);
    CUresult Res = cuEventSynchronize(Event);
    return Plugin::check(Res, "error in cuEventSynchronize: %s");
  }

  /// Get the elapsed time in milliseconds between two events.
  Expected<float> getEventElapsedTimeImpl(void *StartEventPtr,
                                          void *EndEventPtr) override {
    CUevent StartEvent = reinterpret_cast<CUevent>(StartEventPtr);
    CUevent EndEvent = reinterpret_cast<CUevent>(EndEventPtr);

    float ElapsedTime = 0.0f;
    CUresult Res = cuEventElapsedTime(&ElapsedTime, StartEvent, EndEvent);
    if (auto Err = Plugin::check(Res, "error in cuEventElapsedTime: %s"))
      return std::move(Err);

    return ElapsedTime;
  }

  /// Print information about the device.
  Expected<InfoTreeNode> obtainInfoImpl() override {
    char TmpChar[1000];
    const char *TmpCharPtr;
    size_t TmpSt;
    int TmpInt;
    InfoTreeNode Info;

    CUresult Res = cuDriverGetVersion(&TmpInt);
    if (Res == CUDA_SUCCESS)
      // For consistency with other drivers, store the version as a string
      // rather than an integer
      Info.add("CUDA Driver Version", std::to_string(TmpInt), "",
               DeviceInfo::DRIVER_VERSION);

    Info.add("CUDA OpenMP Device Number", DeviceId);

    Res = cuDeviceGetName(TmpChar, 1000, Device);
    if (Res == CUDA_SUCCESS) {
      Info.add("Device Name", TmpChar, "", DeviceInfo::NAME);
      Info.add("Product Name", TmpChar, "", DeviceInfo::PRODUCT_NAME);
    }

    Info.add("Vendor Name", "NVIDIA", "", DeviceInfo::VENDOR);

    Info.add("Vendor ID", uint64_t{4318}, "", DeviceInfo::VENDOR_ID);

    Info.add("Memory Address Size", std::numeric_limits<CUdeviceptr>::digits,
             "bits", DeviceInfo::ADDRESS_BITS);

    Res = cuDeviceTotalMem(&TmpSt, Device);
    if (Res == CUDA_SUCCESS)
      Info.add("Global Memory Size", TmpSt, "bytes",
               DeviceInfo::GLOBAL_MEM_SIZE);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Number of Multiprocessors", TmpInt, "",
               DeviceInfo::NUM_COMPUTE_UNITS);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_GPU_OVERLAP, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Concurrent Copy and Execution", bool(TmpInt));

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_TOTAL_CONSTANT_MEMORY, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Total Constant Memory", TmpInt, "bytes");

    Info.add("Max Shared Memory per Block", MaxBlockSharedMemSize, "bytes",
             DeviceInfo::WORK_GROUP_LOCAL_MEM_SIZE);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_BLOCK, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Registers per Block", TmpInt);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_WARP_SIZE, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Number of Lanes", TmpInt, "", DeviceInfo::NUM_LANES);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Maximum Threads per Block", TmpInt, "",
               DeviceInfo::MAX_WORK_GROUP_SIZE);

    auto &MaxBlock = *Info.add("Maximum Block Dimensions", std::monostate{}, "",
                               DeviceInfo::MAX_WORK_GROUP_SIZE_PER_DIMENSION);
    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X, TmpInt);
    if (Res == CUDA_SUCCESS)
      MaxBlock.add("x", TmpInt);
    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Y, TmpInt);
    if (Res == CUDA_SUCCESS)
      MaxBlock.add("y", TmpInt);
    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Z, TmpInt);
    if (Res == CUDA_SUCCESS)
      MaxBlock.add("z", TmpInt);

    // TODO: I assume CUDA devices have no limit on the amount of threads,
    // verify this
    Info.add("Maximum Grid Size", std::numeric_limits<uint32_t>::max(), "",
             DeviceInfo::MAX_WORK_SIZE);

    auto &MaxGrid = *Info.add("Maximum Grid Dimensions", std::monostate{}, "",
                              DeviceInfo::MAX_WORK_SIZE_PER_DIMENSION);
    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X, TmpInt);
    if (Res == CUDA_SUCCESS)
      MaxGrid.add("x", TmpInt);
    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Y, TmpInt);
    if (Res == CUDA_SUCCESS)
      MaxGrid.add("y", TmpInt);
    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Z, TmpInt);
    if (Res == CUDA_SUCCESS)
      MaxGrid.add("z", TmpInt);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_PITCH, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Maximum Memory Pitch", TmpInt, "bytes");

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_TEXTURE_ALIGNMENT, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Texture Alignment", TmpInt, "bytes");

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_CLOCK_RATE, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Clock Rate", TmpInt / 1000, "MHz",
               DeviceInfo::MAX_CLOCK_FREQUENCY);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Execution Timeout", bool(TmpInt));

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_INTEGRATED, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Integrated Device", bool(TmpInt));

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Can Map Host Memory", bool(TmpInt));

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_COMPUTE_MODE, TmpInt);
    if (Res == CUDA_SUCCESS) {
      if (TmpInt == CU_COMPUTEMODE_DEFAULT)
        TmpCharPtr = "Default";
      else if (TmpInt == CU_COMPUTEMODE_PROHIBITED)
        TmpCharPtr = "Prohibited";
      else if (TmpInt == CU_COMPUTEMODE_EXCLUSIVE_PROCESS)
        TmpCharPtr = "Exclusive process";
      else
        TmpCharPtr = "Unknown";
      Info.add("Compute Mode", TmpCharPtr);
    }

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_CONCURRENT_KERNELS, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Concurrent Kernels", bool(TmpInt));

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_ECC_ENABLED, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("ECC Enabled", bool(TmpInt));

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Memory Clock Rate", TmpInt / 1000, "MHz",
               DeviceInfo::MEMORY_CLOCK_RATE);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Memory Bus Width", TmpInt, "bits");

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_L2_CACHE_SIZE, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("L2 Cache Size", TmpInt, "bytes");

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR,
                           TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Max Threads Per SMP", TmpInt);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_ASYNC_ENGINE_COUNT, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Async Engines", TmpInt);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_UNIFIED_ADDRESSING, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Unified Addressing", bool(TmpInt));

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MANAGED_MEMORY, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Managed Memory", bool(TmpInt));

    Res =
        getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_CONCURRENT_MANAGED_ACCESS, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Concurrent Managed Memory", bool(TmpInt));

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_COMPUTE_PREEMPTION_SUPPORTED,
                           TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Preemption Supported", bool(TmpInt));

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_COOPERATIVE_LAUNCH, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Cooperative Launch", bool(TmpInt), "",
               DeviceInfo::COOPERATIVE_LAUNCH_SUPPORT);

    Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MULTI_GPU_BOARD, TmpInt);
    if (Res == CUDA_SUCCESS)
      Info.add("Multi-Device Boars", bool(TmpInt));

    Info.add("Compute Capabilities", ComputeCapability.str());

    ol_device_fp_capability_flags_t FPFlags =
        OL_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT |
        OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST |
        OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO |
        OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF |
        OL_DEVICE_FP_CAPABILITY_FLAG_INF_NAN |
        OL_DEVICE_FP_CAPABILITY_FLAG_DENORM | OL_DEVICE_FP_CAPABILITY_FLAG_FMA;

    Info.add("Single FP Support", true, "", DeviceInfo::SINGLE_FP_SUPPORT);
    Info.add("Single FP Capabilities", FPFlags, "",
             DeviceInfo::SINGLE_FP_CONFIG);

    Info.add("Double FP Support", true, "", DeviceInfo::DOUBLE_FP_SUPPORT);
    Info.add("Double FP Capabilities", FPFlags, "",
             DeviceInfo::DOUBLE_FP_CONFIG);

    Info.add("Half FP Support", false, "", DeviceInfo::HALF_FP_SUPPORT);
    Info.add("Half FP Capabilities", ol_device_fp_capability_flags_t{0}, "",
             DeviceInfo::HALF_FP_CONFIG);

    return Info;
  }

  /// Getters and setters for stack and heap sizes.
  Error getDeviceStackSize(uint64_t &Value) override {
    return getCtxLimit(CU_LIMIT_STACK_SIZE, Value);
  }
  Error setDeviceStackSize(uint64_t Value) override {
    return setCtxLimit(CU_LIMIT_STACK_SIZE, Value);
  }
  bool hasDeviceHeapSize() override { return true; }
  Error getDeviceHeapSize(uint64_t &Value) override {
    return getCtxLimit(CU_LIMIT_MALLOC_HEAP_SIZE, Value);
  }
  Error setDeviceHeapSize(uint64_t Value) override {
    return setCtxLimit(CU_LIMIT_MALLOC_HEAP_SIZE, Value);
  }
  Error getDeviceMemorySize(uint64_t &Value) override {
    CUresult Res = cuDeviceTotalMem(&Value, Device);
    return Plugin::check(Res, "error in getDeviceMemorySize %s");
  }

  /// CUDA-specific functions for getting and setting context limits.
  Error setCtxLimit(CUlimit Kind, uint64_t Value) {
    CUresult Res = cuCtxSetLimit(Kind, Value);
    return Plugin::check(Res, "error in cuCtxSetLimit: %s");
  }
  Error getCtxLimit(CUlimit Kind, uint64_t &Value) {
    CUresult Res = cuCtxGetLimit(&Value, Kind);
    return Plugin::check(Res, "error in cuCtxGetLimit: %s");
  }

  /// CUDA-specific function to get device attributes.
  Error getDeviceAttr(uint32_t Kind, uint32_t &Value) {
    // TODO: Warn if the new value is larger than the old.
    CUresult Res =
        cuDeviceGetAttribute((int *)&Value, (CUdevice_attribute)Kind, Device);
    return Plugin::check(Res, "error in cuDeviceGetAttribute: %s");
  }

  CUresult getDeviceAttrRaw(uint32_t Kind, int &Value) {
    return cuDeviceGetAttribute(&Value, (CUdevice_attribute)Kind, Device);
  }

  /// See GenericDeviceTy::getComputeUnitKind().
  std::string getComputeUnitKind() const override {
    return ComputeCapability.str();
  }

  /// Returns the clock frequency for the given NVPTX device.
  uint64_t getClockFrequency() const override { return 1000000000; }

private:
  using CUDAStreamManagerTy = GenericDeviceResourceManagerTy<CUDAStreamRef>;
  using CUDAEventManagerTy = GenericDeviceResourceManagerTy<CUDAEventRef>;

  Error callGlobalCtorDtorCommon(GenericPluginTy &Plugin, DeviceImageTy &Image,
                                 bool IsCtor) {
    const char *KernelName = IsCtor ? "nvptx$device$init" : "nvptx$device$fini";
    // Perform a quick check for the named kernel in the image. The kernel
    // should be created by the 'nvptx-lower-ctor-dtor' pass.
    GenericGlobalHandlerTy &Handler = Plugin.getGlobalHandler();
    if (!Handler.isSymbolInImage(*this, Image, KernelName))
      return Plugin::success();

    // The Nvidia backend cannot handle creating the ctor / dtor array
    // automatically so we must create it ourselves. The backend will emit
    // several globals that contain function pointers we can call. These are
    // prefixed with a known name due to Nvidia's lack of section support.
    auto ELFObjOrErr = Handler.getELFObjectFile(Image);
    if (!ELFObjOrErr)
      return ELFObjOrErr.takeError();

    // Search for all symbols that contain a constructor or destructor.
    SmallVector<std::pair<StringRef, uint16_t>> Funcs;
    for (ELFSymbolRef Sym : (*ELFObjOrErr)->symbols()) {
      auto NameOrErr = Sym.getName();
      if (!NameOrErr)
        return NameOrErr.takeError();

      if (!NameOrErr->starts_with(IsCtor ? "__init_array_object_"
                                         : "__fini_array_object_"))
        continue;

      uint16_t Priority;
      if (NameOrErr->rsplit('_').second.getAsInteger(10, Priority))
        return Plugin::error(ErrorCode::INVALID_BINARY,
                             "invalid priority for constructor or destructor");

      Funcs.emplace_back(*NameOrErr, Priority);
    }

    // Sort the created array to be in priority order.
    llvm::sort(Funcs, [=](auto X, auto Y) { return X.second < Y.second; });

    // Allocate a buffer to store all of the known constructor / destructor
    // functions in so we can iterate them on the device.
    auto BufferOrErr =
        allocate(Funcs.size() * sizeof(void *), nullptr, TARGET_ALLOC_DEVICE);
    if (!BufferOrErr)
      return BufferOrErr.takeError();

    void *Buffer = *BufferOrErr;
    if (!Buffer)
      return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
                           "failed to allocate memory for global buffer");

    auto *GlobalPtrStart = reinterpret_cast<uintptr_t *>(Buffer);
    auto *GlobalPtrStop = reinterpret_cast<uintptr_t *>(Buffer) + Funcs.size();

    SmallVector<void *> FunctionPtrs(Funcs.size());
    std::size_t Idx = 0;
    for (auto [Name, Priority] : Funcs) {
      GlobalTy FunctionAddr(Name.str(), sizeof(void *), &FunctionPtrs[Idx++]);
      if (auto Err = Handler.readGlobalFromDevice(*this, Image, FunctionAddr))
        return Err;
    }

    // Copy the local buffer to the device.
    if (auto Err = dataSubmit(GlobalPtrStart, FunctionPtrs.data(),
                              FunctionPtrs.size() * sizeof(void *), nullptr))
      return Err;

    // Copy the created buffer to the appropriate symbols so the kernel can
    // iterate through them.
    GlobalTy StartGlobal(IsCtor ? "__init_array_start" : "__fini_array_start",
                         sizeof(void *), &GlobalPtrStart);
    if (auto Err = Handler.writeGlobalToDevice(*this, Image, StartGlobal))
      return Err;

    GlobalTy StopGlobal(IsCtor ? "__init_array_end" : "__fini_array_end",
                        sizeof(void *), &GlobalPtrStop);
    if (auto Err = Handler.writeGlobalToDevice(*this, Image, StopGlobal))
      return Err;

    CUDAKernelTy CUDAKernel(KernelName);

    if (auto Err = CUDAKernel.init(*this, Image))
      return Err;

    AsyncInfoWrapperTy AsyncInfoWrapper(*this, nullptr);

    KernelLaunchArgsTy LaunchArgs = {};
    uint32_t NumBlocksAndThreads[3] = {1u, 1u, 1u};
    auto Err =
        CUDAKernel.launchImpl(*this, NumBlocksAndThreads, NumBlocksAndThreads,
                              0, LaunchArgs, AsyncInfoWrapper);

    AsyncInfoWrapper.finalize(Err);
    if (Err)
      return Err;

    return free(Buffer, TARGET_ALLOC_DEVICE);
  }

  /// Stream manager for CUDA streams.
  CUDAStreamManagerTy CUDAStreamManager;

  /// Event manager for CUDA events.
  CUDAEventManagerTy CUDAEventManager;

  /// The device's context. This context should be set before performing
  /// operations on the device.
  CUcontext Context = nullptr;

  /// The CUDA device handler.
  CUdevice Device = CU_DEVICE_INVALID;

  /// The compute capability of the corresponding CUDA device.
  struct ComputeCapabilityTy {
    uint32_t Major;
    uint32_t Minor;
    std::string str() const {
      return "sm_" + std::to_string(Major * 10 + Minor);
    }
  } ComputeCapability;

  /// The maximum number of warps that can be resident on all the SMs
  /// simultaneously.
  uint32_t HardwareParallelism = 0;

  /// Device page size.
  size_t Granularity = 0;

  /// Tracker for virtual address reservations.
  VMemTrackerTy<CUmemGenericAllocationHandle> VMemTracker;
};

Error CUDAKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
                               uint32_t NumThreads[3], uint32_t NumBlocks[3],
                               uint32_t DynBlockMemSize,
                               KernelLaunchArgsTy &LaunchArgs,
                               AsyncInfoWrapperTy &AsyncInfoWrapper) const {
  CUDADeviceTy &CUDADevice = static_cast<CUDADeviceTy &>(GenericDevice);

  CUstream Stream;
  if (auto Err = CUDADevice.getStream(AsyncInfoWrapper, Stream))
    return Err;

  // If we are running an RPC server we want to wake up the server thread
  // whenever there is a kernel running and let it sleep otherwise.
  if (GenericDevice.getRPCServer())
    GenericDevice.Plugin.getRPCServer().Thread->notify();

  // In case we require more memory than the current limit.
  if (DynBlockMemSize >= MaxDynBlockMemSize) {
    CUresult AttrResult = cuFuncSetAttribute(
        Func, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, DynBlockMemSize);
    if (auto Err = Plugin::check(
            AttrResult,
            "error in cuFuncSetAttribute while setting the memory limits: %s"))
      return Err;
    MaxDynBlockMemSize = DynBlockMemSize;
  }

  CUlaunchAttribute CoopAttr;
  CoopAttr.id = CU_LAUNCH_ATTRIBUTE_COOPERATIVE;
  CoopAttr.value.cooperative = LaunchArgs.Flags.Cooperative;

  CUlaunchConfig LaunchConfig = {NumBlocks[0],    NumBlocks[1],
                                 NumBlocks[2],    NumThreads[0],
                                 NumThreads[1],   NumThreads[2],
                                 DynBlockMemSize, Stream,
                                 &CoopAttr,       1};

  CUresult Res = cuLaunchKernelEx(&LaunchConfig, Func, LaunchArgs.Args,
                                  /*extra=*/nullptr);

  // Register a callback to indicate when the kernel is complete.
  if (GenericDevice.getRPCServer())
    cuLaunchHostFunc(
        Stream,
        [](void *Data) {
          GenericPluginTy &Plugin = *reinterpret_cast<GenericPluginTy *>(Data);
          Plugin.getRPCServer().Thread->finish();
        },
        &GenericDevice.Plugin);

  return Plugin::check(Res, "error in cuLaunchKernelEx for '%s': %s",
                       getName());
}

Expected<uint32_t>
CUDAKernelTy::getMaxCooperativeGroupCount(GenericDeviceTy &GenericDevice,
                                          const uint32_t NumThreads[3],
                                          uint32_t DynBlockMemSize) const {
  CUDADeviceTy &CUDADevice = static_cast<CUDADeviceTy &>(GenericDevice);

  uint32_t SupportsCooperative = 0;
  if (auto Err = CUDADevice.getDeviceAttr(
          CU_DEVICE_ATTRIBUTE_COOPERATIVE_LAUNCH, SupportsCooperative))
    return Err;

  if (!SupportsCooperative)
    return Plugin::error(ErrorCode::UNSUPPORTED,
                         "device does not support cooperative launch");

  // Calculate total local work size
  size_t LocalWorkSizeTotal = NumThreads[0] * NumThreads[1] * NumThreads[2];

  // Query max active blocks per multiprocessor
  int32_t MaxNumActiveGroupsPerCU = 0;
  CUresult Res = cuOccupancyMaxActiveBlocksPerMultiprocessor(
      &MaxNumActiveGroupsPerCU, Func, LocalWorkSizeTotal, DynBlockMemSize);
  if (auto Err = Plugin::check(
          Res, "error in cuOccupancyMaxActiveBlocksPerMultiprocessor: %s"))
    return Err;

  assert(MaxNumActiveGroupsPerCU >= 0);

  // Handle the case where we can't have all SMs active with at least 1 group
  // per SM. In that case, the device is still able to run 1 work-group, hence
  // we will manually check if it is possible with the available HW resources.
  if (MaxNumActiveGroupsPerCU == 0)
    // Check if we can launch at least 1 work-group
    return (LocalWorkSizeTotal <= MaxNumThreads &&
            DynBlockMemSize <= CUDADevice.getMaxBlockSharedMemSize());

  // Multiply by the number of multiprocessors (compute units) on the device
  uint32_t NumMultiprocessors = 0;
  if (auto Err = CUDADevice.getDeviceAttr(
          CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, NumMultiprocessors))
    return Err;

  return NumMultiprocessors * MaxNumActiveGroupsPerCU;
}

/// Class implementing the CUDA-specific functionalities of the global handler.
class CUDAGlobalHandlerTy final : public GenericGlobalHandlerTy {
public:
  /// Get the metadata of a global from the device. The name and size of the
  /// global is read from DeviceGlobal and the address of the global is written
  /// to DeviceGlobal.
  Error getGlobalMetadataFromDevice(GenericDeviceTy &Device,
                                    DeviceImageTy &Image,
                                    GlobalTy &DeviceGlobal) override {
    CUDADeviceImageTy &CUDAImage = static_cast<CUDADeviceImageTy &>(Image);

    const char *GlobalName = DeviceGlobal.getName().data();

    size_t CUSize;
    CUdeviceptr CUPtr;
    CUresult Res =
        cuModuleGetGlobal(&CUPtr, &CUSize, CUDAImage.getModule(), GlobalName);
    if (auto Err = Plugin::check(Res, "error in cuModuleGetGlobal for '%s': %s",
                                 GlobalName))
      return Err;

    if (DeviceGlobal.getSize() && CUSize != DeviceGlobal.getSize())
      return Plugin::error(
          ErrorCode::INVALID_BINARY,
          "failed to load global '%s' due to size mismatch (%zu != %zu)",
          GlobalName, CUSize, (size_t)DeviceGlobal.getSize());

    DeviceGlobal.setPtr(reinterpret_cast<void *>(CUPtr));
    DeviceGlobal.setSize(CUSize);

    return Plugin::success();
  }
};

struct CUDAPluginContextTy final : public PluginContextTy {
  using PluginContextTy::PluginContextTy;

  Error initAsyncInfoImpl(GenericDeviceTy &Device,
                          AsyncInfoWrapperTy &AsyncInfoWrapper) override {
    auto &CUDADevice = static_cast<CUDADeviceTy &>(Device);
    if (auto Err = CUDADevice.setContext())
      return Err;

    CUstream Stream;
    return CUDADevice.getStream(AsyncInfoWrapper, Stream);
  }

  Expected<PluginAllocInfoTy> getAllocInfo(const void *Ptr) override {
    if (Devices.empty())
      return Plugin::error(error::ErrorCode::NOT_FOUND,
                           "pointer is not a known allocation in this context");

    // Any device in the context can service the query; use the first.
    auto &Ctx0 = static_cast<CUDADeviceTy &>(*Devices.front());
    if (auto Err = Ctx0.setContext())
      return std::move(Err);

    CUdeviceptr CUPtr = reinterpret_cast<CUdeviceptr>(Ptr);

    CUpointer_attribute Attrs[] = {
        CU_POINTER_ATTRIBUTE_MEMORY_TYPE,
        CU_POINTER_ATTRIBUTE_IS_MANAGED,
        CU_POINTER_ATTRIBUTE_DEVICE_ORDINAL,
        CU_POINTER_ATTRIBUTE_RANGE_START_ADDR,
        CU_POINTER_ATTRIBUTE_RANGE_SIZE,
    };
    unsigned MemType = 0;
    int IsManaged = 0;
    int Ordinal = -1;
    CUdeviceptr RangeStart = 0;
    size_t RangeSize = 0;
    void *Data[] = {&MemType, &IsManaged, &Ordinal, &RangeStart, &RangeSize};
    if (CUresult Res = cuPointerGetAttributes(sizeof(Attrs) / sizeof(Attrs[0]),
                                              Attrs, Data, CUPtr))
      return Plugin::error(error::ErrorCode::NOT_FOUND,
                           "cuPointerGetAttributes failed: %d", Res);

    TargetAllocTy Kind = TARGET_ALLOC_DEVICE;
    if (IsManaged)
      Kind = TARGET_ALLOC_SHARED;
    else if (MemType == CU_MEMORYTYPE_HOST)
      Kind = TARGET_ALLOC_HOST;

    // Ordinal is the CUDA driver ordinal (matches CUdevice); compare against
    // that rather than the offload-side device index, which can differ under
    // CUDA_VISIBLE_DEVICES or plugin-side device filtering.
    GenericDeviceTy *OwnerDevice = nullptr;
    for (auto *D : Devices) {
      auto &CD = static_cast<CUDADeviceTy &>(*D);
      if (static_cast<int>(CD.getCUDADevice()) == Ordinal) {
        OwnerDevice = D;
        break;
      }
    }
    if (!OwnerDevice)
      return Plugin::error(error::ErrorCode::NOT_FOUND,
                           "allocation owner is not a device of this context");

    return PluginAllocInfoTy{OwnerDevice, Kind,
                             reinterpret_cast<void *>(RangeStart), RangeSize};
  }
};

/// Class implementing the CUDA-specific functionalities of the plugin.
struct CUDAPluginTy final : public GenericPluginTy {
  /// Create a CUDA plugin.
  CUDAPluginTy() : GenericPluginTy(getTripleArch()) {}

  /// This class should not be copied.
  CUDAPluginTy(const CUDAPluginTy &) = delete;
  CUDAPluginTy(CUDAPluginTy &&) = delete;

  /// Initialize the plugin and return the number of devices.
  Expected<int32_t> initImpl() override {
    CUresult Res = cuInit(0);
    if (Res == CUDA_ERROR_INVALID_HANDLE) {
      // Cannot call cuGetErrorString if dlsym failed.
      ODBG(OLDT_Init) << "Failed to load CUDA shared library";
      return 0;
    }

    if (Res == CUDA_ERROR_NO_DEVICE) {
      // Do not initialize if there are no devices.
      ODBG(OLDT_Init) << "There are no devices supporting CUDA.";
      return 0;
    }

    if (auto Err = Plugin::check(Res, "error in cuInit: %s"))
      return std::move(Err);

    // Get the latest CUDA version supported by the driver.
    int Version;
    Res = cuDriverGetVersion(&Version);
    if (auto Err = Plugin::check(Res, "error in cuDriverGetVersion: %s"))
      return std::move(Err);

    // Verify that the driver supports the minimum CUDA version required.
    constexpr int MinVersion = OFFLOAD_MIN_CUDA_VERSION;
    if (Version < MinVersion) {
      ODBG(OLDT_Init) << "Minimum CUDA version not supported by the driver.";
      return Plugin::error(
          ErrorCode::UNSUPPORTED,
          "CUDA driver does not support minimum CUDA version %d (%d detected)",
          MinVersion, Version);
    }

    // Get the number of devices.
    int NumDevices;
    Res = cuDeviceGetCount(&NumDevices);
    if (auto Err = Plugin::check(Res, "error in cuDeviceGetCount: %s"))
      return std::move(Err);

    // Do not initialize if there are no devices.
    if (NumDevices == 0)
      ODBG(OLDT_Init) << "There are no devices supporting CUDA.";

    return NumDevices;
  }

  /// Deinitialize the plugin.
  Error deinitImpl() override { return Plugin::success(); }

  /// Creates a CUDA device to use for offloading.
  GenericDeviceTy *createDevice(GenericPluginTy &Plugin, int32_t DeviceId,
                                int32_t NumDevices) override {
    return new CUDADeviceTy(Plugin, DeviceId, NumDevices);
  }

  Expected<std::unique_ptr<PluginContextTy>>
  createPluginContext(llvm::ArrayRef<GenericDeviceTy *> Devices) override {
    return std::make_unique<CUDAPluginContextTy>(*this, Devices);
  }

  /// Creates a CUDA global handler.
  GenericGlobalHandlerTy *createGlobalHandler() override {
    return new CUDAGlobalHandlerTy();
  }

  /// Get the ELF code for recognizing the compatible image binary.
  uint16_t getMagicElfBits() const override { return ELF::EM_CUDA; }

  Triple::ArchType getTripleArch() const override {
    // TODO: I think we can drop the support for 32-bit NVPTX devices.
    return Triple::nvptx64;
  }

  const char *getName() const override { return GETNAME(TARGET_NAME); }

  /// Check whether the image is compatible with a CUDA device.
  Expected<bool> isELFCompatible(uint32_t DeviceId,
                                 StringRef Image) const override {
    auto ElfOrErr =
        ELF64LEObjectFile::create(MemoryBufferRef(Image, /*Identifier=*/""),
                                  /*InitContent=*/false);
    if (!ElfOrErr)
      return ElfOrErr.takeError();

    // Get the numeric value for the image's `sm_` value.
    const auto Header = ElfOrErr->getELFFile().getHeader();
    unsigned SM =
        Header.e_ident[ELF::EI_ABIVERSION] == ELF::ELFABIVERSION_CUDA_V1
            ? Header.e_flags & ELF::EF_CUDA_SM
            : (Header.e_flags & ELF::EF_CUDA_SM_MASK) >> ELF::EF_CUDA_SM_OFFSET;

    CUdevice Device;
    CUresult Res = cuDeviceGet(&Device, DeviceId);
    if (auto Err = Plugin::check(Res, "error in cuDeviceGet: %s"))
      return std::move(Err);

    int32_t Major, Minor;
    Res = cuDeviceGetAttribute(
        &Major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, Device);
    if (auto Err = Plugin::check(Res, "error in cuDeviceGetAttribute: %s"))
      return std::move(Err);

    Res = cuDeviceGetAttribute(
        &Minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, Device);
    if (auto Err = Plugin::check(Res, "error in cuDeviceGetAttribute: %s"))
      return std::move(Err);

    int32_t ImageMajor = SM / 10;
    int32_t ImageMinor = SM % 10;

    // A cubin generated for a certain compute capability is supported to
    // run on any GPU with the same major revision and same or higher minor
    // revision.
    return Major == ImageMajor && Minor >= ImageMinor;
  }
};

Error CUDADeviceTy::dataExchangeImpl(const void *SrcPtr,
                                     GenericDeviceTy &DstGenericDevice,
                                     void *DstPtr, int64_t Size,
                                     AsyncInfoWrapperTy &AsyncInfoWrapper) {
  if (auto Err = setContext())
    return Err;

  CUDADeviceTy &DstDevice = static_cast<CUDADeviceTy &>(DstGenericDevice);

  CUresult Res;
  int32_t DstDeviceId = DstDevice.DeviceId;
  CUdeviceptr CUSrcPtr = (CUdeviceptr)SrcPtr;
  CUdeviceptr CUDstPtr = (CUdeviceptr)DstPtr;

  int CanAccessPeer = 0;
  if (DeviceId != DstDeviceId) {
    // Make sure the lock is released before performing the copies.
    std::lock_guard<std::mutex> Lock(PeerAccessesLock);

    switch (PeerAccesses[DstDeviceId]) {
    case PeerAccessState::AVAILABLE:
      CanAccessPeer = 1;
      break;
    case PeerAccessState::UNAVAILABLE:
      CanAccessPeer = 0;
      break;
    case PeerAccessState::PENDING:
      // Check whether the source device can access the destination device.
      Res = cuDeviceCanAccessPeer(&CanAccessPeer, Device, DstDevice.Device);
      if (auto Err = Plugin::check(Res, "Error in cuDeviceCanAccessPeer: %s"))
        return Err;

      if (CanAccessPeer) {
        Res = cuCtxEnablePeerAccess(DstDevice.Context, 0);
        if (Res == CUDA_ERROR_TOO_MANY_PEERS) {
          // Resources may be exhausted due to many P2P links.
          CanAccessPeer = 0;
          ODBG(OLDT_DataTransfer) << "Too many P2P so fall back to D2D memcpy";
        } else if (auto Err =
                       Plugin::check(Res, "error in cuCtxEnablePeerAccess: %s"))
          return Err;
      }
      PeerAccesses[DstDeviceId] = (CanAccessPeer)
                                      ? PeerAccessState::AVAILABLE
                                      : PeerAccessState::UNAVAILABLE;
    }
  }

  CUstream Stream;
  if (auto Err = getStream(AsyncInfoWrapper, Stream))
    return Err;

  if (CanAccessPeer) {
    // TODO: Should we fallback to D2D if peer access fails?
    Res = cuMemcpyPeerAsync(CUDstPtr, Context, CUSrcPtr, DstDevice.Context,
                            Size, Stream);
    return Plugin::check(Res, "error in cuMemcpyPeerAsync: %s");
  }

  // Fallback to D2D copy.
  Res = cuMemcpyDtoDAsync(CUDstPtr, CUSrcPtr, Size, Stream);
  return Plugin::check(Res, "error in cuMemcpyDtoDAsync: %s");
}

/// Map a CUDA driver result code to the corresponding offload error code.
static ErrorCode getOffloadErrorCode(CUresult ResultCode) {
  switch (ResultCode) {
  case CUDA_ERROR_INVALID_VALUE:
    return ErrorCode::INVALID_VALUE;
  case CUDA_ERROR_OUT_OF_MEMORY:
  case CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES:
    return ErrorCode::OUT_OF_RESOURCES;
  case CUDA_ERROR_NOT_INITIALIZED:
  case CUDA_ERROR_DEINITIALIZED:
    return ErrorCode::UNINITIALIZED;
  case CUDA_ERROR_NO_DEVICE:
  case CUDA_ERROR_INVALID_DEVICE:
    return ErrorCode::INVALID_DEVICE;
  case CUDA_ERROR_INVALID_IMAGE:
  case CUDA_ERROR_INVALID_SOURCE:
  case CUDA_ERROR_INVALID_PTX:
  case CUDA_ERROR_UNSUPPORTED_PTX_VERSION:
    return ErrorCode::INVALID_BINARY;
  case CUDA_ERROR_FILE_NOT_FOUND:
  case CUDA_ERROR_OPERATING_SYSTEM:
    return ErrorCode::HOST_IO;
  case CUDA_ERROR_JIT_COMPILER_NOT_FOUND:
    return ErrorCode::HOST_TOOL_NOT_FOUND;
  case CUDA_ERROR_NOT_FOUND:
  case CUDA_ERROR_SHARED_OBJECT_SYMBOL_NOT_FOUND:
    return ErrorCode::NOT_FOUND;
  case CUDA_ERROR_NOT_SUPPORTED:
    return ErrorCode::UNSUPPORTED;
  case CUDA_ERROR_INVALID_HANDLE:
  case CUDA_ERROR_INVALID_CONTEXT:
    return ErrorCode::INVALID_ARGUMENT;
  default:
    return ErrorCode::UNKNOWN;
  }
}

template <typename... ArgsTy>
static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
  CUresult ResultCode = static_cast<CUresult>(Code);
  if (ResultCode == CUDA_SUCCESS)
    return Plugin::success();

  const char *Desc = "Unknown error";
  CUresult Ret = cuGetErrorString(ResultCode, &Desc);
  if (Ret != CUDA_SUCCESS)
    REPORT() << "Unrecognized " GETNAME(TARGET_NAME) " error code " << Code;

  return Plugin::error(getOffloadErrorCode(ResultCode), ErrFmt, Args..., Desc);
}

} // namespace plugin
} // namespace target
} // namespace omp
} // namespace llvm

extern "C" {
llvm::omp::target::plugin::GenericPluginTy *createPlugin_cuda() {
  return new llvm::omp::target::plugin::CUDAPluginTy();
}
}