| //===--------- device.cpp - Target independent OpenMP target RTL ----------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // Functionality for managing devices that are handled by RTL plugins. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "device.h" |
| #include "OffloadEntry.h" |
| #include "OmpAccError.h" |
| #include "OpenMP/Mapping.h" |
| #include "OpenMP/OMPT/Callback.h" |
| #include "OpenMP/OMPT/Interface.h" |
| #include "PluginManager.h" |
| #include "Shared/APITypes.h" |
| #include "Shared/Debug.h" |
| #include "omptarget.h" |
| #include "rtl.h" |
| |
| #include "Shared/EnvironmentVar.h" |
| #include "llvm/Frontend/OpenMP/OMPConstants.h" |
| #include "llvm/Support/Error.h" |
| |
| #include <algorithm> |
| #include <cassert> |
| #include <climits> |
| #include <cstdint> |
| #include <cstdio> |
| #include <mutex> |
| #include <string> |
| #include <thread> |
| |
| #ifdef OMPT_SUPPORT |
| using namespace llvm::omp::target::ompt; |
| #endif |
| |
| using namespace llvm::omp::target; |
| using namespace llvm::omp::target::plugin; |
| using namespace llvm::omp::target::debug; |
| |
| int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device, |
| AsyncInfoTy &AsyncInfo) const { |
| // First, check if the user disabled atomic map transfer/malloc/dealloc. |
| if (!MappingConfig::get().UseEventsForAtomicTransfers) |
| return OFFLOAD_SUCCESS; |
| |
| // We cannot assume the event should not be nullptr because we don't |
| // know if the target support event. But if a target doesn't, |
| // recordEvent should always return success. |
| void *Event = getEvent(); |
| if (Device.recordEvent(&Event, AsyncInfo) != OFFLOAD_SUCCESS) { |
| REPORT() << "Failed to set dependence on event " << Event; |
| return OFFLOAD_FAIL; |
| } |
| |
| setEvent(Event); |
| |
| return OFFLOAD_SUCCESS; |
| } |
| |
| DeviceTy::DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID, |
| ol_device_handle_t DeviceHandle) |
| : DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID), |
| DeviceHandle(DeviceHandle), MappingInfo(*this) {} |
| |
| DeviceTy::~DeviceTy() { |
| if (DeviceID == -1 || !(getInfoLevel() & OMP_INFOTYPE_DUMP_TABLE)) |
| return; |
| |
| ident_t Loc = {0, 0, 0, 0, ";libomptarget;libomptarget;0;0;;"}; |
| dumpTargetPointerMappings(&Loc, *this); |
| } |
| |
| llvm::Error DeviceTy::init() { |
| if (olCreateContext(1, &DeviceHandle, &Context)) { |
| return createError(ErrorCode::BackendFailure, |
| "failed to create context for device %d\n", DeviceID); |
| } |
| |
| // Envar that indicates whether mapped host buffers should be locked |
| // automatically. The possible values are boolean (on/off) and a special: |
| // off: Mapped host buffers are not locked. |
| // on: Mapped host buffers are locked in a best-effort approach. |
| // Failure to lock the buffers are silent. |
| // mandatory: Mapped host buffers are always locked and failures to lock |
| // a buffer results in a fatal error. |
| StringEnvar OMPX_LockMappedBuffers("LIBOMPTARGET_LOCK_MAPPED_HOST_BUFFERS", |
| "off"); |
| bool Enabled; |
| if (StringParser::parse(OMPX_LockMappedBuffers.get().data(), Enabled)) { |
| // Parsed as a boolean value. Enable the feature if necessary. |
| LockMappedBuffers = Enabled; |
| IgnoreLockMappedFailures = true; |
| } else if (OMPX_LockMappedBuffers.get() == "mandatory") { |
| // Enable the feature and failures are fatal. |
| LockMappedBuffers = true; |
| IgnoreLockMappedFailures = false; |
| } else { |
| // Disable by default. |
| ODBG(ODT_Alloc) << "Invalid value LIBOMPTARGET_LOCK_MAPPED_HOST_BUFFERS=" |
| << OMPX_LockMappedBuffers.get(); |
| LockMappedBuffers = false; |
| IgnoreLockMappedFailures = true; |
| } |
| |
| // Enables recording kernels if set. |
| BoolEnvar OMPX_RecordKernel("LIBOMPTARGET_RECORD", false); |
| if (OMPX_RecordKernel) { |
| BoolEnvar OMPX_RecordOutput("LIBOMPTARGET_RECORD_OUTPUT", true); |
| Int64Envar OMPX_RecordMemSize("LIBOMPTARGET_RECORD_MEMSIZE", |
| 8 * 1024 * 1024 * 1024ULL); |
| Int32Envar OMPX_RecordDevice("LIBOMPTARGET_RECORD_DEVICE", 0); |
| StringEnvar OMPX_RecordOutputDir("LIBOMPTARGET_RECORD_DIR", ""); |
| BoolEnvar OMPX_EmitRecordReport("LIBOMPTARGET_RECORD_REPORT", false); |
| StringEnvar OMPX_RecordReportFilename("LIBOMPTARGET_RECORD_REPORT_FILENAME", |
| ""); |
| if (OMPX_RecordDevice != RTLDeviceID) |
| return llvm::Error::success(); |
| |
| // Print report if it was enabled explicitly or a report file was indicated. |
| bool EmitReport = |
| OMPX_EmitRecordReport || !OMPX_RecordReportFilename.get().empty(); |
| |
| int32_t Ret = RTL->initialize_record_replay( |
| RTLDeviceID, OMPX_RecordMemSize, nullptr, |
| /*IsRecord=*/true, /*IsNative=*/true, OMPX_RecordOutput, EmitReport, |
| OMPX_RecordReportFilename.get().c_str(), |
| OMPX_RecordOutputDir.get().c_str()); |
| if (Ret != OFFLOAD_SUCCESS) |
| return createError(ErrorCode::BackendFailure, |
| "failed to initialize RR in device %d\n", DeviceID); |
| } |
| |
| return llvm::Error::success(); |
| } |
| |
| llvm::Error DeviceTy::deinit() { |
| if (olDestroyContext(Context)) { |
| return createError(ErrorCode::BackendFailure, |
| "failed to destroy context for device %d\n", DeviceID); |
| } |
| return llvm::Error::success(); |
| } |
| |
| // Resolve the device address of the global variable \p Name in \p Program, |
| // recording it for kernel record/replay if recording is currently active. |
| llvm::Expected<void *> getAndRecordGlobalAddress(DeviceTy &Device, |
| const ProgramTy &Program, |
| const char *Name) { |
| size_t Size = 0; |
| auto AddrOrErr = Program.getGlobalAddress(Name, &Size); |
| if (!AddrOrErr) |
| return AddrOrErr; |
| |
| GenericDeviceTy &GenericDevice = Device.RTL->getDevice(Device.RTLDeviceID); |
| RecordReplayTy *RecordReplay = GenericDevice.getRecordReplay(); |
| if (RecordReplay && RecordReplay->isRecording()) |
| RecordReplay->addGlobal(Name, Size, *AddrOrErr); |
| |
| return AddrOrErr; |
| } |
| |
| // Extract the mapping of host function pointers to device function pointers |
| // from the entry table. Functions marked as 'indirect' in OpenMP will have |
| // offloading entries generated for them which map the host's function pointer |
| // to a global containing the corresponding function pointer on the device. |
| static llvm::Expected<std::pair<void *, uint64_t>> |
| setupIndirectCallTable(DeviceTy &Device, __tgt_device_image *Image, |
| const ProgramTy &Program) { |
| AsyncInfoTy AsyncInfo(Device); |
| llvm::ArrayRef<llvm::offloading::EntryTy> Entries(Image->EntriesBegin, |
| Image->EntriesEnd); |
| llvm::SmallVector<std::pair<void *, void *>> IndirectCallTable; |
| for (const auto &Entry : Entries) { |
| if (Entry.Kind != llvm::object::OffloadKind::OFK_OpenMP || |
| Entry.Size == 0 || |
| (!(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT) && |
| !(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT_VTABLE))) |
| continue; |
| |
| size_t PtrSize = sizeof(void *); |
| if (Entry.Flags & OMP_DECLARE_TARGET_INDIRECT_VTABLE) { |
| // This is a VTable entry, the current entry is the first index of the |
| // VTable and Entry.Size is the total size of the VTable. Unlike the |
| // indirect function case below, the Global is not of size Entry.Size and |
| // is instead of size PtrSize (sizeof(void*)). |
| void *res; |
| auto VtableOrErr = |
| getAndRecordGlobalAddress(Device, Program, Entry.SymbolName); |
| if (!VtableOrErr) |
| return VtableOrErr.takeError(); |
| void *Vtable = *VtableOrErr; |
| |
| // HstPtr = Entry.Address; |
| if (Device.retrieveData(&res, Vtable, PtrSize, AsyncInfo)) |
| return createError(ErrorCode::InvalidBinary, "failed to load %s", |
| Entry.SymbolName); |
| if (Device.synchronize(AsyncInfo)) |
| return createError(ErrorCode::InvalidBinary, |
| "failed to synchronize after retrieving %s", |
| Entry.SymbolName); |
| // Calculate and emplace entire Vtable from first Vtable byte |
| for (uint64_t i = 0; i < Entry.Size / PtrSize; ++i) { |
| auto &[HstPtr, DevPtr] = IndirectCallTable.emplace_back(); |
| HstPtr = reinterpret_cast<void *>( |
| reinterpret_cast<uintptr_t>(Entry.Address) + i * PtrSize); |
| DevPtr = reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(res) + |
| i * PtrSize); |
| } |
| } else { |
| // Indirect function case: Entry.Size should equal PtrSize since we're |
| // dealing with a single function pointer (not a VTable) |
| assert(Entry.Size == PtrSize && "Global not a function pointer?"); |
| auto &[HstPtr, DevPtr] = IndirectCallTable.emplace_back(); |
| auto PtrOrErr = |
| getAndRecordGlobalAddress(Device, Program, Entry.SymbolName); |
| if (!PtrOrErr) |
| return PtrOrErr.takeError(); |
| void *Ptr = *PtrOrErr; |
| |
| HstPtr = Entry.Address; |
| if (Device.retrieveData(&DevPtr, Ptr, Entry.Size, AsyncInfo)) |
| return createError(ErrorCode::InvalidBinary, "failed to load %s", |
| Entry.SymbolName); |
| } |
| if (Device.synchronize(AsyncInfo)) |
| return createError(ErrorCode::InvalidBinary, |
| "failed to synchronize after retrieving %s", |
| Entry.SymbolName); |
| } |
| |
| // If we do not have any indirect globals we exit early. |
| if (IndirectCallTable.empty()) |
| return std::pair{nullptr, 0}; |
| |
| // Sort the array to allow for more efficient lookup of device pointers. |
| llvm::sort(IndirectCallTable, |
| [](const auto &x, const auto &y) { return x.first < y.first; }); |
| |
| uint64_t TableSize = |
| IndirectCallTable.size() * sizeof(std::pair<void *, void *>); |
| void *DevicePtr = Device.allocData(TableSize, nullptr, TARGET_ALLOC_DEVICE); |
| if (Device.submitData(DevicePtr, IndirectCallTable.data(), TableSize, |
| AsyncInfo)) |
| return createError(ErrorCode::InvalidBinary, "failed to copy data"); |
| // The IndirectCallTable is on the stack, so we must synchronize to ensure |
| // the data is copied before we return. |
| if (Device.synchronize(AsyncInfo)) |
| return createError(ErrorCode::InvalidBinary, |
| "failed to synchronize after copying data"); |
| |
| return std::pair<void *, uint64_t>(DevicePtr, IndirectCallTable.size()); |
| } |
| |
| // Load binary to device and perform global initialization if needed. |
| llvm::Expected<ProgramTy> DeviceTy::loadBinary(__tgt_device_image *Img) { |
| auto ProgramOrErr = ProgramTy::create(Context, DeviceHandle, Img); |
| if (!ProgramOrErr) |
| return ProgramOrErr.takeError(); |
| ProgramTy Program = std::move(*ProgramOrErr); |
| |
| // This symbol is optional. |
| auto DeviceEnvironmentPtrOrErr = |
| getAndRecordGlobalAddress(*this, Program, "__omp_rtl_device_environment"); |
| if (!DeviceEnvironmentPtrOrErr) { |
| llvm::consumeError(DeviceEnvironmentPtrOrErr.takeError()); |
| return std::move(Program); |
| } |
| void *DeviceEnvironmentPtr = *DeviceEnvironmentPtrOrErr; |
| |
| // Obtain a table mapping host function pointers to device function pointers. |
| auto CallTablePairOrErr = setupIndirectCallTable(*this, Img, Program); |
| if (!CallTablePairOrErr) |
| return CallTablePairOrErr.takeError(); |
| |
| GenericDeviceTy &GenericDevice = RTL->getDevice(RTLDeviceID); |
| DeviceEnvironmentTy DeviceEnvironment; |
| DeviceEnvironment.DeviceDebugKind = GenericDevice.getDebugKind(); |
| DeviceEnvironment.NumDevices = RTL->getNumDevices(); |
| // TODO: The device ID used here is not the real device ID used by OpenMP. |
| DeviceEnvironment.DeviceNum = RTLDeviceID; |
| DeviceEnvironment.DynamicMemSize = 0; |
| DeviceEnvironment.ClockFrequency = GenericDevice.getClockFrequency(); |
| DeviceEnvironment.IndirectCallTable = |
| reinterpret_cast<uintptr_t>(CallTablePairOrErr->first); |
| DeviceEnvironment.IndirectCallTableSize = CallTablePairOrErr->second; |
| DeviceEnvironment.HardwareParallelism = |
| GenericDevice.getHardwareParallelism(); |
| |
| AsyncInfoTy AsyncInfo(*this); |
| if (submitData(DeviceEnvironmentPtr, &DeviceEnvironment, |
| sizeof(DeviceEnvironment), AsyncInfo)) |
| return createError(ErrorCode::InvalidBinary, "failed to copy data"); |
| |
| return std::move(Program); |
| } |
| |
| void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) { |
| /// RAII to establish tool anchors before and after data allocation |
| void *TargetPtr = nullptr; |
| OMPT_IF_BUILT(InterfaceRAII TargetDataAllocRAII( |
| RegionInterface.getCallbacks<ompt_target_data_alloc>(), |
| DeviceID, HstPtr, &TargetPtr, Size, |
| /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) |
| |
| TargetPtr = RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind); |
| return TargetPtr; |
| } |
| |
| int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) { |
| /// RAII to establish tool anchors before and after data deletion |
| OMPT_IF_BUILT(InterfaceRAII TargetDataDeleteRAII( |
| RegionInterface.getCallbacks<ompt_target_data_delete>(), |
| DeviceID, TgtAllocBegin, |
| /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) |
| |
| return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind); |
| } |
| |
| // Submit data to device |
| int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, |
| AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry, |
| MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) { |
| if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) |
| MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/true, |
| Entry, HDTTMapPtr); |
| |
| /// RAII to establish tool anchors before and after data submit |
| OMPT_IF_BUILT( |
| InterfaceRAII TargetDataSubmitRAII( |
| RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>(), |
| omp_initial_device, HstPtrBegin, DeviceID, TgtPtrBegin, Size, |
| /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) |
| |
| ol_queue_handle_t queue = AsyncInfo.getQueue(); |
| if (!queue) |
| return OFFLOAD_FAIL; |
| |
| if (auto Res = olMemcpy(queue, TgtPtrBegin, DeviceHandle, HstPtrBegin, |
| PM->getHostDevice(), Size)) { |
| REPORT() << "Failure to copy data from host to device. Pointers: host " |
| << "= " << HstPtrBegin << ", device = " << TgtPtrBegin |
| << ", size = " << Size << ": " << Res->Details; |
| return OFFLOAD_FAIL; |
| } |
| return OFFLOAD_SUCCESS; |
| } |
| |
| // Retrieve data from device |
| int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin, |
| int64_t Size, AsyncInfoTy &AsyncInfo, |
| HostDataToTargetTy *Entry, |
| MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) { |
| if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) |
| MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/false, |
| Entry, HDTTMapPtr); |
| |
| /// RAII to establish tool anchors before and after data retrieval |
| OMPT_IF_BUILT( |
| InterfaceRAII TargetDataRetrieveRAII( |
| RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(), |
| DeviceID, TgtPtrBegin, omp_initial_device, HstPtrBegin, Size, |
| /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) |
| |
| ol_queue_handle_t queue = AsyncInfo.getQueue(); |
| if (!queue) |
| return OFFLOAD_FAIL; |
| if (auto Res = olMemcpy(queue, HstPtrBegin, PM->getHostDevice(), TgtPtrBegin, |
| DeviceHandle, Size)) { |
| REPORT() << "Failure to copy data from device to host. Pointers: host " |
| << "= " << HstPtrBegin << ", device = " << TgtPtrBegin |
| << ", size = " << Size << ": " << Res->Details; |
| return OFFLOAD_FAIL; |
| } |
| return OFFLOAD_SUCCESS; |
| } |
| |
| // Copy data from current device to destination device directly |
| int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr, |
| int64_t Size, AsyncInfoTy &AsyncInfo) { |
| /// RAII to establish tool anchors before and after data exchange |
| /// Note: Despite the fact that this is a data exchange, we use 'from_device' |
| /// operation enum (w.r.t. ompt_target_data_op_t) as there is currently |
| /// no better alternative. It is still possible to distinguish this |
| /// scenario from a real data retrieve by checking if both involved |
| /// device numbers are less than omp_get_num_devices(). |
| OMPT_IF_BUILT( |
| InterfaceRAII TargetDataExchangeRAII( |
| RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(), |
| RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size, |
| /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);) |
| |
| ol_queue_handle_t queue = AsyncInfo.getQueue(); |
| if (!queue) |
| return OFFLOAD_FAIL; |
| if (auto Res = olMemcpy(queue, DstPtr, DstDev.DeviceHandle, SrcPtr, |
| DeviceHandle, Size)) { |
| REPORT() << "Failure to copy data from device (" << RTLDeviceID |
| << ") to device (" << DstDev.RTLDeviceID |
| << "). Pointers: host = " << SrcPtr << ", device = " << DstPtr |
| << ", size = " << Size << ": " << Res->Details; |
| return OFFLOAD_FAIL; |
| } |
| return OFFLOAD_SUCCESS; |
| } |
| |
| llvm::Expected<void *> DeviceTy::registerMemory(void *HstPtr, int64_t Size, |
| bool LockMemory) { |
| void *LockedPtr = nullptr; |
| ol_memory_register_flags_t Flags = |
| LockMemory ? OL_MEMORY_REGISTER_FLAG_LOCK_MEMORY : 0; |
| if (auto Res = olMemRegister(DeviceHandle, HstPtr, Size, Flags, &LockedPtr)) |
| return createError(ErrorCode::Unknown, "failed to lock memory %p: %s", |
| HstPtr, Res->Details); |
| return LockedPtr; |
| } |
| |
| llvm::Error DeviceTy::unregisterMemory(void *HstPtr, bool UnlockMemory) { |
| ol_memory_register_flags_t Flags = |
| UnlockMemory ? OL_MEMORY_REGISTER_FLAG_UNLOCK_MEMORY : 0; |
| if (auto Res = olMemUnregister(DeviceHandle, HstPtr, Flags)) |
| return createError(ErrorCode::Unknown, "failed to unlock memory %p: %s", |
| HstPtr, Res->Details); |
| return llvm::Error::success(); |
| } |
| |
| int32_t DeviceTy::notifyDataMapped(void *HstPtr, int64_t Size) { |
| ODBG(ODT_Mapping) << "Notifying about new mapping: HstPtr=" << HstPtr |
| << ", Size=" << Size; |
| |
| auto LockedPtrOrErr = registerMemory(HstPtr, Size, LockMappedBuffers); |
| if (!LockedPtrOrErr) { |
| if (!IgnoreLockMappedFailures) { |
| REPORT() << "Notifying about data mapping failed: " |
| << llvm::toString(LockedPtrOrErr.takeError()); |
| return OFFLOAD_FAIL; |
| } |
| llvm::consumeError(LockedPtrOrErr.takeError()); |
| } |
| return OFFLOAD_SUCCESS; |
| } |
| |
| int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) { |
| ODBG(ODT_Mapping) << "Notifying about an unmapping: HstPtr=" << HstPtr; |
| |
| if (auto Err = unregisterMemory(HstPtr, LockMappedBuffers)) { |
| if (!IgnoreLockMappedFailures) { |
| REPORT() << "Notifying about data unmapping failed: " |
| << llvm::toString(std::move(Err)); |
| return OFFLOAD_FAIL; |
| } |
| llvm::consumeError(std::move(Err)); |
| } |
| return OFFLOAD_SUCCESS; |
| } |
| |
| /// Resolve \p NumArgs (base pointer, offset) pairs into a flattened array of |
| /// argument-value pointers suitable for a kernel launch, writing the result |
| /// into \p LaunchArgs.NumArgs/Args. |
| static void resolveKernelLaunchParams(void **const TgtArgs, |
| ptrdiff_t *const TgtOffsets, |
| uint32_t NumArgs, |
| llvm::SmallVector<void *> &Args, |
| llvm::SmallVector<void *> &Ptrs, |
| KernelLaunchArgsTy &LaunchArgs) { |
| LaunchArgs.NumArgs = NumArgs; |
| Args.resize(NumArgs); |
| Ptrs.resize(NumArgs); |
| |
| if (NumArgs == 0) |
| return; |
| |
| for (uint32_t I = 0; I < NumArgs; ++I) { |
| Args[I] = reinterpret_cast<void *>(reinterpret_cast<intptr_t>(TgtArgs[I]) + |
| TgtOffsets[I]); |
| Ptrs[I] = &Args[I]; |
| } |
| |
| LaunchArgs.Args = &Ptrs[0]; |
| } |
| |
| // Run region on device |
| int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr, |
| ptrdiff_t *TgtOffsets, KernelArgsTy &KernelArgs, |
| KernelReplayOutcomeTy *ReplayOutcome, |
| AsyncInfoTy &AsyncInfo) { |
| llvm::SmallVector<void *> Args, Ptrs; |
| llvm::SmallVector<int64_t> ArgSizes; |
| |
| KernelLaunchArgsTy LaunchArgs; |
| LaunchArgs.OmpABIVersion = KernelArgs.Version; |
| LaunchArgs.ReplayOutcome = ReplayOutcome; |
| LaunchArgs.ArgSizes = KernelArgs.ArgSizes; |
| LaunchArgs.Tripcount = KernelArgs.Tripcount; |
| LaunchArgs.DynCGroupMem = KernelArgs.DynCGroupMem; |
| llvm::copy(KernelArgs.UserNumBlocks, LaunchArgs.UserNumBlocks); |
| llvm::copy(KernelArgs.UserThreadLimit, LaunchArgs.UserThreadLimit); |
| LaunchArgs.Flags.Cooperative = KernelArgs.Flags.Cooperative; |
| LaunchArgs.Flags.StrictBlocks = KernelArgs.Flags.StrictBlocks; |
| LaunchArgs.Flags.StrictThreads = KernelArgs.Flags.StrictThreads; |
| LaunchArgs.Flags.DynCGroupMemFallback = KernelArgs.Flags.DynCGroupMemFallback; |
| |
| if (KernelArgs.Flags.IsCUDA) { |
| // Kernel languages (CUDA/HIP) pass an already-flattened argument-pointer |
| // array through KernelArgs.ArgPtrs instead of using the OpenMP |
| // base-pointer/offset argument scheme. |
| auto *LaunchParams = |
| reinterpret_cast<KernelLaunchParamsTy *>(KernelArgs.ArgPtrs); |
| LaunchArgs.NumArgs = LaunchParams->NumArgs; |
| LaunchArgs.Args = LaunchParams->Args; |
| } else { |
| resolveKernelLaunchParams(TgtVarsPtr, TgtOffsets, KernelArgs.NumArgs, Args, |
| Ptrs, LaunchArgs); |
| // The dyn_ptr slot is reserved by the host (version >= 4) or by |
| // upgradeKernelArgs (version 3) as the last element of the argument |
| // array. Version 3 device kernels expect it first instead, so rotate it |
| // to the front to match that ABI. |
| if (KernelArgs.NumArgs > 0 && |
| KernelArgs.Version >= OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR) { |
| if (KernelArgs.Version == OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR) { |
| std::rotate(Args.begin(), Args.end() - 1, Args.end()); |
| LaunchArgs.DynPtrSlot = &Args[0]; |
| |
| // Keep ArgSizes in sync with the rotated Args, if present. |
| if (LaunchArgs.ArgSizes) { |
| ArgSizes.assign(LaunchArgs.ArgSizes, |
| LaunchArgs.ArgSizes + KernelArgs.NumArgs); |
| std::rotate(ArgSizes.begin(), ArgSizes.end() - 1, ArgSizes.end()); |
| LaunchArgs.ArgSizes = ArgSizes.data(); |
| } |
| } else { |
| LaunchArgs.DynPtrSlot = &Args[KernelArgs.NumArgs - 1]; |
| } |
| } |
| } |
| |
| return RTL->launch_kernel(RTLDeviceID, TgtEntryPtr, LaunchArgs, AsyncInfo); |
| } |
| |
| static llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, |
| ol_device_type_t Value) { |
| switch (Value) { |
| case OL_DEVICE_TYPE_DEFAULT: |
| return OS << "DEFAULT"; |
| case OL_DEVICE_TYPE_ALL: |
| return OS << "ALL"; |
| case OL_DEVICE_TYPE_GPU: |
| return OS << "GPU"; |
| case OL_DEVICE_TYPE_CPU: |
| return OS << "CPU"; |
| case OL_DEVICE_TYPE_HOST: |
| return OS << "HOST"; |
| default: |
| return OS << "<< INVALID >>"; |
| } |
| } |
| |
| static llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, |
| const ol_dimensions_t &Value) { |
| return OS << "{x: " << Value.x << ", y: " << Value.y << ", z: " << Value.z |
| << "}"; |
| } |
| |
| static void printFPCapabilityFlags(llvm::raw_ostream &OS, |
| ol_device_fp_capability_flags_t Value) { |
| OS << Value << " {"; |
| if (Value & OL_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT) |
| OS << " CORRECTLY_ROUNDED_DIVIDE_SQRT"; |
| if (Value & OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST) |
| OS << " ROUND_TO_NEAREST"; |
| if (Value & OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO) |
| OS << " ROUND_TO_ZERO"; |
| if (Value & OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF) |
| OS << " ROUND_TO_INF"; |
| if (Value & OL_DEVICE_FP_CAPABILITY_FLAG_INF_NAN) |
| OS << " INF_NAN"; |
| if (Value & OL_DEVICE_FP_CAPABILITY_FLAG_DENORM) |
| OS << " DENORM"; |
| if (Value & OL_DEVICE_FP_CAPABILITY_FLAG_FMA) |
| OS << " FMA"; |
| if (Value & OL_DEVICE_FP_CAPABILITY_FLAG_SOFT_FLOAT) |
| OS << " SOFT_FLOAT"; |
| OS << " }"; |
| } |
| |
| // Print a scalar liboffload device info property, if supported. |
| template <typename T> |
| static void printDeviceInfoScalar(ol_device_handle_t DeviceHandle, |
| ol_device_info_t PropName, |
| llvm::StringRef Label, |
| llvm::StringRef Units = "") { |
| T Value{}; |
| if (olGetDeviceInfo(DeviceHandle, PropName, sizeof(Value), &Value)) |
| return; |
| llvm::outs() << " " << Label << ": " << Value; |
| if (!Units.empty()) |
| llvm::outs() << " " << Units; |
| llvm::outs() << "\n"; |
| } |
| |
| // Print a boolean liboffload device info property, if supported. |
| static void printDeviceInfoBool(ol_device_handle_t DeviceHandle, |
| ol_device_info_t PropName, |
| llvm::StringRef Label) { |
| bool Value = false; |
| if (olGetDeviceInfo(DeviceHandle, PropName, sizeof(Value), &Value)) |
| return; |
| llvm::outs() << " " << Label << ": " << (Value ? "Yes" : "No") << "\n"; |
| } |
| |
| // Print a floating point capability liboffload device info property, if |
| // supported. |
| static void printDeviceInfoFPCapability(ol_device_handle_t DeviceHandle, |
| ol_device_info_t PropName, |
| llvm::StringRef Label) { |
| ol_device_fp_capability_flags_t Value{}; |
| if (olGetDeviceInfo(DeviceHandle, PropName, sizeof(Value), &Value)) |
| return; |
| llvm::outs() << " " << Label << ": "; |
| printFPCapabilityFlags(llvm::outs(), Value); |
| llvm::outs() << "\n"; |
| } |
| |
| // Print a string liboffload device info property, if supported. |
| static void printDeviceInfoString(ol_device_handle_t DeviceHandle, |
| ol_device_info_t PropName, |
| llvm::StringRef Label) { |
| size_t Size = 0; |
| if (olGetDeviceInfoSize(DeviceHandle, PropName, &Size) || Size == 0) |
| return; |
| |
| llvm::SmallVector<char> Value(Size); |
| if (olGetDeviceInfo(DeviceHandle, PropName, Size, Value.data())) |
| return; |
| |
| llvm::outs() << " " << Label << ": " << Value.data() << "\n"; |
| } |
| |
| bool DeviceTy::printDeviceInfo() { |
| llvm::outs() << "Device " << DeviceID << ":\n"; |
| printDeviceInfoScalar<ol_device_type_t>(DeviceHandle, OL_DEVICE_INFO_TYPE, |
| "Type"); |
| printDeviceInfoScalar<ol_platform_handle_t>( |
| DeviceHandle, OL_DEVICE_INFO_PLATFORM, "Platform"); |
| printDeviceInfoString(DeviceHandle, OL_DEVICE_INFO_NAME, "Name"); |
| printDeviceInfoString(DeviceHandle, OL_DEVICE_INFO_PRODUCT_NAME, |
| "Product Name"); |
| printDeviceInfoString(DeviceHandle, OL_DEVICE_INFO_UID, "UID"); |
| printDeviceInfoString(DeviceHandle, OL_DEVICE_INFO_VENDOR, "Vendor"); |
| printDeviceInfoString(DeviceHandle, OL_DEVICE_INFO_DRIVER_VERSION, |
| "Driver Version"); |
| printDeviceInfoScalar<uint32_t>( |
| DeviceHandle, OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE, "Max Work Group Size"); |
| printDeviceInfoScalar<ol_dimensions_t>( |
| DeviceHandle, OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE_PER_DIMENSION, |
| "Max Work Group Size Per Dimension"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, OL_DEVICE_INFO_MAX_WORK_SIZE, |
| "Max Work Size"); |
| printDeviceInfoScalar<ol_dimensions_t>( |
| DeviceHandle, OL_DEVICE_INFO_MAX_WORK_SIZE_PER_DIMENSION, |
| "Max Work Size Per Dimension"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, OL_DEVICE_INFO_VENDOR_ID, |
| "Vendor ID"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, |
| OL_DEVICE_INFO_NUM_COMPUTE_UNITS, |
| "Number of Compute Units"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, |
| OL_DEVICE_INFO_MAX_CLOCK_FREQUENCY, |
| "Max Clock Frequency", "MHz"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, |
| OL_DEVICE_INFO_MEMORY_CLOCK_RATE, |
| "Memory Clock Rate", "MHz"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, OL_DEVICE_INFO_ADDRESS_BITS, |
| "Address Bits"); |
| printDeviceInfoScalar<uint64_t>(DeviceHandle, |
| OL_DEVICE_INFO_MAX_MEM_ALLOC_SIZE, |
| "Max Memory Allocation Size", "B"); |
| printDeviceInfoScalar<uint64_t>(DeviceHandle, OL_DEVICE_INFO_GLOBAL_MEM_SIZE, |
| "Global Memory Size", "B"); |
| printDeviceInfoScalar<uint64_t>(DeviceHandle, |
| OL_DEVICE_INFO_WORK_GROUP_LOCAL_MEM_SIZE, |
| "Work Group Local Memory Size", "B"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, OL_DEVICE_INFO_NUM_LANES, |
| "Number of Lanes"); |
| printDeviceInfoBool(DeviceHandle, OL_DEVICE_INFO_SINGLE_FP_SUPPORT, |
| "Single Precision Floating Point Support"); |
| printDeviceInfoFPCapability(DeviceHandle, OL_DEVICE_INFO_SINGLE_FP_CONFIG, |
| "Single Precision Floating Point Capability"); |
| printDeviceInfoBool(DeviceHandle, OL_DEVICE_INFO_DOUBLE_FP_SUPPORT, |
| "Double Precision Floating Point Support"); |
| printDeviceInfoFPCapability(DeviceHandle, OL_DEVICE_INFO_DOUBLE_FP_CONFIG, |
| "Double Precision Floating Point Capability"); |
| printDeviceInfoBool(DeviceHandle, OL_DEVICE_INFO_HALF_FP_SUPPORT, |
| "Half Precision Floating Point Support"); |
| printDeviceInfoFPCapability(DeviceHandle, OL_DEVICE_INFO_HALF_FP_CONFIG, |
| "Half Precision Floating Point Capability"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, |
| OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_CHAR, |
| "Native Vector Width For Char"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, |
| OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_SHORT, |
| "Native Vector Width For Short"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, |
| OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_INT, |
| "Native Vector Width For Int"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, |
| OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG, |
| "Native Vector Width For Long"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, |
| OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_FLOAT, |
| "Native Vector Width For Float"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, |
| OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE, |
| "Native Vector Width For Double"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, |
| OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF, |
| "Native Vector Width For Half"); |
| printDeviceInfoBool(DeviceHandle, OL_DEVICE_INFO_COOPERATIVE_LAUNCH_SUPPORT, |
| "Cooperative Kernel Launch Support"); |
| printDeviceInfoScalar<uint32_t>(DeviceHandle, OL_DEVICE_INFO_DRIVER_ID, |
| "Driver ID"); |
| return true; |
| } |
| |
| int32_t DeviceTy::synchronize(AsyncInfoTy &AsyncInfo) { |
| ol_queue_handle_t Queue = AsyncInfo.getQueue(); |
| if (!Queue) |
| return OFFLOAD_SUCCESS; |
| if (auto Res = olSyncQueue(Queue)) { |
| REPORT() << "Failure to synchronize stream " << Queue << ": " |
| << Res->Details; |
| return OFFLOAD_FAIL; |
| } |
| return OFFLOAD_SUCCESS; |
| } |
| |
| int32_t DeviceTy::queryAsync(AsyncInfoTy &AsyncInfo) { |
| ol_queue_handle_t Queue = AsyncInfo.getQueue(); |
| if (!Queue) |
| return OFFLOAD_SUCCESS; |
| |
| bool isComplete; |
| if (auto Res = olQueryQueue(Queue, &isComplete)) { |
| REPORT() << "Failure to query stream " << Queue << ": " << Res->Details; |
| return OFFLOAD_FAIL; |
| } |
| return OFFLOAD_SUCCESS; |
| } |
| |
| int32_t DeviceTy::recordEvent(void **Event, AsyncInfoTy &AsyncInfo) { |
| ol_event_handle_t NewEvent; |
| if (auto Res = |
| olCreateEvent(AsyncInfo.getQueue(), OL_EVENT_FLAGS_NONE, &NewEvent)) { |
| REPORT() << "Failure to record event: " << Res->Details; |
| return OFFLOAD_FAIL; |
| } |
| |
| if (*Event) { |
| if (auto Res = olDestroyEvent(static_cast<ol_event_handle_t>(*Event))) |
| REPORT() << "Failure to destroy previous event " << *Event << ": " |
| << Res->Details; |
| } |
| |
| *Event = NewEvent; |
| return OFFLOAD_SUCCESS; |
| } |
| |
| int32_t DeviceTy::waitEvent(void *Event, AsyncInfoTy &AsyncInfo) { |
| ol_event_handle_t E = static_cast<ol_event_handle_t>(Event); |
| if (auto Res = olWaitEvents(AsyncInfo.getQueue(), &E, 1)) { |
| REPORT() << "Failure to wait for event " << Event << ": " << Res->Details; |
| return OFFLOAD_FAIL; |
| } |
| return OFFLOAD_SUCCESS; |
| } |
| |
| int32_t DeviceTy::syncEvent(void *Event) { |
| if (auto Res = olSyncEvent(static_cast<ol_event_handle_t>(Event))) { |
| REPORT() << "Failure to synchronize event " << Event << ": " |
| << Res->Details; |
| return OFFLOAD_FAIL; |
| } |
| return OFFLOAD_SUCCESS; |
| } |
| |
| int32_t DeviceTy::destroyEvent(void *Event) { |
| if (auto Res = olDestroyEvent(static_cast<ol_event_handle_t>(Event))) { |
| REPORT() << "Failure to destroy event " << Event << ": " << Res->Details; |
| return OFFLOAD_FAIL; |
| } |
| return OFFLOAD_SUCCESS; |
| } |
| |
| void DeviceTy::dumpOffloadEntries() { |
| fprintf(stderr, "Device %i offload entries:\n", DeviceID); |
| for (auto &It : *DeviceOffloadEntries.getExclusiveAccessor()) { |
| const char *Kind = "kernel"; |
| if (It.second.isLink()) |
| Kind = "link"; |
| else if (It.second.isGlobal()) |
| Kind = "global var."; |
| fprintf(stderr, " %11s: %s\n", Kind, It.second.getNameAsCStr()); |
| } |
| } |
| |
| bool DeviceTy::useAutoZeroCopy() { |
| if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY) |
| return false; |
| return RTL->use_auto_zero_copy(RTLDeviceID); |
| } |
| |
| bool DeviceTy::isAccessiblePtr(const void *Ptr, size_t Size) { |
| bool IsAccessible = false; |
| if (auto Res = olMemIsAccessible(DeviceHandle, Ptr, Size, &IsAccessible)) { |
| REPORT() << "Failure to check pointer accessibility: " << Res->Details; |
| return false; |
| } |
| return IsAccessible; |
| } |
