| //===-- PluginManager.cpp - Plugin loading and communication API ---------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // Functionality for handling plugins. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "PluginManager.h" | |
| #include "OmpAccError.h" | |
| #include "OpenMP/OMPT/Interface.h" | |
| #include "OpenMP/OffloadRTL.h" | |
| #include "Shared/Debug.h" | |
| #include "Shared/Environment.h" | |
| #include "Shared/Profile.h" | |
| #include "device.h" | |
| #include "llvm/ADT/SmallString.h" | |
| #include "llvm/Support/Error.h" | |
| #include "llvm/Support/ErrorHandling.h" | |
| #include <algorithm> | |
| #include <memory> | |
| #include <string> | |
| #ifdef OMPT_SUPPORT | |
| using namespace llvm::omp::target::ompt; | |
| #endif | |
| using namespace llvm; | |
| using namespace llvm::sys; | |
| using namespace llvm::omp::target::error; | |
| using namespace llvm::omp::target::debug; | |
| using namespace llvm::omp::target::helpers; | |
| namespace llvm::offload::tmp { | |
| GenericPluginTy *__ol_tgt_GetPluginFromPlatform(ol_platform_handle_t Platform); | |
| int32_t __ol_tgt_GetPluginDeviceId(ol_device_handle_t Device); | |
| } // namespace llvm::offload::tmp | |
| void PluginManager::init() { | |
| TIMESCOPE(); | |
| ODBG(ODT_Init) << "Loading RTLs"; | |
| if (ol_result_t Res = olInit(nullptr)) | |
| REPORT() << "Failed to initialize liboffload: " << Res->Details; | |
| if (auto Err = iteratePlatforms( | |
| [](ol_platform_handle_t Platform, void *Data) { | |
| auto *PM = static_cast<PluginManager *>(Data); | |
| auto *Plugin = | |
| llvm::offload::tmp::__ol_tgt_GetPluginFromPlatform(Platform); | |
| ODBG(ODT_Init) << "Adding plugin " << Plugin->getName() | |
| << " from liboffload"; | |
| PM->Plugins.push_back(Plugin); | |
| }, | |
| this)) | |
| REPORT() << "Failed to iterate platforms: " << toString(std::move(Err)); | |
| ODBG(ODT_Init) << "RTLs loaded!"; | |
| } | |
| void PluginManager::deinit() { | |
| TIMESCOPE(); | |
| ODBG(ODT_Deinit) << "Unloading RTLs..."; | |
| auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor(); | |
| for (DeviceTy &Device : devices(ExclusiveDevicesAccessor)) { | |
| if (auto Err = Device.deinit()) | |
| REPORT() << "Failed to deinitialize device " << Device.DeviceID << ": " | |
| << toString(std::move(Err)); | |
| } | |
| Plugins.clear(); | |
| if (auto Err = olShutDown()) | |
| REPORT() << "Failed to denitialize liboffload: " << Err->Details; | |
| ODBG(ODT_Deinit) << "RTLs unloaded!"; | |
| } | |
| bool PluginManager::initializeDevice(ol_device_handle_t DeviceHandle) { | |
| if (DeviceIds.find(DeviceHandle) != DeviceIds.end()) { | |
| auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor(); | |
| (*ExclusiveDevicesAccessor)[DeviceIds[DeviceHandle]]->setHasPendingImages( | |
| true); | |
| return true; | |
| } | |
| ol_platform_handle_t PlatformHandle; | |
| if (olGetDeviceInfo(DeviceHandle, OL_DEVICE_INFO_PLATFORM, | |
| sizeof(PlatformHandle), &PlatformHandle)) { | |
| REPORT() << "Failed to get platform while initializing device " | |
| << DeviceHandle; | |
| return false; | |
| } | |
| GenericPluginTy &Plugin = | |
| *llvm::offload::tmp::__ol_tgt_GetPluginFromPlatform(PlatformHandle); | |
| int32_t DeviceId = | |
| llvm::offload::tmp::__ol_tgt_GetPluginDeviceId(DeviceHandle); | |
| // Initialize the device information for the RTL we are about to use. | |
| auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor(); | |
| int32_t UserId = ExclusiveDevicesAccessor->size(); | |
| auto Device = | |
| std::make_unique<DeviceTy>(&Plugin, UserId, DeviceId, DeviceHandle, this); | |
| if (auto Err = Device->init()) { | |
| std::string InfoMsg = toString(std::move(Err)); | |
| ODBG(ODT_Init) << "Failed to init device " << DeviceId << ": " << InfoMsg; | |
| return false; | |
| } | |
| ExclusiveDevicesAccessor->push_back(std::move(Device)); | |
| // We need to map between the liboffload device handle and the OpenMP device | |
| // id. | |
| DeviceIds[DeviceHandle] = UserId; | |
| return true; | |
| } | |
| void PluginManager::initializeAllDevices() { | |
| if (auto Err = iterateDevices( | |
| [this](ol_device_handle_t Device) { initializeDevice(Device); })) { | |
| REPORT() << "Failed to iterate devices: " << toString(std::move(Err)); | |
| } | |
| } | |
| // Returns a pointer to the binary descriptor, upgrading from a legacy format if | |
| // necessary. | |
| __tgt_bin_desc *PluginManager::upgradeLegacyEntries(__tgt_bin_desc *Desc) { | |
| struct LegacyEntryTy { | |
| void *Address; | |
| char *SymbolName; | |
| size_t Size; | |
| int32_t Flags; | |
| int32_t Data; | |
| }; | |
| if (UpgradedDescriptors.contains(Desc)) | |
| return &UpgradedDescriptors[Desc]; | |
| if (Desc->HostEntriesBegin == Desc->HostEntriesEnd || | |
| Desc->HostEntriesBegin->Reserved == 0) | |
| return Desc; | |
| // The new format mandates that each entry starts with eight bytes of zeroes. | |
| // This allows us to detect the old format as this is a null pointer. | |
| llvm::SmallVector<llvm::offloading::EntryTy, 0> &NewEntries = | |
| LegacyEntries.emplace_back(); | |
| for (LegacyEntryTy &Entry : llvm::make_range( | |
| reinterpret_cast<LegacyEntryTy *>(Desc->HostEntriesBegin), | |
| reinterpret_cast<LegacyEntryTy *>(Desc->HostEntriesEnd))) { | |
| llvm::offloading::EntryTy &NewEntry = NewEntries.emplace_back(); | |
| NewEntry.Address = Entry.Address; | |
| NewEntry.Flags = Entry.Flags; | |
| NewEntry.Data = Entry.Data; | |
| NewEntry.Size = Entry.Size; | |
| NewEntry.SymbolName = Entry.SymbolName; | |
| NewEntry.Kind = object::OffloadKind::OFK_OpenMP; | |
| } | |
| // Create a new image struct so we can update the entries list. | |
| llvm::SmallVector<__tgt_device_image, 0> &NewImages = | |
| LegacyImages.emplace_back(); | |
| for (int32_t Image = 0; Image < Desc->NumDeviceImages; ++Image) | |
| NewImages.emplace_back( | |
| __tgt_device_image{Desc->DeviceImages[Image].ImageStart, | |
| Desc->DeviceImages[Image].ImageEnd, | |
| NewEntries.begin(), NewEntries.end()}); | |
| // Create the new binary descriptor containing the newly created memory. | |
| __tgt_bin_desc &NewDesc = UpgradedDescriptors[Desc]; | |
| NewDesc.DeviceImages = NewImages.begin(); | |
| NewDesc.NumDeviceImages = Desc->NumDeviceImages; | |
| NewDesc.HostEntriesBegin = NewEntries.begin(); | |
| NewDesc.HostEntriesEnd = NewEntries.end(); | |
| return &NewDesc; | |
| } | |
| bool PluginManager::registerImageOnDevice( | |
| ol_device_handle_t DeviceHandle, __tgt_bin_desc *Desc, | |
| __tgt_device_image *Img, | |
| llvm::SmallVectorImpl<ol_device_handle_t> &UsedDevices) { | |
| ol_platform_handle_t PlatformHandle; | |
| if (auto Res = olGetDeviceInfo(DeviceHandle, OL_DEVICE_INFO_PLATFORM, | |
| sizeof(PlatformHandle), &PlatformHandle)) { | |
| REPORT() << "Failed to get platform info for device " << DeviceHandle << ":" | |
| << Res->Details; | |
| PlatformHandle = nullptr; | |
| } | |
| llvm::SmallString<256> PlatformName("Unknown"); | |
| if (PlatformHandle) { | |
| size_t PlatformNameSize = 0; | |
| if (olGetPlatformInfoSize(PlatformHandle, OL_PLATFORM_INFO_NAME, | |
| &PlatformNameSize)) | |
| PlatformNameSize = 0; | |
| PlatformName.resize(PlatformNameSize); | |
| if (PlatformNameSize > 0) { | |
| if (olGetPlatformInfo(PlatformHandle, OL_PLATFORM_INFO_NAME, | |
| PlatformNameSize, PlatformName.data())) | |
| PlatformName = "Unknown"; | |
| } else | |
| PlatformName = "Unknown"; | |
| } | |
| // We only want a single matching image to be registered for each binary | |
| // descriptor. This prevents multiple of the same image from being registered | |
| // for the same device in the case that they are mutually compatible, such as | |
| // sm_80 and sm_89. | |
| if (llvm::is_contained(UsedDevices, DeviceHandle)) { | |
| ODBG(ODT_Init) << "Image " << Img->ImageStart | |
| << " is a duplicate, not loaded on RTL " << PlatformName | |
| << " on device " << DeviceHandle; | |
| return false; | |
| } | |
| ODBG(ODT_Init) << "Image " << Img->ImageStart << " with RTL " << PlatformName | |
| << " on device " << DeviceHandle; | |
| if (!initializeDevice(DeviceHandle)) { | |
| ODBG(ODT_Init) << "Skipping image " << Img->ImageStart << " on device " | |
| << DeviceHandle << ": device failed to initialize"; | |
| return false; | |
| } | |
| // Initialize (if necessary) translation table for this library. | |
| std::lock_guard<std::mutex> LG(TrlTblMtx); | |
| if (!HostEntriesBeginToTransTable.count(Desc->HostEntriesBegin)) { | |
| HostEntriesBeginRegistrationOrder.push_back(Desc->HostEntriesBegin); | |
| TranslationTable &TT = HostEntriesBeginToTransTable[Desc->HostEntriesBegin]; | |
| TT.HostTable.EntriesBegin = Desc->HostEntriesBegin; | |
| TT.HostTable.EntriesEnd = Desc->HostEntriesEnd; | |
| } | |
| // Retrieve translation table for this library. | |
| TranslationTable &TT = HostEntriesBeginToTransTable[Desc->HostEntriesBegin]; | |
| ODBG(ODT_Init) << "Registering image " << Img->ImageStart << " with RTL " | |
| << PlatformName; | |
| auto UserId = DeviceIds.at(DeviceHandle); | |
| if (TT.TargetsTable.size() < static_cast<size_t>(UserId + 1)) { | |
| TT.DeviceTables.resize(UserId + 1, {}); | |
| TT.TargetsImages.resize(UserId + 1, nullptr); | |
| TT.TargetsEntries.resize(UserId + 1, {}); | |
| TT.TargetsTable.resize(UserId + 1, nullptr); | |
| } | |
| // Register the image for this target type and invalidate the table. | |
| TT.TargetsImages[UserId] = Img; | |
| TT.TargetsTable[UserId] = nullptr; | |
| UsedDevices.push_back(DeviceHandle); | |
| UsedImages.insert(Img); | |
| return true; | |
| } | |
| void PluginManager::registerLib(__tgt_bin_desc *Desc) { | |
| RTLsMtx.lock(); | |
| // Upgrade the entries from the legacy implementation if necessary. | |
| Desc = upgradeLegacyEntries(Desc); | |
| // Add in all the OpenMP requirements associated with this binary. | |
| for (llvm::offloading::EntryTy &Entry : | |
| llvm::make_range(Desc->HostEntriesBegin, Desc->HostEntriesEnd)) | |
| if (Entry.Kind == object::OffloadKind::OFK_OpenMP && | |
| Entry.Flags == OMP_REGISTER_REQUIRES) | |
| addRequirements(Entry.Data); | |
| // Extract the executable image and extra information if available. | |
| for (int32_t i = 0; i < Desc->NumDeviceImages; ++i) | |
| addDeviceImage(*Desc, Desc->DeviceImages[i]); | |
| // Register the images with the RTLs that understand them, if any. | |
| llvm::SmallVector<ol_device_handle_t> UsedDevices; | |
| for (int32_t i = 0; i < Desc->NumDeviceImages; ++i) { | |
| // Obtain the image and information that was previously extracted. | |
| __tgt_device_image *Img = &Desc->DeviceImages[i]; | |
| struct RegisterImageState { | |
| PluginManager *PM; | |
| __tgt_bin_desc *Desc; | |
| __tgt_device_image *Img; | |
| llvm::SmallVectorImpl<ol_device_handle_t> &UsedDevices; | |
| bool FoundRTL = false; | |
| } State{this, Desc, Img, UsedDevices, false}; | |
| if (ol_result_t Res = olIterateCompatibleDevices( | |
| Img->ImageStart, utils::getPtrDiff(Img->ImageEnd, Img->ImageStart), | |
| [](ol_device_handle_t DeviceHandle, void *Data) { | |
| auto &State = *static_cast<RegisterImageState *>(Data); | |
| if (State.PM->registerImageOnDevice(DeviceHandle, State.Desc, | |
| State.Img, State.UsedDevices)) | |
| State.FoundRTL = true; | |
| return true; | |
| }, | |
| &State)) | |
| REPORT() << "Failed to iterate compatible devices: " << Res->Details; | |
| if (!State.FoundRTL) | |
| ODBG(ODT_Init) << "No RTL found for image " << Img->ImageStart << "!"; | |
| } | |
| RTLsMtx.unlock(); | |
| bool UseAutoZeroCopy = false; | |
| auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor(); | |
| // APUs are homogeneous set of GPUs. Check the first device for | |
| // configuring Auto Zero-Copy. | |
| if (ExclusiveDevicesAccessor->size() > 0) { | |
| auto &Device = *(*ExclusiveDevicesAccessor)[0]; | |
| UseAutoZeroCopy = Device.useAutoZeroCopy(); | |
| } | |
| if (UseAutoZeroCopy) | |
| addRequirements(OMPX_REQ_AUTO_ZERO_COPY); | |
| ODBG(ODT_Init) << "Done registering entries!"; | |
| } | |
| // Temporary forward declaration, old style CTor/DTor handling is going away. | |
| int target(ident_t *Loc, DeviceTy &Device, void *HostPtr, | |
| KernelArgsTy &KernelArgs, AsyncInfoTy &AsyncInfo); | |
| void PluginManager::unregisterLib(__tgt_bin_desc *Desc) { | |
| ODBG(ODT_Deinit) << "Unloading target library!"; | |
| Desc = upgradeLegacyEntries(Desc); | |
| RTLsMtx.lock(); | |
| // Find which RTL understands each image, if any. | |
| for (DeviceImageTy &DI : deviceImages()) { | |
| // Obtain the image and information that was previously extracted. | |
| __tgt_device_image *Img = &DI.getExecutableImage(); | |
| GenericPluginTy *FoundRTL = NULL; | |
| // Scan the RTLs that have associated images until we find one that supports | |
| // the current image. We only need to scan RTLs that are already being used. | |
| for (auto &R : plugins()) { | |
| if (R.is_initialized()) | |
| continue; | |
| // Ensure that we do not use any unused images associated with this RTL. | |
| if (!UsedImages.contains(Img)) | |
| continue; | |
| FoundRTL = &R; | |
| ODBG(ODT_Deinit) << "Unregistered image " << Img->ImageStart | |
| << " from RTL"; | |
| break; | |
| } | |
| // if no RTL was found proceed to unregister the next image | |
| if (!FoundRTL) { | |
| ODBG(ODT_Deinit) << "No RTLs in use support the image " | |
| << Img->ImageStart; | |
| } | |
| } | |
| RTLsMtx.unlock(); | |
| ODBG(ODT_Deinit) << "Done unregistering images!"; | |
| // Remove entries from HostPtrToTableMap | |
| TblMapMtx.lock(); | |
| for (llvm::offloading::EntryTy *Cur = Desc->HostEntriesBegin; | |
| Cur < Desc->HostEntriesEnd; ++Cur) { | |
| if (Cur->Kind == object::OffloadKind::OFK_OpenMP) | |
| HostPtrToTableMap.erase(Cur->Address); | |
| } | |
| // Remove translation table for this descriptor. | |
| auto TransTable = HostEntriesBeginToTransTable.find(Desc->HostEntriesBegin); | |
| if (TransTable != HostEntriesBeginToTransTable.end()) { | |
| ODBG(ODT_Deinit) << "Removing translation table for descriptor " | |
| << Desc->HostEntriesBegin; | |
| HostEntriesBeginToTransTable.erase(TransTable); | |
| } else { | |
| ODBG(ODT_Deinit) << "Translation table for descriptor " | |
| << Desc->HostEntriesBegin << " cannot be found, probably " | |
| << "it has been already removed."; | |
| } | |
| TblMapMtx.unlock(); | |
| ODBG(ODT_Deinit) << "Done unregistering library!"; | |
| } | |
| /// Get the maximum number of threads per team the device supports, further | |
| /// limited by OMP_TEAMS_THREAD_LIMIT if the user set it. | |
| static uint32_t getDeviceThreadLimit( | |
| llvm::omp::target::plugin::GenericDeviceTy &GenericDevice) { | |
| static Int32Envar OMP_TeamsThreadLimit("OMP_TEAMS_THREAD_LIMIT"); | |
| uint32_t ThreadLimit = GenericDevice.getThreadLimit(); | |
| if (OMP_TeamsThreadLimit > 0) | |
| return std::min(ThreadLimit, uint32_t(OMP_TeamsThreadLimit)); | |
| return ThreadLimit; | |
| } | |
| /// Map global data and execute pending ctors | |
| int PluginManager::loadImagesOntoDevice(DeviceTy &Device) { | |
| /* | |
| * Map global data | |
| */ | |
| int32_t DeviceId = Device.DeviceID; | |
| int Rc = OFFLOAD_SUCCESS; | |
| { | |
| std::lock_guard<decltype(TrlTblMtx)> LG(TrlTblMtx); | |
| for (auto *HostEntriesBegin : HostEntriesBeginRegistrationOrder) { | |
| TranslationTable *TransTable = | |
| &HostEntriesBeginToTransTable[HostEntriesBegin]; | |
| ODBG(ODT_Init) << "Trans table " << TransTable->HostTable.EntriesBegin | |
| << " : " << TransTable->HostTable.EntriesEnd; | |
| if (TransTable->HostTable.EntriesBegin == | |
| TransTable->HostTable.EntriesEnd) { | |
| // No host entry so no need to proceed | |
| continue; | |
| } | |
| if (TransTable->TargetsTable[DeviceId] != 0) { | |
| // Library entries have already been processed | |
| continue; | |
| } | |
| // 1) get image. | |
| assert(TransTable->TargetsImages.size() > (size_t)DeviceId && | |
| "Not expecting a device ID outside the table's bounds!"); | |
| __tgt_device_image *Img = TransTable->TargetsImages[DeviceId]; | |
| if (!Img) { | |
| REPORT() << "No image loaded for device id " << DeviceId << "."; | |
| Rc = OFFLOAD_FAIL; | |
| break; | |
| } | |
| // 2) Load the image onto the given device. | |
| auto BinaryOrErr = Device.loadBinary(Img); | |
| if (llvm::Error Err = BinaryOrErr.takeError()) { | |
| REPORT() << "Failed to load image " << llvm::toString(std::move(Err)); | |
| Rc = OFFLOAD_FAIL; | |
| break; | |
| } | |
| // 3) Create the translation table. | |
| llvm::SmallVector<llvm::offloading::EntryTy> &DeviceEntries = | |
| TransTable->TargetsEntries[DeviceId]; | |
| for (llvm::offloading::EntryTy &Entry : | |
| llvm::make_range(Img->EntriesBegin, Img->EntriesEnd)) { | |
| if (Entry.Kind != object::OffloadKind::OFK_OpenMP) | |
| continue; | |
| __tgt_device_binary &Binary = *BinaryOrErr; | |
| llvm::offloading::EntryTy DeviceEntry = Entry; | |
| if (Entry.Size) { | |
| if (!(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT_VTABLE)) | |
| if (Device.RTL->get_global(Binary, Entry.Size, Entry.SymbolName, | |
| &DeviceEntry.Address) != OFFLOAD_SUCCESS) | |
| REPORT() << "Failed to load symbol " << Entry.SymbolName; | |
| // If unified memory is active, the corresponding global is a device | |
| // reference to the host global. We need to initialize the pointer on | |
| // the device to point to the memory on the host. | |
| if (!(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT_VTABLE) && | |
| !(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT) && | |
| ((getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY) || | |
| (getRequirements() & OMPX_REQ_AUTO_ZERO_COPY))) | |
| if (Device.RTL->data_submit(DeviceId, DeviceEntry.Address, | |
| Entry.Address, | |
| Entry.Size) != OFFLOAD_SUCCESS) | |
| REPORT() << "Failed to write symbol for USM " << Entry.SymbolName; | |
| } else if (Entry.Address) { | |
| if (Device.RTL->get_function(Binary, Entry.SymbolName, | |
| &DeviceEntry.Address) != | |
| OFFLOAD_SUCCESS) { | |
| REPORT() << "Failed to load kernel " << Entry.SymbolName; | |
| } else { | |
| // Read this kernel's launch-geometry properties once, from its | |
| // "<name>_kernel_environment" global, and cache them on the device | |
| // for use at launch time. | |
| SmallString<128> EnvName(Entry.SymbolName); | |
| EnvName += "_kernel_environment"; | |
| llvm::omp::target::plugin::GenericDeviceTy &GenericDevice = | |
| Device.RTL->getDevice(Device.RTLDeviceID); | |
| KernelEnvironmentTy KernelEnv{}; | |
| llvm::omp::target::plugin::GlobalTy KernelEnvGlobal( | |
| EnvName, sizeof(KernelEnv), &KernelEnv); | |
| auto &Image = | |
| *reinterpret_cast<llvm::omp::target::plugin::DeviceImageTy *>( | |
| Binary.handle); | |
| if (auto Err = | |
| GenericDevice.Plugin.getGlobalHandler().readGlobalFromImage( | |
| GenericDevice, Image, KernelEnvGlobal)) { | |
| std::string ErrStr = toString(std::move(Err)); | |
| KernelEnv = KernelEnvironmentTy{}; | |
| KernelEnv.Configuration.ExecMode = | |
| llvm::omp::OMP_TGT_EXEC_MODE_BARE; | |
| ODBG(ODT_Mapping) | |
| << "Failed to read kernel environment for '" | |
| << Entry.SymbolName << "' (" << ErrStr << "), using default " | |
| << KernelLaunchInfoTy::getExecutionModeName( | |
| static_cast<llvm::omp::OMPTgtExecModeFlags>( | |
| KernelEnv.Configuration.ExecMode)) | |
| << " execution mode"; | |
| } | |
| auto *Kernel = | |
| reinterpret_cast<llvm::omp::target::plugin::GenericKernelTy *>( | |
| DeviceEntry.Address); | |
| const auto &Cfg = KernelEnv.Configuration; | |
| KernelLaunchInfoTy LaunchInfo; | |
| LaunchInfo.Mode = | |
| static_cast<llvm::omp::OMPTgtExecModeFlags>(Cfg.ExecMode); | |
| LaunchInfo.ReductionDataSize = Cfg.ReductionDataSize; | |
| LaunchInfo.StaticBlockMemSize = Kernel->getStaticBlockMemSize(); | |
| // Max = Config.Max > 0 ? min(Config.Max, Device.Max) : Device.Max, | |
| // further clamped to the kernel function's own driver-reported | |
| // maximum. | |
| uint32_t DeviceThreadLimit = getDeviceThreadLimit(GenericDevice); | |
| LaunchInfo.MaxNumThreads = | |
| std::min(Cfg.MaxThreads > 0 ? std::min(uint32_t(Cfg.MaxThreads), | |
| DeviceThreadLimit) | |
| : DeviceThreadLimit, | |
| Kernel->getMaxThreads()); | |
| LaunchInfo.PreferredNumThreads = | |
| Cfg.MinThreads > 0 | |
| ? std::max(Cfg.MinThreads, | |
| int32_t(GenericDevice.getDefaultNumThreads())) | |
| : GenericDevice.getDefaultNumThreads(); | |
| Device.setKernelLaunchInfo(DeviceEntry.Address, LaunchInfo); | |
| } | |
| } | |
| ODBG(ODT_Mapping) << "Entry point " << Entry.Address << " maps to" | |
| << (Entry.Size ? " global" : "") << " " | |
| << Entry.SymbolName << " (" << DeviceEntry.Address | |
| << ")"; | |
| DeviceEntries.emplace_back(DeviceEntry); | |
| } | |
| // Set the storage for the table and get a pointer to it. | |
| __tgt_target_table DeviceTable{&DeviceEntries[0], | |
| &DeviceEntries[0] + DeviceEntries.size()}; | |
| TransTable->DeviceTables[DeviceId] = DeviceTable; | |
| __tgt_target_table *TargetTable = TransTable->TargetsTable[DeviceId] = | |
| &TransTable->DeviceTables[DeviceId]; | |
| MappingInfoTy::HDTTMapAccessorTy HDTTMap = | |
| Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor(); | |
| __tgt_target_table *HostTable = &TransTable->HostTable; | |
| for (llvm::offloading::EntryTy * | |
| CurrDeviceEntry = TargetTable->EntriesBegin, | |
| *CurrHostEntry = HostTable->EntriesBegin, | |
| *EntryDeviceEnd = TargetTable->EntriesEnd; | |
| CurrDeviceEntry != EntryDeviceEnd; | |
| CurrDeviceEntry++, CurrHostEntry++) { | |
| if (CurrDeviceEntry->Size == 0 || | |
| CurrDeviceEntry->Kind != object::OffloadKind::OFK_OpenMP) | |
| continue; | |
| assert(CurrDeviceEntry->Size == CurrHostEntry->Size && | |
| "data size mismatch"); | |
| // Fortran may use multiple weak declarations for the same symbol, | |
| // therefore we must allow for multiple weak symbols to be loaded from | |
| // the fat binary. Treat these mappings as any other "regular" | |
| // mapping. Add entry to map. | |
| if (Device.getMappingInfo().getTgtPtrBegin( | |
| HDTTMap, CurrHostEntry->Address, CurrHostEntry->Size)) | |
| continue; | |
| void *CurrDeviceEntryAddr = CurrDeviceEntry->Address; | |
| // For indirect mapping, follow the indirection and map the actual | |
| // target. | |
| if (CurrDeviceEntry->Flags & OMP_DECLARE_TARGET_INDIRECT) { | |
| AsyncInfoTy AsyncInfo(Device); | |
| void *DevPtr; | |
| Device.retrieveData(&DevPtr, CurrDeviceEntryAddr, sizeof(void *), | |
| AsyncInfo, /*Entry=*/nullptr, &HDTTMap); | |
| if (AsyncInfo.synchronize() != OFFLOAD_SUCCESS) | |
| return OFFLOAD_FAIL; | |
| CurrDeviceEntryAddr = DevPtr; | |
| } | |
| ODBG(ODT_Mapping) << "Add mapping from host " << CurrHostEntry->Address | |
| << " to device " << CurrDeviceEntry->Address | |
| << " with size " << CurrDeviceEntry->Size | |
| << ", name \"" << CurrDeviceEntry->SymbolName << "\""; | |
| HDTTMap->emplace(new HostDataToTargetTy( | |
| (uintptr_t)CurrHostEntry->Address /*HstPtrBase*/, | |
| (uintptr_t)CurrHostEntry->Address /*HstPtrBegin*/, | |
| (uintptr_t)CurrHostEntry->Address + | |
| CurrHostEntry->Size /*HstPtrEnd*/, | |
| (uintptr_t)CurrDeviceEntryAddr /*TgtAllocBegin*/, | |
| (uintptr_t)CurrDeviceEntryAddr /*TgtPtrBegin*/, | |
| false /*UseHoldRefCount*/, CurrHostEntry->SymbolName, | |
| true /*IsRefCountINF*/)); | |
| // Notify about the new mapping. | |
| if (Device.notifyDataMapped(CurrHostEntry->Address, | |
| CurrHostEntry->Size)) | |
| return OFFLOAD_FAIL; | |
| } | |
| } | |
| Device.setHasPendingImages(false); | |
| } | |
| if (Rc != OFFLOAD_SUCCESS) | |
| return Rc; | |
| static Int32Envar DumpOffloadEntries = | |
| Int32Envar("OMPTARGET_DUMP_OFFLOAD_ENTRIES", -1); | |
| if (DumpOffloadEntries.get() == DeviceId) | |
| Device.dumpOffloadEntries(); | |
| return OFFLOAD_SUCCESS; | |
| } | |
| Expected<DeviceTy &> PluginManager::getDevice(uint32_t DeviceNo) { | |
| DeviceTy *DevicePtr; | |
| { | |
| auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor(); | |
| if (DeviceNo >= ExclusiveDevicesAccessor->size()) | |
| return createError( | |
| ErrorCode::InvalidValue, | |
| "device number '%i' out of range, only %i devices available", | |
| DeviceNo, ExclusiveDevicesAccessor->size()); | |
| DevicePtr = &*(*ExclusiveDevicesAccessor)[DeviceNo]; | |
| } | |
| // Check whether global data has been mapped for this device | |
| if (DevicePtr->hasPendingImages()) | |
| if (loadImagesOntoDevice(*DevicePtr) != OFFLOAD_SUCCESS) | |
| return createError(ErrorCode::BackendFailure, | |
| "failed to load images on device '%i'", DeviceNo); | |
| return *DevicePtr; | |
| } | |
| int PluginManager::getNumActivePlugins() const { | |
| int count = 0; | |
| if (auto Err = iteratePlatforms( | |
| [](ol_platform_handle_t Platform, void *Data) { | |
| bool Active = false; | |
| if (olGetPlatformInfo(Platform, OL_PLATFORM_INFO_ACTIVE, | |
| sizeof(Active), &Active) == OL_SUCCESS && | |
| Active) | |
| ++(*static_cast<int *>(Data)); | |
| }, | |
| static_cast<void *>(&count))) { | |
| consumeError(std::move(Err)); | |
| } | |
| return count; | |
| } |