offload/plugins-nextgen/level_zero/src/L0Program.cpp

//===--- Level Zero Target RTL Implementation -----------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Level Zero Program abstraction.
//
//===----------------------------------------------------------------------===//

#include "L0Plugin.h"
#include "L0Program.h"

namespace llvm::omp::target::plugin {

Error L0GlobalHandlerTy::getGlobalMetadataFromDevice(GenericDeviceTy &Device,
                                                     DeviceImageTy &Image,
                                                     GlobalTy &DeviceGlobal) {
  const char *GlobalName = DeviceGlobal.getName().data();
  size_t SymbolSize = 0;
  void *SymbolAddr = nullptr;

  L0ProgramTy &Program = L0ProgramTy::makeL0Program(Image);
  if (auto Err =
          Program.getSymbolMetadata(GlobalName, &SymbolAddr, &SymbolSize))
    return Err;

  // Save the pointer to the symbol allowing nullptr.
  DeviceGlobal.setPtr(SymbolAddr);
  DeviceGlobal.setSize(SymbolSize);

  return Plugin::success();
}

bool L0GlobalHandlerTy::isExportedSymbol(uint32_t Flags) {
  // Images returned by the Level Zero runtime do not correctly expose kernel
  // functions as global symbols. Bypass the normal ELF handling.here.
  uint32_t Ignored = SymbolRef::SF_Undefined | SymbolRef::SF_Hidden |
                     SymbolRef::SF_FormatSpecific;
  return !(Flags & Ignored);
}

inline L0DeviceTy &L0ProgramTy::getL0Device() const {
  return L0DeviceTy::makeL0Device(getDevice());
}

Error L0ProgramTy::deinit() {
  for (auto *Kernel : Kernels) {
    if (auto Err = Kernel->deinit())
      return Err;
    getL0Device().getPlugin().free(Kernel);
  }
  for (auto Module : Modules) {
    CALL_ZE_RET_ERROR(zeModuleDestroy, Module);
  }
  return Plugin::success();
}

/// Print the contents of a module build or link log. Unless \p FullLog is set,
/// only the first few lines are printed.
static void printBuildLog(ze_module_build_log_handle_t Log, const char *Title,
                          bool FullLog) {
  constexpr size_t MaxLines = 10;

  MESSAGE("%s:", Title);
  size_t LogSize = 0;
  ze_result_t RC;
  CALL_ZE(RC, zeModuleBuildLogGetString, Log, &LogSize, /*LogString=*/nullptr);
  if (RC != ZE_RESULT_SUCCESS) {
    MESSAGE0("  <failed to get build log>");
    return;
  }
  if (LogSize <= 1) {
    MESSAGE0("  <empty>");
    return;
  }
  std::string LogString(LogSize, '\0');
  CALL_ZE(RC, zeModuleBuildLogGetString, Log, &LogSize, LogString.data());
  if (RC != ZE_RESULT_SUCCESS) {
    MESSAGE0("  <failed to get build log>");
    return;
  }

  StringRef Rest(LogString.c_str());
  for (size_t NumLines = 0; !Rest.empty(); ++NumLines) {
    if (!FullLog && NumLines >= MaxLines) {
      MESSAGE0("  (suppressed remaining log)");
      break;
    }
    StringRef Line;
    std::tie(Line, Rest) = Rest.split('\n');
    MESSAGE("  '%.*s'", static_cast<int>(Line.size()), Line.data());
  }
}

Error L0ProgramBuilderTy::addModule(size_t Size, const uint8_t *Image,
                                    const std::string_view CommonBuildOptions,
                                    ze_module_format_t Format) {
  auto &L0Device = getL0Device();
  const ze_module_constants_t SpecConstants =
      L0Device.getPlugin()
          .getOptions()
          .CommonSpecConstants.getModuleConstants();

  std::string BuildOptions(CommonBuildOptions);

  bool IsLibModule =
      BuildOptions.find("-library-compilation") != std::string::npos;

  ze_module_desc_t ModuleDesc{};
  ModuleDesc.stype = ZE_STRUCTURE_TYPE_MODULE_DESC;
  ModuleDesc.pNext = nullptr;
  ModuleDesc.format = Format;
  ze_module_handle_t Module = nullptr;
  ze_module_build_log_handle_t BuildLog = nullptr;

  // Build a single module from a single image.
  ModuleDesc.inputSize = Size;
  ModuleDesc.pInputModule = Image;
  ModuleDesc.pBuildFlags = BuildOptions.c_str();
  ModuleDesc.pConstants = &SpecConstants;
  ze_result_t RC;
  CALL_ZE(RC, zeModuleCreate, getZeContext(), L0Device.getZeDevice(),
          &ModuleDesc, &Module, &BuildLog);
  if (BuildLog) {
    const bool BuildFailed = RC != ZE_RESULT_SUCCESS;
    const bool ShowBuildLog =
        L0Device.getPlugin().getOptions().Flags.ShowBuildLog;
    // Only show the log of library modules (-library-compilation) if their
    // build failed.
    if (BuildFailed || (ShowBuildLog && !IsLibModule)) {
      if (BuildFailed)
        MESSAGE0("Error: module creation failed");
      printBuildLog(BuildLog, "Target build log", ShowBuildLog);
    }
    zeModuleBuildLogDestroy(BuildLog);
  }
  if (RC != ZE_RESULT_SUCCESS) {
    // zeModuleCreate compiles/loads the provided image, so a build failure here
    // means the image itself could not be loaded for this device (e.g. a
    // truncated or malformed binary) rather than a generic JIT failure of an
    // otherwise valid program. Report it as INVALID_BINARY in that case (as
    // opposed to the default mapping of ZE_RESULT_ERROR_MODULE_BUILD_FAILURE
    // to ErrorCode::COMPILE_FAILURE).
    const auto ErrCode = RC == ZE_RESULT_ERROR_MODULE_BUILD_FAILURE
                             ? ErrorCode::INVALID_BINARY
                             : getOffloadErrorCode(RC);
    return Plugin::error(ErrCode, "zeModuleCreate failed with error %d, %s", RC,
                         getZeErrorName(RC));
  }

  // Check if module link is required. We do not need this check for
  // library module.
  if (!RequiresModuleLink && !IsLibModule) {
    ze_module_properties_t Properties = {ZE_STRUCTURE_TYPE_MODULE_PROPERTIES,
                                         nullptr, 0};
    ze_result_t RC;
    CALL_ZE(RC, zeModuleGetProperties, Module, &Properties);
    if (RC == ZE_RESULT_SUCCESS)
      RequiresModuleLink = Properties.flags & ZE_MODULE_PROPERTY_FLAG_IMPORTS;
  }
  // For now, assume the first module contains libraries, globals.
  if (Modules.empty())
    GlobalModule = Module;
  Modules.push_back(Module);
  L0Device.addGlobalModule(Module);
  return Plugin::success();
}

Error L0ProgramBuilderTy::linkModules() {
  auto &L0Device = getL0Device();
  if (!RequiresModuleLink) {
    ODBG(OLDT_Module) << "Module link is not required";
    return Plugin::success();
  }

  if (Modules.empty())
    return Plugin::error(ErrorCode::UNKNOWN,
                         "Invalid number of modules when linking modules");

  ze_module_build_log_handle_t LinkLog = nullptr;
  ze_result_t RC;
  CALL_ZE(RC, zeModuleDynamicLink,
          static_cast<uint32_t>(L0Device.getNumGlobalModules()),
          L0Device.getGlobalModulesArray(), &LinkLog);
  if (LinkLog) {
    const bool LinkFailed = RC != ZE_RESULT_SUCCESS;
    const bool ShowBuildLog =
        L0Device.getPlugin().getOptions().Flags.ShowBuildLog;
    if (LinkFailed || ShowBuildLog) {
      if (LinkFailed)
        MESSAGE0("Error: module link failed");
      printBuildLog(LinkLog, "Target link log", ShowBuildLog);
    }
    zeModuleBuildLogDestroy(LinkLog);
  }
  if (RC != ZE_RESULT_SUCCESS)
    return Plugin::error(getOffloadErrorCode(RC),
                         "zeModuleDynamicLink failed with error %d, %s", RC,
                         getZeErrorName(RC));
  return Plugin::success();
}

static void replaceDriverOptsWithBackendOpts(const L0DeviceTy &Device,
                                             std::string &Options) {
  // Options that need to be replaced with backend-specific options
  static const struct {
    std::string Option;
    std::string BackendOption;
  } OptionTranslationTable[] = {
      {"-ftarget-compile-fast",
       "-igc_opts 'PartitionUnit=1,SubroutineThreshold=50000'"},
      {"-foffload-fp32-prec-div", "-ze-fp32-correctly-rounded-divide-sqrt"},
      {"-foffload-fp32-prec-sqrt", "-ze-fp32-correctly-rounded-divide-sqrt"},
  };

  for (const auto &OptPair : OptionTranslationTable) {
    const size_t Pos = Options.find(OptPair.Option);
    if (Pos != std::string::npos)
      Options.replace(Pos, OptPair.Option.length(), OptPair.BackendOption);
  }
}

Error L0ProgramBuilderTy::buildModules(const std::string_view BuildOptions) {
  auto &L0Device = getL0Device();
  auto Image = getMemoryBuffer();

  // Check if image is an inner OffloadBinary (nested format)
  if (identify_magic(Image.getBuffer()) == file_magic::offload_binary) {
    ODBG(OLDT_Module) << "Processing nested OffloadBinary image";

    // Parse inner OffloadBinary
    auto InnerBinariesOrErr = llvm::object::OffloadBinary::create(Image);
    if (!InnerBinariesOrErr)
      return Plugin::error(
          ErrorCode::INVALID_BINARY, "Failed to parse inner OffloadBinary: %s",
          llvm::toString(InnerBinariesOrErr.takeError()).c_str());

    auto &InnerBinaries = *InnerBinariesOrErr;

    // Should contain exactly one image
    if (InnerBinaries.size() != 1)
      return Plugin::error(ErrorCode::INVALID_BINARY,
                           "Expected single inner OffloadBinary entry, got %zu",
                           InnerBinaries.size());

    const llvm::object::OffloadBinary *InnerBinary = InnerBinaries[0].get();
    llvm::object::ImageKind ImageKind = InnerBinary->getImageKind();

    // Extract image data from inner binary
    llvm::StringRef ImageData = InnerBinary->getImage();
    const uint8_t *ImgBegin =
        reinterpret_cast<const uint8_t *>(ImageData.data());

    // Read metadata from inner binary
    llvm::StringRef Version = InnerBinary->getString("version");
    llvm::StringRef CompileOpts = InnerBinary->getString("compile-opts");
    llvm::StringRef LinkOpts = InnerBinary->getString("link-opts");

    ODBG(OLDT_Module) << "Inner OffloadBinary metadata: version=" << Version
                      << ", kind=" << ImageKind;

    // Build options string combining BuildOptions with compile/link opts
    std::string Options(BuildOptions);
    if (!CompileOpts.empty() || !LinkOpts.empty()) {
      if (!CompileOpts.empty())
        Options += " " + CompileOpts.str();
      if (!LinkOpts.empty())
        Options += " " + LinkOpts.str();
      replaceDriverOptsWithBackendOpts(L0Device, Options);
      ODBG(OLDT_Module) << "Using compile options: " << CompileOpts
                        << ", link options: " << LinkOpts;
    }

    // Determine module format based on image kind
    ze_module_format_t ModuleFormat;
    if (ImageKind == llvm::object::IMG_SPIRV) {
      // SPIR-V intermediate language
      ODBG(OLDT_Module) << "Loading SPIR-V module";
      ModuleFormat = ZE_MODULE_FORMAT_IL_SPIRV;
    } else if (ImageKind == llvm::object::IMG_Object) {
      // Native binary format
      ODBG(OLDT_Module) << "Loading native binary module";
      ModuleFormat = ZE_MODULE_FORMAT_NATIVE;
    } else {
      return Plugin::error(ErrorCode::INVALID_BINARY,
                           "Unsupported image kind %d in inner OffloadBinary",
                           static_cast<int>(ImageKind));
    }

    // Load module into Level Zero
    auto Err = addModule(ImageData.size(), ImgBegin, Options, ModuleFormat);
    if (Err)
      return Err;

    if (RequiresModuleLink) {
      ODBG(OLDT_Module) << "Linking modules after adding OffloadBinary image";
      if (auto Err = linkModules())
        return Err;
    }
    return Plugin::success();
  }

  if (identify_magic(Image.getBuffer()) == file_magic::spirv_object) {
    ODBG(OLDT_Module) << "Processing raw SPIR-V image";
    const uint8_t *ImgBegin =
        reinterpret_cast<const uint8_t *>(Image.getBufferStart());
    auto Err = addModule(Image.getBufferSize(), ImgBegin, BuildOptions,
                         ZE_MODULE_FORMAT_IL_SPIRV);
    if (Err)
      return Err;

    if (RequiresModuleLink) {
      ODBG(OLDT_Module) << "Linking modules after adding SPIR-V image";
      if (auto Err = linkModules())
        return Err;
    }
    return Plugin::success();
  }

  return Plugin::error(ErrorCode::INVALID_BINARY,
                       "Unsupported image format for L0 plugin");
}

Expected<std::unique_ptr<MemoryBuffer>> L0ProgramBuilderTy::getELF() {
  assert(GlobalModule != nullptr && "GlobalModule is null");

  size_t Size = 0;

  CALL_ZE_RET_ERROR(zeModuleGetNativeBinary, GlobalModule, &Size, nullptr);
  std::vector<uint8_t> ELFData(Size);
  CALL_ZE_RET_ERROR(zeModuleGetNativeBinary, GlobalModule, &Size,
                    ELFData.data());
  return MemoryBuffer::getMemBufferCopy(
      StringRef(reinterpret_cast<const char *>(ELFData.data()), Size),
      /*BufferName=*/"L0Program ELF");
}

Error L0ProgramTy::getSymbolMetadata(const char *Name, void **AddrPtr,
                                     size_t *SizePtr) const {
  if (!Name || !AddrPtr || !SizePtr)
    return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                         "Invalid arguments to getSymbolDeviceAddr");

  size_t SymbolSize = 0;
  void *SymbolAddr = nullptr;
  ze_result_t RC;
  for (auto Module : Modules) {
    CALL_ZE(RC, zeModuleGetGlobalPointer, Module, Name, &SymbolSize,
            &SymbolAddr);
    if (RC == ZE_RESULT_SUCCESS && SymbolAddr) {
      *AddrPtr = SymbolAddr;
      *SizePtr = SymbolSize;
      return Plugin::success();
    }
    CALL_ZE(RC, zeModuleGetFunctionPointer, Module, Name, &SymbolAddr);
    if (RC == ZE_RESULT_SUCCESS && SymbolAddr) {
      *AddrPtr = SymbolAddr;
      *SizePtr = 0;
      return Plugin::success();
    }
  }
  return Plugin::error(ErrorCode::NOT_FOUND, "symbol '%s' not found on device",
                       Name);
}

Error L0ProgramTy::readGlobalVariable(const char *Name, size_t Size,
                                      void *HostPtr) {
  size_t SizeDummy = 0;
  void *DevicePtr = nullptr;
  ze_result_t RC;
  CALL_ZE(RC, zeModuleGetGlobalPointer, GlobalModule, Name, &SizeDummy,
          &DevicePtr);
  if (RC != ZE_RESULT_SUCCESS || !DevicePtr) {
    return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                         "Cannot read from device global variable %s", Name);
  }
  return getL0Device().enqueueMemCopyAndSync(HostPtr, DevicePtr, Size);
}

Error L0ProgramTy::writeGlobalVariable(const char *Name, size_t Size,
                                       const void *HostPtr) {
  size_t SizeDummy = 0;
  void *DevicePtr = nullptr;
  ze_result_t RC;
  CALL_ZE(RC, zeModuleGetGlobalPointer, GlobalModule, Name, &SizeDummy,
          &DevicePtr);
  if (RC != ZE_RESULT_SUCCESS || !DevicePtr) {
    return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                         "Cannot write to device global variable %s", Name);
  }
  return getL0Device().enqueueMemCopyAndSync(DevicePtr, HostPtr, Size);
}

Error L0ProgramTy::loadModuleKernels() {
  // We need to build kernels here before filling the offload entries since we
  // don't know which module contains a specific kernel with a name.
  for (auto Module : Modules) {
    uint32_t Count = 0;
    CALL_ZE_RET_ERROR(zeModuleGetKernelNames, Module, &Count,
                      /*Names=*/nullptr);
    if (Count == 0)
      continue;

    llvm::SmallVector<const char *> Names(Count);
    CALL_ZE_RET_ERROR(zeModuleGetKernelNames, Module, &Count, Names.data());

    for (auto *Name : Names) {
      KernelsToModuleMap.emplace(Name, Module);
    }
  }

  return Plugin::success();
}

} // namespace llvm::omp::target::plugin