offload/plugins-nextgen/common/src/GlobalHandler.cpp

//===- GlobalHandler.cpp - Target independent global & env. var handling --===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Target independent global handler and environment manager.
//
//===----------------------------------------------------------------------===//

#include "GlobalHandler.h"
#include "PluginInterface.h"
#include "Utils/ELF.h"

#include "Shared/Utils.h"

#include "llvm/ProfileData/InstrProfData.inc"
#include "llvm/Support/Error.h"

#include <cstring>
#include <string>

using namespace llvm;
using namespace omp;
using namespace target;
using namespace plugin;

Expected<std::unique_ptr<ObjectFile>>
GenericGlobalHandlerTy::getELFObjectFile(DeviceImageTy &Image) {
  assert(utils::elf::isELF(Image.getMemoryBuffer().getBuffer()) &&
         "Input is not an ELF file");

  return ELFObjectFileBase::createELFObjectFile(Image.getMemoryBuffer());
}

Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost(
    GenericDeviceTy &Device, DeviceImageTy &Image, const GlobalTy &HostGlobal,
    bool Device2Host) {

  GlobalTy DeviceGlobal(HostGlobal.getName(), HostGlobal.getSize());

  // Get the metadata from the global on the device.
  if (auto Err = getGlobalMetadataFromDevice(Device, Image, DeviceGlobal))
    return Err;

  // Perform the actual transfer.
  return moveGlobalBetweenDeviceAndHost(Device, HostGlobal, DeviceGlobal,
                                        Device2Host);
}

/// Actually move memory between host and device. See readGlobalFromDevice and
/// writeGlobalToDevice for the interface description.
Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost(
    GenericDeviceTy &Device, const GlobalTy &HostGlobal,
    const GlobalTy &DeviceGlobal, bool Device2Host) {

  // Transfer the data from the source to the destination.
  if (Device2Host) {
    if (auto Err =
            Device.dataRetrieve(HostGlobal.getPtr(), DeviceGlobal.getPtr(),
                                HostGlobal.getSize(), nullptr))
      return Err;
  } else {
    if (auto Err = Device.dataSubmit(DeviceGlobal.getPtr(), HostGlobal.getPtr(),
                                     HostGlobal.getSize(), nullptr))
      return Err;
  }

  DP("Successfully %s %u bytes associated with global symbol '%s' %s the "
     "device "
     "(%p -> %p).\n",
     Device2Host ? "read" : "write", HostGlobal.getSize(),
     HostGlobal.getName().data(), Device2Host ? "from" : "to",
     DeviceGlobal.getPtr(), HostGlobal.getPtr());

  return Plugin::success();
}

bool GenericGlobalHandlerTy::isSymbolInImage(GenericDeviceTy &Device,
                                             DeviceImageTy &Image,
                                             StringRef SymName) {
  // Get the ELF object file for the image. Notice the ELF object may already
  // be created in previous calls, so we can reuse it. If this is unsuccessful
  // just return false as we couldn't find it.
  auto ELFObjOrErr = getELFObjectFile(Image);
  if (!ELFObjOrErr) {
    consumeError(ELFObjOrErr.takeError());
    return false;
  }

  // Search the ELF symbol using the symbol name.
  auto SymOrErr = utils::elf::getSymbol(**ELFObjOrErr, SymName);
  if (!SymOrErr) {
    consumeError(SymOrErr.takeError());
    return false;
  }

  return SymOrErr->has_value();
}

Error GenericGlobalHandlerTy::getGlobalMetadataFromImage(
    GenericDeviceTy &Device, DeviceImageTy &Image, GlobalTy &ImageGlobal) {

  // Get the ELF object file for the image. Notice the ELF object may already
  // be created in previous calls, so we can reuse it.
  auto ELFObj = getELFObjectFile(Image);
  if (!ELFObj)
    return ELFObj.takeError();

  // Search the ELF symbol using the symbol name.
  auto SymOrErr = utils::elf::getSymbol(**ELFObj, ImageGlobal.getName());
  if (!SymOrErr)
    return Plugin::error("Failed ELF lookup of global '%s': %s",
                         ImageGlobal.getName().data(),
                         toString(SymOrErr.takeError()).data());

  if (!SymOrErr->has_value())
    return Plugin::error("Failed to find global symbol '%s' in the ELF image",
                         ImageGlobal.getName().data());

  auto AddrOrErr = utils::elf::getSymbolAddress(**SymOrErr);
  // Get the section to which the symbol belongs.
  if (!AddrOrErr)
    return Plugin::error("Failed to get ELF symbol from global '%s': %s",
                         ImageGlobal.getName().data(),
                         toString(AddrOrErr.takeError()).data());

  // Setup the global symbol's address and size.
  ImageGlobal.setPtr(const_cast<void *>(*AddrOrErr));
  ImageGlobal.setSize((*SymOrErr)->getSize());

  return Plugin::success();
}

Error GenericGlobalHandlerTy::readGlobalFromImage(GenericDeviceTy &Device,
                                                  DeviceImageTy &Image,
                                                  const GlobalTy &HostGlobal) {

  GlobalTy ImageGlobal(HostGlobal.getName(), -1);
  if (auto Err = getGlobalMetadataFromImage(Device, Image, ImageGlobal))
    return Err;

  if (ImageGlobal.getSize() != HostGlobal.getSize())
    return Plugin::error("Transfer failed because global symbol '%s' has "
                         "%u bytes in the ELF image but %u bytes on the host",
                         HostGlobal.getName().data(), ImageGlobal.getSize(),
                         HostGlobal.getSize());

  DP("Global symbol '%s' was found in the ELF image and %u bytes will copied "
     "from %p to %p.\n",
     HostGlobal.getName().data(), HostGlobal.getSize(), ImageGlobal.getPtr(),
     HostGlobal.getPtr());

  assert(Image.getStart() <= ImageGlobal.getPtr() &&
         utils::advancePtr(ImageGlobal.getPtr(), ImageGlobal.getSize()) <
             utils::advancePtr(Image.getStart(), Image.getSize()) &&
         "Attempting to read outside the image!");

  // Perform the copy from the image to the host memory.
  std::memcpy(HostGlobal.getPtr(), ImageGlobal.getPtr(), HostGlobal.getSize());

  return Plugin::success();
}

bool GenericGlobalHandlerTy::hasProfilingGlobals(GenericDeviceTy &Device,
                                                 DeviceImageTy &Image) {
  GlobalTy global(getInstrProfNamesVarName().str(), 0);
  if (auto Err = getGlobalMetadataFromImage(Device, Image, global)) {
    consumeError(std::move(Err));
    return false;
  }
  return true;
}

Expected<GPUProfGlobals>
GenericGlobalHandlerTy::readProfilingGlobals(GenericDeviceTy &Device,
                                             DeviceImageTy &Image) {
  GPUProfGlobals DeviceProfileData;
  auto ObjFile = getELFObjectFile(Image);
  if (!ObjFile)
    return ObjFile.takeError();

  std::unique_ptr<ELFObjectFileBase> ELFObj(
      static_cast<ELFObjectFileBase *>(ObjFile->release()));
  DeviceProfileData.TargetTriple = ELFObj->makeTriple();

  // Iterate through elf symbols
  for (auto &Sym : ELFObj->symbols()) {
    auto NameOrErr = Sym.getName();
    if (!NameOrErr)
      return NameOrErr.takeError();

    // Check if given current global is a profiling global based
    // on name
    if (*NameOrErr == getInstrProfNamesVarName()) {
      // Read in profiled function names
      DeviceProfileData.NamesData = SmallVector<uint8_t>(Sym.getSize(), 0);
      GlobalTy NamesGlobal(NameOrErr->str(), Sym.getSize(),
                           DeviceProfileData.NamesData.data());
      if (auto Err = readGlobalFromDevice(Device, Image, NamesGlobal))
        return Err;
    } else if (NameOrErr->starts_with(getInstrProfCountersVarPrefix())) {
      // Read global variable profiling counts
      SmallVector<int64_t> Counts(Sym.getSize() / sizeof(int64_t), 0);
      GlobalTy CountGlobal(NameOrErr->str(), Sym.getSize(), Counts.data());
      if (auto Err = readGlobalFromDevice(Device, Image, CountGlobal))
        return Err;
      DeviceProfileData.Counts.append(std::move(Counts));
    } else if (NameOrErr->starts_with(getInstrProfDataVarPrefix())) {
      // Read profiling data for this global variable
      __llvm_profile_data Data{};
      GlobalTy DataGlobal(NameOrErr->str(), Sym.getSize(), &Data);
      if (auto Err = readGlobalFromDevice(Device, Image, DataGlobal))
        return Err;
      DeviceProfileData.Data.push_back(std::move(Data));
    } else if (*NameOrErr == INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)) {
      uint64_t RawVersionData;
      GlobalTy RawVersionGlobal(NameOrErr->str(), Sym.getSize(),
                                &RawVersionData);
      if (auto Err = readGlobalFromDevice(Device, Image, RawVersionGlobal))
        return Err;
      DeviceProfileData.Version = RawVersionData;
    }
  }
  return DeviceProfileData;
}

void GPUProfGlobals::dump() const {
  outs() << "======= GPU Profile =======\nTarget: " << TargetTriple.str()
         << "\n";

  outs() << "======== Counters =========\n";
  for (size_t i = 0; i < Counts.size(); i++) {
    if (i > 0 && i % 10 == 0)
      outs() << "\n";
    else if (i != 0)
      outs() << " ";
    outs() << Counts[i];
  }
  outs() << "\n";

  outs() << "========== Data ===========\n";
  for (const auto &ProfData : Data) {
    outs() << "{ ";
// The ProfData.Name maybe array, eg: NumValueSites[IPVK_Last+1] .
// If we print out it directly, we are accessing out of bound data.
// Skip dumping the array for now.
#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)                     \
  if (sizeof(#Name) > 2 && #Name[sizeof(#Name) - 2] == ']') {                  \
    outs() << "[...] ";                                                        \
  } else {                                                                     \
    outs() << ProfData.Name << " ";                                            \
  }
#include "llvm/ProfileData/InstrProfData.inc"
    outs() << "}\n";
  }

  outs() << "======== Functions ========\n";
  std::string s;
  s.reserve(NamesData.size());
  for (uint8_t Name : NamesData) {
    s.push_back((char)Name);
  }

  InstrProfSymtab Symtab;
  if (Error Err = Symtab.create(StringRef(s))) {
    consumeError(std::move(Err));
  }
  Symtab.dumpNames(outs());
  outs() << "===========================\n";
}

Error GPUProfGlobals::write() const {
  if (!__llvm_write_custom_profile)
    return Plugin::error("Could not find symbol __llvm_write_custom_profile. "
                         "The compiler-rt profiling library must be linked for "
                         "GPU PGO to work.");

  size_t DataSize = Data.size() * sizeof(__llvm_profile_data),
         CountsSize = Counts.size() * sizeof(int64_t);
  __llvm_profile_data *DataBegin, *DataEnd;
  char *CountersBegin, *CountersEnd, *NamesBegin, *NamesEnd;

  // Initialize array of contiguous data. We need to make sure each section is
  // contiguous so that the PGO library can compute deltas properly
  SmallVector<uint8_t> ContiguousData(NamesData.size() + DataSize + CountsSize);

  // Compute region pointers
  DataBegin = (__llvm_profile_data *)(ContiguousData.data() + CountsSize);
  DataEnd =
      (__llvm_profile_data *)(ContiguousData.data() + CountsSize + DataSize);
  CountersBegin = (char *)ContiguousData.data();
  CountersEnd = (char *)(ContiguousData.data() + CountsSize);
  NamesBegin = (char *)(ContiguousData.data() + CountsSize + DataSize);
  NamesEnd = (char *)(ContiguousData.data() + CountsSize + DataSize +
                      NamesData.size());

  // Copy data to contiguous buffer
  memcpy(DataBegin, Data.data(), DataSize);
  memcpy(CountersBegin, Counts.data(), CountsSize);
  memcpy(NamesBegin, NamesData.data(), NamesData.size());

  // Invoke compiler-rt entrypoint
  int result = __llvm_write_custom_profile(
      TargetTriple.str().c_str(), DataBegin, DataEnd, CountersBegin,
      CountersEnd, NamesBegin, NamesEnd, &Version);
  if (result != 0)
    return Plugin::error("Error writing GPU PGO data to file");

  return Plugin::success();
}