lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp - llvm-project/llvm - Git at Google

 //===-- PerfJITEventListener.cpp - Tell Linux's perf about JITted code ----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines a JITEventListener object that tells perf about JITted
 // functions, including source line information.
 //
 // Documentation for perf jit integration is available at:
 // https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jitdump-specification.txt
 // https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jit-interface.txt
 //
 //===----------------------------------------------------------------------===//

 #include "llvm/ADT/Twine.h"
 #include "llvm/Config/config.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/ExecutionEngine/JITEventListener.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Object/SymbolSize.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/raw_ostream.h"
 #include <mutex>

 #include <sys/mman.h>  // mmap()
 #include <time.h>      // clock_gettime(), time(), localtime_r() */
 #include <unistd.h>    // for read(), close()

 using namespace llvm;
 using namespace llvm::object;
 typedef DILineInfoSpecifier::FileLineInfoKind FileLineInfoKind;

 namespace {

 // language identifier (XXX: should we generate something better from debug
 // info?)
 #define JIT_LANG "llvm-IR"
 #define LLVM_PERF_JIT_MAGIC                                                    \
   ((uint32_t)'J' << 24 | (uint32_t)'i' << 16 | (uint32_t)'T' << 8 |            \
    (uint32_t)'D')
 #define LLVM_PERF_JIT_VERSION 1

 // bit 0: set if the jitdump file is using an architecture-specific timestamp
 // clock source
 #define JITDUMP_FLAGS_ARCH_TIMESTAMP (1ULL << 0)

 struct LLVMPerfJitHeader;

 class PerfJITEventListener : public JITEventListener {
 public:
   PerfJITEventListener();
   ~PerfJITEventListener() {
     if (MarkerAddr)
       CloseMarker();
   }

   void notifyObjectLoaded(ObjectKey K, const ObjectFile &Obj,
                           const RuntimeDyld::LoadedObjectInfo &L) override;
   void notifyFreeingObject(ObjectKey K) override;

 private:
   bool InitDebuggingDir();
   bool OpenMarker();
   void CloseMarker();
   static bool FillMachine(LLVMPerfJitHeader &hdr);

   void NotifyCode(Expected<llvm::StringRef> &Symbol, uint64_t CodeAddr,
                   uint64_t CodeSize);
   void NotifyDebug(uint64_t CodeAddr, DILineInfoTable Lines);

   // cache lookups
   sys::Process::Pid Pid;

   // base directory for output data
   std::string JitPath;

   // output data stream, closed via Dumpstream
   int DumpFd = -1;

   // output data stream
   std::unique_ptr<raw_fd_ostream> Dumpstream;

   // prevent concurrent dumps from messing up the output file
   sys::Mutex Mutex;

   // perf mmap marker
   void *MarkerAddr = NULL;

   // perf support ready
   bool SuccessfullyInitialized = false;

   // identifier for functions, primarily to identify when moving them around
   uint64_t CodeGeneration = 1;
 };

 // The following are POD struct definitions from the perf jit specification

 enum LLVMPerfJitRecordType {
   JIT_CODE_LOAD = 0,
   JIT_CODE_MOVE = 1, // not emitted, code isn't moved
   JIT_CODE_DEBUG_INFO = 2,
   JIT_CODE_CLOSE = 3,          // not emitted, unnecessary
   JIT_CODE_UNWINDING_INFO = 4, // not emitted

   JIT_CODE_MAX
 };

 struct LLVMPerfJitHeader {
   uint32_t Magic;     // characters "JiTD"
   uint32_t Version;   // header version
   uint32_t TotalSize; // total size of header
   uint32_t ElfMach;   // elf mach target
   uint32_t Pad1;      // reserved
   uint32_t Pid;
   uint64_t Timestamp; // timestamp
   uint64_t Flags;     // flags
 };

 // record prefix (mandatory in each record)
 struct LLVMPerfJitRecordPrefix {
   uint32_t Id; // record type identifier
   uint32_t TotalSize;
   uint64_t Timestamp;
 };

 struct LLVMPerfJitRecordCodeLoad {
   LLVMPerfJitRecordPrefix Prefix;

   uint32_t Pid;
   uint32_t Tid;
   uint64_t Vma;
   uint64_t CodeAddr;
   uint64_t CodeSize;
   uint64_t CodeIndex;
 };

 struct LLVMPerfJitDebugEntry {
   uint64_t Addr;
   int Lineno;  // source line number starting at 1
   int Discrim; // column discriminator, 0 is default
   // followed by null terminated filename, \xff\0 if same as previous entry
 };

 struct LLVMPerfJitRecordDebugInfo {
   LLVMPerfJitRecordPrefix Prefix;

   uint64_t CodeAddr;
   uint64_t NrEntry;
   // followed by NrEntry LLVMPerfJitDebugEntry records
 };

 static inline uint64_t timespec_to_ns(const struct timespec *ts) {
   const uint64_t NanoSecPerSec = 1000000000;
   return ((uint64_t)ts->tv_sec * NanoSecPerSec) + ts->tv_nsec;
 }

 static inline uint64_t perf_get_timestamp(void) {
   struct timespec ts;
   int ret;

   ret = clock_gettime(CLOCK_MONOTONIC, &ts);
   if (ret)
     return 0;

   return timespec_to_ns(&ts);
 }

 PerfJITEventListener::PerfJITEventListener()
     : Pid(sys::Process::getProcessId()) {
   // check if clock-source is supported
   if (!perf_get_timestamp()) {
     errs() << "kernel does not support CLOCK_MONOTONIC\n";
     return;
   }

   if (!InitDebuggingDir()) {
     errs() << "could not initialize debugging directory\n";
     return;
   }

   std::string Filename;
   raw_string_ostream FilenameBuf(Filename);
   FilenameBuf << JitPath << "/jit-" << Pid << ".dump";

   // Need to open ourselves, because we need to hand the FD to OpenMarker() and
   // raw_fd_ostream doesn't expose the FD.
   using sys::fs::openFileForWrite;
   if (auto EC =
           openFileForReadWrite(FilenameBuf.str(), DumpFd,
 			       sys::fs::CD_CreateNew, sys::fs::OF_None)) {
     errs() << "could not open JIT dump file " << FilenameBuf.str() << ": "
            << EC.message() << "\n";
     return;
   }

   Dumpstream = std::make_unique<raw_fd_ostream>(DumpFd, true);

   LLVMPerfJitHeader Header = {0};
   if (!FillMachine(Header))
     return;

   // signal this process emits JIT information
   if (!OpenMarker())
     return;

   // emit dumpstream header
   Header.Magic = LLVM_PERF_JIT_MAGIC;
   Header.Version = LLVM_PERF_JIT_VERSION;
   Header.TotalSize = sizeof(Header);
   Header.Pid = Pid;
   Header.Timestamp = perf_get_timestamp();
   Dumpstream->write(reinterpret_cast<const char *>(&Header), sizeof(Header));

   // Everything initialized, can do profiling now.
   if (!Dumpstream->has_error())
     SuccessfullyInitialized = true;
 }

 void PerfJITEventListener::notifyObjectLoaded(
     ObjectKey K, const ObjectFile &Obj,
     const RuntimeDyld::LoadedObjectInfo &L) {

   if (!SuccessfullyInitialized)
     return;

   OwningBinary<ObjectFile> DebugObjOwner = L.getObjectForDebug(Obj);
   const ObjectFile &DebugObj = *DebugObjOwner.getBinary();

   // Get the address of the object image for use as a unique identifier
   std::unique_ptr<DIContext> Context = DWARFContext::create(DebugObj);

   // Use symbol info to iterate over functions in the object.
   for (const std::pair<SymbolRef, uint64_t> &P : computeSymbolSizes(DebugObj)) {
     SymbolRef Sym = P.first;
     std::string SourceFileName;

     Expected<SymbolRef::Type> SymTypeOrErr = Sym.getType();
     if (!SymTypeOrErr) {
       // There's not much we can with errors here
       consumeError(SymTypeOrErr.takeError());
       continue;
     }
     SymbolRef::Type SymType = *SymTypeOrErr;
     if (SymType != SymbolRef::ST_Function)
       continue;

     Expected<StringRef> Name = Sym.getName();
     if (!Name) {
       consumeError(Name.takeError());
       continue;
     }

     Expected<uint64_t> AddrOrErr = Sym.getAddress();
     if (!AddrOrErr) {
       consumeError(AddrOrErr.takeError());
       continue;
     }
     uint64_t Size = P.second;
     object::SectionedAddress Address;
     Address.Address = *AddrOrErr;

     uint64_t SectionIndex = object::SectionedAddress::UndefSection;
     if (auto SectOrErr = Sym.getSection())
         if (*SectOrErr != Obj.section_end())
             SectionIndex = SectOrErr.get()->getIndex();

     // According to spec debugging info has to come before loading the
     // corresonding code load.
     DILineInfoTable Lines = Context->getLineInfoForAddressRange(
         {*AddrOrErr, SectionIndex}, Size, FileLineInfoKind::AbsoluteFilePath);

     NotifyDebug(*AddrOrErr, Lines);
     NotifyCode(Name, *AddrOrErr, Size);
   }

   Dumpstream->flush();
 }

 void PerfJITEventListener::notifyFreeingObject(ObjectKey K) {
   // perf currently doesn't have an interface for unloading. But munmap()ing the
   // code section does, so that's ok.
 }

 bool PerfJITEventListener::InitDebuggingDir() {
   time_t Time;
   struct tm LocalTime;
   char TimeBuffer[sizeof("YYYYMMDD")];
   SmallString<64> Path;

   // search for location to dump data to
   if (const char *BaseDir = getenv("JITDUMPDIR"))
     Path.append(BaseDir);
   else if (!sys::path::home_directory(Path))
     Path = ".";

   // create debug directory
   Path += "/.debug/jit/";
   if (auto EC = sys::fs::create_directories(Path)) {
     errs() << "could not create jit cache directory " << Path << ": "
            << EC.message() << "\n";
     return false;
   }

   // create unique directory for dump data related to this process
   time(&Time);
   localtime_r(&Time, &LocalTime);
   strftime(TimeBuffer, sizeof(TimeBuffer), "%Y%m%d", &LocalTime);
   Path += JIT_LANG "-jit-";
   Path += TimeBuffer;

   SmallString<128> UniqueDebugDir;

   using sys::fs::createUniqueDirectory;
   if (auto EC = createUniqueDirectory(Path, UniqueDebugDir)) {
     errs() << "could not create unique jit cache directory " << UniqueDebugDir
            << ": " << EC.message() << "\n";
     return false;
   }

   JitPath = std::string(UniqueDebugDir.str());

   return true;
 }

 bool PerfJITEventListener::OpenMarker() {
   // We mmap the jitdump to create an MMAP RECORD in perf.data file.  The mmap
   // is captured either live (perf record running when we mmap) or in deferred
   // mode, via /proc/PID/maps. The MMAP record is used as a marker of a jitdump
   // file for more meta data info about the jitted code. Perf report/annotate
   // detect this special filename and process the jitdump file.
   //
   // Mapping must be PROT_EXEC to ensure it is captured by perf record
   // even when not using -d option.
   MarkerAddr = ::mmap(NULL, sys::Process::getPageSizeEstimate(),
                       PROT_READ | PROT_EXEC, MAP_PRIVATE, DumpFd, 0);

   if (MarkerAddr == MAP_FAILED) {
     errs() << "could not mmap JIT marker\n";
     return false;
   }
   return true;
 }

 void PerfJITEventListener::CloseMarker() {
   if (!MarkerAddr)
     return;

   munmap(MarkerAddr, sys::Process::getPageSizeEstimate());
   MarkerAddr = nullptr;
 }

 bool PerfJITEventListener::FillMachine(LLVMPerfJitHeader &hdr) {
   char id[16];
   struct {
     uint16_t e_type;
     uint16_t e_machine;
   } info;

   size_t RequiredMemory = sizeof(id) + sizeof(info);

   ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
     MemoryBuffer::getFileSlice("/proc/self/exe",
 			       RequiredMemory,
 			       0);

   // This'll not guarantee that enough data was actually read from the
   // underlying file. Instead the trailing part of the buffer would be
   // zeroed. Given the ELF signature check below that seems ok though,
   // it's unlikely that the file ends just after that, and the
   // consequence would just be that perf wouldn't recognize the
   // signature.
   if (auto EC = MB.getError()) {
     errs() << "could not open /proc/self/exe: " << EC.message() << "\n";
     return false;
   }

   memcpy(&id, (*MB)->getBufferStart(), sizeof(id));
   memcpy(&info, (*MB)->getBufferStart() + sizeof(id), sizeof(info));

   // check ELF signature
   if (id[0] != 0x7f || id[1] != 'E' || id[2] != 'L' || id[3] != 'F') {
     errs() << "invalid elf signature\n";
     return false;
   }

   hdr.ElfMach = info.e_machine;

   return true;
 }

 void PerfJITEventListener::NotifyCode(Expected<llvm::StringRef> &Symbol,
                                       uint64_t CodeAddr, uint64_t CodeSize) {
   assert(SuccessfullyInitialized);

   // 0 length functions can't have samples.
   if (CodeSize == 0)
     return;

   LLVMPerfJitRecordCodeLoad rec;
   rec.Prefix.Id = JIT_CODE_LOAD;
   rec.Prefix.TotalSize = sizeof(rec) +        // debug record itself
                          Symbol->size() + 1 + // symbol name
                          CodeSize;            // and code
   rec.Prefix.Timestamp = perf_get_timestamp();

   rec.CodeSize = CodeSize;
   rec.Vma = 0;
   rec.CodeAddr = CodeAddr;
   rec.Pid = Pid;
   rec.Tid = get_threadid();

   // avoid interspersing output
   std::lock_guard<sys::Mutex> Guard(Mutex);

   rec.CodeIndex = CodeGeneration++; // under lock!

   Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec));
   Dumpstream->write(Symbol->data(), Symbol->size() + 1);
   Dumpstream->write(reinterpret_cast<const char *>(CodeAddr), CodeSize);
 }

 void PerfJITEventListener::NotifyDebug(uint64_t CodeAddr,
                                        DILineInfoTable Lines) {
   assert(SuccessfullyInitialized);

   // Didn't get useful debug info.
   if (Lines.empty())
     return;

   LLVMPerfJitRecordDebugInfo rec;
   rec.Prefix.Id = JIT_CODE_DEBUG_INFO;
   rec.Prefix.TotalSize = sizeof(rec); // will be increased further
   rec.Prefix.Timestamp = perf_get_timestamp();
   rec.CodeAddr = CodeAddr;
   rec.NrEntry = Lines.size();

   // compute total size size of record (variable due to filenames)
   DILineInfoTable::iterator Begin = Lines.begin();
   DILineInfoTable::iterator End = Lines.end();
   for (DILineInfoTable::iterator It = Begin; It != End; ++It) {
     DILineInfo &line = It->second;
     rec.Prefix.TotalSize += sizeof(LLVMPerfJitDebugEntry);
     rec.Prefix.TotalSize += line.FileName.size() + 1;
   }

   // The debug_entry describes the source line information. It is defined as
   // follows in order:
   // * uint64_t code_addr: address of function for which the debug information
   // is generated
   // * uint32_t line     : source file line number (starting at 1)
   // * uint32_t discrim  : column discriminator, 0 is default
   // * char name[n]      : source file name in ASCII, including null termination

   // avoid interspersing output
   std::lock_guard<sys::Mutex> Guard(Mutex);

   Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec));

   for (DILineInfoTable::iterator It = Begin; It != End; ++It) {
     LLVMPerfJitDebugEntry LineInfo;
     DILineInfo &Line = It->second;

     LineInfo.Addr = It->first;
     // The function re-created by perf is preceded by a elf
     // header. Need to adjust for that, otherwise the results are
     // wrong.
     LineInfo.Addr += 0x40;
     LineInfo.Lineno = Line.Line;
     LineInfo.Discrim = Line.Discriminator;

     Dumpstream->write(reinterpret_cast<const char *>(&LineInfo),
                       sizeof(LineInfo));
     Dumpstream->write(Line.FileName.c_str(), Line.FileName.size() + 1);
   }
 }

 // There should be only a single event listener per process, otherwise perf gets
 // confused.
 llvm::ManagedStatic<PerfJITEventListener> PerfListener;

 } // end anonymous namespace

 namespace llvm {
 JITEventListener *JITEventListener::createPerfJITEventListener() {
   return &*PerfListener;
 }

 } // namespace llvm

 LLVMJITEventListenerRef LLVMCreatePerfJITEventListener(void)
 {
   return wrap(JITEventListener::createPerfJITEventListener());
 }
	//===-- PerfJITEventListener.cpp - Tell Linux's perf about JITted code ----===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	//
	// This file defines a JITEventListener object that tells perf about JITted
	// functions, including source line information.
	//
	// Documentation for perf jit integration is available at:
	// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jitdump-specification.txt
	// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jit-interface.txt
	//
	//===----------------------------------------------------------------------===//

	#include "llvm/ADT/Twine.h"
	#include "llvm/Config/config.h"
	#include "llvm/DebugInfo/DWARF/DWARFContext.h"
	#include "llvm/ExecutionEngine/JITEventListener.h"
	#include "llvm/Object/ObjectFile.h"
	#include "llvm/Object/SymbolSize.h"
	#include "llvm/Support/Debug.h"
	#include "llvm/Support/Errno.h"
	#include "llvm/Support/FileSystem.h"
	#include "llvm/Support/ManagedStatic.h"
	#include "llvm/Support/MemoryBuffer.h"
	#include "llvm/Support/Mutex.h"
	#include "llvm/Support/Path.h"
	#include "llvm/Support/Process.h"
	#include "llvm/Support/Threading.h"
	#include "llvm/Support/raw_ostream.h"
	#include <mutex>

	#include <sys/mman.h> // mmap()
	#include <time.h> // clock_gettime(), time(), localtime_r() */
	#include <unistd.h> // for read(), close()

	using namespace llvm;
	using namespace llvm::object;
	typedef DILineInfoSpecifier::FileLineInfoKind FileLineInfoKind;

	namespace {

	// language identifier (XXX: should we generate something better from debug
	// info?)
	#define JIT_LANG "llvm-IR"
	#define LLVM_PERF_JIT_MAGIC \
	((uint32_t)'J' << 24 \| (uint32_t)'i' << 16 \| (uint32_t)'T' << 8 \| \
	(uint32_t)'D')
	#define LLVM_PERF_JIT_VERSION 1

	// bit 0: set if the jitdump file is using an architecture-specific timestamp
	// clock source
	#define JITDUMP_FLAGS_ARCH_TIMESTAMP (1ULL << 0)

	struct LLVMPerfJitHeader;

	class PerfJITEventListener : public JITEventListener {
	public:
	PerfJITEventListener();
	~PerfJITEventListener() {
	if (MarkerAddr)
	CloseMarker();
	}

	void notifyObjectLoaded(ObjectKey K, const ObjectFile &Obj,
	const RuntimeDyld::LoadedObjectInfo &L) override;
	void notifyFreeingObject(ObjectKey K) override;

	private:
	bool InitDebuggingDir();
	bool OpenMarker();
	void CloseMarker();
	static bool FillMachine(LLVMPerfJitHeader &hdr);

	void NotifyCode(Expected<llvm::StringRef> &Symbol, uint64_t CodeAddr,
	uint64_t CodeSize);
	void NotifyDebug(uint64_t CodeAddr, DILineInfoTable Lines);

	// cache lookups
	sys::Process::Pid Pid;

	// base directory for output data
	std::string JitPath;

	// output data stream, closed via Dumpstream
	int DumpFd = -1;

	// output data stream
	std::unique_ptr<raw_fd_ostream> Dumpstream;

	// prevent concurrent dumps from messing up the output file
	sys::Mutex Mutex;

	// perf mmap marker
	void *MarkerAddr = NULL;

	// perf support ready
	bool SuccessfullyInitialized = false;

	// identifier for functions, primarily to identify when moving them around
	uint64_t CodeGeneration = 1;
	};

	// The following are POD struct definitions from the perf jit specification

	enum LLVMPerfJitRecordType {
	JIT_CODE_LOAD = 0,
	JIT_CODE_MOVE = 1, // not emitted, code isn't moved
	JIT_CODE_DEBUG_INFO = 2,
	JIT_CODE_CLOSE = 3, // not emitted, unnecessary
	JIT_CODE_UNWINDING_INFO = 4, // not emitted

	JIT_CODE_MAX
	};

	struct LLVMPerfJitHeader {
	uint32_t Magic; // characters "JiTD"
	uint32_t Version; // header version
	uint32_t TotalSize; // total size of header
	uint32_t ElfMach; // elf mach target
	uint32_t Pad1; // reserved
	uint32_t Pid;
	uint64_t Timestamp; // timestamp
	uint64_t Flags; // flags
	};

	// record prefix (mandatory in each record)
	struct LLVMPerfJitRecordPrefix {
	uint32_t Id; // record type identifier
	uint32_t TotalSize;
	uint64_t Timestamp;
	};

	struct LLVMPerfJitRecordCodeLoad {
	LLVMPerfJitRecordPrefix Prefix;

	uint32_t Pid;
	uint32_t Tid;
	uint64_t Vma;
	uint64_t CodeAddr;
	uint64_t CodeSize;
	uint64_t CodeIndex;
	};

	struct LLVMPerfJitDebugEntry {
	uint64_t Addr;
	int Lineno; // source line number starting at 1
	int Discrim; // column discriminator, 0 is default
	// followed by null terminated filename, \xff\0 if same as previous entry
	};

	struct LLVMPerfJitRecordDebugInfo {
	LLVMPerfJitRecordPrefix Prefix;

	uint64_t CodeAddr;
	uint64_t NrEntry;
	// followed by NrEntry LLVMPerfJitDebugEntry records
	};

	static inline uint64_t timespec_to_ns(const struct timespec *ts) {
	const uint64_t NanoSecPerSec = 1000000000;
	return ((uint64_t)ts->tv_sec * NanoSecPerSec) + ts->tv_nsec;
	}

	static inline uint64_t perf_get_timestamp(void) {
	struct timespec ts;
	int ret;

	ret = clock_gettime(CLOCK_MONOTONIC, &ts);
	if (ret)
	return 0;

	return timespec_to_ns(&ts);
	}

	PerfJITEventListener::PerfJITEventListener()
	: Pid(sys::Process::getProcessId()) {
	// check if clock-source is supported
	if (!perf_get_timestamp()) {
	errs() << "kernel does not support CLOCK_MONOTONIC\n";
	return;
	}

	if (!InitDebuggingDir()) {
	errs() << "could not initialize debugging directory\n";
	return;
	}

	std::string Filename;
	raw_string_ostream FilenameBuf(Filename);
	FilenameBuf << JitPath << "/jit-" << Pid << ".dump";

	// Need to open ourselves, because we need to hand the FD to OpenMarker() and
	// raw_fd_ostream doesn't expose the FD.
	using sys::fs::openFileForWrite;
	if (auto EC =
	openFileForReadWrite(FilenameBuf.str(), DumpFd,
	sys::fs::CD_CreateNew, sys::fs::OF_None)) {
	errs() << "could not open JIT dump file " << FilenameBuf.str() << ": "
	<< EC.message() << "\n";
	return;
	}

	Dumpstream = std::make_unique<raw_fd_ostream>(DumpFd, true);

	LLVMPerfJitHeader Header = {0};
	if (!FillMachine(Header))
	return;

	// signal this process emits JIT information
	if (!OpenMarker())
	return;

	// emit dumpstream header
	Header.Magic = LLVM_PERF_JIT_MAGIC;
	Header.Version = LLVM_PERF_JIT_VERSION;
	Header.TotalSize = sizeof(Header);
	Header.Pid = Pid;
	Header.Timestamp = perf_get_timestamp();
	Dumpstream->write(reinterpret_cast<const char *>(&Header), sizeof(Header));

	// Everything initialized, can do profiling now.
	if (!Dumpstream->has_error())
	SuccessfullyInitialized = true;
	}

	void PerfJITEventListener::notifyObjectLoaded(
	ObjectKey K, const ObjectFile &Obj,
	const RuntimeDyld::LoadedObjectInfo &L) {

	if (!SuccessfullyInitialized)
	return;

	OwningBinary<ObjectFile> DebugObjOwner = L.getObjectForDebug(Obj);
	const ObjectFile &DebugObj = *DebugObjOwner.getBinary();

	// Get the address of the object image for use as a unique identifier
	std::unique_ptr<DIContext> Context = DWARFContext::create(DebugObj);

	// Use symbol info to iterate over functions in the object.
	for (const std::pair<SymbolRef, uint64_t> &P : computeSymbolSizes(DebugObj)) {
	SymbolRef Sym = P.first;
	std::string SourceFileName;

	Expected<SymbolRef::Type> SymTypeOrErr = Sym.getType();
	if (!SymTypeOrErr) {
	// There's not much we can with errors here
	consumeError(SymTypeOrErr.takeError());
	continue;
	}
	SymbolRef::Type SymType = *SymTypeOrErr;
	if (SymType != SymbolRef::ST_Function)
	continue;

	Expected<StringRef> Name = Sym.getName();
	if (!Name) {
	consumeError(Name.takeError());
	continue;
	}

	Expected<uint64_t> AddrOrErr = Sym.getAddress();
	if (!AddrOrErr) {
	consumeError(AddrOrErr.takeError());
	continue;
	}
	uint64_t Size = P.second;
	object::SectionedAddress Address;
	Address.Address = *AddrOrErr;

	uint64_t SectionIndex = object::SectionedAddress::UndefSection;
	if (auto SectOrErr = Sym.getSection())
	if (*SectOrErr != Obj.section_end())
	SectionIndex = SectOrErr.get()->getIndex();

	// According to spec debugging info has to come before loading the
	// corresonding code load.
	DILineInfoTable Lines = Context->getLineInfoForAddressRange(
	{*AddrOrErr, SectionIndex}, Size, FileLineInfoKind::AbsoluteFilePath);

	NotifyDebug(*AddrOrErr, Lines);
	NotifyCode(Name, *AddrOrErr, Size);
	}

	Dumpstream->flush();
	}

	void PerfJITEventListener::notifyFreeingObject(ObjectKey K) {
	// perf currently doesn't have an interface for unloading. But munmap()ing the
	// code section does, so that's ok.
	}

	bool PerfJITEventListener::InitDebuggingDir() {
	time_t Time;
	struct tm LocalTime;
	char TimeBuffer[sizeof("YYYYMMDD")];
	SmallString<64> Path;

	// search for location to dump data to
	if (const char *BaseDir = getenv("JITDUMPDIR"))
	Path.append(BaseDir);
	else if (!sys::path::home_directory(Path))
	Path = ".";

	// create debug directory
	Path += "/.debug/jit/";
	if (auto EC = sys::fs::create_directories(Path)) {
	errs() << "could not create jit cache directory " << Path << ": "
	<< EC.message() << "\n";
	return false;
	}

	// create unique directory for dump data related to this process
	time(&Time);
	localtime_r(&Time, &LocalTime);
	strftime(TimeBuffer, sizeof(TimeBuffer), "%Y%m%d", &LocalTime);
	Path += JIT_LANG "-jit-";
	Path += TimeBuffer;

	SmallString<128> UniqueDebugDir;

	using sys::fs::createUniqueDirectory;
	if (auto EC = createUniqueDirectory(Path, UniqueDebugDir)) {
	errs() << "could not create unique jit cache directory " << UniqueDebugDir
	<< ": " << EC.message() << "\n";
	return false;
	}

	JitPath = std::string(UniqueDebugDir.str());

	return true;
	}

	bool PerfJITEventListener::OpenMarker() {
	// We mmap the jitdump to create an MMAP RECORD in perf.data file. The mmap
	// is captured either live (perf record running when we mmap) or in deferred
	// mode, via /proc/PID/maps. The MMAP record is used as a marker of a jitdump
	// file for more meta data info about the jitted code. Perf report/annotate
	// detect this special filename and process the jitdump file.
	//
	// Mapping must be PROT_EXEC to ensure it is captured by perf record
	// even when not using -d option.
	MarkerAddr = ::mmap(NULL, sys::Process::getPageSizeEstimate(),
	PROT_READ \| PROT_EXEC, MAP_PRIVATE, DumpFd, 0);

	if (MarkerAddr == MAP_FAILED) {
	errs() << "could not mmap JIT marker\n";
	return false;
	}
	return true;
	}

	void PerfJITEventListener::CloseMarker() {
	if (!MarkerAddr)
	return;

	munmap(MarkerAddr, sys::Process::getPageSizeEstimate());
	MarkerAddr = nullptr;
	}

	bool PerfJITEventListener::FillMachine(LLVMPerfJitHeader &hdr) {
	char id[16];
	struct {
	uint16_t e_type;
	uint16_t e_machine;
	} info;

	size_t RequiredMemory = sizeof(id) + sizeof(info);

	ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
	MemoryBuffer::getFileSlice("/proc/self/exe",
	RequiredMemory,
	0);

	// This'll not guarantee that enough data was actually read from the
	// underlying file. Instead the trailing part of the buffer would be
	// zeroed. Given the ELF signature check below that seems ok though,
	// it's unlikely that the file ends just after that, and the
	// consequence would just be that perf wouldn't recognize the
	// signature.
	if (auto EC = MB.getError()) {
	errs() << "could not open /proc/self/exe: " << EC.message() << "\n";
	return false;
	}

	memcpy(&id, (*MB)->getBufferStart(), sizeof(id));
	memcpy(&info, (*MB)->getBufferStart() + sizeof(id), sizeof(info));

	// check ELF signature
	if (id[0] != 0x7f \|\| id[1] != 'E' \|\| id[2] != 'L' \|\| id[3] != 'F') {
	errs() << "invalid elf signature\n";
	return false;
	}

	hdr.ElfMach = info.e_machine;

	return true;
	}

	void PerfJITEventListener::NotifyCode(Expected<llvm::StringRef> &Symbol,
	uint64_t CodeAddr, uint64_t CodeSize) {
	assert(SuccessfullyInitialized);

	// 0 length functions can't have samples.
	if (CodeSize == 0)
	return;

	LLVMPerfJitRecordCodeLoad rec;
	rec.Prefix.Id = JIT_CODE_LOAD;
	rec.Prefix.TotalSize = sizeof(rec) + // debug record itself
	Symbol->size() + 1 + // symbol name
	CodeSize; // and code
	rec.Prefix.Timestamp = perf_get_timestamp();

	rec.CodeSize = CodeSize;
	rec.Vma = 0;
	rec.CodeAddr = CodeAddr;
	rec.Pid = Pid;
	rec.Tid = get_threadid();

	// avoid interspersing output
	std::lock_guard<sys::Mutex> Guard(Mutex);

	rec.CodeIndex = CodeGeneration++; // under lock!

	Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec));
	Dumpstream->write(Symbol->data(), Symbol->size() + 1);
	Dumpstream->write(reinterpret_cast<const char *>(CodeAddr), CodeSize);
	}

	void PerfJITEventListener::NotifyDebug(uint64_t CodeAddr,
	DILineInfoTable Lines) {
	assert(SuccessfullyInitialized);

	// Didn't get useful debug info.
	if (Lines.empty())
	return;

	LLVMPerfJitRecordDebugInfo rec;
	rec.Prefix.Id = JIT_CODE_DEBUG_INFO;
	rec.Prefix.TotalSize = sizeof(rec); // will be increased further
	rec.Prefix.Timestamp = perf_get_timestamp();
	rec.CodeAddr = CodeAddr;
	rec.NrEntry = Lines.size();

	// compute total size size of record (variable due to filenames)
	DILineInfoTable::iterator Begin = Lines.begin();
	DILineInfoTable::iterator End = Lines.end();
	for (DILineInfoTable::iterator It = Begin; It != End; ++It) {
	DILineInfo &line = It->second;
	rec.Prefix.TotalSize += sizeof(LLVMPerfJitDebugEntry);
	rec.Prefix.TotalSize += line.FileName.size() + 1;
	}

	// The debug_entry describes the source line information. It is defined as
	// follows in order:
	// * uint64_t code_addr: address of function for which the debug information
	// is generated
	// * uint32_t line : source file line number (starting at 1)
	// * uint32_t discrim : column discriminator, 0 is default
	// * char name[n] : source file name in ASCII, including null termination

	// avoid interspersing output
	std::lock_guard<sys::Mutex> Guard(Mutex);

	Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec));

	for (DILineInfoTable::iterator It = Begin; It != End; ++It) {
	LLVMPerfJitDebugEntry LineInfo;
	DILineInfo &Line = It->second;

	LineInfo.Addr = It->first;
	// The function re-created by perf is preceded by a elf
	// header. Need to adjust for that, otherwise the results are
	// wrong.
	LineInfo.Addr += 0x40;
	LineInfo.Lineno = Line.Line;
	LineInfo.Discrim = Line.Discriminator;

	Dumpstream->write(reinterpret_cast<const char *>(&LineInfo),
	sizeof(LineInfo));
	Dumpstream->write(Line.FileName.c_str(), Line.FileName.size() + 1);
	}
	}

	// There should be only a single event listener per process, otherwise perf gets
	// confused.
	llvm::ManagedStatic<PerfJITEventListener> PerfListener;

	} // end anonymous namespace

	namespace llvm {
	JITEventListener *JITEventListener::createPerfJITEventListener() {
	return &*PerfListener;
	}

	} // namespace llvm

	LLVMJITEventListenerRef LLVMCreatePerfJITEventListener(void)
	{
	return wrap(JITEventListener::createPerfJITEventListener());
	}