llvm/tools/llvm-exegesis/lib/X86/X86Counter.cpp - llvm-project - Git at Google

 //===-- X86Counter.cpp ------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//

 #include "X86Counter.h"

 #if defined(__linux__) && defined(HAVE_LIBPFM) &&                              \
     defined(LIBPFM_HAS_FIELD_CYCLES)

 // FIXME: Use appropriate wrappers for poll.h and mman.h
 // to support Windows and remove this linux-only guard.

 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Errc.h"

 #include <perfmon/perf_event.h>
 #include <perfmon/pfmlib.h>
 #include <perfmon/pfmlib_perf_event.h>

 #include <atomic>
 #include <chrono>
 #include <cstddef>
 #include <cstdint>
 #include <limits>
 #include <memory>
 #include <vector>

 #include <poll.h>
 #include <sys/mman.h>
 #include <unistd.h>

 namespace llvm {
 namespace exegesis {

 // Number of entries in the LBR.
 static constexpr int kLbrEntries = 16;
 static constexpr size_t kBufferPages = 8;
 static const size_t kDataBufferSize = kBufferPages * getpagesize();

 // First page is reserved for perf_event_mmap_page. Data buffer starts on
 // the next page, so we allocate one more page.
 static const size_t kMappedBufferSize = (kBufferPages + 1) * getpagesize();

 // Waits for the LBR perf events.
 static int pollLbrPerfEvent(const int FileDescriptor) {
   struct pollfd PollFd;
   PollFd.fd = FileDescriptor;
   PollFd.events = POLLIN;
   PollFd.revents = 0;
   return poll(&PollFd, 1 /* num of fds */, 10000 /* timeout in ms */);
 }

 // Copies the data-buffer into Buf, given the pointer to MMapped.
 static void copyDataBuffer(void *MMappedBuffer, char *Buf, uint64_t Tail,
                            size_t DataSize) {
   // First page is reserved for perf_event_mmap_page. Data buffer starts on
   // the next page.
   char *Start = reinterpret_cast<char *>(MMappedBuffer) + getpagesize();
   // The LBR buffer is a cyclic buffer, we copy data to another buffer.
   uint64_t Offset = Tail % kDataBufferSize;
   size_t CopySize = kDataBufferSize - Offset;
   memcpy(Buf, Start + Offset, CopySize);
   if (CopySize >= DataSize)
     return;

   memcpy(Buf + CopySize, Start, Offset);
   return;
 }

 // Parses the given data-buffer for stats and fill the CycleArray.
 // If data has been extracted successfully, also modifies the code to jump
 // out the benchmark loop.
 static llvm::Error parseDataBuffer(const char *DataBuf, size_t DataSize,
                                    const void *From, const void *To,
                                    llvm::SmallVector<int64_t, 4> *CycleArray) {
   const char *DataPtr = DataBuf;
   while (DataPtr < DataBuf + DataSize) {
     struct perf_event_header Header;
     memcpy(&Header, DataPtr, sizeof(struct perf_event_header));
     if (Header.type != PERF_RECORD_SAMPLE) {
       // Ignores non-sample records.
       DataPtr += Header.size;
       continue;
     }
     DataPtr += sizeof(Header);
     uint64_t Count = llvm::support::endian::read64(DataPtr, support::native);
     DataPtr += sizeof(Count);

     struct perf_branch_entry Entry;
     memcpy(&Entry, DataPtr, sizeof(struct perf_branch_entry));

     // Read the perf_branch_entry array.
     for (uint64_t i = 0; i < Count; ++i) {
       const uint64_t BlockStart = From == nullptr
                                       ? std::numeric_limits<uint64_t>::min()
                                       : reinterpret_cast<uint64_t>(From);
       const uint64_t BlockEnd = To == nullptr
                                     ? std::numeric_limits<uint64_t>::max()
                                     : reinterpret_cast<uint64_t>(To);

       if (BlockStart <= Entry.from && BlockEnd >= Entry.to)
         CycleArray->push_back(Entry.cycles);

       if (i == Count - 1)
         // We've reached the last entry.
         return llvm::Error::success();

       // Advance to next entry
       DataPtr += sizeof(Entry);
       memcpy(&Entry, DataPtr, sizeof(struct perf_branch_entry));
     }
   }
   return llvm::make_error<llvm::StringError>("Unable to parse databuffer.",
                                              llvm::errc::io_error);
 }

 X86LbrPerfEvent::X86LbrPerfEvent(unsigned SamplingPeriod) {
   assert(SamplingPeriod > 0 && "SamplingPeriod must be positive");
   EventString = "BR_INST_RETIRED.NEAR_TAKEN";
   Attr = new perf_event_attr();
   Attr->size = sizeof(*Attr);
   Attr->type = PERF_TYPE_RAW;
   // FIXME This is SKL's encoding. Not sure if it'll change.
   Attr->config = 0x20c4; // BR_INST_RETIRED.NEAR_TAKEN
   Attr->sample_type = PERF_SAMPLE_BRANCH_STACK;
   // Don't need to specify "USER" because we've already excluded HV and Kernel.
   Attr->branch_sample_type = PERF_SAMPLE_BRANCH_ANY;
   Attr->sample_period = SamplingPeriod;
   Attr->wakeup_events = 1; // We need this even when using ioctl REFRESH.
   Attr->disabled = 1;
   Attr->exclude_kernel = 1;
   Attr->exclude_hv = 1;
   Attr->read_format = PERF_FORMAT_GROUP;

   FullQualifiedEventString = EventString;
 }

 X86LbrCounter::X86LbrCounter(pfm::PerfEvent &&NewEvent)
     : Counter(std::move(NewEvent)) {
   MMappedBuffer = mmap(nullptr, kMappedBufferSize, PROT_READ | PROT_WRITE,
                        MAP_SHARED, FileDescriptor, 0);
   if (MMappedBuffer == MAP_FAILED)
     llvm::errs() << "Failed to mmap buffer.";
 }

 X86LbrCounter::~X86LbrCounter() {
   if (0 != munmap(MMappedBuffer, kMappedBufferSize))
     llvm::errs() << "Failed to munmap buffer.";
 }

 void X86LbrCounter::start() {
   ioctl(FileDescriptor, PERF_EVENT_IOC_REFRESH, 1024 /* kMaxPollsPerFd */);
 }

 llvm::Error X86LbrCounter::checkLbrSupport() {
   // Do a sample read and check if the results contain non-zero values.

   X86LbrCounter counter(X86LbrPerfEvent(123));
   counter.start();

   // Prevent the compiler from unrolling the loop and get rid of all the
   // branches. We need at least 16 iterations.
   int Sum = 0;
   int V = 1;

   volatile int *P = &V;
   auto TimeLimit =
       std::chrono::high_resolution_clock::now() + std::chrono::microseconds(5);

   for (int I = 0;
        I < kLbrEntries || std::chrono::high_resolution_clock::now() < TimeLimit;
        ++I) {
     Sum += *P;
   }

   counter.stop();
   (void)Sum;

   auto ResultOrError = counter.doReadCounter(nullptr, nullptr);
   if (ResultOrError)
     if (!ResultOrError.get().empty())
       // If there is at least one non-zero entry, then LBR is supported.
       for (const int64_t &Value : ResultOrError.get())
         if (Value != 0)
           return Error::success();

   return llvm::make_error<llvm::StringError>(
       "LBR format with cycles is not suppported on the host.",
       llvm::errc::not_supported);
 }

 llvm::Expected<llvm::SmallVector<int64_t, 4>>
 X86LbrCounter::readOrError(StringRef FunctionBytes) const {
   // Disable the event before reading
   ioctl(FileDescriptor, PERF_EVENT_IOC_DISABLE, 0);

   // Find the boundary of the function so that we could filter the LBRs
   // to keep only the relevant records.
   if (FunctionBytes.empty())
     return llvm::make_error<llvm::StringError>("Empty function bytes",
                                                llvm::errc::invalid_argument);
   const void *From = reinterpret_cast<const void *>(FunctionBytes.data());
   const void *To = reinterpret_cast<const void *>(FunctionBytes.data() +
                                                   FunctionBytes.size());
   return doReadCounter(From, To);
 }

 llvm::Expected<llvm::SmallVector<int64_t, 4>>
 X86LbrCounter::doReadCounter(const void *From, const void *To) const {
   // The max number of time-outs/retries before we give up.
   static constexpr int kMaxTimeouts = 160;

   // Parses the LBR buffer and fills CycleArray with the sequence of cycle
   // counts from the buffer.
   llvm::SmallVector<int64_t, 4> CycleArray;
   auto DataBuf = std::make_unique<char[]>(kDataBufferSize);
   int NumTimeouts = 0;
   int PollResult = 0;

   while (PollResult <= 0) {
     PollResult = pollLbrPerfEvent(FileDescriptor);
     if (PollResult > 0)
       break;
     if (PollResult == -1)
       return llvm::make_error<llvm::StringError>("Cannot poll LBR perf event.",
                                                  llvm::errc::io_error);
     if (NumTimeouts++ >= kMaxTimeouts)
       return llvm::make_error<llvm::StringError>(
           "LBR polling still timed out after max number of attempts.",
           llvm::errc::device_or_resource_busy);
   }

   struct perf_event_mmap_page Page;
   memcpy(&Page, MMappedBuffer, sizeof(struct perf_event_mmap_page));

   const uint64_t DataTail = Page.data_tail;
   const uint64_t DataHead = Page.data_head;
   // We're supposed to use a barrier after reading data_head.
   std::atomic_thread_fence(std::memory_order_acq_rel);
   const size_t DataSize = DataHead - DataTail;
   if (DataSize > kDataBufferSize)
     return llvm::make_error<llvm::StringError>(
         "DataSize larger than buffer size.", llvm::errc::invalid_argument);

   copyDataBuffer(MMappedBuffer, DataBuf.get(), DataTail, DataSize);
   llvm::Error error =
       parseDataBuffer(DataBuf.get(), DataSize, From, To, &CycleArray);
   if (!error)
     return CycleArray;
   return std::move(error);
 }

 } // namespace exegesis
 } // namespace llvm

 #endif // defined(__linux__) && defined(HAVE_LIBPFM) &&
        // defined(LIBPFM_HAS_FIELD_CYCLES)
	//===-- X86Counter.cpp ------------------------------------------- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//

	#include "X86Counter.h"

	#if defined(__linux__) && defined(HAVE_LIBPFM) && \
	defined(LIBPFM_HAS_FIELD_CYCLES)

	// FIXME: Use appropriate wrappers for poll.h and mman.h
	// to support Windows and remove this linux-only guard.

	#include "llvm/Support/Endian.h"
	#include "llvm/Support/Errc.h"

	#include <perfmon/perf_event.h>
	#include <perfmon/pfmlib.h>
	#include <perfmon/pfmlib_perf_event.h>

	#include <atomic>
	#include <chrono>
	#include <cstddef>
	#include <cstdint>
	#include <limits>
	#include <memory>
	#include <vector>

	#include <poll.h>
	#include <sys/mman.h>
	#include <unistd.h>

	namespace llvm {
	namespace exegesis {

	// Number of entries in the LBR.
	static constexpr int kLbrEntries = 16;
	static constexpr size_t kBufferPages = 8;
	static const size_t kDataBufferSize = kBufferPages * getpagesize();

	// First page is reserved for perf_event_mmap_page. Data buffer starts on
	// the next page, so we allocate one more page.
	static const size_t kMappedBufferSize = (kBufferPages + 1) * getpagesize();

	// Waits for the LBR perf events.
	static int pollLbrPerfEvent(const int FileDescriptor) {
	struct pollfd PollFd;
	PollFd.fd = FileDescriptor;
	PollFd.events = POLLIN;
	PollFd.revents = 0;
	return poll(&PollFd, 1 /* num of fds /, 10000 / timeout in ms */);
	}

	// Copies the data-buffer into Buf, given the pointer to MMapped.
	static void copyDataBuffer(void MMappedBuffer, char Buf, uint64_t Tail,
	size_t DataSize) {
	// First page is reserved for perf_event_mmap_page. Data buffer starts on
	// the next page.
	char Start = reinterpret_cast<char >(MMappedBuffer) + getpagesize();
	// The LBR buffer is a cyclic buffer, we copy data to another buffer.
	uint64_t Offset = Tail % kDataBufferSize;
	size_t CopySize = kDataBufferSize - Offset;
	memcpy(Buf, Start + Offset, CopySize);
	if (CopySize >= DataSize)
	return;

	memcpy(Buf + CopySize, Start, Offset);
	return;
	}

	// Parses the given data-buffer for stats and fill the CycleArray.
	// If data has been extracted successfully, also modifies the code to jump
	// out the benchmark loop.
	static llvm::Error parseDataBuffer(const char *DataBuf, size_t DataSize,
	const void From, const void To,
	llvm::SmallVector<int64_t, 4> *CycleArray) {
	const char *DataPtr = DataBuf;
	while (DataPtr < DataBuf + DataSize) {
	struct perf_event_header Header;
	memcpy(&Header, DataPtr, sizeof(struct perf_event_header));
	if (Header.type != PERF_RECORD_SAMPLE) {
	// Ignores non-sample records.
	DataPtr += Header.size;
	continue;
	}
	DataPtr += sizeof(Header);
	uint64_t Count = llvm::support::endian::read64(DataPtr, support::native);
	DataPtr += sizeof(Count);

	struct perf_branch_entry Entry;
	memcpy(&Entry, DataPtr, sizeof(struct perf_branch_entry));

	// Read the perf_branch_entry array.
	for (uint64_t i = 0; i < Count; ++i) {
	const uint64_t BlockStart = From == nullptr
	? std::numeric_limits<uint64_t>::min()
	: reinterpret_cast<uint64_t>(From);
	const uint64_t BlockEnd = To == nullptr
	? std::numeric_limits<uint64_t>::max()
	: reinterpret_cast<uint64_t>(To);

	if (BlockStart <= Entry.from && BlockEnd >= Entry.to)
	CycleArray->push_back(Entry.cycles);

	if (i == Count - 1)
	// We've reached the last entry.
	return llvm::Error::success();

	// Advance to next entry
	DataPtr += sizeof(Entry);
	memcpy(&Entry, DataPtr, sizeof(struct perf_branch_entry));
	}
	}
	return llvm::make_error<llvm::StringError>("Unable to parse databuffer.",
	llvm::errc::io_error);
	}

	X86LbrPerfEvent::X86LbrPerfEvent(unsigned SamplingPeriod) {
	assert(SamplingPeriod > 0 && "SamplingPeriod must be positive");
	EventString = "BR_INST_RETIRED.NEAR_TAKEN";
	Attr = new perf_event_attr();
	Attr->size = sizeof(*Attr);
	Attr->type = PERF_TYPE_RAW;
	// FIXME This is SKL's encoding. Not sure if it'll change.
	Attr->config = 0x20c4; // BR_INST_RETIRED.NEAR_TAKEN
	Attr->sample_type = PERF_SAMPLE_BRANCH_STACK;
	// Don't need to specify "USER" because we've already excluded HV and Kernel.
	Attr->branch_sample_type = PERF_SAMPLE_BRANCH_ANY;
	Attr->sample_period = SamplingPeriod;
	Attr->wakeup_events = 1; // We need this even when using ioctl REFRESH.
	Attr->disabled = 1;
	Attr->exclude_kernel = 1;
	Attr->exclude_hv = 1;
	Attr->read_format = PERF_FORMAT_GROUP;

	FullQualifiedEventString = EventString;
	}

	X86LbrCounter::X86LbrCounter(pfm::PerfEvent &&NewEvent)
	: Counter(std::move(NewEvent)) {
	MMappedBuffer = mmap(nullptr, kMappedBufferSize, PROT_READ \| PROT_WRITE,
	MAP_SHARED, FileDescriptor, 0);
	if (MMappedBuffer == MAP_FAILED)
	llvm::errs() << "Failed to mmap buffer.";
	}

	X86LbrCounter::~X86LbrCounter() {
	if (0 != munmap(MMappedBuffer, kMappedBufferSize))
	llvm::errs() << "Failed to munmap buffer.";
	}

	void X86LbrCounter::start() {
	ioctl(FileDescriptor, PERF_EVENT_IOC_REFRESH, 1024 /* kMaxPollsPerFd */);
	}

	llvm::Error X86LbrCounter::checkLbrSupport() {
	// Do a sample read and check if the results contain non-zero values.

	X86LbrCounter counter(X86LbrPerfEvent(123));
	counter.start();

	// Prevent the compiler from unrolling the loop and get rid of all the
	// branches. We need at least 16 iterations.
	int Sum = 0;
	int V = 1;

	volatile int *P = &V;
	auto TimeLimit =
	std::chrono::high_resolution_clock::now() + std::chrono::microseconds(5);

	for (int I = 0;
	I < kLbrEntries \|\| std::chrono::high_resolution_clock::now() < TimeLimit;
	++I) {
	Sum += *P;
	}

	counter.stop();
	(void)Sum;

	auto ResultOrError = counter.doReadCounter(nullptr, nullptr);
	if (ResultOrError)
	if (!ResultOrError.get().empty())
	// If there is at least one non-zero entry, then LBR is supported.
	for (const int64_t &Value : ResultOrError.get())
	if (Value != 0)
	return Error::success();

	return llvm::make_error<llvm::StringError>(
	"LBR format with cycles is not suppported on the host.",
	llvm::errc::not_supported);
	}

	llvm::Expected<llvm::SmallVector<int64_t, 4>>
	X86LbrCounter::readOrError(StringRef FunctionBytes) const {
	// Disable the event before reading
	ioctl(FileDescriptor, PERF_EVENT_IOC_DISABLE, 0);

	// Find the boundary of the function so that we could filter the LBRs
	// to keep only the relevant records.
	if (FunctionBytes.empty())
	return llvm::make_error<llvm::StringError>("Empty function bytes",
	llvm::errc::invalid_argument);
	const void From = reinterpret_cast<const void >(FunctionBytes.data());
	const void To = reinterpret_cast<const void >(FunctionBytes.data() +
	FunctionBytes.size());
	return doReadCounter(From, To);
	}

	llvm::Expected<llvm::SmallVector<int64_t, 4>>
	X86LbrCounter::doReadCounter(const void From, const void To) const {
	// The max number of time-outs/retries before we give up.
	static constexpr int kMaxTimeouts = 160;

	// Parses the LBR buffer and fills CycleArray with the sequence of cycle
	// counts from the buffer.
	llvm::SmallVector<int64_t, 4> CycleArray;
	auto DataBuf = std::make_unique<char[]>(kDataBufferSize);
	int NumTimeouts = 0;
	int PollResult = 0;

	while (PollResult <= 0) {
	PollResult = pollLbrPerfEvent(FileDescriptor);
	if (PollResult > 0)
	break;
	if (PollResult == -1)
	return llvm::make_error<llvm::StringError>("Cannot poll LBR perf event.",
	llvm::errc::io_error);
	if (NumTimeouts++ >= kMaxTimeouts)
	return llvm::make_error<llvm::StringError>(
	"LBR polling still timed out after max number of attempts.",
	llvm::errc::device_or_resource_busy);
	}

	struct perf_event_mmap_page Page;
	memcpy(&Page, MMappedBuffer, sizeof(struct perf_event_mmap_page));

	const uint64_t DataTail = Page.data_tail;
	const uint64_t DataHead = Page.data_head;
	// We're supposed to use a barrier after reading data_head.
	std::atomic_thread_fence(std::memory_order_acq_rel);
	const size_t DataSize = DataHead - DataTail;
	if (DataSize > kDataBufferSize)
	return llvm::make_error<llvm::StringError>(
	"DataSize larger than buffer size.", llvm::errc::invalid_argument);

	copyDataBuffer(MMappedBuffer, DataBuf.get(), DataTail, DataSize);
	llvm::Error error =
	parseDataBuffer(DataBuf.get(), DataSize, From, To, &CycleArray);
	if (!error)
	return CycleArray;
	return std::move(error);
	}

	} // namespace exegesis
	} // namespace llvm

	#endif // defined(__linux__) && defined(HAVE_LIBPFM) &&
	// defined(LIBPFM_HAS_FIELD_CYCLES)