| //===-- Benchmark memory specific tools -------------------------*- C++ -*-===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // This file complements the `benchmark` header with memory specific tools and | |
| // benchmarking facilities. | |
| #ifndef LLVM_LIBC_UTILS_BENCHMARK_MEMORY_BENCHMARK_H | |
| #define LLVM_LIBC_UTILS_BENCHMARK_MEMORY_BENCHMARK_H | |
| #include "LibcBenchmark.h" | |
| #include "LibcFunctionPrototypes.h" | |
| #include "MemorySizeDistributions.h" | |
| #include "llvm/ADT/SmallVector.h" | |
| #include "llvm/Support/Alignment.h" | |
| #include "llvm/Support/MathExtras.h" | |
| #include <cstdint> | |
| #include <optional> | |
| #include <random> | |
| namespace llvm { | |
| namespace libc_benchmarks { | |
| //-------------- | |
| // Configuration | |
| //-------------- | |
| struct StudyConfiguration { | |
| // One of 'memcpy', 'memset', 'memcmp'. | |
| // The underlying implementation is always the llvm libc one. | |
| // e.g. 'memcpy' will test 'LIBC_NAMESPACE::memcpy' | |
| std::string Function; | |
| // The number of trials to run for this benchmark. | |
| // If in SweepMode, each individual sizes are measured 'NumTrials' time. | |
| // i.e 'NumTrials' measurements for 0, 'NumTrials' measurements for 1 ... | |
| uint32_t NumTrials = 1; | |
| // Toggles between Sweep Mode and Distribution Mode (default). | |
| // See 'SweepModeMaxSize' and 'SizeDistributionName' below. | |
| bool IsSweepMode = false; | |
| // Maximum size to use when measuring a ramp of size values (SweepMode). | |
| // The benchmark measures all sizes from 0 to SweepModeMaxSize. | |
| // Note: in sweep mode the same size is sampled several times in a row this | |
| // will allow the processor to learn it and optimize the branching pattern. | |
| // The resulting measurement is likely to be idealized. | |
| uint32_t SweepModeMaxSize = 0; // inclusive | |
| // The name of the distribution to be used to randomize the size parameter. | |
| // This is used when SweepMode is false (default). | |
| std::string SizeDistributionName; | |
| // This parameter allows to control how the buffers are accessed during | |
| // benchmark: | |
| // None : Use a fixed address that is at least cache line aligned, | |
| // 1 : Use random address, | |
| // >1 : Use random address aligned to value. | |
| MaybeAlign AccessAlignment = std::nullopt; | |
| // When Function == 'memcmp', this is the buffers mismatch position. | |
| // 0 : Buffers always compare equal, | |
| // >0 : Buffers compare different at byte N-1. | |
| uint32_t MemcmpMismatchAt = 0; | |
| }; | |
| struct Runtime { | |
| // Details about the Host (cpu name, cpu frequency, cache hierarchy). | |
| HostState Host; | |
| // The framework will populate this value so all data accessed during the | |
| // benchmark will stay in L1 data cache. This includes bookkeeping data. | |
| uint32_t BufferSize = 0; | |
| // This is the number of distinct parameters used in a single batch. | |
| // The framework always tests a batch of randomized parameter to prevent the | |
| // cpu from learning branching patterns. | |
| uint32_t BatchParameterCount = 0; | |
| // The benchmark options that were used to perform the measurement. | |
| // This is decided by the framework. | |
| BenchmarkOptions BenchmarkOptions; | |
| }; | |
| //-------- | |
| // Results | |
| //-------- | |
| // The root object containing all the data (configuration and measurements). | |
| struct Study { | |
| std::string StudyName; | |
| Runtime Runtime; | |
| StudyConfiguration Configuration; | |
| std::vector<Duration> Measurements; | |
| }; | |
| //------ | |
| // Utils | |
| //------ | |
| // Provides an aligned, dynamically allocated buffer. | |
| class AlignedBuffer { | |
| char *const Buffer = nullptr; | |
| size_t Size = 0; | |
| public: | |
| static constexpr size_t Alignment = 512; | |
| explicit AlignedBuffer(size_t Size) | |
| : Buffer(static_cast<char *>( | |
| aligned_alloc(Alignment, alignTo(Size, Alignment)))), | |
| Size(Size) {} | |
| ~AlignedBuffer() { free(Buffer); } | |
| inline char *operator+(size_t Index) { return Buffer + Index; } | |
| inline const char *operator+(size_t Index) const { return Buffer + Index; } | |
| inline char &operator[](size_t Index) { return Buffer[Index]; } | |
| inline const char &operator[](size_t Index) const { return Buffer[Index]; } | |
| inline char *begin() { return Buffer; } | |
| inline char *end() { return Buffer + Size; } | |
| }; | |
| // Helper to generate random buffer offsets that satisfy the configuration | |
| // constraints. | |
| class OffsetDistribution { | |
| std::uniform_int_distribution<uint32_t> Distribution; | |
| uint32_t Factor; | |
| public: | |
| explicit OffsetDistribution(size_t BufferSize, size_t MaxSizeValue, | |
| MaybeAlign AccessAlignment); | |
| template <class Generator> uint32_t operator()(Generator &G) { | |
| return Distribution(G) * Factor; | |
| } | |
| }; | |
| // Helper to generate random buffer offsets that satisfy the configuration | |
| // constraints. It is specifically designed to benchmark `memcmp` functions | |
| // where we may want the Nth byte to differ. | |
| class MismatchOffsetDistribution { | |
| std::uniform_int_distribution<size_t> MismatchIndexSelector; | |
| llvm::SmallVector<uint32_t, 16> MismatchIndices; | |
| const uint32_t MismatchAt; | |
| public: | |
| explicit MismatchOffsetDistribution(size_t BufferSize, size_t MaxSizeValue, | |
| size_t MismatchAt); | |
| explicit operator bool() const { return !MismatchIndices.empty(); } | |
| const llvm::SmallVectorImpl<uint32_t> &getMismatchIndices() const { | |
| return MismatchIndices; | |
| } | |
| template <class Generator> uint32_t operator()(Generator &G, uint32_t Size) { | |
| const uint32_t MismatchIndex = MismatchIndices[MismatchIndexSelector(G)]; | |
| // We need to position the offset so that a mismatch occurs at MismatchAt. | |
| if (Size >= MismatchAt) | |
| return MismatchIndex - MismatchAt; | |
| // Size is too small to trigger the mismatch. | |
| return MismatchIndex - Size - 1; | |
| } | |
| }; | |
| /// This structure holds a vector of ParameterType. | |
| /// It makes sure that BufferCount x BufferSize Bytes and the vector of | |
| /// ParameterType can all fit in the L1 cache. | |
| struct ParameterBatch { | |
| struct ParameterType { | |
| unsigned OffsetBytes : 16; // max : 16 KiB - 1 | |
| unsigned SizeBytes : 16; // max : 16 KiB - 1 | |
| }; | |
| ParameterBatch(size_t BufferCount); | |
| /// Verifies that memory accessed through this parameter is valid. | |
| void checkValid(const ParameterType &) const; | |
| /// Computes the number of bytes processed during within this batch. | |
| size_t getBatchBytes() const; | |
| const size_t BufferSize; | |
| const size_t BatchSize; | |
| std::vector<ParameterType> Parameters; | |
| }; | |
| /// Provides source and destination buffers for the Copy operation as well as | |
| /// the associated size distributions. | |
| struct CopySetup : public ParameterBatch { | |
| CopySetup(); | |
| inline static const ArrayRef<MemorySizeDistribution> getDistributions() { | |
| return getMemcpySizeDistributions(); | |
| } | |
| inline void *Call(ParameterType Parameter, MemcpyFunction Memcpy) { | |
| return Memcpy(DstBuffer + Parameter.OffsetBytes, | |
| SrcBuffer + Parameter.OffsetBytes, Parameter.SizeBytes); | |
| } | |
| private: | |
| AlignedBuffer SrcBuffer; | |
| AlignedBuffer DstBuffer; | |
| }; | |
| /// Provides source and destination buffers for the Move operation as well as | |
| /// the associated size distributions. | |
| struct MoveSetup : public ParameterBatch { | |
| MoveSetup(); | |
| inline static const ArrayRef<MemorySizeDistribution> getDistributions() { | |
| return getMemmoveSizeDistributions(); | |
| } | |
| inline void *Call(ParameterType Parameter, MemmoveFunction Memmove) { | |
| return Memmove(Buffer + ParameterBatch::BufferSize / 3, | |
| Buffer + Parameter.OffsetBytes, Parameter.SizeBytes); | |
| } | |
| private: | |
| AlignedBuffer Buffer; | |
| }; | |
| /// Provides destination buffer for the Set operation as well as the associated | |
| /// size distributions. | |
| struct SetSetup : public ParameterBatch { | |
| SetSetup(); | |
| inline static const ArrayRef<MemorySizeDistribution> getDistributions() { | |
| return getMemsetSizeDistributions(); | |
| } | |
| inline void *Call(ParameterType Parameter, MemsetFunction Memset) { | |
| return Memset(DstBuffer + Parameter.OffsetBytes, | |
| Parameter.OffsetBytes % 0xFF, Parameter.SizeBytes); | |
| } | |
| inline void *Call(ParameterType Parameter, BzeroFunction Bzero) { | |
| Bzero(DstBuffer + Parameter.OffsetBytes, Parameter.SizeBytes); | |
| return DstBuffer.begin(); | |
| } | |
| private: | |
| AlignedBuffer DstBuffer; | |
| }; | |
| /// Provides left and right buffers for the Comparison operation as well as the | |
| /// associated size distributions. | |
| struct ComparisonSetup : public ParameterBatch { | |
| ComparisonSetup(); | |
| inline static const ArrayRef<MemorySizeDistribution> getDistributions() { | |
| return getMemcmpSizeDistributions(); | |
| } | |
| inline int Call(ParameterType Parameter, MemcmpOrBcmpFunction MemcmpOrBcmp) { | |
| return MemcmpOrBcmp(LhsBuffer + Parameter.OffsetBytes, | |
| RhsBuffer + Parameter.OffsetBytes, Parameter.SizeBytes); | |
| } | |
| private: | |
| AlignedBuffer LhsBuffer; | |
| AlignedBuffer RhsBuffer; | |
| }; | |
| } // namespace libc_benchmarks | |
| } // namespace llvm | |
| #endif // LLVM_LIBC_UTILS_BENCHMARK_MEMORY_BENCHMARK_H |