tsd_shared.h

//===-- tsd_shared.h --------------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#ifndef SCUDO_TSD_SHARED_H_
#define SCUDO_TSD_SHARED_H_

#include "tsd.h"

#include "string_utils.h"

#if SCUDO_HAS_PLATFORM_TLS_SLOT
// This is a platform-provided header that needs to be on the include path when
// Scudo is compiled. It must declare a function with the prototype:
//   uintptr_t *getPlatformAllocatorTlsSlot()
// that returns the address of a thread-local word of storage reserved for
// Scudo, that must be zero-initialized in newly created threads.
#include "scudo_platform_tls_slot.h"
#endif

namespace scudo {

template <class Allocator, u32 TSDsArraySize, u32 DefaultTSDCount>
struct TSDRegistrySharedT {
  using ThisT = TSDRegistrySharedT<Allocator, TSDsArraySize, DefaultTSDCount>;

  struct ScopedTSD {
    ALWAYS_INLINE ScopedTSD(ThisT &TSDRegistry) {
      CurrentTSD = TSDRegistry.getTSDAndLock();
      DCHECK_NE(CurrentTSD, nullptr);
    }

    ~ScopedTSD() { CurrentTSD->unlock(); }

    TSD<Allocator> &operator*() { return *CurrentTSD; }

    TSD<Allocator> *operator->() {
      CurrentTSD->assertLocked(/*BypassCheck=*/false);
      return CurrentTSD;
    }

  private:
    TSD<Allocator> *CurrentTSD;
  };

  void init(Allocator *Instance) EXCLUDES(Mutex) {
    ScopedLock L(Mutex);
    // If more than one thread is initializing at the exact same moment, the
    // threads that lose don't need to do anything.
    if (UNLIKELY(atomic_load_relaxed(&Initialized) != 0))
      return;

    Instance->init();
    for (u32 I = 0; I < TSDsArraySize; I++)
      TSDs[I].init(Instance);
    const u32 NumberOfCPUs = getNumberOfCPUs();
    setNumberOfTSDs((NumberOfCPUs == 0) ? DefaultTSDCount
                                        : Min(NumberOfCPUs, DefaultTSDCount));
    atomic_store_relaxed(&Initialized, 1);
  }

  void initOnceMaybe(Allocator *Instance) {
    if (LIKELY(atomic_load_relaxed(&Initialized) != 0))
      return;
    init(Instance); // Sets Initialized.
  }

  void unmapTestOnly(Allocator *Instance) {
    ScopedLock L(Mutex);
    for (u32 I = 0; I < TSDsArraySize; I++) {
      TSDs[I].commitBack(Instance);
      TSDs[I] = {};
    }
    setCurrentTSD(nullptr);
    atomic_store_relaxed(&Initialized, 0);
  }

  void drainCaches(Allocator *Instance) {
    u32 TotalTSDs = atomic_load_relaxed(&NumberOfTSDs);
    for (uptr I = 0; I < TotalTSDs; ++I) {
      TSDs[I].lock();
      Instance->drainCache(&TSDs[I]);
      TSDs[I].unlock();
    }
  }

  ALWAYS_INLINE void initThreadMaybe(Allocator *Instance,
                                     UNUSED bool MinimalInit) {
    if (LIKELY(getCurrentTSD()))
      return;
    initThread(Instance);
  }

  void disable() NO_THREAD_SAFETY_ANALYSIS {
    Mutex.lock();
    for (u32 I = 0; I < TSDsArraySize; I++)
      TSDs[I].lock();
  }

  void enable() NO_THREAD_SAFETY_ANALYSIS {
    for (s32 I = static_cast<s32>(TSDsArraySize - 1); I >= 0; I--)
      TSDs[I].unlock();
    Mutex.unlock();
  }

  bool setOption(Option O, sptr Value) {
    if (O == Option::MaxTSDsCount) {
      ScopedLock L(Mutex);
      return setNumberOfTSDs(static_cast<u32>(Value));
    }
    if (O == Option::ThreadDisableMemInit)
      setDisableMemInit(Value);
    // Not supported by the TSD Registry, but not an error either.
    return true;
  }

  bool getDisableMemInit() const { return *getTlsPtr() & 1; }

  void getStats(ScopedString *Str) {
    u32 TotalTSDs = atomic_load_relaxed(&NumberOfTSDs);
    Str->append("Stats: SharedTSDs: %u available; total %u\n", TotalTSDs,
                TSDsArraySize);
    for (uptr I = 0; I < TotalTSDs; ++I) {
      TSDs[I].lock();
      // Theoretically, we want to mark TSD::lock()/TSD::unlock() with proper
      // thread annotations. However, given the TSD is only locked on shared
      // path, do the assertion in a separate path to avoid confusing the
      // analyzer.
      TSDs[I].assertLocked(/*BypassCheck=*/true);
      Str->append("  Shared TSD[%zu]:\n", I);
      TSDs[I].getSizeClassAllocator().getStats(Str);
      TSDs[I].unlock();
    }
  }

private:
  ALWAYS_INLINE TSD<Allocator> *getTSDAndLock() NO_THREAD_SAFETY_ANALYSIS {
    TSD<Allocator> *TSD = getCurrentTSD();
    DCHECK(TSD);
    if (TSDsArraySize == 1U) {
      // Only 1 TSD, lock and return.
      TSD->lock();
      return TSD;
    }

    if (TSD->tryLock())
      return TSD;
    return getTSDSlow(TSD);
  }

  ALWAYS_INLINE uptr *getTlsPtr() const {
#if SCUDO_HAS_PLATFORM_TLS_SLOT
    return reinterpret_cast<uptr *>(getPlatformAllocatorTlsSlot());
#else
    static thread_local uptr ThreadTSD;
    return &ThreadTSD;
#endif
  }

  static_assert(alignof(TSD<Allocator>) >= 2, "");

  ALWAYS_INLINE void setCurrentTSD(TSD<Allocator> *CurrentTSD) {
    *getTlsPtr() &= 1;
    *getTlsPtr() |= reinterpret_cast<uptr>(CurrentTSD);
  }

  ALWAYS_INLINE TSD<Allocator> *getCurrentTSD() {
    return reinterpret_cast<TSD<Allocator> *>(*getTlsPtr() & ~1ULL);
  }

  // Requires a lock to avoid multiple threads trying to set the TSDs at once.
  bool setNumberOfTSDs(u32 N) REQUIRES(Mutex) {
    DCHECK_GT(N, 0);
    const u32 TotalTSDs = atomic_load_relaxed(&NumberOfTSDs);
    // In order to avoid needing locks for these values, the number of TSDs
    // can never be decreased.
    if (N < TotalTSDs)
      return false;

    if (N > TSDsArraySize)
      N = TSDsArraySize;

    // In order to avoid locks around the CoPrimes, compute all of the
    // new values and then set them in place after.
    u32 NewCoPrimes[TSDsArraySize] = {};
    u32 NewNumberOfCoPrimes = 0;

    // Compute all the coprimes of the TSDs. This will be used to walk the
    // array of TSDs in a random order. For details, see:
    // https://lemire.me/blog/2017/09/18/visiting-all-values-in-an-array-exactly-once-in-random-order/
    for (u32 I = 0; I < N; I++) {
      u32 A = I + 1;
      u32 B = N;
      // Find the GCD between I + 1 and N. If 1, they are coprimes.
      while (B != 0) {
        const u32 T = A;
        A = B;
        B = T % B;
      }
      if (A == 1)
        NewCoPrimes[NewNumberOfCoPrimes++] = I + 1;
    }

    // Set these values in this particular order so that if getTSDSlow
    // is called while they are being modified, nothing crashes, or encounters
    // a zero CoPrime value.
    for (u32 I = 0; I < NewNumberOfCoPrimes; I++) {
      atomic_store(&CoPrimes[I], NewCoPrimes[I], memory_order_release);
    }
    atomic_store(&NumberOfCoPrimes, NewNumberOfCoPrimes, memory_order_release);
    atomic_store(&NumberOfTSDs, N, memory_order_release);
    return true;
  }

  void setDisableMemInit(bool B) {
    *getTlsPtr() &= ~1ULL;
    *getTlsPtr() |= B;
  }

  NOINLINE void initThread(Allocator *Instance) {
    initOnceMaybe(Instance);
    // Initial context assignment is done in a plain round-robin fashion.
    const u32 Index = atomic_fetch_add(&CurrentIndex, 1U, memory_order_relaxed);
    setCurrentTSD(&TSDs[Index % atomic_load_relaxed(&NumberOfTSDs)]);
    Instance->callPostInitCallback();
  }

  // TSDs is an array of locks which is not supported for marking thread-safety
  // capability.
  NOINLINE TSD<Allocator> *getTSDSlow(TSD<Allocator> *CurrentTSD) {
    const u32 TotalTSDs = atomic_load_relaxed(&NumberOfTSDs);
    if (UNLIKELY(TotalTSDs == 1U)) {
      CurrentTSD->lock();
      return CurrentTSD;
    }

    // Use the Precedence of the current TSD as our random seed. Since we are
    // in the slow path, it means that tryLock failed, and as a result it's
    // very likely that said Precedence is non-zero.
    const u32 R = static_cast<u32>(CurrentTSD->getPrecedence());
    const u32 TotalCoPrimes = atomic_load_relaxed(&NumberOfCoPrimes);
    DCHECK_NE(TotalCoPrimes, 0U);
    const u32 Inc = atomic_load_relaxed(&CoPrimes[R % TotalCoPrimes]);

    u32 Index = R % TotalTSDs;
    uptr LowestPrecedence = UINTPTR_MAX;
    TSD<Allocator> *CandidateTSD = nullptr;
    // Go randomly through at most 4 contexts and find a candidate.
    for (u32 I = 0; I < Min(4U, TotalTSDs); I++) {
      if (TSDs[Index].tryLock()) {
        setCurrentTSD(&TSDs[Index]);
        return &TSDs[Index];
      }
      const uptr Precedence = TSDs[Index].getPrecedence();
      // A 0 precedence here means another thread just locked this TSD.
      if (Precedence && Precedence < LowestPrecedence) {
        CandidateTSD = &TSDs[Index];
        LowestPrecedence = Precedence;
      }
      Index += Inc;
      if (Index >= TotalTSDs)
        Index -= TotalTSDs;
    }
    if (CandidateTSD) {
      CandidateTSD->lock();
      setCurrentTSD(CandidateTSD);
      return CandidateTSD;
    }
    // Last resort, stick with the current one.
    CurrentTSD->lock();
    return CurrentTSD;
  }

  atomic_u32 CurrentIndex = {};
  atomic_u32 NumberOfTSDs = {};
  atomic_u32 NumberOfCoPrimes = {};
  atomic_u32 CoPrimes[TSDsArraySize] = {};
  atomic_u8 Initialized = {};
  // Used for global initialization and TSDs access.
  // Acquiring the global initialization should only lock once in normal
  // operation, which is why using it for TSDs access should not cause
  // any interference.
  HybridMutex Mutex;
  TSD<Allocator> TSDs[TSDsArraySize];
};

} // namespace scudo

#endif // SCUDO_TSD_SHARED_H_