| //===- LazyAtomicPointer.----------------------------------------*- C++ -*-===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #ifndef LLVM_ADT_LAZYATOMICPOINTER_H |
| #define LLVM_ADT_LAZYATOMICPOINTER_H |
| |
| #include "llvm/ADT/STLFunctionalExtras.h" |
| #include "llvm/Support/Compiler.h" |
| #include <assert.h> |
| #include <atomic> |
| |
| namespace llvm { |
| |
| /// Atomic pointer that's lock-free, but that can coordinate concurrent writes |
| /// from a lazy generator. Should be reserved for cases where concurrent uses of |
| /// a generator for the same storage is unlikely. |
| /// |
| /// The laziness comes in with \a loadOrGenerate(), which lazily calls the |
| /// provided generator ONLY when the value is currently \c nullptr. With |
| /// concurrent calls, only one generator is called and the rest see that value. |
| /// |
| /// Most other APIs treat an in-flight \a loadOrGenerate() as if \c nullptr |
| /// were stored. APIs that are required to write a value will spin. |
| /// |
| /// The underlying storage is \a std::atomic<uintptr_t>. |
| /// |
| /// TODO: In C++20, use std::atomic<T>::wait() instead of spinning and call |
| /// std::atomic<T>::notify_all() in \a loadOrGenerate(). |
| template <class T> class LazyAtomicPointer { |
| static constexpr uintptr_t getNull() { return 0; } |
| static constexpr uintptr_t getBusy() { return UINTPTR_MAX; } |
| |
| static T *makePointer(uintptr_t Value) { |
| assert(Value != getBusy()); |
| return Value ? reinterpret_cast<T *>(Value) : nullptr; |
| } |
| static uintptr_t makeRaw(T *Value) { |
| uintptr_t Raw = Value ? reinterpret_cast<uintptr_t>(Value) : getNull(); |
| assert(Raw != getBusy()); |
| return Raw; |
| } |
| |
| public: |
| /// Store a value. Waits for concurrent \a loadOrGenerate() calls. |
| void store(T *Value) { return (void)exchange(Value); } |
| |
| /// Set a value. Return the old value. Waits for concurrent \a |
| /// loadOrGenerate() calls. |
| T *exchange(T *Value) { |
| // Note: the call to compare_exchange_weak() fails "spuriously" if the |
| // current value is \a getBusy(), causing the loop to spin. |
| T *Old = nullptr; |
| while (!compare_exchange_weak(Old, Value)) { |
| } |
| return Old; |
| } |
| |
| /// Compare-exchange. Returns \c false if there is a concurrent \a |
| /// loadOrGenerate() call, setting \p ExistingValue to \c nullptr. |
| bool compare_exchange_weak(T *&ExistingValue, T *NewValue) { |
| uintptr_t RawExistingValue = makeRaw(ExistingValue); |
| if (Storage.compare_exchange_weak(RawExistingValue, makeRaw(NewValue))) |
| return true; |
| |
| /// Report the existing value as "None" if busy. |
| if (RawExistingValue == getBusy()) |
| ExistingValue = nullptr; |
| else |
| ExistingValue = makePointer(RawExistingValue); |
| return false; |
| } |
| |
| /// Compare-exchange. Keeps trying if there is a concurrent |
| /// \a loadOrGenerate() call. |
| bool compare_exchange_strong(T *&ExistingValue, T *NewValue) { |
| uintptr_t RawExistingValue = makeRaw(ExistingValue); |
| const uintptr_t OriginalRawExistingValue = RawExistingValue; |
| if (Storage.compare_exchange_strong(RawExistingValue, makeRaw(NewValue))) |
| return true; |
| |
| /// Keep trying as long as it's busy. |
| if (LLVM_UNLIKELY(RawExistingValue == getBusy())) { |
| while (RawExistingValue == getBusy()) { |
| RawExistingValue = OriginalRawExistingValue; |
| if (Storage.compare_exchange_weak(RawExistingValue, makeRaw(NewValue))) |
| return true; |
| } |
| } |
| ExistingValue = makePointer(RawExistingValue); |
| return false; |
| } |
| |
| /// Return the current stored value. Returns \a None if there is a concurrent |
| /// \a loadOrGenerate() in flight. |
| T *load() const { |
| uintptr_t RawValue = Storage.load(); |
| return RawValue == getBusy() ? nullptr : makePointer(RawValue); |
| } |
| |
| /// Get the current value, or call \p Generator to generate a value. |
| /// Guarantees that only one thread's \p Generator will run. |
| /// |
| /// \pre \p Generator doesn't return \c nullptr. |
| T &loadOrGenerate(function_ref<T *()> Generator) { |
| // Return existing value, if already set. |
| uintptr_t Raw = Storage.load(); |
| if (Raw != getNull() && Raw != getBusy()) |
| return *makePointer(Raw); |
| |
| // Try to mark as busy, then generate and store a new value. |
| if (LLVM_LIKELY(Raw == getNull() && |
| Storage.compare_exchange_strong(Raw, getBusy()))) { |
| Raw = makeRaw(Generator()); |
| assert(Raw != getNull() && "Expected non-null from generator"); |
| Storage.store(Raw); |
| return *makePointer(Raw); |
| } |
| |
| // Contended with another generator. Wait for it to complete. |
| while (Raw == getBusy()) |
| Raw = Storage.load(); |
| assert(Raw != getNull() && "Expected non-null from competing generator"); |
| return *makePointer(Raw); |
| } |
| |
| explicit operator bool() const { return load(); } |
| operator T *() const { return load(); } |
| |
| T &operator*() const { |
| T *P = load(); |
| assert(P && "Unexpected null dereference"); |
| return *P; |
| } |
| T *operator->() const { return &operator*(); } |
| |
| LazyAtomicPointer() : Storage(0) {} |
| LazyAtomicPointer(std::nullptr_t) : Storage(0) {} |
| LazyAtomicPointer(T *Value) : Storage(makeRaw(Value)) {} |
| LazyAtomicPointer(const LazyAtomicPointer &RHS) |
| : Storage(makeRaw(RHS.load())) {} |
| |
| LazyAtomicPointer &operator=(std::nullptr_t) { |
| store(nullptr); |
| return *this; |
| } |
| LazyAtomicPointer &operator=(T *RHS) { |
| store(RHS); |
| return *this; |
| } |
| LazyAtomicPointer &operator=(const LazyAtomicPointer &RHS) { |
| store(RHS.load()); |
| return *this; |
| } |
| |
| private: |
| std::atomic<uintptr_t> Storage; |
| }; |
| |
| } // end namespace llvm |
| |
| #endif // LLVM_ADT_LAZYATOMICPOINTER_H |