123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491 |
- #pragma once
-
- // Baselib C++ Atomics
-
- // Interface that sticks closely to std::atomic (as of C++11 for the most part)
- // See: https://en.cppreference.com/w/cpp/atomic/atomic
- //
- // ATTENTION: For free functions (not using the baselib::atomic struct), the caller must ensure that the address of obj is aligned to the size of its respective target. (e.g. 64bit values need to be aligned to 8 bytes)
- // Failure to comply is undefined behavior (in practice depending on compiler & architecture either crash, non-atomicity or slow performance)
- // ALIGN_ATOMIC can be used to ensure this, but generally it is recommended to use the baselib::atomic struct instead!
- //
- // Forwards internally to C atomic header Baselib_Atomic.h (see also Baselib_Atomic_TypeSafe.h for typesafe C atomics)
-
- // -----------------------------------------------------------------------------------------------------------------------------------------
- // Differences to C++11 atomics:
- //
- // * free functions that operate on types other than baselib::atomic
- // * baselib::atomic allows access to its internal value
- // * no zero initialization on baselib::atomic
- // * guaranteed to be lock-free* (optional for std::atomic)
- // * restricts object size from sizeof(char) to 2 * sizeof(void*)
- // * (*) unless build is configured correctly, some platforms may forward to libatomic, which may cause locking and prohibit inlining
- // * compare_exchange
- // * no single parameter versions
- // * compare_exchange is not allowed with (success == memory_order_release && failure == memory_order_acquire)
- // meaning, that unlike defined in the standard, the failure barrier must be weaker (instead of "not stronger") than the success barrier
- // => Use instead: (success == memory_order_acq_rel && failure == memory_order_acquire)
- // -----------------------------------------------------------------------------------------------------------------------------------------
-
- // -----------------------------------------------------------------------------------------------------------------------------------------
- // Why do we provide our own atomics
- //
- // * allows for platform specific modifications if need to be (had bugs as well as suboptimal performance in the past on some platforms)
- // * forbid lockbased implementation
- // * be able to operate on values that aren’t of type std/baselib::atomic<T>
- // * avoid indirections that do not always optimize out as expected
- // * can enforce avoiding use of libatomic
- // * control debug overhead (substantial with std::atomic on some platforms)
- // * striving for more consistent cross platform implementation by providing compiler intrinsic based implementations
- // * allows adding extensions
- // -----------------------------------------------------------------------------------------------------------------------------------------
-
-
- #include "../C/Baselib_Atomic.h"
- #include "Internal/TypeTraits.h"
-
- // Note that aligning by type is not possible with the C compatible COMPILER_ALIGN_AS as MSVC's own alignment attribute does not allow evaluation of sizeof
- #define ALIGN_ATOMIC(TYPE_) alignas(sizeof(TYPE_))
- #define ALIGNED_ATOMIC(TYPE_) ALIGN_ATOMIC(TYPE_) TYPE_
-
- namespace baselib
- {
- BASELIB_CPP_INTERFACE
- {
- enum memory_order_relaxed_t { memory_order_relaxed = 0 }; // Equal to std::memory_order_relaxed
- enum memory_order_acquire_t { memory_order_acquire = 2 }; // Equal to std::memory_order_acquire
- enum memory_order_release_t { memory_order_release = 3 }; // Equal to std::memory_order_release
- enum memory_order_acq_rel_t { memory_order_acq_rel = 4 }; // Equal to std::memory_order_acq_rel
- enum memory_order_seq_cst_t { memory_order_seq_cst = 5 }; // Equal to std::memory_order_seq_cst
-
- namespace detail
- {
- template<typename T, typename ... Rest>
- struct is_any : std::false_type {};
-
- template<typename T, typename First>
- struct is_any<T, First> : std::is_same<T, First> {};
-
- template<typename T, typename First, typename ... Rest>
- struct is_any<T, First, Rest...>
- : std::integral_constant<bool, std::is_same<T, First>::value || is_any<T, Rest...>::value>
- {};
-
- #define TEST_ATOMICS_PREREQUISITES(_TYPE) \
- static_assert(baselib::is_trivially_copyable<_TYPE>::value, "atomic operation operands needs to be trivially copyable"); \
- static_assert(sizeof(_TYPE) <= sizeof(void*) * 2, "atomic operation operands need to be smaller or equal than two pointers");
-
- template<typename T> static inline T fail();
-
- template<typename T, typename MemoryOrder, typename ... AllowedMemoryOrders> static inline T fail_prerequisites()
- {
- TEST_ATOMICS_PREREQUISITES(T);
- static_assert(is_any<MemoryOrder, AllowedMemoryOrders...>::value, "the specified memory ordering is invalid for this atomic operation");
- return fail<T>();
- }
-
- template<typename T, typename MemoryOrderSuccess, typename MemoryOrderFailure> static inline T fail_prerequisites_cmpxchg()
- {
- TEST_ATOMICS_PREREQUISITES(T);
-
- // Special error message for success: release, fail: acquire
- static_assert(!(std::is_same<MemoryOrderSuccess, baselib::memory_order_release_t>::value && std::is_same<MemoryOrderFailure, baselib::memory_order_acquire_t>::value),
- "Unlike std::atomic, baselib does not allow compare_exchange with memory ordering release on success and acquire on failure. Use acq_rel for success instead.\n"
- "This restriction is in place to avoid confusion both by users and implementors on the semantics of such an operation which would not be allowed to do an acquire barrier on load "
- "but still implies a dedicated acquire fence if (and only if) the operation fails. "
- "Scenarios where the user expects acquire on load and release on write are best expressed with acq_rel for success and acquire on failure.");
-
- static_assert(
- // fail: relaxed, success: relaxed/acquire/release/seq_cst
- (std::is_same<MemoryOrderFailure, baselib::memory_order_relaxed_t>::value &&
- is_any<MemoryOrderSuccess, baselib::memory_order_relaxed_t, baselib::memory_order_acquire_t, baselib::memory_order_release_t, baselib::memory_order_seq_cst_t>::value) ||
- // fail: acquire, success acquire/release/seq_cst
- (std::is_same<MemoryOrderFailure, baselib::memory_order_relaxed_t>::value &&
- is_any<MemoryOrderSuccess, baselib::memory_order_acquire_t, baselib::memory_order_release_t, baselib::memory_order_seq_cst_t>::value) ||
- // fail: seq_cst, success: seq_cst
- (std::is_same<MemoryOrderSuccess, baselib::memory_order_seq_cst_t>::value && std::is_same<MemoryOrderFailure, baselib::memory_order_seq_cst_t>::value),
- "the specified combination of memory ordering is invalid for compare exchange operations");
- return fail<T>();
- }
-
- template<typename T, typename MemoryOrder> static inline T fail_prerequisites_alu()
- {
- static_assert(std::is_integral<T>::value, "operands of arithmetic atomic operations need to be integral");
- return fail_prerequisites<T, MemoryOrder,
- baselib::memory_order_relaxed_t,
- baselib::memory_order_acquire_t,
- baselib::memory_order_release_t,
- baselib::memory_order_acq_rel_t,
- baselib::memory_order_seq_cst_t>();
- }
- }
-
- // MACRO generated impl
- // re-directs to Baselib_atomic_ API
- // ----------------------------------------------------------------------------------------------------------------------------------
- #define detail_THREAD_FENCE(order, ...) \
- static FORCE_INLINE void atomic_thread_fence(memory_order_##order##_t order) \
- { \
- return Baselib_atomic_thread_fence_##order(); \
- }
-
- #define detail_LOAD(op, order, id, bits, ...) \
- template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
- static FORCE_INLINE T atomic_load_explicit(const T& obj, memory_order_##order##_t order) \
- { \
- T ret; \
- Baselib_atomic_load_##id##_##order##_v(&obj, &ret); \
- return ret; \
- }
-
- #define detail_LOAD128(op, order, id, bits, ...) \
- template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
- static FORCE_INLINE T atomic_load_explicit(const T& obj, memory_order_##order##_t order) \
- { \
- T ret; \
- Baselib_atomic_load_##id##_##order##_v(const_cast<T*>(&obj), &ret); \
- return ret; \
- }
-
- #define detail_STORE(op, order, id, bits, ...) \
- template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
- static FORCE_INLINE void atomic_store_explicit(T& obj, typename std::common_type<T>::type value, memory_order_##order##_t order)\
- { \
- return Baselib_atomic_store_##id##_##order##_v(&obj, &value); \
- }
-
- #define detail_LOAD_STORE(op, order, id, bits, ...) \
- template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
- static FORCE_INLINE T atomic_##op##_explicit(T& obj, typename std::common_type<T>::type value, memory_order_##order##_t order) \
- { \
- T ret; \
- Baselib_atomic_##op##_##id##_##order##_v(&obj, &value, &ret); \
- return ret; \
- }
-
- #define detail_ALU(op, order, id, bits, ...) \
- template<typename T, typename std::enable_if<baselib::is_integral_of_size<T, bits/8>::value, int>::type = 0> \
- static FORCE_INLINE T atomic_##op##_explicit(T& obj, typename std::common_type<T>::type value, memory_order_##order##_t order) \
- { \
- T ret; \
- Baselib_atomic_##op##_##id##_##order##_v(&obj, &value, &ret); \
- return ret; \
- }
-
- #define detail_CMP_XCHG(op, order1, order2, id, bits, ...) \
- template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
- static FORCE_INLINE bool atomic_##op##_explicit(T& obj, \
- typename std::common_type<T>::type& expected, \
- typename std::common_type<T>::type desired, \
- memory_order_##order1##_t order_success, \
- memory_order_##order2##_t order_failure) \
- { \
- return Baselib_atomic_##op##_##id##_##order1##_##order2##_v(&obj, &expected, &desired); \
- }
-
- #define detail_NOT_SUPPORTED(...)
-
- Baselib_Atomic_FOR_EACH_MEMORY_ORDER(
- detail_THREAD_FENCE
- )
- Baselib_Atomic_FOR_EACH_ATOMIC_OP_MEMORY_ORDER_AND_INT_TYPE(
- detail_LOAD, // load
- detail_STORE, // store
- detail_ALU, // add
- detail_ALU, // and
- detail_ALU, // or
- detail_ALU, // xor
- detail_LOAD_STORE, // exchange
- detail_CMP_XCHG, // compare_exchange_weak
- detail_CMP_XCHG // compare_exchange_strong
- )
-
- #if PLATFORM_ARCH_64
- // 128bit atomics
- Baselib_Atomic_FOR_EACH_ATOMIC_OP_AND_MEMORY_ORDER(
- detail_LOAD128, // load
- detail_STORE, // store
- detail_NOT_SUPPORTED, // add
- detail_NOT_SUPPORTED, // and
- detail_NOT_SUPPORTED, // or
- detail_NOT_SUPPORTED, // xor
- detail_LOAD_STORE, // exchange
- detail_CMP_XCHG, // compare_exchange_weak
- detail_CMP_XCHG, // compare_exchange_strong
- 128, 128)
- #endif
-
- #undef detail_THREAD_FENCE
- #undef detail_LOAD128
- #undef detail_LOAD
- #undef detail_STORE
- #undef detail_LOAD_STORE
- #undef detail_ALU
- #undef detail_CMP_XCHG
- #undef detail_NOT_SUPPORTED
-
- template<typename T, typename MemoryOrder>
- static FORCE_INLINE T atomic_fetch_sub_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
- {
- return baselib::atomic_fetch_add_explicit(obj, 0 - value, order);
- }
-
- // API documentation and default fallback for non-matching types
- // ----------------------------------------------------------------------------------------------------------------------
- template<typename T, typename MemoryOrder>
- static FORCE_INLINE T atomic_load_explicit(const T& obj, MemoryOrder order)
- {
- return detail::fail_prerequisites<T, MemoryOrder, baselib::memory_order_relaxed_t, baselib::memory_order_acquire_t, baselib::memory_order_seq_cst_t>();
- }
-
- template<typename T, typename MemoryOrder>
- static FORCE_INLINE void atomic_store_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
- {
- detail::fail_prerequisites<T, MemoryOrder, baselib::memory_order_relaxed_t, baselib::memory_order_release_t, baselib::memory_order_seq_cst_t>();
- }
-
- template<typename T, typename MemoryOrder>
- static FORCE_INLINE T atomic_fetch_add_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
- {
- return detail::fail_prerequisites_alu<T, MemoryOrder>();
- }
-
- template<typename T, typename MemoryOrder>
- static FORCE_INLINE T atomic_fetch_and_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
- {
- return detail::fail_prerequisites_alu<T, MemoryOrder>();
- }
-
- template<typename T, typename MemoryOrder>
- static FORCE_INLINE T atomic_fetch_or_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
- {
- return detail::fail_prerequisites_alu<T, MemoryOrder>();
- }
-
- template<typename T, typename MemoryOrder>
- static FORCE_INLINE T atomic_fetch_xor_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
- {
- return detail::fail_prerequisites_alu<T, MemoryOrder>();
- }
-
- template<typename T, typename MemoryOrder>
- static FORCE_INLINE T atomic_exchange_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
- {
- return detail::fail_prerequisites<T, MemoryOrder>();
- }
-
- template<typename T, typename MemoryOrderSuccess, typename MemoryOrderFailure>
- static FORCE_INLINE bool atomic_compare_exchange_weak_explicit(T& obj,
- typename std::common_type<T>::type& expected,
- typename std::common_type<T>::type desired,
- MemoryOrderSuccess order_success,
- MemoryOrderFailure order_failure)
- {
- detail::fail_prerequisites_cmpxchg<T, MemoryOrderSuccess, MemoryOrderFailure>();
- return false;
- }
-
- template<typename T, typename MemoryOrderSuccess, typename MemoryOrderFailure>
- static FORCE_INLINE bool atomic_compare_exchange_strong_explicit(T& obj,
- typename std::common_type<T>::type& expected,
- typename std::common_type<T>::type desired,
- MemoryOrderSuccess order_success,
- MemoryOrderFailure order_failure)
- {
- detail::fail_prerequisites_cmpxchg<T, MemoryOrderSuccess, MemoryOrderFailure>();
- return false;
- }
-
- // default memory order (memory_order_seq_cst)
- // ----------------------------------------------------------------------------------------------------------------------
- template<typename T>
- static FORCE_INLINE T atomic_load(const T& obj)
- {
- return baselib::atomic_load_explicit(obj, memory_order_seq_cst);
- }
-
- template<typename T>
- static FORCE_INLINE void atomic_store(T& obj, typename std::common_type<T>::type value)
- {
- return baselib::atomic_store_explicit(obj, value, memory_order_seq_cst);
- }
-
- template<typename T>
- static FORCE_INLINE T atomic_fetch_add(T& obj, typename std::common_type<T>::type value)
- {
- return baselib::atomic_fetch_add_explicit(obj, value, memory_order_seq_cst);
- }
-
- template<typename T>
- static FORCE_INLINE T atomic_fetch_sub(T& obj, typename std::common_type<T>::type value)
- {
- return baselib::atomic_fetch_sub_explicit(obj, value, memory_order_seq_cst);
- }
-
- template<typename T>
- static FORCE_INLINE T atomic_fetch_and(T& obj, typename std::common_type<T>::type value)
- {
- return baselib::atomic_fetch_and_explicit(obj, value, memory_order_seq_cst);
- }
-
- template<typename T>
- static FORCE_INLINE T atomic_fetch_or(T& obj, typename std::common_type<T>::type value)
- {
- return baselib::atomic_fetch_or_explicit(obj, value, memory_order_seq_cst);
- }
-
- template<typename T>
- static FORCE_INLINE T atomic_fetch_xor(T& obj, typename std::common_type<T>::type value)
- {
- return baselib::atomic_fetch_xor_explicit(obj, value, memory_order_seq_cst);
- }
-
- template<typename T>
- static FORCE_INLINE T atomic_exchange(T& obj, typename std::common_type<T>::type value)
- {
- return baselib::atomic_exchange_explicit(obj, value, memory_order_seq_cst);
- }
-
- template<typename T>
- static FORCE_INLINE bool atomic_compare_exchange_weak(T& obj,
- typename std::common_type<T>::type& expected,
- typename std::common_type<T>::type desired)
- {
- return baselib::atomic_compare_exchange_weak_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
- }
-
- template<typename T>
- static FORCE_INLINE bool atomic_compare_exchange_strong(T& obj,
- typename std::common_type<T>::type& expected,
- typename std::common_type<T>::type desired)
- {
- return baselib::atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
- }
-
- template<typename T>
- struct atomic_common
- {
- using value_type = T;
-
- TEST_ATOMICS_PREREQUISITES(T);
-
- ALIGNED_ATOMIC(T) obj;
-
- FORCE_INLINE atomic_common() = default;
-
- // Initializes atomic with a given value. Initialization is not atomic!
- FORCE_INLINE atomic_common(T value)
- {
- obj = value;
- }
-
- FORCE_INLINE operator T() const { return baselib::atomic_load_explicit(obj, memory_order_seq_cst); }
- FORCE_INLINE T operator=(T value) { baselib::atomic_store_explicit(obj, value, memory_order_seq_cst); return value; }
-
- template<typename TMemoryOrder = memory_order_seq_cst_t>
- FORCE_INLINE T load(TMemoryOrder order = memory_order_seq_cst) const
- {
- return baselib::atomic_load_explicit(obj, order);
- }
-
- template<typename TMemoryOrder = memory_order_seq_cst_t>
- FORCE_INLINE void store(T value, TMemoryOrder order = memory_order_seq_cst)
- {
- return baselib::atomic_store_explicit(obj, value, order);
- }
-
- template<typename TMemoryOrder = memory_order_seq_cst_t>
- FORCE_INLINE T exchange(T value, TMemoryOrder order = memory_order_seq_cst)
- {
- return baselib::atomic_exchange_explicit(obj, value, order);
- }
-
- template<typename TMemoryOrderSuccess, typename TMemoryOrderFailure>
- FORCE_INLINE bool compare_exchange_weak(T& expected, T desired, TMemoryOrderSuccess order_success, TMemoryOrderFailure order_failure)
- {
- return baselib::atomic_compare_exchange_weak_explicit(obj, expected, desired, order_success, order_failure);
- }
-
- FORCE_INLINE bool compare_exchange_weak(T& expected, T desired)
- {
- return baselib::atomic_compare_exchange_weak_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
- }
-
- template<typename TMemoryOrderSuccess, typename TMemoryOrderFailure>
- FORCE_INLINE bool compare_exchange_strong(T& expected, T desired, TMemoryOrderSuccess order_success, TMemoryOrderFailure order_failure)
- {
- return baselib::atomic_compare_exchange_strong_explicit(obj, expected, desired, order_success, order_failure);
- }
-
- FORCE_INLINE bool compare_exchange_strong(T& expected, T desired)
- {
- return baselib::atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
- }
- };
-
- template<typename T, bool IsIntegral>
- struct atomic_base {};
-
- // Atomic type for integral types.
- template<typename T>
- struct atomic_base<T, true> : atomic_common<T>
- {
- using atomic_common<T>::atomic_common;
-
- template<typename TMemoryOrder = memory_order_seq_cst_t>
- FORCE_INLINE T fetch_add(T value, TMemoryOrder order = memory_order_seq_cst)
- {
- return baselib::atomic_fetch_add_explicit(atomic_common<T>::obj, value, order);
- }
-
- template<typename TMemoryOrder = memory_order_seq_cst_t>
- FORCE_INLINE T fetch_sub(T value, TMemoryOrder order = memory_order_seq_cst)
- {
- return baselib::atomic_fetch_sub_explicit(atomic_common<T>::obj, value, order);
- }
-
- template<typename TMemoryOrder = memory_order_seq_cst_t>
- FORCE_INLINE T fetch_and(T value, TMemoryOrder order = memory_order_seq_cst)
- {
- return baselib::atomic_fetch_and_explicit(atomic_common<T>::obj, value, order);
- }
-
- template<typename TMemoryOrder = memory_order_seq_cst_t>
- FORCE_INLINE T fetch_or(T value, TMemoryOrder order = memory_order_seq_cst)
- {
- return baselib::atomic_fetch_or_explicit(atomic_common<T>::obj, value, order);
- }
-
- template<typename TMemoryOrder = memory_order_seq_cst_t>
- FORCE_INLINE T fetch_xor(T value, TMemoryOrder order = memory_order_seq_cst)
- {
- return baselib::atomic_fetch_xor_explicit(atomic_common<T>::obj, value, order);
- }
-
- FORCE_INLINE T operator++(int) { return baselib::atomic_fetch_add_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst); }
- FORCE_INLINE T operator--(int) { return baselib::atomic_fetch_sub_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst); }
- FORCE_INLINE T operator++() { return baselib::atomic_fetch_add_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst) + T(1); }
- FORCE_INLINE T operator--() { return baselib::atomic_fetch_sub_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst) - T(1); }
- FORCE_INLINE T operator+=(T value) { return baselib::atomic_fetch_add_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) + value; }
- FORCE_INLINE T operator-=(T value) { return baselib::atomic_fetch_sub_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) - value; }
- FORCE_INLINE T operator&=(T value) { return baselib::atomic_fetch_and_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) & value; }
- FORCE_INLINE T operator|=(T value) { return baselib::atomic_fetch_or_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) | value; }
- FORCE_INLINE T operator^=(T value) { return baselib::atomic_fetch_xor_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) ^ value; }
- };
-
- // Atomic type for non-integral types.
- template<typename T>
- struct atomic_base<T, false> : atomic_common<T>
- {
- using atomic_common<T>::atomic_common;
- };
-
- template<typename T>
- struct atomic : atomic_base<T, std::is_integral<T>::value>
- {
- using atomic_base<T, std::is_integral<T>::value>::atomic_base;
- };
-
- #undef TEST_ATOMICS_PREREQUISITES
- }
- }
|