diff --git a/Fleece/Support/AtomicRetained.hh b/Fleece/Support/AtomicRetained.hh new file mode 100644 index 00000000..f8630503 --- /dev/null +++ b/Fleece/Support/AtomicRetained.hh @@ -0,0 +1,193 @@ +// +// AtomicRetained.cc +// +// Copyright 2016-Present Couchbase, Inc. +// +// Use of this software is governed by the Business Source License included +// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +// in that file, in accordance with the Business Source License, use of this +// software will be governed by the Apache License, Version 2.0, included in +// the file licenses/APL2.txt. +// + +#pragma once +#include "fleece/RefCounted.hh" + +namespace fleece { + + namespace internal { + class AtomicWrapper { + public: + explicit AtomicWrapper(uintptr_t ref) noexcept; + explicit AtomicWrapper(const void* FL_NULLABLE ref) noexcept + :AtomicWrapper(reinterpret_cast(ref)) { } + + /// Safe access to `_ref` inside a callback. Returns whatever the callback returned. + template FN> + auto use(FN fn) const noexcept { + uintptr_t ref = getAndLock(); + auto result = fn(ref); + setAndUnlock(ref, ref); + return result; + } + /// Atomically swaps `_ref` with `newRef` and returns the old value. + uintptr_t exchangeWith(uintptr_t newRef) noexcept; + + private: + uintptr_t getAndLock() const noexcept; + void setAndUnlock(uintptr_t oldRef, uintptr_t newRef) const noexcept; + + mutable std::atomic _ref; + }; + } + + + /** A fully thread-safe version of `Retained` that supports concurrent gets and sets. + * It's a lot slower than `Retained`, so use it only when necessary: in situations where it + * will be called on multiple threads. */ + template + class AtomicRetained { + public: + using T_ptr = typename Retained::template nullable_if::ptr; + using T_nullable_ptr = T* FL_NULLABLE; + + AtomicRetained() noexcept requires (N==MaybeNull) :_ref(nullptr) { } + AtomicRetained(std::nullptr_t) noexcept requires (N==MaybeNull) :AtomicRetained() { } + AtomicRetained(T_ptr t) noexcept :_ref(_retain(t)) { } + + AtomicRetained(const AtomicRetained &r) noexcept :_ref(r._getRef()) { } + AtomicRetained(AtomicRetained &&r) noexcept :_ref(std::move(r).detach()) { } + + template requires (std::derived_from && N >= UN) + AtomicRetained(const AtomicRetained &r) noexcept :_ref(r._getRef()) { } + template requires (std::derived_from && N >= UN) + AtomicRetained(AtomicRetained &&r) noexcept :_ref(std::move(r).detach()) { } + + ~AtomicRetained() noexcept {release(_unretainedGet());} + + AtomicRetained& operator=(T_ptr r) & noexcept { + auto oldRef = _exchangeWith(_retain(r)); + release(oldRef); + return *this; + } + + AtomicRetained& operator=(const AtomicRetained &r) & noexcept { + *this = r.get(); return *this; + } + + template requires(std::derived_from && N >= UN) + AtomicRetained& operator=(const AtomicRetained &r) & noexcept { + *this = r.get(); return *this; + } + + template requires(std::derived_from && N >= UN) + AtomicRetained& operator=(const Retained &r) & noexcept { + *this = r.get(); return *this; + } + + AtomicRetained& operator= (AtomicRetained &&r) & noexcept { + T_ptr newRef = std::move(r).detach(); // retained + auto oldRef = _exchangeWith(newRef); + release(oldRef); + return *this; + } + + template requires(std::derived_from && N >= UN) + AtomicRetained& operator= (AtomicRetained &&r) & noexcept { + T_ptr newRef = std::move(r).detach(); // newRef is already retained + auto oldRef = _exchangeWith(newRef); + release(oldRef); + return *this; + } + + explicit operator bool () const FLPURE {return N==NonNull || (get() != nullptr);} + + // AtomicRetained doesn't dereference to `T*`, rather to `Retained`. + // This prevents a concurrent mutation from releasing the object out from under you, + // since that temporary Retained object keeps it alive for the duration of the expression. + + operator Retained () const & noexcept LIFETIMEBOUND FLPURE STEPOVER {return _get();} + Retained operator-> () const noexcept LIFETIMEBOUND FLPURE STEPOVER {return _get();} + Retained get() const noexcept LIFETIMEBOUND FLPURE STEPOVER {return _get();} + + /// Converts any AtomicRetained to non-nullable form (AtomicRef), or throws if its value is nullptr. + AtomicRetained asRef() const & noexcept(!N) {return AtomicRetained(get());} + AtomicRetained asRef() && noexcept(!N) { + return AtomicRetained(std::move(*this).detach(), false); // note: non-retaining constructor + } + + /// Converts a AtomicRetained into a raw pointer with a +1 reference that must be released. + /// Used in C++ functions that bridge to C and return C references. + /// @note The opposite of this is \ref adopt. + [[nodiscard]] + T_ptr detach() && noexcept { + auto oldRef = _exchangeWith(nullptr); + return oldRef; + } + + /// Converts a raw pointer with a +1 reference into a AtomicRetained object. + /// This has no effect on the object's ref-count; the existing +1 ref will be released when + /// the AtomicRetained destructs. */ + [[nodiscard]] static AtomicRetained adopt(T_ptr t) noexcept { + return AtomicRetained(t, false); + } + + private: + template friend class AtomicRetained; + + AtomicRetained(T_ptr t, bool) noexcept(N==MaybeNull) :_ref(t) { // private no-retain ctor + if constexpr (N == NonNull) { + if (t == nullptr) [[unlikely]] + _failNullRef(); + } + } + + Retained _get() const { + return Retained::adopt(_getRef()); + } + + T_nullable_ptr _getRef() const { + return _ref.use( [](auto r) { + return retain(reinterpret_cast(r)); + } ); + } + + T_nullable_ptr _unretainedGet() const { + return reinterpret_cast(_ref.use( [](auto r) {return r;} )); + } + + T_nullable_ptr _exchangeWith(T_nullable_ptr ref) { + return reinterpret_cast(_ref.exchangeWith(reinterpret_cast(ref))); + } + + static T_ptr _retain(T_ptr t) noexcept { + if constexpr (N == NonNull && std::derived_from) + t->_retain(); // this is faster, and it detects illegal null (by signal) + else + retain(t); + return t; + } + + static void _release(T_ptr t) noexcept { + if constexpr (N == NonNull && std::derived_from) + t->_release(); // this is faster, and it detects illegal null (by signal) + else + release(t); + } + + internal::AtomicWrapper _ref; + }; + + + /// Ref is an alias for a non-nullable AtomicRetained. + template using AtomicRef = AtomicRetained; + + /// NullableRef is an alias for a (default) nullable AtomicRetained. + template using AtomicNullableRef = AtomicRetained; + + /// RetainedConst is an alias for a AtomicRetained that holds a const pointer. + template using AtomicRetainedConst = AtomicRetained; + + + template AtomicRetained(T* FL_NULLABLE) -> AtomicRetained; // deduction guide +} diff --git a/Fleece/Support/RefCounted.cc b/Fleece/Support/RefCounted.cc index 2e05f7cc..a8b3c3bc 100644 --- a/Fleece/Support/RefCounted.cc +++ b/Fleece/Support/RefCounted.cc @@ -11,7 +11,9 @@ // #include "fleece/RefCounted.hh" +#include "AtomicRetained.hh" #include "Backtrace.hh" +#include "betterassert.hh" #include #include #include @@ -131,5 +133,47 @@ namespace fleece { } + namespace internal { + /// Tag bit that's added to `_ref` while accessing it. + /// We can't use the low bit (1) because mutable Fleece Values already use that as a tag. + static constexpr uintptr_t kBusyMask = uintptr_t(1) << 63; + + AtomicWrapper::AtomicWrapper(uintptr_t ref) noexcept + :_ref(ref) + { + assert((ref & kBusyMask) == 0); + } + + + uintptr_t AtomicWrapper::exchangeWith(uintptr_t newRef) noexcept { + uintptr_t oldRef = getAndLock(); + setAndUnlock(oldRef, newRef); + return oldRef; + } + + /// Loads and returns the value of `_ref`, while atomically setting `_ref`s high bit + /// to mark it as busy. If the high bit is set, busy-waits until it's cleared. + /// (The busy-wait should be OK since the high bit will only be set very briefly.) + /// @warning This MUST be followed ASAP by `setAndUnlock`. + uintptr_t AtomicWrapper::getAndLock() const noexcept { + uintptr_t r = _ref.load(std::memory_order_acquire); + while (true) { + if (r & kBusyMask) + r = _ref.load(std::memory_order_acquire); + else if (_ref.compare_exchange_strong(r, r | kBusyMask, std::memory_order_acquire)) + break; + } + assert((r & kBusyMask) == 0); + return r; + } + + /// Changes `_ref`s value from `oldRef` to `newRef`, clearing the busy bit. + /// MUST only be called after `getAndLock`. + void AtomicWrapper::setAndUnlock(uintptr_t oldRef, uintptr_t newRef) const noexcept { + uintptr_t r = oldRef | kBusyMask; + bool ok = _ref.compare_exchange_strong(r, newRef, std::memory_order_release); + assert(ok); + } + } } diff --git a/Tests/PerfTests.cc b/Tests/PerfTests.cc index 25acdc82..4568415c 100644 --- a/Tests/PerfTests.cc +++ b/Tests/PerfTests.cc @@ -59,8 +59,8 @@ TEST_CASE("GetUVarint performance", "[.Perf]") { } bench.stop(); CHECK(result != 1); // bogus - fprintf(stderr, "n = %16llx; %2zd bytes; time = %.3f ns\n", - (long long)n, nBytes, + fprintf(stderr, "n = %16llx; %2zu bytes; time = %.3f ns\n", + (unsigned long long)n, nBytes, bench.elapsed() / kNRounds * 1.0e9); d *= 1.5; } diff --git a/Tests/SupportTests.cc b/Tests/SupportTests.cc index 01e1e477..c7b1e39b 100644 --- a/Tests/SupportTests.cc +++ b/Tests/SupportTests.cc @@ -14,6 +14,7 @@ #include "fleece/InstanceCounted.hh" #include "FleeceTests.hh" #include "FleeceImpl.hh" +#include "AtomicRetained.hh" #include "Backtrace.hh" #include "ConcurrentMap.hh" #include "Bitmap.hh" @@ -234,7 +235,7 @@ TEST_CASE("ConcurrentMap concurrency", "[ConcurrentMap]") { constexpr size_t bufSize = 10; for (int i = 0; i < kSize; i++) { char keybuf[bufSize]; - snprintf(keybuf, bufSize, "%x", i); + snprintf(keybuf, bufSize, "%x", unsigned(i)); keys.push_back(keybuf); } @@ -400,3 +401,22 @@ TEST_CASE("InstanceCounted") { } CHECK(InstanceCounted::liveInstanceCount() == n); } + + +struct AtomicRetainedTest : RefCounted { + int i = 0; + string str; +}; + + +TEST_CASE("AtomicRetained") { + // This is mostly to check for compile errors. + AtomicRetained r = new AtomicRetainedTest(); + CHECK(r->i == 0); + AtomicRetained r2 = r; + AtomicRetained r3 = std::move(r2); + CHECK(r->refCount() == 3); + // `r` is one reference, `r3` is another, and the temporary created by `->` was the third. + + AtomicRef rr = new AtomicRetainedTest(); +}