Implemented AtomicRetained / AtomicRef

Fleece/Support/AtomicRetained.hh

@@ -0,0 +1,193 @@
+//
+// AtomicRetained.cc
+//
+// Copyright 2016-Present Couchbase, Inc.
+//
+// Use of this software is governed by the Business Source License included
+// in the file licenses/BSL-Couchbase.txt.  As of the Change Date specified
+// in that file, in accordance with the Business Source License, use of this
+// software will be governed by the Apache License, Version 2.0, included in
+// the file licenses/APL2.txt.
+//
+
+#pragma once
+#include "fleece/RefCounted.hh"
+
+namespace fleece {
+
+    namespace internal {
+        class AtomicWrapper {
+        public:
+            explicit AtomicWrapper(uintptr_t ref) noexcept;
+            explicit AtomicWrapper(const void* FL_NULLABLE ref) noexcept
+                :AtomicWrapper(reinterpret_cast<uintptr_t>(ref)) { }
+
+            /// Safe access to `_ref` inside a callback. Returns whatever the callback returned.
+            template<std::invocable<uintptr_t> FN>
+            auto use(FN fn) const noexcept {
+                uintptr_t ref = getAndLock();
+                auto result = fn(ref);
+                setAndUnlock(ref, ref);
+                return result;
+            }
+            /// Atomically swaps `_ref` with `newRef` and returns the old value.
+            uintptr_t exchangeWith(uintptr_t newRef) noexcept;
+
+        private:
+            uintptr_t getAndLock() const noexcept;
+            void setAndUnlock(uintptr_t oldRef, uintptr_t newRef) const noexcept;
+
+            mutable std::atomic<uintptr_t> _ref;
+        };
+    }
+
+
+    /** A fully thread-safe version of `Retained` that supports concurrent gets and sets.
+     *  It's a lot slower than `Retained`, so use it only when necessary: in situations where it
+     *  will be called on multiple threads. */
+    template <class T, Nullability N = MaybeNull>
+    class AtomicRetained {
+    public:
+        using T_ptr = typename Retained<T,N>::template nullable_if<T,N>::ptr;
+        using T_nullable_ptr = T* FL_NULLABLE;
+
+        AtomicRetained() noexcept requires (N==MaybeNull)                 :_ref(nullptr) { }
+        AtomicRetained(std::nullptr_t) noexcept requires (N==MaybeNull)   :AtomicRetained() { }
+        AtomicRetained(T_ptr t) noexcept                                  :_ref(_retain(t)) { }
+
+        AtomicRetained(const AtomicRetained &r) noexcept        :_ref(r._getRef()) { }
+        AtomicRetained(AtomicRetained &&r) noexcept             :_ref(std::move(r).detach()) { }
+
+        template <typename U, Nullability UN> requires (std::derived_from<U,T> && N >= UN)
+        AtomicRetained(const AtomicRetained<U,UN> &r) noexcept  :_ref(r._getRef()) { }
+        template <typename U, Nullability UN> requires (std::derived_from<U,T> && N >= UN)
+        AtomicRetained(AtomicRetained<U,UN> &&r) noexcept       :_ref(std::move(r).detach()) { }
+
+        ~AtomicRetained() noexcept                              {release(_unretainedGet());}
+
+        AtomicRetained& operator=(T_ptr r) & noexcept {
+            auto oldRef = _exchangeWith(_retain(r));
+            release(oldRef);
+            return *this;
+        }
+
+        AtomicRetained& operator=(const AtomicRetained &r) & noexcept {
+            *this = r.get(); return *this;
+        }
+
+        template <typename U, Nullability UN> requires(std::derived_from<U,T> && N >= UN)
+        AtomicRetained& operator=(const AtomicRetained<U,UN> &r) & noexcept {
+            *this = r.get(); return *this;
+        }
+
+        template <typename U, Nullability UN> requires(std::derived_from<U,T> && N >= UN)
+        AtomicRetained& operator=(const Retained<U,UN> &r) & noexcept {
+            *this = r.get(); return *this;
+        }
+
+        AtomicRetained& operator= (AtomicRetained &&r) & noexcept {
+            T_ptr newRef = std::move(r).detach();  // retained
+            auto oldRef = _exchangeWith(newRef);
+            release(oldRef);
+            return *this;
+        }
+
+        template <typename U, Nullability UN> requires(std::derived_from<U,T> && N >= UN)
+        AtomicRetained& operator= (AtomicRetained<U,UN> &&r) & noexcept {
+            T_ptr newRef = std::move(r).detach(); // newRef is already retained
+            auto oldRef = _exchangeWith(newRef);
+            release(oldRef);
+            return *this;
+        }
+
+        explicit operator bool () const FLPURE          {return N==NonNull || (get() != nullptr);}
+
+        // AtomicRetained doesn't dereference to `T*`, rather to `Retained<T>`.
+        // This prevents a concurrent mutation from releasing the object out from under you,
+        // since that temporary Retained object keeps it alive for the duration of the expression.
+
+        operator Retained<T,N> () const & noexcept LIFETIMEBOUND FLPURE STEPOVER {return _get();}
+        Retained<T,N> operator-> () const noexcept LIFETIMEBOUND FLPURE STEPOVER {return _get();}
+        Retained<T,N> get() const noexcept LIFETIMEBOUND FLPURE STEPOVER         {return _get();}
+
+        /// Converts any AtomicRetained to non-nullable form (AtomicRef), or throws if its value is nullptr.
+        AtomicRetained<T,NonNull> asRef() const & noexcept(!N) {return AtomicRetained<T,NonNull>(get());}
+        AtomicRetained<T,NonNull> asRef() && noexcept(!N) {
+            return AtomicRetained<T,NonNull>(std::move(*this).detach(), false);  // note: non-retaining constructor
+        }
+
+        /// Converts a AtomicRetained into a raw pointer with a +1 reference that must be released.
+        /// Used in C++ functions that bridge to C and return C references.
+        /// @note  The opposite of this is \ref adopt.
+        [[nodiscard]]
+        T_ptr detach() && noexcept {
+            auto oldRef = _exchangeWith(nullptr);
+            return oldRef;
+        }
+
+        /// Converts a raw pointer with a +1 reference into a AtomicRetained object.
+        /// This has no effect on the object's ref-count; the existing +1 ref will be released when
+        /// the AtomicRetained destructs. */
+        [[nodiscard]] static AtomicRetained adopt(T_ptr t) noexcept {
+            return AtomicRetained(t, false);
+        }
+
+    private:
+        template <class U, Nullability UN> friend class AtomicRetained;
+
+        AtomicRetained(T_ptr t, bool) noexcept(N==MaybeNull) :_ref(t) { // private no-retain ctor
+            if constexpr (N == NonNull) {
+                if (t == nullptr) [[unlikely]]
+                    _failNullRef();
+            }
+        }
+
+        Retained<T,N> _get() const {
+            return Retained<T,N>::adopt(_getRef());
+        }
+
+        T_nullable_ptr _getRef() const {
+            return _ref.use( [](auto r) {
+                return retain(reinterpret_cast<T_nullable_ptr>(r));
+            } );
+        }
+
+        T_nullable_ptr _unretainedGet() const {
+            return reinterpret_cast<T_ptr>(_ref.use( [](auto r) {return r;} ));
+        }
+
+        T_nullable_ptr _exchangeWith(T_nullable_ptr ref) {
+            return reinterpret_cast<T_nullable_ptr>(_ref.exchangeWith(reinterpret_cast<uintptr_t>(ref)));
+        }
+
+        static T_ptr _retain(T_ptr t) noexcept {
+            if constexpr (N == NonNull && std::derived_from<T, AtomicRetained>)
+                t->_retain(); // this is faster, and it detects illegal null (by signal)
+            else
+                retain(t);
+            return t;
+        }
+
+        static void _release(T_ptr t) noexcept {
+            if constexpr (N == NonNull && std::derived_from<T, AtomicRetained>)
+                t->_release(); // this is faster, and it detects illegal null (by signal)
+            else
+                release(t);
+        }
+
+        internal::AtomicWrapper _ref;
+    };
+
+
+    /// Ref<T> is an alias for a non-nullable AtomicRetained<T>.
+    template <class T> using AtomicRef = AtomicRetained<T, NonNull>;
+
+    /// NullableRef<T> is an alias for a (default) nullable AtomicRetained<T>.
+    template <class T> using AtomicNullableRef = AtomicRetained<T, MaybeNull>;
+
+    /// RetainedConst is an alias for a AtomicRetained that holds a const pointer.
+    template <class T> using AtomicRetainedConst = AtomicRetained<const T>;
+
+
+    template <class T> AtomicRetained(T* FL_NULLABLE) -> AtomicRetained<T>; // deduction guide
+}

Fleece/Support/RefCounted.cc

@@ -11,7 +11,9 @@
 //
 
 #include "fleece/RefCounted.hh"
+#include "AtomicRetained.hh"
 #include "Backtrace.hh"
+#include "betterassert.hh"
 #include <stdexcept>
 #include <stdio.h>
 #include <stdlib.h>
@@ -131,5 +133,47 @@ namespace fleece {
     }
 
 
+    namespace internal {
+        /// Tag bit that's added to `_ref` while accessing it.
+        /// We can't use the low bit (1) because mutable Fleece Values already use that as a tag.
+        static constexpr uintptr_t kBusyMask = uintptr_t(1) << 63;
+
+        AtomicWrapper::AtomicWrapper(uintptr_t ref) noexcept
+        :_ref(ref)
+        {
+            assert((ref & kBusyMask) == 0);
+        }
+
+
+        uintptr_t AtomicWrapper::exchangeWith(uintptr_t newRef) noexcept {
+            uintptr_t oldRef = getAndLock();
+            setAndUnlock(oldRef, newRef);
+            return oldRef;
+        }
+
+        /// Loads and returns the value of `_ref`, while atomically setting `_ref`s high bit
+        /// to mark it as busy. If the high bit is set, busy-waits until it's cleared.
+        /// (The busy-wait should be OK since the high bit will only be set very briefly.)
+        /// @warning This MUST be followed ASAP by `setAndUnlock`.
+        uintptr_t AtomicWrapper::getAndLock() const noexcept {
+            uintptr_t r = _ref.load(std::memory_order_acquire);
+            while (true) {
+                if (r & kBusyMask)
+                    r = _ref.load(std::memory_order_acquire);
+                else if (_ref.compare_exchange_strong(r, r | kBusyMask, std::memory_order_acquire))
+                    break;
+            }
+            assert((r & kBusyMask) == 0);
+            return r;
+        }
+
+        /// Changes `_ref`s value from `oldRef` to `newRef`, clearing the busy bit.
+        /// MUST only be called after `getAndLock`.
+        void AtomicWrapper::setAndUnlock(uintptr_t oldRef, uintptr_t newRef) const noexcept {
+            uintptr_t r = oldRef | kBusyMask;
+            bool ok = _ref.compare_exchange_strong(r, newRef, std::memory_order_release);
+            assert(ok);
+        }
+    }
 
 }

Tests/PerfTests.cc

@@ -59,8 +59,8 @@ TEST_CASE("GetUVarint performance", "[.Perf]") {
         }
         bench.stop();
         CHECK(result != 1); // bogus
-        fprintf(stderr, "n = %16llx; %2zd bytes; time = %.3f ns\n",
-                (long long)n, nBytes,
+        fprintf(stderr, "n = %16llx; %2zu bytes; time = %.3f ns\n",
+                (unsigned long long)n, nBytes,
                 bench.elapsed() / kNRounds * 1.0e9);
          d *= 1.5;
     }

Tests/SupportTests.cc

@@ -14,6 +14,7 @@
 #include "fleece/InstanceCounted.hh"
 #include "FleeceTests.hh"
 #include "FleeceImpl.hh"
+#include "AtomicRetained.hh"
 #include "Backtrace.hh"
 #include "ConcurrentMap.hh"
 #include "Bitmap.hh"
@@ -234,7 +235,7 @@ TEST_CASE("ConcurrentMap concurrency", "[ConcurrentMap]") {
     constexpr size_t bufSize = 10;
     for (int i = 0; i < kSize; i++) {
         char keybuf[bufSize];
-        snprintf(keybuf, bufSize, "%x", i);
+        snprintf(keybuf, bufSize, "%x", unsigned(i));
         keys.push_back(keybuf);
     }
 
@@ -400,3 +401,22 @@ TEST_CASE("InstanceCounted") {
     }
     CHECK(InstanceCounted::liveInstanceCount() == n);
 }
+
+
+struct AtomicRetainedTest : RefCounted {
+    int i = 0;
+    string str;
+};
+
+
+TEST_CASE("AtomicRetained") {
+    // This is mostly to check for compile errors.
+    AtomicRetained r = new AtomicRetainedTest();
+    CHECK(r->i == 0);
+    AtomicRetained r2 = r;
+    AtomicRetained r3 = std::move(r2);
+    CHECK(r->refCount() == 3);
+    // `r` is one reference, `r3` is another, and the temporary created by `->` was the third.
+
+    AtomicRef<AtomicRetainedTest> rr = new AtomicRetainedTest();
+}