diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 60556842..83d0af27 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -2,9 +2,9 @@ name: build on: push: - branches: [ "main", "v2dev" ] + branches: [ "main", "radix" ] pull_request: - branches: [ "main", "v2dev", "releases/**" ] + branches: [ "main", "releases/**" ] workflow_dispatch: workflow_call: diff --git a/CMakeLists.txt b/CMakeLists.txt index a89ecf72..43b68dfe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ set(SEQ_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/seq") # create source file list file(GLOB SeqSources "seq/*.hpp" + "seq/internal/*.hpp" ) add_library(seq INTERFACE ${SeqSources}) @@ -56,15 +57,16 @@ install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/seq" # Configure and install seq.pc -configure_file(seq.pc.in seq.pc @ONLY) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/seq.pc +configure_file(seq.pc.in ${CMAKE_BINARY_DIR}/seq.pc @ONLY) +install(FILES ${CMAKE_BINARY_DIR}/seq.pc DESTINATION ${SEQ_INSTALL_DATAROOTDIR}/pkgconfig) # Configure and install seq_config.hpp -configure_file(seq_config.hpp.in seq_config.hpp @ONLY) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/seq_config.hpp - DESTINATION "${CMAKE_INSTALL_PREFIX}/include/seq") +configure_file(seq_config.hpp.in ${CMAKE_BINARY_DIR}/seq_config.hpp @ONLY) +install(FILES ${CMAKE_BINARY_DIR}/seq_config.hpp + DESTINATION "include/seq") + # Configure and install seqConfig.cmake and seqConfigVersion.cmake include(CMakePackageConfigHelpers) diff --git a/LICENSES/MPMCQueue.txt b/LICENSES/MPMCQueue.txt new file mode 100644 index 00000000..d49012fb --- /dev/null +++ b/LICENSES/MPMCQueue.txt @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2018 Erik Rigtorp + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/LICENSES/modycamel.txt b/LICENSES/modycamel.txt new file mode 100644 index 00000000..51933897 --- /dev/null +++ b/LICENSES/modycamel.txt @@ -0,0 +1,62 @@ +This license file applies to everything in this repository except that which +is explicitly annotated as being written by other authors, i.e. the Boost +queue (included in the benchmarks for comparison), Intel's TBB library (ditto), +dlib::pipe (ditto), +the CDSChecker tool (used for verification), the Relacy model checker (ditto), +and Jeff Preshing's semaphore implementation (used in the blocking queue) which +has a zlib license (embedded in lightweightsempahore.h). + +--- + +Simplified BSD License: + +Copyright (c) 2013-2016, Cameron Desrochers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this list of +conditions and the following disclaimer in the documentation and/or other materials +provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +I have also chosen to dual-license under the Boost Software License as an alternative to +the Simplified BSD license above: + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index 8e23e860..04547f30 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,20 @@ [![CTest](https://github.com/Thermadiag/seq/actions/workflows/build-macos.yml/badge.svg?branch=main)](https://github.com/Thermadiag/seq/actions/workflows/build-macos.yml) [![CTest](https://github.com/Thermadiag/seq/actions/workflows/build-windows.yml/badge.svg?branch=main)](https://github.com/Thermadiag/seq/actions/workflows/build-windows.yml) -Transitioning to v2.0 +For what it's worth, this library and its documentation were created without the help of AI. + +Transitioning to v2.1 --------------------- -*seq* v2.0 introduced a lot of changes. See this [note](docs/v2.md) for more details and explanations. +*seq* v2.1 introduced a lot of changes. See [changelog](docs/changelog.md) for more details and explanations. Purpose ------- The *seq* library is a header-only collection of original C++17 STL-like containers and related tools. -*seq* library does not try to reimplement already existing container classes present in other libraries like folly, abseil, boost and (of course) std. Instead, it provides new features (or a combination of features) that are usually not present in other libraries. Some low level API like bits manipulation or hashing functions are not new, but must be defined to keep the seq library self dependent. +*seq* library does not try to reimplement already existing container classes present in other libraries like folly, abseil, boost and (of course) std. +Instead, it provides new features (or a combination of features) that are usually not present in other libraries. Some low level API like bits manipulation or hashing functions are not new, but must be defined to keep the seq library self dependent. Among other things (see modules below), the *seq* library defines several container classes as alternatives to STL containers or providing features not present in the STL. These containers generally adhere to the properties of STL containers (in C++17 version), though there are often some associated API differences and/or implementation details which differ from the standard library. @@ -23,7 +26,7 @@ The *seq* containers are not necessarly drop-in replacement for their STL counte Currently, the *containers* module provide 5 types of containers: - Sequential random-access containers: - - [seq::devector](docs/devector.md): double ended vector that optimized for front and back operations. Similar interface to `std::deque`. + - [seq::devector](docs/devector.md): double ended vector optimized for front AND back operations. Similar interface to `std::deque`. - [seq::tiered_vector](docs/tiered_vector.md): tiered vector implementation optimized for fast insertion and deletion in the middle. Similar interface to `std::deque`. - Sequential stable non random-access container: `seq::sequence`, fast stable list-like container. - Sorted containers: @@ -36,28 +39,26 @@ Currently, the *containers* module provide 5 types of containers: - Hash tables: - [seq::ordered_set](docs/ordered_set.md): Ordered robin-hood hash table with backward shift deletion. Drop-in replacement for `std::unordered_set` (except for the bucket and node interface) with iterator/reference stability, with performances close to 'flat' hash maps. `seq::ordered_set` preserves the insertion order. - `seq::ordered_map`: associative version of `seq::ordered_set`. - - [seq::radix_hash_set](docs/radix_tree.md): radix based hash table with a similar interface to `std::unordered_set`. Uses incremental rehash (no memory peak) with a very small memory footprint. + - [seq::radix_hash_set](docs/radix_tree.md): radix based hash table with a similar interface to `std::unordered_set`. Uses incremental rehash (no memory peak), ideal when low memory footprint and low latency are required. - `seq::radix_hash_map`: associative version of `seq::radix_hash_set`. - [seq::concurrent_map](docs/concurrent_map.md) and `seq::concurrent_set`: higly scalable concurrent hash tables with interfaces similar to `boost::concurrent_flat_set/map`. - Strings: - [seq::tiny_string](docs/tiny_string.md): relocatable string-like class with configurable Small String Optimization and tiny memory footprint. Makes most string containers faster. - +- Other: + - [seq::concurrent_queue](docs/concurrent_queue.md): fast thread-safe queue designed for Multi-Producer Multi-Consumer (MPMC) scenarios. Content ------- -The library is divided in 7 small modules: +The library is divided in 5 small modules: - [bits](docs/bits.md): low-level bits manipulation utilities - [hash](docs/hash.md): tiny hashing framework -- [charconv](docs/charconv.md): fast arithmetic to/from string conversion -- [format](docs/format.md): fast and type safe formatting tools - [containers](docs/containers.md): main module, collection of original containers: double ended vector, tiered-vector, ordered hash map, flat map based on tiered-vector, compressed vector... - [any](docs/any.md): type-erasing polymorphic object wrapper used to build heterogeneous containers, including hash tables and sorted containers. - [algorithm](docs/algorithm.md): a (small) collection of algorithm include the `net_sort` stable sorting algorithm. A cmake project is provided for installation and compilation of tests/benchmarks. -*seq* library was tested with gcc 10.1.0 and 13.2.0 (Windows and Linux), msvc 19.43 (Windows), ClangCL 12.0.0 (Windows). Design ------ @@ -83,6 +84,18 @@ Currently, the following options are provided: - SEQ_BUILD_TESTS(OFF): build all tests - SEQ_BUILD_BENCHS(OFF): build all benchmarks +It is also possible to directly copy/paste the *seq* folder into your project, while slightly dirty :). + + +Benchmarks +---------- + +The library provides several (small) benchmarks and more will be added: +- Benchmark of [concurrent hash tables](docs/concurrent_map.md) at the end of the page. Its goal is to compare `seq::concurrent_set/map` to other implementations. +- Benchmark on [sorted containers](docs/sorted_benchmark.md). Its goal is to compare `seq::flat_set/map` and `seq::radix_set/map` to other implementations. +- Very tiny benchmark on [concurrent queues](docs/concurrent_queue.md) to compare `seq::concurrent_queue` with other implementations. +- [Memory and latency benchmark](docs/latency_benchmark.md) on hash tables to compare `seq::radix_hash_set/map` with other implementations. + Acknowledgements ---------------- @@ -94,11 +107,14 @@ The `net_sort` stable sorting algorithm uses several ideas originaly coming (I t Benchmarks (in `seq/benchs`) compare the performances of the *seq* library with other great libraries that I use in personnal or professional projects: - plf: used for the plf::colony container, - gtl: used for its gtl::btree_set and gtl::parallel_flat_hash_map, -- boost: used for boost::flat_set, boost::unordered_flat_set and boost::concurrent_flat_map, +- boost: used for boost::flat_set, boost::unordered_flat_set, boost::concurrent_flat_map, boost::lockfree::queue, - unordered_dense: used for ankerl::unordered_dense::set, - TBB: used for tbb::concurrent_unordered_map and tbb::concurrent_hash_map. +- TSL: numerous Tessil's hash tables. +- Modycamel: used for its MPMC concurrent queue. +- MPMCQueue: Rigtorp's MPMC concurrent queue. Some of these libraries are included in the `seq/benchs` folder. -seq:: library and this page Copyright (c) 2025, Victor Moncada +seq:: library and this page Copyright (c) 2026, Victor Moncada diff --git a/benchs/CMakeLists.txt b/benchs/CMakeLists.txt index 3efd7c51..65dad79d 100644 --- a/benchs/CMakeLists.txt +++ b/benchs/CMakeLists.txt @@ -6,16 +6,17 @@ enable_testing() # create the testing file and list of tests create_test_sourcelist (Benchs seq_benchs.cpp - bench_format.cpp bench_hash.cpp bench_concurrent_hash.cpp bench_map.cpp + bench_map_csv.cpp bench_sequence.cpp - bench_text_stream.cpp bench_tiered_vector.cpp bench_tiny_string.cpp - bench_boost_unordered_benchmarks.cpp + #bench_boost_unordered_benchmarks.cpp bench_sort.cpp + bench_concurrent_queue.cpp +bench_hash_latency.cpp ) # add the executable @@ -60,6 +61,12 @@ if(TEST_BOOST_INSERT_ERASE) target_compile_definitions(seq_benchs PRIVATE -DTEST_BOOST_INSERT_ERASE) endif() + +find_package(OpenMP ) +if(${OpenMP_FOUND}) +target_link_libraries(seq_benchs PRIVATE OpenMP::OpenMP_CXX) +endif() + # msvc warnings generated by seq_benchs.cpp target_compile_definitions(seq_benchs PRIVATE -D_CRT_SECURE_NO_WARNINGS) diff --git a/benchs/MPMCQueue.h b/benchs/MPMCQueue.h new file mode 100644 index 00000000..a31e8267 --- /dev/null +++ b/benchs/MPMCQueue.h @@ -0,0 +1,295 @@ +/* +Copyright (c) 2020 Erik Rigtorp + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + */ + +#pragma once + +#include +#include +#include // offsetof +#include +#include +#include // std::hardware_destructive_interference_size +#include + +#ifndef __cpp_aligned_new +#ifdef _WIN32 +#include // _aligned_malloc +#else +#include // posix_memalign +#endif +#endif + +namespace rigtorp { +namespace mpmc { +#if defined(__cpp_lib_hardware_interference_size) && !defined(__APPLE__) +static constexpr size_t hardwareInterferenceSize = + std::hardware_destructive_interference_size; +#else +static constexpr size_t hardwareInterferenceSize = 64; +#endif + +#if defined(__cpp_aligned_new) +template using AlignedAllocator = std::allocator; +#else +template struct AlignedAllocator { + using value_type = T; + + T *allocate(std::size_t n) { + if (n > std::numeric_limits::max() / sizeof(T)) { + throw std::bad_array_new_length(); + } +#ifdef _WIN32 + auto *p = static_cast(_aligned_malloc(sizeof(T) * n, alignof(T))); + if (p == nullptr) { + throw std::bad_alloc(); + } +#else + T *p; + if (posix_memalign(reinterpret_cast(&p), alignof(T), + sizeof(T) * n) != 0) { + throw std::bad_alloc(); + } +#endif + return p; + } + + void deallocate(T *p, std::size_t) { +#ifdef _WIN32 + _aligned_free(p); +#else + free(p); +#endif + } +}; +#endif + +template struct Slot { + ~Slot() noexcept { + if (turn & 1) { + destroy(); + } + } + + template void construct(Args &&...args) noexcept { + static_assert(std::is_nothrow_constructible::value, + "T must be nothrow constructible with Args&&..."); + new (&storage) T(std::forward(args)...); + } + + void destroy() noexcept { + static_assert(std::is_nothrow_destructible::value, + "T must be nothrow destructible"); + reinterpret_cast(&storage)->~T(); + } + + T &&move() noexcept { return reinterpret_cast(storage); } + + // Align to avoid false sharing between adjacent slots + alignas(hardwareInterferenceSize) std::atomic turn = {0}; + typename std::aligned_storage::type storage; +}; + +template >> +class Queue { +private: + static_assert(std::is_nothrow_copy_assignable::value || + std::is_nothrow_move_assignable::value, + "T must be nothrow copy or move assignable"); + + static_assert(std::is_nothrow_destructible::value, + "T must be nothrow destructible"); + +public: + explicit Queue(const size_t capacity, + const Allocator &allocator = Allocator()) + : capacity_(capacity), allocator_(allocator), head_(0), tail_(0) { + if (capacity_ < 1) { + throw std::invalid_argument("capacity < 1"); + } + // Allocate one extra slot to prevent false sharing on the last slot + slots_ = allocator_.allocate(capacity_ + 1); + // Allocators are not required to honor alignment for over-aligned types + // (see http://eel.is/c++draft/allocator.requirements#10) so we verify + // alignment here + if (reinterpret_cast(slots_) % alignof(Slot) != 0) { + allocator_.deallocate(slots_, capacity_ + 1); + throw std::bad_alloc(); + } + for (size_t i = 0; i < capacity_; ++i) { + new (&slots_[i]) Slot(); + } + static_assert( + alignof(Slot) == hardwareInterferenceSize, + "Slot must be aligned to cache line boundary to prevent false sharing"); + static_assert(sizeof(Slot) % hardwareInterferenceSize == 0, + "Slot size must be a multiple of cache line size to prevent " + "false sharing between adjacent slots"); + static_assert(sizeof(Queue) % hardwareInterferenceSize == 0, + "Queue size must be a multiple of cache line size to " + "prevent false sharing between adjacent queues"); + static_assert( + offsetof(Queue, tail_) - offsetof(Queue, head_) == + static_cast(hardwareInterferenceSize), + "head and tail must be a cache line apart to prevent false sharing"); + } + + ~Queue() noexcept { + for (size_t i = 0; i < capacity_; ++i) { + slots_[i].~Slot(); + } + allocator_.deallocate(slots_, capacity_ + 1); + } + + // non-copyable and non-movable + Queue(const Queue &) = delete; + Queue &operator=(const Queue &) = delete; + + template void emplace(Args &&...args) noexcept { + static_assert(std::is_nothrow_constructible::value, + "T must be nothrow constructible with Args&&..."); + auto const head = head_.fetch_add(1); + auto &slot = slots_[idx(head)]; + while (turn(head) * 2 != slot.turn.load(std::memory_order_acquire)) + ; + slot.construct(std::forward(args)...); + slot.turn.store(turn(head) * 2 + 1, std::memory_order_release); + } + + template bool try_emplace(Args &&...args) noexcept { + static_assert(std::is_nothrow_constructible::value, + "T must be nothrow constructible with Args&&..."); + auto head = head_.load(std::memory_order_acquire); + for (;;) { + auto &slot = slots_[idx(head)]; + if (turn(head) * 2 == slot.turn.load(std::memory_order_acquire)) { + if (head_.compare_exchange_strong(head, head + 1)) { + slot.construct(std::forward(args)...); + slot.turn.store(turn(head) * 2 + 1, std::memory_order_release); + return true; + } + } else { + auto const prevHead = head; + head = head_.load(std::memory_order_acquire); + if (head == prevHead) { + return false; + } + } + } + } + + void push(const T &v) noexcept { + static_assert(std::is_nothrow_copy_constructible::value, + "T must be nothrow copy constructible"); + emplace(v); + } + + template ::value>::type> + void push(P &&v) noexcept { + emplace(std::forward

(v)); + } + + bool try_push(const T &v) noexcept { + static_assert(std::is_nothrow_copy_constructible::value, + "T must be nothrow copy constructible"); + return try_emplace(v); + } + + template ::value>::type> + bool try_push(P &&v) noexcept { + return try_emplace(std::forward

(v)); + } + + void pop(T &v) noexcept { + auto const tail = tail_.fetch_add(1); + auto &slot = slots_[idx(tail)]; + while (turn(tail) * 2 + 1 != slot.turn.load(std::memory_order_acquire)) + ; + v = slot.move(); + slot.destroy(); + slot.turn.store(turn(tail) * 2 + 2, std::memory_order_release); + } + + bool try_pop(T &v) noexcept { + auto tail = tail_.load(std::memory_order_acquire); + for (;;) { + auto &slot = slots_[idx(tail)]; + if (turn(tail) * 2 + 1 == slot.turn.load(std::memory_order_acquire)) { + if (tail_.compare_exchange_strong(tail, tail + 1)) { + v = slot.move(); + slot.destroy(); + slot.turn.store(turn(tail) * 2 + 2, std::memory_order_release); + return true; + } + } else { + auto const prevTail = tail; + tail = tail_.load(std::memory_order_acquire); + if (tail == prevTail) { + return false; + } + } + } + } + + /// Returns the number of elements in the queue. + /// The size can be negative when the queue is empty and there is at least one + /// reader waiting. Since this is a concurrent queue the size is only a best + /// effort guess until all reader and writer threads have been joined. + ptrdiff_t size() const noexcept { + // TODO: How can we deal with wrapped queue on 32bit? + return static_cast(head_.load(std::memory_order_relaxed) - + tail_.load(std::memory_order_relaxed)); + } + + /// Returns true if the queue is empty. + /// Since this is a concurrent queue this is only a best effort guess + /// until all reader and writer threads have been joined. + bool empty() const noexcept { return size() <= 0; } + +private: + constexpr size_t idx(size_t i) const noexcept { return i % capacity_; } + + constexpr size_t turn(size_t i) const noexcept { return i / capacity_; } + +private: + const size_t capacity_; + Slot *slots_; +#if defined(__has_cpp_attribute) && __has_cpp_attribute(no_unique_address) + Allocator allocator_ [[no_unique_address]]; +#else + Allocator allocator_; +#endif + + // Align to avoid false sharing between head_ and tail_ + alignas(hardwareInterferenceSize) std::atomic head_; + alignas(hardwareInterferenceSize) std::atomic tail_; +}; +} // namespace mpmc + +template >> +using MPMCQueue = mpmc::Queue; + +} // namespace rigtorp diff --git a/benchs/bench_concurrent_hash.cpp b/benchs/bench_concurrent_hash.cpp index 2e83f6dd..024b65ac 100644 --- a/benchs/bench_concurrent_hash.cpp +++ b/benchs/bench_concurrent_hash.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -30,11 +30,9 @@ #include #include -#include -#include #include #include -#include +#include #include #include @@ -59,6 +57,8 @@ #include +#include "testing.hpp" + using namespace seq; inline bool get_res(bool r) @@ -502,7 +502,7 @@ void test_concurrent_map(size_t count, const char* name, Rng rng) seq::random_shuffle(keys.begin(), keys.end(), seed); // if (seed < 4) // continue; - for (size_t threads = 1; threads < 20; ++threads) { + for (size_t threads = 1; threads <= 16; ++threads) { Map set; // reserve(set,keys.size()); @@ -529,7 +529,7 @@ void test_concurrent_hash_maps(size_t count, const Gen& gen) test_concurrent_map, std::equal_to, std::allocator>, 5, std::shared_mutex>>( count, "gtl::parallel_flat_hash_map", gen); - + #ifdef BOOST_CONCURRENT_MAP_FOUND test_concurrent_map, std::equal_to>>(count, "boost::concurrent_flat_map", gen); #endif diff --git a/benchs/bench_concurrent_queue.cpp b/benchs/bench_concurrent_queue.cpp new file mode 100644 index 00000000..1b55ed2e --- /dev/null +++ b/benchs/bench_concurrent_queue.cpp @@ -0,0 +1,498 @@ +#include "testing.hpp" +#include +#include +#include + +#include "concurrentqueue.h" +#include "MPMCQueue.h" +#include +#include +#include +using namespace rigtorp::mpmc; +#if BOOST_FOUND +#include +template +using boost_queue = boost::lockfree::queue; +#endif + + + +template +using concurrent_set = seq::concurrent_set, std::equal_to<>, std::allocator, seq::low_concurrency>; + + + +template +struct queue +{ + using lock_type = std::mutex; + lock_type lock; + seq::sequence deq; + +public: + using value_type = T; + queue() = default; + + template + void emplace(Args&&... args) + { + std::scoped_lock guard(lock); + deq.emplace_back(std::forward(args)...); + } + + bool pop(T& val) noexcept + { + std::scoped_lock guard(lock); + if (deq.empty()) + return false; + val = std::move(deq.front()); + deq.pop_front(); + return true; + } + size_t size() const { return deq.size(); } + + void clear() + { + std::scoped_lock guard(lock); + deq.clear(); + } +}; + +template +bool push(Queue& q, Args&&... t) +{ + q.emplace(std::forward(t)...); + return true; +} +template +bool push(moodycamel::ConcurrentQueue& q, Args&&... t) +{ + q.enqueue(std::forward(t)...); + return true; +} + +#if BOOST_FOUND +template +bool push(boost_queue& q, Args&&... t) +{ + return (q.push(T{std::forward(t)...} )); +} +#endif + + +template +bool push(Queue& q, Args&&... t) +{ + q.emplace(std::forward(t)...); + return true; +} +template +bool push(concurrent_set& q, Args&&... t) +{ + q.emplace(std::forward(t)...); + return true; +} + +template +bool pop(Queue& q, T& v) +{ + return q.pop(v); +} +template +bool pop(moodycamel::ConcurrentQueue& q, T& v) +{ + return q.try_dequeue(v); +} +template +bool pop(Queue& q, T& v) +{ + return q.try_pop(v); +} +#if BOOST_FOUND +template +bool pop(boost_queue& q, T &v) +{ + return q.pop(v); +} +#endif +template +bool pop(concurrent_set& q, T& v) +{ + + q.visit_all([&](const auto& f) { + v = f; + return false; + }); + q.erase(v); + return true; +} + +template +void launch_push(Queue& q, const std::vector& data, int threads) +{ +#pragma omp parallel for num_threads(threads) + for (int i = 0; i < (int)data.size(); ++i) + push(q, data[i]); +} + +template +void launch_pop(Queue& q, int N, int threads) +{ +#pragma omp parallel for num_threads(threads) + for (int i = 0; i < N; ++i) { + T v; + pop(q, v); + } +} + +template +void unbalanced(Queue& q, const std::vector& data, int balance, int threads) +{ +#pragma omp parallel for num_threads(threads) + for (int i = 0; i < threads; ++i) { + + for (size_t j = 0; j < data.size(); ++j) { + if (j % balance == 0) { + T v; + pop(q, v); + } + else { + push(q, data[j]); + } + } + } +} + +template +class safe_counter +{ + static constexpr unsigned count = 32; + static constexpr unsigned mask = count - 1; + std::atomic d_cnts[count]; + + static unsigned global_id() noexcept + { + static std::atomic cnt{ 0 }; + return cnt.fetch_add(1); + } + SEQ_ALWAYS_INLINE unsigned thread_id() noexcept + { + thread_local unsigned id = global_id() & mask; + return id; + } + +public: + safe_counter() noexcept { memset((void*)d_cnts, 0, sizeof(d_cnts)); } + + SEQ_ALWAYS_INLINE void add(T v = 1) noexcept { d_cnts[thread_id()].fetch_add(v); } + SEQ_ALWAYS_INLINE void sub(T v = 1) noexcept { d_cnts[thread_id()].fetch_sub(v); } + + SEQ_ALWAYS_INLINE T value() const noexcept + { + T ret = 0; + for (unsigned i = 0; i < count; ++i) + ret += d_cnts[i].load(std::memory_order_relaxed); + return ret; + } +}; + +template +void push_thread(Queue& q, std::atomic& start, std::atomic& stop, safe_counter& cnt) +{ + while (!start) + ; // std::this_thread::yield(); + T val = 0; + size_t r = 0; + while (!stop) { + if(push(q, val++)) + if ((++r & 31) == 0) + cnt.add(32); + } +} + + +static constexpr size_t max_pop = 10000000; + + +template +void pop_thread(Queue& q, std::atomic& start, std::atomic& stop, safe_counter& cnt) +{ + while (!start) + ; // std::this_thread::yield(); + T val; + size_t r = 0; + while (!stop) { + if (pop(q, val)) { + cnt.add(); + if ((++r & 31) == 0) + if SEQ_UNLIKELY(cnt.value() >= max_pop) + stop.store(true); + } + else + // Give time for additional push + //std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } +} + +static uint64_t msecs() +{ + return (uint64_t)std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +struct Ret +{ + uint64_t push_cnt = 0; + uint64_t pop_cnt = 0; + uint64_t elapsed_ms = 0; + std::string name; +}; + +template +Ret test_queue(Queue& q, int threads) +{ + std::vector> all_threads((size_t)threads); + std::atomic start_push{ false }, start_pop{ false }, stop{ false }; + safe_counter push_cnt, pop_cnt; + for (int i = 0; i < threads; ++i) { + all_threads[(size_t)i].first = std::thread([&]() { push_thread(q, start_push, stop, push_cnt); }); + all_threads[(size_t)i].second = std::thread([&]() { pop_thread(q, start_pop, stop, pop_cnt); }); + } + + + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // for (int i = 0; i < 10000; ++i) + // push(q, T{}); + + start_push.store(true); + //std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + start_pop.store(true); + + /* auto st = msecs(); + while (msecs() - st < 2000) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + stop.store(true);*/ + auto st = msecs(); + for (size_t i = 0; i < (size_t)threads; ++i) { + all_threads[i].first.join(); + all_threads[i].second.join(); + } + + //T v; + //while(pop(q,v)) + // ; + auto el = msecs() - st; + + return { push_cnt.value(), pop_cnt.value() ,el}; +} +template +Ret test_queue_name(const char* name, int threads = 1) +{ + Ret p; + if constexpr (std::is_same_v,QueueType>){ + QueueType q(max_pop * 2); + p = test_queue(q, threads); + } +#if BOOST_FOUND + else if constexpr (std::is_same_v, QueueType>) { + QueueType q(0); + p = test_queue(q, threads); + } +#endif + else{ + QueueType q; + p = test_queue(q, threads); + } + p.name = name; + std::cout << name << ": " << p.push_cnt << " " << p.pop_cnt << " " << p.elapsed_ms << std::endl; + return p; +} + + +bool to_csv(const std::vector < std::vector>& res, const char* filename) +{ + std::ofstream fout(filename); + if (!fout) + return false; + fout << "\"sep=\t\"" << std::endl; + + fout << "Threads\t"; + for (size_t i = 0; i < res[1].size(); ++i) { + fout << res[1][i].name <<"\t"; + } + fout << std::endl; + + for(size_t i = 1; i < res.size(); ++i){ + fout << i <<"\t"; + auto & vec = res[i]; + for (size_t j = 0; j < vec.size(); ++j) + fout << vec[j].elapsed_ms <<"\t"; + fout << std::endl; + + } + fout << std::endl; + + return true; +} + + + +int bench_concurrent_queue(int, char** const) +{ + // These benchmarks are not well formalized! + + int threads = 16; + int count = 1000000; + + /*std::vector> rets(threads + 1); + for(int th = 1; th <= threads; ++th){ + std::cout << th << " threads" << std::endl; + rets[(size_t)th].push_back( test_queue_name, int>("std::deque", th)); + rets[(size_t)th].push_back( test_queue_name, int>("seq::concurrent_queue", th)); + //rets[(size_t)th].push_back( test_queue_name, int>("rigtorp::mpmc::Queue", th)); + rets[(size_t)th].push_back( test_queue_name, int>("moodycamel::ConcurrentQueue", th)); +#if BOOST_FOUND + rets[(size_t)th].push_back( test_queue_name, int>("boost::lockfree::queue", th)); +#endif + } + to_csv(rets,"bench_concurrent_queue.csv"); + */ + struct Test + { + int val; + char pad[32]; + + Test& operator=(int v) + { + val = v; + return *this; + } + }; + using queue_type = int; + + std::vector vals((size_t)count); + for (size_t i = 0; i < vals.size(); ++i) + vals[i] = (int)i; + + // Parallel push + + { + + queue q; + + seq::tick(); + launch_push(q, vals, threads); + auto el = seq::tock_ms(); + std::cout << "push queue: " << el << " ms " << q.size() << std::endl; + + seq::concurrent_queue fifo; + // fifo.reserve(vals.size()); + seq::tick(); + launch_push(fifo, vals, threads); + el = seq::tock_ms(); + std::cout << "push fifo: " << el << " ms " << fifo.size() << std::endl; + + seq::tick(); + moodycamel::ConcurrentQueue mq; + launch_push(mq, vals, threads); + el = seq::tock_ms(); + std::cout << "push concurrent modycamel: " << el << " ms " << mq.size_approx() << std::endl; + + #if BOOST_FOUND + seq::tick(); + boost_queue bq{ 0 }; + bq.reserve_unsafe(vals.size()); + launch_push(bq, vals, threads); + el = seq::tock_ms(); + std::cout << "push concurrent boost: " << el << " ms " << std::endl; + #endif + + + Queue mpq(vals.size()); + seq::tick(); + launch_push(mpq, vals, threads); + el = seq::tock_ms(); + std::cout << "push concurrent MPMC: " << el << " ms " << mpq.size() << std::endl; + + /* seq::detail::TQueue ts; //(vals.size()); + seq::tick(); + launch_push(ts, vals, threads); + el = seq::tock_ms(); + std::cout << "push concurrent TQueue: " << el << " ms " << ts.size() << std::endl; + */ + + // Parallel pop + // threads = 1; + + seq::tick(); + launch_pop(q, count, threads); + el = seq::tock_ms(); + std::cout << "pop queue: " << el << " ms " << q.size() << std::endl; + + seq::tick(); + launch_pop(fifo, count, threads); + el = seq::tock_ms(); + std::cout << "pop fifo: " << el << " ms " << q.size() << std::endl; + + seq::tick(); + launch_pop(mq, count, threads); + el = seq::tock_ms(); + std::cout << "pop concurrent modycamel: " << el << " ms " << mq.size_approx() << std::endl; + + #if BOOST_FOUND + seq::tick(); + launch_pop(bq, count, threads); + el = seq::tock_ms(); + std::cout << "pop concurrent boost: " << el << " ms " << std::endl; + #endif + + seq::tick(); + launch_pop(mpq, count, threads); + el = seq::tock_ms(); + std::cout << "pop concurrent MPMC: " << el << " ms " << mpq.size() << std::endl; + + /* seq::tick(); + launch_pop(ts, count, threads); + el = seq::tock_ms(); + std::cout << "pop concurrent TQueue: " << el << " ms " << ts.size() << std::endl;*/ + } + + // while (true) + + { + int unbalance = 3; + queue q; + seq::concurrent_queue fifo; + moodycamel::ConcurrentQueue mq; + + seq::tick(); + unbalanced(q, vals, unbalance, threads); + auto el = seq::tock_ms(); + std::cout << "push/pop queue: " << el << " ms " << q.size() << std::endl; + q.clear(); + + seq::tick(); + unbalanced(fifo, vals, unbalance, threads); + el = seq::tock_ms(); + std::cout << "push/pop concurrent fifo: " << el << " ms " << fifo.size() << std::endl; + // cq.clear(); + + seq::tick(); + unbalanced(mq, vals, unbalance, threads); + el = seq::tock_ms(); + std::cout << "push/pop concurrent modycamel: " << el << " ms " << mq.size_approx() << std::endl; + + #if BOOST_FOUND + boost_queue bq{0}; + seq::tick(); + unbalanced(bq, vals, unbalance, threads); + el = seq::tock_ms(); + std::cout << "push/pop concurrent boost: " << el << " ms " << std::endl; + #endif + } + + return 0; +} \ No newline at end of file diff --git a/benchs/bench_format.cpp b/benchs/bench_format.cpp deleted file mode 100644 index c82aa0b9..00000000 --- a/benchs/bench_format.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#include -#include -#include -#include - -#ifdef SEQ_HAS_CPP_20 -#include -#endif - -int bench_format(int, char ** const) -{ - using namespace seq; - - // Generate 4M double values - using ftype = double; - random_float_genertor rgn; - std::vector vec_d; - for (int i = 0; i < 4000000; ++i) - vec_d.push_back(rgn()); - - // Null ostream object - nullbuf n; - std::ostream oss(&n); - oss.sync_with_stdio(false); - - // Build a table of 4 * 1000000 double values separated by a '|'. All values are centered on a 20 characters space - tick(); - oss << std::setprecision(6); - for (size_t i = 0; i < vec_d.size() / 4; ++i) - { - oss << std::left << std::setw(20) << vec_d[i * 4] << "|"; - oss << std::left << std::setw(20) << vec_d[i * 4 + 1] << "|"; - oss << std::left << std::setw(20) << vec_d[i * 4 + 2] << "|"; - oss << std::left << std::setw(20) << vec_d[i * 4 + 3] << "|"; - oss << std::endl; - } - size_t el = tock_ms(); - std::cout << "Write table with streams: " << el << " ms" << std::endl; - - - // Build the same table with format module - - // Create the format object - auto slot = _g().p(6).l(20); // floating point slot with a precision of 6 and left-aligned on a 20 characters width - auto f = join("|",slot, slot, slot, slot, ""); - tick(); - for (size_t i = 0; i < vec_d.size() / 4; ++i) - oss << f(vec_d[i * 4], vec_d[i * 4 + 1], vec_d[i * 4 + 2], vec_d[i * 4 + 3]) << std::endl; - el = tock_ms(); - std::cout << "Write table with seq formatting module: " << el << " ms" << std::endl; - - - // Compare to std::format for C++20 compilers -#ifdef SEQ_HAS_CPP_20 - tick(); - for (size_t i = 0; i < vec_d.size() / 4; ++i) - std::format_to( - std::ostreambuf_iterator(oss), - "{:^20.6g} | {:^20.6g} | {:^20.6g} | {:^20.6g}\n", - vec_d[i * 4], vec_d[i * 4 + 1], vec_d[i * 4 + 2], vec_d[i * 4 + 3]); - el = tock_ms(); - std::cout << "Write table with std::format : " << el << " ms" << std::endl; -#endif - - // Just for comparison, directly dump the double values without the '|' character (but keeping alignment) - - tick(); - auto f2 = g().l(20); - for (size_t i = 0; i < vec_d.size(); ++i) - oss << f2(vec_d[i]); - el = tock_ms(); - std::cout << "Write left-aligned double with seq::fmt: " << el << " ms" << std::endl; - - - - // use std::ostream::bad() to make sure the above tests are not simply ignored by the compiler - if (oss.bad()) - std::cout << "error" << std::endl; - - - return 0; -} \ No newline at end of file diff --git a/benchs/bench_hash.cpp b/benchs/bench_hash.cpp index bc543963..1425cb30 100644 --- a/benchs/bench_hash.cpp +++ b/benchs/bench_hash.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -25,6 +25,7 @@ #include #include #include +#include #ifdef _MSC_VER #include @@ -36,24 +37,66 @@ #include #endif -#include +#include + #include #include #include -#include +#include #include #include #include -//#include "flat_hash_map.hpp" +#include "testing.hpp" -#ifdef SEQ_HAS_CPP_17 #include "ankerl/unordered_dense.h" #include -#endif + using namespace seq; +template +struct ValueHolder +{ + std::unique_ptr ptr; + + ValueHolder() + : ptr(new T()) + { + } + template + ValueHolder(Args&&... args) + : ptr(new T(std::forward(args)...)) + { + } + ValueHolder(ValueHolder&& ) noexcept = default; + ValueHolder& operator=(ValueHolder&&) noexcept = default; + + ValueHolder(const ValueHolder& other) + : ptr(new T(*other.ptr)) + { + } + + operator const T&() const noexcept { return *ptr; } + + friend bool operator==(const ValueHolder& l, const ValueHolder& r) { return *l.ptr == *r.ptr; } + friend bool operator==(const T& l, const ValueHolder& r) { return l == *r.ptr; } + friend bool operator==(const ValueHolder& l, const T& r) { return *l.ptr == r; } +}; + +namespace seq +{ + template + struct hasher> + { + using is_transparent=int; + using is_avalanching=int; + size_t operator()(const ValueHolder& v) const noexcept { return hasher{}(*v.ptr); } + template + size_t operator()(const U& v) const noexcept { return hasher{}(v); } + }; +} + template void erase(C& set, Iter it) { @@ -66,7 +109,7 @@ void reserve(C& set, size_t count) } template -void reserve(radix_set & , size_t ) +void reserve(radix_set&, size_t) { } @@ -75,6 +118,7 @@ inline size_t to_size_t(const T& t) { return static_cast(t); } + inline size_t to_size_t(const std::string& t) { return t.size(); @@ -89,6 +133,11 @@ inline size_t to_size_t(const seq::hold_any& t) { return reinterpret_cast(t.data()); } +template +inline size_t to_size_t(const ValueHolder& t) +{ + return to_size_t(*t.ptr); +} template size_t count_iter(Iter start, Iter end) @@ -124,8 +173,8 @@ size_t walk_set(flat_hash_set& set) size_t sum = 0; set.cvisit_all([&](const auto& v) { sum += to_size_t(v); }); return sum; -}*/ - +} +*/ inline bool test_insert(bool v) { return v; @@ -174,7 +223,6 @@ struct LaunchTest start_mem = get_memory_usage(); C s; tick(); - // s.reserve(success.size()); reserve(s, success.size()); for (int i = 0; i < success.size(); ++i) { @@ -280,8 +328,7 @@ struct LaunchTest int i = 0; size_t target = set.size() / 2; while (set.size() > target) { - if (set.erase(success[i]) != 1) - SEQ_TEST(false); + SEQ_TEST(set.erase(success[i]) == 1); i++; } eraset = tock_ms(); @@ -347,43 +394,47 @@ void test_hash(int count, Gen gen, bool save_keys = false) std::sort(keys.begin(), keys.end()); size_t size = std::unique(keys.begin(), keys.end()) - keys.begin(); keys.resize(size); - seq::random_shuffle(keys.begin(), keys.end(), 1); + std::shuffle(keys.begin(), keys.end(), std::mt19937(1)); } // warmup { ordered_set, std::allocator> set; - set.max_load_factor(0.85); + // set.max_load_factor(0.85); test_hash_set("seq::ordered_set", set, keys, f, false); } { ordered_set> set; - set.max_load_factor(0.85); + // set.max_load_factor(0.85); test_hash_set("seq::ordered_set", set, keys, f); } -#ifdef SEQ_HAS_CPP_17 + { ankerl::unordered_dense::set> set; - //set.max_load_factor(0.85); + // set.max_load_factor(0.85); test_hash_set("ankerl::unordered_dense::set", set, keys, f); } -#endif { concurrent_set, std::allocator, seq::no_concurrency> set; - //set.max_load_factor(0.85); + // set.max_load_factor(0.8); test_hash_set("seq::concurrent_set", set, keys, f); } /* { - flat_hash_set, std::allocator > set; - set.max_load_factor(0.6); + flat_hash_set, std::allocator> set; + // set.max_load_factor(0.8); test_hash_set("seq::flat_hash_set", set, keys, f); }*/ + { radix_hash_set> set; test_hash_set("seq::radix_hash_set", set, keys, f); } + { + tsl::sparse_set> set; + test_hash_set("tsl::sparse_set", set, keys, f); + } { robin_hood::unordered_flat_set> set; test_hash_set("robin_hood::unordered_flat_set", set, keys, f); @@ -395,7 +446,8 @@ void test_hash(int count, Gen gen, bool save_keys = false) } #endif #ifdef BOOST_UNORDERED_MAP_FOUND - { + { + // Compile error with boost::unordered_flat_set and seq::hasher boost::unordered_flat_set> set; test_hash_set("boost::unordered_flat_set", set, keys, f); } @@ -406,17 +458,41 @@ void test_hash(int count, Gen gen, bool save_keys = false) set.max_load_factor(0.85); test_hash_set("std::unordered_set", set, keys, f); } + } +struct Test +{ + size_t v; + size_t data[4]; + Test(size_t i = 0) noexcept + : v(i) + { + } + bool operator==(const Test& o) const noexcept { return v == o.v; } +}; +template<> +struct seq::hasher +{ + size_t operator()(const Test& t) const noexcept { return hasher{}(t.v); } +}; + + + + int bench_hash(int, char** const) { - - + using tvec = tiered_vector; + + std::vector vv(10); + tvec vec{ vv.begin(), vv.end() }; + test_hash>(8000000, [](size_t i) { return (i); }); test_hash>(8000000, [](size_t i) { return (i); }); - random_float_genertor rng; - test_hash>(8000000, [&rng](size_t i) { return rng(); }); + std::mt19937 e2(0); + std::uniform_real_distribution<> rng; + test_hash>(8000000, [&](size_t i) { return rng(e2); }); test_hash>(2500000, [](size_t i) { return generate_random_string(63, false); }); diff --git a/benchs/bench_hash_latency.cpp b/benchs/bench_hash_latency.cpp new file mode 100644 index 00000000..908cf82d --- /dev/null +++ b/benchs/bench_hash_latency.cpp @@ -0,0 +1,291 @@ +#include "seq/timer.hpp" +#include "seq/radix_hash_map.hpp" +#include "robin_hood/robin_hood.h" +#include +#include "ankerl/unordered_dense.h" +#include + +#include "testing.hpp" +#include +#include +#include +#include +#include + +using namespace seq; + + +inline void _reset_memory_usage() +{ +#if defined(WIN32) || defined(_WIN32) + SetProcessWorkingSetSize(GetCurrentProcess(), static_cast(-1), static_cast(-1)); +#endif +} + +inline auto _get_memory_usage() -> size_t +{ +#if defined(WIN32) || defined(_WIN32) + static HANDLE currentProcessHandle = GetCurrentProcess(); + PROCESS_MEMORY_COUNTERS_EX memoryCounters; // = { 0 };; + memset(&memoryCounters, 0, sizeof(memoryCounters)); + if (GetProcessMemoryInfo(currentProcessHandle, reinterpret_cast(&memoryCounters), sizeof(memoryCounters))) { + return /*memoryCounters.PrivateUsage + */ memoryCounters.WorkingSetSize; + } + return 0; +#else + return 0; +#endif +} + + +static void measure_max_mem(std::atomic& start, std::atomic& stop, std::atomic& max_mem) +{ + while (!start) + ; + + while (!stop) { + size_t mem = _get_memory_usage(); + auto max = max_mem.load(); + + if (mem > 100000000000ull) { // more than 100GB + std::this_thread::yield(); + continue; + } + + while (mem > max) { + if (max_mem.compare_exchange_strong(max,mem)) + break; + mem = std::max(mem, _get_memory_usage()); + } + } +} + +struct LatenctyMem +{ + uint64_t max_nano = 0; + uint64_t max_mem = 0; +}; + +struct LatenctyMemResult +{ + std::vector results; + std::string name; +}; + +struct LatenctyFind +{ + uint64_t max_find = 0; + uint64_t max_failed = 0; +}; + +struct LatenctyFindResult +{ + std::vector results; + std::string name; +}; + +#define MAX_POINTS 1000 + +template +LatenctyMemResult test_hash_map_insert(const char* name) +{ + static constexpr uint64_t multiplier = 10000; + _reset_memory_usage(); + T map; + + std::mt19937 e(0); + std::uniform_int_distribution rng; + LatenctyMemResult ret; + ret.results.resize(MAX_POINTS); + ret.name = name; + size_t count = ret.results.size() * multiplier; + + std::atomic start = false, stop=false; + std::atomic max_mem = 0; + std::thread th([&]() { measure_max_mem(start, stop, max_mem); }); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + size_t start_mem = _get_memory_usage(); + start = true; + + timer t; + uint64_t el_max = 0; + + for (size_t i = 0; i< count; ++i) { + auto val = rng(e); + t.tick(); + map.emplace(val); + auto el = t.tock(); + el_max = std::max(el, el_max); + + std::this_thread::yield(); + + if (i % multiplier == 0) { + ret.results[i / multiplier] = { el_max, max_mem.load() - start_mem }; + el_max = 0; + } + } + + stop = true; + th.join(); + + // Find max time/max mem + uint64_t max_memv = 0; + uint64_t max_nanov = 0; + for (auto& m : ret.results) { + max_memv = std::max(max_memv, m.max_mem); + max_nanov = std::max(max_nanov, m.max_nano); + } + + std::cout << name << ": max " << (double)max_memv / (1024 * 1024) << " MB, " << max_nanov << " ns" << std::endl; + + return ret; +} + + +template +LatenctyFindResult test_hash_map_find(const char* name) +{ + static constexpr uint64_t multiplier = 10000; + T map; + + LatenctyFindResult ret; + ret.results.resize(MAX_POINTS); + ret.name = name; + size_t count = ret.results.size() * multiplier; + + timer t; + uint64_t el_max = 0; + uint64_t el_max_fail = 0; + std::vector rn_vals(multiplier); + + for (size_t i = 0; i < count; ++i) { + auto val = i; + map.emplace(val); + + if (i && i % multiplier == 0) { + + { + std::mt19937 e(i); + std::uniform_int_distribution rng(0, i); + // Find the last multiplier values; + for (auto& v : rn_vals) + v = rng(e); + + t.tick(); + for (size_t j = 0; j < 100; ++j) { + SEQ_TEST(map.count(rn_vals[j]) == 1); + } + auto el = t.tock(); + el_max = el / 100; + } + std::mt19937 e(i); + std::uniform_int_distribution rng(i+1, std::numeric_limits::max()); + for (auto& v : rn_vals) + v = rng(e); + // Find fail multiplier values; + t.tick(); + for (size_t j = 0; j < 100; ++j) { + SEQ_TEST(map.count(rn_vals[j]) == 0); + } + auto el = t.tock(); + el_max_fail = el / 100; + + ret.results[i / multiplier] = { el_max, el_max_fail }; + el_max = el_max_fail = 0; + } + } + + // Find max time/max mem + uint64_t max_find = 0; + uint64_t max_find_fail = 0; + for (auto& m : ret.results) { + max_find = std::max(max_find, m.max_find); + max_find_fail = std::max(max_find_fail, m.max_failed); + } + + std::cout << name << ": max find " << max_find << " ns, fail " << max_find_fail << " ns" << std::endl; + + return ret; +} + + + + +bool to_csv(const std::vector& res, const char* filename) +{ + std::ofstream fout(filename); + if (!fout) + return false; + fout << "\"sep=\t\"" << std::endl; + + fout << "Count\t"; + for (size_t i = 0; i < res.size(); ++i) { + fout << res[i].name << " peak mem(MB)\t" << res[i].name << " peak latency(ns)\t"; + } + fout << std::endl; + + for (size_t c = 0; c < MAX_POINTS; ++c) { + fout << c * MAX_POINTS << "\t"; + //auto& vec = res[c]; + for (size_t i = 0; i < res.size(); ++i) { + fout << (double)res[i].results[c].max_mem/(1024*1024) << "\t" << res[i].results[c].max_nano << "\t"; + } + fout << std::endl; + } + + return true; +} + +bool to_csv(const std::vector& res, const char* filename) +{ + std::ofstream fout(filename); + if (!fout) + return false; + fout << "\"sep=\t\"" << std::endl; + + fout << "Count\t"; + for (size_t i = 0; i < res.size(); ++i) { + fout << res[i].name << "\t" << res[i].name << "\t"; + } + fout << std::endl; + + for (size_t c = 0; c < MAX_POINTS; ++c) { + fout << c * MAX_POINTS << "\t"; + // auto& vec = res[c]; + for (size_t i = 0; i < res.size(); ++i) { + fout << res[i].results[c].max_find << "\t" << res[i].results[c].max_failed << "\t"; + } + fout << std::endl; + } + + return true; +} + + + + +int bench_hash_latency(int, char** const) +{ + using Hash = seq::hasher; + + /* { + std::vector ret; + ret.push_back(test_hash_map_insert>("tsl::sparse_set")); + ret.push_back(test_hash_map_insert>("gtl::flat_hash_set")); + ret.push_back(test_hash_map_insert>("ankerl::unordered_dense::set")); + ret.push_back(test_hash_map_insert>("seq::radix_hash_set")); + ret.push_back(test_hash_map_insert>("std::unordered_set")); + to_csv(ret, "bench_hash_latency.csv"); + }*/ + { + std::vector ret; + ret.push_back(test_hash_map_find>("tsl::sparse_set")); + ret.push_back(test_hash_map_find>("gtl::flat_hash_set")); + ret.push_back(test_hash_map_find>("ankerl::unordered_dense::set")); + ret.push_back(test_hash_map_find>("seq::radix_hash_set")); + ret.push_back(test_hash_map_find>("std::unordered_set")); + to_csv(ret, "bench_hash_find_latency.csv"); + } + return 0; +} \ No newline at end of file diff --git a/benchs/bench_map.cpp b/benchs/bench_map.cpp index 9b41bc40..57e1245f 100644 --- a/benchs/bench_map.cpp +++ b/benchs/bench_map.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,8 +24,7 @@ #include #include -#include -#include +#include #include #include "gtl/btree.hpp" @@ -36,6 +35,9 @@ #include #include #include +#include + +#include "testing.hpp" using namespace seq; @@ -357,7 +359,7 @@ void test_map(size_t count, Gen gen, Extract e = Extract()) std::sort(vec.begin(), vec.end(), less); auto it = std::unique(vec.begin(), vec.end(), equal); vec.resize(it - vec.begin()); - seq::random_shuffle(vec.begin(), vec.end(), 1); + std::shuffle(vec.begin(), vec.end(), std::mt19937(1)); // std::reverse(vec.begin(), vec.end()); std::cout << "vector size: " << vec.size() << std::endl; @@ -393,10 +395,6 @@ void test_map(size_t count, Gen gen, Extract e = Extract()) test_set>("std::map", vec, f); } - - - - /* template void test_small_map_repeat(const char * name, int count, int repeat, Format f) @@ -457,12 +455,11 @@ void test_small_map(int count, int repeat) } */ - int bench_map(int, char** const) { - + using string = tstring; - + // test random tuple { std::random_device dev; @@ -503,6 +500,6 @@ int bench_map(int, char** const) return seq::r_any(generate_random_string(13, true)); } }); - + return 0; } diff --git a/benchs/bench_map_csv.cpp b/benchs/bench_map_csv.cpp new file mode 100644 index 00000000..c4bf30bc --- /dev/null +++ b/benchs/bench_map_csv.cpp @@ -0,0 +1,291 @@ + +#include +#include +#include +#include +#include +#include +#include "gtl/btree.hpp" + +#ifdef BOOST_FOUND +#include "boost/container/flat_set.hpp" +#endif + +#include +#include +#include +#include +#include + +#include "testing.hpp" + +using namespace seq; + +template +struct is_boost_set : std::false_type +{ +}; +#ifdef BOOST_FOUND +template +struct is_boost_set> : std::true_type +{ +}; +#endif + +using measure_type = std::vector>; + +#define STEP 65536 + +template +measure_type bench_insert(Set& s, const std::vector& vec) +{ + + timer t; + + std::vector> elapsed; + elapsed.reserve(vec.size() / STEP + 2); + + if (is_boost_set::value) { + for (size_t i = 0; i < vec.size(); ++i) { + if (((i & (STEP-1)) == 0 && i)|| i == 1000) { + elapsed.push_back({ i, 0 }); + } + } + return elapsed; + } + + t.tick(); + + for (size_t i = 0; i < vec.size(); ++i) { + s.insert(vec[i]); + if (((i & (STEP-1)) == 0 && i)|| i == 1000) { + auto el = t.tock(); + elapsed.push_back({ i, el / (size_t)(i == 1000 ? 1000 : STEP) }); + t.tick(); + } + } + return elapsed; +} + +template +measure_type bench_erase(Set& s, const std::vector& vec) +{ + timer t; + + std::vector> elapsed; + elapsed.reserve(vec.size() / STEP + 2); + + if (is_boost_set::value) { + for (size_t i = 0; i < vec.size(); ++i) { + if (((i & (STEP-1)) == 0 && i)|| i == 1000) { + elapsed.push_back({ i, 0 }); + } + } + return elapsed; + } + + std::vector idx(vec.size()); + for (size_t i = 0; i < vec.size(); ++i) { + s.insert(vec[i]); + idx.push_back(i); + } + std::shuffle(idx.begin(), idx.end(), std::random_device{}); + + t.tick(); + + for (size_t i = 0; i < vec.size(); ++i) { + s.erase(vec[idx[i]]); + if (((i & (STEP-1)) == 0 && i)|| i == 1000) { + auto el = t.tock(); + elapsed.push_back({ vec.size() - i, el / (size_t)(i == 1000 ? 1000 : STEP) }); + t.tick(); + } + } + std::sort(elapsed.begin(), elapsed.end()); + return elapsed; +} + +template +measure_type bench_find_success(Set& s, const std::vector& vec) +{ + std::vector> elapsed; + elapsed.reserve(vec.size() / STEP + 2); + std::vector idx; + idx.reserve(vec.size()); + + for (size_t i = 0; i < vec.size(); ++i) { + s.insert(vec[i]); + idx.push_back(i); + if (((i & (STEP-1)) == 0 && i)|| i == 1000) { + + auto tmp = idx; + std::shuffle(tmp.begin(), tmp.end(), std::random_device{}); + timer t; + t.tick(); + + // perform look up of all inserted values + for (auto id : tmp) + SEQ_TEST(s.count(vec[id]) == 1); + + auto el = t.tock(); + elapsed.push_back({ i, el / tmp.size() }); + } + } + return elapsed; +} + +template +measure_type bench_find_failed(Set& s, const std::vector& vec, const std::vector& failed) +{ + std::vector> elapsed; + elapsed.reserve(vec.size() / STEP + 2); + + for (size_t i = 0; i < vec.size(); ++i) { + s.insert(vec[i]); + if (((i & (STEP-1)) == 0 && i)|| i == 1000) { + + timer t; + t.tick(); + + // perform look up of all inserted values + for (const auto& v : failed) { + SEQ_TEST(s.count(v) == 0); + } + + auto el = t.tock(); + elapsed.push_back({ i, el / failed.size() }); + } + } + return elapsed; +} + +static void print_measure(const measure_type& m) +{ + for (size_t i = 0; i < m.size(); ++i) + std::cout << m[i].first << " " << m[i].second << std::endl; +} + +struct SetResults +{ + std::string name; + measure_type insert; + measure_type erase; + measure_type find_success; + measure_type find_fail; +}; + +bool to_csv(const std::vector < SetResults>& res, const char* filename) +{ + std::ofstream fout(filename); + if (!fout) + return false; + fout << "\"sep=\t\"" << std::endl; + + fout << "\t"; + for (size_t i = 0; i < res.size(); ++i) { + auto n = res[i].name; + fout << n + "\t\t\t\t"; + } + fout << std::endl; + + fout << "count\t"; + for (size_t i = 0; i < res.size(); ++i) { + auto n = res[i].name; + fout << "insert\t" <<"erase\t" <<"find\t" << "fail\t" ; + } + fout << std::endl; + + size_t rows = res[0].insert.size(); + for (size_t j = 0; j < rows; ++j) { + + fout << res[0].insert[j].first << "\t"; + for (size_t i = 0; i < res.size(); ++i) { + auto & r = res[i]; + fout << r.insert[j].second << "\t" << r.erase[j].second << "\t" << r.find_success[j].second << "\t" << r.find_fail[j].second << "\t" ; + } + fout << std::endl; + + } + return true; +} + +template +SetResults bench_set(const char *name, const std::vector& keys, const std::vector& failed) +{ + std::cout <<"bench "< keys(1024*1024 + 5000); + for (size_t i = 0; i < keys.size(); ++i) + keys[i] = i; + + std::shuffle(keys.begin(), keys.end(), std::random_device{}); + std::vector failed(keys.begin() + keys.size() - 5000, keys.end()); + keys.erase(keys.begin() + keys.size() - failed.size(), keys.end()); + + std::vector res; + +//#ifdef BOOST_FOUND +// res.push_back(bench_set>("boost.flat_set", keys, failed)); +//#endif + res.push_back( bench_set>("flat_set", keys, failed)); + res.push_back( bench_set>("radix_set", keys, failed)); + + res.push_back( bench_set>("gtl::btree_set", keys, failed)); + res.push_back( bench_set>("std::set" ,keys, failed)); + + to_csv(res, "bench_map_size_t.csv"); + }*/ + + { + std::vector keys(1024 * 1024 * 4 + 5000); + for (size_t i = 0; i < keys.size(); ++i) + keys[i] = generate_random_string(63,true); + auto it = seq::unique(keys.begin(), keys.end()); + keys.resize(it - keys.begin()); + + std::shuffle(keys.begin(), keys.end(), std::random_device{}); + std::vector failed(keys.begin() + keys.size() - 5000, keys.end()); + keys.erase(keys.begin() + keys.size() - failed.size(), keys.end()); + + std::vector res; + + // #ifdef BOOST_FOUND + // res.push_back(bench_set>("boost.flat_set", keys, failed)); + // #endif + res.push_back(bench_set>("flat_set", keys, failed)); + res.push_back(bench_set>("radix_set", keys, failed)); + + res.push_back(bench_set>("gtl::btree_set", keys, failed)); + res.push_back(bench_set>("std::set", keys, failed)); + + to_csv(res, "bench_map_short_tstring.csv"); + } + + return 0; +} diff --git a/benchs/bench_sequence.cpp b/benchs/bench_sequence.cpp index 41abd4a5..e06e8631 100644 --- a/benchs/bench_sequence.cpp +++ b/benchs/bench_sequence.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,11 +26,15 @@ #define NOMINMAX #include -#include -#include +#include #include "plf/plf_colony.hpp" #include +#include + +#include + +#include "testing.hpp" #undef min #undef max @@ -46,16 +50,17 @@ void test_sequence_vs_colony(size_t count) using seq_type = sequence>; using colony_type = plf::colony; using list_type = std::list; + using deque_type = std::deque; std::cout << std::endl; - std::cout << "Compare performances of seq::sequence, plf::colony and std::list " << std::endl; + std::cout << "Compare performances of seq::sequence, plf::colony, std::deque and std::list " << std::endl; std::cout << std::endl; - std::cout << fmt(fmt("method").l(30), "|", fmt("plf::colony").c(20), "|", fmt("seq::sequence").c(20), "|", fmt("std::list").c(20), "|") << std::endl; - std::cout << fmt(str().l(30).f('-'), "|", str().c(20).f('-'), "|", str().c(20).f('-'), "|", str().c(20).f('-'), "|") << std::endl; + std::cout << fmt(fmt("method").l(30), "|", fmt("plf::colony").c(20), "|", fmt("seq::sequence").c(20), "|", fmt("std::deque").c(20), "|", fmt("std::list").c(20), "|") << std::endl; + std::cout << fmt(str().l(30).f('-'), "|", str().c(20).f('-'), "|", str().c(20).f('-'),"|", str().c(20).f('-'), "|", str().c(20).f('-'), "|") << std::endl; - auto f = fmt(pos<0, 2, 4, 6>(), str().l(30), "|", fmt(pos<0>(), size_t(), " ms").c(20), "|", fmt(pos<0>(), size_t(), " ms").c(20), "|", fmt(pos<0>(), size_t(), " ms").c(20), "|"); + auto f = fmt(pos<0, 2, 4, 6, 8>(), str().l(30), "|", fmt(pos<0>(), size_t(), " ms").c(20), "|", fmt(pos<0>(), size_t(), " ms").c(20), "|", fmt(pos<0>(), size_t(), " ms").c(20), "|", fmt(pos<0>(), size_t(), " ms").c(20), "|"); @@ -64,8 +69,9 @@ void test_sequence_vs_colony(size_t count) colony_type col1; seq_type seq1; list_type lst; + deque_type deq; - size_t col_t, seq_t, lst_t; + size_t col_t, seq_t, lst_t, deq_t; for (size_t i = 0; i < count; ++i) shufle.push_back((T)i); @@ -85,15 +91,22 @@ void test_sequence_vs_colony(size_t count) seq1.push_back((shufle[i])); seq_t = tock_ms(); + tick(); + for (int i = 0; i < count; ++i) + deq.push_back((shufle[i])); + deq_t = tock_ms(); + tick(); for (int i = 0; i < count; ++i) lst.push_back((shufle[i])); lst_t = tock_ms(); - std::cout << f("insert(reserve)", col_t, seq_t, lst_t) << std::endl; + std::cout << f("insert(reserve)", col_t, seq_t, deq_t, lst_t) << std::endl; col1 = colony_type{}; seq1 = seq_type{}; + deq.clear(); + lst.clear(); tick(); @@ -106,12 +119,17 @@ void test_sequence_vs_colony(size_t count) seq1.insert((shufle[i])); seq_t = tock_ms(); + tick(); + for (int i = 0; i < count; ++i) + deq.push_back((shufle[i])); + deq_t = tock_ms(); + tick(); for (int i = 0; i < count; ++i) lst.push_back((shufle[i])); lst_t = tock_ms(); - std::cout << f("insert", col_t, seq_t, lst_t) << std::endl; + std::cout << f("insert", col_t, seq_t,deq_t, lst_t) << std::endl; @@ -123,16 +141,21 @@ void test_sequence_vs_colony(size_t count) seq1.clear(); seq_t = tock_ms(); + tick(); + deq.clear(); + deq_t = tock_ms(); + tick(); lst.clear(); lst_t = tock_ms(); SEQ_TEST(seq1.size() == 0 && col1.size() == 0 && lst.size() == 0); - std::cout << f("clear", col_t, seq_t, lst_t) << std::endl; + std::cout << f("clear", col_t, seq_t, deq_t,lst_t) << std::endl; for (int i = 0; i < count; ++i) { col1.insert((shufle[i])); seq1.insert((shufle[i])); + deq.push_back((shufle[i])); lst.push_back((shufle[i])); } @@ -144,18 +167,23 @@ void test_sequence_vs_colony(size_t count) seq1.erase(seq1.begin(), seq1.end()); seq_t = tock_ms(); + tick(); + deq.erase(deq.begin(), deq.end()); + deq_t = tock_ms(); + tick(); lst.erase(lst.begin(), lst.end()); lst_t = tock_ms(); SEQ_TEST(seq1.size() == 0 && col1.size() == 0 && lst.size() == 0); - std::cout << f("erase(begin(),end())", col_t, seq_t, lst_t) << std::endl; + std::cout << f("erase(begin(),end())", col_t, seq_t, deq_t, lst_t) << std::endl; for (int i = 0; i < count; ++i) { col1.insert((shufle[i])); seq1.insert((shufle[i])); + deq.push_back((shufle[i])); lst.push_back((shufle[i])); } T sum = 0; @@ -170,12 +198,17 @@ void test_sequence_vs_colony(size_t count) sum += *it; seq_t = tock_ms(); + tick(); + for (auto it = deq.begin(); it != deq.end(); ++it) + sum += *it; + deq_t = tock_ms(); + tick(); for (auto it = lst.begin(); it != lst.end(); ++it) sum += *it; lst_t = tock_ms(); - std::cout << f("iterate", col_t, seq_t, lst_t) << std::endl; print_null(sum); + std::cout << f("iterate", col_t, seq_t, deq_t, lst_t) << std::endl; print_null(sum); @@ -193,6 +226,12 @@ void test_sequence_vs_colony(size_t count) } seq_t = tock_ms(); + tick(); + //for (auto it = deq.begin(); it != deq.end(); ++it) { + // it = deq.erase(it); + //} + deq_t = tock_ms(); + tick(); for (auto it = lst.begin(); it != lst.end(); ++it) { it = lst.erase(it); @@ -200,7 +239,7 @@ void test_sequence_vs_colony(size_t count) lst_t = tock_ms(); SEQ_TEST(seq1.size() == col1.size() && col1.size() == lst.size() ); - std::cout << f("erase half", col_t, seq_t, lst_t) << std::endl; + std::cout << f("erase half", col_t, seq_t, deq_t, lst_t) << std::endl; tick(); @@ -213,22 +252,29 @@ void test_sequence_vs_colony(size_t count) seq1.insert(shufle[i]); seq_t = tock_ms(); + tick(); + //for (int i = 0; i < shufle.size(); ++i) + // deq.push_back(shufle[i]); + deq_t = tock_ms(); + tick(); for (int i = 0; i < shufle.size(); ++i) lst.push_back(shufle[i]); lst_t = tock_ms(); - std::cout << f("insert again", col_t, seq_t, lst_t) << std::endl; + std::cout << f("insert again", col_t, seq_t, deq_t, lst_t) << std::endl; col1.clear(); seq1.clear(); lst.clear(); + deq.clear(); for (int i = 0; i < count; ++i) { seq1.push_back((shufle[i])); col1.insert((shufle[i])); lst.push_back((shufle[i])); + deq.push_back((shufle[i])); } @@ -240,11 +286,15 @@ void test_sequence_vs_colony(size_t count) seq1.sort(); seq_t = tock_ms(); + tick(); + std::sort(deq.begin(),deq.end()); + deq_t = tock_ms(); + tick(); lst.sort(); lst_t = tock_ms(); - std::cout << f("sort", col_t, seq_t, lst_t) << std::endl; + std::cout << f("sort", col_t, seq_t, deq_t, lst_t) << std::endl; } diff --git a/benchs/bench_sort.cpp b/benchs/bench_sort.cpp index d39580c3..3ca71265 100644 --- a/benchs/bench_sort.cpp +++ b/benchs/bench_sort.cpp @@ -1,16 +1,17 @@ #include #include +#include -#include -#include +#include #include -#include +#include #ifdef BOOST_FOUND #include #endif #include "pdqsort.hpp" +#include "testing.hpp" template void indisort(T start, size_t size, Cmp c) diff --git a/benchs/bench_text_stream.cpp b/benchs/bench_text_stream.cpp deleted file mode 100644 index 4e1a0fe5..00000000 --- a/benchs/bench_text_stream.cpp +++ /dev/null @@ -1,331 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Victor Moncada - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - - -#define NOMINMAX - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - - -using namespace seq; - - -inline std::vector generate_random_integers(size_t count) -{ - // Seed with a real random value, if available - std::random_device r1,r2,r3,r4; - std::default_random_engine e1(r1()), e2(r2()), e3(r3()), e4(r4()); - // Choose a random mean between 1 and 6 - - std::uniform_int_distribution uniform_dist_very_small(std::numeric_limits::min(), std::numeric_limits::max()); - std::uniform_int_distribution uniform_dist_small(std::numeric_limits::min(), std::numeric_limits::max()); - std::uniform_int_distribution uniform_dist_medium(std::numeric_limits::min(), std::numeric_limits::max()); - std::uniform_int_distribution uniform_dist_big(std::numeric_limits::min(), std::numeric_limits::max()); - std::vector vec(count); - for (size_t i = 0; i < count; ++i) { - std::int64_t val; - if ((i & 3) == 0) val = uniform_dist_very_small(e1) & 255; - else if ((i & 3) == 1) val = uniform_dist_small(e2); - else if ((i & 3) == 2) val = uniform_dist_medium(e3); - else val = uniform_dist_big(e4); - vec[i] = (val); - } - seq::random_shuffle(vec.begin(), vec.end()); - return vec; -} - -template -std::vector generate_random_float(size_t count) -{ - std::vector res(count); - - // Half is the result of casting random integers to float - std::vector tmp = generate_random_integers(count / 2); - for (size_t i = 0; i < tmp.size(); ++i) - res[i] = static_cast(tmp[i]); - - // Remaining half - random_float_genertor rng; - for (size_t i = count / 2; i < count; ++i) - { - res[i] = rng(); - - // remove infinit values as std::istream is not able to read them - while(std::isinf(res[i])) - res[i] = rng(); - } - - seq::random_shuffle(res.begin(), res.end()); - return res; -} - -template::value> -struct Generator -{ - using type = std::int64_t; - static std::vector generate(size_t count) - { - return generate_random_integers(count); - } -}; -template -struct Generator -{ - using type = T; - static std::vector generate(size_t count) - { - return generate_random_float(count); - } -}; - - - - -/// @brief Compare reading numeric values using seq::from_stream, strto* and std::istream -template -void test_read_numeric(size_t count) -{ - using type = typename Generator::type; - auto values = Generator::generate(count); - - std::cout << std::endl; - std::cout << "Test reading " << count << " values of type " << typeid(type).name() << std::endl; - std::cout << std::endl; - std::cout << fmt(fmt("Method").l(30), "|", fmt("Read").c(20), "|") << std::endl; - std::cout << fmt(str().c(30).f('-'), "|", str().c(20).f('-'), "|") << std::endl; - - auto f = fmt(pos<0,2>(), fmt("Method").l(30), "|", fmt().c(20), "|"); - - - - type sum = 0; - - //generate big string of values - std::ostringstream sout; - for (int i = 0; i < count; ++i) - sout << values[i] << " "; - std::string str = sout.str(); - - //test buffer_input_stream - tick(); - sum = 0; - seq::buffer_input_stream in(str); - while (in) { - type i; - seq::from_stream(in, i); - sum += i; - } - size_t b_i_s = tock_ms(); - print_null(sum); - - std::cout << f("seq::buffer_input_stream", b_i_s) << std::endl; - - //test std_input_stream - tick(); - sum = 0; - std::istringstream sin(str); - seq::std_input_stream seqin(sin); - while (seqin) { - type i; - seq::from_stream(seqin, i); - sum += i; - } - size_t s_i_s = tock_ms(); - print_null(sum); - - std::cout << f("seq::std_input_stream", s_i_s) << std::endl; - - //test strto* familly - tick(); - sum = 0; - const char* pos = str.data(); - while (true) { - char* end = (char*)str.data() + str.size(); - type i=0; - if (std::is_integral::value) - i = static_cast(strtoll(pos, &end, 10)); - else if (std::is_same::value) { - i = static_cast(strtold(pos, &end)); - } - else if (std::is_same::value) - i = static_cast(strtod(pos, &end)); - else if (std::is_same::value) - i = static_cast(strtof(pos, &end)); - if (end == NULL || end == pos) - break; - sum += i; - pos = end; - } - size_t strto = tock_ms(); - print_null(sum); - - std::cout << f("strto* familly", strto) << std::endl; - - //test istringstream - tick(); - sum = 0; - std::istringstream istr(str); - istr.sync_with_stdio(false); - while (istr) { - type i; - istr >> i; - sum += i; - } - size_t istream = tock_ms(); - print_null(sum); - - std::cout << f("std::istringstream", istream) << std::endl; -} - - - - -template::value> -struct Writer -{ - static auto write(char * first, char * last, const T& v, int ) - { - return seq::to_chars(first, last, v); - } -}; -template -struct Writer -{ - static auto write(char* first, char* last, const T& v, int precision) - { - return seq::to_chars(first, last, v, format, precision); - } -}; - -/// @brief Compare writing numeric values using seq::to_chars, seq::fmt, printf and std::ostream -template -void test_write_numeric(size_t count, int precision = 6) -{ - using type = typename Generator::type; - auto values = Generator::generate(count); - - std::cout << std::endl; - if(std::is_integral::value) - std::cout << "Test writing " << count << " values of type " << typeid(type).name() << std::endl; - else - std::cout << "Test writing " << count << " values of type " << typeid(type).name() << - " with format " << (format == seq::general ? "'general'" : format == seq::scientific ? "'scientific'" : "'fixed'") << " and precision " << precision << std::endl; - std::cout << std::endl; - - std::cout << fmt(fmt("Method").l(30), "|", fmt("Write").c(20), "|") << std::endl; - std::cout << fmt(str().c(30).f('-'), "|", str().c(20).f('-'), "|") << std::endl; - - auto f = fmt(pos<0, 2>(), fmt("Method").l(30), "|", fmt().c(20), "|"); - size_t sum = 0; - char buff[5000]; - - std::string pfmt; - { - std::ostringstream oss; - if (std::is_integral::value) - oss << "%lld"; - else if (std::is_same::value) - oss << "%." << precision << (format == seq::general ? "g" : format == seq::scientific ? "e" : "f"); - else if (std::is_same::value) - oss << "%." << precision << (format == seq::general ? "g" : format == seq::scientific ? "e" : "f"); - else if (std::is_same::value) - oss << "%." << precision << (format == seq::general ? "Lg" : format == seq::scientific ? "Le" : "Lf"); - pfmt = oss.str(); - } - - - tick(); - sum = 0; - for (size_t i = 0; i < count; ++i) - { - *Writer::write(buff, buff + sizeof(buff), values[i], precision).ptr = 0; - sum += buff[0]; - } - size_t t_c = tock_ms(); - - tick(); - sum = 0; - for (size_t i = 0; i < count; ++i) - { - if (std::is_integral::value) - fmt(values[i]).to_chars(buff); - else - fmt(values[i]).t (format == seq::general ? 'g' : format == seq::scientific ? 'e' : 'f').p(precision).to_chars(buff); - sum += buff[0]; - } - size_t f_m = tock_ms(); - - - tick(); - sum = 0; - for (size_t i = 0; i < count; ++i) - { - snprintf(buff,sizeof(buff), pfmt.c_str(), values[i]); - sum += buff[0]; - } - size_t prtf = tock_ms(); - - - tick(); - nullbuf n; - std::ostream oss(&n); - for (int i = 0; i < count; ++i) - { - oss << values[i]; - } - size_t ostr = tock_ms(); - - std::cout << f("seq::to_chars", t_c) << std::endl; - std::cout << f("seq::fmt", f_m) << std::endl; - std::cout << f("snprintf", prtf) << std::endl; - std::cout << f("std::ostream", ostr) << std::endl; - -} - - - -int bench_text_stream(int, char** const) -{ - test_write_numeric(1000000); - test_write_numeric(1000000, 12); - test_write_numeric(1000000, 12); - test_write_numeric(1000000, 12); - test_read_numeric(1000000); - test_read_numeric(1000000); - test_read_numeric(1000000); - - // last benchmark, as strtold is buggy on gcc - test_read_numeric(1000000); - - return 0; -} \ No newline at end of file diff --git a/benchs/bench_tiered_vector.cpp b/benchs/bench_tiered_vector.cpp index 1eaaca85..24dcf4c9 100644 --- a/benchs/bench_tiered_vector.cpp +++ b/benchs/bench_tiered_vector.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,8 +24,7 @@ #include #include -#include -#include +#include #include #include @@ -34,6 +33,8 @@ #include #include +#include "testing.hpp" + using namespace seq; template diff --git a/benchs/bench_tiny_string.cpp b/benchs/bench_tiny_string.cpp index 8e05dd2a..a5427aed 100644 --- a/benchs/bench_tiny_string.cpp +++ b/benchs/bench_tiny_string.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,16 +24,18 @@ #include -#include -#include +#include #include #include +#include extern "C" { #include } +#include "testing.hpp" + using namespace seq; diff --git a/benchs/concurrentqueue.h b/benchs/concurrentqueue.h new file mode 100644 index 00000000..5f46eb9a --- /dev/null +++ b/benchs/concurrentqueue.h @@ -0,0 +1,3769 @@ +// Provides a C++11 implementation of a multi-producer, multi-consumer lock-free queue. +// An overview, including benchmark results, is provided here: +// http://moodycamel.com/blog/2014/a-fast-general-purpose-lock-free-queue-for-c++ +// The full design is also described in excruciating detail at: +// http://moodycamel.com/blog/2014/detailed-design-of-a-lock-free-queue + +// Simplified BSD license: +// Copyright (c) 2013-2020, Cameron Desrochers. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// - Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// - Redistributions in binary form must reproduce the above copyright notice, this list of +// conditions and the following disclaimer in the documentation and/or other materials +// provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +// OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Also dual-licensed under the Boost Software License (see LICENSE.md) + +#pragma once + +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) +// Disable -Wconversion warnings (spuriously triggered when Traits::size_t and +// Traits::index_t are set to < 32 bits, causing integer promotion, causing warnings +// upon assigning any computed values) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" + +#ifdef MCDBGQ_USE_RELACY +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#endif +#endif + +#if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17) +// VS2019 with /W4 warns about constant conditional expressions but unless /std=c++17 or higher +// does not support `if constexpr`, so we have no choice but to simply disable the warning +#pragma warning(push) +#pragma warning(disable: 4127) // conditional expression is constant +#endif + +#if defined(__APPLE__) +#include "TargetConditionals.h" +#endif + +#ifdef MCDBGQ_USE_RELACY +#include "relacy/relacy_std.hpp" +#include "relacy_shims.h" +// We only use malloc/free anyway, and the delete macro messes up `= delete` method declarations. +// We'll override the default trait malloc ourselves without a macro. +#undef new +#undef delete +#undef malloc +#undef free +#else +#include // Requires C++11. Sorry VS2010. +#include +#endif +#include // for max_align_t +#include +#include +#include +#include +#include +#include +#include // for CHAR_BIT +#include +#include // partly for __WINPTHREADS_VERSION if on MinGW-w64 w/ POSIX threading +#include // used for thread exit synchronization + +// Platform-specific definitions of a numeric thread ID type and an invalid value +namespace moodycamel { namespace details { + template struct thread_id_converter { + typedef thread_id_t thread_id_numeric_size_t; + typedef thread_id_t thread_id_hash_t; + static thread_id_hash_t prehash(thread_id_t const& x) { return x; } + }; +} } +#if defined(MCDBGQ_USE_RELACY) +namespace moodycamel { namespace details { + typedef std::uint32_t thread_id_t; + static const thread_id_t invalid_thread_id = 0xFFFFFFFFU; + static const thread_id_t invalid_thread_id2 = 0xFFFFFFFEU; + static inline thread_id_t thread_id() { return rl::thread_index(); } +} } +#elif defined(_WIN32) || defined(__WINDOWS__) || defined(__WIN32__) +// No sense pulling in windows.h in a header, we'll manually declare the function +// we use and rely on backwards-compatibility for this not to break +extern "C" __declspec(dllimport) unsigned long __stdcall GetCurrentThreadId(void); +namespace moodycamel { namespace details { + static_assert(sizeof(unsigned long) == sizeof(std::uint32_t), "Expected size of unsigned long to be 32 bits on Windows"); + typedef std::uint32_t thread_id_t; + static const thread_id_t invalid_thread_id = 0; // See http://blogs.msdn.com/b/oldnewthing/archive/2004/02/23/78395.aspx + static const thread_id_t invalid_thread_id2 = 0xFFFFFFFFU; // Not technically guaranteed to be invalid, but is never used in practice. Note that all Win32 thread IDs are presently multiples of 4. + static inline thread_id_t thread_id() { return static_cast(::GetCurrentThreadId()); } +} } +#elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || (defined(__APPLE__) && TARGET_OS_IPHONE) || defined(__MVS__) || defined(MOODYCAMEL_NO_THREAD_LOCAL) +namespace moodycamel { namespace details { + static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, "std::thread::id is expected to be either 4 or 8 bytes"); + + typedef std::thread::id thread_id_t; + static const thread_id_t invalid_thread_id; // Default ctor creates invalid ID + + // Note we don't define a invalid_thread_id2 since std::thread::id doesn't have one; it's + // only used if MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is defined anyway, which it won't + // be. + static inline thread_id_t thread_id() { return std::this_thread::get_id(); } + + template struct thread_id_size { }; + template<> struct thread_id_size<4> { typedef std::uint32_t numeric_t; }; + template<> struct thread_id_size<8> { typedef std::uint64_t numeric_t; }; + + template<> struct thread_id_converter { + typedef thread_id_size::numeric_t thread_id_numeric_size_t; +#ifndef __APPLE__ + typedef std::size_t thread_id_hash_t; +#else + typedef thread_id_numeric_size_t thread_id_hash_t; +#endif + + static thread_id_hash_t prehash(thread_id_t const& x) + { +#ifndef __APPLE__ + return std::hash()(x); +#else + return *reinterpret_cast(&x); +#endif + } + }; +} } +#else +// Use a nice trick from this answer: http://stackoverflow.com/a/8438730/21475 +// In order to get a numeric thread ID in a platform-independent way, we use a thread-local +// static variable's address as a thread identifier :-) +#if defined(__GNUC__) || defined(__INTEL_COMPILER) +#define MOODYCAMEL_THREADLOCAL __thread +#elif defined(_MSC_VER) +#define MOODYCAMEL_THREADLOCAL __declspec(thread) +#else +// Assume C++11 compliant compiler +#define MOODYCAMEL_THREADLOCAL thread_local +#endif +namespace moodycamel { namespace details { + typedef std::uintptr_t thread_id_t; + static const thread_id_t invalid_thread_id = 0; // Address can't be nullptr + static const thread_id_t invalid_thread_id2 = 1; // Member accesses off a null pointer are also generally invalid. Plus it's not aligned. + inline thread_id_t thread_id() { static MOODYCAMEL_THREADLOCAL int x; return reinterpret_cast(&x); } +} } +#endif + +// Constexpr if +#ifndef MOODYCAMEL_CONSTEXPR_IF +#if (defined(_MSC_VER) && defined(_HAS_CXX17) && _HAS_CXX17) || __cplusplus > 201402L +#define MOODYCAMEL_CONSTEXPR_IF if constexpr +#define MOODYCAMEL_MAYBE_UNUSED [[maybe_unused]] +#else +#define MOODYCAMEL_CONSTEXPR_IF if +#define MOODYCAMEL_MAYBE_UNUSED +#endif +#endif + +// Exceptions +#ifndef MOODYCAMEL_EXCEPTIONS_ENABLED +#if (defined(_MSC_VER) && defined(_CPPUNWIND)) || (defined(__GNUC__) && defined(__EXCEPTIONS)) || (!defined(_MSC_VER) && !defined(__GNUC__)) +#define MOODYCAMEL_EXCEPTIONS_ENABLED +#endif +#endif +#ifdef MOODYCAMEL_EXCEPTIONS_ENABLED +#define MOODYCAMEL_TRY try +#define MOODYCAMEL_CATCH(...) catch(__VA_ARGS__) +#define MOODYCAMEL_RETHROW throw +#define MOODYCAMEL_THROW(expr) throw (expr) +#else +#define MOODYCAMEL_TRY MOODYCAMEL_CONSTEXPR_IF (true) +#define MOODYCAMEL_CATCH(...) else MOODYCAMEL_CONSTEXPR_IF (false) +#define MOODYCAMEL_RETHROW +#define MOODYCAMEL_THROW(expr) +#endif + +#ifndef MOODYCAMEL_NOEXCEPT +#if !defined(MOODYCAMEL_EXCEPTIONS_ENABLED) +#define MOODYCAMEL_NOEXCEPT +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) true +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) true +#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1800 +// VS2012's std::is_nothrow_[move_]constructible is broken and returns true when it shouldn't :-( +// We have to assume *all* non-trivial constructors may throw on VS2012! +#define MOODYCAMEL_NOEXCEPT _NOEXCEPT +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference::value && std::is_move_constructible::value ? std::is_trivially_move_constructible::value : std::is_trivially_copy_constructible::value) +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference::value && std::is_move_assignable::value ? std::is_trivially_move_assignable::value || std::is_nothrow_move_assignable::value : std::is_trivially_copy_assignable::value || std::is_nothrow_copy_assignable::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)) +#elif defined(_MSC_VER) && defined(_NOEXCEPT) && _MSC_VER < 1900 +#define MOODYCAMEL_NOEXCEPT _NOEXCEPT +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) (std::is_rvalue_reference::value && std::is_move_constructible::value ? std::is_trivially_move_constructible::value || std::is_nothrow_move_constructible::value : std::is_trivially_copy_constructible::value || std::is_nothrow_copy_constructible::value) +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) ((std::is_rvalue_reference::value && std::is_move_assignable::value ? std::is_trivially_move_assignable::value || std::is_nothrow_move_assignable::value : std::is_trivially_copy_assignable::value || std::is_nothrow_copy_assignable::value) && MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr)) +#else +#define MOODYCAMEL_NOEXCEPT noexcept +#define MOODYCAMEL_NOEXCEPT_CTOR(type, valueType, expr) noexcept(expr) +#define MOODYCAMEL_NOEXCEPT_ASSIGN(type, valueType, expr) noexcept(expr) +#endif +#endif + +#ifndef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#ifdef MCDBGQ_USE_RELACY +#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#else +// VS2013 doesn't support `thread_local`, and MinGW-w64 w/ POSIX threading has a crippling bug: http://sourceforge.net/p/mingw-w64/bugs/445 +// g++ <=4.7 doesn't support thread_local either. +// Finally, iOS/ARM doesn't have support for it either, and g++/ARM allows it to compile but it's unconfirmed to actually work +#if (!defined(_MSC_VER) || _MSC_VER >= 1900) && (!defined(__MINGW32__) && !defined(__MINGW64__) || !defined(__WINPTHREADS_VERSION)) && (!defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && (!defined(__APPLE__) || !TARGET_OS_IPHONE) && !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(__MVS__) && !defined(MOODYCAMEL_NO_THREAD_LOCAL) +// Assume `thread_local` is fully supported in all other C++11 compilers/platforms +#define MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED // tentatively enabled for now; years ago several users report having problems with it on +#endif +#endif +#endif + +// VS2012 doesn't support deleted functions. +// In this case, we declare the function normally but don't define it. A link error will be generated if the function is called. +#ifndef MOODYCAMEL_DELETE_FUNCTION +#if defined(_MSC_VER) && _MSC_VER < 1800 +#define MOODYCAMEL_DELETE_FUNCTION +#else +#define MOODYCAMEL_DELETE_FUNCTION = delete +#endif +#endif + +namespace moodycamel { namespace details { +#ifndef MOODYCAMEL_ALIGNAS +// VS2013 doesn't support alignas or alignof, and align() requires a constant literal +#if defined(_MSC_VER) && _MSC_VER <= 1800 +#define MOODYCAMEL_ALIGNAS(alignment) __declspec(align(alignment)) +#define MOODYCAMEL_ALIGNOF(obj) __alignof(obj) +#define MOODYCAMEL_ALIGNED_TYPE_LIKE(T, obj) typename details::Vs2013Aligned::value, T>::type + template struct Vs2013Aligned { }; // default, unsupported alignment + template struct Vs2013Aligned<1, T> { typedef __declspec(align(1)) T type; }; + template struct Vs2013Aligned<2, T> { typedef __declspec(align(2)) T type; }; + template struct Vs2013Aligned<4, T> { typedef __declspec(align(4)) T type; }; + template struct Vs2013Aligned<8, T> { typedef __declspec(align(8)) T type; }; + template struct Vs2013Aligned<16, T> { typedef __declspec(align(16)) T type; }; + template struct Vs2013Aligned<32, T> { typedef __declspec(align(32)) T type; }; + template struct Vs2013Aligned<64, T> { typedef __declspec(align(64)) T type; }; + template struct Vs2013Aligned<128, T> { typedef __declspec(align(128)) T type; }; + template struct Vs2013Aligned<256, T> { typedef __declspec(align(256)) T type; }; +#else + template struct identity { typedef T type; }; +#define MOODYCAMEL_ALIGNAS(alignment) alignas(alignment) +#define MOODYCAMEL_ALIGNOF(obj) alignof(obj) +#define MOODYCAMEL_ALIGNED_TYPE_LIKE(T, obj) alignas(alignof(obj)) typename details::identity::type +#endif +#endif +} } + + +// TSAN can false report races in lock-free code. To enable TSAN to be used from projects that use this one, +// we can apply per-function compile-time suppression. +// See https://clang.llvm.org/docs/ThreadSanitizer.html#has-feature-thread-sanitizer +#define MOODYCAMEL_NO_TSAN +#if defined(__has_feature) + #if __has_feature(thread_sanitizer) + #undef MOODYCAMEL_NO_TSAN + #define MOODYCAMEL_NO_TSAN __attribute__((no_sanitize("thread"))) + #endif // TSAN +#endif // TSAN + +// Compiler-specific likely/unlikely hints +namespace moodycamel { namespace details { +#if defined(__GNUC__) + static inline bool (likely)(bool x) { return __builtin_expect((x), true); } + static inline bool (unlikely)(bool x) { return __builtin_expect((x), false); } +#else + static inline bool (likely)(bool x) { return x; } + static inline bool (unlikely)(bool x) { return x; } +#endif +} } + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG +#include "internal/concurrentqueue_internal_debug.h" +#endif + +namespace moodycamel { +namespace details { + template + struct const_numeric_max { + static_assert(std::is_integral::value, "const_numeric_max can only be used with integers"); + static const T value = std::numeric_limits::is_signed + ? (static_cast(1) << (sizeof(T) * CHAR_BIT - 1)) - static_cast(1) + : static_cast(-1); + }; + +#if defined(__GLIBCXX__) + typedef ::max_align_t std_max_align_t; // libstdc++ forgot to add it to std:: for a while +#else + typedef std::max_align_t std_max_align_t; // Others (e.g. MSVC) insist it can *only* be accessed via std:: +#endif + + // Some platforms have incorrectly set max_align_t to a type with <8 bytes alignment even while supporting + // 8-byte aligned scalar values (*cough* 32-bit iOS). Work around this with our own union. See issue #64. + typedef union { + std_max_align_t x; + long long y; + void* z; + } max_align_t; +} + +// Default traits for the ConcurrentQueue. To change some of the +// traits without re-implementing all of them, inherit from this +// struct and shadow the declarations you wish to be different; +// since the traits are used as a template type parameter, the +// shadowed declarations will be used where defined, and the defaults +// otherwise. +struct ConcurrentQueueDefaultTraits +{ + // General-purpose size type. std::size_t is strongly recommended. + typedef std::size_t size_t; + + // The type used for the enqueue and dequeue indices. Must be at least as + // large as size_t. Should be significantly larger than the number of elements + // you expect to hold at once, especially if you have a high turnover rate; + // for example, on 32-bit x86, if you expect to have over a hundred million + // elements or pump several million elements through your queue in a very + // short space of time, using a 32-bit type *may* trigger a race condition. + // A 64-bit int type is recommended in that case, and in practice will + // prevent a race condition no matter the usage of the queue. Note that + // whether the queue is lock-free with a 64-int type depends on the whether + // std::atomic is lock-free, which is platform-specific. + typedef std::size_t index_t; + + // Internally, all elements are enqueued and dequeued from multi-element + // blocks; this is the smallest controllable unit. If you expect few elements + // but many producers, a smaller block size should be favoured. For few producers + // and/or many elements, a larger block size is preferred. A sane default + // is provided. Must be a power of 2. + static const size_t BLOCK_SIZE = 32; + + // For explicit producers (i.e. when using a producer token), the block is + // checked for being empty by iterating through a list of flags, one per element. + // For large block sizes, this is too inefficient, and switching to an atomic + // counter-based approach is faster. The switch is made for block sizes strictly + // larger than this threshold. + static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = 32; + + // How many full blocks can be expected for a single explicit producer? This should + // reflect that number's maximum for optimal performance. Must be a power of 2. + static const size_t EXPLICIT_INITIAL_INDEX_SIZE = 32; + + // How many full blocks can be expected for a single implicit producer? This should + // reflect that number's maximum for optimal performance. Must be a power of 2. + // Note: This impacts the maximum number of elements that can be enqueued by a + // single implicit producer when using try_enqueue/try_enqeue_bulk exclusively (which + // cannot allocate), since it limits the number of blocks that the producer can hold to + // store elements. When pre-allocating blocks for use with try-enqueueing, configure + // this initial size to the desired maximum number of blocks per implicit producer. + // Alternately, use the regular enqueue methods, which can grow the index as needed. + static const size_t IMPLICIT_INITIAL_INDEX_SIZE = 32; + + // The initial size of the hash table mapping thread IDs to implicit producers. + // Note that the hash is resized every time it becomes half full. + // Must be a power of two, and either 0 or at least 1. If 0, implicit production + // (using the enqueue methods without an explicit producer token) is disabled. + static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = 32; + + // Controls the number of items that an explicit consumer (i.e. one with a token) + // must consume before it causes all consumers to rotate and move on to the next + // internal queue. + static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = 256; + + // The maximum number of elements (inclusive) that can be enqueued to a sub-queue. + // Enqueue operations that would cause this limit to be surpassed will fail. Note + // that this limit is enforced at the block level (for performance reasons), i.e. + // it's rounded up to the nearest block size. + static const size_t MAX_SUBQUEUE_SIZE = details::const_numeric_max::value; + + // The number of times to spin before sleeping when waiting on a semaphore. + // Recommended values are on the order of 1000-10000 unless the number of + // consumer threads exceeds the number of idle cores (in which case try 0-100). + // Only affects instances of the BlockingConcurrentQueue. + static const int MAX_SEMA_SPINS = 10000; + + // Whether to recycle dynamically-allocated blocks into an internal free list or + // not. If false, only pre-allocated blocks (controlled by the constructor + // arguments) will be recycled, and all others will be `free`d back to the heap. + // Note that blocks consumed by explicit producers are only freed on destruction + // of the queue (not following destruction of the token) regardless of this trait. + static const bool RECYCLE_ALLOCATED_BLOCKS = false; + + +#ifndef MCDBGQ_USE_RELACY + // Memory allocation can be customized if needed. + // malloc should return nullptr on failure, and handle alignment like std::malloc. +#if defined(malloc) || defined(free) + // Gah, this is 2015, stop defining macros that break standard code already! + // Work around malloc/free being special macros: + static inline void* WORKAROUND_malloc(size_t size) { return malloc(size); } + static inline void WORKAROUND_free(void* ptr) { return free(ptr); } + static inline void* (malloc)(size_t size) { return WORKAROUND_malloc(size); } + static inline void (free)(void* ptr) { return WORKAROUND_free(ptr); } +#else + static inline void* malloc(size_t size) { return std::malloc(size); } + static inline void free(void* ptr) { return std::free(ptr); } +#endif +#else + // Debug versions when running under the Relacy race detector (ignore + // these in user code) + static inline void* malloc(size_t size) { return rl::rl_malloc(size, $); } + static inline void free(void* ptr) { return rl::rl_free(ptr, $); } +#endif +}; + + +// When producing or consuming many elements, the most efficient way is to: +// 1) Use one of the bulk-operation methods of the queue with a token +// 2) Failing that, use the bulk-operation methods without a token +// 3) Failing that, create a token and use that with the single-item methods +// 4) Failing that, use the single-parameter methods of the queue +// Having said that, don't create tokens willy-nilly -- ideally there should be +// a maximum of one token per thread (of each kind). +struct ProducerToken; +struct ConsumerToken; + +template class ConcurrentQueue; +template class BlockingConcurrentQueue; +class ConcurrentQueueTests; + + +namespace details +{ + struct ConcurrentQueueProducerTypelessBase + { + ConcurrentQueueProducerTypelessBase* next; + std::atomic inactive; + ProducerToken* token; + + ConcurrentQueueProducerTypelessBase() + : next(nullptr), inactive(false), token(nullptr) + { + } + }; + + template struct _hash_32_or_64 { + static inline std::uint32_t hash(std::uint32_t h) + { + // MurmurHash3 finalizer -- see https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp + // Since the thread ID is already unique, all we really want to do is propagate that + // uniqueness evenly across all the bits, so that we can use a subset of the bits while + // reducing collisions significantly + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + return h ^ (h >> 16); + } + }; + template<> struct _hash_32_or_64<1> { + static inline std::uint64_t hash(std::uint64_t h) + { + h ^= h >> 33; + h *= 0xff51afd7ed558ccd; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53; + return h ^ (h >> 33); + } + }; + template struct hash_32_or_64 : public _hash_32_or_64<(size > 4)> { }; + + static inline size_t hash_thread_id(thread_id_t id) + { + static_assert(sizeof(thread_id_t) <= 8, "Expected a platform where thread IDs are at most 64-bit values"); +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuseless-cast" +#endif + return static_cast(hash_32_or_64::thread_id_hash_t)>::hash( + thread_id_converter::prehash(id))); +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + } + + template + static inline bool circular_less_than(T a, T b) + { + static_assert(std::is_integral::value && !std::numeric_limits::is_signed, "circular_less_than is intended to be used only with unsigned integer types"); + return static_cast(a - b) > static_cast(static_cast(1) << (static_cast(sizeof(T) * CHAR_BIT - 1))); + // Note: extra parens around rhs of operator<< is MSVC bug: https://developercommunity2.visualstudio.com/t/C4554-triggers-when-both-lhs-and-rhs-is/10034931 + // silencing the bug requires #pragma warning(disable: 4554) around the calling code and has no effect when done here. + } + + template + static inline char* align_for(char* ptr) + { + const std::size_t alignment = std::alignment_of::value; + return ptr + (alignment - (reinterpret_cast(ptr) % alignment)) % alignment; + } + + template + static inline T ceil_to_pow_2(T x) + { + static_assert(std::is_integral::value && !std::numeric_limits::is_signed, "ceil_to_pow_2 is intended to be used only with unsigned integer types"); + + // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + --x; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + for (std::size_t i = 1; i < sizeof(T); i <<= 1) { + x |= x >> (i << 3); + } + ++x; + return x; + } + + template + static inline void swap_relaxed(std::atomic& left, std::atomic& right) + { + T temp = left.load(std::memory_order_relaxed); + left.store(right.load(std::memory_order_relaxed), std::memory_order_relaxed); + right.store(temp, std::memory_order_relaxed); + } + + template + static inline T const& nomove(T const& x) + { + return x; + } + + template + struct nomove_if + { + template + static inline T const& eval(T const& x) + { + return x; + } + }; + + template<> + struct nomove_if + { + template + static inline auto eval(U&& x) + -> decltype(std::forward(x)) + { + return std::forward(x); + } + }; + + template + static inline auto deref_noexcept(It& it) MOODYCAMEL_NOEXCEPT -> decltype(*it) + { + return *it; + } + +#if defined(__clang__) || !defined(__GNUC__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) + template struct is_trivially_destructible : std::is_trivially_destructible { }; +#else + template struct is_trivially_destructible : std::has_trivial_destructor { }; +#endif + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED +#ifdef MCDBGQ_USE_RELACY + typedef RelacyThreadExitListener ThreadExitListener; + typedef RelacyThreadExitNotifier ThreadExitNotifier; +#else + class ThreadExitNotifier; + + struct ThreadExitListener + { + typedef void (*callback_t)(void*); + callback_t callback; + void* userData; + + ThreadExitListener* next; // reserved for use by the ThreadExitNotifier + ThreadExitNotifier* chain; // reserved for use by the ThreadExitNotifier + }; + + class ThreadExitNotifier + { + public: + static void subscribe(ThreadExitListener* listener) + { + auto& tlsInst = instance(); + std::lock_guard guard(mutex()); + listener->next = tlsInst.tail; + listener->chain = &tlsInst; + tlsInst.tail = listener; + } + + static void unsubscribe(ThreadExitListener* listener) + { + std::lock_guard guard(mutex()); + if (!listener->chain) { + return; // race with ~ThreadExitNotifier + } + auto& tlsInst = *listener->chain; + listener->chain = nullptr; + ThreadExitListener** prev = &tlsInst.tail; + for (auto ptr = tlsInst.tail; ptr != nullptr; ptr = ptr->next) { + if (ptr == listener) { + *prev = ptr->next; + break; + } + prev = &ptr->next; + } + } + + private: + ThreadExitNotifier() : tail(nullptr) { } + ThreadExitNotifier(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; + ThreadExitNotifier& operator=(ThreadExitNotifier const&) MOODYCAMEL_DELETE_FUNCTION; + + ~ThreadExitNotifier() + { + // This thread is about to exit, let everyone know! + assert(this == &instance() && "If this assert fails, you likely have a buggy compiler! Change the preprocessor conditions such that MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED is no longer defined."); + std::lock_guard guard(mutex()); + for (auto ptr = tail; ptr != nullptr; ptr = ptr->next) { + ptr->chain = nullptr; + ptr->callback(ptr->userData); + } + } + + // Thread-local + static inline ThreadExitNotifier& instance() + { + static thread_local ThreadExitNotifier notifier; + return notifier; + } + + static inline std::mutex& mutex() + { + // Must be static because the ThreadExitNotifier could be destroyed while unsubscribe is called + static std::mutex mutex; + return mutex; + } + + private: + ThreadExitListener* tail; + }; +#endif +#endif + + template struct static_is_lock_free_num { enum { value = 0 }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_CHAR_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_SHORT_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_INT_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_LONG_LOCK_FREE }; }; + template<> struct static_is_lock_free_num { enum { value = ATOMIC_LLONG_LOCK_FREE }; }; + template struct static_is_lock_free : static_is_lock_free_num::type> { }; + template<> struct static_is_lock_free { enum { value = ATOMIC_BOOL_LOCK_FREE }; }; + template struct static_is_lock_free { enum { value = ATOMIC_POINTER_LOCK_FREE }; }; +} + + +struct ProducerToken +{ + template + explicit ProducerToken(ConcurrentQueue& queue); + + template + explicit ProducerToken(BlockingConcurrentQueue& queue); + + ProducerToken(ProducerToken&& other) MOODYCAMEL_NOEXCEPT + : producer(other.producer) + { + other.producer = nullptr; + if (producer != nullptr) { + producer->token = this; + } + } + + inline ProducerToken& operator=(ProducerToken&& other) MOODYCAMEL_NOEXCEPT + { + swap(other); + return *this; + } + + void swap(ProducerToken& other) MOODYCAMEL_NOEXCEPT + { + std::swap(producer, other.producer); + if (producer != nullptr) { + producer->token = this; + } + if (other.producer != nullptr) { + other.producer->token = &other; + } + } + + // A token is always valid unless: + // 1) Memory allocation failed during construction + // 2) It was moved via the move constructor + // (Note: assignment does a swap, leaving both potentially valid) + // 3) The associated queue was destroyed + // Note that if valid() returns true, that only indicates + // that the token is valid for use with a specific queue, + // but not which one; that's up to the user to track. + inline bool valid() const { return producer != nullptr; } + + ~ProducerToken() + { + if (producer != nullptr) { + producer->token = nullptr; + producer->inactive.store(true, std::memory_order_release); + } + } + + // Disable copying and assignment + ProducerToken(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; + ProducerToken& operator=(ProducerToken const&) MOODYCAMEL_DELETE_FUNCTION; + +private: + template friend class ConcurrentQueue; + friend class ConcurrentQueueTests; + +protected: + details::ConcurrentQueueProducerTypelessBase* producer; +}; + + +struct ConsumerToken +{ + template + explicit ConsumerToken(ConcurrentQueue& q); + + template + explicit ConsumerToken(BlockingConcurrentQueue& q); + + ConsumerToken(ConsumerToken&& other) MOODYCAMEL_NOEXCEPT + : initialOffset(other.initialOffset), lastKnownGlobalOffset(other.lastKnownGlobalOffset), itemsConsumedFromCurrent(other.itemsConsumedFromCurrent), currentProducer(other.currentProducer), desiredProducer(other.desiredProducer) + { + } + + inline ConsumerToken& operator=(ConsumerToken&& other) MOODYCAMEL_NOEXCEPT + { + swap(other); + return *this; + } + + void swap(ConsumerToken& other) MOODYCAMEL_NOEXCEPT + { + std::swap(initialOffset, other.initialOffset); + std::swap(lastKnownGlobalOffset, other.lastKnownGlobalOffset); + std::swap(itemsConsumedFromCurrent, other.itemsConsumedFromCurrent); + std::swap(currentProducer, other.currentProducer); + std::swap(desiredProducer, other.desiredProducer); + } + + // Disable copying and assignment + ConsumerToken(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; + ConsumerToken& operator=(ConsumerToken const&) MOODYCAMEL_DELETE_FUNCTION; + +private: + template friend class ConcurrentQueue; + friend class ConcurrentQueueTests; + +private: // but shared with ConcurrentQueue + std::uint32_t initialOffset; + std::uint32_t lastKnownGlobalOffset; + std::uint32_t itemsConsumedFromCurrent; + details::ConcurrentQueueProducerTypelessBase* currentProducer; + details::ConcurrentQueueProducerTypelessBase* desiredProducer; +}; + +// Need to forward-declare this swap because it's in a namespace. +// See http://stackoverflow.com/questions/4492062/why-does-a-c-friend-class-need-a-forward-declaration-only-in-other-namespaces +template +inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, typename ConcurrentQueue::ImplicitProducerKVP& b) MOODYCAMEL_NOEXCEPT; + + +template +class ConcurrentQueue +{ +public: + typedef ::moodycamel::ProducerToken producer_token_t; + typedef ::moodycamel::ConsumerToken consumer_token_t; + + typedef typename Traits::index_t index_t; + typedef typename Traits::size_t size_t; + + static const size_t BLOCK_SIZE = static_cast(Traits::BLOCK_SIZE); + static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = static_cast(Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD); + static const size_t EXPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::EXPLICIT_INITIAL_INDEX_SIZE); + static const size_t IMPLICIT_INITIAL_INDEX_SIZE = static_cast(Traits::IMPLICIT_INITIAL_INDEX_SIZE); + static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = static_cast(Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE); + static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = static_cast(Traits::EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE); +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4307) // + integral constant overflow (that's what the ternary expression is for!) +#pragma warning(disable: 4309) // static_cast: Truncation of constant value +#endif + static const size_t MAX_SUBQUEUE_SIZE = (details::const_numeric_max::value - static_cast(Traits::MAX_SUBQUEUE_SIZE) < BLOCK_SIZE) ? details::const_numeric_max::value : ((static_cast(Traits::MAX_SUBQUEUE_SIZE) + (BLOCK_SIZE - 1)) / BLOCK_SIZE * BLOCK_SIZE); +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + static_assert(!std::numeric_limits::is_signed && std::is_integral::value, "Traits::size_t must be an unsigned integral type"); + static_assert(!std::numeric_limits::is_signed && std::is_integral::value, "Traits::index_t must be an unsigned integral type"); + static_assert(sizeof(index_t) >= sizeof(size_t), "Traits::index_t must be at least as wide as Traits::size_t"); + static_assert((BLOCK_SIZE > 1) && !(BLOCK_SIZE & (BLOCK_SIZE - 1)), "Traits::BLOCK_SIZE must be a power of 2 (and at least 2)"); + static_assert((EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD > 1) && !(EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD & (EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD - 1)), "Traits::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD must be a power of 2 (and greater than 1)"); + static_assert((EXPLICIT_INITIAL_INDEX_SIZE > 1) && !(EXPLICIT_INITIAL_INDEX_SIZE & (EXPLICIT_INITIAL_INDEX_SIZE - 1)), "Traits::EXPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); + static_assert((IMPLICIT_INITIAL_INDEX_SIZE > 1) && !(IMPLICIT_INITIAL_INDEX_SIZE & (IMPLICIT_INITIAL_INDEX_SIZE - 1)), "Traits::IMPLICIT_INITIAL_INDEX_SIZE must be a power of 2 (and greater than 1)"); + static_assert((INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) || !(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE & (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE - 1)), "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be a power of 2"); + static_assert(INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0 || INITIAL_IMPLICIT_PRODUCER_HASH_SIZE >= 1, "Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE must be at least 1 (or 0 to disable implicit enqueueing)"); + +public: + // Creates a queue with at least `capacity` element slots; note that the + // actual number of elements that can be inserted without additional memory + // allocation depends on the number of producers and the block size (e.g. if + // the block size is equal to `capacity`, only a single block will be allocated + // up-front, which means only a single producer will be able to enqueue elements + // without an extra allocation -- blocks aren't shared between producers). + // This method is not thread safe -- it is up to the user to ensure that the + // queue is fully constructed before it starts being used by other threads (this + // includes making the memory effects of construction visible, possibly with a + // memory barrier). + explicit ConcurrentQueue(size_t capacity = 32 * BLOCK_SIZE) + : producerListTail(nullptr), + producerCount(0), + initialBlockPoolIndex(0), + nextExplicitConsumerId(0), + globalExplicitConsumerOffset(0) + { + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + populate_initial_implicit_producer_hash(); + populate_initial_block_list(capacity / BLOCK_SIZE + ((capacity & (BLOCK_SIZE - 1)) == 0 ? 0 : 1)); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + // Track all the producers using a fully-resolved typed list for + // each kind; this makes it possible to debug them starting from + // the root queue object (otherwise wacky casts are needed that + // don't compile in the debugger's expression evaluator). + explicitProducers.store(nullptr, std::memory_order_relaxed); + implicitProducers.store(nullptr, std::memory_order_relaxed); +#endif + } + + // Computes the correct amount of pre-allocated blocks for you based + // on the minimum number of elements you want available at any given + // time, and the maximum concurrent number of each type of producer. + ConcurrentQueue(size_t minCapacity, size_t maxExplicitProducers, size_t maxImplicitProducers) + : producerListTail(nullptr), + producerCount(0), + initialBlockPoolIndex(0), + nextExplicitConsumerId(0), + globalExplicitConsumerOffset(0) + { + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + populate_initial_implicit_producer_hash(); + size_t blocks = (((minCapacity + BLOCK_SIZE - 1) / BLOCK_SIZE) - 1) * (maxExplicitProducers + 1) + 2 * (maxExplicitProducers + maxImplicitProducers); + populate_initial_block_list(blocks); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + explicitProducers.store(nullptr, std::memory_order_relaxed); + implicitProducers.store(nullptr, std::memory_order_relaxed); +#endif + } + + // Note: The queue should not be accessed concurrently while it's + // being deleted. It's up to the user to synchronize this. + // This method is not thread safe. + ~ConcurrentQueue() + { + // Destroy producers + auto ptr = producerListTail.load(std::memory_order_relaxed); + while (ptr != nullptr) { + auto next = ptr->next_prod(); + if (ptr->token != nullptr) { + ptr->token->producer = nullptr; + } + destroy(ptr); + ptr = next; + } + + // Destroy implicit producer hash tables + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE != 0) { + auto hash = implicitProducerHash.load(std::memory_order_relaxed); + while (hash != nullptr) { + auto prev = hash->prev; + if (prev != nullptr) { // The last hash is part of this object and was not allocated dynamically + for (size_t i = 0; i != hash->capacity; ++i) { + hash->entries[i].~ImplicitProducerKVP(); + } + hash->~ImplicitProducerHash(); + (Traits::free)(hash); + } + hash = prev; + } + } + + // Destroy global free list + auto block = freeList.head_unsafe(); + while (block != nullptr) { + auto next = block->freeListNext.load(std::memory_order_relaxed); + if (block->dynamicallyAllocated) { + destroy(block); + } + block = next; + } + + // Destroy initial free list + destroy_array(initialBlockPool, initialBlockPoolSize); + } + + // Disable copying and copy assignment + ConcurrentQueue(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; + ConcurrentQueue& operator=(ConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION; + + // Moving is supported, but note that it is *not* a thread-safe operation. + // Nobody can use the queue while it's being moved, and the memory effects + // of that move must be propagated to other threads before they can use it. + // Note: When a queue is moved, its tokens are still valid but can only be + // used with the destination queue (i.e. semantically they are moved along + // with the queue itself). + ConcurrentQueue(ConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT + : producerListTail(other.producerListTail.load(std::memory_order_relaxed)), + producerCount(other.producerCount.load(std::memory_order_relaxed)), + initialBlockPoolIndex(other.initialBlockPoolIndex.load(std::memory_order_relaxed)), + initialBlockPool(other.initialBlockPool), + initialBlockPoolSize(other.initialBlockPoolSize), + freeList(std::move(other.freeList)), + nextExplicitConsumerId(other.nextExplicitConsumerId.load(std::memory_order_relaxed)), + globalExplicitConsumerOffset(other.globalExplicitConsumerOffset.load(std::memory_order_relaxed)) + { + // Move the other one into this, and leave the other one as an empty queue + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + populate_initial_implicit_producer_hash(); + swap_implicit_producer_hashes(other); + + other.producerListTail.store(nullptr, std::memory_order_relaxed); + other.producerCount.store(0, std::memory_order_relaxed); + other.nextExplicitConsumerId.store(0, std::memory_order_relaxed); + other.globalExplicitConsumerOffset.store(0, std::memory_order_relaxed); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + explicitProducers.store(other.explicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.explicitProducers.store(nullptr, std::memory_order_relaxed); + implicitProducers.store(other.implicitProducers.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.implicitProducers.store(nullptr, std::memory_order_relaxed); +#endif + + other.initialBlockPoolIndex.store(0, std::memory_order_relaxed); + other.initialBlockPoolSize = 0; + other.initialBlockPool = nullptr; + + reown_producers(); + } + + inline ConcurrentQueue& operator=(ConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT + { + return swap_internal(other); + } + + // Swaps this queue's state with the other's. Not thread-safe. + // Swapping two queues does not invalidate their tokens, however + // the tokens that were created for one queue must be used with + // only the swapped queue (i.e. the tokens are tied to the + // queue's movable state, not the object itself). + inline void swap(ConcurrentQueue& other) MOODYCAMEL_NOEXCEPT + { + swap_internal(other); + } + +private: + ConcurrentQueue& swap_internal(ConcurrentQueue& other) + { + if (this == &other) { + return *this; + } + + details::swap_relaxed(producerListTail, other.producerListTail); + details::swap_relaxed(producerCount, other.producerCount); + details::swap_relaxed(initialBlockPoolIndex, other.initialBlockPoolIndex); + std::swap(initialBlockPool, other.initialBlockPool); + std::swap(initialBlockPoolSize, other.initialBlockPoolSize); + freeList.swap(other.freeList); + details::swap_relaxed(nextExplicitConsumerId, other.nextExplicitConsumerId); + details::swap_relaxed(globalExplicitConsumerOffset, other.globalExplicitConsumerOffset); + + swap_implicit_producer_hashes(other); + + reown_producers(); + other.reown_producers(); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + details::swap_relaxed(explicitProducers, other.explicitProducers); + details::swap_relaxed(implicitProducers, other.implicitProducers); +#endif + + return *this; + } + +public: + // Enqueues a single item (by copying it). + // Allocates memory if required. Only fails if memory allocation fails (or implicit + // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, + // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(T const& item) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue(item); + } + + // Enqueues a single item (by moving it, if possible). + // Allocates memory if required. Only fails if memory allocation fails (or implicit + // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0, + // or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(T&& item) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue(std::move(item)); + } + + // Enqueues a single item (by copying it) using an explicit producer token. + // Allocates memory if required. Only fails if memory allocation fails (or + // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(producer_token_t const& token, T const& item) + { + return inner_enqueue(token, item); + } + + // Enqueues a single item (by moving it, if possible) using an explicit producer token. + // Allocates memory if required. Only fails if memory allocation fails (or + // Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Thread-safe. + inline bool enqueue(producer_token_t const& token, T&& item) + { + return inner_enqueue(token, std::move(item)); + } + + // Enqueues several items. + // Allocates memory if required. Only fails if memory allocation fails (or + // implicit production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE + // is 0, or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Note: Use std::make_move_iterator if the elements should be moved instead of copied. + // Thread-safe. + template + bool enqueue_bulk(It itemFirst, size_t count) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue_bulk(itemFirst, count); + } + + // Enqueues several items using an explicit producer token. + // Allocates memory if required. Only fails if memory allocation fails + // (or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed). + // Note: Use std::make_move_iterator if the elements should be moved + // instead of copied. + // Thread-safe. + template + bool enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) + { + return inner_enqueue_bulk(token, itemFirst, count); + } + + // Enqueues a single item (by copying it). + // Does not allocate memory. Fails if not enough room to enqueue (or implicit + // production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE + // is 0). + // Note: If using only try_enqueue/try_enqueue_bulk with pre-allocated blocks, configure + // Traits::IMPLICIT_INITIAL_INDEX_SIZE appropriately to ensure the index has sufficient + // capacity for the number of blocks each producer may need. + // Thread-safe. + inline bool try_enqueue(T const& item) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue(item); + } + + // Enqueues a single item (by moving it, if possible). + // Does not allocate memory (except for one-time implicit producer). + // Fails if not enough room to enqueue (or implicit production is + // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). + // Note: If using only try_enqueue/try_enqueue_bulk with pre-allocated blocks, configure + // Traits::IMPLICIT_INITIAL_INDEX_SIZE appropriately to ensure the index has sufficient + // capacity for the number of blocks each producer may need. + // Thread-safe. + inline bool try_enqueue(T&& item) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue(std::move(item)); + } + + // Enqueues a single item (by copying it) using an explicit producer token. + // Does not allocate memory. Fails if not enough room to enqueue. + // Thread-safe. + inline bool try_enqueue(producer_token_t const& token, T const& item) + { + return inner_enqueue(token, item); + } + + // Enqueues a single item (by moving it, if possible) using an explicit producer token. + // Does not allocate memory. Fails if not enough room to enqueue. + // Thread-safe. + inline bool try_enqueue(producer_token_t const& token, T&& item) + { + return inner_enqueue(token, std::move(item)); + } + + // Enqueues several items. + // Does not allocate memory (except for one-time implicit producer). + // Fails if not enough room to enqueue (or implicit production is + // disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0). + // Note: If using only try_enqueue/try_enqueue_bulk with pre-allocated blocks, configure + // Traits::IMPLICIT_INITIAL_INDEX_SIZE appropriately to ensure the index has sufficient + // capacity for the number of blocks each producer may need. + // Note: Use std::make_move_iterator if the elements should be moved + // instead of copied. + // Thread-safe. + template + bool try_enqueue_bulk(It itemFirst, size_t count) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) return false; + else return inner_enqueue_bulk(itemFirst, count); + } + + // Enqueues several items using an explicit producer token. + // Does not allocate memory. Fails if not enough room to enqueue. + // Note: Use std::make_move_iterator if the elements should be moved + // instead of copied. + // Thread-safe. + template + bool try_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) + { + return inner_enqueue_bulk(token, itemFirst, count); + } + + + + // Attempts to dequeue from the queue. + // Returns false if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + bool try_dequeue(U& item) + { + // Instead of simply trying each producer in turn (which could cause needless contention on the first + // producer), we score them heuristically. + size_t nonEmptyCount = 0; + ProducerBase* best = nullptr; + size_t bestSize = 0; + for (auto ptr = producerListTail.load(std::memory_order_acquire); nonEmptyCount < 3 && ptr != nullptr; ptr = ptr->next_prod()) { + auto size = ptr->size_approx(); + if (size > 0) { + if (size > bestSize) { + bestSize = size; + best = ptr; + } + ++nonEmptyCount; + } + } + + // If there was at least one non-empty queue but it appears empty at the time + // we try to dequeue from it, we need to make sure every queue's been tried + if (nonEmptyCount > 0) { + if ((details::likely)(best->dequeue(item))) { + return true; + } + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + if (ptr != best && ptr->dequeue(item)) { + return true; + } + } + } + return false; + } + + // Attempts to dequeue from the queue. + // Returns false if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // This differs from the try_dequeue(item) method in that this one does + // not attempt to reduce contention by interleaving the order that producer + // streams are dequeued from. So, using this method can reduce overall throughput + // under contention, but will give more predictable results in single-threaded + // consumer scenarios. This is mostly only useful for internal unit tests. + // Never allocates. Thread-safe. + template + bool try_dequeue_non_interleaved(U& item) + { + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + if (ptr->dequeue(item)) { + return true; + } + } + return false; + } + + // Attempts to dequeue from the queue using an explicit consumer token. + // Returns false if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + bool try_dequeue(consumer_token_t& token, U& item) + { + // The idea is roughly as follows: + // Every 256 items from one producer, make everyone rotate (increase the global offset) -> this means the highest efficiency consumer dictates the rotation speed of everyone else, more or less + // If you see that the global offset has changed, you must reset your consumption counter and move to your designated place + // If there's no items where you're supposed to be, keep moving until you find a producer with some items + // If the global offset has not changed but you've run out of items to consume, move over from your current position until you find an producer with something in it + + if (token.desiredProducer == nullptr || token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) { + if (!update_current_producer_after_rotation(token)) { + return false; + } + } + + // If there was at least one non-empty queue but it appears empty at the time + // we try to dequeue from it, we need to make sure every queue's been tried + if (static_cast(token.currentProducer)->dequeue(item)) { + if (++token.itemsConsumedFromCurrent == EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) { + globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed); + } + return true; + } + + auto tail = producerListTail.load(std::memory_order_acquire); + auto ptr = static_cast(token.currentProducer)->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + while (ptr != static_cast(token.currentProducer)) { + if (ptr->dequeue(item)) { + token.currentProducer = ptr; + token.itemsConsumedFromCurrent = 1; + return true; + } + ptr = ptr->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + } + return false; + } + + // Attempts to dequeue several elements from the queue. + // Returns the number of items actually dequeued. + // Returns 0 if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + size_t try_dequeue_bulk(It itemFirst, size_t max) + { + size_t count = 0; + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + count += ptr->dequeue_bulk(itemFirst, max - count); + if (count == max) { + break; + } + } + return count; + } + + // Attempts to dequeue several elements from the queue using an explicit consumer token. + // Returns the number of items actually dequeued. + // Returns 0 if all producer streams appeared empty at the time they + // were checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + size_t try_dequeue_bulk(consumer_token_t& token, It itemFirst, size_t max) + { + if (token.desiredProducer == nullptr || token.lastKnownGlobalOffset != globalExplicitConsumerOffset.load(std::memory_order_relaxed)) { + if (!update_current_producer_after_rotation(token)) { + return 0; + } + } + + size_t count = static_cast(token.currentProducer)->dequeue_bulk(itemFirst, max); + if (count == max) { + if ((token.itemsConsumedFromCurrent += static_cast(max)) >= EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE) { + globalExplicitConsumerOffset.fetch_add(1, std::memory_order_relaxed); + } + return max; + } + token.itemsConsumedFromCurrent += static_cast(count); + max -= count; + + auto tail = producerListTail.load(std::memory_order_acquire); + auto ptr = static_cast(token.currentProducer)->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + while (ptr != static_cast(token.currentProducer)) { + auto dequeued = ptr->dequeue_bulk(itemFirst, max); + count += dequeued; + if (dequeued != 0) { + token.currentProducer = ptr; + token.itemsConsumedFromCurrent = static_cast(dequeued); + } + if (dequeued == max) { + break; + } + max -= dequeued; + ptr = ptr->next_prod(); + if (ptr == nullptr) { + ptr = tail; + } + } + return count; + } + + + + // Attempts to dequeue from a specific producer's inner queue. + // If you happen to know which producer you want to dequeue from, this + // is significantly faster than using the general-case try_dequeue methods. + // Returns false if the producer's queue appeared empty at the time it + // was checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + inline bool try_dequeue_from_producer(producer_token_t const& producer, U& item) + { + return static_cast(producer.producer)->dequeue(item); + } + + // Attempts to dequeue several elements from a specific producer's inner queue. + // Returns the number of items actually dequeued. + // If you happen to know which producer you want to dequeue from, this + // is significantly faster than using the general-case try_dequeue methods. + // Returns 0 if the producer's queue appeared empty at the time it + // was checked (so, the queue is likely but not guaranteed to be empty). + // Never allocates. Thread-safe. + template + inline size_t try_dequeue_bulk_from_producer(producer_token_t const& producer, It itemFirst, size_t max) + { + return static_cast(producer.producer)->dequeue_bulk(itemFirst, max); + } + + + // Returns an estimate of the total number of elements currently in the queue. This + // estimate is only accurate if the queue has completely stabilized before it is called + // (i.e. all enqueue and dequeue operations have completed and their memory effects are + // visible on the calling thread, and no further operations start while this method is + // being called). + // Thread-safe. + size_t size_approx() const + { + size_t size = 0; + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + size += ptr->size_approx(); + } + return size; + } + + + // Returns true if the underlying atomic variables used by + // the queue are lock-free (they should be on most platforms). + // Thread-safe. + static constexpr bool is_lock_free() + { + return + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::value == 2 && + details::static_is_lock_free::thread_id_numeric_size_t>::value == 2; + } + + +private: + friend struct ProducerToken; + friend struct ConsumerToken; + struct ExplicitProducer; + friend struct ExplicitProducer; + struct ImplicitProducer; + friend struct ImplicitProducer; + friend class ConcurrentQueueTests; + + enum AllocationMode { CanAlloc, CannotAlloc }; + + + /////////////////////////////// + // Queue methods + /////////////////////////////// + + template + inline bool inner_enqueue(producer_token_t const& token, U&& element) + { + return static_cast(token.producer)->ConcurrentQueue::ExplicitProducer::template enqueue(std::forward(element)); + } + + template + inline bool inner_enqueue(U&& element) + { + auto producer = get_or_add_implicit_producer(); + return producer == nullptr ? false : producer->ConcurrentQueue::ImplicitProducer::template enqueue(std::forward(element)); + } + + template + inline bool inner_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count) + { + return static_cast(token.producer)->ConcurrentQueue::ExplicitProducer::template enqueue_bulk(itemFirst, count); + } + + template + inline bool inner_enqueue_bulk(It itemFirst, size_t count) + { + auto producer = get_or_add_implicit_producer(); + return producer == nullptr ? false : producer->ConcurrentQueue::ImplicitProducer::template enqueue_bulk(itemFirst, count); + } + + inline bool update_current_producer_after_rotation(consumer_token_t& token) + { + // Ah, there's been a rotation, figure out where we should be! + auto tail = producerListTail.load(std::memory_order_acquire); + if (token.desiredProducer == nullptr && tail == nullptr) { + return false; + } + auto prodCount = producerCount.load(std::memory_order_relaxed); + auto globalOffset = globalExplicitConsumerOffset.load(std::memory_order_relaxed); + if ((details::unlikely)(token.desiredProducer == nullptr)) { + // Aha, first time we're dequeueing anything. + // Figure out our local position + // Note: offset is from start, not end, but we're traversing from end -- subtract from count first + std::uint32_t offset = prodCount - 1 - (token.initialOffset % prodCount); + token.desiredProducer = tail; + for (std::uint32_t i = 0; i != offset; ++i) { + token.desiredProducer = static_cast(token.desiredProducer)->next_prod(); + if (token.desiredProducer == nullptr) { + token.desiredProducer = tail; + } + } + } + + std::uint32_t delta = globalOffset - token.lastKnownGlobalOffset; + if (delta >= prodCount) { + delta = delta % prodCount; + } + for (std::uint32_t i = 0; i != delta; ++i) { + token.desiredProducer = static_cast(token.desiredProducer)->next_prod(); + if (token.desiredProducer == nullptr) { + token.desiredProducer = tail; + } + } + + token.lastKnownGlobalOffset = globalOffset; + token.currentProducer = token.desiredProducer; + token.itemsConsumedFromCurrent = 0; + return true; + } + + + /////////////////////////// + // Free list + /////////////////////////// + + template + struct FreeListNode + { + FreeListNode() : freeListRefs(0), freeListNext(nullptr) { } + + std::atomic freeListRefs; + std::atomic freeListNext; + }; + + // A simple CAS-based lock-free free list. Not the fastest thing in the world under heavy contention, but + // simple and correct (assuming nodes are never freed until after the free list is destroyed), and fairly + // speedy under low contention. + template // N must inherit FreeListNode or have the same fields (and initialization of them) + struct FreeList + { + FreeList() : freeListHead(nullptr) { } + FreeList(FreeList&& other) : freeListHead(other.freeListHead.load(std::memory_order_relaxed)) { other.freeListHead.store(nullptr, std::memory_order_relaxed); } + void swap(FreeList& other) { details::swap_relaxed(freeListHead, other.freeListHead); } + + FreeList(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; + FreeList& operator=(FreeList const&) MOODYCAMEL_DELETE_FUNCTION; + + inline void add(N* node) + { +#ifdef MCDBGQ_NOLOCKFREE_FREELIST + debug::DebugLock lock(mutex); +#endif + // We know that the should-be-on-freelist bit is 0 at this point, so it's safe to + // set it using a fetch_add + if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST, std::memory_order_acq_rel) == 0) { + // Oh look! We were the last ones referencing this node, and we know + // we want to add it to the free list, so let's do it! + add_knowing_refcount_is_zero(node); + } + } + + inline N* try_get() + { +#ifdef MCDBGQ_NOLOCKFREE_FREELIST + debug::DebugLock lock(mutex); +#endif + auto head = freeListHead.load(std::memory_order_acquire); + while (head != nullptr) { + auto prevHead = head; + auto refs = head->freeListRefs.load(std::memory_order_relaxed); + if ((refs & REFS_MASK) == 0 || !head->freeListRefs.compare_exchange_strong(refs, refs + 1, std::memory_order_acquire)) { + head = freeListHead.load(std::memory_order_acquire); + continue; + } + + // Good, reference count has been incremented (it wasn't at zero), which means we can read the + // next and not worry about it changing between now and the time we do the CAS + auto next = head->freeListNext.load(std::memory_order_relaxed); + if (freeListHead.compare_exchange_strong(head, next, std::memory_order_acquire, std::memory_order_relaxed)) { + // Yay, got the node. This means it was on the list, which means shouldBeOnFreeList must be false no + // matter the refcount (because nobody else knows it's been taken off yet, it can't have been put back on). + assert((head->freeListRefs.load(std::memory_order_relaxed) & SHOULD_BE_ON_FREELIST) == 0); + + // Decrease refcount twice, once for our ref, and once for the list's ref + head->freeListRefs.fetch_sub(2, std::memory_order_release); + return head; + } + + // OK, the head must have changed on us, but we still need to decrease the refcount we increased. + // Note that we don't need to release any memory effects, but we do need to ensure that the reference + // count decrement happens-after the CAS on the head. + refs = prevHead->freeListRefs.fetch_sub(1, std::memory_order_acq_rel); + if (refs == SHOULD_BE_ON_FREELIST + 1) { + add_knowing_refcount_is_zero(prevHead); + } + } + + return nullptr; + } + + // Useful for traversing the list when there's no contention (e.g. to destroy remaining nodes) + N* head_unsafe() const { return freeListHead.load(std::memory_order_relaxed); } + + private: + inline void add_knowing_refcount_is_zero(N* node) + { + // Since the refcount is zero, and nobody can increase it once it's zero (except us, and we run + // only one copy of this method per node at a time, i.e. the single thread case), then we know + // we can safely change the next pointer of the node; however, once the refcount is back above + // zero, then other threads could increase it (happens under heavy contention, when the refcount + // goes to zero in between a load and a refcount increment of a node in try_get, then back up to + // something non-zero, then the refcount increment is done by the other thread) -- so, if the CAS + // to add the node to the actual list fails, decrease the refcount and leave the add operation to + // the next thread who puts the refcount back at zero (which could be us, hence the loop). + auto head = freeListHead.load(std::memory_order_relaxed); + while (true) { + node->freeListNext.store(head, std::memory_order_relaxed); + node->freeListRefs.store(1, std::memory_order_release); + if (!freeListHead.compare_exchange_strong(head, node, std::memory_order_release, std::memory_order_relaxed)) { + // Hmm, the add failed, but we can only try again when the refcount goes back to zero + if (node->freeListRefs.fetch_add(SHOULD_BE_ON_FREELIST - 1, std::memory_order_acq_rel) == 1) { + continue; + } + } + return; + } + } + + private: + // Implemented like a stack, but where node order doesn't matter (nodes are inserted out of order under contention) + std::atomic freeListHead; + + static const std::uint32_t REFS_MASK = 0x7FFFFFFF; + static const std::uint32_t SHOULD_BE_ON_FREELIST = 0x80000000; + +#ifdef MCDBGQ_NOLOCKFREE_FREELIST + debug::DebugMutex mutex; +#endif + }; + + + /////////////////////////// + // Block + /////////////////////////// + + enum InnerQueueContext { implicit_context = 0, explicit_context = 1 }; + + struct Block + { + Block() + : next(nullptr), elementsCompletelyDequeued(0), freeListRefs(0), freeListNext(nullptr), dynamicallyAllocated(true) + { +#ifdef MCDBGQ_TRACKMEM + owner = nullptr; +#endif + } + + template + inline bool is_empty() const + { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Check flags + for (size_t i = 0; i < BLOCK_SIZE; ++i) { + if (!emptyFlags[i].load(std::memory_order_relaxed)) { + return false; + } + } + + // Aha, empty; make sure we have all other memory effects that happened before the empty flags were set + std::atomic_thread_fence(std::memory_order_acquire); + return true; + } + else { + // Check counter + if (elementsCompletelyDequeued.load(std::memory_order_relaxed) == BLOCK_SIZE) { + std::atomic_thread_fence(std::memory_order_acquire); + return true; + } + assert(elementsCompletelyDequeued.load(std::memory_order_relaxed) <= BLOCK_SIZE); + return false; + } + } + + // Returns true if the block is now empty (does not apply in explicit context) + template + inline bool set_empty(MOODYCAMEL_MAYBE_UNUSED index_t i) + { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Set flag + assert(!emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].load(std::memory_order_relaxed)); + emptyFlags[BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1))].store(true, std::memory_order_release); + return false; + } + else { + // Increment counter + auto prevVal = elementsCompletelyDequeued.fetch_add(1, std::memory_order_acq_rel); + assert(prevVal < BLOCK_SIZE); + return prevVal == BLOCK_SIZE - 1; + } + } + + // Sets multiple contiguous item statuses to 'empty' (assumes no wrapping and count > 0). + // Returns true if the block is now empty (does not apply in explicit context). + template + inline bool set_many_empty(MOODYCAMEL_MAYBE_UNUSED index_t i, size_t count) + { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Set flags + std::atomic_thread_fence(std::memory_order_release); + i = BLOCK_SIZE - 1 - static_cast(i & static_cast(BLOCK_SIZE - 1)) - count + 1; + for (size_t j = 0; j != count; ++j) { + assert(!emptyFlags[i + j].load(std::memory_order_relaxed)); + emptyFlags[i + j].store(true, std::memory_order_relaxed); + } + return false; + } + else { + // Increment counter + auto prevVal = elementsCompletelyDequeued.fetch_add(count, std::memory_order_acq_rel); + assert(prevVal + count <= BLOCK_SIZE); + return prevVal + count == BLOCK_SIZE; + } + } + + template + inline void set_all_empty() + { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Set all flags + for (size_t i = 0; i != BLOCK_SIZE; ++i) { + emptyFlags[i].store(true, std::memory_order_relaxed); + } + } + else { + // Reset counter + elementsCompletelyDequeued.store(BLOCK_SIZE, std::memory_order_relaxed); + } + } + + template + inline void reset_empty() + { + MOODYCAMEL_CONSTEXPR_IF (context == explicit_context && BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD) { + // Reset flags + for (size_t i = 0; i != BLOCK_SIZE; ++i) { + emptyFlags[i].store(false, std::memory_order_relaxed); + } + } + else { + // Reset counter + elementsCompletelyDequeued.store(0, std::memory_order_relaxed); + } + } + + inline T* operator[](index_t idx) MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } + inline T const* operator[](index_t idx) const MOODYCAMEL_NOEXCEPT { return static_cast(static_cast(elements)) + static_cast(idx & static_cast(BLOCK_SIZE - 1)); } + + private: + static_assert(std::alignment_of::value <= sizeof(T), "The queue does not support types with an alignment greater than their size at this time"); + MOODYCAMEL_ALIGNED_TYPE_LIKE(char[sizeof(T) * BLOCK_SIZE], T) elements; + public: + Block* next; + std::atomic elementsCompletelyDequeued; + std::atomic emptyFlags[BLOCK_SIZE <= EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD ? BLOCK_SIZE : 1]; + public: + std::atomic freeListRefs; + std::atomic freeListNext; + bool dynamicallyAllocated; // Perhaps a better name for this would be 'isNotPartOfInitialBlockPool' + +#ifdef MCDBGQ_TRACKMEM + void* owner; +#endif + }; + static_assert(std::alignment_of::value >= std::alignment_of::value, "Internal error: Blocks must be at least as aligned as the type they are wrapping"); + + +#ifdef MCDBGQ_TRACKMEM +public: + struct MemStats; +private: +#endif + + /////////////////////////// + // Producer base + /////////////////////////// + + struct ProducerBase : public details::ConcurrentQueueProducerTypelessBase + { + ProducerBase(ConcurrentQueue* parent_, bool isExplicit_) : + tailIndex(0), + headIndex(0), + dequeueOptimisticCount(0), + dequeueOvercommit(0), + tailBlock(nullptr), + isExplicit(isExplicit_), + parent(parent_) + { + } + + virtual ~ProducerBase() { } + + template + inline bool dequeue(U& element) + { + if (isExplicit) { + return static_cast(this)->dequeue(element); + } + else { + return static_cast(this)->dequeue(element); + } + } + + template + inline size_t dequeue_bulk(It& itemFirst, size_t max) + { + if (isExplicit) { + return static_cast(this)->dequeue_bulk(itemFirst, max); + } + else { + return static_cast(this)->dequeue_bulk(itemFirst, max); + } + } + + inline ProducerBase* next_prod() const { return static_cast(next); } + + inline size_t size_approx() const + { + auto tail = tailIndex.load(std::memory_order_relaxed); + auto head = headIndex.load(std::memory_order_relaxed); + return details::circular_less_than(head, tail) ? static_cast(tail - head) : 0; + } + + inline index_t getTail() const { return tailIndex.load(std::memory_order_relaxed); } + protected: + std::atomic tailIndex; // Where to enqueue to next + std::atomic headIndex; // Where to dequeue from next + + std::atomic dequeueOptimisticCount; + std::atomic dequeueOvercommit; + + Block* tailBlock; + + public: + bool isExplicit; + ConcurrentQueue* parent; + + protected: +#ifdef MCDBGQ_TRACKMEM + friend struct MemStats; +#endif + }; + + + /////////////////////////// + // Explicit queue + /////////////////////////// + + struct ExplicitProducer : public ProducerBase + { + explicit ExplicitProducer(ConcurrentQueue* parent_) : + ProducerBase(parent_, true), + blockIndex(nullptr), + pr_blockIndexSlotsUsed(0), + pr_blockIndexSize(EXPLICIT_INITIAL_INDEX_SIZE >> 1), + pr_blockIndexFront(0), + pr_blockIndexEntries(nullptr), + pr_blockIndexRaw(nullptr) + { + size_t poolBasedIndexSize = details::ceil_to_pow_2(parent_->initialBlockPoolSize) >> 1; + if (poolBasedIndexSize > pr_blockIndexSize) { + pr_blockIndexSize = poolBasedIndexSize; + } + + new_block_index(0); // This creates an index with double the number of current entries, i.e. EXPLICIT_INITIAL_INDEX_SIZE + } + + ~ExplicitProducer() + { + // Destruct any elements not yet dequeued. + // Since we're in the destructor, we can assume all elements + // are either completely dequeued or completely not (no halfways). + if (this->tailBlock != nullptr) { // Note this means there must be a block index too + // First find the block that's partially dequeued, if any + Block* halfDequeuedBlock = nullptr; + if ((this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) != 0) { + // The head's not on a block boundary, meaning a block somewhere is partially dequeued + // (or the head block is the tail block and was fully dequeued, but the head/tail are still not on a boundary) + size_t i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & (pr_blockIndexSize - 1); + while (details::circular_less_than(pr_blockIndexEntries[i].base + BLOCK_SIZE, this->headIndex.load(std::memory_order_relaxed))) { + i = (i + 1) & (pr_blockIndexSize - 1); + } + assert(details::circular_less_than(pr_blockIndexEntries[i].base, this->headIndex.load(std::memory_order_relaxed))); + halfDequeuedBlock = pr_blockIndexEntries[i].block; + } + + // Start at the head block (note the first line in the loop gives us the head from the tail on the first iteration) + auto block = this->tailBlock; + do { + block = block->next; + if (block->ConcurrentQueue::Block::template is_empty()) { + continue; + } + + size_t i = 0; // Offset into block + if (block == halfDequeuedBlock) { + i = static_cast(this->headIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)); + } + + // Walk through all the items in the block; if this is the tail block, we need to stop when we reach the tail index + auto lastValidIndex = (this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)) == 0 ? BLOCK_SIZE : static_cast(this->tailIndex.load(std::memory_order_relaxed) & static_cast(BLOCK_SIZE - 1)); + while (i != BLOCK_SIZE && (block != this->tailBlock || i != lastValidIndex)) { + (*block)[i++]->~T(); + } + } while (block != this->tailBlock); + } + + // Destroy all blocks that we own + if (this->tailBlock != nullptr) { + auto block = this->tailBlock; + do { + auto nextBlock = block->next; + this->parent->add_block_to_free_list(block); + block = nextBlock; + } while (block != this->tailBlock); + } + + // Destroy the block indices + auto header = static_cast(pr_blockIndexRaw); + while (header != nullptr) { + auto prev = static_cast(header->prev); + header->~BlockIndexHeader(); + (Traits::free)(header); + header = prev; + } + } + + template + inline bool enqueue(U&& element) + { + index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); + index_t newTailIndex = 1 + currentTailIndex; + if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + // We reached the end of a block, start a new one + auto startBlock = this->tailBlock; + auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; + if (this->tailBlock != nullptr && this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { + // We can re-use the block ahead of us, it's empty! + this->tailBlock = this->tailBlock->next; + this->tailBlock->ConcurrentQueue::Block::template reset_empty(); + + // We'll put the block on the block index (guaranteed to be room since we're conceptually removing the + // last block from it first -- except instead of removing then adding, we can just overwrite). + // Note that there must be a valid block index here, since even if allocation failed in the ctor, + // it would have been re-attempted when adding the first block to the queue; since there is such + // a block, a block index must have been successfully allocated. + } + else { + // Whatever head value we see here is >= the last value we saw here (relatively), + // and <= its current value. Since we have the most recent tail, the head must be + // <= to it. + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) + || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { + // We can't enqueue in another block because there's not enough leeway -- the + // tail could surpass the head by the time the block fills up! (Or we'll exceed + // the size limit, if the second part of the condition was true.) + return false; + } + // We're going to need a new block; check that the block index has room + if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize) { + // Hmm, the circular block index is already full -- we'll need + // to allocate a new index. Note pr_blockIndexRaw can only be nullptr if + // the initial allocation failed in the constructor. + + MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) { + return false; + } + else if (!new_block_index(pr_blockIndexSlotsUsed)) { + return false; + } + } + + // Insert a new block in the circular linked list + auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); + if (newBlock == nullptr) { + return false; + } +#ifdef MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template reset_empty(); + if (this->tailBlock == nullptr) { + newBlock->next = newBlock; + } + else { + newBlock->next = this->tailBlock->next; + this->tailBlock->next = newBlock; + } + this->tailBlock = newBlock; + ++pr_blockIndexSlotsUsed; + } + + MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { + // The constructor may throw. We want the element not to appear in the queue in + // that case (without corrupting the queue): + MOODYCAMEL_TRY { + new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); + } + MOODYCAMEL_CATCH (...) { + // Revert change to the current block, but leave the new block available + // for next time + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? this->tailBlock : startBlock; + MOODYCAMEL_RETHROW; + } + } + else { + (void)startBlock; + (void)originalBlockIndexSlotsUsed; + } + + // Add block to block index + auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; + entry.base = currentTailIndex; + entry.block = this->tailBlock; + blockIndex.load(std::memory_order_relaxed)->front.store(pr_blockIndexFront, std::memory_order_release); + pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); + + MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + } + + // Enqueue + new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); + + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + + template + bool dequeue(U& element) + { + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + if (details::circular_less_than(this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) { + // Might be something to dequeue, let's give it a try + + // Note that this if is purely for performance purposes in the common case when the queue is + // empty and the values are eventually consistent -- we may enter here spuriously. + + // Note that whatever the values of overcommit and tail are, they are not going to change (unless we + // change them) and must be the same value at this point (inside the if) as when the if condition was + // evaluated. + + // We insert an acquire fence here to synchronize-with the release upon incrementing dequeueOvercommit below. + // This ensures that whatever the value we got loaded into overcommit, the load of dequeueOptisticCount in + // the fetch_add below will result in a value at least as recent as that (and therefore at least as large). + // Note that I believe a compiler (signal) fence here would be sufficient due to the nature of fetch_add (all + // read-modify-write operations are guaranteed to work on the latest value in the modification order), but + // unfortunately that can't be shown to be correct using only the C++11 standard. + // See http://stackoverflow.com/questions/18223161/what-are-the-c11-memory-ordering-guarantees-in-this-corner-case + std::atomic_thread_fence(std::memory_order_acquire); + + // Increment optimistic counter, then check if it went over the boundary + auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed); + + // Note that since dequeueOvercommit must be <= dequeueOptimisticCount (because dequeueOvercommit is only ever + // incremented after dequeueOptimisticCount -- this is enforced in the `else` block below), and since we now + // have a version of dequeueOptimisticCount that is at least as recent as overcommit (due to the release upon + // incrementing dequeueOvercommit and the acquire above that synchronizes with it), overcommit <= myDequeueCount. + // However, we can't assert this since both dequeueOptimisticCount and dequeueOvercommit may (independently) + // overflow; in such a case, though, the logic still holds since the difference between the two is maintained. + + // Note that we reload tail here in case it changed; it will be the same value as before or greater, since + // this load is sequenced after (happens after) the earlier load above. This is supported by read-read + // coherency (as defined in the standard), explained here: http://en.cppreference.com/w/cpp/atomic/memory_order + tail = this->tailIndex.load(std::memory_order_acquire); + if ((details::likely)(details::circular_less_than(myDequeueCount - overcommit, tail))) { + // Guaranteed to be at least one element to dequeue! + + // Get the index. Note that since there's guaranteed to be at least one element, this + // will never exceed tail. We need to do an acquire-release fence here since it's possible + // that whatever condition got us to this point was for an earlier enqueued element (that + // we already see the memory effects for), but that by the time we increment somebody else + // has incremented it, and we need to see the memory effects for *that* element, which is + // in such a case is necessarily visible on the thread that incremented it in the first + // place with the more current condition (they must have acquired a tail that is at least + // as recent). + auto index = this->headIndex.fetch_add(1, std::memory_order_acq_rel); + + + // Determine which block the element is in + + auto localBlockIndex = blockIndex.load(std::memory_order_acquire); + auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); + + // We need to be careful here about subtracting and dividing because of index wrap-around. + // When an index wraps, we need to preserve the sign of the offset when dividing it by the + // block size (in order to get a correct signed block count offset in all cases): + auto headBase = localBlockIndex->entries[localBlockIndexHead].base; + auto blockBaseIndex = index & ~static_cast(BLOCK_SIZE - 1); + auto offset = static_cast(static_cast::type>(blockBaseIndex - headBase) / static_cast::type>(BLOCK_SIZE)); + auto block = localBlockIndex->entries[(localBlockIndexHead + offset) & (localBlockIndex->size - 1)].block; + + // Dequeue + auto& el = *((*block)[index]); + if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, element = std::move(el))) { + // Make sure the element is still fully dequeued and destroyed even if the assignment + // throws + struct Guard { + Block* block; + index_t index; + + ~Guard() + { + (*block)[index]->~T(); + block->ConcurrentQueue::Block::template set_empty(index); + } + } guard = { block, index }; + + element = std::move(el); // NOLINT + } + else { + element = std::move(el); // NOLINT + el.~T(); // NOLINT + block->ConcurrentQueue::Block::template set_empty(index); + } + + return true; + } + else { + // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent + this->dequeueOvercommit.fetch_add(1, std::memory_order_release); // Release so that the fetch_add on dequeueOptimisticCount is guaranteed to happen before this write + } + } + + return false; + } + + template + bool MOODYCAMEL_NO_TSAN enqueue_bulk(It itemFirst, size_t count) + { + // First, we need to make sure we have enough room to enqueue all of the elements; + // this means pre-allocating blocks and putting them in the block index (but only if + // all the allocations succeeded). + index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed); + auto startBlock = this->tailBlock; + auto originalBlockIndexFront = pr_blockIndexFront; + auto originalBlockIndexSlotsUsed = pr_blockIndexSlotsUsed; + + Block* firstAllocatedBlock = nullptr; + + // Figure out how many blocks we'll need to allocate, and do so + size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); + index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + if (blockBaseDiff > 0) { + // Allocate as many blocks as possible from ahead + while (blockBaseDiff > 0 && this->tailBlock != nullptr && this->tailBlock->next != firstAllocatedBlock && this->tailBlock->next->ConcurrentQueue::Block::template is_empty()) { + blockBaseDiff -= static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE); + + this->tailBlock = this->tailBlock->next; + firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; + + auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; + entry.base = currentTailIndex; + entry.block = this->tailBlock; + pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); + } + + // Now allocate as many blocks as necessary from the block pool + while (blockBaseDiff > 0) { + blockBaseDiff -= static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE); + + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); + if (pr_blockIndexRaw == nullptr || pr_blockIndexSlotsUsed == pr_blockIndexSize || full) { + MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) { + // Failed to allocate, undo changes (but keep injected blocks) + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + return false; + } + else if (full || !new_block_index(originalBlockIndexSlotsUsed)) { + // Failed to allocate, undo changes (but keep injected blocks) + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + return false; + } + + // pr_blockIndexFront is updated inside new_block_index, so we need to + // update our fallback value too (since we keep the new index even if we + // later fail) + originalBlockIndexFront = originalBlockIndexSlotsUsed; + } + + // Insert a new block in the circular linked list + auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); + if (newBlock == nullptr) { + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + return false; + } + +#ifdef MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template set_all_empty(); + if (this->tailBlock == nullptr) { + newBlock->next = newBlock; + } + else { + newBlock->next = this->tailBlock->next; + this->tailBlock->next = newBlock; + } + this->tailBlock = newBlock; + firstAllocatedBlock = firstAllocatedBlock == nullptr ? this->tailBlock : firstAllocatedBlock; + + ++pr_blockIndexSlotsUsed; + + auto& entry = blockIndex.load(std::memory_order_relaxed)->entries[pr_blockIndexFront]; + entry.base = currentTailIndex; + entry.block = this->tailBlock; + pr_blockIndexFront = (pr_blockIndexFront + 1) & (pr_blockIndexSize - 1); + } + + // Excellent, all allocations succeeded. Reset each block's emptiness before we fill them up, and + // publish the new block index front + auto block = firstAllocatedBlock; + while (true) { + block->ConcurrentQueue::Block::template reset_empty(); + if (block == this->tailBlock) { + break; + } + block = block->next; + } + + MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { + blockIndex.load(std::memory_order_relaxed)->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release); + } + } + + // Enqueue, one block at a time + index_t newTailIndex = startTailIndex + static_cast(count); + currentTailIndex = startTailIndex; + auto endBlock = this->tailBlock; + this->tailBlock = startBlock; + assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { + this->tailBlock = firstAllocatedBlock; + } + while (true) { + index_t stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(newTailIndex, stopIndex)) { + stopIndex = newTailIndex; + } + MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { + while (currentTailIndex != stopIndex) { + new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++); + } + } + else { + MOODYCAMEL_TRY { + while (currentTailIndex != stopIndex) { + // Must use copy constructor even if move constructor is available + // because we may have to revert if there's an exception. + // Sorry about the horrible templated next line, but it was the only way + // to disable moving *at compile time*, which is important because a type + // may only define a (noexcept) move constructor, and so calls to the + // cctor will not compile, even if they are in an if branch that will never + // be executed + new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if(nullptr)) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst)); + ++currentTailIndex; + ++itemFirst; + } + } + MOODYCAMEL_CATCH (...) { + // Oh dear, an exception's been thrown -- destroy the elements that + // were enqueued so far and revert the entire bulk operation (we'll keep + // any allocated blocks in our linked list for later, though). + auto constructedStopIndex = currentTailIndex; + auto lastBlockEnqueued = this->tailBlock; + + pr_blockIndexFront = originalBlockIndexFront; + pr_blockIndexSlotsUsed = originalBlockIndexSlotsUsed; + this->tailBlock = startBlock == nullptr ? firstAllocatedBlock : startBlock; + + if (!details::is_trivially_destructible::value) { + auto block = startBlock; + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + block = firstAllocatedBlock; + } + currentTailIndex = startTailIndex; + while (true) { + stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(constructedStopIndex, stopIndex)) { + stopIndex = constructedStopIndex; + } + while (currentTailIndex != stopIndex) { + (*block)[currentTailIndex++]->~T(); + } + if (block == lastBlockEnqueued) { + break; + } + block = block->next; + } + } + MOODYCAMEL_RETHROW; + } + } + + if (this->tailBlock == endBlock) { + assert(currentTailIndex == newTailIndex); + break; + } + this->tailBlock = this->tailBlock->next; + } + + MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { + if (firstAllocatedBlock != nullptr) + blockIndex.load(std::memory_order_relaxed)->front.store((pr_blockIndexFront - 1) & (pr_blockIndexSize - 1), std::memory_order_release); + } + + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + + template + size_t dequeue_bulk(It& itemFirst, size_t max) + { + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + auto desiredCount = static_cast(tail - (this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit)); + if (details::circular_less_than(0, desiredCount)) { + desiredCount = desiredCount < max ? desiredCount : max; + std::atomic_thread_fence(std::memory_order_acquire); + + auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed); + + tail = this->tailIndex.load(std::memory_order_acquire); + auto actualCount = static_cast(tail - (myDequeueCount - overcommit)); + if (details::circular_less_than(0, actualCount)) { + actualCount = desiredCount < actualCount ? desiredCount : actualCount; + if (actualCount < desiredCount) { + this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release); + } + + // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this + // will never exceed tail. + auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel); + + // Determine which block the first element is in + auto localBlockIndex = blockIndex.load(std::memory_order_acquire); + auto localBlockIndexHead = localBlockIndex->front.load(std::memory_order_acquire); + + auto headBase = localBlockIndex->entries[localBlockIndexHead].base; + auto firstBlockBaseIndex = firstIndex & ~static_cast(BLOCK_SIZE - 1); + auto offset = static_cast(static_cast::type>(firstBlockBaseIndex - headBase) / static_cast::type>(BLOCK_SIZE)); + auto indexIndex = (localBlockIndexHead + offset) & (localBlockIndex->size - 1); + + // Iterate the blocks and dequeue + auto index = firstIndex; + do { + auto firstIndexInBlock = index; + index_t endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + auto block = localBlockIndex->entries[indexIndex].block; + if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst++ = std::move(el); + el.~T(); + ++index; + } + } + else { + MOODYCAMEL_TRY { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst = std::move(el); + ++itemFirst; + el.~T(); + ++index; + } + } + MOODYCAMEL_CATCH (...) { + // It's too late to revert the dequeue, but we can make sure that all + // the dequeued objects are properly destroyed and the block index + // (and empty count) are properly updated before we propagate the exception + do { + block = localBlockIndex->entries[indexIndex].block; + while (index != endIndex) { + (*block)[index++]->~T(); + } + block->ConcurrentQueue::Block::template set_many_empty(firstIndexInBlock, static_cast(endIndex - firstIndexInBlock)); + indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); + + firstIndexInBlock = index; + endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + } while (index != firstIndex + actualCount); + + MOODYCAMEL_RETHROW; + } + } + block->ConcurrentQueue::Block::template set_many_empty(firstIndexInBlock, static_cast(endIndex - firstIndexInBlock)); + indexIndex = (indexIndex + 1) & (localBlockIndex->size - 1); + } while (index != firstIndex + actualCount); + + return actualCount; + } + else { + // Wasn't anything to dequeue after all; make the effective dequeue count eventually consistent + this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release); + } + } + + return 0; + } + + private: + struct BlockIndexEntry + { + index_t base; + Block* block; + }; + + struct BlockIndexHeader + { + size_t size; + std::atomic front; // Current slot (not next, like pr_blockIndexFront) + BlockIndexEntry* entries; + void* prev; + }; + + + bool new_block_index(size_t numberOfFilledSlotsToExpose) + { + auto prevBlockSizeMask = pr_blockIndexSize - 1; + + // Create the new block + pr_blockIndexSize <<= 1; + auto newRawPtr = static_cast((Traits::malloc)(sizeof(BlockIndexHeader) + std::alignment_of::value - 1 + sizeof(BlockIndexEntry) * pr_blockIndexSize)); + if (newRawPtr == nullptr) { + pr_blockIndexSize >>= 1; // Reset to allow graceful retry + return false; + } + + auto newBlockIndexEntries = reinterpret_cast(details::align_for(newRawPtr + sizeof(BlockIndexHeader))); + + // Copy in all the old indices, if any + size_t j = 0; + if (pr_blockIndexSlotsUsed != 0) { + auto i = (pr_blockIndexFront - pr_blockIndexSlotsUsed) & prevBlockSizeMask; + do { + newBlockIndexEntries[j++] = pr_blockIndexEntries[i]; + i = (i + 1) & prevBlockSizeMask; + } while (i != pr_blockIndexFront); + } + + // Update everything + auto header = new (newRawPtr) BlockIndexHeader; + header->size = pr_blockIndexSize; + header->front.store(numberOfFilledSlotsToExpose - 1, std::memory_order_relaxed); + header->entries = newBlockIndexEntries; + header->prev = pr_blockIndexRaw; // we link the new block to the old one so we can free it later + + pr_blockIndexFront = j; + pr_blockIndexEntries = newBlockIndexEntries; + pr_blockIndexRaw = newRawPtr; + blockIndex.store(header, std::memory_order_release); + + return true; + } + + private: + std::atomic blockIndex; + + // To be used by producer only -- consumer must use the ones in referenced by blockIndex + size_t pr_blockIndexSlotsUsed; + size_t pr_blockIndexSize; + size_t pr_blockIndexFront; // Next slot (not current) + BlockIndexEntry* pr_blockIndexEntries; + void* pr_blockIndexRaw; + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + public: + ExplicitProducer* nextExplicitProducer; + private: +#endif + +#ifdef MCDBGQ_TRACKMEM + friend struct MemStats; +#endif + }; + + + ////////////////////////////////// + // Implicit queue + ////////////////////////////////// + + struct ImplicitProducer : public ProducerBase + { + ImplicitProducer(ConcurrentQueue* parent_) : + ProducerBase(parent_, false), + nextBlockIndexCapacity(IMPLICIT_INITIAL_INDEX_SIZE), + blockIndex(nullptr) + { + new_block_index(); + } + + ~ImplicitProducer() + { + // Note that since we're in the destructor we can assume that all enqueue/dequeue operations + // completed already; this means that all undequeued elements are placed contiguously across + // contiguous blocks, and that only the first and last remaining blocks can be only partially + // empty (all other remaining blocks must be completely full). + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + // Unregister ourselves for thread termination notification + if (!this->inactive.load(std::memory_order_relaxed)) { + details::ThreadExitNotifier::unsubscribe(&threadExitListener); + } +#endif + + // Destroy all remaining elements! + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto index = this->headIndex.load(std::memory_order_relaxed); + Block* block = nullptr; + assert(index == tail || details::circular_less_than(index, tail)); + bool forceFreeLastBlock = index != tail; // If we enter the loop, then the last (tail) block will not be freed + while (index != tail) { + if ((index & static_cast(BLOCK_SIZE - 1)) == 0 || block == nullptr) { + if (block != nullptr) { + // Free the old block + this->parent->add_block_to_free_list(block); + } + + block = get_block_index_entry_for_index(index)->value.load(std::memory_order_relaxed); + } + + ((*block)[index])->~T(); + ++index; + } + // Even if the queue is empty, there's still one block that's not on the free list + // (unless the head index reached the end of it, in which case the tail will be poised + // to create a new block). + if (this->tailBlock != nullptr && (forceFreeLastBlock || (tail & static_cast(BLOCK_SIZE - 1)) != 0)) { + this->parent->add_block_to_free_list(this->tailBlock); + } + + // Destroy block index + auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); + if (localBlockIndex != nullptr) { + for (size_t i = 0; i != localBlockIndex->capacity; ++i) { + localBlockIndex->index[i]->~BlockIndexEntry(); + } + do { + auto prev = localBlockIndex->prev; + localBlockIndex->~BlockIndexHeader(); + (Traits::free)(localBlockIndex); + localBlockIndex = prev; + } while (localBlockIndex != nullptr); + } + } + + template + inline bool enqueue(U&& element) + { + index_t currentTailIndex = this->tailIndex.load(std::memory_order_relaxed); + index_t newTailIndex = 1 + currentTailIndex; + if ((currentTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + // We reached the end of a block, start a new one + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + if (!details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head))) { + return false; + } +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + // Find out where we'll be inserting this block in the block index + BlockIndexEntry* idxEntry; + if (!insert_block_index_entry(idxEntry, currentTailIndex)) { + return false; + } + + // Get ahold of a new block + auto newBlock = this->parent->ConcurrentQueue::template requisition_block(); + if (newBlock == nullptr) { + rewind_block_index_tail(); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + return false; + } +#ifdef MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template reset_empty(); + + MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { + // May throw, try to insert now before we publish the fact that we have this new block + MOODYCAMEL_TRY { + new ((*newBlock)[currentTailIndex]) T(std::forward(element)); + } + MOODYCAMEL_CATCH (...) { + rewind_block_index_tail(); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + this->parent->add_block_to_free_list(newBlock); + MOODYCAMEL_RETHROW; + } + } + + // Insert the new block into the index + idxEntry->value.store(newBlock, std::memory_order_relaxed); + + this->tailBlock = newBlock; + + MOODYCAMEL_CONSTEXPR_IF (!MOODYCAMEL_NOEXCEPT_CTOR(T, U, new (static_cast(nullptr)) T(std::forward(element)))) { + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + } + + // Enqueue + new ((*this->tailBlock)[currentTailIndex]) T(std::forward(element)); + + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } + + template + bool dequeue(U& element) + { + // See ExplicitProducer::dequeue for rationale and explanation + index_t tail = this->tailIndex.load(std::memory_order_relaxed); + index_t overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + if (details::circular_less_than(this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit, tail)) { + std::atomic_thread_fence(std::memory_order_acquire); + + index_t myDequeueCount = this->dequeueOptimisticCount.fetch_add(1, std::memory_order_relaxed); + tail = this->tailIndex.load(std::memory_order_acquire); + if ((details::likely)(details::circular_less_than(myDequeueCount - overcommit, tail))) { + index_t index = this->headIndex.fetch_add(1, std::memory_order_acq_rel); + + // Determine which block the element is in + auto entry = get_block_index_entry_for_index(index); + + // Dequeue + auto block = entry->value.load(std::memory_order_relaxed); + auto& el = *((*block)[index]); + + if (!MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, element = std::move(el))) { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + // Note: Acquiring the mutex with every dequeue instead of only when a block + // is released is very sub-optimal, but it is, after all, purely debug code. + debug::DebugLock lock(producer->mutex); +#endif + struct Guard { + Block* block; + index_t index; + BlockIndexEntry* entry; + ConcurrentQueue* parent; + + ~Guard() + { + (*block)[index]->~T(); + if (block->ConcurrentQueue::Block::template set_empty(index)) { + entry->value.store(nullptr, std::memory_order_relaxed); + parent->add_block_to_free_list(block); + } + } + } guard = { block, index, entry, this->parent }; + + element = std::move(el); // NOLINT + } + else { + element = std::move(el); // NOLINT + el.~T(); // NOLINT + + if (block->ConcurrentQueue::Block::template set_empty(index)) { + { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + // Add the block back into the global free pool (and remove from block index) + entry->value.store(nullptr, std::memory_order_relaxed); + } + this->parent->add_block_to_free_list(block); // releases the above store + } + } + + return true; + } + else { + this->dequeueOvercommit.fetch_add(1, std::memory_order_release); + } + } + + return false; + } + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4706) // assignment within conditional expression +#endif + template + bool enqueue_bulk(It itemFirst, size_t count) + { + // First, we need to make sure we have enough room to enqueue all of the elements; + // this means pre-allocating blocks and putting them in the block index (but only if + // all the allocations succeeded). + + // Note that the tailBlock we start off with may not be owned by us any more; + // this happens if it was filled up exactly to the top (setting tailIndex to + // the first index of the next block which is not yet allocated), then dequeued + // completely (putting it on the free list) before we enqueue again. + + index_t startTailIndex = this->tailIndex.load(std::memory_order_relaxed); + auto startBlock = this->tailBlock; + Block* firstAllocatedBlock = nullptr; + auto endBlock = this->tailBlock; + + // Figure out how many blocks we'll need to allocate, and do so + size_t blockBaseDiff = ((startTailIndex + count - 1) & ~static_cast(BLOCK_SIZE - 1)) - ((startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1)); + index_t currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + if (blockBaseDiff > 0) { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + do { + blockBaseDiff -= static_cast(BLOCK_SIZE); + currentTailIndex += static_cast(BLOCK_SIZE); + + // Find out where we'll be inserting this block in the block index + BlockIndexEntry* idxEntry = nullptr; // initialization here unnecessary but compiler can't always tell + Block* newBlock; + bool indexInserted = false; + auto head = this->headIndex.load(std::memory_order_relaxed); + assert(!details::circular_less_than(currentTailIndex, head)); + bool full = !details::circular_less_than(head, currentTailIndex + BLOCK_SIZE) || (MAX_SUBQUEUE_SIZE != details::const_numeric_max::value && (MAX_SUBQUEUE_SIZE == 0 || MAX_SUBQUEUE_SIZE - BLOCK_SIZE < currentTailIndex - head)); + + if (full || !(indexInserted = insert_block_index_entry(idxEntry, currentTailIndex)) || (newBlock = this->parent->ConcurrentQueue::template requisition_block()) == nullptr) { + // Index allocation or block allocation failed; revert any other allocations + // and index insertions done so far for this operation + if (indexInserted) { + rewind_block_index_tail(); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + } + currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { + currentTailIndex += static_cast(BLOCK_SIZE); + idxEntry = get_block_index_entry_for_index(currentTailIndex); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + rewind_block_index_tail(); + } + this->parent->add_blocks_to_free_list(firstAllocatedBlock); + this->tailBlock = startBlock; + + return false; + } + +#ifdef MCDBGQ_TRACKMEM + newBlock->owner = this; +#endif + newBlock->ConcurrentQueue::Block::template reset_empty(); + newBlock->next = nullptr; + + // Insert the new block into the index + idxEntry->value.store(newBlock, std::memory_order_relaxed); + + // Store the chain of blocks so that we can undo if later allocations fail, + // and so that we can find the blocks when we do the actual enqueueing + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr) { + assert(this->tailBlock != nullptr); + this->tailBlock->next = newBlock; + } + this->tailBlock = newBlock; + endBlock = newBlock; + firstAllocatedBlock = firstAllocatedBlock == nullptr ? newBlock : firstAllocatedBlock; + } while (blockBaseDiff > 0); + } + + // Enqueue, one block at a time + index_t newTailIndex = startTailIndex + static_cast(count); + currentTailIndex = startTailIndex; + this->tailBlock = startBlock; + assert((startTailIndex & static_cast(BLOCK_SIZE - 1)) != 0 || firstAllocatedBlock != nullptr || count == 0); + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0 && firstAllocatedBlock != nullptr) { + this->tailBlock = firstAllocatedBlock; + } + while (true) { + index_t stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(newTailIndex, stopIndex)) { + stopIndex = newTailIndex; + } + MOODYCAMEL_CONSTEXPR_IF (MOODYCAMEL_NOEXCEPT_CTOR(T, decltype(*itemFirst), new (static_cast(nullptr)) T(details::deref_noexcept(itemFirst)))) { + while (currentTailIndex != stopIndex) { + new ((*this->tailBlock)[currentTailIndex++]) T(*itemFirst++); + } + } + else { + MOODYCAMEL_TRY { + while (currentTailIndex != stopIndex) { + new ((*this->tailBlock)[currentTailIndex]) T(details::nomove_if(nullptr)) T(details::deref_noexcept(itemFirst)))>::eval(*itemFirst)); + ++currentTailIndex; + ++itemFirst; + } + } + MOODYCAMEL_CATCH (...) { + auto constructedStopIndex = currentTailIndex; + auto lastBlockEnqueued = this->tailBlock; + + if (!details::is_trivially_destructible::value) { + auto block = startBlock; + if ((startTailIndex & static_cast(BLOCK_SIZE - 1)) == 0) { + block = firstAllocatedBlock; + } + currentTailIndex = startTailIndex; + while (true) { + stopIndex = (currentTailIndex & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + if (details::circular_less_than(constructedStopIndex, stopIndex)) { + stopIndex = constructedStopIndex; + } + while (currentTailIndex != stopIndex) { + (*block)[currentTailIndex++]->~T(); + } + if (block == lastBlockEnqueued) { + break; + } + block = block->next; + } + } + + currentTailIndex = (startTailIndex - 1) & ~static_cast(BLOCK_SIZE - 1); + for (auto block = firstAllocatedBlock; block != nullptr; block = block->next) { + currentTailIndex += static_cast(BLOCK_SIZE); + auto idxEntry = get_block_index_entry_for_index(currentTailIndex); + idxEntry->value.store(nullptr, std::memory_order_relaxed); + rewind_block_index_tail(); + } + this->parent->add_blocks_to_free_list(firstAllocatedBlock); + this->tailBlock = startBlock; + MOODYCAMEL_RETHROW; + } + } + + if (this->tailBlock == endBlock) { + assert(currentTailIndex == newTailIndex); + break; + } + this->tailBlock = this->tailBlock->next; + } + this->tailIndex.store(newTailIndex, std::memory_order_release); + return true; + } +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + template + size_t dequeue_bulk(It& itemFirst, size_t max) + { + auto tail = this->tailIndex.load(std::memory_order_relaxed); + auto overcommit = this->dequeueOvercommit.load(std::memory_order_relaxed); + auto desiredCount = static_cast(tail - (this->dequeueOptimisticCount.load(std::memory_order_relaxed) - overcommit)); + if (details::circular_less_than(0, desiredCount)) { + desiredCount = desiredCount < max ? desiredCount : max; + std::atomic_thread_fence(std::memory_order_acquire); + + auto myDequeueCount = this->dequeueOptimisticCount.fetch_add(desiredCount, std::memory_order_relaxed); + + tail = this->tailIndex.load(std::memory_order_acquire); + auto actualCount = static_cast(tail - (myDequeueCount - overcommit)); + if (details::circular_less_than(0, actualCount)) { + actualCount = desiredCount < actualCount ? desiredCount : actualCount; + if (actualCount < desiredCount) { + this->dequeueOvercommit.fetch_add(desiredCount - actualCount, std::memory_order_release); + } + + // Get the first index. Note that since there's guaranteed to be at least actualCount elements, this + // will never exceed tail. + auto firstIndex = this->headIndex.fetch_add(actualCount, std::memory_order_acq_rel); + + // Iterate the blocks and dequeue + auto index = firstIndex; + BlockIndexHeader* localBlockIndex; + auto indexIndex = get_block_index_index_for_index(index, localBlockIndex); + do { + auto blockStartIndex = index; + index_t endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + + auto entry = localBlockIndex->index[indexIndex]; + auto block = entry->value.load(std::memory_order_relaxed); + if (MOODYCAMEL_NOEXCEPT_ASSIGN(T, T&&, details::deref_noexcept(itemFirst) = std::move((*(*block)[index])))) { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst++ = std::move(el); + el.~T(); + ++index; + } + } + else { + MOODYCAMEL_TRY { + while (index != endIndex) { + auto& el = *((*block)[index]); + *itemFirst = std::move(el); + ++itemFirst; + el.~T(); + ++index; + } + } + MOODYCAMEL_CATCH (...) { + do { + entry = localBlockIndex->index[indexIndex]; + block = entry->value.load(std::memory_order_relaxed); + while (index != endIndex) { + (*block)[index++]->~T(); + } + + if (block->ConcurrentQueue::Block::template set_many_empty(blockStartIndex, static_cast(endIndex - blockStartIndex))) { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + entry->value.store(nullptr, std::memory_order_relaxed); + this->parent->add_block_to_free_list(block); + } + indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); + + blockStartIndex = index; + endIndex = (index & ~static_cast(BLOCK_SIZE - 1)) + static_cast(BLOCK_SIZE); + endIndex = details::circular_less_than(firstIndex + static_cast(actualCount), endIndex) ? firstIndex + static_cast(actualCount) : endIndex; + } while (index != firstIndex + actualCount); + + MOODYCAMEL_RETHROW; + } + } + if (block->ConcurrentQueue::Block::template set_many_empty(blockStartIndex, static_cast(endIndex - blockStartIndex))) { + { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + // Note that the set_many_empty above did a release, meaning that anybody who acquires the block + // we're about to free can use it safely since our writes (and reads!) will have happened-before then. + entry->value.store(nullptr, std::memory_order_relaxed); + } + this->parent->add_block_to_free_list(block); // releases the above store + } + indexIndex = (indexIndex + 1) & (localBlockIndex->capacity - 1); + } while (index != firstIndex + actualCount); + + return actualCount; + } + else { + this->dequeueOvercommit.fetch_add(desiredCount, std::memory_order_release); + } + } + + return 0; + } + + private: + // The block size must be > 1, so any number with the low bit set is an invalid block base index + static const index_t INVALID_BLOCK_BASE = 1; + + struct BlockIndexEntry + { + std::atomic key; + std::atomic value; + }; + + struct BlockIndexHeader + { + size_t capacity; + std::atomic tail; + BlockIndexEntry* entries; + BlockIndexEntry** index; + BlockIndexHeader* prev; + }; + + template + inline bool insert_block_index_entry(BlockIndexEntry*& idxEntry, index_t blockStartIndex) + { + auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); // We're the only writer thread, relaxed is OK + if (localBlockIndex == nullptr) { + return false; // this can happen if new_block_index failed in the constructor + } + size_t newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1); + idxEntry = localBlockIndex->index[newTail]; + if (idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE || + idxEntry->value.load(std::memory_order_relaxed) == nullptr) { + + idxEntry->key.store(blockStartIndex, std::memory_order_relaxed); + localBlockIndex->tail.store(newTail, std::memory_order_release); + return true; + } + + // No room in the old block index, try to allocate another one! + MOODYCAMEL_CONSTEXPR_IF (allocMode == CannotAlloc) { + return false; + } + else if (!new_block_index()) { + return false; + } + else { + localBlockIndex = blockIndex.load(std::memory_order_relaxed); + newTail = (localBlockIndex->tail.load(std::memory_order_relaxed) + 1) & (localBlockIndex->capacity - 1); + idxEntry = localBlockIndex->index[newTail]; + assert(idxEntry->key.load(std::memory_order_relaxed) == INVALID_BLOCK_BASE); + idxEntry->key.store(blockStartIndex, std::memory_order_relaxed); + localBlockIndex->tail.store(newTail, std::memory_order_release); + return true; + } + } + + inline void rewind_block_index_tail() + { + auto localBlockIndex = blockIndex.load(std::memory_order_relaxed); + localBlockIndex->tail.store((localBlockIndex->tail.load(std::memory_order_relaxed) - 1) & (localBlockIndex->capacity - 1), std::memory_order_relaxed); + } + + inline BlockIndexEntry* get_block_index_entry_for_index(index_t index) const + { + BlockIndexHeader* localBlockIndex; + auto idx = get_block_index_index_for_index(index, localBlockIndex); + return localBlockIndex->index[idx]; + } + + inline size_t get_block_index_index_for_index(index_t index, BlockIndexHeader*& localBlockIndex) const + { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + debug::DebugLock lock(mutex); +#endif + index &= ~static_cast(BLOCK_SIZE - 1); + localBlockIndex = blockIndex.load(std::memory_order_acquire); + auto tail = localBlockIndex->tail.load(std::memory_order_acquire); + auto tailBase = localBlockIndex->index[tail]->key.load(std::memory_order_relaxed); + assert(tailBase != INVALID_BLOCK_BASE); + // Note: Must use division instead of shift because the index may wrap around, causing a negative + // offset, whose negativity we want to preserve + auto offset = static_cast(static_cast::type>(index - tailBase) / static_cast::type>(BLOCK_SIZE)); + size_t idx = (tail + offset) & (localBlockIndex->capacity - 1); + assert(localBlockIndex->index[idx]->key.load(std::memory_order_relaxed) == index && localBlockIndex->index[idx]->value.load(std::memory_order_relaxed) != nullptr); + return idx; + } + + bool new_block_index() + { + auto prev = blockIndex.load(std::memory_order_relaxed); + size_t prevCapacity = prev == nullptr ? 0 : prev->capacity; + auto entryCount = prev == nullptr ? nextBlockIndexCapacity : prevCapacity; + auto raw = static_cast((Traits::malloc)( + sizeof(BlockIndexHeader) + + std::alignment_of::value - 1 + sizeof(BlockIndexEntry) * entryCount + + std::alignment_of::value - 1 + sizeof(BlockIndexEntry*) * nextBlockIndexCapacity)); + if (raw == nullptr) { + return false; + } + + auto header = new (raw) BlockIndexHeader; + auto entries = reinterpret_cast(details::align_for(raw + sizeof(BlockIndexHeader))); + auto index = reinterpret_cast(details::align_for(reinterpret_cast(entries) + sizeof(BlockIndexEntry) * entryCount)); + if (prev != nullptr) { + auto prevTail = prev->tail.load(std::memory_order_relaxed); + auto prevPos = prevTail; + size_t i = 0; + do { + prevPos = (prevPos + 1) & (prev->capacity - 1); + index[i++] = prev->index[prevPos]; + } while (prevPos != prevTail); + assert(i == prevCapacity); + } + for (size_t i = 0; i != entryCount; ++i) { + new (entries + i) BlockIndexEntry; + entries[i].key.store(INVALID_BLOCK_BASE, std::memory_order_relaxed); + index[prevCapacity + i] = entries + i; + } + header->prev = prev; + header->entries = entries; + header->index = index; + header->capacity = nextBlockIndexCapacity; + header->tail.store((prevCapacity - 1) & (nextBlockIndexCapacity - 1), std::memory_order_relaxed); + + blockIndex.store(header, std::memory_order_release); + + nextBlockIndexCapacity <<= 1; + + return true; + } + + private: + size_t nextBlockIndexCapacity; + std::atomic blockIndex; + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + public: + details::ThreadExitListener threadExitListener; + private: +#endif + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + public: + ImplicitProducer* nextImplicitProducer; + private: +#endif + +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODBLOCKINDEX + mutable debug::DebugMutex mutex; +#endif +#ifdef MCDBGQ_TRACKMEM + friend struct MemStats; +#endif + }; + + + ////////////////////////////////// + // Block pool manipulation + ////////////////////////////////// + + void populate_initial_block_list(size_t blockCount) + { + initialBlockPoolSize = blockCount; + if (initialBlockPoolSize == 0) { + initialBlockPool = nullptr; + return; + } + + initialBlockPool = create_array(blockCount); + if (initialBlockPool == nullptr) { + initialBlockPoolSize = 0; + } + for (size_t i = 0; i < initialBlockPoolSize; ++i) { + initialBlockPool[i].dynamicallyAllocated = false; + } + } + + inline Block* try_get_block_from_initial_pool() + { + if (initialBlockPoolIndex.load(std::memory_order_relaxed) >= initialBlockPoolSize) { + return nullptr; + } + + auto index = initialBlockPoolIndex.fetch_add(1, std::memory_order_relaxed); + + return index < initialBlockPoolSize ? (initialBlockPool + index) : nullptr; + } + + inline void add_block_to_free_list(Block* block) + { +#ifdef MCDBGQ_TRACKMEM + block->owner = nullptr; +#endif + if (!Traits::RECYCLE_ALLOCATED_BLOCKS && block->dynamicallyAllocated) { + destroy(block); + } + else { + freeList.add(block); + } + } + + inline void add_blocks_to_free_list(Block* block) + { + while (block != nullptr) { + auto next = block->next; + add_block_to_free_list(block); + block = next; + } + } + + inline Block* try_get_block_from_free_list() + { + return freeList.try_get(); + } + + // Gets a free block from one of the memory pools, or allocates a new one (if applicable) + template + Block* requisition_block() + { + auto block = try_get_block_from_initial_pool(); + if (block != nullptr) { + return block; + } + + block = try_get_block_from_free_list(); + if (block != nullptr) { + return block; + } + + MOODYCAMEL_CONSTEXPR_IF (canAlloc == CanAlloc) { + return create(); + } + else { + return nullptr; + } + } + + +#ifdef MCDBGQ_TRACKMEM + public: + struct MemStats { + size_t allocatedBlocks; + size_t usedBlocks; + size_t freeBlocks; + size_t ownedBlocksExplicit; + size_t ownedBlocksImplicit; + size_t implicitProducers; + size_t explicitProducers; + size_t elementsEnqueued; + size_t blockClassBytes; + size_t queueClassBytes; + size_t implicitBlockIndexBytes; + size_t explicitBlockIndexBytes; + + friend class ConcurrentQueue; + + private: + static MemStats getFor(ConcurrentQueue* q) + { + MemStats stats = { 0 }; + + stats.elementsEnqueued = q->size_approx(); + + auto block = q->freeList.head_unsafe(); + while (block != nullptr) { + ++stats.allocatedBlocks; + ++stats.freeBlocks; + block = block->freeListNext.load(std::memory_order_relaxed); + } + + for (auto ptr = q->producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + bool implicit = dynamic_cast(ptr) != nullptr; + stats.implicitProducers += implicit ? 1 : 0; + stats.explicitProducers += implicit ? 0 : 1; + + if (implicit) { + auto prod = static_cast(ptr); + stats.queueClassBytes += sizeof(ImplicitProducer); + auto head = prod->headIndex.load(std::memory_order_relaxed); + auto tail = prod->tailIndex.load(std::memory_order_relaxed); + auto hash = prod->blockIndex.load(std::memory_order_relaxed); + if (hash != nullptr) { + for (size_t i = 0; i != hash->capacity; ++i) { + if (hash->index[i]->key.load(std::memory_order_relaxed) != ImplicitProducer::INVALID_BLOCK_BASE && hash->index[i]->value.load(std::memory_order_relaxed) != nullptr) { + ++stats.allocatedBlocks; + ++stats.ownedBlocksImplicit; + } + } + stats.implicitBlockIndexBytes += hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry); + for (; hash != nullptr; hash = hash->prev) { + stats.implicitBlockIndexBytes += sizeof(typename ImplicitProducer::BlockIndexHeader) + hash->capacity * sizeof(typename ImplicitProducer::BlockIndexEntry*); + } + } + for (; details::circular_less_than(head, tail); head += BLOCK_SIZE) { + //auto block = prod->get_block_index_entry_for_index(head); + ++stats.usedBlocks; + } + } + else { + auto prod = static_cast(ptr); + stats.queueClassBytes += sizeof(ExplicitProducer); + auto tailBlock = prod->tailBlock; + bool wasNonEmpty = false; + if (tailBlock != nullptr) { + auto block = tailBlock; + do { + ++stats.allocatedBlocks; + if (!block->ConcurrentQueue::Block::template is_empty() || wasNonEmpty) { + ++stats.usedBlocks; + wasNonEmpty = wasNonEmpty || block != tailBlock; + } + ++stats.ownedBlocksExplicit; + block = block->next; + } while (block != tailBlock); + } + auto index = prod->blockIndex.load(std::memory_order_relaxed); + while (index != nullptr) { + stats.explicitBlockIndexBytes += sizeof(typename ExplicitProducer::BlockIndexHeader) + index->size * sizeof(typename ExplicitProducer::BlockIndexEntry); + index = static_cast(index->prev); + } + } + } + + auto freeOnInitialPool = q->initialBlockPoolIndex.load(std::memory_order_relaxed) >= q->initialBlockPoolSize ? 0 : q->initialBlockPoolSize - q->initialBlockPoolIndex.load(std::memory_order_relaxed); + stats.allocatedBlocks += freeOnInitialPool; + stats.freeBlocks += freeOnInitialPool; + + stats.blockClassBytes = sizeof(Block) * stats.allocatedBlocks; + stats.queueClassBytes += sizeof(ConcurrentQueue); + + return stats; + } + }; + + // For debugging only. Not thread-safe. + MemStats getMemStats() + { + return MemStats::getFor(this); + } + private: + friend struct MemStats; +#endif + + + ////////////////////////////////// + // Producer list manipulation + ////////////////////////////////// + + ProducerBase* recycle_or_create_producer(bool isExplicit) + { +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugLock lock(implicitProdMutex); +#endif + // Try to re-use one first + for (auto ptr = producerListTail.load(std::memory_order_acquire); ptr != nullptr; ptr = ptr->next_prod()) { + if (ptr->inactive.load(std::memory_order_relaxed) && ptr->isExplicit == isExplicit) { + bool expected = true; + if (ptr->inactive.compare_exchange_strong(expected, /* desired */ false, std::memory_order_acquire, std::memory_order_relaxed)) { + // We caught one! It's been marked as activated, the caller can have it + return ptr; + } + } + } + + return add_producer(isExplicit ? static_cast(create(this)) : create(this)); + } + + ProducerBase* add_producer(ProducerBase* producer) + { + // Handle failed memory allocation + if (producer == nullptr) { + return nullptr; + } + + producerCount.fetch_add(1, std::memory_order_relaxed); + + // Add it to the lock-free list + auto prevTail = producerListTail.load(std::memory_order_relaxed); + do { + producer->next = prevTail; + } while (!producerListTail.compare_exchange_weak(prevTail, producer, std::memory_order_release, std::memory_order_relaxed)); + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + if (producer->isExplicit) { + auto prevTailExplicit = explicitProducers.load(std::memory_order_relaxed); + do { + static_cast(producer)->nextExplicitProducer = prevTailExplicit; + } while (!explicitProducers.compare_exchange_weak(prevTailExplicit, static_cast(producer), std::memory_order_release, std::memory_order_relaxed)); + } + else { + auto prevTailImplicit = implicitProducers.load(std::memory_order_relaxed); + do { + static_cast(producer)->nextImplicitProducer = prevTailImplicit; + } while (!implicitProducers.compare_exchange_weak(prevTailImplicit, static_cast(producer), std::memory_order_release, std::memory_order_relaxed)); + } +#endif + + return producer; + } + + void reown_producers() + { + // After another instance is moved-into/swapped-with this one, all the + // producers we stole still think their parents are the other queue. + // So fix them up! + for (auto ptr = producerListTail.load(std::memory_order_relaxed); ptr != nullptr; ptr = ptr->next_prod()) { + ptr->parent = this; + } + } + + + ////////////////////////////////// + // Implicit producer hash + ////////////////////////////////// + + struct ImplicitProducerKVP + { + std::atomic key; + ImplicitProducer* value; // No need for atomicity since it's only read by the thread that sets it in the first place + + ImplicitProducerKVP() : value(nullptr) { } + + ImplicitProducerKVP(ImplicitProducerKVP&& other) MOODYCAMEL_NOEXCEPT + { + key.store(other.key.load(std::memory_order_relaxed), std::memory_order_relaxed); + value = other.value; + } + + inline ImplicitProducerKVP& operator=(ImplicitProducerKVP&& other) MOODYCAMEL_NOEXCEPT + { + swap(other); + return *this; + } + + inline void swap(ImplicitProducerKVP& other) MOODYCAMEL_NOEXCEPT + { + if (this != &other) { + details::swap_relaxed(key, other.key); + std::swap(value, other.value); + } + } + }; + + template + friend void moodycamel::swap(typename ConcurrentQueue::ImplicitProducerKVP&, typename ConcurrentQueue::ImplicitProducerKVP&) MOODYCAMEL_NOEXCEPT; + + struct ImplicitProducerHash + { + size_t capacity; + ImplicitProducerKVP* entries; + ImplicitProducerHash* prev; + }; + + inline void populate_initial_implicit_producer_hash() + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) { + return; + } + else { + implicitProducerHashCount.store(0, std::memory_order_relaxed); + auto hash = &initialImplicitProducerHash; + hash->capacity = INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; + hash->entries = &initialImplicitProducerHashEntries[0]; + for (size_t i = 0; i != INITIAL_IMPLICIT_PRODUCER_HASH_SIZE; ++i) { + initialImplicitProducerHashEntries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed); + } + hash->prev = nullptr; + implicitProducerHash.store(hash, std::memory_order_relaxed); + } + } + + void swap_implicit_producer_hashes(ConcurrentQueue& other) + { + MOODYCAMEL_CONSTEXPR_IF (INITIAL_IMPLICIT_PRODUCER_HASH_SIZE == 0) { + return; + } + else { + // Swap (assumes our implicit producer hash is initialized) + initialImplicitProducerHashEntries.swap(other.initialImplicitProducerHashEntries); + initialImplicitProducerHash.entries = &initialImplicitProducerHashEntries[0]; + other.initialImplicitProducerHash.entries = &other.initialImplicitProducerHashEntries[0]; + + details::swap_relaxed(implicitProducerHashCount, other.implicitProducerHashCount); + + details::swap_relaxed(implicitProducerHash, other.implicitProducerHash); + if (implicitProducerHash.load(std::memory_order_relaxed) == &other.initialImplicitProducerHash) { + implicitProducerHash.store(&initialImplicitProducerHash, std::memory_order_relaxed); + } + else { + ImplicitProducerHash* hash; + for (hash = implicitProducerHash.load(std::memory_order_relaxed); hash->prev != &other.initialImplicitProducerHash; hash = hash->prev) { + continue; + } + hash->prev = &initialImplicitProducerHash; + } + if (other.implicitProducerHash.load(std::memory_order_relaxed) == &initialImplicitProducerHash) { + other.implicitProducerHash.store(&other.initialImplicitProducerHash, std::memory_order_relaxed); + } + else { + ImplicitProducerHash* hash; + for (hash = other.implicitProducerHash.load(std::memory_order_relaxed); hash->prev != &initialImplicitProducerHash; hash = hash->prev) { + continue; + } + hash->prev = &other.initialImplicitProducerHash; + } + } + } + + // Only fails (returns nullptr) if memory allocation fails + ImplicitProducer* get_or_add_implicit_producer() + { + // Note that since the data is essentially thread-local (key is thread ID), + // there's a reduced need for fences (memory ordering is already consistent + // for any individual thread), except for the current table itself. + + // Start by looking for the thread ID in the current and all previous hash tables. + // If it's not found, it must not be in there yet, since this same thread would + // have added it previously to one of the tables that we traversed. + + // Code and algorithm adapted from http://preshing.com/20130605/the-worlds-simplest-lock-free-hash-table + +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugLock lock(implicitProdMutex); +#endif + + auto id = details::thread_id(); + auto hashedId = details::hash_thread_id(id); + + auto mainHash = implicitProducerHash.load(std::memory_order_acquire); + assert(mainHash != nullptr); // silence clang-tidy and MSVC warnings (hash cannot be null) + for (auto hash = mainHash; hash != nullptr; hash = hash->prev) { + // Look for the id in this hash + auto index = hashedId; + while (true) { // Not an infinite loop because at least one slot is free in the hash table + index &= hash->capacity - 1u; + + auto probedKey = hash->entries[index].key.load(std::memory_order_relaxed); + if (probedKey == id) { + // Found it! If we had to search several hashes deep, though, we should lazily add it + // to the current main hash table to avoid the extended search next time. + // Note there's guaranteed to be room in the current hash table since every subsequent + // table implicitly reserves space for all previous tables (there's only one + // implicitProducerHashCount). + auto value = hash->entries[index].value; + if (hash != mainHash) { + index = hashedId; + while (true) { + index &= mainHash->capacity - 1u; + auto empty = details::invalid_thread_id; +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + auto reusable = details::invalid_thread_id2; + if (mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_seq_cst, std::memory_order_relaxed) || + mainHash->entries[index].key.compare_exchange_strong(reusable, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { +#else + if (mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { +#endif + mainHash->entries[index].value = value; + break; + } + ++index; + } + } + + return value; + } + if (probedKey == details::invalid_thread_id) { + break; // Not in this hash table + } + ++index; + } + } + + // Insert! + auto newCount = 1 + implicitProducerHashCount.fetch_add(1, std::memory_order_relaxed); + while (true) { + // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) + if (newCount >= (mainHash->capacity >> 1) && !implicitProducerHashResizeInProgress.test_and_set(std::memory_order_acquire)) { + // We've acquired the resize lock, try to allocate a bigger hash table. + // Note the acquire fence synchronizes with the release fence at the end of this block, and hence when + // we reload implicitProducerHash it must be the most recent version (it only gets changed within this + // locked block). + mainHash = implicitProducerHash.load(std::memory_order_acquire); + if (newCount >= (mainHash->capacity >> 1)) { + size_t newCapacity = mainHash->capacity << 1; + while (newCount >= (newCapacity >> 1)) { + newCapacity <<= 1; + } + auto raw = static_cast((Traits::malloc)(sizeof(ImplicitProducerHash) + std::alignment_of::value - 1 + sizeof(ImplicitProducerKVP) * newCapacity)); + if (raw == nullptr) { + // Allocation failed + implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); + implicitProducerHashResizeInProgress.clear(std::memory_order_relaxed); + return nullptr; + } + + auto newHash = new (raw) ImplicitProducerHash; + newHash->capacity = static_cast(newCapacity); + newHash->entries = reinterpret_cast(details::align_for(raw + sizeof(ImplicitProducerHash))); + for (size_t i = 0; i != newCapacity; ++i) { + new (newHash->entries + i) ImplicitProducerKVP; + newHash->entries[i].key.store(details::invalid_thread_id, std::memory_order_relaxed); + } + newHash->prev = mainHash; + implicitProducerHash.store(newHash, std::memory_order_release); + implicitProducerHashResizeInProgress.clear(std::memory_order_release); + mainHash = newHash; + } + else { + implicitProducerHashResizeInProgress.clear(std::memory_order_release); + } + } + + // If it's < three-quarters full, add to the old one anyway so that we don't have to wait for the next table + // to finish being allocated by another thread (and if we just finished allocating above, the condition will + // always be true) + if (newCount < (mainHash->capacity >> 1) + (mainHash->capacity >> 2)) { + auto producer = static_cast(recycle_or_create_producer(false)); + if (producer == nullptr) { + implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); + return nullptr; + } + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + producer->threadExitListener.callback = &ConcurrentQueue::implicit_producer_thread_exited_callback; + producer->threadExitListener.userData = producer; + details::ThreadExitNotifier::subscribe(&producer->threadExitListener); +#endif + + auto index = hashedId; + while (true) { + index &= mainHash->capacity - 1u; + auto empty = details::invalid_thread_id; +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + auto reusable = details::invalid_thread_id2; + if (mainHash->entries[index].key.compare_exchange_strong(reusable, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { + implicitProducerHashCount.fetch_sub(1, std::memory_order_relaxed); // already counted as a used slot + mainHash->entries[index].value = producer; + break; + } +#endif + if (mainHash->entries[index].key.compare_exchange_strong(empty, id, std::memory_order_seq_cst, std::memory_order_relaxed)) { + mainHash->entries[index].value = producer; + break; + } + ++index; + } + return producer; + } + + // Hmm, the old hash is quite full and somebody else is busy allocating a new one. + // We need to wait for the allocating thread to finish (if it succeeds, we add, if not, + // we try to allocate ourselves). + mainHash = implicitProducerHash.load(std::memory_order_acquire); + } + } + +#ifdef MOODYCAMEL_CPP11_THREAD_LOCAL_SUPPORTED + void implicit_producer_thread_exited(ImplicitProducer* producer) + { + // Remove from hash +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugLock lock(implicitProdMutex); +#endif + auto hash = implicitProducerHash.load(std::memory_order_acquire); + assert(hash != nullptr); // The thread exit listener is only registered if we were added to a hash in the first place + auto id = details::thread_id(); + auto hashedId = details::hash_thread_id(id); + details::thread_id_t probedKey; + + // We need to traverse all the hashes just in case other threads aren't on the current one yet and are + // trying to add an entry thinking there's a free slot (because they reused a producer) + for (; hash != nullptr; hash = hash->prev) { + auto index = hashedId; + do { + index &= hash->capacity - 1u; + probedKey = id; + if (hash->entries[index].key.compare_exchange_strong(probedKey, details::invalid_thread_id2, std::memory_order_seq_cst, std::memory_order_relaxed)) { + break; + } + ++index; + } while (probedKey != details::invalid_thread_id); // Can happen if the hash has changed but we weren't put back in it yet, or if we weren't added to this hash in the first place + } + + // Mark the queue as being recyclable + producer->inactive.store(true, std::memory_order_release); + } + + static void implicit_producer_thread_exited_callback(void* userData) + { + auto producer = static_cast(userData); + auto queue = producer->parent; + queue->implicit_producer_thread_exited(producer); + } +#endif + + ////////////////////////////////// + // Utility functions + ////////////////////////////////// + + template + static inline void* aligned_malloc(size_t size) + { + MOODYCAMEL_CONSTEXPR_IF (std::alignment_of::value <= std::alignment_of::value) + return (Traits::malloc)(size); + else { + size_t alignment = std::alignment_of::value; + void* raw = (Traits::malloc)(size + alignment - 1 + sizeof(void*)); + if (!raw) + return nullptr; + char* ptr = details::align_for(reinterpret_cast(raw) + sizeof(void*)); + *(reinterpret_cast(ptr) - 1) = raw; + return ptr; + } + } + + template + static inline void aligned_free(void* ptr) + { + MOODYCAMEL_CONSTEXPR_IF (std::alignment_of::value <= std::alignment_of::value) + return (Traits::free)(ptr); + else + (Traits::free)(ptr ? *(reinterpret_cast(ptr) - 1) : nullptr); + } + + template + static inline U* create_array(size_t count) + { + assert(count > 0); + U* p = static_cast(aligned_malloc(sizeof(U) * count)); + if (p == nullptr) + return nullptr; + + for (size_t i = 0; i != count; ++i) + new (p + i) U(); + return p; + } + + template + static inline void destroy_array(U* p, size_t count) + { + if (p != nullptr) { + assert(count > 0); + for (size_t i = count; i != 0; ) + (p + --i)->~U(); + } + aligned_free(p); + } + + template + static inline U* create() + { + void* p = aligned_malloc(sizeof(U)); + return p != nullptr ? new (p) U : nullptr; + } + + template + static inline U* create(A1&& a1) + { + void* p = aligned_malloc(sizeof(U)); + return p != nullptr ? new (p) U(std::forward(a1)) : nullptr; + } + + template + static inline void destroy(U* p) + { + if (p != nullptr) + p->~U(); + aligned_free(p); + } + +private: + std::atomic producerListTail; + std::atomic producerCount; + + std::atomic initialBlockPoolIndex; + Block* initialBlockPool; + size_t initialBlockPoolSize; + +#ifndef MCDBGQ_USEDEBUGFREELIST + FreeList freeList; +#else + debug::DebugFreeList freeList; +#endif + + std::atomic implicitProducerHash; + std::atomic implicitProducerHashCount; // Number of slots logically used + ImplicitProducerHash initialImplicitProducerHash; + std::array initialImplicitProducerHashEntries; + std::atomic_flag implicitProducerHashResizeInProgress; + + std::atomic nextExplicitConsumerId; + std::atomic globalExplicitConsumerOffset; + +#ifdef MCDBGQ_NOLOCKFREE_IMPLICITPRODHASH + debug::DebugMutex implicitProdMutex; +#endif + +#ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG + std::atomic explicitProducers; + std::atomic implicitProducers; +#endif +}; + + +template +ProducerToken::ProducerToken(ConcurrentQueue& queue) + : producer(queue.recycle_or_create_producer(true)) +{ + if (producer != nullptr) { + producer->token = this; + } +} + +template +ProducerToken::ProducerToken(BlockingConcurrentQueue& queue) + : producer(reinterpret_cast*>(&queue)->recycle_or_create_producer(true)) +{ + if (producer != nullptr) { + producer->token = this; + } +} + +template +ConsumerToken::ConsumerToken(ConcurrentQueue& queue) + : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) +{ + initialOffset = queue.nextExplicitConsumerId.fetch_add(1, std::memory_order_release); + lastKnownGlobalOffset = static_cast(-1); +} + +template +ConsumerToken::ConsumerToken(BlockingConcurrentQueue& queue) + : itemsConsumedFromCurrent(0), currentProducer(nullptr), desiredProducer(nullptr) +{ + initialOffset = reinterpret_cast*>(&queue)->nextExplicitConsumerId.fetch_add(1, std::memory_order_release); + lastKnownGlobalOffset = static_cast(-1); +} + +template +inline void swap(ConcurrentQueue& a, ConcurrentQueue& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +inline void swap(ProducerToken& a, ProducerToken& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +inline void swap(ConsumerToken& a, ConsumerToken& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +template +inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, typename ConcurrentQueue::ImplicitProducerKVP& b) MOODYCAMEL_NOEXCEPT +{ + a.swap(b); +} + +} + +#if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17) +#pragma warning(pop) +#endif + +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) +#pragma GCC diagnostic pop +#endif diff --git a/benchs/flat_hash_map.hpp b/benchs/flat_hash_map.hpp index 877db4c2..03b7e2da 100644 --- a/benchs/flat_hash_map.hpp +++ b/benchs/flat_hash_map.hpp @@ -122,9 +122,193 @@ namespace seq size_t d_next_target; // next size before rehash size_t d_hash_mask; // hash mask chain_count_type d_chain_count; // number of chained nodes, used to optimize detection of needed rehash on insert - float d_max_load_factor = 0.75f; + float d_max_load_factor = 0.7f; chain_node_type* first_free = nullptr; + template + static SEQ_CONCURRENT_INLINE auto FindWithTh(std::uint8_t th, const Equal& eq, K&& key, const std::uint8_t* hashs, const Value* values) noexcept( + noexcept(eq(extract_key::key(*values), std::forward(key)))) -> const Value* + { + if constexpr (Size == 4) { + if (!hashs[0]) + return nullptr; + // no SSE variant (way slower) + std::uint32_t _th; + memset(&_th, th, sizeof(_th)); + + // do first 3 values + std::uint32_t found = MoveMask4((*reinterpret_cast(hashs)) ^ _th) >> 8u; + if (found) { + SEQ_PREFETCH(values); + do { + unsigned pos = bit_scan_forward_32(found) >> 3u; + if (eq(extract_key::key(values[pos]), std::forward(key))) + return values + pos; + reinterpret_cast(&found)[pos] = 0; + } while (found); + } + return nullptr; + } + else if constexpr (Size == 8) { + if (!hashs[0]) + return nullptr; + // no SSE variant (way slower) + std::uint64_t _th; + memset(&_th, th, sizeof(_th)); + + // do first 7 values + std::uint64_t found = MoveMask8((*reinterpret_cast(hashs)) ^ _th) >> 8u; + if (found) { + SEQ_PREFETCH(values); + do { + unsigned pos = bit_scan_forward_64(found) >> 3u; + if (eq(extract_key::key(values[pos]), std::forward(key))) + return values + pos; + reinterpret_cast(&found)[pos] = 0; + } while (found); + } + return nullptr; + } + else if constexpr (Size == 16) { +#if defined(__SSE2__) + // SSE movemask + if (!hashs[0]) + return nullptr; + auto hs = _mm_load_si128(reinterpret_cast(hashs)); + auto mask = (unsigned)_mm_movemask_epi8(_mm_cmpeq_epi8(hs, _mm_set1_epi8(static_cast(th)))) >> 1; + if (mask) { + SEQ_PREFETCH(values); + do { + unsigned pos = bit_scan_forward_32(mask); + if (eq(extract_key::key(values[pos]), std::forward(key))) + return values + pos; + mask &= mask - 1u; + } while (mask); + } +#endif + return nullptr; + } + SEQ_UNREACHABLE(); + } + + /// @brief Find given value in a dense node. + /// Performs lookup on the full chain. + template + auto FindInDense(std::uint8_t th, const Equal& eq, K&& key, const ConcurrentDenseNode* n, F&& f) const + noexcept(noexcept(eq(std::forward(key), extract_key::key(std::declval())))) -> size_t + { + do { + auto v = FindWithTh(th, eq, std::forward(key), n->hashs, n->values()); + if (v) { + std::forward(f)(const_cast(*v)); + return 1; + } + } while ((n = n->right)); + return 0; + } + + /// @brief Find given value in main node. + /// Performs lookup on the full chain. + template + SEQ_CONCURRENT_INLINE auto FindInNode(std::uint8_t th, const Equal& eq, K&& key, const ConcurrentHashNode* node, const ConcurrentValueNode* values, F&& f) const + noexcept(noexcept(eq(std::forward(key), extract_key::key(std::declval())))) -> size_t + { + if (auto v = FindWithTh(th, eq, std::forward(key), node->hashs, values->values())) { + std::forward(f)(const_cast(*v)); + return 1; + } + if (node->full() && values->right) + return FindInDense(th, eq, std::forward(key), values->right, std::forward(f)); + return 0; + } + + /// @brief Returns index of the next null hash value + /// Only used when an exception is thrown during rehash + auto FindFreeSlotInNode(ConcurrentHashNode* node, ConcurrentValueNode* values) noexcept -> std::pair + { + // Look for a free slot in the node chain + unsigned idx = FindIndexZero(node->hashs); + if (idx != static_cast(-1)) + return { values->values() + idx, node->hashs + idx + 1 }; + auto* d = values->right; + while (d) { + idx = FindIndexZero(d->hashs); + if (idx != static_cast(-1)) + return { d->values() + idx, d->hashs + idx + 1 }; + d = d->right; + } + return { nullptr, nullptr }; + } + + /// @brief Insert value in a new dense node + template + auto InsertNewDense(std::uint8_t th, Node* n, K&& key, Args&&... args) -> std::pair + { + chain_node_allocator al{ get_allocator() }; + chain_node_type* d = al.allocate(1); + + memset(d->hashs, 0, sizeof(d->hashs)); + d->right = nullptr; + d->left = reinterpret_cast(n); + + try { + Policy::emplace(d->values(), std::forward(key), std::forward(args)...); + } + catch (...) { + // destroy dense node + al.deallocate(d, 1); + throw; + } + d->hashs[++d->hashs[0]] = th; + n->right = d; + ++d_chain_count; + return { d->values(), true }; + } + + /// @brief Insert value in dense node if it does not already exist + template + auto FindInsertDense(std::uint8_t th, const Equal& eq, ConcurrentDenseNode* n, K&& key, Args&&... args) -> std::pair + { + auto valid = n; + do { + if constexpr (CheckExists) { + auto v = FindWithTh(th, eq, extract_key::key(std::forward(key)), n->hashs, n->values()); + if (v) + return { const_cast(v), false }; + } + valid = n; + } while ((n = n->right)); + if SEQ_UNLIKELY (valid->full()) + return InsertNewDense(th, valid, std::forward(key), std::forward(args)...); + + // might throw, fine + auto p = Policy::emplace(valid->values() + valid->count(), std::forward(key), std::forward(args)...); + valid->hashs[++valid->hashs[0]] = th; + return { p, true }; + } + + /// @brief Insert value if it does not already exist + template + SEQ_CONCURRENT_INLINE auto FindInsertNode(std::uint8_t th, const Equal& eq, ConcurrentHashNode* node, ConcurrentValueNode* values, K&& key, Args&&... args) + -> std::pair + { + if constexpr (CheckExists) { + auto v = FindWithTh(th, eq, extract_key::key(std::forward(key)), node->hashs, values->values()); + if (v) + return { const_cast(v), false }; + } + if SEQ_UNLIKELY (node->full()) { + if (values->right) + return FindInsertDense(th, eq, values->right, std::forward(key), std::forward(args)...); + return InsertNewDense(th, values, std::forward(key), std::forward(args)...); + } + // might throw, fine + auto p = Policy::emplace(values->values() + node->count(), std::forward(key), std::forward(args)...); + node->hashs[++node->hashs[0]] = th; + return { p, true }; + } + + SEQ_ALWAYS_INLINE auto get_allocator() const noexcept { return static_cast(*this); } static auto get_static_node() noexcept -> node_type* @@ -202,9 +386,7 @@ namespace seq d_buckets[i].for_each(d_values + i, [&](std::uint8_t* hashs, unsigned j, Value& val) { auto pos = hash_key(extract_key::key(val)) & new_hash_mask; - FindInsertNode(d_chain_count, - //chain_node_allocator{ get_allocator() }, - ChainAllocator{ this }, + FindInsertNode< InsertFlatPolicy, false>( hashs[j + 1], this->key_eq(), buckets + pos, @@ -302,7 +484,7 @@ namespace seq SEQ_ALWAYS_INLINE void rehash_on_insert() { // Rehash on insert - if SEQ_UNLIKELY ((d_size >= d_next_target) && (d_buckets == get_static_node() || d_chain_count > ((d_hash_mask + 1) >> 5))) + if SEQ_UNLIKELY ((d_size >= d_next_target) && (d_buckets == get_static_node() || d_chain_count > ((d_hash_mask + 1) >> 4))) // delay rehash as long as there are few chains rehash_on_next_target(d_size); } @@ -317,8 +499,7 @@ namespace seq auto th = node_type::tiny_hash(hash); size_t pos = (hash & d_hash_mask); - auto p = FindInsertNode( - d_chain_count, ChainAllocator{ this }, th, this->key_eq(), d_buckets + pos, d_values + pos, std::forward(key), std::forward(args)...); + auto p = FindInsertNode< Policy, true>(th, this->key_eq(), d_buckets + pos, d_values + pos, std::forward(key), std::forward(args)...); if (!p.second) { // Key exist: call functor std::forward(fun)(*p.first); @@ -484,14 +665,14 @@ namespace seq { size_t hash = hash_key(key); size_t pos = (hash & d_hash_mask); - return FindInNode(node_type::tiny_hash(hash), this->key_eq(), key, d_buckets + pos, d_values + pos, std::forward(f)); + return FindInNode(node_type::tiny_hash(hash), this->key_eq(), key, d_buckets + pos, d_values + pos, std::forward(f)); } template SEQ_ALWAYS_INLINE size_t visit(const K& key, F&& f) { size_t hash = hash_key(key); size_t pos = (hash & d_hash_mask); - return FindInNode(node_type::tiny_hash(hash), this->key_eq(), key, d_buckets + pos, d_values + pos, std::forward(f)); + return FindInNode(node_type::tiny_hash(hash), this->key_eq(), key, d_buckets + pos, d_values + pos, std::forward(f)); } SEQ_ALWAYS_INLINE bool contains_value(const Value& key_value) const @@ -579,7 +760,7 @@ namespace seq auto* d = values->right; do { // Look in dense node - auto found = FindWithTh(th, this->key_eq(), key, d->hashs, d->values()); + auto found = FindWithTh< chain_concurrent_node_size>(th, this->key_eq(), key, d->hashs, d->values()); if (found) { // Erase from dense node if functor returns true if (!std::forward(fun)(const_cast(*found))) @@ -607,7 +788,7 @@ namespace seq auto values = d_values + pos; auto bucket = d_buckets + pos; // Find in main bucket - auto found = FindWithTh(th, this->key_eq(), key, bucket->hashs, values->values()); + auto found = FindWithTh< max_concurrent_node_size>(th, this->key_eq(), key, bucket->hashs, values->values()); if (found) { // Erase from main bucket if functor returns true if (!std::forward(fun)(const_cast(*found))) diff --git a/benchs/testing.hpp b/benchs/testing.hpp new file mode 100644 index 00000000..07e8ea8c --- /dev/null +++ b/benchs/testing.hpp @@ -0,0 +1,544 @@ +/** + * MIT License + * + * Copyright (c) 2026 Victor Moncada + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef SEQ_TESTING_HPP +#define SEQ_TESTING_HPP + +#if defined(WIN32) || defined(_WIN32) +#include +#include +#else +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "seq/bits.hpp" +#include "seq/internal/utils.hpp" +#include "seq/timer.hpp" +#include "seq/legacy/format.hpp" + + +#ifdef max +#undef max +#endif + +#ifdef min +#undef min +#endif + +#ifdef __clang__ +// remove these warnings for testing +#pragma clang diagnostic ignored "-Wfloat-equal" +#pragma clang diagnostic ignored "-Wformat-nonliteral" +#endif + +namespace seq +{ + /// @brief Exception thrown for failed tests + class test_error : public std::runtime_error + { + public: + test_error(const std::string& str) + : std::runtime_error(str) + { + } + }; + + /// @brief Streambuf that stores the number of outputed characters + class streambuf_size : public std::streambuf + { + std::streambuf* sbuf{ nullptr }; + std::ostream* oss{ nullptr }; + size_t size{ 0 }; + + virtual int overflow(int c) override + { + size++; + return sbuf->sputc(static_cast(c)); + } + virtual int sync() override { return sbuf->pubsync(); } + + public: + streambuf_size(std::ostream& o) + : sbuf(o.rdbuf()) + , oss(&o) + { + oss->rdbuf(this); + } + virtual ~streambuf_size() noexcept override { oss->rdbuf(sbuf); } + size_t get_size() const { return size; } + }; + + namespace detail + { + template + std::string to_string(const T& value) + { + std::ostringstream ss; + ss << value; + return ss.str(); + } + } + +} + +/// @brief Very basic testing macro that throws seq::test_error if condition is not met. +#define SEQ_TEST(...) \ + do { \ + if (!(__VA_ARGS__)) { \ + throw seq::test_error("testing error at file " __FILE__ "(" + seq::detail::to_string(__LINE__) + "): " #__VA_ARGS__); \ + } \ + } while (false) +// if(! (__VA_ARGS__) ) {throw seq::test_error("testing error at file " __FILE__ "(" + seq::detail::to_string(__LINE__) + "): " #__VA_ARGS__); } + +/// @brief Test if writting given argument to a std::ostream produces the string 'result', throws seq::test_error if not. +#define SEQ_TEST_TO_OSTREAM(result, ...) \ + do { \ + std::ostringstream oss; \ + oss << (__VA_ARGS__); \ + oss.flush(); \ + if (oss.str() != result) { \ + std::string v = seq::detail::to_string(__LINE__); \ + throw seq::test_error(("testing error at file " __FILE__ "(" + v + "): \"" + std::string(result) + "\" == " #__VA_ARGS__).c_str()); \ + } \ + } while (false) + +/// @brief Test if given statement throws a 'exception' object. If not, throws seq::test_error. +#define SEQ_TEST_THROW(exception, ...) \ + do { \ + bool has_thrown = false; \ + try { \ + __VA_ARGS__; \ + } \ + catch (const exception&) { \ + has_thrown = true; \ + } \ + catch (...) { \ + } \ + if (!has_thrown) { \ + std::string v = seq::detail::to_string(__LINE__); \ + throw seq::test_error(("testing error at file " __FILE__ "(" + v + "): " #__VA_ARGS__).c_str()); \ + } \ + } while (false) + +/// @brief Test module +#define SEQ_TEST_MODULE(name, ...) \ + do { \ + seq::streambuf_size str(std::cout); \ + size_t size = 0; \ + bool ok = true; \ + try { \ + std::cout << "TEST MODULE " << #name << "... "; \ + std::cout.flush(); \ + size = str.get_size(); \ + __VA_ARGS__; \ + } \ + catch (const seq::test_error& e) { \ + std::cout << std::endl; \ + ok = false; \ + std::cerr << "TEST FAILURE IN MODULE " << #name << ": " << e.what() << std::endl; \ + } \ + catch (const std::exception& e) { \ + std::cout << std::endl; \ + ok = false; \ + std::cerr << "UNEXPECTED ERROR IN MODULE " << #name << " (std::exception): " << e.what() << std::endl; \ + } \ + catch (...) { \ + std::cout << std::endl; \ + ok = false; \ + std::cerr << "UNEXPECTED ERROR IN MODULE " << #name << std::endl; \ + } \ + if (ok) { \ + if (str.get_size() != size) \ + std::cout << std::endl; \ + std::cout << "SUCCESS" << std::endl; \ + } \ + } while (false) + +/// @brief Test module +#define SEQ_TEST_MODULE_RETURN(name, ret_value, ...) \ + do { \ + seq::streambuf_size str(std::cout); \ + size_t size = 0; \ + bool ok = true; \ + try { \ + std::cout << "TEST MODULE " << #name << "... "; \ + std::cout.flush(); \ + size = str.get_size(); \ + __VA_ARGS__; \ + } \ + catch (const seq::test_error& e) { \ + std::cout << std::endl; \ + ok = false; \ + std::cerr << "TEST FAILURE IN MODULE " << #name << ": " << e.what() << std::endl; \ + } \ + catch (const std::exception& e) { \ + std::cout << std::endl; \ + ok = false; \ + std::cerr << "UNEXPECTED ERROR IN MODULE " << #name << " (std::exception): " << e.what() << std::endl; \ + } \ + catch (...) { \ + std::cout << std::endl; \ + ok = false; \ + std::cerr << "UNEXPECTED ERROR IN MODULE " << #name << std::endl; \ + } \ + if (ok) { \ + if (str.get_size() != size) \ + std::cout << std::endl; \ + std::cout << "SUCCESS" << std::endl; \ + } \ + else \ + return ret_value; \ + } while (false) + +namespace seq +{ + namespace detail + { + static inline auto msecs_since_epoch() -> uint64_t + { + using namespace std::chrono; + return static_cast(duration_cast(system_clock::now().time_since_epoch()).count()); + } + + static inline timer& get_clock() + { + thread_local timer clock; + return clock; + } + } + + /// @brief For tests only, reset timer for calling thread + inline void tick() + { + + detail::get_clock().tick(); + } + + /// @brief For tests only, returns elapsed milliseconds since last call to tick() + inline auto tock_ms() -> std::uint64_t + { + return detail::get_clock().tock() / 1000000; + } + + /// @brief Similar to C++11 (and deprecated) std::random_shuffle + template + void random_shuffle(Iter begin, Iter end, uint_fast32_t seed = 0) + { + std::mt19937 g(seed); + if (seed) + g.seed(seed); + std::shuffle(begin, end, g); + } + + /// @brief Similar to C++14 std::equal + template + bool equal(Iter1 first, Iter1 last, Iter2 first2, Iter2 last2, BinaryPredicate pred) + { + if (first == last && first2 == last2) + return true; + if (first == last || first2 == last2) + return false; + + if constexpr (is_random_access_v && is_random_access_v) + if (std::distance(first, last) != std::distance(first2, last2)) + return false; + + for (; first != last; ++first, ++first2) { + if (first2 == last2 || !pred(*first, *first2)) + return false; + } + return true; + } + template + bool equal(Iter1 first, Iter1 last, Iter2 first2, Iter2 last2) + { + using T1 = typename std::iterator_traits::value_type; + using T2 = typename std::iterator_traits::value_type; + return seq::equal(first, last, first2, last2, [](const T1& a, const T2& b) { return a == b; }); + } + template + bool equal(Iter1 first, Iter1 last, Iter2 first2) + { + for (; first != last; ++first, ++first2) { + if (!(*first == *first2)) { + return false; + } + } + return true; + } + + inline void reset_memory_usage() + { +#if defined(WIN32) || defined(_WIN32) + SetProcessWorkingSetSize(GetCurrentProcess(), static_cast(-1), static_cast(-1)); +#endif + } + + inline auto get_memory_usage() -> size_t + { +#if defined(WIN32) || defined(_WIN32) + Sleep(50); + HANDLE currentProcessHandle = GetCurrentProcess(); + PROCESS_MEMORY_COUNTERS_EX memoryCounters; // = { 0 };; + memset(&memoryCounters, 0, sizeof(memoryCounters)); + if (GetProcessMemoryInfo(currentProcessHandle, reinterpret_cast(&memoryCounters), sizeof(memoryCounters))) { + return /*memoryCounters.PrivateUsage + */ memoryCounters.WorkingSetSize; + } + return 0; +#else + return 0; +#endif + } + + template> + struct basic_nullbuf : std::basic_streambuf + { + using base_type = std::basic_streambuf; + using int_type = typename base_type::int_type; + using traits_type = typename base_type::traits_type; + + virtual auto overflow(int_type c) -> int_type override { return traits_type::not_eof(c); } + }; + + /// @brief For tests only, alias for null buffer, to be used with c++ iostreams + using nullbuf = basic_nullbuf; + + struct disable_ostream + { + std::ostream* d_oss; + std::streambuf* d_buf; + nullbuf d_null; + disable_ostream(std::ostream& oss) + : d_oss(&oss) + { + d_buf = oss.rdbuf(); + oss.rdbuf(&d_null); + } + ~disable_ostream() { d_oss->rdbuf(d_buf); } + }; + + template + void print_null(const T& v) + { + static nullbuf n; + auto b = std::cout.rdbuf(&n); + std::cout << v << std::endl; + std::cout.rdbuf(b); + } + + /// @brief For tests only, generate a random string of given max size + template + auto generate_random_string(int max_size, bool fixed = false) -> String + { + using value_type = typename String::value_type; + size_t size = static_cast(fixed ? max_size : rand() % max_size); + String res(size, 0); + for (size_t i = 0; i < size; ++i) + res[i] = static_cast((rand() & 63) + 33); + + return res; + } + + namespace detail + { + template + struct Multiply + { + static auto multiply(T value) -> T { return static_cast((static_cast(rand()) + static_cast(rand())) * 14695981039346656037ULL * value); } + template + static auto read(Stream& str) -> T + { + T r; + seq::from_stream(str, r); + return r; + } + }; + template<> + struct Multiply + { + static auto multiply(long double value) -> long double + { + return static_cast((static_cast(rand()) + static_cast(rand()))) * 1.4695981039346656037 * value; + } + template + static auto read(Stream& str) -> long double + { + long double r; + seq::from_stream(str, r); + return r; + } + }; + template<> + struct Multiply + { + static auto multiply(double value) -> double { return static_cast((static_cast(rand()) + static_cast(rand()))) * 1.4695981039346656037 * value; } + template + static auto read(Stream& str) -> double + { + double r; + seq::from_stream(str, r); + return r; + } + }; + template<> + struct Multiply + { + static auto multiply(float value) -> float + { + return static_cast((static_cast(rand()) + static_cast(rand())) * 1.4695981039346656037 * static_cast(value)); + } + template + static auto read(Stream& str) -> float + { + float r; + seq::from_stream(str, r); + return r; + } + }; + } + + /// @brief For tests only, generate random floating point number on the whole representable range (including potential infinit values) + template + class random_float_genertor + { + static constexpr unsigned mask = sizeof(Float) == 4 ? 31 : sizeof(Float) == 8 ? 255 : 4095; + std::mt19937_64 d_rand; + unsigned count; + + unsigned get_rand() + { + return (d_rand()) & 0xFFFFFFFFu; + // return rand(); + } + + public: + random_float_genertor(unsigned seed = 0) + : d_rand(seed) + , count(0) + { + // srand(seed); + } + + auto operator()() -> Float + { + const bool type = get_rand() & 1; + Float sign1 = (get_rand() & 1) ? static_cast(-1) : static_cast(1); + Float res; + if (type) + res = (sign1 * static_cast(detail::Multiply::multiply(static_cast(count++ * static_cast(get_rand()))))); + else + res = (sign1 * static_cast(detail::Multiply::multiply(static_cast(count++ * static_cast(get_rand()))) * + std::pow(static_cast(10.), static_cast(sign1) * static_cast(static_cast(get_rand()) & mask)))); + + return res; + } + }; + + template + struct debug_allocator + { + typedef T value_type; + typedef T* pointer; + typedef const T* const_pointer; + typedef T& reference; + typedef const T& const_reference; + using size_type = size_t; + using difference_type = ptrdiff_t; + using is_always_equal = std::false_type; + using propagate_on_container_swap = std::true_type; + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + + template + struct rebind + { + using other = debug_allocator; + }; + + std::shared_ptr> d_count; + + debug_allocator() + : d_count(new std::atomic(0)) + { + } + debug_allocator(const debug_allocator& other) + : d_count(other.d_count) + { + } + template + debug_allocator(const debug_allocator& other) + : d_count(other.d_count) + { + } + ~debug_allocator() {} + debug_allocator& operator=(const debug_allocator& other) + { + d_count = other.d_count; + return *this; + } + + bool operator==(const debug_allocator& other) const { return d_count == other.d_count; } + bool operator!=(const debug_allocator& other) const { return d_count != other.d_count; } + + void deallocate(T* p, const size_t count) + { + std::allocator{}.deallocate(p, count); + (*d_count) -= count * sizeof(T); + // printf("deallocate %i elems (%i B) of type %s\n", (int)count, (int)(count * sizeof(T)), typeid(T).name()); + SEQ_ASSERT_DEBUG(*d_count >= 0, ""); + } + T* allocate(const size_t count) + { + T* p = std::allocator{}.allocate(count); + (*d_count) += count * sizeof(T); + // printf("allocate %i elems (%i B) of type %s\n", (int)count,(int)(count*sizeof(T)), typeid(T).name()); + return p; + } + T* allocate(const size_t count, const void*) { return allocate(count); } + size_t max_size() const noexcept { return static_cast(-1) / sizeof(T); } + }; + + template + std::int64_t get_alloc_bytes(const debug_allocator& al) + { + return *al.d_count; + } + template + std::int64_t get_alloc_bytes(const T&) + { + return 0; + } +} +#endif diff --git a/benchs/tsl/sparse_growth_policy.h b/benchs/tsl/sparse_growth_policy.h new file mode 100644 index 00000000..0b7622b5 --- /dev/null +++ b/benchs/tsl/sparse_growth_policy.h @@ -0,0 +1,341 @@ +/** + * MIT License + * + * Copyright (c) 2017 Thibaut Goetghebuer-Planchon + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef TSL_SPARSE_GROWTH_POLICY_H +#define TSL_SPARSE_GROWTH_POLICY_H + +#include +#include +#include +#include +#include +#include +#include +#include + + +#ifndef TSL_NO_EXCEPTIONS +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || \ + (defined(_MSC_VER) && defined(_CPPUNWIND))) +#define TSL_NO_EXCEPTIONS 0 +#else +#define TSL_NO_EXCEPTIONS 1 +#endif +#endif + +#if TSL_NO_EXCEPTIONS +#include +#ifdef TSL_DEBUG +#include +#define TSL_SH_THROW_OR_ABORT(ex, msg) \ + do { \ + std::fprintf(stderr, "error: %s\n", msg); \ + std::abort(); \ + } while (0) +#else +#define TSL_SH_THROW_OR_ABORT(ex, msg) std::abort() +#endif +#define TSL_SH_TRY if (true) +#define TSL_SH_CATCH(x) if (false) +#define TSL_SH_RETRHOW +#else +#include + +#define TSL_SH_THROW_OR_ABORT(ex, msg) throw ex(msg) +#define TSL_SH_TRY try +#define TSL_SH_CATCH(x) catch (x) +#define TSL_SH_RETRHOW throw +#endif + +namespace tsl { +namespace sh { + +/** + * Grow the hash table by a factor of GrowthFactor keeping the bucket count to a + * power of two. It allows the table to use a mask operation instead of a modulo + * operation to map a hash to a bucket. + * + * GrowthFactor must be a power of two >= 2. + */ +template +class power_of_two_growth_policy { + public: + /** + * Called on the hash table creation and on rehash. The number of buckets for + * the table is passed in parameter. This number is a minimum, the policy may + * update this value with a higher value if needed (but not lower). + * + * If 0 is given, min_bucket_count_in_out must still be 0 after the policy + * creation and bucket_for_hash must always return 0 in this case. + */ + explicit power_of_two_growth_policy(std::size_t &min_bucket_count_in_out) { + if (min_bucket_count_in_out > max_bucket_count()) { + TSL_SH_THROW_OR_ABORT(std::length_error, + "The hash table exceeds its maximum size."); + } + + if (min_bucket_count_in_out > 0) { + min_bucket_count_in_out = + round_up_to_power_of_two(min_bucket_count_in_out); + m_mask = min_bucket_count_in_out - 1; + } else { + m_mask = 0; + } + } + + /** + * Return the bucket [0, bucket_count()) to which the hash belongs. + * If bucket_count() is 0, it must always return 0. + */ + std::size_t bucket_for_hash(std::size_t hash) const noexcept { + return hash & m_mask; + } + + /** + * Return the number of buckets that should be used on next growth. + */ + std::size_t next_bucket_count() const { + if ((m_mask + 1) > max_bucket_count() / GrowthFactor) { + TSL_SH_THROW_OR_ABORT(std::length_error, + "The hash table exceeds its maximum size."); + } + + return (m_mask + 1) * GrowthFactor; + } + + /** + * Return the maximum number of buckets supported by the policy. + */ + std::size_t max_bucket_count() const { + // Largest power of two. + return (std::numeric_limits::max() / 2) + 1; + } + + /** + * Reset the growth policy as if it was created with a bucket count of 0. + * After a clear, the policy must always return 0 when bucket_for_hash is + * called. + */ + void clear() noexcept { m_mask = 0; } + + private: + static std::size_t round_up_to_power_of_two(std::size_t value) { + if (is_power_of_two(value)) { + return value; + } + + if (value == 0) { + return 1; + } + + --value; + for (std::size_t i = 1; i < sizeof(std::size_t) * CHAR_BIT; i *= 2) { + value |= value >> i; + } + + return value + 1; + } + + static constexpr bool is_power_of_two(std::size_t value) { + return value != 0 && (value & (value - 1)) == 0; + } + + protected: + static_assert(is_power_of_two(GrowthFactor) && GrowthFactor >= 2, + "GrowthFactor must be a power of two >= 2."); + + std::size_t m_mask; +}; + +/** + * Grow the hash table by GrowthFactor::num / GrowthFactor::den and use a modulo + * to map a hash to a bucket. Slower but it can be useful if you want a slower + * growth. + */ +template > +class mod_growth_policy { + public: + explicit mod_growth_policy(std::size_t &min_bucket_count_in_out) { + if (min_bucket_count_in_out > max_bucket_count()) { + TSL_SH_THROW_OR_ABORT(std::length_error, + "The hash table exceeds its maximum size."); + } + + if (min_bucket_count_in_out > 0) { + m_mod = min_bucket_count_in_out; + } else { + m_mod = 1; + } + } + + std::size_t bucket_for_hash(std::size_t hash) const noexcept { + return hash % m_mod; + } + + std::size_t next_bucket_count() const { + if (m_mod == max_bucket_count()) { + TSL_SH_THROW_OR_ABORT(std::length_error, + "The hash table exceeds its maximum size."); + } + + const double next_bucket_count = + std::ceil(double(m_mod) * REHASH_SIZE_MULTIPLICATION_FACTOR); + if (!std::isnormal(next_bucket_count)) { + TSL_SH_THROW_OR_ABORT(std::length_error, + "The hash table exceeds its maximum size."); + } + + if (next_bucket_count > double(max_bucket_count())) { + return max_bucket_count(); + } else { + return std::size_t(next_bucket_count); + } + } + + std::size_t max_bucket_count() const { return MAX_BUCKET_COUNT; } + + void clear() noexcept { m_mod = 1; } + + private: + static constexpr double REHASH_SIZE_MULTIPLICATION_FACTOR = + 1.0 * GrowthFactor::num / GrowthFactor::den; + static const std::size_t MAX_BUCKET_COUNT = + std::size_t(double(std::numeric_limits::max() / + REHASH_SIZE_MULTIPLICATION_FACTOR)); + + static_assert(REHASH_SIZE_MULTIPLICATION_FACTOR >= 1.1, + "Growth factor should be >= 1.1."); + + std::size_t m_mod; +}; + +/** + * Grow the hash table by using prime numbers as bucket count. Slower than + * tsl::sh::power_of_two_growth_policy in general but will probably distribute + * the values around better in the buckets with a poor hash function. + * + * To allow the compiler to optimize the modulo operation, a lookup table is + * used with constant primes numbers. + * + * With a switch the code would look like: + * \code + * switch(iprime) { // iprime is the current prime of the hash table + * case 0: hash % 5ul; + * break; + * case 1: hash % 17ul; + * break; + * case 2: hash % 29ul; + * break; + * ... + * } + * \endcode + * + * Due to the constant variable in the modulo the compiler is able to optimize + * the operation by a series of multiplications, substractions and shifts. + * + * The 'hash % 5' could become something like 'hash - (hash * 0xCCCCCCCD) >> 34) + * * 5' in a 64 bits environment. + */ +class prime_growth_policy { + public: + explicit prime_growth_policy(std::size_t &min_bucket_count_in_out) { + auto it_prime = std::lower_bound(primes().begin(), primes().end(), + min_bucket_count_in_out); + if (it_prime == primes().end()) { + TSL_SH_THROW_OR_ABORT(std::length_error, + "The hash table exceeds its maximum size."); + } + + m_iprime = + static_cast(std::distance(primes().begin(), it_prime)); + if (min_bucket_count_in_out > 0) { + min_bucket_count_in_out = *it_prime; + } else { + min_bucket_count_in_out = 0; + } + } + + std::size_t bucket_for_hash(std::size_t hash) const noexcept { + return mod_prime()[m_iprime](hash); + } + + std::size_t next_bucket_count() const { + if (m_iprime + 1 >= primes().size()) { + TSL_SH_THROW_OR_ABORT(std::length_error, + "The hash table exceeds its maximum size."); + } + + return primes()[m_iprime + 1]; + } + + std::size_t max_bucket_count() const { return primes().back(); } + + void clear() noexcept { m_iprime = 0; } + + private: + static const std::array &primes() { + static const std::array PRIMES = { + {1ul, 5ul, 17ul, 29ul, 37ul, + 53ul, 67ul, 79ul, 97ul, 131ul, + 193ul, 257ul, 389ul, 521ul, 769ul, + 1031ul, 1543ul, 2053ul, 3079ul, 6151ul, + 12289ul, 24593ul, 49157ul, 98317ul, 196613ul, + 393241ul, 786433ul, 1572869ul, 3145739ul, 6291469ul, + 12582917ul, 25165843ul, 50331653ul, 100663319ul, 201326611ul, + 402653189ul, 805306457ul, 1610612741ul, 3221225473ul, 4294967291ul}}; + + static_assert( + std::numeric_limits::max() >= PRIMES.size(), + "The type of m_iprime is not big enough."); + + return PRIMES; + } + + static const std::array &mod_prime() { + // MOD_PRIME[iprime](hash) returns hash % PRIMES[iprime]. This table allows + // for faster modulo as the compiler can optimize the modulo code better + // with a constant known at the compilation. + static const std::array MOD_PRIME = { + {&mod<0>, &mod<1>, &mod<2>, &mod<3>, &mod<4>, &mod<5>, &mod<6>, + &mod<7>, &mod<8>, &mod<9>, &mod<10>, &mod<11>, &mod<12>, &mod<13>, + &mod<14>, &mod<15>, &mod<16>, &mod<17>, &mod<18>, &mod<19>, &mod<20>, + &mod<21>, &mod<22>, &mod<23>, &mod<24>, &mod<25>, &mod<26>, &mod<27>, + &mod<28>, &mod<29>, &mod<30>, &mod<31>, &mod<32>, &mod<33>, &mod<34>, + &mod<35>, &mod<36>, &mod<37>, &mod<38>, &mod<39>}}; + + return MOD_PRIME; + } + + template + static std::size_t mod(std::size_t hash) { + return hash % primes()[IPrime]; + } + + private: + unsigned int m_iprime; +}; + +} // namespace sh +} // namespace tsl + +#endif diff --git a/benchs/tsl/sparse_hash.h b/benchs/tsl/sparse_hash.h new file mode 100644 index 00000000..0a6eaa0b --- /dev/null +++ b/benchs/tsl/sparse_hash.h @@ -0,0 +1,2236 @@ +/** + * MIT License + * + * Copyright (c) 2017 Thibaut Goetghebuer-Planchon + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef TSL_SPARSE_HASH_H +#define TSL_SPARSE_HASH_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sparse_growth_policy.h" + +#ifdef __INTEL_COMPILER +#include // For _popcnt32 and _popcnt64 +#endif + +#ifdef _MSC_VER +#include // For __cpuid, __popcnt and __popcnt64 +#endif + +#ifdef TSL_DEBUG +#define tsl_sh_assert(expr) assert(expr) +#else +#define tsl_sh_assert(expr) (static_cast(0)) +#endif + +namespace tsl { + +namespace sh { +enum class probing { linear, quadratic }; + +enum class exception_safety { basic, strong }; + +enum class sparsity { high, medium, low }; +} // namespace sh + +namespace detail_popcount { +/** + * Define the popcount(ll) methods and pick-up the best depending on the + * compiler. + */ + +// From Wikipedia: https://en.wikipedia.org/wiki/Hamming_weight +inline int fallback_popcountll(unsigned long long int x) { + static_assert( + sizeof(unsigned long long int) == sizeof(std::uint64_t), + "sizeof(unsigned long long int) must be equal to sizeof(std::uint64_t). " + "Open a feature request if you need support for a platform where it " + "isn't the case."); + + const std::uint64_t m1 = 0x5555555555555555ull; + const std::uint64_t m2 = 0x3333333333333333ull; + const std::uint64_t m4 = 0x0f0f0f0f0f0f0f0full; + const std::uint64_t h01 = 0x0101010101010101ull; + + x -= (x >> 1ull) & m1; + x = (x & m2) + ((x >> 2ull) & m2); + x = (x + (x >> 4ull)) & m4; + return static_cast((x * h01) >> (64ull - 8ull)); +} + +inline int fallback_popcount(unsigned int x) { + static_assert(sizeof(unsigned int) == sizeof(std::uint32_t) || + sizeof(unsigned int) == sizeof(std::uint64_t), + "sizeof(unsigned int) must be equal to sizeof(std::uint32_t) " + "or sizeof(std::uint64_t). " + "Open a feature request if you need support for a platform " + "where it isn't the case."); + + if (sizeof(unsigned int) == sizeof(std::uint32_t)) { + const std::uint32_t m1 = 0x55555555; + const std::uint32_t m2 = 0x33333333; + const std::uint32_t m4 = 0x0f0f0f0f; + const std::uint32_t h01 = 0x01010101; + + x -= (x >> 1) & m1; + x = (x & m2) + ((x >> 2) & m2); + x = (x + (x >> 4)) & m4; + return static_cast((x * h01) >> (32 - 8)); + } else { + return fallback_popcountll(x); + } +} + +#if defined(__clang__) || defined(__GNUC__) +inline int popcountll(unsigned long long int value) { + return __builtin_popcountll(value); +} + +inline int popcount(unsigned int value) { return __builtin_popcount(value); } + +#elif defined(_MSC_VER) +/** + * We need to check for popcount support at runtime on Windows with __cpuid + * See https://msdn.microsoft.com/en-us/library/bb385231.aspx + */ +inline bool has_popcount_support() { + int cpu_infos[4]; + __cpuid(cpu_infos, 1); + return (cpu_infos[2] & (1 << 23)) != 0; +} + +inline int popcountll(unsigned long long int value) { +#ifdef _WIN64 + static_assert( + sizeof(unsigned long long int) == sizeof(std::int64_t), + "sizeof(unsigned long long int) must be equal to sizeof(std::int64_t). "); + + static const bool has_popcount = has_popcount_support(); + return has_popcount + ? static_cast(__popcnt64(static_cast(value))) + : fallback_popcountll(value); +#else + return fallback_popcountll(value); +#endif +} + +inline int popcount(unsigned int value) { + static_assert(sizeof(unsigned int) == sizeof(std::int32_t), + "sizeof(unsigned int) must be equal to sizeof(std::int32_t). "); + + static const bool has_popcount = has_popcount_support(); + return has_popcount + ? static_cast(__popcnt(static_cast(value))) + : fallback_popcount(value); +} + +#elif defined(__INTEL_COMPILER) +inline int popcountll(unsigned long long int value) { + static_assert(sizeof(unsigned long long int) == sizeof(__int64), ""); + return _popcnt64(static_cast<__int64>(value)); +} + +inline int popcount(unsigned int value) { + return _popcnt32(static_cast(value)); +} + +#else +inline int popcountll(unsigned long long int x) { + return fallback_popcountll(x); +} + +inline int popcount(unsigned int x) { return fallback_popcount(x); } + +#endif +} // namespace detail_popcount + +namespace detail_sparse_hash { + +template +struct make_void { + using type = void; +}; + +template +struct has_is_transparent : std::false_type {}; + +template +struct has_is_transparent::type> + : std::true_type {}; + +template +struct is_power_of_two_policy : std::false_type {}; + +template +struct is_power_of_two_policy> + : std::true_type {}; + +inline constexpr bool is_power_of_two(std::size_t value) { + return value != 0 && (value & (value - 1)) == 0; +} + +inline std::size_t round_up_to_power_of_two(std::size_t value) { + if (is_power_of_two(value)) { + return value; + } + + if (value == 0) { + return 1; + } + + --value; + for (std::size_t i = 1; i < sizeof(std::size_t) * CHAR_BIT; i *= 2) { + value |= value >> i; + } + + return value + 1; +} + +template +static T numeric_cast(U value, + const char *error_message = "numeric_cast() failed.") { + T ret = static_cast(value); + if (static_cast(ret) != value) { + TSL_SH_THROW_OR_ABORT(std::runtime_error, error_message); + } + + const bool is_same_signedness = + (std::is_unsigned::value && std::is_unsigned::value) || + (std::is_signed::value && std::is_signed::value); + if (!is_same_signedness && (ret < T{}) != (value < U{})) { + TSL_SH_THROW_OR_ABORT(std::runtime_error, error_message); + } + + return ret; +} + +/** + * Fixed size type used to represent size_type values on serialization. Need to + * be big enough to represent a std::size_t on 32 and 64 bits platforms, and + * must be the same size on both platforms. + */ +using slz_size_type = std::uint64_t; +static_assert(std::numeric_limits::max() >= + std::numeric_limits::max(), + "slz_size_type must be >= std::size_t"); + +template +static T deserialize_value(Deserializer &deserializer) { + // MSVC < 2017 is not conformant, circumvent the problem by removing the + // template keyword +#if defined(_MSC_VER) && _MSC_VER < 1910 + return deserializer.Deserializer::operator()(); +#else + return deserializer.Deserializer::template operator()(); +#endif +} + +/** + * WARNING: the sparse_array class doesn't free the ressources allocated through + * the allocator passed in parameter in each method. You have to manually call + * `clear(Allocator&)` when you don't need a sparse_array object anymore. + * + * The reason is that the sparse_array doesn't store the allocator to avoid + * wasting space in each sparse_array when the allocator has a size > 0. It only + * allocates/deallocates objects with the allocator that is passed in parameter. + * + * + * + * Index denotes a value between [0, BITMAP_NB_BITS), it is an index similar to + * std::vector. Offset denotes the real position in `m_values` corresponding to + * an index. + * + * We are using raw pointers instead of std::vector to avoid loosing + * 2*sizeof(size_t) bytes to store the capacity and size of the vector in each + * sparse_array. We know we can only store up to BITMAP_NB_BITS elements in the + * array, we don't need such big types. + * + * + * T must be nothrow move constructible and/or copy constructible. + * Behaviour is undefined if the destructor of T throws an exception. + * + * See https://smerity.com/articles/2015/google_sparsehash.html for details on + * the idea behinds the implementation. + * + * TODO Check to use std::realloc and std::memmove when possible + */ +template +class sparse_array { + public: + using value_type = T; + using size_type = std::uint_least8_t; + using allocator_type = Allocator; + using iterator = value_type *; + using const_iterator = const value_type *; + + private: + static const size_type CAPACITY_GROWTH_STEP = + (Sparsity == tsl::sh::sparsity::high) ? 2 + : (Sparsity == tsl::sh::sparsity::medium) + ? 4 + : 8; // (Sparsity == tsl::sh::sparsity::low) + + /** + * Bitmap size configuration. + * Use 32 bits for the bitmap on 32-bits or less environnement as popcount on + * 64 bits numbers is slow on these environnement. Use 64 bits bitmap + * otherwise. + */ +#if SIZE_MAX <= UINT32_MAX + using bitmap_type = std::uint_least32_t; + static const std::size_t BITMAP_NB_BITS = 32; + static const std::size_t BUCKET_SHIFT = 5; +#else + using bitmap_type = std::uint_least64_t; + static const std::size_t BITMAP_NB_BITS = 64; + static const std::size_t BUCKET_SHIFT = 6; +#endif + + static const std::size_t BUCKET_MASK = BITMAP_NB_BITS - 1; + + static_assert(is_power_of_two(BITMAP_NB_BITS), + "BITMAP_NB_BITS must be a power of two."); + static_assert(std::numeric_limits::digits >= BITMAP_NB_BITS, + "bitmap_type must be able to hold at least BITMAP_NB_BITS."); + static_assert((std::size_t(1) << BUCKET_SHIFT) == BITMAP_NB_BITS, + "(1 << BUCKET_SHIFT) must be equal to BITMAP_NB_BITS."); + static_assert(std::numeric_limits::max() >= BITMAP_NB_BITS, + "size_type must be big enough to hold BITMAP_NB_BITS."); + static_assert(std::is_unsigned::value, + "bitmap_type must be unsigned."); + static_assert((std::numeric_limits::max() & BUCKET_MASK) == + BITMAP_NB_BITS - 1, + ""); + + public: + /** + * Map an ibucket [0, bucket_count) in the hash table to a sparse_ibucket + * (a sparse_array holds multiple buckets, so there is less sparse_array than + * bucket_count). + * + * The bucket ibucket is in + * m_sparse_buckets[sparse_ibucket(ibucket)][index_in_sparse_bucket(ibucket)] + * instead of something like m_buckets[ibucket] in a classical hash table. + */ + static std::size_t sparse_ibucket(std::size_t ibucket) { + return ibucket >> BUCKET_SHIFT; + } + + /** + * Map an ibucket [0, bucket_count) in the hash table to an index in the + * sparse_array which corresponds to the bucket. + * + * The bucket ibucket is in + * m_sparse_buckets[sparse_ibucket(ibucket)][index_in_sparse_bucket(ibucket)] + * instead of something like m_buckets[ibucket] in a classical hash table. + */ + static typename sparse_array::size_type index_in_sparse_bucket( + std::size_t ibucket) { + return static_cast( + ibucket & sparse_array::BUCKET_MASK); + } + + static std::size_t nb_sparse_buckets(std::size_t bucket_count) noexcept { + if (bucket_count == 0) { + return 0; + } + + return std::max( + 1, sparse_ibucket(tsl::detail_sparse_hash::round_up_to_power_of_two( + bucket_count))); + } + + public: + sparse_array() noexcept + : m_values(nullptr), + m_bitmap_vals(0), + m_bitmap_deleted_vals(0), + m_nb_elements(0), + m_capacity(0), + m_last_array(false) {} + + explicit sparse_array(bool last_bucket) noexcept + : m_values(nullptr), + m_bitmap_vals(0), + m_bitmap_deleted_vals(0), + m_nb_elements(0), + m_capacity(0), + m_last_array(last_bucket) {} + + sparse_array(size_type capacity, Allocator &alloc) + : m_values(nullptr), + m_bitmap_vals(0), + m_bitmap_deleted_vals(0), + m_nb_elements(0), + m_capacity(capacity), + m_last_array(false) { + if (m_capacity > 0) { + m_values = alloc.allocate(m_capacity); + tsl_sh_assert(m_values != + nullptr); // allocate should throw if there is a failure + } + } + + sparse_array(const sparse_array &other, Allocator &alloc) + : m_values(nullptr), + m_bitmap_vals(other.m_bitmap_vals), + m_bitmap_deleted_vals(other.m_bitmap_deleted_vals), + m_nb_elements(0), + m_capacity(other.m_capacity), + m_last_array(other.m_last_array) { + tsl_sh_assert(other.m_capacity >= other.m_nb_elements); + if (m_capacity == 0) { + return; + } + + m_values = alloc.allocate(m_capacity); + tsl_sh_assert(m_values != + nullptr); // allocate should throw if there is a failure + TSL_SH_TRY { + for (size_type i = 0; i < other.m_nb_elements; i++) { + construct_value(alloc, m_values + i, other.m_values[i]); + m_nb_elements++; + } + } + TSL_SH_CATCH(...) { + clear(alloc); + TSL_SH_RETRHOW; + } + } + + sparse_array(sparse_array &&other) noexcept + : m_values(other.m_values), + m_bitmap_vals(other.m_bitmap_vals), + m_bitmap_deleted_vals(other.m_bitmap_deleted_vals), + m_nb_elements(other.m_nb_elements), + m_capacity(other.m_capacity), + m_last_array(other.m_last_array) { + other.m_values = nullptr; + other.m_bitmap_vals = 0; + other.m_bitmap_deleted_vals = 0; + other.m_nb_elements = 0; + other.m_capacity = 0; + } + + sparse_array(sparse_array &&other, Allocator &alloc) + : m_values(nullptr), + m_bitmap_vals(other.m_bitmap_vals), + m_bitmap_deleted_vals(other.m_bitmap_deleted_vals), + m_nb_elements(0), + m_capacity(other.m_capacity), + m_last_array(other.m_last_array) { + tsl_sh_assert(other.m_capacity >= other.m_nb_elements); + if (m_capacity == 0) { + return; + } + + m_values = alloc.allocate(m_capacity); + tsl_sh_assert(m_values != + nullptr); // allocate should throw if there is a failure + TSL_SH_TRY { + for (size_type i = 0; i < other.m_nb_elements; i++) { + construct_value(alloc, m_values + i, std::move(other.m_values[i])); + m_nb_elements++; + } + } + TSL_SH_CATCH(...) { + clear(alloc); + TSL_SH_RETRHOW; + } + } + + sparse_array &operator=(const sparse_array &) = delete; + sparse_array &operator=(sparse_array &&) = delete; + + ~sparse_array() noexcept { + // The code that manages the sparse_array must have called clear before + // destruction. See documentation of sparse_array for more details. + tsl_sh_assert(m_capacity == 0 && m_nb_elements == 0 && m_values == nullptr); + } + + iterator begin() noexcept { return m_values; } + iterator end() noexcept { return m_values + m_nb_elements; } + const_iterator begin() const noexcept { return cbegin(); } + const_iterator end() const noexcept { return cend(); } + const_iterator cbegin() const noexcept { return m_values; } + const_iterator cend() const noexcept { return m_values + m_nb_elements; } + + bool empty() const noexcept { return m_nb_elements == 0; } + + size_type size() const noexcept { return m_nb_elements; } + + void clear(allocator_type &alloc) noexcept { + destroy_and_deallocate_values(alloc, m_values, m_nb_elements, m_capacity); + + m_values = nullptr; + m_bitmap_vals = 0; + m_bitmap_deleted_vals = 0; + m_nb_elements = 0; + m_capacity = 0; + } + + bool last() const noexcept { return m_last_array; } + + void set_as_last() noexcept { m_last_array = true; } + + bool has_value(size_type index) const noexcept { + tsl_sh_assert(index < BITMAP_NB_BITS); + return (m_bitmap_vals & (bitmap_type(1) << index)) != 0; + } + + bool has_deleted_value(size_type index) const noexcept { + tsl_sh_assert(index < BITMAP_NB_BITS); + return (m_bitmap_deleted_vals & (bitmap_type(1) << index)) != 0; + } + + iterator value(size_type index) noexcept { + tsl_sh_assert(has_value(index)); + return m_values + index_to_offset(index); + } + + const_iterator value(size_type index) const noexcept { + tsl_sh_assert(has_value(index)); + return m_values + index_to_offset(index); + } + + /** + * Return iterator to set value. + */ + template + iterator set(allocator_type &alloc, size_type index, Args &&...value_args) { + tsl_sh_assert(!has_value(index)); + + const size_type offset = index_to_offset(index); + insert_at_offset(alloc, offset, std::forward(value_args)...); + + m_bitmap_vals = (m_bitmap_vals | (bitmap_type(1) << index)); + m_bitmap_deleted_vals = + (m_bitmap_deleted_vals & ~(bitmap_type(1) << index)); + + m_nb_elements++; + + tsl_sh_assert(has_value(index)); + tsl_sh_assert(!has_deleted_value(index)); + + return m_values + offset; + } + + iterator erase(allocator_type &alloc, iterator position) { + const size_type offset = + static_cast(std::distance(begin(), position)); + return erase(alloc, position, offset_to_index(offset)); + } + + // Return the next value or end if no next value + iterator erase(allocator_type &alloc, iterator position, size_type index) { + tsl_sh_assert(has_value(index)); + tsl_sh_assert(!has_deleted_value(index)); + + const size_type offset = + static_cast(std::distance(begin(), position)); + erase_at_offset(alloc, offset); + + m_bitmap_vals = (m_bitmap_vals & ~(bitmap_type(1) << index)); + m_bitmap_deleted_vals = (m_bitmap_deleted_vals | (bitmap_type(1) << index)); + + m_nb_elements--; + + tsl_sh_assert(!has_value(index)); + tsl_sh_assert(has_deleted_value(index)); + + return m_values + offset; + } + + void swap(sparse_array &other) { + using std::swap; + + swap(m_values, other.m_values); + swap(m_bitmap_vals, other.m_bitmap_vals); + swap(m_bitmap_deleted_vals, other.m_bitmap_deleted_vals); + swap(m_nb_elements, other.m_nb_elements); + swap(m_capacity, other.m_capacity); + swap(m_last_array, other.m_last_array); + } + + static iterator mutable_iterator(const_iterator pos) { + return const_cast(pos); + } + + template + void serialize(Serializer &serializer) const { + const slz_size_type sparse_bucket_size = m_nb_elements; + serializer(sparse_bucket_size); + + const slz_size_type bitmap_vals = m_bitmap_vals; + serializer(bitmap_vals); + + const slz_size_type bitmap_deleted_vals = m_bitmap_deleted_vals; + serializer(bitmap_deleted_vals); + + for (const value_type &value : *this) { + serializer(value); + } + } + + template + static sparse_array deserialize_hash_compatible(Deserializer &deserializer, + Allocator &alloc) { + const slz_size_type sparse_bucket_size = + deserialize_value(deserializer); + const slz_size_type bitmap_vals = + deserialize_value(deserializer); + const slz_size_type bitmap_deleted_vals = + deserialize_value(deserializer); + + if (sparse_bucket_size > BITMAP_NB_BITS) { + TSL_SH_THROW_OR_ABORT( + std::runtime_error, + "Deserialized sparse_bucket_size is too big for the platform. " + "Maximum should be BITMAP_NB_BITS."); + } + + sparse_array sarray; + if (sparse_bucket_size == 0) { + return sarray; + } + + sarray.m_bitmap_vals = numeric_cast( + bitmap_vals, "Deserialized bitmap_vals is too big."); + sarray.m_bitmap_deleted_vals = numeric_cast( + bitmap_deleted_vals, "Deserialized bitmap_deleted_vals is too big."); + + sarray.m_capacity = numeric_cast( + sparse_bucket_size, "Deserialized sparse_bucket_size is too big."); + sarray.m_values = alloc.allocate(sarray.m_capacity); + + TSL_SH_TRY { + for (size_type ivalue = 0; ivalue < sarray.m_capacity; ivalue++) { + construct_value(alloc, sarray.m_values + ivalue, + deserialize_value(deserializer)); + sarray.m_nb_elements++; + } + } + TSL_SH_CATCH(...) { + sarray.clear(alloc); + TSL_SH_RETRHOW; + } + + return sarray; + } + + /** + * Deserialize the values of the bucket and insert them all in sparse_hash + * through sparse_hash.insert(...). + */ + template + static void deserialize_values_into_sparse_hash(Deserializer &deserializer, + SparseHash &sparse_hash) { + const slz_size_type sparse_bucket_size = + deserialize_value(deserializer); + + const slz_size_type bitmap_vals = + deserialize_value(deserializer); + static_cast(bitmap_vals); // Ignore, not needed + + const slz_size_type bitmap_deleted_vals = + deserialize_value(deserializer); + static_cast(bitmap_deleted_vals); // Ignore, not needed + + for (slz_size_type ivalue = 0; ivalue < sparse_bucket_size; ivalue++) { + sparse_hash.insert(deserialize_value(deserializer)); + } + } + + private: + template + static void construct_value(allocator_type &alloc, value_type *value, + Args &&...value_args) { + std::allocator_traits::construct( + alloc, value, std::forward(value_args)...); + } + + static void destroy_value(allocator_type &alloc, value_type *value) noexcept { + std::allocator_traits::destroy(alloc, value); + } + + static void destroy_and_deallocate_values( + allocator_type &alloc, value_type *values, size_type nb_values, + size_type capacity_values) noexcept { + for (size_type i = 0; i < nb_values; i++) { + destroy_value(alloc, values + i); + } + + alloc.deallocate(values, capacity_values); + } + + static size_type popcount(bitmap_type val) noexcept { + if (sizeof(bitmap_type) <= sizeof(unsigned int)) { + return static_cast( + tsl::detail_popcount::popcount(static_cast(val))); + } else { + return static_cast(tsl::detail_popcount::popcountll(val)); + } + } + + size_type index_to_offset(size_type index) const noexcept { + tsl_sh_assert(index < BITMAP_NB_BITS); + return popcount(m_bitmap_vals & + ((bitmap_type(1) << index) - bitmap_type(1))); + } + + // TODO optimize + size_type offset_to_index(size_type offset) const noexcept { + tsl_sh_assert(offset < m_nb_elements); + + bitmap_type bitmap_vals = m_bitmap_vals; + size_type index = 0; + size_type nb_ones = 0; + + while (bitmap_vals != 0) { + if ((bitmap_vals & 0x1) == 1) { + if (nb_ones == offset) { + break; + } + + nb_ones++; + } + + index++; + bitmap_vals = bitmap_vals >> 1; + } + + return index; + } + + size_type next_capacity() const noexcept { + return static_cast(m_capacity + CAPACITY_GROWTH_STEP); + } + + /** + * Insertion + * + * Two situations: + * - Either we are in a situation where + * std::is_nothrow_move_constructible::value is true. In this + * case, on insertion we just reallocate m_values when we reach its capacity + * (i.e. m_nb_elements == m_capacity), otherwise we just put the new value at + * its appropriate place. We can easily keep the strong exception guarantee as + * moving the values around is safe. + * - Otherwise we are in a situation where + * std::is_nothrow_move_constructible::value is false. In this + * case on EACH insertion we allocate a new area of m_nb_elements + 1 where we + * copy the values of m_values into it and put the new value there. On + * success, we set m_values to this new area. Even if slower, it's the only + * way to preserve to strong exception guarantee. + */ + template ::value>::type * = nullptr> + void insert_at_offset(allocator_type &alloc, size_type offset, + Args &&...value_args) { + if (m_nb_elements < m_capacity) { + insert_at_offset_no_realloc(alloc, offset, + std::forward(value_args)...); + } else { + insert_at_offset_realloc(alloc, offset, next_capacity(), + std::forward(value_args)...); + } + } + + template ::value>::type * = nullptr> + void insert_at_offset(allocator_type &alloc, size_type offset, + Args &&...value_args) { + insert_at_offset_realloc(alloc, offset, m_nb_elements + 1, + std::forward(value_args)...); + } + + template ::value>::type * = nullptr> + void insert_at_offset_no_realloc(allocator_type &alloc, size_type offset, + Args &&...value_args) { + tsl_sh_assert(offset <= m_nb_elements); + tsl_sh_assert(m_nb_elements < m_capacity); + + for (size_type i = m_nb_elements; i > offset; i--) { + construct_value(alloc, m_values + i, std::move(m_values[i - 1])); + destroy_value(alloc, m_values + i - 1); + } + + TSL_SH_TRY { + construct_value(alloc, m_values + offset, + std::forward(value_args)...); + } + TSL_SH_CATCH(...) { + for (size_type i = offset; i < m_nb_elements; i++) { + construct_value(alloc, m_values + i, std::move(m_values[i + 1])); + destroy_value(alloc, m_values + i + 1); + } + TSL_SH_RETRHOW; + } + } + + template ::value>::type * = nullptr> + void insert_at_offset_realloc(allocator_type &alloc, size_type offset, + size_type new_capacity, Args &&...value_args) { + tsl_sh_assert(new_capacity > m_nb_elements); + + value_type *new_values = alloc.allocate(new_capacity); + // Allocate should throw if there is a failure + tsl_sh_assert(new_values != nullptr); + + TSL_SH_TRY { + construct_value(alloc, new_values + offset, + std::forward(value_args)...); + } + TSL_SH_CATCH(...) { + alloc.deallocate(new_values, new_capacity); + TSL_SH_RETRHOW; + } + + // Should not throw from here + for (size_type i = 0; i < offset; i++) { + construct_value(alloc, new_values + i, std::move(m_values[i])); + } + + for (size_type i = offset; i < m_nb_elements; i++) { + construct_value(alloc, new_values + i + 1, std::move(m_values[i])); + } + + destroy_and_deallocate_values(alloc, m_values, m_nb_elements, m_capacity); + + m_values = new_values; + m_capacity = new_capacity; + } + + template ::value>::type * = nullptr> + void insert_at_offset_realloc(allocator_type &alloc, size_type offset, + size_type new_capacity, Args &&...value_args) { + tsl_sh_assert(new_capacity > m_nb_elements); + + value_type *new_values = alloc.allocate(new_capacity); + // Allocate should throw if there is a failure + tsl_sh_assert(new_values != nullptr); + + size_type nb_new_values = 0; + TSL_SH_TRY { + for (size_type i = 0; i < offset; i++) { + construct_value(alloc, new_values + i, m_values[i]); + nb_new_values++; + } + + construct_value(alloc, new_values + offset, + std::forward(value_args)...); + nb_new_values++; + + for (size_type i = offset; i < m_nb_elements; i++) { + construct_value(alloc, new_values + i + 1, m_values[i]); + nb_new_values++; + } + } + TSL_SH_CATCH(...) { + destroy_and_deallocate_values(alloc, new_values, nb_new_values, + new_capacity); + TSL_SH_RETRHOW; + } + + tsl_sh_assert(nb_new_values == m_nb_elements + 1); + + destroy_and_deallocate_values(alloc, m_values, m_nb_elements, m_capacity); + + m_values = new_values; + m_capacity = new_capacity; + } + + /** + * Erasure + * + * Two situations: + * - Either we are in a situation where + * std::is_nothrow_move_constructible::value is true. Simply + * destroy the value and left-shift move the value on the right of offset. + * - Otherwise we are in a situation where + * std::is_nothrow_move_constructible::value is false. Copy all + * the values except the one at offset into a new heap area. On success, we + * set m_values to this new area. Even if slower, it's the only way to + * preserve to strong exception guarantee. + */ + template ::value>::type * = nullptr> + void erase_at_offset(allocator_type &alloc, size_type offset) noexcept { + tsl_sh_assert(offset < m_nb_elements); + + destroy_value(alloc, m_values + offset); + + for (size_type i = offset + 1; i < m_nb_elements; i++) { + construct_value(alloc, m_values + i - 1, std::move(m_values[i])); + destroy_value(alloc, m_values + i); + } + } + + template ::value>::type * = nullptr> + void erase_at_offset(allocator_type &alloc, size_type offset) { + tsl_sh_assert(offset < m_nb_elements); + + // Erasing the last element, don't need to reallocate. We keep the capacity. + if (offset + 1 == m_nb_elements) { + destroy_value(alloc, m_values + offset); + return; + } + + tsl_sh_assert(m_nb_elements > 1); + const size_type new_capacity = m_nb_elements - 1; + + value_type *new_values = alloc.allocate(new_capacity); + // Allocate should throw if there is a failure + tsl_sh_assert(new_values != nullptr); + + size_type nb_new_values = 0; + TSL_SH_TRY { + for (size_type i = 0; i < m_nb_elements; i++) { + if (i != offset) { + construct_value(alloc, new_values + nb_new_values, m_values[i]); + nb_new_values++; + } + } + } + TSL_SH_CATCH(...) { + destroy_and_deallocate_values(alloc, new_values, nb_new_values, + new_capacity); + TSL_SH_RETRHOW; + } + + tsl_sh_assert(nb_new_values == m_nb_elements - 1); + + destroy_and_deallocate_values(alloc, m_values, m_nb_elements, m_capacity); + + m_values = new_values; + m_capacity = new_capacity; + } + + private: + value_type *m_values; + + bitmap_type m_bitmap_vals; + bitmap_type m_bitmap_deleted_vals; + + size_type m_nb_elements; + size_type m_capacity; + bool m_last_array; +}; + +/** + * Internal common class used by `sparse_map` and `sparse_set`. + * + * `ValueType` is what will be stored by `sparse_hash` (usually `std::pair` for map and `Key` for set). + * + * `KeySelect` should be a `FunctionObject` which takes a `ValueType` in + * parameter and returns a reference to the key. + * + * `ValueSelect` should be a `FunctionObject` which takes a `ValueType` in + * parameter and returns a reference to the value. `ValueSelect` should be void + * if there is no value (in a set for example). + * + * The strong exception guarantee only holds if `ExceptionSafety` is set to + * `tsl::sh::exception_safety::strong`. + * + * `ValueType` must be nothrow move constructible and/or copy constructible. + * Behaviour is undefined if the destructor of `ValueType` throws. + * + * + * The class holds its buckets in a 2-dimensional fashion. Instead of having a + * linear `std::vector` for [0, bucket_count) where each bucket stores + * one value, we have a `std::vector` (m_sparse_buckets_data) + * where each `sparse_array` stores multiple values (up to + * `sparse_array::BITMAP_NB_BITS`). To convert a one dimensional `ibucket` + * position to a position in `std::vector` and a position in + * `sparse_array`, use respectively the methods + * `sparse_array::sparse_ibucket(ibucket)` and + * `sparse_array::index_in_sparse_bucket(ibucket)`. + */ +template +class sparse_hash : private Allocator, + private Hash, + private KeyEqual, + private GrowthPolicy { + private: + template + using has_mapped_type = + typename std::integral_constant::value>; + + static_assert( + noexcept(std::declval().bucket_for_hash(std::size_t(0))), + "GrowthPolicy::bucket_for_hash must be noexcept."); + static_assert(noexcept(std::declval().clear()), + "GrowthPolicy::clear must be noexcept."); + + public: + template + class sparse_iterator; + + using key_type = typename KeySelect::key_type; + using value_type = ValueType; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using hasher = Hash; + using key_equal = KeyEqual; + using allocator_type = Allocator; + using reference = value_type &; + using const_reference = const value_type &; + using pointer = value_type *; + using const_pointer = const value_type *; + using iterator = sparse_iterator; + using const_iterator = sparse_iterator; + + private: + using sparse_array = + tsl::detail_sparse_hash::sparse_array; + + using sparse_buckets_allocator = typename std::allocator_traits< + allocator_type>::template rebind_alloc; + using sparse_buckets_container = + std::vector; + + public: + /** + * The `operator*()` and `operator->()` methods return a const reference and + * const pointer respectively to the stored value type (`Key` for a set, + * `std::pair` for a map). + * + * In case of a map, to get a mutable reference to the value `T` associated to + * a key (the `.second` in the stored pair), you have to call `value()`. + */ + template + class sparse_iterator { + friend class sparse_hash; + + private: + using sparse_bucket_iterator = typename std::conditional< + IsConst, typename sparse_buckets_container::const_iterator, + typename sparse_buckets_container::iterator>::type; + + using sparse_array_iterator = + typename std::conditional::type; + + /** + * sparse_array_it should be nullptr if sparse_bucket_it == + * m_sparse_buckets_data.end(). (TODO better way?) + */ + sparse_iterator(sparse_bucket_iterator sparse_bucket_it, + sparse_array_iterator sparse_array_it) + : m_sparse_buckets_it(sparse_bucket_it), + m_sparse_array_it(sparse_array_it) {} + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = const typename sparse_hash::value_type; + using difference_type = std::ptrdiff_t; + using reference = value_type &; + using pointer = value_type *; + + sparse_iterator() noexcept {} + + // Copy constructor from iterator to const_iterator. + template ::type * = nullptr> + sparse_iterator(const sparse_iterator &other) noexcept + : m_sparse_buckets_it(other.m_sparse_buckets_it), + m_sparse_array_it(other.m_sparse_array_it) {} + + sparse_iterator(const sparse_iterator &other) = default; + sparse_iterator(sparse_iterator &&other) = default; + sparse_iterator &operator=(const sparse_iterator &other) = default; + sparse_iterator &operator=(sparse_iterator &&other) = default; + + const typename sparse_hash::key_type &key() const { + return KeySelect()(*m_sparse_array_it); + } + + template ::value && + IsConst>::type * = nullptr> + const typename U::value_type &value() const { + return U()(*m_sparse_array_it); + } + + template ::value && + !IsConst>::type * = nullptr> + typename U::value_type &value() { + return U()(*m_sparse_array_it); + } + + reference operator*() const { return *m_sparse_array_it; } + + pointer operator->() const { return std::addressof(*m_sparse_array_it); } + + sparse_iterator &operator++() { + tsl_sh_assert(m_sparse_array_it != nullptr); + ++m_sparse_array_it; + + if (m_sparse_array_it == m_sparse_buckets_it->end()) { + do { + if (m_sparse_buckets_it->last()) { + ++m_sparse_buckets_it; + m_sparse_array_it = nullptr; + return *this; + } + + ++m_sparse_buckets_it; + } while (m_sparse_buckets_it->empty()); + + m_sparse_array_it = m_sparse_buckets_it->begin(); + } + + return *this; + } + + sparse_iterator operator++(int) { + sparse_iterator tmp(*this); + ++*this; + + return tmp; + } + + friend bool operator==(const sparse_iterator &lhs, + const sparse_iterator &rhs) { + return lhs.m_sparse_buckets_it == rhs.m_sparse_buckets_it && + lhs.m_sparse_array_it == rhs.m_sparse_array_it; + } + + friend bool operator!=(const sparse_iterator &lhs, + const sparse_iterator &rhs) { + return !(lhs == rhs); + } + + private: + sparse_bucket_iterator m_sparse_buckets_it; + sparse_array_iterator m_sparse_array_it; + }; + + public: + sparse_hash(size_type bucket_count, const Hash &hash, const KeyEqual &equal, + const Allocator &alloc, float max_load_factor) + : Allocator(alloc), + Hash(hash), + KeyEqual(equal), + GrowthPolicy(bucket_count), + m_sparse_buckets_data(alloc), + m_sparse_buckets(static_empty_sparse_bucket_ptr()), + m_bucket_count(bucket_count), + m_nb_elements(0), + m_nb_deleted_buckets(0) { + if (m_bucket_count > max_bucket_count()) { + TSL_SH_THROW_OR_ABORT(std::length_error, + "The map exceeds its maximum size."); + } + + if (m_bucket_count > 0) { + /* + * We can't use the `vector(size_type count, const Allocator& alloc)` + * constructor as it's only available in C++14 and we need to support + * C++11. We thus must resize after using the `vector(const Allocator& + * alloc)` constructor. + * + * We can't use `vector(size_type count, const T& value, const Allocator& + * alloc)` as it requires the value T to be copyable. + */ + m_sparse_buckets_data.resize( + sparse_array::nb_sparse_buckets(bucket_count)); + m_sparse_buckets = m_sparse_buckets_data.data(); + + tsl_sh_assert(!m_sparse_buckets_data.empty()); + m_sparse_buckets_data.back().set_as_last(); + } + + this->max_load_factor(max_load_factor); + + // Check in the constructor instead of outside of a function to avoid + // compilation issues when value_type is not complete. + static_assert(std::is_nothrow_move_constructible::value || + std::is_copy_constructible::value, + "Key, and T if present, must be nothrow move constructible " + "and/or copy constructible."); + } + + ~sparse_hash() { clear(); } + + sparse_hash(const sparse_hash &other) + : Allocator(std::allocator_traits< + Allocator>::select_on_container_copy_construction(other)), + Hash(other), + KeyEqual(other), + GrowthPolicy(other), + m_sparse_buckets_data( + std::allocator_traits< + Allocator>::select_on_container_copy_construction(other)), + m_bucket_count(other.m_bucket_count), + m_nb_elements(other.m_nb_elements), + m_nb_deleted_buckets(other.m_nb_deleted_buckets), + m_load_threshold_rehash(other.m_load_threshold_rehash), + m_load_threshold_clear_deleted(other.m_load_threshold_clear_deleted), + m_max_load_factor(other.m_max_load_factor) { + copy_buckets_from(other), + m_sparse_buckets = m_sparse_buckets_data.empty() + ? static_empty_sparse_bucket_ptr() + : m_sparse_buckets_data.data(); + } + + sparse_hash(sparse_hash &&other) noexcept( + std::is_nothrow_move_constructible::value + &&std::is_nothrow_move_constructible::value + &&std::is_nothrow_move_constructible::value + &&std::is_nothrow_move_constructible::value + &&std::is_nothrow_move_constructible< + sparse_buckets_container>::value) + : Allocator(std::move(other)), + Hash(std::move(other)), + KeyEqual(std::move(other)), + GrowthPolicy(std::move(other)), + m_sparse_buckets_data(std::move(other.m_sparse_buckets_data)), + m_sparse_buckets(m_sparse_buckets_data.empty() + ? static_empty_sparse_bucket_ptr() + : m_sparse_buckets_data.data()), + m_bucket_count(other.m_bucket_count), + m_nb_elements(other.m_nb_elements), + m_nb_deleted_buckets(other.m_nb_deleted_buckets), + m_load_threshold_rehash(other.m_load_threshold_rehash), + m_load_threshold_clear_deleted(other.m_load_threshold_clear_deleted), + m_max_load_factor(other.m_max_load_factor) { + other.GrowthPolicy::clear(); + other.m_sparse_buckets_data.clear(); + other.m_sparse_buckets = static_empty_sparse_bucket_ptr(); + other.m_bucket_count = 0; + other.m_nb_elements = 0; + other.m_nb_deleted_buckets = 0; + other.m_load_threshold_rehash = 0; + other.m_load_threshold_clear_deleted = 0; + } + + sparse_hash &operator=(const sparse_hash &other) { + if (this != &other) { + clear(); + + if (std::allocator_traits< + Allocator>::propagate_on_container_copy_assignment::value) { + Allocator::operator=(other); + } + + Hash::operator=(other); + KeyEqual::operator=(other); + GrowthPolicy::operator=(other); + + if (std::allocator_traits< + Allocator>::propagate_on_container_copy_assignment::value) { + m_sparse_buckets_data = + sparse_buckets_container(static_cast(other)); + } else { + if (m_sparse_buckets_data.size() != + other.m_sparse_buckets_data.size()) { + m_sparse_buckets_data = + sparse_buckets_container(static_cast(*this)); + } else { + m_sparse_buckets_data.clear(); + } + } + + copy_buckets_from(other); + m_sparse_buckets = m_sparse_buckets_data.empty() + ? static_empty_sparse_bucket_ptr() + : m_sparse_buckets_data.data(); + + m_bucket_count = other.m_bucket_count; + m_nb_elements = other.m_nb_elements; + m_nb_deleted_buckets = other.m_nb_deleted_buckets; + m_load_threshold_rehash = other.m_load_threshold_rehash; + m_load_threshold_clear_deleted = other.m_load_threshold_clear_deleted; + m_max_load_factor = other.m_max_load_factor; + } + + return *this; + } + + sparse_hash &operator=(sparse_hash &&other) { + clear(); + + if (std::allocator_traits< + Allocator>::propagate_on_container_move_assignment::value) { + static_cast(*this) = + std::move(static_cast(other)); + m_sparse_buckets_data = std::move(other.m_sparse_buckets_data); + } else if (static_cast(*this) != + static_cast(other)) { + move_buckets_from(std::move(other)); + } else { + static_cast(*this) = + std::move(static_cast(other)); + m_sparse_buckets_data = std::move(other.m_sparse_buckets_data); + } + + m_sparse_buckets = m_sparse_buckets_data.empty() + ? static_empty_sparse_bucket_ptr() + : m_sparse_buckets_data.data(); + + static_cast(*this) = std::move(static_cast(other)); + static_cast(*this) = std::move(static_cast(other)); + static_cast(*this) = + std::move(static_cast(other)); + m_bucket_count = other.m_bucket_count; + m_nb_elements = other.m_nb_elements; + m_nb_deleted_buckets = other.m_nb_deleted_buckets; + m_load_threshold_rehash = other.m_load_threshold_rehash; + m_load_threshold_clear_deleted = other.m_load_threshold_clear_deleted; + m_max_load_factor = other.m_max_load_factor; + + other.GrowthPolicy::clear(); + other.m_sparse_buckets_data.clear(); + other.m_sparse_buckets = static_empty_sparse_bucket_ptr(); + other.m_bucket_count = 0; + other.m_nb_elements = 0; + other.m_nb_deleted_buckets = 0; + other.m_load_threshold_rehash = 0; + other.m_load_threshold_clear_deleted = 0; + + return *this; + } + + allocator_type get_allocator() const { + return static_cast(*this); + } + + /* + * Iterators + */ + iterator begin() noexcept { + auto begin = m_sparse_buckets_data.begin(); + while (begin != m_sparse_buckets_data.end() && begin->empty()) { + ++begin; + } + + return iterator(begin, (begin != m_sparse_buckets_data.end()) + ? begin->begin() + : nullptr); + } + + const_iterator begin() const noexcept { return cbegin(); } + + const_iterator cbegin() const noexcept { + auto begin = m_sparse_buckets_data.cbegin(); + while (begin != m_sparse_buckets_data.cend() && begin->empty()) { + ++begin; + } + + return const_iterator(begin, (begin != m_sparse_buckets_data.cend()) + ? begin->cbegin() + : nullptr); + } + + iterator end() noexcept { + return iterator(m_sparse_buckets_data.end(), nullptr); + } + + const_iterator end() const noexcept { return cend(); } + + const_iterator cend() const noexcept { + return const_iterator(m_sparse_buckets_data.cend(), nullptr); + } + + /* + * Capacity + */ + bool empty() const noexcept { return m_nb_elements == 0; } + + size_type size() const noexcept { return m_nb_elements; } + + size_type max_size() const noexcept { + return std::min(std::allocator_traits::max_size(), + m_sparse_buckets_data.max_size()); + } + + /* + * Modifiers + */ + void clear() noexcept { + for (auto &bucket : m_sparse_buckets_data) { + bucket.clear(*this); + } + + m_nb_elements = 0; + m_nb_deleted_buckets = 0; + } + + template + std::pair insert(P &&value) { + return insert_impl(KeySelect()(value), std::forward

(value)); + } + + template + iterator insert_hint(const_iterator hint, P &&value) { + if (hint != cend() && + compare_keys(KeySelect()(*hint), KeySelect()(value))) { + return mutable_iterator(hint); + } + + return insert(std::forward

(value)).first; + } + + template + void insert(InputIt first, InputIt last) { + if (std::is_base_of< + std::forward_iterator_tag, + typename std::iterator_traits::iterator_category>::value) { + const auto nb_elements_insert = std::distance(first, last); + const size_type nb_free_buckets = m_load_threshold_rehash - size(); + tsl_sh_assert(m_load_threshold_rehash >= size()); + + if (nb_elements_insert > 0 && + nb_free_buckets < size_type(nb_elements_insert)) { + reserve(size() + size_type(nb_elements_insert)); + } + } + + for (; first != last; ++first) { + insert(*first); + } + } + + template + std::pair insert_or_assign(K &&key, M &&obj) { + auto it = try_emplace(std::forward(key), std::forward(obj)); + if (!it.second) { + it.first.value() = std::forward(obj); + } + + return it; + } + + template + iterator insert_or_assign(const_iterator hint, K &&key, M &&obj) { + if (hint != cend() && compare_keys(KeySelect()(*hint), key)) { + auto it = mutable_iterator(hint); + it.value() = std::forward(obj); + + return it; + } + + return insert_or_assign(std::forward(key), std::forward(obj)).first; + } + + template + std::pair emplace(Args &&...args) { + return insert(value_type(std::forward(args)...)); + } + + template + iterator emplace_hint(const_iterator hint, Args &&...args) { + return insert_hint(hint, value_type(std::forward(args)...)); + } + + template + std::pair try_emplace(K &&key, Args &&...args) { + return insert_impl(key, std::piecewise_construct, + std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(args)...)); + } + + template + iterator try_emplace_hint(const_iterator hint, K &&key, Args &&...args) { + if (hint != cend() && compare_keys(KeySelect()(*hint), key)) { + return mutable_iterator(hint); + } + + return try_emplace(std::forward(key), std::forward(args)...).first; + } + + /** + * Here to avoid `template size_type erase(const K& key)` being used + * when we use an iterator instead of a const_iterator. + */ + iterator erase(iterator pos) { + tsl_sh_assert(pos != end() && m_nb_elements > 0); + auto it_sparse_array_next = + pos.m_sparse_buckets_it->erase(*this, pos.m_sparse_array_it); + m_nb_elements--; + m_nb_deleted_buckets++; + + if (it_sparse_array_next == pos.m_sparse_buckets_it->end()) { + auto it_sparse_buckets_next = pos.m_sparse_buckets_it; + do { + ++it_sparse_buckets_next; + } while (it_sparse_buckets_next != m_sparse_buckets_data.end() && + it_sparse_buckets_next->empty()); + + if (it_sparse_buckets_next == m_sparse_buckets_data.end()) { + return end(); + } else { + return iterator(it_sparse_buckets_next, + it_sparse_buckets_next->begin()); + } + } else { + return iterator(pos.m_sparse_buckets_it, it_sparse_array_next); + } + } + + iterator erase(const_iterator pos) { return erase(mutable_iterator(pos)); } + + iterator erase(const_iterator first, const_iterator last) { + if (first == last) { + return mutable_iterator(first); + } + + // TODO Optimize, could avoid the call to std::distance. + const size_type nb_elements_to_erase = + static_cast(std::distance(first, last)); + auto to_delete = mutable_iterator(first); + for (size_type i = 0; i < nb_elements_to_erase; i++) { + to_delete = erase(to_delete); + } + + return to_delete; + } + + template + size_type erase(const K &key) { + return erase(key, hash_key(key)); + } + + template + size_type erase(const K &key, std::size_t hash) { + return erase_impl(key, hash); + } + + void swap(sparse_hash &other) { + using std::swap; + + if (std::allocator_traits::propagate_on_container_swap::value) { + swap(static_cast(*this), static_cast(other)); + } else { + tsl_sh_assert(static_cast(*this) == + static_cast(other)); + } + + swap(static_cast(*this), static_cast(other)); + swap(static_cast(*this), static_cast(other)); + swap(static_cast(*this), + static_cast(other)); + swap(m_sparse_buckets_data, other.m_sparse_buckets_data); + swap(m_sparse_buckets, other.m_sparse_buckets); + swap(m_bucket_count, other.m_bucket_count); + swap(m_nb_elements, other.m_nb_elements); + swap(m_nb_deleted_buckets, other.m_nb_deleted_buckets); + swap(m_load_threshold_rehash, other.m_load_threshold_rehash); + swap(m_load_threshold_clear_deleted, other.m_load_threshold_clear_deleted); + swap(m_max_load_factor, other.m_max_load_factor); + } + + /* + * Lookup + */ + template < + class K, class U = ValueSelect, + typename std::enable_if::value>::type * = nullptr> + typename U::value_type &at(const K &key) { + return at(key, hash_key(key)); + } + + template < + class K, class U = ValueSelect, + typename std::enable_if::value>::type * = nullptr> + typename U::value_type &at(const K &key, std::size_t hash) { + return const_cast( + static_cast(this)->at(key, hash)); + } + + template < + class K, class U = ValueSelect, + typename std::enable_if::value>::type * = nullptr> + const typename U::value_type &at(const K &key) const { + return at(key, hash_key(key)); + } + + template < + class K, class U = ValueSelect, + typename std::enable_if::value>::type * = nullptr> + const typename U::value_type &at(const K &key, std::size_t hash) const { + auto it = find(key, hash); + if (it != cend()) { + return it.value(); + } else { + TSL_SH_THROW_OR_ABORT(std::out_of_range, "Couldn't find key."); + } + } + + template < + class K, class U = ValueSelect, + typename std::enable_if::value>::type * = nullptr> + typename U::value_type &operator[](K &&key) { + return try_emplace(std::forward(key)).first.value(); + } + + template + bool contains(const K &key) const { + return contains(key, hash_key(key)); + } + + template + bool contains(const K &key, std::size_t hash) const { + return count(key, hash) != 0; + } + + template + size_type count(const K &key) const { + return count(key, hash_key(key)); + } + + template + size_type count(const K &key, std::size_t hash) const { + if (find(key, hash) != cend()) { + return 1; + } else { + return 0; + } + } + + template + iterator find(const K &key) { + return find_impl(key, hash_key(key)); + } + + template + iterator find(const K &key, std::size_t hash) { + return find_impl(key, hash); + } + + template + const_iterator find(const K &key) const { + return find_impl(key, hash_key(key)); + } + + template + const_iterator find(const K &key, std::size_t hash) const { + return find_impl(key, hash); + } + + template + std::pair equal_range(const K &key) { + return equal_range(key, hash_key(key)); + } + + template + std::pair equal_range(const K &key, std::size_t hash) { + iterator it = find(key, hash); + return std::make_pair(it, (it == end()) ? it : std::next(it)); + } + + template + std::pair equal_range(const K &key) const { + return equal_range(key, hash_key(key)); + } + + template + std::pair equal_range( + const K &key, std::size_t hash) const { + const_iterator it = find(key, hash); + return std::make_pair(it, (it == cend()) ? it : std::next(it)); + } + + /* + * Bucket interface + */ + size_type bucket_count() const { return m_bucket_count; } + + size_type max_bucket_count() const { + return m_sparse_buckets_data.max_size(); + } + + /* + * Hash policy + */ + float load_factor() const { + if (bucket_count() == 0) { + return 0; + } + + return float(m_nb_elements) / float(bucket_count()); + } + + float max_load_factor() const { return m_max_load_factor; } + + void max_load_factor(float ml) { + m_max_load_factor = std::max(0.1f, std::min(ml, 0.8f)); + m_load_threshold_rehash = + size_type(float(bucket_count()) * m_max_load_factor); + + const float max_load_factor_with_deleted_buckets = + m_max_load_factor + 0.5f * (1.0f - m_max_load_factor); + tsl_sh_assert(max_load_factor_with_deleted_buckets > 0.0f && + max_load_factor_with_deleted_buckets <= 1.0f); + m_load_threshold_clear_deleted = + size_type(float(bucket_count()) * max_load_factor_with_deleted_buckets); + } + + void rehash(size_type count) { + count = std::max(count, + size_type(std::ceil(float(size()) / max_load_factor()))); + rehash_impl(count); + } + + void reserve(size_type count) { + rehash(size_type(std::ceil(float(count) / max_load_factor()))); + } + + /* + * Observers + */ + hasher hash_function() const { return static_cast(*this); } + + key_equal key_eq() const { return static_cast(*this); } + + /* + * Other + */ + iterator mutable_iterator(const_iterator pos) { + auto it_sparse_buckets = + m_sparse_buckets_data.begin() + + std::distance(m_sparse_buckets_data.cbegin(), pos.m_sparse_buckets_it); + + return iterator(it_sparse_buckets, + sparse_array::mutable_iterator(pos.m_sparse_array_it)); + } + + template + void serialize(Serializer &serializer) const { + serialize_impl(serializer); + } + + template + void deserialize(Deserializer &deserializer, bool hash_compatible) { + deserialize_impl(deserializer, hash_compatible); + } + + private: + template + std::size_t hash_key(const K &key) const { + return Hash::operator()(key); + } + + template + bool compare_keys(const K1 &key1, const K2 &key2) const { + return KeyEqual::operator()(key1, key2); + } + + size_type bucket_for_hash(std::size_t hash) const { + const std::size_t bucket = GrowthPolicy::bucket_for_hash(hash); + tsl_sh_assert(sparse_array::sparse_ibucket(bucket) < + m_sparse_buckets_data.size() || + (bucket == 0 && m_sparse_buckets_data.empty())); + + return bucket; + } + + template ::value>::type * = + nullptr> + size_type next_bucket(size_type ibucket, size_type iprobe) const { + (void)iprobe; + if (Probing == tsl::sh::probing::linear) { + return (ibucket + 1) & this->m_mask; + } else { + tsl_sh_assert(Probing == tsl::sh::probing::quadratic); + return (ibucket + iprobe) & this->m_mask; + } + } + + template ::value>::type * = + nullptr> + size_type next_bucket(size_type ibucket, size_type iprobe) const { + (void)iprobe; + if (Probing == tsl::sh::probing::linear) { + ibucket++; + return (ibucket != bucket_count()) ? ibucket : 0; + } else { + tsl_sh_assert(Probing == tsl::sh::probing::quadratic); + ibucket += iprobe; + return (ibucket < bucket_count()) ? ibucket : ibucket % bucket_count(); + } + } + + // TODO encapsulate m_sparse_buckets_data to avoid the managing the allocator + void copy_buckets_from(const sparse_hash &other) { + m_sparse_buckets_data.reserve(other.m_sparse_buckets_data.size()); + + TSL_SH_TRY { + for (const auto &bucket : other.m_sparse_buckets_data) { + m_sparse_buckets_data.emplace_back(bucket, + static_cast(*this)); + } + } + TSL_SH_CATCH(...) { + clear(); + TSL_SH_RETRHOW; + } + + tsl_sh_assert(m_sparse_buckets_data.empty() || + m_sparse_buckets_data.back().last()); + } + + void move_buckets_from(sparse_hash &&other) { + m_sparse_buckets_data.reserve(other.m_sparse_buckets_data.size()); + + TSL_SH_TRY { + for (auto &&bucket : other.m_sparse_buckets_data) { + m_sparse_buckets_data.emplace_back(std::move(bucket), + static_cast(*this)); + } + } + TSL_SH_CATCH(...) { + clear(); + TSL_SH_RETRHOW; + } + + tsl_sh_assert(m_sparse_buckets_data.empty() || + m_sparse_buckets_data.back().last()); + } + + template + std::pair insert_impl(const K &key, + Args &&...value_type_args) { + /** + * We must insert the value in the first empty or deleted bucket we find. If + * we first find a deleted bucket, we still have to continue the search + * until we find an empty bucket or until we have searched all the buckets + * to be sure that the value is not in the hash table. We thus remember the + * position, if any, of the first deleted bucket we have encountered so we + * can insert it there if needed. + */ + bool found_first_deleted_bucket = false; + std::size_t sparse_ibucket_first_deleted = 0; + typename sparse_array::size_type index_in_sparse_bucket_first_deleted = 0; + + const std::size_t hash = hash_key(key); + std::size_t ibucket = bucket_for_hash(hash); + + std::size_t probe = 0; + while (true) { + std::size_t sparse_ibucket = sparse_array::sparse_ibucket(ibucket); + auto index_in_sparse_bucket = + sparse_array::index_in_sparse_bucket(ibucket); + + if (m_sparse_buckets[sparse_ibucket].has_value(index_in_sparse_bucket)) { + auto value_it = + m_sparse_buckets[sparse_ibucket].value(index_in_sparse_bucket); + if (compare_keys(key, KeySelect()(*value_it))) { + return std::make_pair( + iterator(m_sparse_buckets_data.begin() + sparse_ibucket, + value_it), + false); + } + } else if (m_sparse_buckets[sparse_ibucket].has_deleted_value( + index_in_sparse_bucket) && + probe < m_bucket_count) { + if (!found_first_deleted_bucket) { + found_first_deleted_bucket = true; + sparse_ibucket_first_deleted = sparse_ibucket; + index_in_sparse_bucket_first_deleted = index_in_sparse_bucket; + } + } else { + /** + * At this point we are sure that the value does not exist + * in the hash table. + * First check if we satisfy load and delete thresholds, and if not, + * rehash the hash table (and therefore start over). Otherwise, just + * insert the value into the appropriate bucket. + */ + if (size() >= m_load_threshold_rehash) { + rehash_impl(GrowthPolicy::next_bucket_count()); + return insert_impl(key, std::forward(value_type_args)...); + } else if (size() + m_nb_deleted_buckets >= + m_load_threshold_clear_deleted) { + clear_deleted_buckets(); + return insert_impl(key, std::forward(value_type_args)...); + } + + if (found_first_deleted_bucket) { + auto it = insert_in_bucket(sparse_ibucket_first_deleted, + index_in_sparse_bucket_first_deleted, + std::forward(value_type_args)...); + m_nb_deleted_buckets--; + + return it; + } + + return insert_in_bucket(sparse_ibucket, index_in_sparse_bucket, + std::forward(value_type_args)...); + } + + probe++; + ibucket = next_bucket(ibucket, probe); + } + } + + template + std::pair insert_in_bucket( + std::size_t sparse_ibucket, + typename sparse_array::size_type index_in_sparse_bucket, + Args &&...value_type_args) { + auto value_it = m_sparse_buckets[sparse_ibucket].set( + *this, index_in_sparse_bucket, std::forward(value_type_args)...); + m_nb_elements++; + + return std::make_pair( + iterator(m_sparse_buckets_data.begin() + sparse_ibucket, value_it), + true); + } + + template + size_type erase_impl(const K &key, std::size_t hash) { + std::size_t ibucket = bucket_for_hash(hash); + + std::size_t probe = 0; + while (true) { + const std::size_t sparse_ibucket = sparse_array::sparse_ibucket(ibucket); + const auto index_in_sparse_bucket = + sparse_array::index_in_sparse_bucket(ibucket); + + if (m_sparse_buckets[sparse_ibucket].has_value(index_in_sparse_bucket)) { + auto value_it = + m_sparse_buckets[sparse_ibucket].value(index_in_sparse_bucket); + if (compare_keys(key, KeySelect()(*value_it))) { + m_sparse_buckets[sparse_ibucket].erase(*this, value_it, + index_in_sparse_bucket); + m_nb_elements--; + m_nb_deleted_buckets++; + + return 1; + } + } else if (!m_sparse_buckets[sparse_ibucket].has_deleted_value( + index_in_sparse_bucket) || + probe >= m_bucket_count) { + return 0; + } + + probe++; + ibucket = next_bucket(ibucket, probe); + } + } + + template + iterator find_impl(const K &key, std::size_t hash) { + return mutable_iterator( + static_cast(this)->find(key, hash)); + } + + template + const_iterator find_impl(const K &key, std::size_t hash) const { + std::size_t ibucket = bucket_for_hash(hash); + + std::size_t probe = 0; + while (true) { + const std::size_t sparse_ibucket = sparse_array::sparse_ibucket(ibucket); + const auto index_in_sparse_bucket = + sparse_array::index_in_sparse_bucket(ibucket); + + if (m_sparse_buckets[sparse_ibucket].has_value(index_in_sparse_bucket)) { + auto value_it = + m_sparse_buckets[sparse_ibucket].value(index_in_sparse_bucket); + if (compare_keys(key, KeySelect()(*value_it))) { + return const_iterator(m_sparse_buckets_data.cbegin() + sparse_ibucket, + value_it); + } + } else if (!m_sparse_buckets[sparse_ibucket].has_deleted_value( + index_in_sparse_bucket) || + probe >= m_bucket_count) { + return cend(); + } + + probe++; + ibucket = next_bucket(ibucket, probe); + } + } + + void clear_deleted_buckets() { + // TODO could be optimized, we could do it in-place instead of allocating a + // new bucket array. + rehash_impl(m_bucket_count); + tsl_sh_assert(m_nb_deleted_buckets == 0); + } + + template ::type + * = nullptr> + void rehash_impl(size_type count) { + sparse_hash new_table(count, static_cast(*this), + static_cast(*this), + static_cast(*this), m_max_load_factor); + + for (auto &bucket : m_sparse_buckets_data) { + for (auto &val : bucket) { + new_table.insert_on_rehash(std::move(val)); + } + + // TODO try to reuse some of the memory + bucket.clear(*this); + } + + new_table.swap(*this); + } + + /** + * TODO: For now we copy each element into the new map. We could move + * them if they are nothrow_move_constructible without triggering + * any exception if we reserve enough space in the sparse arrays beforehand. + */ + template ::type * = nullptr> + void rehash_impl(size_type count) { + sparse_hash new_table(count, static_cast(*this), + static_cast(*this), + static_cast(*this), m_max_load_factor); + + for (const auto &bucket : m_sparse_buckets_data) { + for (const auto &val : bucket) { + new_table.insert_on_rehash(val); + } + } + + new_table.swap(*this); + } + + template + void insert_on_rehash(K &&key_value) { + const std::size_t hash = hash_key(KeySelect()(key_value)); + std::size_t ibucket = bucket_for_hash(hash); + + std::size_t probe = 0; + while (true) { + std::size_t sparse_ibucket = sparse_array::sparse_ibucket(ibucket); + auto index_in_sparse_bucket = + sparse_array::index_in_sparse_bucket(ibucket); + + if (!m_sparse_buckets[sparse_ibucket].has_value(index_in_sparse_bucket)) { + m_sparse_buckets[sparse_ibucket].set(*this, index_in_sparse_bucket, + std::forward(key_value)); + m_nb_elements++; + + return; + } else { + tsl_sh_assert( + !compare_keys(KeySelect()(key_value), + KeySelect()(*m_sparse_buckets[sparse_ibucket].value( + index_in_sparse_bucket)))); + } + + probe++; + ibucket = next_bucket(ibucket, probe); + } + } + + template + void serialize_impl(Serializer &serializer) const { + const slz_size_type version = SERIALIZATION_PROTOCOL_VERSION; + serializer(version); + + const slz_size_type bucket_count = m_bucket_count; + serializer(bucket_count); + + const slz_size_type nb_sparse_buckets = m_sparse_buckets_data.size(); + serializer(nb_sparse_buckets); + + const slz_size_type nb_elements = m_nb_elements; + serializer(nb_elements); + + const slz_size_type nb_deleted_buckets = m_nb_deleted_buckets; + serializer(nb_deleted_buckets); + + const float max_load_factor = m_max_load_factor; + serializer(max_load_factor); + + for (const auto &bucket : m_sparse_buckets_data) { + bucket.serialize(serializer); + } + } + + template + void deserialize_impl(Deserializer &deserializer, bool hash_compatible) { + tsl_sh_assert( + m_bucket_count == 0 && + m_sparse_buckets_data.empty()); // Current hash table must be empty + + const slz_size_type version = + deserialize_value(deserializer); + // For now we only have one version of the serialization protocol. + // If it doesn't match there is a problem with the file. + if (version != SERIALIZATION_PROTOCOL_VERSION) { + TSL_SH_THROW_OR_ABORT(std::runtime_error, + "Can't deserialize the sparse_map/set. The " + "protocol version header is invalid."); + } + + const slz_size_type bucket_count_ds = + deserialize_value(deserializer); + const slz_size_type nb_sparse_buckets = + deserialize_value(deserializer); + const slz_size_type nb_elements = + deserialize_value(deserializer); + const slz_size_type nb_deleted_buckets = + deserialize_value(deserializer); + const float max_load_factor = deserialize_value(deserializer); + + if (!hash_compatible) { + this->max_load_factor(max_load_factor); + reserve(numeric_cast(nb_elements, + "Deserialized nb_elements is too big.")); + for (slz_size_type ibucket = 0; ibucket < nb_sparse_buckets; ibucket++) { + sparse_array::deserialize_values_into_sparse_hash(deserializer, *this); + } + } else { + m_bucket_count = numeric_cast( + bucket_count_ds, "Deserialized bucket_count is too big."); + + GrowthPolicy::operator=(GrowthPolicy(m_bucket_count)); + // GrowthPolicy should not modify the bucket count we got from + // deserialization + if (m_bucket_count != bucket_count_ds) { + TSL_SH_THROW_OR_ABORT(std::runtime_error, + "The GrowthPolicy is not the same even though " + "hash_compatible is true."); + } + + if (nb_sparse_buckets != + sparse_array::nb_sparse_buckets(m_bucket_count)) { + TSL_SH_THROW_OR_ABORT(std::runtime_error, + "Deserialized nb_sparse_buckets is invalid."); + } + + m_nb_elements = numeric_cast( + nb_elements, "Deserialized nb_elements is too big."); + m_nb_deleted_buckets = numeric_cast( + nb_deleted_buckets, "Deserialized nb_deleted_buckets is too big."); + + m_sparse_buckets_data.reserve(numeric_cast( + nb_sparse_buckets, "Deserialized nb_sparse_buckets is too big.")); + for (slz_size_type ibucket = 0; ibucket < nb_sparse_buckets; ibucket++) { + m_sparse_buckets_data.emplace_back( + sparse_array::deserialize_hash_compatible( + deserializer, static_cast(*this))); + } + + if (!m_sparse_buckets_data.empty()) { + m_sparse_buckets_data.back().set_as_last(); + m_sparse_buckets = m_sparse_buckets_data.data(); + } + + this->max_load_factor(max_load_factor); + if (load_factor() > this->max_load_factor()) { + TSL_SH_THROW_OR_ABORT( + std::runtime_error, + "Invalid max_load_factor. Check that the serializer and " + "deserializer support " + "floats correctly as they can be converted implicitely to ints."); + } + } + } + + public: + static const size_type DEFAULT_INIT_BUCKET_COUNT = 0; + static constexpr float DEFAULT_MAX_LOAD_FACTOR = 0.5f; + + /** + * Protocol version currenlty used for serialization. + */ + static const slz_size_type SERIALIZATION_PROTOCOL_VERSION = 1; + + /** + * Return an always valid pointer to an static empty bucket_entry with + * last_bucket() == true. + */ + sparse_array *static_empty_sparse_bucket_ptr() { + static sparse_array empty_sparse_bucket(true); + return &empty_sparse_bucket; + } + + private: + sparse_buckets_container m_sparse_buckets_data; + + /** + * Points to m_sparse_buckets_data.data() if !m_sparse_buckets_data.empty() + * otherwise points to static_empty_sparse_bucket_ptr. This variable is useful + * to avoid the cost of checking if m_sparse_buckets_data is empty when trying + * to find an element. + * + * TODO Remove m_sparse_buckets_data and only use a pointer instead of a + * pointer+vector to save some space in the sparse_hash object. + */ + sparse_array *m_sparse_buckets; + + size_type m_bucket_count; + size_type m_nb_elements; + size_type m_nb_deleted_buckets; + + /** + * Maximum that m_nb_elements can reach before a rehash occurs automatically + * to grow the hash table. + */ + size_type m_load_threshold_rehash; + + /** + * Maximum that m_nb_elements + m_nb_deleted_buckets can reach before cleaning + * up the buckets marked as deleted. + */ + size_type m_load_threshold_clear_deleted; + float m_max_load_factor; +}; + +} // namespace detail_sparse_hash +} // namespace tsl + +#endif diff --git a/benchs/tsl/sparse_map.h b/benchs/tsl/sparse_map.h new file mode 100644 index 00000000..601742d8 --- /dev/null +++ b/benchs/tsl/sparse_map.h @@ -0,0 +1,800 @@ +/** + * MIT License + * + * Copyright (c) 2017 Thibaut Goetghebuer-Planchon + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef TSL_SPARSE_MAP_H +#define TSL_SPARSE_MAP_H + +#include +#include +#include +#include +#include +#include + +#include "sparse_hash.h" + +namespace tsl { + +/** + * Implementation of a sparse hash map using open-addressing with quadratic + * probing. The goal on the hash map is to be the most memory efficient + * possible, even at low load factor, while keeping reasonable performances. + * + * `GrowthPolicy` defines how the map grows and consequently how a hash value is + * mapped to a bucket. By default the map uses + * `tsl::sh::power_of_two_growth_policy`. This policy keeps the number of + * buckets to a power of two and uses a mask to map the hash to a bucket instead + * of the slow modulo. Other growth policies are available and you may define + * your own growth policy, check `tsl::sh::power_of_two_growth_policy` for the + * interface. + * + * `ExceptionSafety` defines the exception guarantee provided by the class. By + * default only the basic exception safety is guaranteed which mean that all + * resources used by the hash map will be freed (no memory leaks) but the hash + * map may end-up in an undefined state if an exception is thrown (undefined + * here means that some elements may be missing). This can ONLY happen on rehash + * (either on insert or if `rehash` is called explicitly) and will occur if the + * Allocator can't allocate memory (`std::bad_alloc`) or if the copy constructor + * (when a nothrow move constructor is not available) throws an exception. This + * can be avoided by calling `reserve` beforehand. This basic guarantee is + * similar to the one of `google::sparse_hash_map` and `spp::sparse_hash_map`. + * It is possible to ask for the strong exception guarantee with + * `tsl::sh::exception_safety::strong`, the drawback is that the map will be + * slower on rehashes and will also need more memory on rehashes. + * + * `Sparsity` defines how much the hash set will compromise between insertion + * speed and memory usage. A high sparsity means less memory usage but longer + * insertion times, and vice-versa for low sparsity. The default + * `tsl::sh::sparsity::medium` sparsity offers a good compromise. It doesn't + * change the lookup speed. + * + * `Key` and `T` must be nothrow move constructible and/or copy constructible. + * + * If the destructor of `Key` or `T` throws an exception, the behaviour of the + * class is undefined. + * + * Iterators invalidation: + * - clear, operator=, reserve, rehash: always invalidate the iterators. + * - insert, emplace, emplace_hint, operator[]: if there is an effective + * insert, invalidate the iterators. + * - erase: always invalidate the iterators. + */ +template , + class KeyEqual = std::equal_to, + class Allocator = std::allocator>, + class GrowthPolicy = tsl::sh::power_of_two_growth_policy<2>, + tsl::sh::exception_safety ExceptionSafety = + tsl::sh::exception_safety::basic, + tsl::sh::sparsity Sparsity = tsl::sh::sparsity::medium> +class sparse_map { + private: + template + using has_is_transparent = tsl::detail_sparse_hash::has_is_transparent; + + class KeySelect { + public: + using key_type = Key; + + const key_type &operator()( + const std::pair &key_value) const noexcept { + return key_value.first; + } + + key_type &operator()(std::pair &key_value) noexcept { + return key_value.first; + } + }; + + class ValueSelect { + public: + using value_type = T; + + const value_type &operator()( + const std::pair &key_value) const noexcept { + return key_value.second; + } + + value_type &operator()(std::pair &key_value) noexcept { + return key_value.second; + } + }; + + using ht = detail_sparse_hash::sparse_hash< + std::pair, KeySelect, ValueSelect, Hash, KeyEqual, Allocator, + GrowthPolicy, ExceptionSafety, Sparsity, tsl::sh::probing::quadratic>; + + public: + using key_type = typename ht::key_type; + using mapped_type = T; + using value_type = typename ht::value_type; + using size_type = typename ht::size_type; + using difference_type = typename ht::difference_type; + using hasher = typename ht::hasher; + using key_equal = typename ht::key_equal; + using allocator_type = typename ht::allocator_type; + using reference = typename ht::reference; + using const_reference = typename ht::const_reference; + using pointer = typename ht::pointer; + using const_pointer = typename ht::const_pointer; + using iterator = typename ht::iterator; + using const_iterator = typename ht::const_iterator; + + public: + /* + * Constructors + */ + sparse_map() : sparse_map(ht::DEFAULT_INIT_BUCKET_COUNT) {} + + explicit sparse_map(size_type bucket_count, const Hash &hash = Hash(), + const KeyEqual &equal = KeyEqual(), + const Allocator &alloc = Allocator()) + : m_ht(bucket_count, hash, equal, alloc, ht::DEFAULT_MAX_LOAD_FACTOR) {} + + sparse_map(size_type bucket_count, const Allocator &alloc) + : sparse_map(bucket_count, Hash(), KeyEqual(), alloc) {} + + sparse_map(size_type bucket_count, const Hash &hash, const Allocator &alloc) + : sparse_map(bucket_count, hash, KeyEqual(), alloc) {} + + explicit sparse_map(const Allocator &alloc) + : sparse_map(ht::DEFAULT_INIT_BUCKET_COUNT, alloc) {} + + template + sparse_map(InputIt first, InputIt last, + size_type bucket_count = ht::DEFAULT_INIT_BUCKET_COUNT, + const Hash &hash = Hash(), const KeyEqual &equal = KeyEqual(), + const Allocator &alloc = Allocator()) + : sparse_map(bucket_count, hash, equal, alloc) { + insert(first, last); + } + + template + sparse_map(InputIt first, InputIt last, size_type bucket_count, + const Allocator &alloc) + : sparse_map(first, last, bucket_count, Hash(), KeyEqual(), alloc) {} + + template + sparse_map(InputIt first, InputIt last, size_type bucket_count, + const Hash &hash, const Allocator &alloc) + : sparse_map(first, last, bucket_count, hash, KeyEqual(), alloc) {} + + sparse_map(std::initializer_list init, + size_type bucket_count = ht::DEFAULT_INIT_BUCKET_COUNT, + const Hash &hash = Hash(), const KeyEqual &equal = KeyEqual(), + const Allocator &alloc = Allocator()) + : sparse_map(init.begin(), init.end(), bucket_count, hash, equal, alloc) { + } + + sparse_map(std::initializer_list init, size_type bucket_count, + const Allocator &alloc) + : sparse_map(init.begin(), init.end(), bucket_count, Hash(), KeyEqual(), + alloc) {} + + sparse_map(std::initializer_list init, size_type bucket_count, + const Hash &hash, const Allocator &alloc) + : sparse_map(init.begin(), init.end(), bucket_count, hash, KeyEqual(), + alloc) {} + + sparse_map &operator=(std::initializer_list ilist) { + m_ht.clear(); + + m_ht.reserve(ilist.size()); + m_ht.insert(ilist.begin(), ilist.end()); + + return *this; + } + + allocator_type get_allocator() const { return m_ht.get_allocator(); } + + /* + * Iterators + */ + iterator begin() noexcept { return m_ht.begin(); } + const_iterator begin() const noexcept { return m_ht.begin(); } + const_iterator cbegin() const noexcept { return m_ht.cbegin(); } + + iterator end() noexcept { return m_ht.end(); } + const_iterator end() const noexcept { return m_ht.end(); } + const_iterator cend() const noexcept { return m_ht.cend(); } + + /* + * Capacity + */ + bool empty() const noexcept { return m_ht.empty(); } + size_type size() const noexcept { return m_ht.size(); } + size_type max_size() const noexcept { return m_ht.max_size(); } + + /* + * Modifiers + */ + void clear() noexcept { m_ht.clear(); } + + std::pair insert(const value_type &value) { + return m_ht.insert(value); + } + + template ::value>::type * = nullptr> + std::pair insert(P &&value) { + return m_ht.emplace(std::forward

(value)); + } + + std::pair insert(value_type &&value) { + return m_ht.insert(std::move(value)); + } + + iterator insert(const_iterator hint, const value_type &value) { + return m_ht.insert_hint(hint, value); + } + + template ::value>::type * = nullptr> + iterator insert(const_iterator hint, P &&value) { + return m_ht.emplace_hint(hint, std::forward

(value)); + } + + iterator insert(const_iterator hint, value_type &&value) { + return m_ht.insert_hint(hint, std::move(value)); + } + + template + void insert(InputIt first, InputIt last) { + m_ht.insert(first, last); + } + + void insert(std::initializer_list ilist) { + m_ht.insert(ilist.begin(), ilist.end()); + } + + template + std::pair insert_or_assign(const key_type &k, M &&obj) { + return m_ht.insert_or_assign(k, std::forward(obj)); + } + + template + std::pair insert_or_assign(key_type &&k, M &&obj) { + return m_ht.insert_or_assign(std::move(k), std::forward(obj)); + } + + template + iterator insert_or_assign(const_iterator hint, const key_type &k, M &&obj) { + return m_ht.insert_or_assign(hint, k, std::forward(obj)); + } + + template + iterator insert_or_assign(const_iterator hint, key_type &&k, M &&obj) { + return m_ht.insert_or_assign(hint, std::move(k), std::forward(obj)); + } + + /** + * Due to the way elements are stored, emplace will need to move or copy the + * key-value once. The method is equivalent to + * `insert(value_type(std::forward(args)...));`. + * + * Mainly here for compatibility with the `std::unordered_map` interface. + */ + template + std::pair emplace(Args &&...args) { + return m_ht.emplace(std::forward(args)...); + } + + /** + * Due to the way elements are stored, emplace_hint will need to move or copy + * the key-value once. The method is equivalent to `insert(hint, + * value_type(std::forward(args)...));`. + * + * Mainly here for compatibility with the `std::unordered_map` interface. + */ + template + iterator emplace_hint(const_iterator hint, Args &&...args) { + return m_ht.emplace_hint(hint, std::forward(args)...); + } + + template + std::pair try_emplace(const key_type &k, Args &&...args) { + return m_ht.try_emplace(k, std::forward(args)...); + } + + template + std::pair try_emplace(key_type &&k, Args &&...args) { + return m_ht.try_emplace(std::move(k), std::forward(args)...); + } + + template + iterator try_emplace(const_iterator hint, const key_type &k, Args &&...args) { + return m_ht.try_emplace_hint(hint, k, std::forward(args)...); + } + + template + iterator try_emplace(const_iterator hint, key_type &&k, Args &&...args) { + return m_ht.try_emplace_hint(hint, std::move(k), + std::forward(args)...); + } + + iterator erase(iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator first, const_iterator last) { + return m_ht.erase(first, last); + } + size_type erase(const key_type &key) { return m_ht.erase(key); } + + /** + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + size_type erase(const key_type &key, std::size_t precalculated_hash) { + return m_ht.erase(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * `KeyEqual::is_transparent` exists. If so, `K` must be hashable and + * comparable to `Key`. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + size_type erase(const K &key) { + return m_ht.erase(key); + } + + /** + * @copydoc erase(const K& key) + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + size_type erase(const K &key, std::size_t precalculated_hash) { + return m_ht.erase(key, precalculated_hash); + } + + void swap(sparse_map &other) { other.m_ht.swap(m_ht); } + + /* + * Lookup + */ + T &at(const Key &key) { return m_ht.at(key); } + + /** + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + T &at(const Key &key, std::size_t precalculated_hash) { + return m_ht.at(key, precalculated_hash); + } + + const T &at(const Key &key) const { return m_ht.at(key); } + + /** + * @copydoc at(const Key& key, std::size_t precalculated_hash) + */ + const T &at(const Key &key, std::size_t precalculated_hash) const { + return m_ht.at(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * `KeyEqual::is_transparent` exists. If so, `K` must be hashable and + * comparable to `Key`. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + T &at(const K &key) { + return m_ht.at(key); + } + + /** + * @copydoc at(const K& key) + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + T &at(const K &key, std::size_t precalculated_hash) { + return m_ht.at(key, precalculated_hash); + } + + /** + * @copydoc at(const K& key) + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + const T &at(const K &key) const { + return m_ht.at(key); + } + + /** + * @copydoc at(const K& key, std::size_t precalculated_hash) + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + const T &at(const K &key, std::size_t precalculated_hash) const { + return m_ht.at(key, precalculated_hash); + } + + T &operator[](const Key &key) { return m_ht[key]; } + T &operator[](Key &&key) { return m_ht[std::move(key)]; } + + size_type count(const Key &key) const { return m_ht.count(key); } + + /** + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + size_type count(const Key &key, std::size_t precalculated_hash) const { + return m_ht.count(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * `KeyEqual::is_transparent` exists. If so, `K` must be hashable and + * comparable to `Key`. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + size_type count(const K &key) const { + return m_ht.count(key); + } + + /** + * @copydoc count(const K& key) const + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + size_type count(const K &key, std::size_t precalculated_hash) const { + return m_ht.count(key, precalculated_hash); + } + + iterator find(const Key &key) { return m_ht.find(key); } + + /** + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + iterator find(const Key &key, std::size_t precalculated_hash) { + return m_ht.find(key, precalculated_hash); + } + + const_iterator find(const Key &key) const { return m_ht.find(key); } + + /** + * @copydoc find(const Key& key, std::size_t precalculated_hash) + */ + const_iterator find(const Key &key, std::size_t precalculated_hash) const { + return m_ht.find(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * `KeyEqual::is_transparent` exists. If so, `K` must be hashable and + * comparable to `Key`. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + iterator find(const K &key) { + return m_ht.find(key); + } + + /** + * @copydoc find(const K& key) + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + iterator find(const K &key, std::size_t precalculated_hash) { + return m_ht.find(key, precalculated_hash); + } + + /** + * @copydoc find(const K& key) + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + const_iterator find(const K &key) const { + return m_ht.find(key); + } + + /** + * @copydoc find(const K& key) + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + const_iterator find(const K &key, std::size_t precalculated_hash) const { + return m_ht.find(key, precalculated_hash); + } + + bool contains(const Key &key) const { return m_ht.contains(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The + * hash value should be the same as hash_function()(key). Useful to speed-up + * the lookup if you already have the hash. + */ + bool contains(const Key &key, std::size_t precalculated_hash) const { + return m_ht.contains(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * KeyEqual::is_transparent exists. If so, K must be hashable and comparable + * to Key. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + bool contains(const K &key) const { + return m_ht.contains(key); + } + + /** + * @copydoc contains(const K& key) const + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The + * hash value should be the same as hash_function()(key). Useful to speed-up + * the lookup if you already have the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + bool contains(const K &key, std::size_t precalculated_hash) const { + return m_ht.contains(key, precalculated_hash); + } + + std::pair equal_range(const Key &key) { + return m_ht.equal_range(key); + } + + /** + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + std::pair equal_range(const Key &key, + std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + std::pair equal_range(const Key &key) const { + return m_ht.equal_range(key); + } + + /** + * @copydoc equal_range(const Key& key, std::size_t precalculated_hash) + */ + std::pair equal_range( + const Key &key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * `KeyEqual::is_transparent` exists. If so, `K` must be hashable and + * comparable to `Key`. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + std::pair equal_range(const K &key) { + return m_ht.equal_range(key); + } + + /** + * @copydoc equal_range(const K& key) + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + std::pair equal_range(const K &key, + std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * @copydoc equal_range(const K& key) + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + std::pair equal_range(const K &key) const { + return m_ht.equal_range(key); + } + + /** + * @copydoc equal_range(const K& key, std::size_t precalculated_hash) + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + std::pair equal_range( + const K &key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + /* + * Bucket interface + */ + size_type bucket_count() const { return m_ht.bucket_count(); } + size_type max_bucket_count() const { return m_ht.max_bucket_count(); } + + /* + * Hash policy + */ + float load_factor() const { return m_ht.load_factor(); } + float max_load_factor() const { return m_ht.max_load_factor(); } + void max_load_factor(float ml) { m_ht.max_load_factor(ml); } + + void rehash(size_type count) { m_ht.rehash(count); } + void reserve(size_type count) { m_ht.reserve(count); } + + /* + * Observers + */ + hasher hash_function() const { return m_ht.hash_function(); } + key_equal key_eq() const { return m_ht.key_eq(); } + + /* + * Other + */ + + /** + * Convert a `const_iterator` to an `iterator`. + */ + iterator mutable_iterator(const_iterator pos) { + return m_ht.mutable_iterator(pos); + } + + /** + * Serialize the map through the `serializer` parameter. + * + * The `serializer` parameter must be a function object that supports the + * following call: + * - `template void operator()(const U& value);` where the types + * `std::uint64_t`, `float` and `std::pair` must be supported for U. + * + * The implementation leaves binary compatibility (endianness, IEEE 754 for + * floats, ...) of the types it serializes in the hands of the `Serializer` + * function object if compatibility is required. + */ + template + void serialize(Serializer &serializer) const { + m_ht.serialize(serializer); + } + + /** + * Deserialize a previously serialized map through the `deserializer` + * parameter. + * + * The `deserializer` parameter must be a function object that supports the + * following calls: + * - `template U operator()();` where the types `std::uint64_t`, + * `float` and `std::pair` must be supported for U. + * + * If the deserialized hash map type is hash compatible with the serialized + * map, the deserialization process can be sped up by setting + * `hash_compatible` to true. To be hash compatible, the Hash, KeyEqual and + * GrowthPolicy must behave the same way than the ones used on the serialized + * map. The `std::size_t` must also be of the same size as the one on the + * platform used to serialize the map. If these criteria are not met, the + * behaviour is undefined with `hash_compatible` sets to true. + * + * The behaviour is undefined if the type `Key` and `T` of the `sparse_map` + * are not the same as the types used during serialization. + * + * The implementation leaves binary compatibility (endianness, IEEE 754 for + * floats, size of int, ...) of the types it deserializes in the hands of the + * `Deserializer` function object if compatibility is required. + */ + template + static sparse_map deserialize(Deserializer &deserializer, + bool hash_compatible = false) { + sparse_map map(0); + map.m_ht.deserialize(deserializer, hash_compatible); + + return map; + } + + friend bool operator==(const sparse_map &lhs, const sparse_map &rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + + for (const auto &element_lhs : lhs) { + const auto it_element_rhs = rhs.find(element_lhs.first); + if (it_element_rhs == rhs.cend() || + element_lhs.second != it_element_rhs->second) { + return false; + } + } + + return true; + } + + friend bool operator!=(const sparse_map &lhs, const sparse_map &rhs) { + return !operator==(lhs, rhs); + } + + friend void swap(sparse_map &lhs, sparse_map &rhs) { lhs.swap(rhs); } + + private: + ht m_ht; +}; + +/** + * Same as `tsl::sparse_map`. + */ +template , + class KeyEqual = std::equal_to, + class Allocator = std::allocator>> +using sparse_pg_map = + sparse_map; + +} // end namespace tsl + +#endif diff --git a/benchs/tsl/sparse_set.h b/benchs/tsl/sparse_set.h new file mode 100644 index 00000000..3ce6a588 --- /dev/null +++ b/benchs/tsl/sparse_set.h @@ -0,0 +1,655 @@ +/** + * MIT License + * + * Copyright (c) 2017 Thibaut Goetghebuer-Planchon + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef TSL_SPARSE_SET_H +#define TSL_SPARSE_SET_H + +#include +#include +#include +#include +#include +#include + +#include "sparse_hash.h" + +namespace tsl { + +/** + * Implementation of a sparse hash set using open-addressing with quadratic + * probing. The goal on the hash set is to be the most memory efficient + * possible, even at low load factor, while keeping reasonable performances. + * + * `GrowthPolicy` defines how the set grows and consequently how a hash value is + * mapped to a bucket. By default the set uses + * `tsl::sh::power_of_two_growth_policy`. This policy keeps the number of + * buckets to a power of two and uses a mask to map the hash to a bucket instead + * of the slow modulo. Other growth policies are available and you may define + * your own growth policy, check `tsl::sh::power_of_two_growth_policy` for the + * interface. + * + * `ExceptionSafety` defines the exception guarantee provided by the class. By + * default only the basic exception safety is guaranteed which mean that all + * resources used by the hash set will be freed (no memory leaks) but the hash + * set may end-up in an undefined state if an exception is thrown (undefined + * here means that some elements may be missing). This can ONLY happen on rehash + * (either on insert or if `rehash` is called explicitly) and will occur if the + * Allocator can't allocate memory (`std::bad_alloc`) or if the copy constructor + * (when a nothrow move constructor is not available) throws an exception. This + * can be avoided by calling `reserve` beforehand. This basic guarantee is + * similar to the one of `google::sparse_hash_map` and `spp::sparse_hash_map`. + * It is possible to ask for the strong exception guarantee with + * `tsl::sh::exception_safety::strong`, the drawback is that the set will be + * slower on rehashes and will also need more memory on rehashes. + * + * `Sparsity` defines how much the hash set will compromise between insertion + * speed and memory usage. A high sparsity means less memory usage but longer + * insertion times, and vice-versa for low sparsity. The default + * `tsl::sh::sparsity::medium` sparsity offers a good compromise. It doesn't + * change the lookup speed. + * + * `Key` must be nothrow move constructible and/or copy constructible. + * + * If the destructor of `Key` throws an exception, the behaviour of the class is + * undefined. + * + * Iterators invalidation: + * - clear, operator=, reserve, rehash: always invalidate the iterators. + * - insert, emplace, emplace_hint: if there is an effective insert, invalidate + * the iterators. + * - erase: always invalidate the iterators. + */ +template , + class KeyEqual = std::equal_to, + class Allocator = std::allocator, + class GrowthPolicy = tsl::sh::power_of_two_growth_policy<2>, + tsl::sh::exception_safety ExceptionSafety = + tsl::sh::exception_safety::basic, + tsl::sh::sparsity Sparsity = tsl::sh::sparsity::medium> +class sparse_set { + private: + template + using has_is_transparent = tsl::detail_sparse_hash::has_is_transparent; + + class KeySelect { + public: + using key_type = Key; + + const key_type &operator()(const Key &key) const noexcept { return key; } + + key_type &operator()(Key &key) noexcept { return key; } + }; + + using ht = + detail_sparse_hash::sparse_hash; + + public: + using key_type = typename ht::key_type; + using value_type = typename ht::value_type; + using size_type = typename ht::size_type; + using difference_type = typename ht::difference_type; + using hasher = typename ht::hasher; + using key_equal = typename ht::key_equal; + using allocator_type = typename ht::allocator_type; + using reference = typename ht::reference; + using const_reference = typename ht::const_reference; + using pointer = typename ht::pointer; + using const_pointer = typename ht::const_pointer; + using iterator = typename ht::iterator; + using const_iterator = typename ht::const_iterator; + + /* + * Constructors + */ + sparse_set() : sparse_set(ht::DEFAULT_INIT_BUCKET_COUNT) {} + + explicit sparse_set(size_type bucket_count, const Hash &hash = Hash(), + const KeyEqual &equal = KeyEqual(), + const Allocator &alloc = Allocator()) + : m_ht(bucket_count, hash, equal, alloc, ht::DEFAULT_MAX_LOAD_FACTOR) {} + + sparse_set(size_type bucket_count, const Allocator &alloc) + : sparse_set(bucket_count, Hash(), KeyEqual(), alloc) {} + + sparse_set(size_type bucket_count, const Hash &hash, const Allocator &alloc) + : sparse_set(bucket_count, hash, KeyEqual(), alloc) {} + + explicit sparse_set(const Allocator &alloc) + : sparse_set(ht::DEFAULT_INIT_BUCKET_COUNT, alloc) {} + + template + sparse_set(InputIt first, InputIt last, + size_type bucket_count = ht::DEFAULT_INIT_BUCKET_COUNT, + const Hash &hash = Hash(), const KeyEqual &equal = KeyEqual(), + const Allocator &alloc = Allocator()) + : sparse_set(bucket_count, hash, equal, alloc) { + insert(first, last); + } + + template + sparse_set(InputIt first, InputIt last, size_type bucket_count, + const Allocator &alloc) + : sparse_set(first, last, bucket_count, Hash(), KeyEqual(), alloc) {} + + template + sparse_set(InputIt first, InputIt last, size_type bucket_count, + const Hash &hash, const Allocator &alloc) + : sparse_set(first, last, bucket_count, hash, KeyEqual(), alloc) {} + + sparse_set(std::initializer_list init, + size_type bucket_count = ht::DEFAULT_INIT_BUCKET_COUNT, + const Hash &hash = Hash(), const KeyEqual &equal = KeyEqual(), + const Allocator &alloc = Allocator()) + : sparse_set(init.begin(), init.end(), bucket_count, hash, equal, alloc) { + } + + sparse_set(std::initializer_list init, size_type bucket_count, + const Allocator &alloc) + : sparse_set(init.begin(), init.end(), bucket_count, Hash(), KeyEqual(), + alloc) {} + + sparse_set(std::initializer_list init, size_type bucket_count, + const Hash &hash, const Allocator &alloc) + : sparse_set(init.begin(), init.end(), bucket_count, hash, KeyEqual(), + alloc) {} + + sparse_set &operator=(std::initializer_list ilist) { + m_ht.clear(); + + m_ht.reserve(ilist.size()); + m_ht.insert(ilist.begin(), ilist.end()); + + return *this; + } + + allocator_type get_allocator() const { return m_ht.get_allocator(); } + + /* + * Iterators + */ + iterator begin() noexcept { return m_ht.begin(); } + const_iterator begin() const noexcept { return m_ht.begin(); } + const_iterator cbegin() const noexcept { return m_ht.cbegin(); } + + iterator end() noexcept { return m_ht.end(); } + const_iterator end() const noexcept { return m_ht.end(); } + const_iterator cend() const noexcept { return m_ht.cend(); } + + /* + * Capacity + */ + bool empty() const noexcept { return m_ht.empty(); } + size_type size() const noexcept { return m_ht.size(); } + size_type max_size() const noexcept { return m_ht.max_size(); } + + /* + * Modifiers + */ + void clear() noexcept { m_ht.clear(); } + + std::pair insert(const value_type &value) { + return m_ht.insert(value); + } + + std::pair insert(value_type &&value) { + return m_ht.insert(std::move(value)); + } + + iterator insert(const_iterator hint, const value_type &value) { + return m_ht.insert_hint(hint, value); + } + + iterator insert(const_iterator hint, value_type &&value) { + return m_ht.insert_hint(hint, std::move(value)); + } + + template + void insert(InputIt first, InputIt last) { + m_ht.insert(first, last); + } + + void insert(std::initializer_list ilist) { + m_ht.insert(ilist.begin(), ilist.end()); + } + + /** + * Due to the way elements are stored, emplace will need to move or copy the + * key-value once. The method is equivalent to + * `insert(value_type(std::forward(args)...));`. + * + * Mainly here for compatibility with the `std::unordered_map` interface. + */ + template + std::pair emplace(Args &&...args) { + return m_ht.emplace(std::forward(args)...); + } + + /** + * Due to the way elements are stored, emplace_hint will need to move or copy + * the key-value once. The method is equivalent to `insert(hint, + * value_type(std::forward(args)...));`. + * + * Mainly here for compatibility with the `std::unordered_map` interface. + */ + template + iterator emplace_hint(const_iterator hint, Args &&...args) { + return m_ht.emplace_hint(hint, std::forward(args)...); + } + + iterator erase(iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator pos) { return m_ht.erase(pos); } + iterator erase(const_iterator first, const_iterator last) { + return m_ht.erase(first, last); + } + size_type erase(const key_type &key) { return m_ht.erase(key); } + + /** + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + size_type erase(const key_type &key, std::size_t precalculated_hash) { + return m_ht.erase(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * `KeyEqual::is_transparent` exists. If so, `K` must be hashable and + * comparable to `Key`. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + size_type erase(const K &key) { + return m_ht.erase(key); + } + + /** + * @copydoc erase(const K& key) + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + size_type erase(const K &key, std::size_t precalculated_hash) { + return m_ht.erase(key, precalculated_hash); + } + + void swap(sparse_set &other) { other.m_ht.swap(m_ht); } + + /* + * Lookup + */ + size_type count(const Key &key) const { return m_ht.count(key); } + + /** + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + size_type count(const Key &key, std::size_t precalculated_hash) const { + return m_ht.count(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * `KeyEqual::is_transparent` exists. If so, `K` must be hashable and + * comparable to `Key`. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + size_type count(const K &key) const { + return m_ht.count(key); + } + + /** + * @copydoc count(const K& key) const + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + size_type count(const K &key, std::size_t precalculated_hash) const { + return m_ht.count(key, precalculated_hash); + } + + iterator find(const Key &key) { return m_ht.find(key); } + + /** + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + iterator find(const Key &key, std::size_t precalculated_hash) { + return m_ht.find(key, precalculated_hash); + } + + const_iterator find(const Key &key) const { return m_ht.find(key); } + + /** + * @copydoc find(const Key& key, std::size_t precalculated_hash) + */ + const_iterator find(const Key &key, std::size_t precalculated_hash) const { + return m_ht.find(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * `KeyEqual::is_transparent` exists. If so, `K` must be hashable and + * comparable to `Key`. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + iterator find(const K &key) { + return m_ht.find(key); + } + + /** + * @copydoc find(const K& key) + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + iterator find(const K &key, std::size_t precalculated_hash) { + return m_ht.find(key, precalculated_hash); + } + + /** + * @copydoc find(const K& key) + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + const_iterator find(const K &key) const { + return m_ht.find(key); + } + + /** + * @copydoc find(const K& key) + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + const_iterator find(const K &key, std::size_t precalculated_hash) const { + return m_ht.find(key, precalculated_hash); + } + + bool contains(const Key &key) const { return m_ht.contains(key); } + + /** + * Use the hash value 'precalculated_hash' instead of hashing the key. The + * hash value should be the same as hash_function()(key). Useful to speed-up + * the lookup if you already have the hash. + */ + bool contains(const Key &key, std::size_t precalculated_hash) const { + return m_ht.contains(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * KeyEqual::is_transparent exists. If so, K must be hashable and comparable + * to Key. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + bool contains(const K &key) const { + return m_ht.contains(key); + } + + /** + * @copydoc contains(const K& key) const + * + * Use the hash value 'precalculated_hash' instead of hashing the key. The + * hash value should be the same as hash_function()(key). Useful to speed-up + * the lookup if you already have the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + bool contains(const K &key, std::size_t precalculated_hash) const { + return m_ht.contains(key, precalculated_hash); + } + + std::pair equal_range(const Key &key) { + return m_ht.equal_range(key); + } + + /** + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + std::pair equal_range(const Key &key, + std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + std::pair equal_range(const Key &key) const { + return m_ht.equal_range(key); + } + + /** + * @copydoc equal_range(const Key& key, std::size_t precalculated_hash) + */ + std::pair equal_range( + const Key &key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * This overload only participates in the overload resolution if the typedef + * `KeyEqual::is_transparent` exists. If so, `K` must be hashable and + * comparable to `Key`. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + std::pair equal_range(const K &key) { + return m_ht.equal_range(key); + } + + /** + * @copydoc equal_range(const K& key) + * + * Use the hash value `precalculated_hash` instead of hashing the key. The + * hash value should be the same as `hash_function()(key)`, otherwise the + * behaviour is undefined. Useful to speed-up the lookup if you already have + * the hash. + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + std::pair equal_range(const K &key, + std::size_t precalculated_hash) { + return m_ht.equal_range(key, precalculated_hash); + } + + /** + * @copydoc equal_range(const K& key) + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + std::pair equal_range(const K &key) const { + return m_ht.equal_range(key); + } + + /** + * @copydoc equal_range(const K& key, std::size_t precalculated_hash) + */ + template < + class K, class KE = KeyEqual, + typename std::enable_if::value>::type * = nullptr> + std::pair equal_range( + const K &key, std::size_t precalculated_hash) const { + return m_ht.equal_range(key, precalculated_hash); + } + + /* + * Bucket interface + */ + size_type bucket_count() const { return m_ht.bucket_count(); } + size_type max_bucket_count() const { return m_ht.max_bucket_count(); } + + /* + * Hash policy + */ + float load_factor() const { return m_ht.load_factor(); } + float max_load_factor() const { return m_ht.max_load_factor(); } + void max_load_factor(float ml) { m_ht.max_load_factor(ml); } + + void rehash(size_type count) { m_ht.rehash(count); } + void reserve(size_type count) { m_ht.reserve(count); } + + /* + * Observers + */ + hasher hash_function() const { return m_ht.hash_function(); } + key_equal key_eq() const { return m_ht.key_eq(); } + + /* + * Other + */ + + /** + * Convert a `const_iterator` to an `iterator`. + */ + iterator mutable_iterator(const_iterator pos) { + return m_ht.mutable_iterator(pos); + } + + /** + * Serialize the set through the `serializer` parameter. + * + * The `serializer` parameter must be a function object that supports the + * following call: + * - `void operator()(const U& value);` where the types `std::uint64_t`, + * `float` and `Key` must be supported for U. + * + * The implementation leaves binary compatibility (endianness, IEEE 754 for + * floats, ...) of the types it serializes in the hands of the `Serializer` + * function object if compatibility is required. + */ + template + void serialize(Serializer &serializer) const { + m_ht.serialize(serializer); + } + + /** + * Deserialize a previously serialized set through the `deserializer` + * parameter. + * + * The `deserializer` parameter must be a function object that supports the + * following calls: + * - `template U operator()();` where the types `std::uint64_t`, + * `float` and `Key` must be supported for U. + * + * If the deserialized hash set type is hash compatible with the serialized + * set, the deserialization process can be sped up by setting + * `hash_compatible` to true. To be hash compatible, the Hash, KeyEqual and + * GrowthPolicy must behave the same way than the ones used on the serialized + * set. The `std::size_t` must also be of the same size as the one on the + * platform used to serialize the set. If these criteria are not met, the + * behaviour is undefined with `hash_compatible` sets to true. + * + * The behaviour is undefined if the type `Key` of the `sparse_set` is not the + * same as the type used during serialization. + * + * The implementation leaves binary compatibility (endianness, IEEE 754 for + * floats, size of int, ...) of the types it deserializes in the hands of the + * `Deserializer` function object if compatibility is required. + */ + template + static sparse_set deserialize(Deserializer &deserializer, + bool hash_compatible = false) { + sparse_set set(0); + set.m_ht.deserialize(deserializer, hash_compatible); + + return set; + } + + friend bool operator==(const sparse_set &lhs, const sparse_set &rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + + for (const auto &element_lhs : lhs) { + const auto it_element_rhs = rhs.find(element_lhs); + if (it_element_rhs == rhs.cend()) { + return false; + } + } + + return true; + } + + friend bool operator!=(const sparse_set &lhs, const sparse_set &rhs) { + return !operator==(lhs, rhs); + } + + friend void swap(sparse_set &lhs, sparse_set &rhs) { lhs.swap(rhs); } + + private: + ht m_ht; +}; + +/** + * Same as `tsl::sparse_set`. + */ +template , + class KeyEqual = std::equal_to, + class Allocator = std::allocator> +using sparse_pg_set = + sparse_set; + +} // end namespace tsl + +#endif diff --git a/docs/any.md b/docs/any.md index 6b03a9fd..5cc15775 100644 --- a/docs/any.md +++ b/docs/any.md @@ -211,7 +211,7 @@ assert(set.find(2.5) != set.end()); // use heterogeneous lookup assert(set.find("hello") != set.end()); // use heterogeneous lookup assert(set.find(tstring("world")) != set.end()); // use heterogeneous lookup assert(set.find(tstring_view("ok")) != set.end()) ; // use heterogeneous lookup -assert(set.find("ok") == set.end()) ; // "ok" not found has we compare 2 const char* -> pointer comparison, not string comparison +assert(set.find("ok") != set.end()) ; // we compare const char* with any containing const char* -> string comparison assert(set.find("no") == set.end()); // failed lookup ``` @@ -227,7 +227,6 @@ Two `hold_any` are considered equal if: - They hold the same type and both underlying objects compare equals. If the type does not provide a comparison operator, the operator==() always return false. - They both hold an arithmetic value of possibly different types, and these values compare equals. - They both hold a string like object (std::string, seq::tstring, seq::tstring_view, std::string_view, char*, const char*) that compare equals. - Note that a const char* can be compared to another string object (like std::string) using string comparison, but comparing two const char* will result in a pointer comparison! It is possible to register a comparison function for unrelated types using `seq::register_any_equal_comparison()` function. diff --git a/docs/v2.md b/docs/changelog.md similarity index 63% rename from docs/v2.md rename to docs/changelog.md index d1036343..56a45f99 100644 --- a/docs/v2.md +++ b/docs/changelog.md @@ -1,13 +1,39 @@ +# seq v2.1 -Notes on version 2 of seq library ---------------------------------- +The version 2.1 introduced additional breaking changes: +- The modules *charconv* and *format* are now deprecated. They are still available, but moved to the *legacy* folder. + Indeed, while I really like these, they do not belong to a library about containers. They will eventually be moved to another library, and removed from *seq*. +- The *tagged_pointer.hpp* file have been moved to the *internal* folder (private API). + +Additional changes: + +- The hashing framework has a better handling of transparent keys. +- The hashing framework now supports std::chrono::time_point and std::chrono::duration. +- The `seq::net_sort` algorithm has been slightly optimized/refactored and moved to *net_sort.hpp* header. +- The `radix_set/map` lower_bound() method has been corrected (compilation error). +- `radix_set/map` now supports std::chrono::time_point and std::chrono::duration as key. +- `radix_set/map` no longer have typedef `prefix_iterator` and `prefix_const_iterator`. Instead, member prefix_range() now returns a std::pair of regular iterators. +- All radix-based containers (`seq::radix_set/map`, `seq::radix_hash_set/map`) have been internally simplified. +- The `seq::hold_any` class has been refactored and simplified. A `seq::hold_any` containing a char* or const char* is now considered holding a string. Comparison of `seq::hold_any` containing a `char*` or `const char*` results in string comparison. +- New container: [`seq::concurrent_queue`](concurrent_queue.md). + +The markdown documentation has been updated accordingly, and new benchmark results were added. Full list of benchmarks now: +- Benchmark of [concurrent hash tables](concurrent_map.md) at the end of the page. Its goal is to compare `seq::concurrent_set/map` to other implementations. +- Benchmark on [sorted containers](sorted_benchmark.md). Its goal is to compare `seq::flat_set/map` and `seq::radix_set/map` to other implementations. +- Very tiny benchmark on [concurrent queues](concurrent_queue.md) to compare `seq::concurrent_queue` with other implementations. +- [Memory and latency benchmark](latency_benchmark.md) on hash tables to compare `seq::radix_hash_set/map` with other implementations. + + + +# seq v2.0 + The version 2 of `seq` introduced several changes on all modules, which are listed below. -Library wide changes --------------------- +## Library wide changes + - The biggest change is the library requirement which was upgraded to C++17. Indeed, working with C++14 was painfull and all compilers I now work with support at least C++17. - Another big change is the full removal of the `cvector` class (compressed vector-like container). Indeed, `cvector` relied on a compression algorithm that was heavily refactored and upgraded, up to the point where it did not belong to a library about containers... @@ -16,41 +42,35 @@ Library wide changes - [Pdqsort](https://github.com/orlp/pdqsort) is not used anymore within the library. Instead the `net_sort` algorithm (from [algorithm](algorithm.md) module) is used everywhere. - The `memory` module (deprecated in v1.3) was removed. -[bits](bits.md) --------------------- +## [bits](bits.md) - Internal refactoring. - Updated SEQ_LIKELY/SEQ_UNLIKELY to use c++20 [[likely]]/[[unlikely]] attributes if available. - Added class `fast_rand`: fast 32 bits random number generator. -[hash](hash.md) --------------------- +## [hash](hash.md) - Internal refactoring. - Moved implementation which was in `hash.cpp` to `hash_impl.hpp` -[charconv](charconv.md) ----------------------------- +## [charconv](charconv.md) - Internal refactoring - Removed file `charconv.cpp` - All functions to read/write integral/floating numbers are now template, and work on any character type instead of just `char`. -[format](format.md) ------------------------- +## [format](format.md) - Full refactoring of the module. - All functions now work with any character type including `wchar_t`, `char16_t`, `char32_t` and `char8_t` (if available). -[any](any.md) ------------------- +## [any](any.md) - Minor refactoring to simplify the code. - Now relies on seq::hasher instead of std::hash. -[containers](containers.md) --------------------------------- +## [containers](containers.md) - Sequential random-access containers: - [seq::devector](devector.md): @@ -87,10 +107,9 @@ Library wide changes - Added several type traits to help detect string types and character types. -[algorithm](algorithm.md) -------------------------- +## [algorithm](algorithm.md) -New module, provides several iterator based algorithms including the `net_sort` sorting one. +New module, provides several iterator based algorithms including the `net_sort` sorting one. diff --git a/docs/concurrent_queue.md b/docs/concurrent_queue.md new file mode 100644 index 00000000..e882768f --- /dev/null +++ b/docs/concurrent_queue.md @@ -0,0 +1,28 @@ +# Concurrent queue + +The new container `seq::concurrent_queue` was added in *seq v2.1*. As its name implies, it is a concurrent (thread safe) FIFO (First In First Out) designed for Multi-Producer Multi-Consumer (MPMC) scenarios. + +I tried several concurrent queue implementations but they all had their own limitations: +- [moodycamel::ConcurrentQueue](https://github.com/cameron314/concurrentqueue): very fast, but no guarantee on the dequeuing order which is a big stop for me. +- [boost::lockfree::queue](https://www.boost.org/doc/libs/1_53_0/doc/html/lockfree.html): I did not manage to get any performances out of it, maybe I configured it wrongly. But the type limitation is also a big no. +- [MPMCQueue](https://github.com/rigtorp/MPMCQueue): very good one, but sadly only work with preallocated storage. + +In the end I decided to roll my own implentation with the following criteria: +- Being faster than a regular std::mutex plus std::deque, +- Having the possibility to preallocate memory up-front, +- Always ensure FIFO behavior, +- Not being to harsh on the type requirement, +- Providing strong exception guarantee, +- Providing unsafe but usefull API (like iteration) + +The resulting `seq::concurrent_queue` is not fully lock-free nor wait-free, but combines atomic-based operations with locks to provide a certain level of concurrency. + +Below image is a simple benchmark of multiple concurrent queue implementations: [moodycamel::ConcurrentQueue](https://github.com/cameron314/concurrentqueue), [boost::lockfree::queue](https://www.boost.org/doc/libs/1_53_0/doc/html/lockfree.html), `seq::concurrent_queue` and a simple queue + mutex. +In this benchmark, N threads continuously push new values while N threads continuously try to dequeue elements. We stop the process when 10M elements have been successfully dequeued and measure the time it took (lower is better). +In order to simulate a real steady-state MPMC scenario, the dequeuing threads sleep 10ms each time the dequeue operation fails in order to give some room to the pushing threads for adding new values. +The benchmark ran on a Linux server equipped with 2 *Intel(R) Xeon(R) Gold 5220R CPU @ 2.20GHz* of 24 CPU cores each. + +![Dequeue](images/concurrent_queue.svg) + +`seq::concurrent_queue` performs really well and only start to get outpaced by `moodycamel::ConcurrentQueue` with 10 producer threads and 10 consumer threads. The choice between the 2 implementations really depend of your need regarding the dequeuing order. + diff --git a/docs/flat_set.md b/docs/flat_set.md index 3279cc41..2ff3a11e 100644 --- a/docs/flat_set.md +++ b/docs/flat_set.md @@ -43,6 +43,8 @@ All ``seq::flat_set`` operations only provide *basic exception guarantee*, exact ## Performances +For a more recent benchmark, see [here](sorted_benchmark.md). + Performances of `seq::flat_set` has been measured and compared to std::set, std::unordered_set, boost::flat_set and phmap::btree_set (based on abseil btree_set). The following table show the results when compiled with gcc 10.1.0 (-O3) for msys2 on Windows 10, using Intel(R) Core(TM) i7-10850H at 2.70GHz. Measured operations are: - Insert successfully a range of 1M unique double randomly shuffled using set_class::insert(first,last) diff --git a/docs/hash.md b/docs/hash.md index 2d8d757d..01350f0f 100644 --- a/docs/hash.md +++ b/docs/hash.md @@ -17,6 +17,7 @@ The *hash* module also provides its own hashing class called `seq::hasher` that, - `std::unique_ptr` and `std::shared_ptr` - `std::tuple` and `std::pair` - `std::basic_string`, `std::basic_string_view` and `seq::tiny_string` (`` must be included). +- `std::chrono::time_point` and `std::chrono::duration` For string types, `seq::hasher` uses a seeded version of [komihash](https://github.com/avaneev/komihash). komihash is a very fast hash function that passes all [SMhasher](https://github.com/rurban/smhasher) tests, and is especially efficient on small strings. diff --git a/docs/images/concurrent_queue.svg b/docs/images/concurrent_queue.svg new file mode 100644 index 00000000..e9f7dcdf --- /dev/null +++ b/docs/images/concurrent_queue.svg @@ -0,0 +1,863 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 1000 + + + 2000 + + + 3000 + + + 4000 + + + 5000 + + + 6000 + + + 7000 + + + 8000 + + + 9000 + + + 10000 + + + 0 + + + 2 + + + 4 + + + 6 + + + 8 + + + 10 + + + 12 + + + 14 + + + 16 + + + 18 + + + Milliseconds + + + Number of threads + + + Dequeue time per number of threads + + + + + + queue + mutex + + + + + + seq::concurrent_queue + + + + + + moodycamel::ConcurrentQueue + + + + + + boost::lockfree::queue + + + + diff --git a/docs/images/find_latency.svg b/docs/images/find_latency.svg new file mode 100644 index 00000000..ac4739c2 --- /dev/null +++ b/docs/images/find_latency.svg @@ -0,0 +1,868 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 500 + + + 1000 + + + 1500 + + + 2000 + + + 2500 + + + 3000 + + + 0 + + + 100000 + + + 200000 + + + 300000 + + + 400000 + + + 500000 + + + 600000 + + + 700000 + + + 800000 + + + 900000 + + + 1000000 + + + Max lookup time (ns) + + + Number of elements + + + Successfull lookup maximum latency + + + + + + tsl::sparse_set (2251ns) + + + + + + gtl::flat_hash_set (1326) + + + + + + ankerl::unordered_dense::set (2634) + + + + + + seq::radix_hash_set (1083) + + + + + + std::unordered_set (974) + + + + diff --git a/docs/images/insert_latency.svg b/docs/images/insert_latency.svg new file mode 100644 index 00000000..9d80e993 --- /dev/null +++ b/docs/images/insert_latency.svg @@ -0,0 +1,843 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + 50000001 + + + 100000001 + + + 150000001 + + + 200000001 + + + 250000001 + + + 300000001 + + + 350000001 + + + 400000001 + + + 0 + + + 100000 + + + 200000 + + + 300000 + + + 400000 + + + 500000 + + + 600000 + + + 700000 + + + 800000 + + + 900000 + + + 1000000 + + + Latency peak (ns) + + + Number of elements + + + Maximum latency per 10000 insertions (ns) + + + + + + tsl::sparse_set (359ms) + + + + + + gtl::flat_hash_set (66ms) + + + + + + ankerl::unordered_dense::set (171ms) + + + + + + seq::radix_hash_set (4,7ms) + + + + diff --git a/docs/images/level_merging.svg b/docs/images/level_merging.svg new file mode 100644 index 00000000..34550005 --- /dev/null +++ b/docs/images/level_merging.svg @@ -0,0 +1,1254 @@ + + + + + + + + + + + + + 0 + + 1 + + 2 + + 3 + + Leaf (1) + + Leaf (7) + + Null Leaf (0) + + Leaf (2) + + + + + + 0 + + 1 + + 2 + + 3 + + Leaf (12) + + Leaf (7) + + Null Leaf (0) + + Leaf (1) + + 0 + + 1 + + 2 + + 3 + + 0 + + 1 + + 2 + + 3 + + 0 + + 1 + + 2 + + 3 + + 0 + + 1 + + 2 + + 3 + + + + + + Leaf (5) + + Leaf (6) + + Leaf (2) + + Leaf (32) + + More leaves… + + + + + + + + + + + + Root directory(4) + + Root directory (4) + + Level 1 directoriesTree state after 10 insertionsTree state after 500 insertionsTree state after 500 insertions and level merging + + 1 + + 2 + + 3 + + 0 + + 1 + + 2 + + 3 + + Root directory (16) + + 4 + + 5 + + 6 + + 7 + + 8 + + 9 + + 10 + + 11 + + 12 + + 13 + + 14 + + 15 + + Leaf (12) + + Leaf (7) + + Null Leaf (0) + + Leaf (1) + + Leaf (5) + + Leaf (6) + + Leaf (2) + + Leaf (32) + + More leaves… + + + + + + + + + + + + + + + diff --git a/docs/images/map_insert.svg b/docs/images/map_insert.svg new file mode 100644 index 00000000..c8cd8e0c --- /dev/null +++ b/docs/images/map_insert.svg @@ -0,0 +1,2043 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 500 + + + 1000 + + + 1500 + + + 2000 + + + 2500 + + + 0 + + + 1000000 + + + 2000000 + + + 3000000 + + + 4000000 + + + Nanoseconds per insert + + + Container size + + + Insertion time (uint64_t) + + + + + + seq::flat_set + + + + + + seq::radix_set + + + + + + gtl::btree_set + + + + + + std::set + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 500 + + + 1000 + + + 1500 + + + 2000 + + + 2500 + + + 3000 + + + 3500 + + + 4000 + + + 4500 + + + 0 + + + 1000000 + + + 2000000 + + + 3000000 + + + 4000000 + + + Nanoseconds per insert + + + Container size + + + Insertion time (short string) + + + + + + flat_set + + + + + + radix_set + + + + + + gtl::btree_set + + + + + + std::set + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 1000 + + + 2000 + + + 3000 + + + 4000 + + + 5000 + + + 6000 + + + 7000 + + + 0 + + + 1000000 + + + 2000000 + + + 3000000 + + + 4000000 + + + Nanoseconds per insert + + + Container size + + + Insertion time (long string) + + + + + + flat_set + + + + + + radix_set + + + + + + gtl::btree_set + + + + + + std::set + + + + diff --git a/docs/images/map_lookup.svg b/docs/images/map_lookup.svg new file mode 100644 index 00000000..43f1a460 --- /dev/null +++ b/docs/images/map_lookup.svg @@ -0,0 +1,2043 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 200 + + + 400 + + + 600 + + + 800 + + + 1000 + + + 1200 + + + 1400 + + + 0 + + + 1000000 + + + 2000000 + + + 3000000 + + + 4000000 + + + Nanoseconds per lookup + + + Container + + + size + + + Successfull lookup time (uint64_t) + + + + + + seq::flat_set + + + + + + seq::radix_set + + + + + + gtl::btree_set + + + + + + std::set + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 200 + + + 400 + + + 600 + + + 800 + + + 1000 + + + 1200 + + + 0 + + + 1000000 + + + 2000000 + + + 3000000 + + + 4000000 + + + Nanoseconds per lookup + + + Container + + + size + + + Successfull lookup time (short string) + + + + + + flat_set + + + + + + radix_set + + + + + + gtl::btree_set + + + + + + std::set + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 500 + + + 1000 + + + 1500 + + + 2000 + + + 2500 + + + 0 + + + 1000000 + + + 2000000 + + + 3000000 + + + 4000000 + + + Nanoseconds per lookup + + + Container + + + size + + + Successfull lookup time (long string) + + + + + + flat_set + + + + + + radix_set + + + + + + gtl::btree_set + + + + + + std::set + + + + diff --git a/docs/images/memory_peak.svg b/docs/images/memory_peak.svg new file mode 100644 index 00000000..fc2a9aac --- /dev/null +++ b/docs/images/memory_peak.svg @@ -0,0 +1,868 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + 100 + + + 200 + + + 300 + + + 400 + + + 500 + + + 600 + + + 0 + + + 100000 + + + 200000 + + + 300000 + + + 400000 + + + 500000 + + + 600000 + + + 700000 + + + 800000 + + + 900000 + + + 1000000 + + + Memory peak (MB) + + + Number of elements + + + Memory peak every 10000 insertions (MB) + + + + + + tsl::sparse_set + + + + + + gtl::flat_hash_set + + + + + + ankerl::unordered_dense::set + + + + + + seq::radix_hash_set + + + + + + std::unordered_set + + + + diff --git a/docs/latency_benchmark.md b/docs/latency_benchmark.md new file mode 100644 index 00000000..58afd010 --- /dev/null +++ b/docs/latency_benchmark.md @@ -0,0 +1,85 @@ +# Hash table latency benchmark + +A lot of exhaustive hash table benchmarks can be found online, like for instance: +- From [jacksonallan](https://jacksonallan.github.io/c_cpp_hash_tables_benchmark/) +- From [Tessil](https://tessil.github.io/2016/08/29/benchmark-hopscotch-map.html) +- From [Martin Ankerl](https://martin.ankerl.com/2022/08/27/hashmap-bench-01/) +- From [skarupke](https://attractivechaos.wordpress.com/2018/01/13/revisiting-hash-table-performance/) +- From [renzibei](https://github.com/renzibei/hashtable-bench) + +In fact, every time a new hash table appear, a new benchmark is provided as well. I love these benchmarks, so let me follow the tradition and provide my own. Except that this time we're going to focus on slightly ignored metrics: +- The memory peak of several hast table implementations (this measure can be found on some benchmarks) +- The latency (maximum time to insert or find a value) which is mostly ignored. + +The goal here is to show the performances of the `seq::radix_hash_set/map` in this regard, which is based on the [Variable Arity Radix Tree](radix_tree.md) (VART), a concept of my own derived from the *Burst Trie*. +We do not compare ALL possible implementations in this benchmark, only a few hash tables (all great in their own regard) representing very different storage/probing strategies: +- [tsl::sparse_set](https://github.com/Tessil/sparse-map): low memory hash table (similar to `google::sparse_hash_set`) using quadratic probing, +- [gtl::flat_hash_set](https://github.com/greg7mdp/gtl): swiss table derived from `absl::flat_hash_set` using quadratic probing, +- [ankerl::unordered_dense::set](https://github.com/martinus/unordered_dense): double storage strategy using robin hood probing, +- `std::unordered_set`: the standard C++ hash table using chaining. We use the msvc implementation which is quite performant considering its requirements (and compared to libstdc++ one), +- [seq::radix_hash_set](radix_tree.md): *seq* library hash table based on VART and using incremental rehash. + +As far as I know, all hash tables use a factor of 2 growth policy. + +All benchmarks ran on an *Intel(R) Core(TM) i7-10850H* at 2.70GHz and were compiled with msvc 19.44.35219. + +## Memory peak benchmark + +The first benchmark measures the memory peak of each hash table based on the number of elements it contains. For that we insert 10M integers of 64 bits in each table using `seq::hasher` hash function. +Every 10000 insertions, the program memory peak is recorded. Therefore, we do NOT see the release of memory happening at the end of rehash, which is Ok since we are only interested in the memory peak. +The following graph shows the memory peak in MB based on the number of inserted values for each hash table. + +![MemoryPeak](images/memory_peak.svg) + +All implementations have a distinctive behavior: +- `std::unordered_set` is as expected the most memory hungry. Its memory usage pattern is composed of a peak during rehash, followed by a linear pattern due to the multitude of small allocations (one per entry). +- `gtl::flat_hash_set` behaves like a *regular* open-addressing hash table, with a single memory peak at each rehash. +- `ankerl::unordered_dense::set` displays multiple memory peaks as it uses a double storage strategy: one for the hash table (element index + probe distance) and one for the values (regular std::vector). Both storage double their capacities at different times. +- `tsl::sparse_set` displays tiny memory peaks (when the bucket table is effectively doubled) followed by a linear pattern (when each bucket receive new values). +- `seq::radix_hash_set` has a very low, almost linear memory pattern. This is thanks to it incremental rehash strategy as explained [here](radix_tree.md). + +Now, let's see if the memory usage is correlated to the insertion latency. + +## Insertion latency benchmark + +The goal here is to measure the MAXIMUM time required for an insertion. This measure is usefull for low latency systems, where the hash table maximum size cannot be predicted up-front. +For that, we insert 10M values within the hash table and measure each individual insertion time using the most precise possible measurement method (QueryPerformanceCounter on Windows). +For this benchmark, the measure precision is not an issue as we expect high values during rehash. The following graph displays the maximum insertion time in nanoseconds based on the number of elements. The maximum time reached is displayed in the legend. +I removed the `std::unordered_set` curve which is too high and makes it impossible to interpret the results (and log scale does not help either). Just know that it's roughly 2 times slower than the second slowest (`tsl::sparse_set`). + +![InsertLatency](images/insert_latency.svg) + +Some conclusions: +- `gtl::flat_hash_set` is rather fast to rehash, but still needs 66ms to insert a single value at most. +- `ankerl::unordered_dense::set` rehash process is slightly slower, as robin hood insertion process might require to move around several elements. We also see secondary peaks corresponding to the vector resize. +- `tsl::sparse_set` rehash process is... slow. No wonder as it requires a lot of small allocations/deallocations, that's the price of a small memory usage. If you're unlucky, a single insertion in a big table might take more than... 300ms. +- `seq::radix_hash_set` has a very low latency compared to other implementations. Its biggest insertion time (4.7ms) is 10 times lower than the second fastest (`gtl::flat_hash_set`). + +So far so good, we see that `seq::radix_hash_set` manages to combine very low memory usage with low insertion latency. But this is also the case for a regular `std::map`... that we usually don't want to use for single point lookup. +To be complete, we also need to measure the latency of successfull and failed lookup operations. + +## Lookup latency benchmark + +In this benchmark, we want to know the maximum time taken by a single `find()` operation, weither successfull or not. +The process is slightly different than with previous benchmark: +- We insert 10M elements to a hash table, +- Every 10k elements, we measure the time it takes to look for 100 random existing values and compute the average time. We cannot just measure a single lookup which is way too fast for our measurement method. +- Every 10k elements, we measure in the same way the average time to look for 100 random non existing values. + +In the end failed lookup latencies are very similar to the successfull ones, therefore I just give the later: + +![FindLatency](images/find_latency.svg) + +First (good) news, all hash table behave in O(1) complexity. We already knew that, but it's still pleasing to the eyes. As for the (light) interpretation: +- `gtl::flat_hash_set` is rather fast with a low dispersion, like `seq::radix_hash_set`. +- `ankerl::unordered_dense::set` and `tsl::sparse_set` have a higher average lookup time with a higher dispersion. +- And the winner is... **std::unordered_set**!! This is the first benchmark I see with `std::unordered_set` on top (well, the good top). + +These results contradict most online lookup benchmarks. Indeed, `ankerl::unordered_dense::set` is usually very fast, while `std::unordered_set` not so much (although the msvc implementation is really good for lookup). +I suspect that traditional benchmarks usually loop through all existing values in the hash table, which probably tends to highlight 2 things: +- the cache locality differences between each implementation (even if the access pattern is random), +- the efficiency of the probing strategy. + +In this becnhmark, we only look for a few values in-between lots of insertions (that might slightly trash the cache). Therefore we probably are in a cold cache scenario that gives different results. Weither this scenario is more realistic or not depends on your use-case. + +As a side note, we confirmed that `seq::radix_hash_set` has a low lookup latency in addition to low insertion latency and low memory footprint! \ No newline at end of file diff --git a/docs/radix_tree.md b/docs/radix_tree.md index 211fd77b..d9a48e88 100644 --- a/docs/radix_tree.md +++ b/docs/radix_tree.md @@ -26,11 +26,16 @@ The VART is a Burst Trie with the following specificities: - Each leaf node is a flat sorted array of keys (or pairs of key/value for the radix_map) containing at most MAX keys, where MAX depends on the key size (typically 64). - Intermediate nodes are called 'directories' and can potentially have an unlimited number of children (other directories or leaf nodes). - Since keys are stored 'as is' without compression, `seq::radix_set` and `seq::radix_map` can provide the same interface as `std::set` and `std::map`. -- Keys can be arithmetic types, pointers, standard strings or wide strings, vectors, arrays... (anything with a `data()` and `size()` member). +- Keys can be arithmetic types, pointers, std::chrono time_point and duration, standard strings or wide strings, vectors, arrays... (anything with a `data()` and `size()` member). - Keys are unique within the tree. For the rest of the documentation, we consider a VART with a start arity of 4 and maximum leaf node size of 64 keys. +The `seq::radix_set` class has been [benchmarked](sorted_benchmark.md) against other sorted containers. + +The `seq::radix_hash_set` memory usage and latency was [benchmarked](docs/latency_benchmark.md) against other hash tables. + + ## Insertion process @@ -56,44 +61,13 @@ VART also uses path compression to reduce the tree depth for keys sharing common At some point during the insertion process, a directory (the root one or an intermediate directory) will be entirely filled with sub-directories. These sub-directories can be removed to efficiently reduce the tree depth. For that, the parent directory arity is multiplied by 4, the sub-directories are suppressed, and their children nodes (other directories or leaf nodes) are reinserted into the parent directory. We don't need to process the keys in order to find the position of each node within the new parent directory, as it is the combination of the sub-directory position and its child position. -Below is a simplified code sample describing this process: - -```cpp - -// here, 'parent' is the parent directory -// create a new parent directory with an arity equals to parent_arity * 4 -// Note that the arity is stored as its log2 representation (number of bits) -directory* new_parent = directory::create(parent->bit_length + 2); - -// Walk through the sub-directories -for (unsigned i = 0; i < parent->child_count(); ++i) -{ - // get sub-directory - directory* sub_dir = parent->child(i); +Below is a an image displaying the radix tree state after 10 insertions (state 1), after 500 insertions (state 2), and after the level merging (state 3). We can see that inserting 500 values creates intermediate directories (level 1 directories, state 2). +Once the root directory only contains sub-directories (state 2), the root arity is increased, level 1 directories are suppressed and leaves are linked to the root directory (state 3). - // walk through all nodes within sub_dir - for (unsigned j = 0; j < sub_dir->child_count(); ++j) - { - void* child = sub_dir->child(j); // child could be null, a directory or a leaf node - - // compute the child position within new_parent - unsigned position = j | (i << sub_dir->bit_length); - - // add child to new_parent - new_parent->set_child(position, child); - } - - // we can safely destroy sub_dir as we moved all its children to new_parent - directory::destroy(sub_dir); -} - -// we can safely destroy parent as we moved its content to new_parent -directory::destroy(parent); - -``` +![LevelMerging](images/level_merging.svg) -This level merging process is done on insertion step 4, during the keys dispatching to new nodes. The above code sample is straightforward and only works when all subdirectories have the same arity of 4. +This level merging process is done on insertion step 4, during the keys dispatching to new nodes. The above image is straightforward and only works when all subdirectories have the same arity of 4. Other scenarios are possible: the sub-directories have variable arities (because of other level mergings), or arities greater than 4, or store a non 0 prefix length. All these cases are not described here, but properly handled by the VART. The insertion process is in fact more complex than described previously: because of this level merging, each intermediate directory might consume more than 2 bits of the key to get its child index, depending on its arity. @@ -102,7 +76,7 @@ The level merging will reduce the depth of some branches of the tree. For an eve The `seq::radix_hash_set` and `seq::radix_hash_map` classes (hash tables) use this property to provide fast lookup. -## Performances +## Complexity Most types of radix trees (like ART one) support O(k) operations (where k is the size of the key in bytes). `seq::radix_set/map` does not depend on the key sizes, but instead on their distribution. In terms of big O notation, `seq::radix_set/map` behaves in the following way: @@ -133,7 +107,7 @@ Starting version 2 of `seq`, the VART rebalance itself on erasure for `seq::radi Both `seq::radix_hash_set` and `seq::radix_hash_map` uses VART behind the scene, except that the tree structure is built upon the hashed representation of the keys instead of the keys themselves. The step 4 of the insertion process (key dispatching) requires to rehash the keys within a leaf node in order to find their new locations. Therefore, the tree grows using incremental rehash by chunks of 64 keys. This is very similar to *extendible hashing*, except that a poor hash function will result in an unbalanced tree (still with a low memory footprint) instead of a huge and sparsely populated root directory. With a good hash function, the tree usually becomes a flat array (root directory) of leaf nodes, ensuring fast lookups. -A `seq::radix_hash_set/map` grows rather smoothly for a hash table, with an almost linear memory pattern. The absence of memory peak makes it one of the least memory gready hash table implementation. +A `seq::radix_hash_set/map` grows rather smoothly for a hash table, with an almost [linear memory pattern](docs/latency_benchmark.md). The absence of memory peak makes it one of the least memory gready hash table implementation. Another benefit of `seq::radix_hash_set/map` is its lower latency on insert/erase operations thanks to the incremental rehash, making it more suitbale for firm real-time applications. ### Collision resolution diff --git a/docs/sorted_benchmark.md b/docs/sorted_benchmark.md new file mode 100644 index 00000000..9e6436ec --- /dev/null +++ b/docs/sorted_benchmark.md @@ -0,0 +1,46 @@ +# Sorted containers benchmark + +This document summarizes benchmark results between various sorted containers to demonstrate their differences in speed and complexity. The goal is mostly to place the *seq* containers compared to other well known implementations. The following containers are tested: +- `std::set`: this is the standard version of a sorted container, always a good idea to position ourself based on it. Its implementation is usually based on a Red-Black (RB) tree. +- [`gtl::btree_set`](https://github.com/greg7mdp/gtl): B-tree set derived from [`absl::btree_set`](https://abseil.io/about/design/btree). Its internal nodes can store multiple values for usually better performances than a regular RB-tree. The number of stored values per node depends on the value type size. +- [seq::flat_set](flat_set.md): flat set data structure within the *seq* library offering random access in addition to *standard* features. `seq::flat_set` is implemented on top of `seq::tiered_vector` for fast insertion/deletion of single element as opposed to most flat containers like [`boost::container::flat_set`](https://www.boost.org/doc/libs/1_53_0/doc/html/boost/container/flat_set.html). +- [seq::radix_set](radix_tree.md): Radix tree using what I called Variable Arity Radix Tree ([VART](radix_tree.md)). I created this data structure based on the regular Burst Trie. + +The benchmarks test 2 operations only: successfull insert and successfull lookup based on the number of entries in the container. For all tested containers, erasure and failed lookups are very similar to tested operations in terms of speed and are not displayed here. + +Additional operations could have been tested, like iteration performances, failed insertion, range insertion, different input distributions... I might do it in the future, but my spare time is currently quite limited. Here are just a few hints based on internal benchmarks that I did not take the time to properly formalize: +- `std::set` is the slowest in all tested scenarios for all operations. Indeed, its requirements (iterator/reference stability) require an allocation/deallocation for each insertion/deletion, and a lot of cache misses for lookup. +- `seq::radix_set` is almost always the fastest in all tested scenarios, except for iteration where it is beaten by `seq::flat_set` which is as fast as a `std::deque`. +- `seq::radix_set` is not comparison based, and can be slower than other containers with adverserial input distribution, like 'a', 'aa', 'aaa',... However, internal guardrails ensure that its performances remain comparable to `seq::flat_set` with an additional time constant. +- Both `gtl::btree_set` and `seq::flat_set` are very sensible to the value type size. In addition, `seq::flat_set` is MUCH faster for insertion/deletion with relocatable types (based on `seq::is_relocatable` type trait). For large value type size, `gtl::btree_set` behaves exactly like `std::set`. +- `seq::radix_set` and `seq::flat_set` are both faster when using range insertion. + +The 2 benchmarks ran on Windows 11 containing an *Intel(R) Core(TM) i7-10850H* at 2.70GHz. + +## Successfull insertion benchmark + +The following graphs show the average insertion time (one by one) in nanoseconds based on the container current size (lower is better) for 3 different value types: +- random 64 bits integer, +- random short ascii `[seq::tstring](tiny_string.md)` of 15 characters (using Small String Optimization), +- random long ascii string of 64 characters (each requiring an allocation). + +![Insert](images/map_insert.svg) + +A few things can be said: +- `seq::radix_set` is always the fastest. Its insertion complexity is always in-between O(1) for random input, up to O(sqrt(N)) for adverserial input (similar to `seq::flat_set`) and unlike most radix trees that perform in O(k) (k is the input length). +- `seq::flat_set` quickly become the slowest container with its O(sqrt(N)) complexity. However, for 8 bytes value, it remains faster than `std::set` until 1M values, which is impressive for a flat map (as compared to boost::flat_set for instance). +- For long strings, finding the insertion location will trigger several cache misses on most containers. `seq::radix_set` is way faster as it mostly only use the inserted value itself to build its internal insertion path. + +## Successfull lookup benchmark + +The following graphs show the average successfull lookup time in nanoseconds based on the container current size (lower is better) for 3 different value types: +- random 64 bits integer, +- random short ascii `seq::tstring` of 15 characters (using Small String Optimization), +- random long ascii string of 64 characters (each requiring an allocation). + +![Lookup](images/map_lookup.svg) + +Results here are slightly different than for the insertion benchmark, and we clearly see the O(log(N)) asymptotic complexity for most containers. Some remarks: +- For random input of fixed length size (which is the case here as the strings are bounded), `seq::radix_set` has a complexity of O(1) as almost each input creates a unique lookup path. In fact, for truely random integers, `seq::radix_set` behaves almost exactly like a hash table. Its worst complexity (with adverserial input) is always O(log(N)) as it degenerate to a flat map with an additional time constant. +- `seq::flat_map` performs in O(log(N)) as it uses a kind of binary search internally, like most flat map implementations. It is very close to `gtl::btree_set` in performances, and will outpace it for bigger value type size. + diff --git a/seq/algorithm.hpp b/seq/algorithm.hpp index b3a6f5ee..f1ed7b5d 100644 --- a/seq/algorithm.hpp +++ b/seq/algorithm.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -25,1329 +25,11 @@ #ifndef SEQ_ALGORITHM_HPP #define SEQ_ALGORITHM_HPP -#include "bits.hpp" -#include "type_traits.hpp" +#include "net_sort.hpp" #include "internal/concurrent_hash_table.hpp" -#include -#include -#include -#include -#include - -#define SEQ_ALGO_ASSERT_DEBUG(condition, msg) SEQ_ASSERT_DEBUG(condition, msg) - namespace seq { - namespace algo_detail - { - // Unspecified length - static constexpr unsigned Unspecified = (unsigned)-1; - - /// @brief Iterator wrapper for bidirectional iterator - template - class IterWrapper - { - template - void increment_iter(Iter& it, Diff d) - { - if constexpr (has_plus_equal::value) - it += d; - else - std::advance(it, d); - } - - public: - using value_type = typename std::iterator_traits::value_type; - using iterator_category = typename std::iterator_traits::iterator_category; - using difference_type = typename std::iterator_traits::difference_type; - using reference = typename std::iterator_traits::reference; - using pointer = typename std::iterator_traits::pointer; - - Iter iter; - ptrdiff_t pos; - - SEQ_ALWAYS_INLINE IterWrapper(Iter it = Iter(), std::ptrdiff_t p = 0) noexcept - : iter(it) - , pos(p) - { - } - SEQ_ALWAYS_INLINE auto operator*() const noexcept -> reference { return (*this->iter); } - SEQ_ALWAYS_INLINE auto operator++() noexcept -> IterWrapper& - { - ++iter; - ++pos; - return *this; - } - SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> IterWrapper - { - IterWrapper it = *this; - ++(*this); - return it; - } - SEQ_ALWAYS_INLINE auto operator--() noexcept -> IterWrapper& - { - --iter; - --pos; - return *this; - } - SEQ_ALWAYS_INLINE auto operator--(int) noexcept -> IterWrapper - { - IterWrapper it = *this; - --(*this); - return it; - } - template - SEQ_ALWAYS_INLINE auto operator+=(Diff diff) noexcept -> IterWrapper& - { - pos += static_cast(diff); - increment_iter(iter, static_cast(diff)); - return *this; - } - template - SEQ_ALWAYS_INLINE auto operator-=(Diff diff) noexcept -> IterWrapper& - { - pos -= static_cast(diff); - increment_iter(iter, -static_cast(diff)); - return *this; - } - }; - template - SEQ_ALWAYS_INLINE auto operator+(const IterWrapper& it, typename IterWrapper::difference_type diff) noexcept - { - IterWrapper res = it; - res += diff; - return res; - } - template - SEQ_ALWAYS_INLINE auto operator-(const IterWrapper& it, typename IterWrapper::difference_type diff) noexcept - { - IterWrapper res = it; - res -= diff; - return res; - } - template - SEQ_ALWAYS_INLINE auto operator-(const IterWrapper& it1, const IterWrapper& it2) noexcept - { - return it1.pos - it2.pos; - } - template - SEQ_ALWAYS_INLINE auto operator==(const IterWrapper& it1, const IterWrapper& it2) noexcept - { - return it1.iter == it2.iter; - } - template - SEQ_ALWAYS_INLINE auto operator!=(const IterWrapper& it1, const IterWrapper& it2) noexcept - { - return it1.iter != it2.iter; - } - - // noexcept specifier for inplace merge/sort - template - struct NothrowSort - { - // Check if sorting given type with given comparator does not throw - static constexpr bool value = std::is_nothrow_move_assignable_v && std::is_nothrow_move_constructible_v && std::is_nothrow_constructible_v && - noexcept(std::declval()(std::declval(), std::declval())); - }; - // noexcept specifier for inplace merge/sort - template - struct NothrowSortIter : public NothrowSort::value_type, Cmp> - { - }; - - // Interpret element as bitfield for relocatable types - template - struct Bitfield - { - alignas(T) char data[sizeof(T)]; - }; - template - SEQ_ALWAYS_INLINE Bitfield& as_bits(T& ref) noexcept - { - return (Bitfield&)ref; - } - template - SEQ_ALWAYS_INLINE Bitfield& as_bits(T&& ref) noexcept - { - return (Bitfield&)ref; - } - template - SEQ_ALWAYS_INLINE T& as_type(Bitfield& d) - { - return (T&)d; - } - - // similar to std::next, but use ++ operator when no distance is specified - template - SEQ_ALWAYS_INLINE auto iter_next(Iter it) noexcept - { - return ++it; - } - template - SEQ_ALWAYS_INLINE auto iter_next(Iter it, Diff d) noexcept - { - return it + d; - } - - // similar to std::prev, but use -- operator when no distance is specified - template - SEQ_ALWAYS_INLINE auto iter_prev(Iter it) noexcept - { - return --it; - } - template - SEQ_ALWAYS_INLINE auto iter_prev(Iter it, Diff d) noexcept - { - return it - d; - } - - template - auto unwrap_iter(Iter it) noexcept - { - return it; - } - template - auto unwrap_iter(std::move_iterator it) noexcept - { - return unwrap_iter(it.base()); - } - template - auto unwrap_iter(std::reverse_iterator it) noexcept - { - return unwrap_iter(it.base()); - } - - // compare iterators for equality without triggering compile error - template - static SEQ_ALWAYS_INLINE bool iter_equal(Iter1 it1, Iter2 it2) noexcept - { - using type1 = decltype(unwrap_iter(it1)); - using type2 = decltype(unwrap_iter(it2)); - if constexpr (std::is_same_v) - return unwrap_iter(it1) == unwrap_iter(it2); - else - return false; - } - - // similar to std::distance with an overload for IterWrapper that supports subtracting 2 iterators - template - static SEQ_ALWAYS_INLINE auto iter_distance(Iter first, Iter last) noexcept - { - return std::distance(first, last); - } - template - static SEQ_ALWAYS_INLINE auto iter_distance(const IterWrapper& first, const IterWrapper& last) noexcept - { - return last - first; - } - - template - auto wrap_iter(Iter it, ptrdiff_t d = 0) - { - if constexpr (is_random_access::value) - return it; - else - return IterWrapper(it, d); - } - - template - static void merge_inplace_left_subproblem(Iter f0, - size_t n0, - Iter f1, - size_t n1, - Iter& f0_0, - size_t& n0_0, - Iter& f0_1, - size_t& n0_1, - Iter& f1_0, - size_t& n1_0, - Iter& f1_1, - size_t& n1_1, - Cmp r) noexcept(NothrowSortIter::value) - { - // Subroutine of inplace_merge_n - SEQ_ALGO_ASSERT_DEBUG((size_t)iter_distance(f0, f1) == n0, ""); - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(f0, iter_next(f0, n0), r) && std::is_sorted(f1, iter_next(f1, n1), r), ""); - SEQ_ALGO_ASSERT_DEBUG(n0 > 0, ""); - SEQ_ALGO_ASSERT_DEBUG(n1 > 0, ""); - - f0_0 = f0; - n0_0 = n0 >> 1; - f0_1 = f0; - f0_1 = iter_next(f0_1, n0_0); - f1_1 = std::lower_bound(f1, iter_next(f1, n1), *f0_1, r); - f1_0 = std::rotate(f0_1, f1, f1_1); - n0_1 = iter_distance(f0_1, f1_0); - ++f1_0; - n1_0 = (n0 - n0_0) - 1; - n1_1 = n1 - n0_1; - } - - template - static void merge_inplace_right_subproblem(Iter f0, - size_t n0, - Iter f1, - size_t n1, - Iter& f0_0, - size_t& n0_0, - Iter& f0_1, - size_t& n0_1, - Iter& f1_0, - size_t& n1_0, - Iter& f1_1, - size_t& n1_1, - Cmp r) noexcept(NothrowSortIter::value) - { - // Subroutine of inplace_merge_n - SEQ_ALGO_ASSERT_DEBUG((size_t)iter_distance(f0, f1) == n0, ""); - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(f0, iter_next(f0, n0), r) && std::is_sorted(f1, iter_next(f1, n1), r), ""); - SEQ_ALGO_ASSERT_DEBUG(n0 > 0, ""); - SEQ_ALGO_ASSERT_DEBUG(n1 > 0, ""); - - f0_0 = f0; - n0_1 = n1 >> 1; - f1_1 = f1; - f1_1 = iter_next(f1_1, n0_1); - f0_1 = std::upper_bound(f0, iter_next(f0, n0), *f1_1, r); - ++f1_1; - f1_0 = std::rotate(f0_1, f1, f1_1); - n0_0 = iter_distance(f0_0, f0_1); - n1_0 = n0 - n0_0; - n1_1 = (n1 - n0_1) - 1; - } - - template - SEQ_ALWAYS_INLINE Out copy_internal(Iter begin, Iter end, Out out) - { - // direct std::copy call - return std::copy((begin), (end), (out)); - } - template - SEQ_ALWAYS_INLINE Out copy_internal(std::move_iterator begin, std::move_iterator end, Out out) - { - // Let the compiler decide to use memmove if necesary - return std::move((begin.base()), (end.base()), (out)); - } - - template - inline Out merge_forward(Iter1 first1, Iter1 end1, Iter2 first2, Iter2 end2, Out out, Cmp c) - { - // Merge 2 range forward - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(first1, end1, c), ""); - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(first2, end2, c), ""); - SEQ_DEBUG_ONLY(Out dst = out;) - - if constexpr (is_random_access::value && is_random_access::value) { - - // Check for unbalanced merge - auto s1 = iter_distance(first1, end1); - auto s2 = iter_distance(first2, end2); - - if (s1 * 32 < s2) { - // Left is way smaller than right - for (; first1 != end1; ++first1) { - if (first2 != end2) { - while (first1 != end1 && !c(*first2, *first1)) - *out++ = (*first1++); - if (first1 == end1) - break; - - auto found = std::lower_bound(first2, end2, *first1, c); - out = copy_internal(first2, found, out); - first2 = found; - } - *out++ = (*first1); - } - goto end; - } - else if (s2 * 32 < s1) { - // Right is way smaller than left - for (; first2 != end2; ++first2) { - if (first1 != end1) { - while (first2 != end2 && c(*first2, *first1)) - *out++ = (*first2++); - if (first2 == end2) - break; - - auto found = std::upper_bound(first1, end1, *first2, c); - out = copy_internal(first1, found, out); - first1 = found; - } - *out++ = (*first2); - } - goto end; - } - } - - // More efficient merge than std::merge (usually) - - while (first2 != end2) { - while (first1 != end1 && !c(*first2, *first1)) { - - *out = (*first1); - ++out; - ++first1; - } - - if (first1 == end1) - break; - - *out = (*first2); - ++out; - ++first2; - - while (first2 != end2 && c(*first2, *first1)) { - - *out = (*first2); - ++out; - ++first2; - } - - *out = (*first1); - ++out; - ++first1; - } - - end: - out = copy_internal(first1, end1, out); - if (Overlap && iter_equal(first2, out)) { - // The last range is already inplace: - // just advance output iterator if not - // a std::reverse_iterator (used by - // merge_backward which do not use the result). - if constexpr (!is_reverse_iterator::value) - out = iter_next(out, iter_distance(first2, end2)); - } - else - out = copy_internal(first2, end2, out); - - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(dst, out, c), ""); - return out; - } - - template - static SEQ_ALWAYS_INLINE std::pair merge_tails(Iter* first, Iter* second, Out& out_left, Out& out_right, Cmp c) - { - // Merge tails and advance - bool left_order = c(*first[1], *first[0]); - bool right_order = !c(*second[1], *second[0]); - *out_left = std::move(*first[left_order]); - *out_right = std::move(*second[right_order]); - ++out_left; - --out_right; - first[1] += left_order; - first[0] += !left_order; - second[1] -= right_order; - second[0] -= !right_order; - return { left_order, right_order }; - } - - template - static SEQ_ALWAYS_INLINE void finish_bidirectional_merge(Iter* first, Iter* second, Out out_left, Cmp c) - { - bool finish_left = (second[0] < first[0]); - bool finish_right = (second[1] < first[1]); - - if (!finish_left && !finish_right) { - merge_forward( - std::make_move_iterator(first[0]), std::make_move_iterator(++second[0]), std::make_move_iterator(first[1]), std::make_move_iterator(++second[1]), out_left, c); - } - else if (finish_left) - std::move((first[1]), (++second[1]), (out_left)); - else if (finish_right) - std::move((first[0]), (++second[0]), (out_left)); - } - - template - static Out merge_move_bidirectional(Iter first1, Iter last1, Iter first2, Iter last2, Out out, Cmp c, Out* out_end = nullptr) noexcept(NothrowSortIter::value) - { - using T = typename std::iterator_traits::value_type; - - // Merge 2 sorted ranges to given output. - // Uses the fastest available method: standard forward merge - // or branchless merge from both ends for random access iterators and relocatable types. - - if constexpr (is_random_access::value && is_relocatable::value) { - - // Branchless merge from both ends - // Only truly faster with trivial comparison function, - // which is usually the case for relocatable types. - - Out out_left = out; - Iter first[2] = { first1, first2 }; - Iter second[2] = { iter_prev(last1), iter_prev(last2) }; - - if constexpr (Count != Unspecified) { - Out res = out_end ? *out_end : iter_next(out, Count * 2); - Out out_right = iter_prev(res); - - while (first[0] < second[0] && first[1] < second[1]) - merge_tails(first, second, out_left, out_right, c); - - finish_bidirectional_merge(first, second, out_left, c); - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(out, res, c), ""); - return res; - } - else { - - ptrdiff_t dist1 = iter_distance(first1, last1); - ptrdiff_t dist2 = iter_distance(first2, last2); - - Out res = out_end ? *out_end : iter_next(out, dist1 + dist2); - Out out_right = iter_prev(res); - - // Unbalanced merge from both ends. - // For the first part of the merge (1/16 of the smallest range), - // check if the order is pseud random. If not, finish with - // merge_forward(). - - const ptrdiff_t iter_count = std::min(dist1, dist2); - const ptrdiff_t stop = iter_count / 16; - ptrdiff_t order = 0; - ptrdiff_t count = 0; - - std::pair prev_order = { true, true }; - - if (first[0] < second[0] && first[1] < second[1]) { - - prev_order = merge_tails(first, second, out_left, out_right, c); - - while (count < stop && first[0] < second[0] && first[1] < second[1]) { - auto ord = merge_tails(first, second, out_left, out_right, c); - order += (ptrdiff_t)(ord.first == prev_order.first) + (ptrdiff_t)(ord.second == prev_order.second); - prev_order = ord; - ++count; - } - if (order <= stop + stop / 2) { - // Balanced merging: keep using bidirectional merge - while (first[0] < second[0] && first[1] < second[1]) - merge_tails(first, second, out_left, out_right, c); - } - - // Finish with merge_forward - finish_bidirectional_merge(first, second, out_left, c); - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(out, res, c), ""); - return res; - } - } - } - // Standard forward merge - return merge_forward(std::make_move_iterator(first1), std::make_move_iterator(last1), std::make_move_iterator(first2), std::make_move_iterator(last2), out, c); - } - - template - void merge_backward(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Out out_end, Cmp c) noexcept(NothrowSortIter::value) - { - // Merge backward implemented in terms of merge_forward - merge_forward(std::make_reverse_iterator(last2), - std::make_reverse_iterator(first2), - std::make_reverse_iterator(last1), - std::make_reverse_iterator(first1), - std::make_reverse_iterator(out_end), - [c](const auto& a, const auto& b) { return c(b, a); }); - } - - template - static void merge_with_buffer(Iter first, size_t n0, Iter middle, size_t n1, Iter e1, Cmp r, B buffer) noexcept(NothrowSortIter::value) - { - // Inplace merge 2 ranges using provided buffer. - // Moves as few elements as possible to the temporary buffer. - if (n0 <= n1) { - auto blast = std::move((first), (middle), buffer); - merge_forward(std::make_move_iterator(buffer), std::make_move_iterator(blast), std::make_move_iterator(middle), std::make_move_iterator(e1), first, r); - } - else { - auto last = e1; - auto blast = std::move((middle), (last), buffer); - merge_backward(std::make_move_iterator(first), std::make_move_iterator(middle), std::make_move_iterator(buffer), std::make_move_iterator(blast), last, r); - } - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(first, iter_next(middle, n1), r), ""); - } - - template - void rotate_one_right(Iter first, Iter mid, Iter last) - { - // Exchanges the range [first, mid) with [mid, last) - // pre: distance(mid, last) is 1 - using type = typename std::iterator_traits::value_type; - type tmp(std::move(*mid)); - std::move_backward((first), (mid), (last)); - *first = std::move(tmp); - } - - template - static void merge_adaptive_n(Iter f0, size_t n0, Iter f1, size_t n1, Iter e1, Cmp r, B buffer) noexcept(NothrowSortIter::value) - { - // Inplace merge with buffer, first published by Dudzin'sky and Dydek in 1981 IPL 12(1):5-8 - // Implementation from: https://www.jmeiners.com/efficient-programming-with-components/15_merge_inplace.html - - SEQ_ALGO_ASSERT_DEBUG((size_t)iter_distance(f0, f1) == n0, ""); - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(f0, iter_next(f0, n0), r), ""); - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(f1, iter_next(f1, n1), r), ""); - - if constexpr (FirstChecks) { - // Perform the firsts, easy checks - - if (!n0 || !n1 || !r(*f1, *iter_prev(f1))) - // One of the 2 ranges is empty, or already sorted - return; - - if (r(*iter_prev(e1), *f0)) { - // Simple rotation needed - std::rotate((f0), (f1), (e1)); - return; - } - - // The following checks come from msvc STL implementation - // and help a LOT in some situations - - // Increment f0 as long as it is smaller than first value of second range - for (;;) { - if (f0 == f1) - // We reach the end of first range: already in order - return; - if (r(*f1, *f0)) - break; - ++f0; - --n0; - } - - auto highest = iter_prev(f1); - do { - --e1; - --n1; - if (f1 == e1) { // rotate only element remaining in right partition to the beginning, without allocating - rotate_one_right(f0, f1, ++e1); - return; - } - } while (!r(*e1, *highest)); // found that *highest goes in *e1's position - - ++e1; - ++n1; - } - - if (n0 <= buffer.size || n1 <= buffer.size) - // We have enough buffer: merge - return merge_with_buffer((f0), n0, (f1), n1, (e1), r, (buffer.first)); - - // Rotate left or right range - Iter f0_0, f0_1, f1_0, f1_1; - size_t n0_0, n0_1, n1_0, n1_1; - if (n0 < n1) - merge_inplace_left_subproblem(f0, n0, f1, n1, f0_0, n0_0, f0_1, n0_1, f1_0, n1_0, f1_1, n1_1, r); - else - merge_inplace_right_subproblem(f0, n0, f1, n1, f0_0, n0_0, f0_1, n0_1, f1_0, n1_0, f1_1, n1_1, r); - - // Recurse on each range - merge_adaptive_n(f0_0, n0_0, f0_1, n0_1, iter_next(f0_1, n0_1), r, buffer); - merge_adaptive_n(f1_0, n1_0, f1_1, n1_1, iter_next(f1_1, n1_1), r, buffer); - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(f0, e1), ""); - } - - template - SEQ_ALWAYS_INLINE Iter insertion_sort_n(Iter begin, unsigned count, Cmp l) noexcept(NothrowSortIter::value) - { - // Standard in-place insertion sort working on bidirectional iterators, - // but using a number of values to sort instead of an end iterator. - - using T = typename std::iterator_traits::value_type; - if SEQ_UNLIKELY (count < 2) - return count == 0 ? begin : iter_next(begin); - - auto cur = begin; - auto prev = cur++; - for (; count > 1; --count) { - if (l(*cur, *prev)) { - auto sift = cur; - T tmp = std::move(*sift); - do { - *sift = std::move(*prev); - --sift; - } while (sift != begin && l(tmp, *(--prev))); - *sift = std::move(tmp); - } - prev = cur++; - } - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(begin, cur, l), ""); - return cur; - } - - template - static void reverse_sort(Iter begin, Iter end, Cmp l) noexcept(NothrowSortIter::value) - { - // Inplace reverse range. - // The range must be sorted in descending order. - // Stable version of std::reverse. - - if (begin == end) - return; - Iter start = begin; - Iter prev = begin++; - while (begin != end) { - // Loop through non equal values - while (l(*begin, *prev)) { - prev = begin++; - if SEQ_UNLIKELY (begin == end) - goto reverse_full; - } - // Find full equal range and reverse it - Iter start_equal = prev++; - ++begin; - while (begin != end && !l(*begin, *prev)) - prev = begin++; - std::reverse((start_equal), (begin)); - } - - reverse_full: - // Reverse the full sequence. - // Equal ranges will get back their natural orders. - std::reverse((start), (end)); - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(start, end, [l](const auto& a, const auto& b) { return l(a, b); }), ""); - } - - template - static void ping_pong_merge_4(Iter it0, Iter it1, Iter it2, Iter it3, Iter it4, Cmp c, Storage tmp) noexcept(NothrowSortIter::value) - { - // Ping pong merge 4 sorted ranges using provided buffer. - - if ((size_t)(it4 - it0) <= tmp.size) { - const bool s0 = !c(*it1, *iter_prev(it1)); - const bool s1 = !c(*it2, *iter_prev(it2)); - const bool s2 = !c(*it3, *iter_prev(it3)); - if (s0 && s1 && s2) - return; - - decltype(tmp.first) middle, end; - if (!s0) - middle = merge_move_bidirectional((it0), (it1), (it1), (it2), tmp.first, c); - else { - auto dst = std::move((it0), (it1), tmp.first); - middle = std::move((it1), (it2), dst); - } - if (!s2) - end = merge_move_bidirectional((it2), (it3), (it3), (it4), middle, c); - else { - auto dst = std::move((it2), (it3), middle); - end = std::move((it3), (it4), dst); - } - if (c(*middle, *iter_prev(middle))) - merge_move_bidirectional(tmp.first, (middle), (middle), (end), it0, c, &it4); - else { - auto dst = std::move((tmp.first), (middle), it0); - std::move((middle), (end), dst); - } - } - else { - merge_adaptive_n(it0, it1 - it0, it1, it2 - it1, it2, c, tmp); - merge_adaptive_n(it2, it3 - it2, it3, it4 - it3, it4, c, tmp); - merge_adaptive_n(it0, it2 - it0, it2, it4 - it2, it4, c, tmp); - } - } - template - static void ping_pong_merge_3(Iter it0, Iter it1, Iter it2, Iter it3, Cmp c, Storage tmp) noexcept(NothrowSortIter::value) - { - // Ping pong merge 3 sorted ranges using provided buffer. - - if ((size_t)(it2 - it0) <= tmp.size) { - const bool s0 = !c(*it1, *iter_prev(it1)); - const bool s1 = !c(*it2, *iter_prev(it2)); - if (s0 && s1) - return; - - auto middle = tmp.first; - if (!s0) - middle = merge_move_bidirectional((it0), (it1), (it1), (it2), tmp.first, c); - else { - auto dst = std::move((it0), (it1), tmp.first); - middle = std::move((it1), (it2), dst); - } - if (c(*it2, *iter_prev(middle))) - merge_forward(std::make_move_iterator(tmp.first), std::make_move_iterator(middle), std::make_move_iterator(it2), std::make_move_iterator(it3), it0, c); - else - std::move((tmp.first), (middle), it0); - } - else { - merge_adaptive_n(it0, it1 - it0, it1, it2 - it1, it2, c, tmp); - merge_adaptive_n(it0, it2 - it0, it2, it3 - it2, it3, c, tmp); - } - } - - template::pointer> - static void merge_sorted_runs_with_buffer(Iter* iters, size_t start, size_t last, Cmp cmp, Buffer buf) noexcept(NothrowSortIter::value) - { - // Inplace merge already sorted ranges represented by an array of iterators. - // Supports bidirectional iterators. - // Internally calls merge_adaptive_n with provided buffer. - - auto size = last - start; - - if (size > 4) { - auto quarter = size / 4; - auto half = quarter * 2; - auto quarter3 = quarter * 3; - // merge each 4 ranges - merge_sorted_runs_with_buffer(iters, start, start + quarter, cmp, buf); - merge_sorted_runs_with_buffer(iters, start + quarter, start + half, cmp, buf); - merge_sorted_runs_with_buffer(iters, start + half, start + quarter3, cmp, buf); - merge_sorted_runs_with_buffer(iters, start + quarter3, last, cmp, buf); - return ping_pong_merge_4(iters[start], iters[start + quarter], iters[start + half], iters[start + quarter3], iters[last], cmp, buf); - } - - switch (size) { - case 4: - ping_pong_merge_4(iters[start], iters[start + 1], iters[start + 2], iters[start + 3], iters[start + 4], cmp, buf); - break; - case 3: - ping_pong_merge_3(iters[start], iters[start + 1], iters[start + 2], iters[start + 3], cmp, buf); - break; - case 2: - merge_adaptive_n(iters[start], iters[start + 1] - iters[start], iters[start + 1], iters[last] - iters[start + 1], iters[last], cmp, buf); - break; - default: - break; - } - } - - template - SEQ_ALWAYS_INLINE void swap_branchless(T&& a, T&& b, bool b_is_less) noexcept - { - // Swap elements based on is_less. - // Uses branchless swap for relocatable types. - - if constexpr (is_relocatable::value) { - auto tmp = as_bits(a); - as_bits(a) = b_is_less ? as_bits(b) : tmp; - as_bits(b) = b_is_less ? tmp : as_bits(b); - } - else { - using std::swap; - if (b_is_less) - swap(a, b); - } - } - -#define CHECK_2(l, r) swap_branchless(a##l, a##r, cmp(a##r, a##l)) -#define CHECK_2_NO_OVERLAPP(a, b, c, d) \ - CHECK_2(a, b); \ - CHECK_2(c, d) -#define CHECK_4_NO_OVERLAPP(a, b, c, d, e, f, g, h) \ - CHECK_2_NO_OVERLAPP(a, b, c, d); \ - CHECK_2_NO_OVERLAPP(e, f, g, h) - - template - SEQ_ALWAYS_INLINE void network_sort_8(T&& a0, T&& a1, T&& a2, T&& a3, T&& a4, T&& a5, T&& a6, T&& a7, Cmp cmp) - { - CHECK_4_NO_OVERLAPP(0, 1, 2, 3, 4, 5, 6, 7); - CHECK_4_NO_OVERLAPP(0, 2, 1, 3, 4, 6, 5, 7); - CHECK_4_NO_OVERLAPP(0, 4, 1, 5, 2, 6, 3, 7); - CHECK_2_NO_OVERLAPP(2, 4, 3, 5); - CHECK_2_NO_OVERLAPP(1, 4, 3, 6); - CHECK_2_NO_OVERLAPP(1, 2, 3, 4); - CHECK_2(5, 6); - } - -#undef CHECK_2 -#undef CHECK_2_NO_OVERLAPP -#undef CHECK_4_NO_OVERLAPP - - template - Iter atom_sort_8(Iter vals, unsigned count, Cmp cmp) noexcept(NothrowSortIter::value) - { - // Sort up to 8 values - if constexpr (is_random_access::value) { - if (N == 8 || count == 8) { - // Sort 8 values using a sorting netork - network_sort_8(vals[0], vals[1], vals[2], vals[3], vals[4], vals[5], vals[6], vals[7], cmp); - return vals + 8; - } - } - - return insertion_sort_n(vals, N == Unspecified ? count : N, cmp); - } - - template - Out atom_sort_64(Iter& first, Out out, Cmp c) noexcept(NothrowSortIter::value) - { - // Sort 64 values to output - - auto it0 = atom_sort_8<8>(first, 8, c); - auto it1 = atom_sort_8<8>(it0, 8, c); - auto it2 = atom_sort_8<8>(it1, 8, c); - auto it3 = atom_sort_8<8>(it2, 8, c); - auto it4 = atom_sort_8<8>(it3, 8, c); - auto it5 = atom_sort_8<8>(it4, 8, c); - auto it6 = atom_sort_8<8>(it5, 8, c); - auto it7 = atom_sort_8<8>(it6, 8, c); - - auto o0 = merge_move_bidirectional<8>(first, it0, it0, it1, out, c); - auto o1 = merge_move_bidirectional<8>(it1, it2, it2, it3, o0, c); - auto o2 = merge_move_bidirectional<8>(it3, it4, it4, it5, o1, c); - auto o3 = merge_move_bidirectional<8>(it5, it6, it6, it7, o2, c); - auto d0 = merge_move_bidirectional<16>(out, o0, o0, o1, first, c); - auto d1 = merge_move_bidirectional<16>(o1, o2, o2, o3, d0, c); - auto r = merge_move_bidirectional<32>(first, d0, d0, d1, out, c); - first = d1; - return r; - } - - template - Iter sort_128(Iter vals, unsigned count, Cmp c, Buffer buf) noexcept(NothrowSortIter::value) - { - // Sort inplace up to 128 values using provided buffer - - if (count == 128 && buf.size >= 128) { - auto src = (vals); - auto it0 = atom_sort_64((src), (buf.first), c); - auto it1 = atom_sort_64((src), (it0), c); - return merge_move_bidirectional<64>((buf.first), (it0), (it0), (it1), vals, c); - } - else { - // Buffer too small or less than 128 elements - Iter iters[17] = { vals }; - unsigned cnt = 1; - unsigned rem = count; - Iter src = vals; - while (rem) { - auto p = atom_sort_8(src, std::min(rem, 8u), c); - rem -= (unsigned)(p - src); - iters[cnt++] = src = p; - } - merge_sorted_runs_with_buffer(iters, 0, cnt - 1, c, buf); - return src; - } - } - - template - static std::pair try_wave_sort(T begin, size_t size, size_t min_dist, Cmp c, Buffer buf) noexcept(NothrowSortIter::value) - { - // Attempt to sort the range [begin,end). - // Find consecutive sorted runs (ascending or descending), up to IterCount-1. - // Stop when reaching end iterator or runs limit. - // Supports bidirectional iterators. - - if SEQ_UNLIKELY (size == 0) - return { begin, 0 }; - - T start = begin; - T prev = begin; - T iters[IterCount] = { begin++ }; - bool ascending[IterCount]; - unsigned cnt = 1; - size_t dist = 1; - - if (size == 1) - return { begin, 1 }; - - ascending[0] = !c(*begin, *prev); - prev = begin; - ++begin; - ++dist; - - for (; dist != size; ++begin, ++prev, ++dist) { - // Find consecutive ascending or descending runs - const bool as = ascending[cnt - 1]; - if (as) { - for (; dist != size && !c(*begin, *prev); ++dist) - prev = begin++; - } - else { - for (; dist != size && !c(*prev, *begin); ++dist) - prev = begin++; - } - - // Stop before adding the start of a new sorted range, - // or the last range will have a size of 1... - if (cnt == (IterCount - 1)) - break; - - ascending[cnt] = !as; - iters[cnt++] = begin; - if (cnt > 1 && begin == iter_next(iters[cnt - 2])) { - ascending[cnt - 2] = !as; - --cnt; - } - if SEQ_UNLIKELY (dist == size) - break; - } - - if (dist < min_dist) - // We were not able to sort up to min_dist elements, - // returns the start iterator to notify this failure. - return { start, 0 }; - - // add last iterator - if (iters[cnt - 1u] != begin) - iters[cnt++] = begin; - - // reverse descending ranges - for (size_t i = 0; i < cnt - 1; ++i) { - if (!ascending[i]) - // stable reverse - reverse_sort(iters[i], iters[i + 1], c); - } - - // inplace merge runs - merge_sorted_runs_with_buffer(iters, 0u, cnt - 1u, c, buf); - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(start, begin, c), ""); - return { begin, dist }; - } - - template - static void generic_merge_sort_internal(Iter begin, Iter end, size_t size, Cmp l, Fn sort_sub_range, Buff buf, size_t min_size = 0) noexcept(NothrowSortIter::value) - { - // Generic merge sort that uses a custom sort function for small chunks. - // Supports bidirectional iterators. - - size_t remaining = size; - Iter start = begin; - Iter last_start = begin; - size_t new_min_size = std::numeric_limits::max(); - - do { - size_t cnt = 1; - size_t cum_dist = 0; - Iter iters[MaxIters] = { begin }; - - do { - // Sort any number of elements (up to remaining). - // sort_sub_range must return a std::pair. - std::pair r; - if (min_size == 0) - // First pass: sort input chunk - r = sort_sub_range(begin, remaining, l); - else { - // Next passes : identify sorted range - if (remaining >= min_size) { - // We need to identify the sorted range with is_sorted_until() - - // Increment it by min_size -1, starting from begin or from end - auto it = begin; - - if constexpr (!is_random_access::value) { - size_t d = (size_t)iter_distance(begin, end); - if (min_size > d / 2) - it = iter_prev(end, d - min_size + 1); - } - if (it == begin) - it = iter_next(begin, min_size - 1); - - auto p = std::is_sorted_until((it), (end), l); - r = { p, min_size - 1 + (size_t)iter_distance(it, p) }; - } - else { - // No need to call is_sorted_until(), we know - // the sorted range goes to the end. - r = { end, remaining }; - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(begin, r.first, l), ""); - } - } - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(begin, r.first, l), ""); - - // Store new start position - // and update remaining elements - - iters[cnt++] = begin = r.first; - remaining -= r.second; - cum_dist += r.second; - - } while (remaining != 0 && cnt < MaxIters); - - if (remaining == 0 && cnt > 1 && iters[cnt - 1] == iters[cnt - 2]) { - // Special case when remaining is 0: the last iterator might be equal to the previous one. - // We must remove it to avoid range of size 0 - --cnt; - } - - // Merge all sorted runs - merge_sorted_runs_with_buffer(iters, 0u, cnt - 1u, l, buf); - - // Update new min size if this is NOT the last chunk - if (remaining || new_min_size == std::numeric_limits::max()) { - new_min_size = std::min(new_min_size, cum_dist); - } - - last_start = iters[0]; - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(iters[0], iters[cnt - 1u], l), ""); - - } while (remaining != 0); - - // Compute end iterator during the first pass - if (min_size == 0) - end = begin; - - if (last_start != start) { - // We need a next pass if the number of iterators was not enough - return generic_merge_sort_internal(start, end, size, l, sort_sub_range, buf, new_min_size); - } - } - - template - static void merge_sort_internal(Iter begin, size_t size, Cmp l, Buffer buf) noexcept(NothrowSortIter::value) - { - // Bottom-up merge sort. - // Small chunks of up to 128 elements are sorted using insertion sort or sorting netwrok and ping-pong merge. - // If possible, use wave sort on longer runs. - // Supports bidirectional iterators. - - if (size < 128) { - auto r = try_wave_sort<5>(begin, size, size, l, buf); - if (r.first == begin) - // Failed - sort_128(begin, (unsigned)size, l, buf); - return; - } - - generic_merge_sort_internal<65>( - begin, - begin, - size, - l, - [&](Iter b, size_t remaining, auto l) { - // Try wave sort first, as it might consume a lot more - // than the default 128 elements (possibly the whole sequence) - auto r = try_wave_sort<5>(b, remaining, std::min(remaining, (size_t)128u), l, buf); - if (r.first != b) { - // Success, retrieve new start position and number of sorted elements - return r; - } - - // Failure, use sort_128() - unsigned cnt = (unsigned)std::min(remaining, (size_t)128u); - auto it = sort_128(b, cnt, l, buf); - return std::make_pair(it, (size_t)cnt); - }, - buf); - - SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(begin, iter_next(begin, size), l), ""); - } - - // Create the internal buffer used by net_sort() - template - static size_t sort_buffer_size(const Buffer& buf, size_t count) noexcept - { - if (buf.size == 0) - return (count); - if (buf.size == std::numeric_limits::max() - 1) - return (count / 8u); - if (buf.size == std::numeric_limits::max() - 2) - return (count / 32u); - return (count / 128u); - } - - } // end algo_detail - - /// @brief Buffer object used by net_sort(), net_sort_size() and inplace_merge() - template - struct buffer - { - Iter first; - size_t size = 0; - }; - - /// @brief Default sort buffer size, uses input size/2 bytes - static constexpr buffer default_buffer{ nullptr, 0 }; - - /// @brief Medium sort buffer size, uses input size/16 bytes - static constexpr buffer medium_buffer{ nullptr, std::numeric_limits::max() - 1u }; - - /// @brief Small sort buffer size, uses input size/64 bytes - static constexpr buffer small_buffer{ nullptr, std::numeric_limits::max() - 2u }; - - /// @brief Tiny sort buffer size, uses input size/256 bytes - static constexpr buffer tiny_buffer{ nullptr, std::numeric_limits::max() - 3u }; - - /// @brief Null buffer, uses (slow) bufferless merge sort - static constexpr buffer null_buffer{ nullptr, 0 }; - - /// @brief Stable merge algorithm similar to std::merge. - /// - /// This algorithm is usually more efficient than regular - /// std::merge, at least on msvc. It provides a better - /// handling of consecutive ordered values, and has a - /// special case for unbalanced merging (one range is - /// way smaller than the other). - /// - template> - Out merge(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Out out, Cmp c = Cmp()) - { - return algo_detail::merge_forward(first1, last1, first2, last2, out, c); - } - - /// @brief Inplace stable merge algorithm similar to std::inplace_merge. - /// - /// The main difference with std::inplace_merge is that this version - /// uses a user provided buffer for merging. This function only allocate - /// memory if provided buffer is one of 'default_buffer', 'medium_buffer', - /// 'small_buffer' or 'tiny_buffer'. 0 sized buffer are supported. - /// - template, class Buffer = buffer> - void inplace_merge(Iter first, Iter middle, Iter last, Cmp c = Cmp(), Buffer buf = Buffer()) - { - using namespace algo_detail; - using Key = typename std::iterator_traits::value_type; - - if (middle == first || middle == last) - return; - - // Perform the firsts, easy checks - if (!c(*middle, *std::prev(middle))) - // One of the 2 ranges is empty, or already sorted - return; - - if (c(*std::prev(last), *first)) { - // Simple rotation needed - std::rotate(first, middle, last); - return; - } - - // The following checks come from msvc STL implementation - // and help a LOT in some situations - - // Increment first as long as it is smaller than first value of second range - for (;;) { - if (first == middle) - // We reach the end of first range: already in order - return; - if (c(*middle, *first)) - break; - ++first; - } - - auto highest = iter_prev(middle); - do { - --last; - if (middle == last) { // rotate only element remaining in right partition to the beginning, without allocating - rotate_one_right(first, middle, ++last); - return; - } - } while (!c(*last, *highest)); // found that *highest goes in *last's position - - ++last; - - // Now go through merge_adaptive_n - - auto s1 = (size_t)std::distance(first, middle); - auto s2 = (size_t)std::distance(middle, last); - - if constexpr (std::is_same_v, Buffer>) { - // Compute buffer size - size_t min_size = std::min(s1, s2); - size_t buf_size = sort_buffer_size(buf, min_size); - std::vector buf_(buf_size); - return merge_adaptive_n(wrap_iter(first), s1, wrap_iter(middle, s1), s2, wrap_iter(last, s1 + s2), c, buffer{ buf_.data(), buf_.size() }); - } - else - // Use provided buffer - return merge_adaptive_n(wrap_iter(first), s1, wrap_iter(middle, s1), s2, wrap_iter(last, s1 + s2), c, buf); - } - - template, class Buffer = buffer> - void inplace_merge(Iter* iters, size_t count, Cmp c = Cmp(), Buffer buf = Buffer()) - { - using namespace algo_detail; - using Key = typename std::iterator_traits::value_type; - - if (count <= 2) - // Nothing to do - return; - - if constexpr (std::is_same_v, Buffer>) { - // Compute buffer size - size_t buf_size = sort_buffer_size(buf, std::distance(iters[0], iters[count - 1]) / 2); - std::vector buf_(buf_size); - merge_sorted_runs_with_buffer(iters, 0, count - 1, c, buffer{ buf_.data(), buf_.size() }); - } - else - // Use provided buffer - merge_sorted_runs_with_buffer(iters, 0, count - 1, c, buf); - } - - /// @brief Reverse a range already sorted in descending order while preserving stability. - /// - /// The full range is reversed except equal values - /// of which order is preserved. - /// - template> - void reverse_descending(Iter first, Iter last, Cmp c = Cmp()) - { - algo_detail::reverse_sort(first, last, c); - } - - /// @brief Stable merge sort algorithm using an external buffer. - /// - /// net sort is a merge sort algorithm with the following specificities: - /// - /// - Bottom-up merging instead of the more traditional top-down approach, - /// - Small blocks of 8 elements are sorted using a sorting network, - /// - Bidirectional merging is used for relocatable types, - /// - Ping-pong merge is used to merge 4 sorted ranges, - /// - Can work without allocating memory through a (potentially null) user provided buffer, - /// - Works on bidirectional iterators. - /// - /// If provided buffer is one of 'seq::default_buffer', 'seq::medium_buffer', 'seq::small_buffer' - /// or 'seq::tiny_buffer', this function will try to allocate memory. - /// - /// From my tests on multiple input types, net_sort() is always faster than std::stable_sort(). - /// - /// net_sort_size() and net_sort() work on bidirectional iterators. - /// Using net_sort_size() instead of net_sort() is faster when the range size is already known. - /// - /// All credits to scandum (https://github.com/scandum) for its quadsort algorithm from which - /// I took several ideas (bidirectional merge and ping-pong merge). - /// - template, class Buffer = buffer> - void net_sort_size(Iter begin, size_t size, Cmp cmp = Cmp(), Buffer buf = Buffer()) - { - using namespace algo_detail; - using Key = typename std::iterator_traits::value_type; - - if (size < 16) { - insertion_sort_n(begin, (unsigned)size, cmp); - return; - } - - if constexpr (std::is_same_v, Buffer>) { - // Compute buffer size - size_t buf_size = sort_buffer_size(buf, size / 2); - if (buf_size < 16) - buf_size = 16; - std::vector buf_(buf_size); - return merge_sort_internal(wrap_iter(begin, 0), size, cmp, seq::buffer{ buf_.data(), buf_.size() }); - } - else { - // Use provided buffer - return merge_sort_internal(wrap_iter(begin, 0), size, cmp, buf); - } - } - - /// @brief Stable merge sort algorithm using an external buffer. - /// - /// net sort is a merge sort algorithm with the following specificities: - /// - /// - Bottom-up merging instead of the more traditional top-down approach, - /// - Small blocks of 8 elements are sorted using a sorting network, - /// - Bidirectional merging is used for relocatable types, - /// - Ping-pong merge is used to merge 4 sorted ranges, - /// - Can work without allocating memory through a (potentially null) user provided buffer, - /// - Works on bidirectional iterators. - /// - /// If provided buffer is one of 'seq::default_buffer', 'seq::medium_buffer', 'seq::small_buffer' - /// or 'seq::tiny_buffer', this function will try to allocate memory. - /// - /// From my tests on multiple input types, net_sort() is always faster than std::stable_sort(). - /// - /// net_sort_size() and net_sort() work on bidirectional iterators. - /// Using net_sort_size() instead of net_sort() is faster when the range size is already known. - /// - /// Full credits to scandum (https://github.com/scandum) for its quadsort algorithm from which - /// I took several ideas (bidirectional merge and ping-pong merge). - /// - template, class Buffer = buffer> - void net_sort(Iter begin, Iter end, Cmp cmp = Cmp(), Buffer buffer = Buffer()) - { - net_sort_size(begin, std::distance(begin, end), cmp, buffer); - } namespace algo_detail { @@ -1391,7 +73,7 @@ namespace seq /// function for non trival element type. /// template::value_type>, class Equal = std::equal_to<>> - Iter unique(Iter first, Iter last, const Hash& h = Hash(), const Equal& eq = Equal()) + Iter unique(Iter first, Iter last, const Hash& h = {}, const Equal& eq = {}) { using namespace algo_detail; using Key = typename std::iterator_traits::value_type; diff --git a/seq/any.hpp b/seq/any.hpp index ed1d39cb..5b31609c 100644 --- a/seq/any.hpp +++ b/seq/any.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,12 +32,14 @@ The any module provides the seq::hold_any class and related functions to provide a type-erasing polymorphic object wrapper. */ -#include "format.hpp" -#include "tagged_pointer.hpp" +#include "internal/tagged_pointer.hpp" #include "tiny_string.hpp" #include "hash.hpp" #include +#include +#include +#include #ifndef SEQ_ANY_MALLOC #define SEQ_ANY_MALLOC operator new // allocation function used within seq::hold_any @@ -54,6 +56,32 @@ The any module provides the seq::hold_any class and related functions to namespace seq { + /// @brief Enumeration of supported features by a hold_any object + enum supported_any_operation + { + EqualComparable = 1, + LessComparable = 2, + Ostreamable = 4, + Istreamable = 8 + }; + + /// @brief Returns supported operations for given type. + /// The result is a combination of supported_any_operation eunm. + template + constexpr int supported_type_operations() + { + int ret = 0; + if constexpr (is_equal_comparable::value) + ret |= EqualComparable; + if constexpr (is_less_comparable::value) + ret |= LessComparable; + if constexpr (is_ostreamable::value) + ret |= Ostreamable; + if constexpr (is_istreamable::value) + ret |= Istreamable; + return ret; + } + /// @brief Exception class similar to std::bad_function_call but storing a custom message class bad_any_function_call : public std::bad_function_call { @@ -72,6 +100,21 @@ namespace seq } }; + // forward declaration + template + class hold_any; + + + template + struct is_hold_any : std::false_type + { + }; + + template + struct is_hold_any> : std::true_type + { + }; + namespace detail { template @@ -79,7 +122,7 @@ namespace seq { // Generate a unique id starting from Start static std::atomic cnt = { Start }; // start index for custom types - return cnt++; + return cnt.fetch_add(1); } } // namespace detail @@ -98,38 +141,40 @@ namespace seq return 5; else if constexpr (std::is_same_v) return 6; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 7; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 8; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 9; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 10; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 11; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 12; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 13; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 14; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 15; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 16; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 17; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 18; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 19; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 20; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 21; - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) return 22; + else if constexpr (std::is_same_v) + return 23; else { return 0; } @@ -145,41 +190,41 @@ namespace seq if constexpr (v != 0) return v; else { - static constexpr int last_id = 22; + static constexpr int last_id = 23; static int id = detail::build_type_id(); return id; } } /// @brief Returns true if given type id corresponds to a signed integral type - SEQ_ALWAYS_INLINE auto is_signed_integral_type(int id) noexcept -> bool + SEQ_ALWAYS_INLINE constexpr bool is_signed_integral_type(int id) noexcept { - return id != 0 && id <= get_type_id(); + return id != 0 && id <= get_base_type_id(); } /// @brief Returns true if given type id corresponds to an unsigned integral type - SEQ_ALWAYS_INLINE auto is_unsigned_integral_type(int id) noexcept -> bool + SEQ_ALWAYS_INLINE constexpr bool is_unsigned_integral_type(int id) noexcept { - return id >= get_type_id() && id <= get_type_id(); + return id >= get_base_type_id() && id <= get_base_type_id(); } /// @brief Returns true if given type id corresponds to an integral type - SEQ_ALWAYS_INLINE auto is_integral_type(int id) noexcept -> bool + SEQ_ALWAYS_INLINE constexpr bool is_integral_type(int id) noexcept { - return id != 0 && id <= get_type_id(); + return id != 0 && id <= get_base_type_id(); } /// @brief Returns true if given type id corresponds to a floating point type - SEQ_ALWAYS_INLINE auto is_floating_point_type(int id) noexcept -> bool + SEQ_ALWAYS_INLINE constexpr bool is_floating_point_type(int id) noexcept { - return id >= get_type_id() && id <= get_type_id(); + return id >= get_base_type_id() && id <= get_base_type_id(); } /// @brief Returns true if given type id corresponds to an arithmetic type (floating point or integral) - SEQ_ALWAYS_INLINE auto is_arithmetic_type(int id) noexcept -> bool + SEQ_ALWAYS_INLINE constexpr bool is_arithmetic_type(int id) noexcept { - return id != 0 && id <= get_type_id(); + return id != 0 && id <= get_base_type_id(); } /// @brief Returns true if given type id corresponds to a string type (std::string, tstring, tstring_view or const char*) - SEQ_ALWAYS_INLINE auto is_string_type(int id) noexcept -> bool + SEQ_ALWAYS_INLINE constexpr bool is_string_type(int id) noexcept { - return id >= get_type_id() && id <= get_type_id(); + return id >= get_base_type_id() && id <= get_base_type_id(); } namespace detail @@ -202,16 +247,10 @@ namespace seq pointer = 16 // pointer type }; - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - // - // LOTS of partial specialization because C++11 does not support 'if constexpr ()' ... - // - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - /// @brief Hash value only if is_hashable::value is true, /// otherwise throw bad_any_function_call template - auto hash_for_val(const void* in) -> size_t + static SEQ_ALWAYS_INLINE auto hash_for_val(const void* in) -> size_t { if constexpr (is_hashable::value) { if constexpr (is_character_pointer::value) { @@ -233,30 +272,9 @@ namespace seq } } - /// @brief Format to string - template - void format_val(std::string& out, const void* in, const width_format& wfmt, const numeric_format& nfmt) - { - if constexpr (!is_formattable::value) - throw seq::bad_any_function_call("data type is not formattable"); - else if constexpr (std::is_pointer_v) { - const void * ptr = reinterpret_cast(*static_cast(in)); - auto f = fmt(ptr); - f.set_width_format(wfmt); - f.set_numeric_format(nfmt); - f.append(out); - } - else { - auto f = fmt(const_cast(*static_cast(in))); - f.set_width_format(wfmt); - f.set_numeric_format(nfmt); - f.append(out); - } - } - /// @brief Cast T to an arithmetic type template - T cast_arithmetic(const void* in, const TypeInfo* in_p) + static T cast_arithmetic(const void* in, const TypeInfo* in_p) { if constexpr (!std::is_arithmetic_v) { throw std::bad_cast(); @@ -264,6 +282,8 @@ namespace seq } else { switch (in_p->type_id()) { + case get_base_type_id(): + return static_cast(*static_cast(in)); case get_base_type_id(): return static_cast(*static_cast(in)); case get_base_type_id(): @@ -307,10 +327,36 @@ namespace seq template T arithmetic_from_string(const String& in) { - if constexpr (std::is_arithmetic_v) { + if constexpr (std::is_same_v) + return (bool)arithmetic_from_string(in); + else if constexpr (std::is_same_v) + return (char16_t)arithmetic_from_string(in); + else if constexpr (std::is_same_v) + return (char32_t)arithmetic_from_string(in); + else if constexpr (std::is_same_v) + return (long double)arithmetic_from_string(in); + else if constexpr (std::is_arithmetic_v) { T res; - if (from_chars(string_data(in), string_data(in) + string_size(in), res).ec != std::errc()) +#if __cpp_lib_to_chars + if constexpr(std::is_floating_point_v){ + if (std::from_chars(string_data(in), string_data(in) + string_size(in), res).ec != std::errc()) + throw std::bad_cast(); + } + else{ + // Add the 'base' argument, mandatory with clang on macosx(?) + if (std::from_chars(string_data(in), string_data(in) + string_size(in), res, 10).ec != std::errc()) + throw std::bad_cast(); + } +#else + std::istringstream iss{std::string(in)}; + if constexpr (is_istreamable::value){ + if(!(iss >> res)) + throw std::bad_cast(); + } + else throw std::bad_cast(); +#endif + return res; } else { @@ -319,50 +365,87 @@ namespace seq } } - /// @brief Convert arithmetic object to allcoated string object + template + String arithmetic_to_string(const T& in) + { + if constexpr (std::is_same_v) + return arithmetic_to_string((uint8_t)in); + else if constexpr (std::is_integral_v) { + String tmp(19, (char)0); + auto res = std::to_chars(tmp.data(), tmp.data() + tmp.size(), in); + if (res.ec != std::errc()) + throw std::bad_cast(); + tmp.resize(res.ptr - tmp.data()); + return tmp; + } + else if constexpr (std::is_same_v) + return arithmetic_to_string((uint16_t)in); + else if constexpr (std::is_same_v) + return arithmetic_to_string((uint32_t)in); + else if constexpr (std::is_same_v) + return arithmetic_to_string((double)in); + else if constexpr (std::is_floating_point_v) { +#if __cpp_lib_to_chars + String tmp(24, (char)0); + auto res = std::to_chars(tmp.data(), tmp.data() + tmp.size(), in); + if (res.ec != std::errc()) + throw std::bad_cast(); + tmp.resize(res.ptr - tmp.data()); + return tmp; +#else + std::ostringstream oss; + oss << in; + return oss.str(); +#endif + } + else { + throw std::bad_cast(); + return {}; + } + } + + /// @brief Convert arithmetic object to allocated string object template String arithmetic_to_string(const void* in, const TypeInfo* in_p) { if constexpr (is_allocated_string::value) { - String res; switch (in_p->type_id()) { case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); case get_base_type_id(): - return fmt(*static_cast(in)).append(res); + return arithmetic_to_string(*static_cast(in)); default: throw std::bad_cast(); break; } - return res; } else { throw std::bad_cast(); @@ -439,33 +522,37 @@ namespace seq /// @brief Stream object to std::ostream or throw std::bad_function_call template - typename std::enable_if::value, void>::type ostream_any(std::ostream& oss, const void* in) + void ostream_any(std::ostream& oss, const void* in) { + if constexpr (is_ostreamable::value) { + #if defined(__clang__) && defined(_MSC_VER) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wmicrosoft-cast" #endif - oss << *static_cast(in); + oss << *static_cast(in); #if defined(__clang__) && defined(_MSC_VER) #pragma clang diagnostic pop #endif - } - template - typename std::enable_if::value, void>::type ostream_any(std::ostream&, const void*) - { - throw seq::bad_any_function_call("data type does not streamable to std::ostream"); + } + else { + (void)oss; + (void)in; + throw seq::bad_any_function_call("data type does not streamable to std::ostream"); + } } /// @brief Read object to std::istream or throw std::bad_function_call template - typename std::enable_if::value, void>::type istream_any(std::istream& iss, void* in) + void istream_any(std::istream& iss, void* in) { - iss >> *static_cast(in); - } - template - typename std::enable_if::value, void>::type istream_any(std::istream&, void*) - { - throw seq::bad_any_function_call("data type does not streamable from std::istream"); + if constexpr (is_istreamable::value) + iss >> *static_cast(in); + else { + (void)iss; + (void)in; + throw seq::bad_any_function_call("data type does not streamable from std::istream"); + } } /// @brief Hash object using hasher or throw std::bad_function_call @@ -479,82 +566,71 @@ namespace seq } /// @brief Compare equal 2 objects of same type template - SEQ_ALWAYS_INLINE typename std::enable_if::value, bool>::type compare_equal_any(const void* a, const void* b) - { - SEQ_COMPARE_FLOAT(return *static_cast(a) == *static_cast(b);) - } - template - SEQ_ALWAYS_INLINE typename std::enable_if::value, bool>::type compare_equal_any(const void*, const void*) + SEQ_ALWAYS_INLINE bool compare_equal_any(const void* a, const void* b) { - return false; + if constexpr (is_equal_comparable::value) { + SEQ_COMPARE_FLOAT(return *static_cast(a) == *static_cast(b);) + } + else { + (void)a; + (void)b; + return false; + } } /// @brief Compare 2 objects of same type for less than, or throw std::bad_function_call template - SEQ_ALWAYS_INLINE typename std::enable_if::value, bool>::type compare_less_any(const void* a, const void* b) + SEQ_ALWAYS_INLINE bool compare_less_any(const void* a, const void* b) { + if constexpr (is_less_comparable::value) { + #ifdef __clang__ - _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wordered-compare-function-pointers\"") + _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wordered-compare-function-pointers\"") #endif - if constexpr (is_function_pointer::value) return reinterpret_cast(*static_cast(a)) < reinterpret_cast(*static_cast(b)); - else return *static_cast(a) < *static_cast(b); + if constexpr (is_function_pointer::value) return reinterpret_cast(*static_cast(a)) < + reinterpret_cast(*static_cast(b)); + else return *static_cast(a) < *static_cast(b); #ifdef __clang__ - _Pragma("clang diagnostic pop") + _Pragma("clang diagnostic pop") #endif - } - template - SEQ_ALWAYS_INLINE typename std::enable_if::value, bool>::type compare_less_any(const void*, const void*) - { - throw seq::bad_any_function_call("data type does not provide a less operator"); - return false; - } - - /// @brief Format object or throw std::bad_function_call - template - SEQ_ALWAYS_INLINE void format_any(std::string& out, const void* in, const width_format& wfmt, const numeric_format& nfmt) - { - format_val(out, in, wfmt, nfmt); + } + else { + (void)a; + (void)b; + throw seq::bad_any_function_call("data type does not provide a less operator"); + return false; + } } template - SEQ_ALWAYS_INLINE typename std::enable_if, void>::type assign_value(T& dst, const T& src) - { - dst = src; - } - template - SEQ_ALWAYS_INLINE typename std::enable_if, void>::type assign_value(T&, const T&) + SEQ_ALWAYS_INLINE void assign_value(T& dst, const T& src) { + if constexpr (std::is_copy_assignable_v) + dst = src; } template - SEQ_ALWAYS_INLINE typename std::enable_if, void>::type move_assign_value(T& dst, T& src) - { - dst = std::move(src); - } - template - SEQ_ALWAYS_INLINE typename std::enable_if, void>::type move_assign_value(T&, T&) + SEQ_ALWAYS_INLINE void move_assign_value(T& dst, T& src) { + if constexpr (std::is_move_assignable_v) + dst = std::move(src); } template - SEQ_ALWAYS_INLINE typename std::enable_if, void>::type copy_construct_value(void* dst, const T& src) - { - new (dst) T(src); - } - template - SEQ_ALWAYS_INLINE typename std::enable_if, void>::type copy_construct_value(void*, const T&) + SEQ_ALWAYS_INLINE void copy_construct_value(void* dst, const T& src) { + if constexpr (std::is_copy_constructible_v) + new (dst) T(src); } template - SEQ_ALWAYS_INLINE typename std::enable_if, bool>::type compare_equal_arithmetic(long double a, const T& b) + SEQ_ALWAYS_INLINE bool compare_equal_arithmetic(long double a, const T& b) { - SEQ_COMPARE_FLOAT(return a == static_cast(b);) - } - template - SEQ_ALWAYS_INLINE typename std::enable_if, bool>::type compare_equal_arithmetic(long double, const T&) - { - return false; + if constexpr (std::is_arithmetic_v) { + SEQ_COMPARE_FLOAT(return a == static_cast(b);) + } + else + return false; } template @@ -562,10 +638,10 @@ namespace seq { // Copy object to destination buffer, allocating buffer if dst size is not big enough - if (!std::is_copy_assignable_v && !std::is_copy_constructible_v) + if constexpr (!std::is_copy_assignable_v && !std::is_copy_constructible_v) throw seq::bad_any_function_call("data type is not copyable"); - if (in_p == out_p && std::is_copy_assignable_v) { + if (std::is_copy_assignable_v && in_p == out_p) { // same type, just copy // get the dst pointer void* dst = sizeof(T) > out_storage_size ? read_void_p(out_storage) : out_storage; @@ -596,7 +672,7 @@ namespace seq { // Move object to destination buffer, allocating buffer if dst size is not big enough - if (in_p == out_p && std::is_move_assignable_v) { + if (std::is_move_assignable_v && in_p == out_p) { // same type, just copy // get the dst pointer void* dst = sizeof(T) > out_storage_size ? read_void_p(out_storage) : out_storage; @@ -641,6 +717,8 @@ namespace seq virtual ~any_type_info() {} /// @brief Returns the size of underlying type virtual auto sizeof_type() const noexcept -> size_t = 0; + /// @brief Returns supported operations by the type + virtual auto supported_operations() const noexcept -> int = 0; /// @brief Destroy object virtual void destroy_any(void* in) const noexcept = 0; /// @brief Compare 2 objects of same type for equality @@ -649,8 +727,6 @@ namespace seq virtual auto less_any(const void* a, const void* o) const -> bool = 0; /// @brief Hash object of underlying type virtual auto hash_any(const void* in) const -> size_t = 0; - /// @brief Format object of underlying type into output string - virtual void format_any(std::string& out, const void* in, const width_format& wfmt, const numeric_format& nfmt) const = 0; /// @brief Stream object of underlying type to a std::ostream object virtual void ostream_any(const void* in, std::ostream& oss) const = 0; /// @brief Read object of underlying type from a std::istream object @@ -664,11 +740,11 @@ namespace seq struct any_typed_type_info : virtual any_type_info { auto sizeof_type() const noexcept -> size_t override { return sizeof(T); } + auto supported_operations() const noexcept -> int override { return supported_type_operations(); } void destroy_any(void* in) const noexcept override { static_cast(in)->~T(); } auto equal_any(const void* a, const void* b) const -> bool override { return detail::compare_equal_any(a, b); } auto less_any(const void* a, const void* b) const -> bool override { return detail::compare_less_any(a, b); } auto hash_any(const void* in) const -> size_t override { return detail::hash_any(in); } - void format_any(std::string& out, const void* in, const width_format& wfmt, const numeric_format& nfmt) const override { detail::format_any(out, in, wfmt, nfmt); } void ostream_any(const void* in, std::ostream& oss) const override { return detail::ostream_any(oss, in); } void istream_any(void* in, std::istream& iss) const override { return detail::istream_any(iss, in); } void copy_any(const any_type_info* in_p, const void* in, const any_type_info* out_p, void* out_storage, unsigned out_storage_size) const override @@ -728,13 +804,13 @@ namespace seq } /// @brief Default less comparison template - SEQ_ALWAYS_INLINE auto default_less_comparison(const void* a, const void* b) -> bool + SEQ_ALWAYS_INLINE bool default_less_comparison(const void* a, const void* b) { return *static_cast(a) < *static_cast(b); } /// @brief Less comparison based on functor template - SEQ_ALWAYS_INLINE auto default_less_comparison_with_functor(const Fun& fun, const void* a, const void* b) -> bool + SEQ_ALWAYS_INLINE bool default_less_comparison_with_functor(const Fun& fun, const void* a, const void* b) { return fun(*static_cast(a), *static_cast(b)); } @@ -755,13 +831,13 @@ namespace seq } /// @brief Default equal comparison template - SEQ_ALWAYS_INLINE auto default_equal_comparison(const void* a, const void* b) -> bool + SEQ_ALWAYS_INLINE bool default_equal_comparison(const void* a, const void* b) { return *static_cast(a) == *static_cast(b); } /// @brief Equal comparison based on functor template - SEQ_ALWAYS_INLINE auto default_equal_comparison_with_functor(const Fun& fun, const void* a, const void* b) -> bool + SEQ_ALWAYS_INLINE bool default_equal_comparison_with_functor(const Fun& fun, const void* a, const void* b) { return fun(*static_cast(a), *static_cast(b)); } @@ -867,7 +943,7 @@ namespace seq } // to arithmetic conversion - if (std::is_arithmetic_v) { + if constexpr (std::is_arithmetic_v) { if (info->type_id() <= get_type_id()) return detail::cast_arithmetic(this->data(), info); else if (info->type_id() == get_type_id()) @@ -885,7 +961,7 @@ namespace seq } // convert to tstring_view, only valid for string types - if (is_generic_string_view::value) { + else if constexpr (is_generic_string_view::value) { if (info->type_id() == get_type_id()) return detail::string_to_view(*static_cast(this->data())); else if (info->type_id() == get_type_id()) @@ -901,7 +977,7 @@ namespace seq } // to string type conversion - if (is_allocated_string::value) { + else if constexpr (is_allocated_string::value) { if (info->type_id() == get_type_id()) return detail::string_to_string(*static_cast(this->data())); else if (info->type_id() == get_type_id()) @@ -953,6 +1029,9 @@ namespace seq SEQ_ALWAYS_INLINE auto empty() const noexcept -> bool { return this->d_type_info.full() == 0; } /// @brief Returns the size of underlying type, 0 if empty. SEQ_ALWAYS_INLINE auto sizeof_type() const noexcept -> size_t { return empty() ? 0 : type()->sizeof_type(); } + /// @brief Returns the supported operations of underlying type, 0 if empty. + /// The result is a combination of supported_any_operation enum. + SEQ_ALWAYS_INLINE auto supported_operations() const noexcept -> int { return empty() ? 0 : type()->supported_operations(); } /// @brief Cast any object to given type. /// Throw a std::bad_cast on error. @@ -1585,6 +1664,7 @@ namespace seq { // Emplace construction using type = typename std::decay::type; + static_assert(std::is_same_v || !is_hold_any::value, "forbidden nested hold_any"); auto* i_t = this->type(); auto* o_t = get_type(); @@ -1713,6 +1793,8 @@ namespace seq { // remove const ref to T using type = typename std::decay::type; + static_assert(std::is_same_v || !is_hold_any::value, "forbidden nested hold_any"); + // set the type info pointer this->d_type_info = get_type(); @@ -1806,40 +1888,13 @@ namespace seq } } + /// @brief Assign any kind of object except a hold_any template::type>, void>::type> auto operator=(T&& value) -> hold_any& { using type = typename std::decay::type; - auto* i_t = this->type(); - auto* o_t = get_type(); - - if (o_t == i_t) { - // same type - *static_cast(this->data()) = std::forward(value); - } - else { - // different types, clear this object - reset(); - // set type info pointer - this->d_type_info = o_t; - void* d = nullptr; - try { - // create memory chunk and tags, might throw - d = this->template alloc(); - // copy, might throw - new (d) type(std::forward(value)); - } - catch (...) { - // free memory chunk if needed - if (d && (this->d_type_info.tag() & detail::big_size)) - SEQ_ANY_FREE(d); - // reset type info pointer and tags - this->d_type_info.set_full(0); - throw; - } - } - + emplace_args(std::forward(value)); return *this; } @@ -1891,130 +1946,144 @@ namespace seq /// @brief Returns true if underlying object is equal to other template - auto equal_to(T&& other) const -> bool + bool equal_to(const T& other) const { - using type = typename std::decay::type; + if constexpr (std::is_same_v) + return *this == other; + else { - // one empty: return false - if (this->empty()) - return false; + using type = typename std::decay::type; - const type_info_type* otype = get_type(); + // one empty: return false + if (this->empty()) + return false; - // same type: use equal_any - if (this->type() == otype) - return this->type()->equal_any(this->data(), &other); + const type_info_type* otype = get_type(); - int a_id = this->type_id(); + // same type: use equal_any + if (this->type() == otype) + return this->type()->equal_any(this->data(), &other); - // arithmetic comparison - if (is_arithmetic_type(a_id) && std::is_arithmetic_v) { - return detail::compare_equal_arithmetic(this->template cast(), std::forward(other)); - } + int a_id = this->type_id(); - // string comparison - if (is_string_type(a_id) && is_generic_char_string::value) { - return detail::compare_string(this->template cast(), std::forward(other)) == 0; - } + // arithmetic comparison + if (std::is_arithmetic_v && is_arithmetic_type(a_id)) { + return detail::compare_equal_arithmetic(this->template cast(), (other)); + } - // use registered comparison - if (a_id >= static_cast(detail::get_equal_comparison().size())) - return false; - const auto& converts = detail::get_equal_comparison()[static_cast(a_id)]; // info->d_convert; - int b_id = get_type_id(); - if (b_id >= static_cast(converts.size()) || !converts[static_cast(b_id)]) - return false; - return converts[static_cast(b_id)](this->data(), &other); + // string comparison + if (is_generic_char_string::value && is_string_type(a_id)) { + return detail::compare_string(this->template cast(), (other)) == 0; + } + + // use registered comparison + if (a_id >= static_cast(detail::get_equal_comparison().size())) + return false; + const auto& converts = detail::get_equal_comparison()[static_cast(a_id)]; // info->d_convert; + int b_id = get_type_id(); + if (b_id >= static_cast(converts.size()) || !converts[static_cast(b_id)]) + return false; + return converts[static_cast(b_id)](this->data(), &other); + } } /// @brief Returns true if underlying object compares less than other template - auto less_than(T&& other) const -> bool + bool less_than(const T& other) const { - using type = typename std::decay::type; + if constexpr (std::is_same_v) + return *this < other; + else { + using type = typename std::decay::type; - if (this->empty()) - return true; + if (this->empty()) + return true; - const type_info_type* otype = get_type(); + const type_info_type* otype = get_type(); - // same type: use less_any - if (this->type() == otype) - return otype->less_any(this->data(), &other); + // same type: use less_any + if (this->type() == otype) + return otype->less_any(this->data(), &other); - int a_id = this->type_id(); + int a_id = this->type_id(); - // arithmetic comparison - if (is_arithmetic_type(a_id) && std::is_arithmetic_v) { - if (is_integral_type(a_id) && std::is_integral_v) { - if (is_signed_integral_type(a_id) != std::is_signed_v || is_unsigned_integral_type(a_id)) // different sign: convert to unsigned integral - return detail::less_arithmetic(this->template cast(), other); - else - return detail::less_arithmetic(this->template cast(), other); + // arithmetic comparison + if (std::is_arithmetic_v && is_arithmetic_type(a_id)) { + if (is_integral_type(a_id) && std::is_integral_v) { + if (is_signed_integral_type(a_id) != std::is_signed_v || is_unsigned_integral_type(a_id)) // different sign: convert to unsigned integral + return detail::less_arithmetic(this->template cast(), other); + else + return detail::less_arithmetic(this->template cast(), other); + } + return detail::less_arithmetic(this->template cast(), other); } - return detail::less_arithmetic(this->template cast(), other); - } - // string comparison - if (is_string_type(a_id) && is_generic_char_string::value) { - return detail::compare_string(this->template cast(), other) < 0; - } + // string comparison + if (is_generic_char_string::value && is_string_type(a_id)) { + return detail::compare_string(this->template cast(), other) < 0; + } - // use registered comparison - int b_id = get_type_id(); - if (a_id >= static_cast(detail::get_less_comparison().size())) - return a_id < b_id; - const auto& converts = detail::get_less_comparison()[static_cast(a_id)]; // info->d_convert; - if (b_id >= static_cast(converts.size()) || !converts[static_cast(b_id)]) - return a_id < b_id; - return converts[static_cast(b_id)](this->data(), &other); + // use registered comparison + int b_id = get_type_id(); + if (a_id >= static_cast(detail::get_less_comparison().size())) + return a_id < b_id; + const auto& converts = detail::get_less_comparison()[static_cast(a_id)]; // info->d_convert; + if (b_id >= static_cast(converts.size()) || !converts[static_cast(b_id)]) + return a_id < b_id; + return converts[static_cast(b_id)](this->data(), &other); + } } /// @brief Returns true if underlying object compares greater than other template - auto greater_than(T&& other) const -> bool + bool greater_than(const T& other) const { - using type = typename std::decay::type; + if constexpr (std::is_same_v) + return *this > other; + else { - if (this->empty()) - return true; + using type = typename std::decay::type; - const type_info_type* otype = get_type(); + if (this->empty()) + return true; - // same type: use less_any - if (this->type() == otype) - return otype->less_any(&other, this->data()); + const type_info_type* otype = get_type(); - int a_id = this->type_id(); + // same type: use less_any + if (this->type() == otype) + return otype->less_any(&other, this->data()); - // arithmetic comparison - if (is_arithmetic_type(a_id) && std::is_arithmetic_v) { - if (is_integral_type(a_id) && std::is_integral_v) { - if (is_signed_integral_type(a_id) != std::is_signed_v || is_unsigned_integral_type(a_id)) // different sign: convert to unsigned integral - return detail::greater_arithmetic(this->template cast(), other); - else - return detail::greater_arithmetic(this->template cast(), other); + int a_id = this->type_id(); + + // arithmetic comparison + if (std::is_arithmetic_v && is_arithmetic_type(a_id)) { + if (is_integral_type(a_id) && std::is_integral_v) { + if (is_signed_integral_type(a_id) != std::is_signed_v || is_unsigned_integral_type(a_id)) // different sign: convert to unsigned integral + return detail::greater_arithmetic(this->template cast(), other); + else + return detail::greater_arithmetic(this->template cast(), other); + } + return detail::greater_arithmetic(this->template cast(), other); } - return detail::greater_arithmetic(this->template cast(), other); - } - // string comparison - if (is_string_type(a_id) && is_generic_char_string::value) { - return detail::compare_string(this->template cast(), other) > 0; - } + // string comparison + if (is_generic_char_string::value && is_string_type(a_id)) { + return detail::compare_string(this->template cast(), other) > 0; + } - // use registered comparison - int b_id = get_type_id(); - if (b_id >= static_cast(detail::get_less_comparison().size())) - return b_id < a_id; - const auto& converts = detail::get_less_comparison()[static_cast(b_id)]; // info->d_convert; - if (a_id >= static_cast(converts.size()) || !converts[static_cast(a_id)]) - return b_id < a_id; - return converts[static_cast(a_id)](&other, this->data()); + // use registered comparison + int b_id = get_type_id(); + if (b_id >= static_cast(detail::get_less_comparison().size())) + return b_id < a_id; + const auto& converts = detail::get_less_comparison()[static_cast(b_id)]; // info->d_convert; + if (a_id >= static_cast(converts.size()) || !converts[static_cast(a_id)]) + return b_id < a_id; + return converts[static_cast(a_id)](&other, this->data()); + } } /// @brief Equality comparison operator - auto operator==(const hold_any& other) const -> bool + bool operator==(const hold_any& other) const { const bool a_empty = this->empty(); const bool b_empty = other.empty(); @@ -2051,8 +2120,9 @@ namespace seq return false; return converts[static_cast(b_id)](this->data(), other.data()); } - SEQ_ALWAYS_INLINE auto operator!=(const hold_any& other) const -> bool { return !(*this == other); } - auto operator<(const hold_any& other) const -> bool + SEQ_ALWAYS_INLINE bool operator!=(const hold_any& other) const { return !(*this == other); } + + bool operator<(const hold_any& other) const { const bool a_empty = this->empty(); const bool b_empty = other.empty(); @@ -2095,9 +2165,9 @@ namespace seq return a_id < b_id; return converts[static_cast(b_id)](this->data(), other.data()); } - SEQ_ALWAYS_INLINE auto operator<=(const hold_any& other) const -> bool { return !(other < *this); } - SEQ_ALWAYS_INLINE auto operator>(const hold_any& other) const -> bool { return other < *this; } - SEQ_ALWAYS_INLINE auto operator>=(const hold_any& other) const -> bool { return !(*this < other); } + SEQ_ALWAYS_INLINE bool operator<=(const hold_any& other) const { return !(other < *this); } + SEQ_ALWAYS_INLINE bool operator>(const hold_any& other) const { return other < *this; } + SEQ_ALWAYS_INLINE bool operator>=(const hold_any& other) const { return !(*this < other); } }; /// @brief Register a conversion function based on explicit conversion from type T to type U @@ -2111,7 +2181,7 @@ namespace seq auto& converts = detail::get_converters()[in_id]; if (converts.size() <= out_id) converts.resize(out_id + 1); - converts[out_id] = std::function(detail::default_convert); + converts[out_id] = detail::default_convert; } /// @brief Register a conversion function using given functor template @@ -2124,7 +2194,8 @@ namespace seq auto& converts = detail::get_converters()[in_id]; if (converts.size() <= out_id) converts.resize(out_id + 1); - converts[out_id] = std::function(std::bind(detail::default_convert_with_functor, fun, std::placeholders::_1, std::placeholders::_2)); + + converts[out_id] = [fun](const void* in, void* out) { detail::default_convert_with_functor(fun, in, out); }; } /// @brief Register the comparison function T() < U() @@ -2138,7 +2209,7 @@ namespace seq auto& converts = detail::get_less_comparison()[in_id]; if (converts.size() <= out_id) converts.resize(out_id + 1); - converts[out_id] = std::function(detail::default_less_comparison); + converts[out_id] = detail::default_less_comparison; } /// @brief Register a comparison function between types T and U based on given functor template @@ -2151,7 +2222,8 @@ namespace seq auto& converts = detail::get_less_comparison()[in_id]; if (converts.size() <= out_id) converts.resize(out_id + 1); - converts[out_id] = std::function(std::bind(detail::default_less_comparison_with_functor, fun, std::placeholders::_1, std::placeholders::_2)); + + converts[out_id] = [fun](const void* l, const void* r) { return detail::default_less_comparison_with_functor(fun, l, r); }; } /// @brief Register the comparison function T() == U() @@ -2165,7 +2237,7 @@ namespace seq auto& converts = detail::get_equal_comparison()[in_id]; if (converts.size() <= out_id) converts.resize(out_id + 1); - converts[out_id] = std::function(detail::default_equal_comparison); + converts[out_id] = detail::default_equal_comparison; } /// @brief Register a comparison function between types T and U based on given functor @@ -2179,8 +2251,7 @@ namespace seq auto& converts = detail::get_equal_comparison()[in_id]; if (converts.size() <= out_id) converts.resize(out_id + 1); - converts[out_id] = - std::function(std::bind(detail::default_equal_comparison_with_functor, fun, std::placeholders::_1, std::placeholders::_2)); + converts[out_id] = [fun](const void* l, const void* r) { return detail::default_equal_comparison_with_functor(fun, l, r); }; } // operator == with any other type, mandatory for heterogeneous lookup in hash table @@ -2309,30 +2380,6 @@ namespace seq return res; } - template - class ostream_format, Slot> : public base_ostream_format, ostream_format, Slot>> - { - using base_type = base_ostream_format, ostream_format, Slot>>; - - public: - ostream_format() - : base_type() - { - } - explicit ostream_format(const hold_any& v) - : base_type(v) - { - } - - auto to_string(std::string& out) const -> size_t - { - size_t prev = out.size(); - if (!this->value().empty()) - this->value().type()->format_any(out, this->value().data(), this->width_fmt(), this->numeric_fmt()); - return out.size() - prev; - } - }; - using any = hold_any<>; using r_any = hold_any; using nh_any = hold_any; @@ -2344,42 +2391,14 @@ namespace seq static constexpr bool value = hold_any::relocatable; }; - template - struct is_hold_any : std::false_type - { - }; - - template - struct is_hold_any> : std::true_type - { - }; - - namespace detail - { - // Force type nh_any to be stored inline in formatting object - template - struct inline_value_storage> : std::true_type - { - }; - } - - /// @brief Create placeholder for any type for the formatting module - SEQ_ALWAYS_INLINE auto _any() -> ostream_format - { - return ostream_format(); - } - SEQ_ALWAYS_INLINE auto _a() -> ostream_format - { - return ostream_format(); - } // specialization of seq::hasher for hold_any template class hasher> { public: - using is_transparent = void; - using is_avalanching = void; + using is_transparent = std::true_type; + using is_avalanching = std::true_type; auto operator()(const seq::hold_any& a) const -> size_t { return (a.hash()); } @@ -2396,9 +2415,8 @@ namespace seq template auto operator()(const T& value) const -> size_t { - using type = T; // For any type, use the type info for this type and call hash_any - const auto* type_inf = seq::hold_any::template get_type(); + const auto* type_inf = seq::hold_any::template get_type(); return (type_inf->hash_any(&value)); } }; @@ -2420,7 +2438,8 @@ namespace std class hash> { public: - using is_transparent = void; + using is_transparent = std::true_type; + using is_avalanching = std::true_type; auto operator()(const seq::hold_any& a) const -> size_t { return a.hash(); } @@ -2437,9 +2456,8 @@ namespace std template auto operator()(const T& value) const -> size_t { - using type = T; // For any type, use the type info for this type and call hash_any - const auto* type_inf = seq::hold_any::template get_type(); + const auto* type_inf = seq::hold_any::template get_type(); return type_inf->hash_any(&value); } }; diff --git a/seq/bits.hpp b/seq/bits.hpp index a155780b..6e646372 100644 --- a/seq/bits.hpp +++ b/seq/bits.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/seq/concurrent_map.hpp b/seq/concurrent_map.hpp index 253e0eea..f986e33a 100644 --- a/seq/concurrent_map.hpp +++ b/seq/concurrent_map.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -94,7 +94,7 @@ namespace seq this->rehash(n); } - template + template::value, int> = 0> concurrent_set(InputIterator f, InputIterator l, size_type n = 0, const hasher& hf = hasher(), const key_equal& eql = key_equal(), const allocator_type& a = allocator_type()) : base_type(hf, eql, a) { @@ -113,7 +113,7 @@ namespace seq { } - template + template::value, int> = 0> concurrent_set(InputIterator f, InputIterator l, allocator_type const& a) : concurrent_set(f, l, 0, hasher(), key_equal(), a) { @@ -150,13 +150,13 @@ namespace seq { } - template + template::value, int> = 0> concurrent_set(InputIterator f, InputIterator l, size_type n, const allocator_type& a) : concurrent_set(f, l, n, hasher(), key_equal(), a) { } - template + template::value, int> = 0> concurrent_set(InputIterator f, InputIterator l, size_type n, const hasher& hf, const allocator_type& a) : concurrent_set(f, l, n, hf, key_equal(), a) { @@ -201,7 +201,7 @@ namespace seq SEQ_CONCURRENT_INLINE auto key_eq() const -> key_equal { return base_type::get_key_eq(); } SEQ_CONCURRENT_INLINE void clear() { base_type::clear(); } - SEQ_CONCURRENT_INLINE void rehash(size_t n) { base_type::rehash(n); } + SEQ_CONCURRENT_INLINE void rehash(size_t n) { base_type::rehash((size_t)(n / (double)max_load_factor())); } SEQ_CONCURRENT_INLINE void reserve(size_t size) { base_type::reserve(size); } SEQ_CONCURRENT_INLINE void swap(concurrent_set& other) noexcept(noexcept(std::declval().swap(std::declval()))) { base_type::swap(other); } @@ -510,7 +510,7 @@ namespace seq this->rehash(n); } - template + template::value, int> = 0> concurrent_map(InputIterator f, InputIterator l, size_type n = 0, const hasher& hf = hasher(), const key_equal& eql = key_equal(), const allocator_type& a = allocator_type()) : base_type(hf, eql, a) { @@ -529,7 +529,7 @@ namespace seq { } - template + template::value, int> = 0> concurrent_map(InputIterator f, InputIterator l, allocator_type const& a) : concurrent_map(f, l, 0, hasher(), key_equal(), a) { @@ -566,13 +566,13 @@ namespace seq { } - template + template::value, int> = 0> concurrent_map(InputIterator f, InputIterator l, size_type n, const allocator_type& a) : concurrent_map(f, l, n, hasher(), key_equal(), a) { } - template + template::value, int> = 0> concurrent_map(InputIterator f, InputIterator l, size_type n, const hasher& hf, const allocator_type& a) : concurrent_map(f, l, n, hf, key_equal(), a) { @@ -617,7 +617,7 @@ namespace seq SEQ_CONCURRENT_INLINE auto key_eq() const -> key_equal { return base_type::get_key_eq(); } SEQ_CONCURRENT_INLINE void clear() { base_type::clear(); } - SEQ_CONCURRENT_INLINE void rehash(size_t n) { base_type::rehash(n); } + SEQ_CONCURRENT_INLINE void rehash(size_t n) { base_type::rehash((size_t)(n / (double)max_load_factor())); } SEQ_CONCURRENT_INLINE void reserve(size_t size) { base_type::reserve(size); } SEQ_CONCURRENT_INLINE void swap(concurrent_map& other) noexcept(noexcept(std::declval().swap(std::declval()))) { base_type::swap(other); } diff --git a/seq/concurrent_queue.hpp b/seq/concurrent_queue.hpp new file mode 100644 index 00000000..377ca5be --- /dev/null +++ b/seq/concurrent_queue.hpp @@ -0,0 +1,763 @@ +/** + * MIT License + * + * Copyright (c) 2026 Victor Moncada + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef SEQ_CONCURRENT_QUEUE_HPP +#define SEQ_CONCURRENT_QUEUE_HPP + +#include + +#include "lock.hpp" +#include "internal/utils.hpp" + +namespace seq +{ + namespace detail + { + + // Internal implementation of the concurrent queue + template> + class QueueImpl : private Allocator + { + // Create mask value based on bit count + template + static constexpr uint64_t QueueMask() + { + if constexpr (Count == 64) + return std::numeric_limits::max(); + else + return ((1ull << count) - 1ull); + } + + public: + using value_type = T; + using size_type = uint64_t; + using lock_type = std::mutex; + using atomic_type = std::atomic; + + // Number of elements per bucket (power of 2) + static constexpr size_type count = sizeof(T) <= 32 ? 64 : sizeof(T) <= 64 ? 32 : sizeof(T) <= 128 ? 16 : sizeof(T) <= 256 ? 8 : 4; + + // Mask value to retrieve bucket position based on head/tail position + static constexpr size_type mask_full = QueueMask(); + + // Invalid position + static constexpr size_type invalid = std::numeric_limits::max(); + + struct Bucket; + using atomic_bucket = std::atomic; + + struct BaseBucket + { + atomic_bucket prev{ nullptr }; // Linked list + atomic_bucket next{ nullptr }; // Linked list + size_type head_start = mask_full; // Bucket index + + bool is_end() const noexcept { return head_start == mask_full; } + }; + // Bucket structure, stores up to count elements + struct Bucket : BaseBucket + { + atomic_type cnt{ mask_full }; // Mask of created (constructed) elements + atomic_type pop{ 0 }; // Mask of poped (removed) elements + alignas(T) char values[sizeof(T) * count] = { '\0' }; // Elements + + // Return pointer to elements + SEQ_ALWAYS_INLINE T* ptr() noexcept { return (T*)values; } + + // Return bitmask of valid elements + size_type valid_mask() const noexcept { return cnt.load(std::memory_order_relaxed) & (~pop.load(std::memory_order_relaxed)); } + + // Return index of first valid element + uint16_t first_valid_index() const noexcept + { + auto mask = valid_mask(); + return mask ? (uint16_t)bit_scan_forward_64(mask) : 0; + } + + // Return index of last valid element + uint16_t last_valid_index() const noexcept + { + auto mask = valid_mask(); + return mask ? (uint16_t)bit_scan_reverse_64(mask) : 0; + } + + // Return if given index contains a valid element + bool is_valid(unsigned idx) const noexcept + { + auto mask = valid_mask(); + return mask & (1ull << idx); + } + }; + + private: + // Head (insert) position + atomic_type d_head{ 0 }; + + // Tail (pop) position + atomic_type d_tail{ 0 }; + + // End bucket of linked list + BaseBucket d_end; + + // Free list of buckets + atomic_bucket d_free{ nullptr }; + atomic_type d_free_count{ 0 }; + lock_type d_free_lock; + + template + static SEQ_ALWAYS_INLINE size_type add(atomic_type& a, EarlyStop f = {}) noexcept + { + // Increment atomic counter using CAS. + // This is the main bottleneck for concurrent insert/pop, + // and using CAS + yield is way more effective than fetch_add(). + + // An early stop condition can be provided. + + auto val = a.load(std::memory_order_relaxed); + uint8_t cnt = 0; + for (;;) { + // Early stop condition + if constexpr (!std::is_same_v) + if (f(val)) + return invalid; + + if (a.compare_exchange_strong(val, val + 1)) + return val; + + + for (uint8_t i = 0; i < cnt +1; ++i) + std::this_thread::yield(); + cnt = ((cnt + 1) & 31); + } + } + + SEQ_ALWAYS_INLINE void ensure_has_bucket() + { + // Make sure we have at least one free bucket available + if (!d_free.load(std::memory_order_relaxed)) + free_bucket(new (allocate_from(get_allocator())) Bucket()); + } + + Bucket* pop_bucket() noexcept + { + // Remove and return bucket from the free list. + auto* b = d_free.load(std::memory_order_relaxed); + while (b && !d_free.compare_exchange_strong(b, b->next.load(std::memory_order_relaxed))) + ; + + if (b) + d_free_count.fetch_sub(count); + return b; + } + + void free_bucket(Bucket* q) noexcept + { + // Add bucket to free list. + auto fr = d_free.load(std::memory_order_relaxed); + for (;;) { + q->next.store(fr, std::memory_order_release); + if (d_free.compare_exchange_strong(fr, q)) + break; + } + d_free_count.fetch_add(count); + } + + void make_bucket(size_type head_start, T&& val) noexcept + { + // Create a bucket of one element using the internal free list of buckets + Bucket* b = pop_bucket(); + b->cnt.store(1); + b->pop.store(0); + b->head_start = head_start; + new (b->ptr()) T(std::move(val)); + b->next.store(end_bucket(), std::memory_order_release); + + std::scoped_lock ll(d_free_lock); + + // Add to list + auto last = d_end.prev.load(std::memory_order_relaxed); + b->prev.store(last, std::memory_order_release); + last->next.store(b, std::memory_order_release); + end_bucket()->prev.store(b, std::memory_order_release); + } + + void remove_bucket(Bucket* first) noexcept + { + { + std::scoped_lock ll(d_free_lock); + + // Remove from list and bucket to the free list + auto next = first->next.load(std::memory_order_relaxed); + end_bucket()->next.store(next, std::memory_order_release); + next->prev.store(end_bucket(), std::memory_order_release); + } + free_bucket(first); + } + + static SEQ_ALWAYS_INLINE void destroy(T* v) noexcept + { + // Destroy element without throwing + if constexpr (!std::is_trivially_destructible_v) { + if constexpr (std::is_nothrow_destructible_v) + v->~T(); + else { + try { + v->~T(); + } + catch (...) { + } + } + } + } + + template + SEQ_ALWAYS_INLINE bool pop_internal(F fun) noexcept + { + // Get the tail position and increment it. + // Fail if the tail already reached the head position. + auto pos = add(d_tail, [&](auto p) { return p >= d_head.load(std::memory_order_relaxed); }); + // Empty + if (pos == invalid) + return false; + + auto idx = pos & (count - 1); + auto head_start = pos / count; + + for (;;) { + auto first = d_end.next.load(std::memory_order_relaxed); + + // Check if empty or if this is the right bucket + if (first == end_bucket() || first->head_start != head_start) { + std::this_thread::yield(); + continue; + } + + // Check if value is valid + auto idx_bits = (1ull << idx); + if (!(first->cnt.load(std::memory_order_relaxed) & idx_bits)) + continue; + + // Retrieve/destroy value + fun(first->ptr()[idx]); + // Mark as poped + auto poped = first->pop.fetch_or(idx_bits, std::memory_order_relaxed) | idx_bits; + + if (poped == mask_full) + // Destroy bucket if all values were removed + remove_bucket(first); + + return true; + } + } + + SEQ_ALWAYS_INLINE void push_internal(uint64_t pos, T&& val) noexcept + { + auto idx = pos & (count - 1); + auto head_start = pos / count; + + for (;;) { + auto last = d_end.prev.load(std::memory_order_relaxed); + if (idx != 0) { + if (head_start != last->head_start) + continue; + new (last->ptr() + idx) T(std::move(val)); + last->cnt.fetch_or(1ull << idx, std::memory_order_relaxed); + return; + } + else { + if (last != end_bucket()) { + // Ensure that we ARE in the last bucket and that it is full + if (head_start != last->head_start + 1 || last->cnt.load(std::memory_order_relaxed) != mask_full) + continue; + } + + return make_bucket(head_start, std::move(val)); + } + } + } + + public: + QueueImpl(const Allocator& al = {}) + : Allocator(al) + { + // Initialize linked list + d_end.next.store(end_bucket()); + d_end.prev.store(end_bucket()); + } + + ~QueueImpl() noexcept + { + // Remove entries + clear(); + + // Remove current bucket + auto* start = d_end.next.load(std::memory_order_relaxed); + if (start != end_bucket()) + free_bucket(start); + + // Free pending buckets + shrink_to_fit(); + } + + Bucket* end_bucket() const noexcept { return (Bucket*)&d_end; } + + const Allocator& get_allocator() const noexcept { return *this; } + + void shrink_to_fit() noexcept + { + // Free pending buckets. + // This function is NOT thread safe. + while (Bucket* b = pop_bucket()) + deallocate_from(get_allocator(), b); + } + + void reserve(size_t count) + { + // Allocate enough buckets to store up to count elements + auto s = size(); + if (s >= count) + return; + s = count - s; + + std::scoped_lock ll(d_free_lock); + while (d_free_count < s) { + Bucket* b = new (allocate_from(get_allocator())) Bucket(); + free_bucket(b); + } + } + + void clear() noexcept + { + while (!empty()) + pop(); + } + + template + SEQ_ALWAYS_INLINE void emplace(Args&&... args) + { + // Create value upfront as this might throw + T val{ std::forward(args)... }; + + // Increment head position, and make sure a bucket is availble if needed BEFORE increment + // to avoid potential bad_alloc exception to corrupt the head state. + auto pos = add(d_head, [&](auto v) { + if ((v & (count - 1)) == 0) + ensure_has_bucket(); + return false; + }); + push_internal(pos, std::move(val)); + } + + template + SEQ_ALWAYS_INLINE bool try_emplace(Args&&... args) + { + // Create value upfront as this might throw + T val{ std::forward(args)... }; + + // Increment head position, and make sure a bucket is availble if needed BEFORE increment + // to avoid potential bad_alloc exception to corrupt the head state. + auto pos = add(d_head, [&](auto v) { + if ((v & (count - 1)) == 0) + return (!d_free.load(std::memory_order_relaxed)); + return false; + }); + if (pos == invalid) + return false; + push_internal(pos, std::move(val)); + return true; + } + + SEQ_ALWAYS_INLINE bool pop() noexcept + { + return pop_internal([](T& v) { destroy(&v); }); + } + SEQ_ALWAYS_INLINE bool pop(T& val) noexcept + { + auto f = [&](T& v) { + val = std::move(v); + destroy(&v); + }; + return pop_internal(f); + } + SEQ_ALWAYS_INLINE size_t size() const noexcept + { + auto h = d_head.load(std::memory_order_relaxed); + auto t = d_tail.load(std::memory_order_relaxed); + return h > t ? (h - t) : 0; + } + SEQ_ALWAYS_INLINE bool empty() const noexcept { return size() == 0; } + }; + + // Const iterator fo concurrent_queue (unsafe) + template + struct QueueConstIterator + { + using value_type = typename Queue::value_type; + using bucket_type = typename Queue::Bucket; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using iterator_category = std::bidirectional_iterator_tag; + using difference_type = std::ptrdiff_t; + using size_type = std::uint64_t; + static constexpr size_type count = Queue::count; + + QueueConstIterator() noexcept = default; + QueueConstIterator(bucket_type* b) noexcept // end + : d_bucket(b) + { + } + QueueConstIterator(bucket_type* b, unsigned p) noexcept + : d_bucket(b) + , d_pos(p) + , d_first(b->first_valid_index()) + , d_last(b->last_valid_index()) + { + } + + auto operator++() noexcept -> QueueConstIterator& + { + SEQ_ASSERT_DEBUG(d_bucket && !d_bucket->is_end(), "invalid operation on end iterator"); + if (d_pos++ == d_last) { + d_bucket = d_bucket->next.load(std::memory_order_relaxed); + if (d_bucket->is_end()) + d_pos = d_first = d_last = 0; + else { + d_pos = d_first = d_bucket->first_valid_index(); + d_last = d_bucket->last_valid_index(); + } + } + return *this; + } + auto operator++(int) noexcept -> QueueConstIterator + { + auto ret = *this; + ++(*this); + return ret; + } + auto operator--() noexcept -> QueueConstIterator& + { + if (d_pos-- == d_first) { + d_bucket = d_bucket->prev.load(std::memory_order_relaxed); + if (d_bucket->is_end()) + d_pos = d_first = d_last = 0; + else { + d_first = d_bucket->first_valid_index(); + d_pos = d_last = d_bucket->last_valid_index(); + } + } + return *this; + } + auto operator--(int) noexcept -> QueueConstIterator + { + auto ret = *this; + --(*this); + return ret; + } + auto operator*() const noexcept -> const_reference + { + SEQ_ASSERT_DEBUG(d_bucket && !d_bucket->is_end(), "invalid operation on end iterator"); + SEQ_ASSERT_DEBUG(d_bucket->is_valid(d_pos), "iterator points to an empty location"); + return d_bucket->ptr()[d_pos]; + } + auto operator->() const noexcept -> const_pointer { return std::pointer_traits::pointer_to(**this); } + + bool operator==(const QueueConstIterator& other) const noexcept { return d_bucket == other.d_bucket && d_pos == other.d_pos; } + bool operator!=(const QueueConstIterator& other) const noexcept { return !operator==(other); } + + private: + bucket_type* d_bucket = nullptr; + unsigned d_pos = 0; + uint16_t d_first = 0; + uint16_t d_last = 0; + }; + + // Iterator fo concurrent_queue (unsafe) + template + struct QueueIterator : public QueueConstIterator + { + using base_type = QueueConstIterator; + using bucket_type = typename base_type::bucket_type; + using value_type = typename base_type::value_type; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using iterator_category = std::bidirectional_iterator_tag; + using difference_type = std::ptrdiff_t; + using size_type = std::uint64_t; + + QueueIterator() noexcept = default; + QueueIterator(const base_type& other) noexcept + : base_type(other) + { + } + QueueIterator(bucket_type* b) noexcept + : base_type(b) + { + } + QueueIterator(bucket_type* b, unsigned p) noexcept + : base_type(b, p) + { + } + auto operator*() const noexcept -> reference { return const_cast(base_type::operator*()); } + auto operator->() const noexcept -> pointer { return std::pointer_traits::pointer_to(**this); } + auto operator++() noexcept -> QueueIterator& + { + base_type::operator++(); + return *this; + } + auto operator++(int) noexcept -> QueueIterator + { + QueueIterator ret = *this; + base_type::operator++(); + return ret; + } + auto operator--() noexcept -> QueueIterator& + { + base_type::operator--(); + return *this; + } + auto operator--(int) noexcept -> QueueIterator + { + QueueIterator ret = *this; + base_type::operator--(); + return ret; + } + bool operator==(const base_type& other) const noexcept { return base_type::operator==(other); } + bool operator!=(const base_type& other) const noexcept { return base_type::operator!=(other); } + }; + + } + + /// @brief Thread-safe FIFO container. + /// @tparam T Value type + /// @tparam Allocator Allocator type + /// + /// seq::concurrent_queue is a thread-safe unbounded queue (or FIFO) + /// aiming at better performances than using std::queue + std::mutex. + /// It is designed for Multi-Producer, Multi-Consumer (MPMC) scenarios. + /// + /// seq::concurrent_queue is not lock-free nor wait-free, but combines + /// atomic-based operations with locks to provide a certain level of + /// concurrency. In all tested scenarios, concurrent_queue is faster + /// than a regular std::queue with std::mutex, with a gain of up to + /// a factor 2 in some situations. + /// + /// In addition, concurrent_queue provides unsafe but usefull features + /// like iteration. + /// + template> + class concurrent_queue : private Allocator + { + using lock_type = spinlock; + using queue_type = detail::QueueImpl; + + public: + static_assert(std::is_nothrow_move_constructible_v); + static_assert(std::is_nothrow_move_assignable_v); + + using value_type = T; + using reference = T&; + using pointer = T*; + using size_type = uint64_t; + using allocator_type = Allocator; + using iterator = detail::QueueIterator; + using const_iterator = detail::QueueConstIterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + // Safe API + + /// @brief Default constructor + concurrent_queue(const Allocator al = {}) + : Allocator(al) + { + } + + /// @brief Construct and reserve enough space to store at least count elements + concurrent_queue(size_type count, const Allocator al = {}) + : Allocator(al) + { + reserve(count); + } + + /// @brief Move constructor + /// It is safe to call this while other is still being used. + concurrent_queue(concurrent_queue&& other) noexcept(std::is_nothrow_move_constructible_v) + : Allocator(std::move(static_cast(other))) + { + std::scoped_lock ll(other.d_data_lock); + d_data.store(other.d_data.exchange(nullptr)); + } + + /// @brief Move assignment operator. + /// It is safe to call this operator while both queues are being used. + concurrent_queue& operator=(concurrent_queue&& other) noexcept + { + swap(other); + return *this; + } + + /// @brief Destructor + ~concurrent_queue() + { + if (auto* d = d_data.load()) { + d->~queue_type(); + deallocate_from(get_allocator(), d); + } + } + + /// @brief Returns the queue allocator + const allocator_type& get_allocator() const noexcept { return static_cast(*this); } + + /// @brief Remove all elements of the queue. + /// This function might never return if other thread(s) constantly push new values to the queue. + void clear() noexcept + { + auto d = d_data.load(std::memory_order_relaxed); + if (d) + d->clear(); + } + + /// @brief Swap 2 queues. + /// It is safe to call swap while both queues are being used. + void swap(concurrent_queue& other) noexcept(noexcept(swap_allocator(std::declval(), std::declval()))) + { + std::scoped_lock ll(d_data_lock, other.d_data_lock); + + swap_allocator(static_cast(*this), static_cast(other)); + + auto d = d_data.load(); + d_data.store(other.d_data.load()); + other.d_data.store(d); + } + + /// @brief Reserve enough space to hold at least count elements. + void reserve(size_t count) { data()->reserve(count); } + + /// @brief Push an element to the back of the queue. + /// Strong exception guarantee. + template + SEQ_ALWAYS_INLINE void emplace(Args&&... args) + { + return data()->emplace(std::forward(args)...); + } + /// @brief Push an element to the back of the queue. + /// Strong exception guarantee. + SEQ_ALWAYS_INLINE void push(const T& v) { return emplace(v); } + /// @brief Push an element to the back of the queue. + /// Strong exception guarantee. + SEQ_ALWAYS_INLINE void push(T&& v) { return emplace(std::move(v)); } + + /// @brief Try to retrieve the front element of the queue. + SEQ_ALWAYS_INLINE bool pop(T& v) noexcept + { + auto d = d_data.load(std::memory_order_relaxed); + return d ? d->pop(v) : false; + } + /// @brief Try to discard the front element of the queue. + SEQ_ALWAYS_INLINE bool pop() noexcept + { + auto d = d_data.load(std::memory_order_relaxed); + return d ? d->pop() : false; + } + + /// @brief Returns an estimation of the queue size. + SEQ_ALWAYS_INLINE auto size() const noexcept + { + auto d = d_data.load(std::memory_order_relaxed); + return d ? d->size() : 0; + } + /// @brief Returns true if the queue is empty. + SEQ_ALWAYS_INLINE bool empty() const noexcept { return size() == 0; } + + // Unsafe API + + /// @brief Release unused memory. + void shrink_to_fit() noexcept + { + auto d = d_data.load(std::memory_order_relaxed); + if (d) + d->shrink_to_fit(); + } + + auto cbegin() const noexcept -> const_iterator + { + auto d = d_data.load(std::memory_order_relaxed); + if (!d) + return const_iterator(); + auto bucket = d->end_bucket()->next.load(std::memory_order_relaxed); + if (bucket == d->end_bucket()) + return end(); + return const_iterator(bucket, bucket->first_valid_index()); + } + auto begin() noexcept -> iterator { return cbegin(); } + auto begin() const noexcept -> const_iterator { return cbegin(); } + + auto cend() const noexcept -> const_iterator + { + auto d = d_data.load(std::memory_order_relaxed); + return d ? const_iterator(d->end_bucket()) : const_iterator(); + } + auto end() noexcept -> iterator { return cend(); } + auto end() const noexcept -> const_iterator { return cend(); } + + auto rbegin() noexcept { return reverse_iterator(end()); } + auto crbegin() const noexcept { return const_reverse_iterator(end()); } + auto rbegin() const noexcept { return const_reverse_iterator(end()); } + + auto rend() noexcept { return reverse_iterator(begin()); } + auto crend() const noexcept { return const_reverse_iterator(begin()); } + auto rend() const noexcept { return const_reverse_iterator(begin()); } + + private: + SEQ_ALWAYS_INLINE queue_type* data() const + { + auto d = d_data.load(std::memory_order_relaxed); + if SEQ_LIKELY (d) + return d; + return const_cast(this)->make_data(); + } + + queue_type* make_data() + { + std::scoped_lock ll(d_data_lock); + auto d = new (allocate_from(get_allocator())) queue_type(get_allocator()); + queue_type* prev = nullptr; + if (!d_data.compare_exchange_strong(prev, d)) { + d->~queue_type(); + deallocate_from(get_allocator(), d); + d = prev; + } + return d; + } + + std::atomic d_data{ nullptr }; + lock_type d_data_lock; + }; +} + +#endif \ No newline at end of file diff --git a/seq/devector.hpp b/seq/devector.hpp index d622ee6e..91944ea7 100644 --- a/seq/devector.hpp +++ b/seq/devector.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ #include "type_traits.hpp" #include "bits.hpp" -#include "utils.hpp" +#include "internal/utils.hpp" #include @@ -688,7 +688,7 @@ namespace seq /// @param first the range to copy the elements from /// @param last the range to copy the elements from /// @param alloc allocator to use for all memory allocations of this container - template + template::value, int> = 0> devector(InputIt first, InputIt last, const Allocator& alloc = Allocator()) : base_type(alloc) { diff --git a/seq/flat_map.hpp b/seq/flat_map.hpp index c3575397..da5a099b 100644 --- a/seq/flat_map.hpp +++ b/seq/flat_map.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -47,7 +47,7 @@ See the documentation of each class for more details. #include "tiered_vector.hpp" #include "tiny_string.hpp" -#include "utils.hpp" +#include "internal/utils.hpp" #include "internal/binary_search.hpp" /** \addtogroup containers @@ -89,7 +89,7 @@ namespace seq }; template - SEQ_ALWAYS_INLINE int compare_value(const L& l, const R& r) noexcept + static SEQ_ALWAYS_INLINE int compare_value(const L& l, const R& r) noexcept { // Compare 2 values // Always return 0 if they are NOT comparable @@ -182,7 +182,7 @@ namespace seq template size_t sort_deque(Deque& d, size_t begin, Less le) { - if (Stable) { + if constexpr (Stable) { net_sort(d.begin() + begin, d.end(), le); return begin; } @@ -190,7 +190,7 @@ namespace seq using T = typename Deque::value_type; static constexpr bool nothrow = std::is_nothrow_move_constructible_v && std::is_nothrow_move_assignable_v; - if (!nothrow) { + if constexpr (!nothrow) { net_sort(d.begin() + begin, d.end(), le); return begin; } @@ -202,14 +202,14 @@ namespace seq template void sort_full_deque(Deque& d, Less le) { - if (Stable) { + if constexpr (Stable) { net_sort(d.begin(), d.end(), le); } else { using T = typename Deque::value_type; static constexpr bool nothrow = std::is_nothrow_move_constructible_v && std::is_nothrow_move_assignable_v; - if (!nothrow) { + if constexpr (!nothrow) { net_sort(d.begin(), d.end(), le); } else { @@ -241,38 +241,6 @@ namespace seq if SEQ_UNLIKELY (b_index == buckets.size()) return { d.size(), false }; - // TEST - /* { - // find inside bucket - const auto* bucket = buckets[b_index].bucket; - - using pos_type = int; - const T* begin_ptr = bucket->begin_ptr(); - const T* ptr; - pos_type partition_size; - pos_type offset = 0; - const bool low_half = begin_ptr != bucket->buffer() && // begin is not 0 - (bucket->begin + bucket->size) > bucket->max_size_ && // begin + size overflow - le(*(bucket->buffer() + bucket->max_size1), value); // value to took for is not in the upper half (between begin and buffer end) - - if (low_half) { - ptr = bucket->buffer(); - partition_size = ((bucket->begin + bucket->size) & bucket->max_size1); - offset = (bucket->max_size_ - bucket->begin); - } - else { - ptr = begin_ptr; - partition_size = (std::min(bucket->size, static_cast(bucket->max_size_ - bucket->begin))); - } - - auto _low = lower_bound(ptr, partition_size, value, le); - _low.first += offset; - size_t r = static_cast(_low.first) + - (b_index != 0 ? static_cast(buckets[0]->size) + static_cast(b_index - 1) * static_cast(bucket->max_size_) : 0); - return { r, _low.second }; - } - */ - // find inside bucket const auto* bucket = buckets[b_index].bucket; @@ -301,7 +269,7 @@ namespace seq /// @brief Optimized version of std::upper_bound(begin(),end(),value,le); /// Only works for sorted tiered_vector. template - auto tvector_upper_bound(const Deque& d, const U& value, const Less& le) noexcept(noexcept(le(std::declval(), value))) -> size_t + SEQ_ALWAYS_INLINE auto tvector_upper_bound(const Deque& d, const U& value, const Less& le) noexcept(noexcept(le(std::declval(), value))) -> size_t { return tvector_lower_bound(d, value, [&le](const auto& a, const auto& b) { return !le(b, a); }).first; } @@ -625,15 +593,15 @@ namespace seq return *this; } - auto get_allocator() const noexcept -> const Allocator& { return d_deque.get_allocator(); } - auto get_allocator() noexcept -> Allocator& { return d_deque.get_allocator(); } + SEQ_ALWAYS_INLINE auto get_allocator() const noexcept -> const Allocator& { return d_deque.get_allocator(); } + SEQ_ALWAYS_INLINE auto get_allocator() noexcept -> Allocator& { return d_deque.get_allocator(); } - auto empty() const noexcept -> bool { return d_deque.empty(); } - auto size() const noexcept -> size_type { return d_deque.size(); } - auto max_size() const noexcept -> size_type { return d_deque.max_size(); } - void clear() noexcept { d_deque.clear(); } + SEQ_ALWAYS_INLINE auto empty() const noexcept -> bool { return d_deque.empty(); } + SEQ_ALWAYS_INLINE auto size() const noexcept -> size_type { return d_deque.size(); } + SEQ_ALWAYS_INLINE auto max_size() const noexcept -> size_type { return d_deque.max_size(); } + SEQ_ALWAYS_INLINE void clear() noexcept { d_deque.clear(); } - auto container() const noexcept -> const container_type& { return d_deque; } + SEQ_ALWAYS_INLINE auto container() const noexcept -> const container_type& { return d_deque; } auto extract() noexcept -> container_type { @@ -655,23 +623,22 @@ namespace seq } } - iterator iterator_at(size_t pos) noexcept { return d_deque.iterator_at(pos); } - const_iterator iterator_at(size_t pos) const noexcept { return d_deque.iterator_at(pos); } + SEQ_ALWAYS_INLINE iterator iterator_at(size_t pos) noexcept { return d_deque.iterator_at(pos); } + SEQ_ALWAYS_INLINE const_iterator iterator_at(size_t pos) const noexcept { return d_deque.iterator_at(pos); } template - auto emplace_pos_multi(Policy, K&& key, Args&&... args) -> std::pair + SEQ_ALWAYS_INLINE auto emplace_pos_multi(Policy, K&& key, Args&&... args) -> std::pair { size_t pos = tvector_upper_bound(d_deque, key, base()); Policy::emplace(d_deque, pos, std::forward(key), std::forward(args)...); return std::pair(pos, true); } template - auto emplace_pos(Policy p, K&& key, Args&&... args) -> std::pair + SEQ_ALWAYS_INLINE auto emplace_pos(Policy p, K&& key, Args&&... args) -> std::pair { if constexpr (!Unique) return emplace_pos_multi(p, std::forward(key), std::forward(args)...); - - if constexpr (Comparable) { + else if constexpr (Comparable) { auto l = tvector_lower_bound(d_deque, key, base()); if (l.second) return { l.first, false }; @@ -679,27 +646,28 @@ namespace seq Policy::emplace(d_deque, l.first, std::forward(key), std::forward(args)...); return l; } - - std::pair res; - res.first = tvector_lower_bound(d_deque, key, base()).first; - res.second = !(res.first != d_deque.size() && !(*this)(key, d_deque[res.first])); - if (res.second) - Policy::emplace(d_deque, res.first, std::forward(key), std::forward(args)...); - return res; + else { + std::pair res; + res.first = tvector_lower_bound(d_deque, key, base()).first; + res.second = !(res.first != d_deque.size() && !(*this)(key, d_deque[res.first])); + if (res.second) + Policy::emplace(d_deque, res.first, std::forward(key), std::forward(args)...); + return res; + } } template - auto emplace(K&& key, Args&&... args) -> std::pair + SEQ_ALWAYS_INLINE auto emplace(K&& key, Args&&... args) -> std::pair { return emplace_pos(FlatEmplacePolicy{}, std::forward(key), std::forward(args)...); } template - auto try_emplace(K&& key, Args&&... args) -> std::pair + SEQ_ALWAYS_INLINE auto try_emplace(K&& key, Args&&... args) -> std::pair { return emplace_pos(TryFlatEmplacePolicy{}, std::forward(key), std::forward(args)...); } template - auto emplace_pos_hint(const_iterator hint, Policy p, K&& key, Args&&... args) -> std::pair + SEQ_ALWAYS_INLINE auto emplace_pos_hint(const_iterator hint, Policy p, K&& key, Args&&... args) -> std::pair { if constexpr (!Unique) { if (hint != end() && (*this)(key, *hint)) { @@ -711,24 +679,26 @@ namespace seq } return emplace_pos(p, std::forward(key), std::forward(args)...); } - if (hint != end() && (*this)(key, *hint)) { - // value is < hint, check that value before hint is smaller - auto h = hint; - if (hint == begin() || (*this)(*(--hint), key)) { - Policy::emplace(d_deque, static_cast(h.pos), std::forward(key), std::forward(args)...); - return std::pair(static_cast(h.pos), true); + else { + if (hint != end() && (*this)(key, *hint)) { + // value is < hint, check that value before hint is smaller + auto h = hint; + if (hint == begin() || (*this)(*(--hint), key)) { + Policy::emplace(d_deque, static_cast(h.pos), std::forward(key), std::forward(args)...); + return std::pair(static_cast(h.pos), true); + } } + return emplace_pos(p, std::forward(key), std::forward(args)...); } - return emplace_pos(p, std::forward(key), std::forward(args)...); } template - auto emplace_hint(const_iterator hint, K&& key, Args&&... args) -> std::pair + SEQ_ALWAYS_INLINE auto emplace_hint(const_iterator hint, K&& key, Args&&... args) -> std::pair { return emplace_pos_hint(hint, FlatEmplacePolicy{}, std::forward(key), std::forward(args)...); } template - auto try_emplace_hint(const_iterator hint, K&& key, Args&&... args) -> std::pair + SEQ_ALWAYS_INLINE auto try_emplace_hint(const_iterator hint, K&& key, Args&&... args) -> std::pair { return emplace_pos_hint(hint, TryFlatEmplacePolicy{}, std::forward(key), std::forward(args)...); } @@ -792,13 +762,13 @@ namespace seq } assign_cat(first, last, typename std::iterator_traits::iterator_category()); } - void erase_pos(size_type pos) { d_deque.erase(pos); } - auto erase(const_iterator pos) -> iterator { return d_deque.erase(pos); } - auto erase(iterator pos) -> iterator { return d_deque.erase(pos); } - auto erase(const_iterator first, const_iterator last) -> iterator { return d_deque.erase(first, last); } - void erase(size_t first, size_t last) { d_deque.erase(first, last); } + SEQ_ALWAYS_INLINE void erase_pos(size_type pos) { d_deque.erase(pos); } + SEQ_ALWAYS_INLINE auto erase(const_iterator pos) -> iterator { return d_deque.erase(pos); } + SEQ_ALWAYS_INLINE auto erase(iterator pos) -> iterator { return d_deque.erase(pos); } + SEQ_ALWAYS_INLINE auto erase(const_iterator first, const_iterator last) -> iterator { return d_deque.erase(first, last); } + SEQ_ALWAYS_INLINE void erase(size_t first, size_t last) { d_deque.erase(first, last); } template - auto erase(const K& key) -> size_type + SEQ_ALWAYS_INLINE auto erase(const K& key) -> size_type { if constexpr (Unique) { size_t pos = tvector_binary_search(d_deque, key, base()); @@ -875,10 +845,12 @@ namespace seq size_t pos = tvector_binary_search(d_deque, x, base()); return (pos == d_deque.size() ? 0 : 1); } - size_t low = tvector_lower_bound(d_deque, x, base()).first; - if (low == d_deque.size()) - return 0; - return tvector_upper_bound(d_deque, x, base()) - low; + else { + size_t low = tvector_lower_bound(d_deque, x, base()).first; + if (low == d_deque.size()) + return 0; + return tvector_upper_bound(d_deque, x, base()) - low; + } } template SEQ_INLINE_BINARY_SEARCH auto equal_range(const K& x) -> std::pair @@ -888,8 +860,10 @@ namespace seq return std::pair(low, low); if constexpr (Unique) return std::pair(low, low + 1); - iterator up = upper_bound(x); - return std::pair(low, up); + else { + iterator up = upper_bound(x); + return std::pair(low, up); + } } template SEQ_INLINE_BINARY_SEARCH auto equal_range(const K& x) const -> std::pair @@ -899,8 +873,10 @@ namespace seq return std::pair(low, low); if constexpr (Unique) return std::pair(low, low + 1); - const_iterator up = upper_bound(x); - return std::pair(low, up); + else { + const_iterator up = upper_bound(x); + return std::pair(low, up); + } } template SEQ_INLINE_BINARY_SEARCH auto equal_range_pos(const K& x) const -> std::pair @@ -910,14 +886,16 @@ namespace seq return std::pair(low, low); if constexpr (Unique) return std::pair(low, low + 1); - size_t up = upper_bound_pos(x); - return std::pair(low, up); + else { + size_t up = upper_bound_pos(x); + return std::pair(low, up); + } } template void merge(flat_tree& source) { - if (Unique) { + if constexpr (Unique) { container_type out; typename flat_tree::container_type rem; detail::unique_merge_move(out, begin(), end(), source.begin(), source.end(), base(), &rem); @@ -933,24 +911,24 @@ namespace seq } } - auto begin() noexcept -> iterator { return d_deque.begin(); } - auto begin() const noexcept -> const_iterator { return d_deque.cbegin(); } - auto cbegin() const noexcept -> const_iterator { return d_deque.cbegin(); } + SEQ_ALWAYS_INLINE auto begin() noexcept -> iterator { return d_deque.begin(); } + SEQ_ALWAYS_INLINE auto begin() const noexcept -> const_iterator { return d_deque.cbegin(); } + SEQ_ALWAYS_INLINE auto cbegin() const noexcept -> const_iterator { return d_deque.cbegin(); } - auto end() noexcept -> iterator { return d_deque.end(); } - auto end() const noexcept -> const_iterator { return d_deque.cend(); } - auto cend() const noexcept -> const_iterator { return d_deque.cend(); } + SEQ_ALWAYS_INLINE auto end() noexcept -> iterator { return d_deque.end(); } + SEQ_ALWAYS_INLINE auto end() const noexcept -> const_iterator { return d_deque.cend(); } + SEQ_ALWAYS_INLINE auto cend() const noexcept -> const_iterator { return d_deque.cend(); } - auto rbegin() noexcept -> reverse_iterator { return d_deque.rbegin(); } - auto rbegin() const noexcept -> const_reverse_iterator { return d_deque.crbegin(); } - auto crbegin() const noexcept -> const_reverse_iterator { return d_deque.crbegin(); } + SEQ_ALWAYS_INLINE auto rbegin() noexcept -> reverse_iterator { return d_deque.rbegin(); } + SEQ_ALWAYS_INLINE auto rbegin() const noexcept -> const_reverse_iterator { return d_deque.crbegin(); } + SEQ_ALWAYS_INLINE auto crbegin() const noexcept -> const_reverse_iterator { return d_deque.crbegin(); } - auto rend() noexcept -> reverse_iterator { return d_deque.rend(); } - auto rend() const noexcept -> const_reverse_iterator { return d_deque.crend(); } - auto crend() const noexcept -> const_reverse_iterator { return d_deque.crend(); } + SEQ_ALWAYS_INLINE auto rend() noexcept -> reverse_iterator { return d_deque.rend(); } + SEQ_ALWAYS_INLINE auto rend() const noexcept -> const_reverse_iterator { return d_deque.crend(); } + SEQ_ALWAYS_INLINE auto crend() const noexcept -> const_reverse_iterator { return d_deque.crend(); } - auto key_comp() const -> key_compare { return static_cast(base()); } - auto value_comp() const -> value_compare { return base(); } + SEQ_ALWAYS_INLINE auto key_comp() const -> key_compare { return static_cast(base()); } + SEQ_ALWAYS_INLINE auto value_comp() const -> value_compare { return base(); } }; } // end namespace detail @@ -1111,7 +1089,7 @@ namespace seq /// @param last the range to copy the elements from /// @param comp comparison function object to use for all comparisons of keys /// @param alloc allocator to use for all memory allocations of this container - template + template::value, int> = 0> flat_set(InputIt first, InputIt last, const Compare& comp = Compare(), const Allocator& alloc = Allocator()) : d_tree(first, last, comp, alloc) { @@ -1122,7 +1100,7 @@ namespace seq /// @param first the range to copy the elements from /// @param last the range to copy the elements from /// @param alloc allocator to use for all memory allocations of this container - template + template::value, int> = 0> flat_set(InputIt first, InputIt last, const Allocator& alloc) : flat_set(first, last, Compare(), alloc) { @@ -1200,8 +1178,8 @@ namespace seq return *this; } - auto extract() noexcept -> container_type { return d_tree.extract(); } - void replace(container_type&& c) noexcept { d_tree.replace(std::move(c)); } + SEQ_ALWAYS_INLINE auto extract() noexcept -> container_type { return d_tree.extract(); } + SEQ_ALWAYS_INLINE void replace(container_type&& c) noexcept { d_tree.replace(std::move(c)); } /// @brief Returns container's allocator SEQ_ALWAYS_INLINE auto get_allocator() const noexcept -> const Allocator& { return d_tree.get_allocator(); } @@ -1582,12 +1560,12 @@ namespace seq : base_type(alloc) { } - template + template::value, int> = 0> flat_multiset(InputIt first, InputIt last, const Compare& comp = Compare(), const Allocator& alloc = Allocator()) : base_type(first, last, comp, alloc) { } - template + template::value, int> = 0> flat_multiset(InputIt first, InputIt last, const Allocator& alloc) : flat_multiset(first, last, Compare(), alloc) { @@ -1722,12 +1700,12 @@ namespace seq : d_tree(alloc) { } - template + template::value, int> = 0> flat_map(InputIt first, InputIt last, const Compare& comp = Compare(), const Allocator& alloc = Allocator()) : d_tree(first, last, comp, alloc) { } - template + template::value, int> = 0> flat_map(InputIt first, InputIt last, const Allocator& alloc) : flat_map(first, last, Compare(), alloc) { @@ -1774,8 +1752,8 @@ namespace seq return *this; } - auto extract() noexcept -> container_type { return d_tree.extract(); } - void replace(container_type&& c) noexcept { d_tree.replace(std::move(c)); } + SEQ_ALWAYS_INLINE auto extract() noexcept -> container_type { return d_tree.extract(); } + SEQ_ALWAYS_INLINE void replace(container_type&& c) noexcept { d_tree.replace(std::move(c)); } SEQ_ALWAYS_INLINE auto get_allocator() const noexcept -> const Allocator& { return d_tree.get_allocator(); } SEQ_ALWAYS_INLINE auto get_allocator() noexcept -> Allocator& { return d_tree.get_allocator(); } @@ -2145,12 +2123,12 @@ namespace seq : base_type(alloc) { } - template + template::value, int> = 0> flat_multimap(InputIt first, InputIt last, const Compare& comp = Compare(), const Allocator& alloc = Allocator()) : base_type(first, last, comp, alloc) { } - template + template::value, int> = 0> flat_multimap(InputIt first, InputIt last, const Allocator& alloc) : flat_multimap(first, last, Compare(), alloc) { diff --git a/seq/hash.hpp b/seq/hash.hpp index 8374568e..88bfe44c 100644 --- a/seq/hash.hpp +++ b/seq/hash.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,28 +29,20 @@ /**\defgroup hash Hash: small collection of hash utilities -The hash module provides several hash-related functions: - - seq::hash_combine: combine 2 hash values - - seq::hash_finalize: mix hash value for better avalanching - - seq::hash_bytes_murmur64: murmurhash2 algorithm - - seq::hash_bytes_fnv1a: fnv1a hash algorithm working on chunks of 4 to 8 bytes - - seq::hash_bytes_fnv1a_slow: standard fnv1a hash algorithm - -Note that the specialization of std::hash for seq::tiny_string uses murmurhash2 algorithm. - */ /** \addtogroup hash * @{ */ -#include "utils.hpp" +#include "internal/utils.hpp" #include "internal/hash_impl.hpp" #include #include #include #include #include +#include namespace seq { @@ -118,7 +110,10 @@ namespace seq } /// @brief Mix input hash value for better avalanching - SEQ_ALWAYS_INLINE size_t hash_finalize(size_t h) noexcept { return detail::Mixin::mix(h); } + SEQ_ALWAYS_INLINE size_t hash_finalize(size_t h) noexcept + { + return detail::Mixin::mix(h); + } /// @brief Combine 2 hash values. Uses murmurhash2 mixin. /// @param seed in/out seed value @@ -148,7 +143,6 @@ namespace seq return detail::HashVal::hash(h, v); } - inline auto hash_bytes_murmur64(const void* ptr, size_t size) noexcept { return detail::hash_bytes_murmur64_impl(ptr, size); @@ -162,7 +156,6 @@ namespace seq return detail::hash_bytes_komihash_impl(ptr, size); } - template struct hasher : public std::hash { @@ -173,8 +166,13 @@ namespace seq template<> \ struct hasher \ { \ - using is_avalanching = int; \ - SEQ_ALWAYS_INLINE size_t operator()(T v) const noexcept { return hash_finalize(static_cast(v)); } \ + using is_avalanching = void; \ + using is_transparent = void; \ + template \ + SEQ_ALWAYS_INLINE size_t operator()(const U& v) const noexcept \ + { \ + return hash_finalize(static_cast(static_cast(v))); \ + } \ } SEQ_INTEGRAL_HASH_FUNCTION(bool); @@ -199,15 +197,17 @@ namespace seq template<> struct hasher { - using is_avalanching = int; - SEQ_ALWAYS_INLINE size_t operator()(float v) const noexcept + using is_avalanching = void; + using is_transparent = void; + template + SEQ_ALWAYS_INLINE size_t operator()(const U& v) const noexcept { union { float fv; std::uint32_t uv; }; - fv = v; + fv = (float)v; return hash_finalize(uv); } }; @@ -215,31 +215,42 @@ namespace seq template<> struct hasher { - using is_avalanching = int; - SEQ_ALWAYS_INLINE size_t operator()(double v) const noexcept + using is_avalanching = void; + using is_transparent = void; + template + SEQ_ALWAYS_INLINE size_t operator()(const U& v) const noexcept { union { double fv; std::uint64_t uv; }; - fv = v; + fv = (double)v; return hash_finalize(static_cast(uv)); } }; + template<> + struct hasher : hasher + { + }; template struct hasher { - using is_avalanching = int; - SEQ_ALWAYS_INLINE size_t operator()(T* ptr) const noexcept { return hash_finalize(reinterpret_cast(ptr)); } + using is_avalanching = void; + using is_transparent = void; + template + SEQ_ALWAYS_INLINE size_t operator()(U* ptr) const noexcept + { + return hash_finalize(reinterpret_cast(ptr)); + } }; template struct hasher> { - using is_avalanching = int; - using is_transparent = int; + using is_avalanching = void; + using is_transparent = void; SEQ_ALWAYS_INLINE size_t operator()(const std::unique_ptr& ptr) const noexcept { return hash_finalize(reinterpret_cast(ptr.get())); } SEQ_ALWAYS_INLINE size_t operator()(const T* ptr) const noexcept { return hash_finalize(reinterpret_cast(ptr)); } }; @@ -247,8 +258,8 @@ namespace seq template struct hasher> { - using is_avalanching = int; - using is_transparent = int; + using is_avalanching = void; + using is_transparent = void; SEQ_ALWAYS_INLINE size_t operator()(const std::shared_ptr& ptr) const noexcept { return hash_finalize(reinterpret_cast(ptr.get())); } SEQ_ALWAYS_INLINE size_t operator()(const T* ptr) const noexcept { return hash_finalize(reinterpret_cast(ptr)); } }; @@ -256,8 +267,8 @@ namespace seq template struct hasher, void>::type> { - using is_avalanching = int; - using is_transparent = int; + using is_avalanching = void; + using is_transparent = void; SEQ_ALWAYS_INLINE size_t operator()(Enum e) const noexcept { using Underlying = typename std::underlying_type::type; @@ -273,8 +284,10 @@ namespace seq template struct hasher> { - using is_avalanching = int; - SEQ_ALWAYS_INLINE size_t operator()(const std::pair& p) const noexcept + using is_avalanching = void; + using is_transparent = void; + template + SEQ_ALWAYS_INLINE size_t operator()(const std::pair& p) const noexcept { size_t s = hasher{}(p.first); hash_combine(s, hasher{}(p.second)); @@ -287,7 +300,8 @@ namespace seq template::value - 1> struct HashTuple { - static SEQ_ALWAYS_INLINE void apply(size_t& seed, Tuple const& tuple) noexcept + template + static SEQ_ALWAYS_INLINE void apply(size_t& seed, OtherTuple const& tuple) noexcept { using elem_type = typename std::tuple_element::type; HashTuple::apply(seed, tuple); @@ -298,7 +312,8 @@ namespace seq template struct HashTuple { - static SEQ_ALWAYS_INLINE void apply(size_t& seed, Tuple const& tuple) noexcept + template + static SEQ_ALWAYS_INLINE void apply(size_t& seed, OtherTuple const& tuple) noexcept { using elem_type = typename std::tuple_element<0, Tuple>::type; hash_combine(seed, hasher{}(std::get<0>(tuple))); @@ -309,14 +324,47 @@ namespace seq template struct hasher> { - using is_avalanching = int; - SEQ_ALWAYS_INLINE size_t operator()(const std::tuple& t) const noexcept + using is_avalanching = void; + using is_transparent = void; + template + SEQ_ALWAYS_INLINE size_t operator()(const OtherTuple& t) const noexcept { size_t seed = 0; detail::HashTuple>::apply(seed, t); return seed; } }; + + template + struct hasher> + { + using is_avalanching = void; + using is_transparent = void; + using type = std::chrono::duration; + + size_t operator()(const type& s) const { return hasher{}(s.count()); } + template, void>> + size_t operator()(T s) const + { + return hasher{}(static_cast(s)); + } + }; + + template + struct hasher> + { + using is_avalanching = void; + using is_transparent = void; + using type = std::chrono::time_point; + using integral = decltype(type{}.time_since_epoch().count()); + + size_t operator()(const type& s) const {return hasher{}(s.time_since_epoch().count()); } + template, void>> + size_t operator()(T s) const + { + return hasher{}(static_cast(s)); + } + }; } /** @}*/ diff --git a/seq/internal/binary_search.hpp b/seq/internal/binary_search.hpp index 4668ead3..6fb0cd1d 100644 --- a/seq/internal/binary_search.hpp +++ b/seq/internal/binary_search.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -27,6 +27,7 @@ #define SEQ_BINARY_SEARCH_HPP #include "../type_traits.hpp" +#include "../bits.hpp" #include namespace seq diff --git a/seq/internal/concurrent_hash_table.hpp b/seq/internal/concurrent_hash_table.hpp index 569d6d54..247bc5c4 100644 --- a/seq/internal/concurrent_hash_table.hpp +++ b/seq/internal/concurrent_hash_table.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -27,8 +27,8 @@ #include "simd.hpp" #include "hash_utils.hpp" +#include "utils.hpp" #include "../lock.hpp" -#include "../utils.hpp" #include "../hash.hpp" #include "../bits.hpp" @@ -73,7 +73,7 @@ namespace seq enum { max_concurrent_node_size = 16, - chain_concurrent_node_size = 16 + chain_concurrent_node_size = 4 }; #else enum @@ -288,12 +288,12 @@ namespace seq /// @brief Apply functor and return boolean value (true if void() functor, functor result otherwise) template - static SEQ_CONCURRENT_INLINE bool ApplyFunctor(F&& f, Args&&... args) noexcept(noexcept(std::forward(f)(std::forward(args)...))) + static SEQ_CONCURRENT_INLINE bool ApplyFunctor(F& f, Args&&... args) noexcept(noexcept(f(std::forward(args)...))) { - if constexpr (std::is_same_v()(std::forward(args)...))>) - return std::forward(f)(std::forward(args)...); + if constexpr (std::is_convertible_v < std::invoke_result_t, bool>) + return f(std::forward(args)...); else { - std::forward(f)(std::forward(args)...); + f(std::forward(args)...); return true; } } @@ -318,78 +318,38 @@ namespace seq std::uint64_t tmp = (word & 0x7F7F7F7F7F7F7F7FULL) + 0x7F7F7F7F7F7F7F7FULL; return ~(tmp | word | 0x7F7F7F7F7F7F7F7FULL); } + /// @brief Movemask function of 4 bytes word + static SEQ_CONCURRENT_INLINE auto MoveMask4(std::uint32_t word) noexcept -> std::uint32_t + { + std::uint32_t tmp = (word & 0x7F7F7F7FU) + 0x7F7F7F7FUL; + return ~(tmp | word | 0x7F7F7F7FU); + } /// @brief Returns slot index with a tiny hash value of 0 template static SEQ_CONCURRENT_INLINE auto FindIndexZero(const std::uint8_t* hashs) noexcept -> unsigned { - if constexpr (Size == 8) { - std::uint64_t _th = 0; - std::uint64_t found = MoveMask8((*reinterpret_cast(hashs)) ^ _th) >> 8u; + if constexpr (Size == 4) { + std::uint32_t found = MoveMask4((*reinterpret_cast(hashs)) ^ 0u) >> 8u; + if (found) + return bit_scan_forward_32(found) >> 3u; + return static_cast(-1); + } + else if constexpr (Size == 8) { + std::uint64_t found = MoveMask8((*reinterpret_cast(hashs)) ^ 0ull) >> 8u; if (found) return bit_scan_forward_64(found) >> 3u; return static_cast(-1); } - if constexpr (Size == 16) { + else if constexpr (Size == 16) { #if defined(__SSE2__) - auto hs = _mm_loadu_si128(reinterpret_cast(hashs)); - int mask = _mm_movemask_epi8(_mm_cmpeq_epi8(hs, _mm_set1_epi8(0))) >> 1; + auto hs = _mm_load_si128(reinterpret_cast(hashs)); + int mask = _mm_movemask_epi8(_mm_cmpeq_epi8(hs, _mm_setzero_si128())) >> 1; if (mask) return bit_scan_forward_32(static_cast(mask)); #endif return static_cast(-1); } - - SEQ_UNREACHABLE(); - } - - /// @brief Find given value based on its small hash representation and hashs/values arrays - template - static SEQ_CONCURRENT_INLINE auto FindWithTh(std::uint8_t th, const Equal& eq, K&& key, const std::uint8_t* hashs, const Value* values) noexcept(noexcept(eq(ExtractKey::key(*values), - std::forward(key)))) - -> const Value* - { - if constexpr (Size == 8) { - if (!hashs[0]) - return nullptr; - // no SSE variant (way slower) - std::uint64_t _th; - memset(&_th, th, sizeof(_th)); - - // do first 7 values - std::uint64_t found = MoveMask8((*reinterpret_cast(hashs)) ^ _th) >> 8u; - if (found) { - SEQ_PREFETCH(values); - do { - unsigned pos = bit_scan_forward_64(found) >> 3u; - if (eq(ExtractKey::key(values[pos]), std::forward(key))) - return values + pos; - reinterpret_cast(&found)[pos] = 0; - } while (found); - } - return nullptr; - } - - if constexpr (Size == 16) { -#if defined(__SSE2__) - // SSE movemask - if (!hashs[0]) - return nullptr; - auto hs = _mm_loadu_si128(reinterpret_cast(hashs)); - int mask = _mm_movemask_epi8(_mm_cmpeq_epi8(hs, _mm_set1_epi8(static_cast(th)))) >> 1; - if (mask) { - SEQ_PREFETCH(values); - do { - unsigned pos = bit_scan_forward_32(static_cast(mask)); - if (eq(ExtractKey::key(values[pos]), std::forward(key))) - return values + pos; - mask &= mask - 1; - } while (mask); - } -#endif - return nullptr; - } - SEQ_UNREACHABLE(); } @@ -431,7 +391,7 @@ namespace seq }; /// @brief Hash node of max_concurrent_node_size tiny hashes - struct ConcurrentHashNode + struct alignas(16) ConcurrentHashNode { static constexpr unsigned size = max_concurrent_node_size; static constexpr unsigned shift = (size == 32 ? 5 : (size == 16 ? 4 : 3)); @@ -462,16 +422,16 @@ namespace seq } } template - bool for_each_until(const ConcurrentValueNode* n, F&& f) const noexcept(noexcept(std::forward(f)(std::declval(), 0, std::declval()))) + bool for_each_until(const ConcurrentValueNode* n, F f) const noexcept(noexcept(std::forward(f)(std::declval(), 0, std::declval()))) { for (unsigned i = 0; i < count(); ++i) - if (!ApplyFunctor(std::forward(f), hashs, i, n->values()[i])) + if (!ApplyFunctor(f, hashs, i, n->values()[i])) return false; if (full() && n->right) { const ConcurrentDenseNode* d = n->right; do { for (unsigned i = 0; i < d->count(); ++i) - if (!ApplyFunctor(std::forward(f), d->hashs, i, d->values()[i])) + if (!ApplyFunctor(f, d->hashs, i, d->values()[i])) return false; } while ((d = d->right)); } @@ -487,132 +447,10 @@ namespace seq bool for_each_until(ConcurrentValueNode* n, F&& f) noexcept(noexcept(std::forward(f)(std::declval(), 0, std::declval()))) { return static_cast(this)->for_each_until( - n, [&f](const std::uint8_t* hs, unsigned i, const T& v) { std::forward(f)(const_cast(hs), i, const_cast(v)); }); + n, [&f](const std::uint8_t* hs, unsigned i, const T& v) { return std::forward(f)(const_cast(hs), i, const_cast(v)); }); } }; - /// @brief Find given value in a dense node. - /// Performs lookup on the full chain. - template - auto FindInDense(std::uint8_t th, const Equal& eq, K&& key, const ConcurrentDenseNode* n, F&& f) noexcept(noexcept(eq(std::forward(key), - ExtractKey::key(std::declval())))) -> size_t - { - do { - auto v = FindWithTh(th, eq, std::forward(key), n->hashs, n->values()); - if (v) { - std::forward(f)(const_cast(*v)); - return 1; - } - } while ((n = n->right)); - return 0; - } - - /// @brief Find given value in main node. - /// Performs lookup on the full chain. - template - SEQ_CONCURRENT_INLINE auto FindInNode(std::uint8_t th, const Equal& eq, K&& key, const ConcurrentHashNode* node, const ConcurrentValueNode* values, F&& f) noexcept( - noexcept(eq(std::forward(key), ExtractKey::key(std::declval())))) -> size_t - { - if (auto v = FindWithTh(th, eq, std::forward(key), node->hashs, values->values())) { - std::forward(f)(const_cast(*v)); - return 1; - } - if (node->full() && values->right) - return FindInDense(th, eq, std::forward(key), values->right, std::forward(f)); - return 0; - } - - /// @brief Returns index of the next null hash value - /// Only used when an exception is thrown during rehash - template - auto FindFreeSlotInNode(ConcurrentHashNode* node, ConcurrentValueNode* values) noexcept -> std::pair - { - // Look for a free slot in the node chain - unsigned idx = FindIndexZero(node->hashs); - if (idx != static_cast(-1)) - return { values->values() + idx, node->hashs + idx + 1 }; - auto* d = values->right; - while (d) { - idx = FindIndexZero(d->hashs); - if (idx != static_cast(-1)) - return { d->values() + idx, d->hashs + idx + 1 }; - d = d->right; - } - return { nullptr, nullptr }; - } - - /// @brief Insert value in a new dense node - template - auto InsertNewDense(ChainCount& counter, Allocator al, std::uint8_t th, Node* n, K&& key, Args&&... args) -> std::pair - { - using value_type = typename Node::value_type; - //using Alloc = typename std::allocator_traits::template rebind_alloc>; - //Alloc a = al; - //ConcurrentDenseNode* d = a.allocate(1); - ConcurrentDenseNode* d = al.allocate(1); - - memset(d->hashs,0,sizeof(d->hashs)); - d->right = nullptr; - d->left = reinterpret_cast*>(n); - - try { - Policy::emplace(d->values(), std::forward(key), std::forward(args)...); - } - catch (...) { - // destroy dense node - al.deallocate(d, 1); - throw; - } - d->hashs[++d->hashs[0]] = th; - n->right = d; - ++counter; - return { d->values(), true }; - } - - /// @brief Insert value in dense node if it does not already exist - template - auto FindInsertDense(ChainCount& counter, const Allocator& al, std::uint8_t th, const Equal& eq, ConcurrentDenseNode* n, K&& key, Args&&... args) -> std::pair - { - auto valid = n; - do { - if constexpr (CheckExists) { - auto v = FindWithTh(th, eq, ExtractKey::key(std::forward(key)), n->hashs, n->values()); - if (v) - return { const_cast(v), false }; - } - valid = n; - } while ((n = n->right)); - if (valid->full()) - return InsertNewDense(counter, al, th, valid, std::forward(key), std::forward(args)...); - - // might throw, fine - auto p = Policy::emplace(valid->values() + valid->count(), std::forward(key), std::forward(args)...); - valid->hashs[++valid->hashs[0]] = th; - return { p, true }; - } - - /// @brief Insert value if it does not already exist - template - SEQ_CONCURRENT_INLINE auto - FindInsertNode(ChainCount& counter, const Allocator& al, std::uint8_t th, const Equal& eq, ConcurrentHashNode* node, ConcurrentValueNode* values, K&& key, Args&&... args) - -> std::pair - { - if constexpr (CheckExists) { - auto v = FindWithTh(th, eq, ExtractKey::key(std::forward(key)), node->hashs, values->values()); - if (v) - return { const_cast(v), false }; - } - if (node->full()) { - if (values->right) - return FindInsertDense(counter, al, th, eq, values->right, std::forward(key), std::forward(args)...); - return InsertNewDense(counter, al, th, values, std::forward(key), std::forward(args)...); - } - // might throw, fine - auto p = Policy::emplace(values->values() + node->count(), std::forward(key), std::forward(args)...); - node->hashs[++node->hashs[0]] = th; - return { p, true }; - } - /// @brief Standard insert policy struct InsertConcurrentPolicy { @@ -685,6 +523,190 @@ namespace seq std::condition_variable d_rehash_condition; chain_count_type d_chain_count; // number of chained nodes, used to optimize detection of needed rehash on insert + /// @brief Find given value based on its small hash representation and hashs/values arrays + template + static SEQ_CONCURRENT_INLINE auto FindWithTh(std::uint8_t th, const Equal& eq, K&& key, const std::uint8_t* hashs, const Value* values) noexcept( + noexcept(eq(extract_key::key(*values), std::forward(key)))) -> const Value* + { + if constexpr (Size == 4) { + if (!hashs[0]) + return nullptr; + // no SSE variant (way slower) + std::uint32_t _th; + memset(&_th, th, sizeof(_th)); + + // do first 3 values + std::uint32_t found = MoveMask4((*reinterpret_cast(hashs)) ^ _th) >> 8u; + if (found) { + SEQ_PREFETCH(values); + do { + unsigned pos = bit_scan_forward_32(found) >> 3u; + if (eq(extract_key::key(values[pos]), std::forward(key))) + return values + pos; + reinterpret_cast(&found)[pos] = 0; + } while (found); + } + return nullptr; + } + else if constexpr (Size == 8) { + if (!hashs[0]) + return nullptr; + // no SSE variant (way slower) + std::uint64_t _th; + memset(&_th, th, sizeof(_th)); + + // do first 7 values + std::uint64_t found = MoveMask8((*reinterpret_cast(hashs)) ^ _th) >> 8u; + if (found) { + SEQ_PREFETCH(values); + do { + unsigned pos = bit_scan_forward_64(found) >> 3u; + if (eq(extract_key::key(values[pos]), std::forward(key))) + return values + pos; + reinterpret_cast(&found)[pos] = 0; + } while (found); + } + return nullptr; + } + else if constexpr (Size == 16) { +#if defined(__SSE2__) + // SSE movemask + if (!hashs[0]) + return nullptr; + auto hs = _mm_load_si128(reinterpret_cast(hashs)); + auto mask = (unsigned)_mm_movemask_epi8(_mm_cmpeq_epi8(hs, _mm_set1_epi8(static_cast(th)))) >> 1; + if (mask) { + SEQ_PREFETCH(values); + do { + unsigned pos = bit_scan_forward_32(mask); + if (eq(extract_key::key(values[pos]), std::forward(key))) + return values + pos; + mask &= mask - 1u; + } while (mask); + } +#endif + return nullptr; + } + SEQ_UNREACHABLE(); + } + + /// @brief Find given value in a dense node. + /// Performs lookup on the full chain. + template + auto FindInDense(std::uint8_t th, const Equal& eq, K&& key, const ConcurrentDenseNode* n, F&& f) const + noexcept(noexcept(eq(std::forward(key), extract_key::key(std::declval())))) -> size_t + { + do { + auto v = FindWithTh(th, eq, std::forward(key), n->hashs, n->values()); + if (v) { + std::forward(f)(const_cast(*v)); + return 1; + } + } while ((n = n->right)); + return 0; + } + + /// @brief Find given value in main node. + /// Performs lookup on the full chain. + template + SEQ_CONCURRENT_INLINE auto FindInNode(std::uint8_t th, const Equal& eq, K&& key, const ConcurrentHashNode* node, const ConcurrentValueNode* values, F&& f) const + noexcept(noexcept(eq(std::forward(key), extract_key::key(std::declval())))) -> size_t + { + if (auto v = FindWithTh(th, eq, std::forward(key), node->hashs, values->values())) { + std::forward(f)(const_cast(*v)); + return 1; + } + if (node->full() && values->right) + return FindInDense(th, eq, std::forward(key), values->right, std::forward(f)); + return 0; + } + + /// @brief Returns index of the next null hash value + /// Only used when an exception is thrown during rehash + auto FindFreeSlotInNode(ConcurrentHashNode* node, ConcurrentValueNode* values) noexcept -> std::pair + { + // Look for a free slot in the node chain + unsigned idx = FindIndexZero(node->hashs); + if (idx != static_cast(-1)) + return { values->values() + idx, node->hashs + idx + 1 }; + auto* d = values->right; + while (d) { + idx = FindIndexZero(d->hashs); + if (idx != static_cast(-1)) + return { d->values() + idx, d->hashs + idx + 1 }; + d = d->right; + } + return { nullptr, nullptr }; + } + + /// @brief Insert value in a new dense node + template + auto InsertNewDense(std::uint8_t th, Node* n, K&& key, Args&&... args) -> std::pair + { + chain_node_allocator al{ get_allocator() }; + chain_node_type* d = al.allocate(1); + + memset(d->hashs, 0, sizeof(d->hashs)); + d->right = nullptr; + d->left = reinterpret_cast(n); + + try { + Policy::emplace(d->values(), std::forward(key), std::forward(args)...); + } + catch (...) { + // destroy dense node + al.deallocate(d, 1); + throw; + } + d->hashs[++d->hashs[0]] = th; + n->right = d; + ++d_chain_count; + return { d->values(), true }; + } + + /// @brief Insert value in dense node if it does not already exist + template + auto FindInsertDense(std::uint8_t th, const Equal& eq, ConcurrentDenseNode* n, K&& key, Args&&... args) -> std::pair + { + auto valid = n; + do { + if constexpr (CheckExists) { + auto v = FindWithTh(th, eq, extract_key::key(std::forward(key)), n->hashs, n->values()); + if (v) + return { const_cast(v), false }; + } + valid = n; + } while ((n = n->right)); + if SEQ_UNLIKELY (valid->full()) + return InsertNewDense(th, valid, std::forward(key), std::forward(args)...); + + // might throw, fine + auto p = Policy::emplace(valid->values() + valid->count(), std::forward(key), std::forward(args)...); + valid->hashs[++valid->hashs[0]] = th; + return { p, true }; + } + + /// @brief Insert value if it does not already exist + template + SEQ_CONCURRENT_INLINE auto FindInsertNode(std::uint8_t th, const Equal& eq, ConcurrentHashNode* node, ConcurrentValueNode* values, K&& key, Args&&... args) + -> std::pair + { + if constexpr (CheckExists) { + auto v = FindWithTh(th, eq, extract_key::key(std::forward(key)), node->hashs, values->values()); + if (v) + return { const_cast(v), false }; + } + if SEQ_UNLIKELY (node->full()) { + if (values->right) + return FindInsertDense(th, eq, values->right, std::forward(key), std::forward(args)...); + return InsertNewDense(th, values, std::forward(key), std::forward(args)...); + } + // might throw, fine + auto p = Policy::emplace(values->values() + node->count(), std::forward(key), std::forward(args)...); + node->hashs[++node->hashs[0]] = th; + return { p, true }; + } + SEQ_ALWAYS_INLINE auto get_allocator() const noexcept { return d_data->get_allocator(); } SEQ_ALWAYS_INLINE auto key_eq() const noexcept { return d_data->key_eq(); } @@ -703,10 +725,8 @@ namespace seq auto make_value_nodes(size_t count = 1) -> value_node_type* { value_node_type* n = value_node_allocator{ get_allocator() }.allocate(count); - //memset(n, 0, count * sizeof(value_node_type)); - for(size_t i=0; i < count; ++i) - n[i].right = nullptr; //TEST - + for (size_t i = 0; i < count; ++i) + n[i].right = nullptr; return n; } @@ -732,24 +752,21 @@ namespace seq SEQ_ALWAYS_INLINE void lock(node_lock& lock) { - if (lock.try_lock()) - return; while (!lock.try_lock()) { if (d_in_rehash.load(std::memory_order_relaxed)) { std::unique_lock ll(d_rehash_mutex); d_rehash_condition.wait_for(ll, std::chrono::milliseconds(1), [&lock]() { return !lock.is_locked(); }); } - else + else { std::this_thread::yield(); + } } } SEQ_ALWAYS_INLINE void lock_shared(node_lock& lock) const { - if (lock.try_lock()) - return; - this_type* _this = const_cast(this); while (!lock.try_lock_shared()) { if (d_in_rehash.load(std::memory_order_relaxed)) { + this_type* _this = const_cast(this); std::unique_lock ll(_this->d_rehash_mutex); _this->d_rehash_condition.wait_for(ll, std::chrono::milliseconds(1), [&lock]() { return !lock.is_locked(); }); } @@ -803,11 +820,10 @@ namespace seq d_buckets[i].for_each(d_values + i, [&](std::uint8_t* hashs, unsigned j, Value& val) { auto pos = hash_key(extract_key::key(val)) & new_hash_mask; - FindInsertNode( - d_chain_count, chain_node_allocator{ get_allocator() }, hashs[j + 1], key_eq(), buckets + pos, values + pos, std::move_if_noexcept(val)); + FindInsertNode(hashs[j + 1], key_eq(), buckets + pos, values + pos, std::move_if_noexcept(val)); - if (std::is_nothrow_move_constructible_v) { - if (!std::is_trivially_destructible_v) + if constexpr (std::is_nothrow_move_constructible_v) { + if constexpr (!std::is_trivially_destructible_v) val.~Value(); // mark position as destroyed hashs[j + 1] = 0; @@ -906,7 +922,7 @@ namespace seq // Rehash for given size if (size == 0) return rehash_internal(0, false); - size_t new_hash_mask = size - 1ULL; + size_t new_hash_mask = (size ) - 1ULL; if ((size & (size - 1ULL)) != 0ULL) // non power of 2 new_hash_mask = (1ULL << (1ULL + (bit_scan_reverse_64(size)))) - 1ULL; new_hash_mask >>= node_type::shift; @@ -924,9 +940,10 @@ namespace seq { // Rehash on insert size_t s = AtomicLoad(d_size); - if SEQ_UNLIKELY ((s >= d_next_target) && (d_buckets == get_static_node() || AtomicLoad(d_chain_count) > ((d_hash_mask + 1) >> 5))) + if SEQ_UNLIKELY ((s >= d_next_target) && (d_buckets == get_static_node() || AtomicLoad(d_chain_count) > ((d_hash_mask + 1) >> 4))) { // delay rehash as long as there are few chains rehash_on_next_target(s); + } } SEQ_NOINLINE(auto) update_lock(lock_array* locks, size_t hash, size_t& hash_mask, size_t& pos, node_lock*& l) @@ -1018,11 +1035,10 @@ namespace seq pos = this->template get_node(AtomicLoad(d_locks), hash, ll); else pos = (hash & d_hash_mask); - // Grad lock + // Grab lock LockUnique lock(ll, true); - auto p = FindInsertNode( - d_chain_count, chain_node_allocator{ get_allocator() } ,th, key_eq(), d_buckets + pos, d_values + pos, std::forward(key), std::forward(args)...); + auto p = FindInsertNode(th, key_eq(), d_buckets + pos, d_values + pos, std::forward(key), std::forward(args)...); if (!p.second) { // Key exist: call functor std::forward(fun)(*p.first); @@ -1113,6 +1129,28 @@ namespace seq } } + /// @brief Erase key if found AND is fun(value) returns true. + /// Returns number of erased entries (1 or 0). + template + auto erase_key_dense(node_type* bucket, value_node_type* values, uint8_t th, F&& fun, const K& key) -> size_t + { + auto* d = values->right; + do { + // Look in dense node + auto found = FindWithTh(th, d_data->key_eq(), key, d->hashs, d->values()); + if (found) { + // Erase from dense node if functor returns true + if (!std::forward(fun)(const_cast(*found))) + return 0; + + erase_from_dense(bucket, values, d, static_cast(found - d->values())); + --d_size; + return 1; + } + } while ((d = d->right)); + return 0; + } + bool contains_value(const Value& key_value) const { // Returns true if given value (key AND value) exists in this table @@ -1217,7 +1255,7 @@ namespace seq pos = (hash & d_hash_mask); LockShared ll(lock, true); - return FindInNode(node_type::tiny_hash(hash), d_data->key_eq(), key, d_buckets + pos, d_values + pos, std::forward(f)); + return FindInNode(node_type::tiny_hash(hash), d_data->key_eq(), key, d_buckets + pos, d_values + pos, std::forward(f)); } template SEQ_CONCURRENT_INLINE size_t visit_hash(size_t hash, const K& key, F&& f) @@ -1230,7 +1268,7 @@ namespace seq else pos = (hash & d_hash_mask); LockUnique ll(lock, true); - return FindInNode(node_type::tiny_hash(hash), d_data->key_eq(), key, d_buckets + pos, d_values + pos, std::forward(f)); + return FindInNode(node_type::tiny_hash(hash), d_data->key_eq(), key, d_buckets + pos, d_values + pos, std::forward(f)); } /// @brief Visit all entries and call functor for each of them. @@ -1262,7 +1300,7 @@ namespace seq return true; } template - bool visit_all(F&& fun) + bool visit_all(F fun) { // Avoid rehash while calling visit_all LockShared lock(const_cast(d_rehash_lock)); @@ -1279,7 +1317,7 @@ namespace seq node_type* n = d_buckets + i; value_node_type* v = d_values + i; - if (!n->for_each_until(v, [&](auto, auto, Value& val) { return std::forward(fun)(val); })) + if (!n->for_each_until(v, [&](auto, auto, Value& val) { return fun(val); })) return false; if (locks) ++iter; @@ -1287,6 +1325,16 @@ namespace seq return true; } + SEQ_CONCURRENT_INLINE bool need_rehash_on_insert() + { + // Rehash on insert + size_t s = AtomicLoad(d_size); + if SEQ_UNLIKELY ((s >= d_next_target) && (d_buckets == get_static_node() || AtomicLoad(d_chain_count) > ((d_hash_mask + 1) >> 5))) + // delay rehash as long as there are few chains + return true; + return false; + } + /// @brief Insert entry based on provided policy template SEQ_CONCURRENT_INLINE auto emplace_policy(size_t hash, K&& key, Args&&... args) -> bool @@ -1308,28 +1356,6 @@ namespace seq return insert_policy(hash, std::forward(fun), std::forward(key), std::forward(args)...); } - /// @brief Erase key if found AND is fun(value) returns true. - /// Returns number of erased entries (1 or 0). - template - auto erase_key_dense(node_type* bucket, value_node_type* values, uint8_t th, F&& fun, const K& key) -> size_t - { - auto* d = values->right; - do { - // Look in dense node - auto found = FindWithTh(th, d_data->key_eq(), key, d->hashs, d->values()); - if (found) { - // Erase from dense node if functor returns true - if (!std::forward(fun)(const_cast(*found))) - return 0; - - erase_from_dense(bucket, values, d, static_cast(found - d->values())); - --d_size; - return 1; - } - } while ((d = d->right)); - return 0; - } - /// @brief Erase key if found AND is fun(value) returns true. /// Returns number of erased entries (1 or 0). template @@ -1352,7 +1378,7 @@ namespace seq auto values = d_values + pos; auto bucket = d_buckets + pos; // Find in main bucket - auto found = FindWithTh(th, d_data->key_eq(), key, bucket->hashs, values->values()); + auto found = FindWithTh(th, d_data->key_eq(), key, bucket->hashs, values->values()); if (found) { // Erase from main bucket if functor returns true if (!std::forward(fun)(const_cast(*found))) @@ -1888,6 +1914,16 @@ namespace seq return this->emplace_policy([](const auto&) {}, std::forward(key), std::forward(args)...); } + /* SEQ_CONCURRENT_INLINE bool need_rehash_on_insert() const noexcept + { + + }*/ + template + SEQ_CONCURRENT_INLINE auto emplace_policy_hash_no_exist_no_rehash(size_t hash, K&& key, Args&&... args) -> bool + { + return get_data()->at(index_from_hash(hash)).template insert_policy(hash, [](const auto&) {}, std::forward(key), std::forward(args)...); + } + template SEQ_CONCURRENT_INLINE auto emplace_policy(F&& fun, K&& key, Args&&... args) -> bool { @@ -1896,6 +1932,12 @@ namespace seq return d->at(index_from_hash(hash)).template emplace_policy_visit(hash, std::forward(fun), std::forward(key), std::forward(args)...); } + template + SEQ_CONCURRENT_INLINE auto emplace_policy_hash(F&& fun, size_t hash, K&& key, Args&&... args) -> bool + { + return get_data()->at(index_from_hash(hash)).template emplace_policy_visit(hash, std::forward(fun), std::forward(key), std::forward(args)...); + } + template void insert(Iter first, Iter last) { @@ -1915,6 +1957,14 @@ namespace seq return d->at(index_from_hash(hash)).visit_hash(hash, key, std::forward(fun)); } template + SEQ_CONCURRENT_INLINE auto visit_hash(size_t hash, const K& key, F&& fun) const -> size_t + { + const PrivateData* d = get_data(); + if SEQ_UNLIKELY (!d) + return 0; + return d->at(index_from_hash(hash)).visit_hash(hash, key, std::forward(fun)); + } + template SEQ_CONCURRENT_INLINE auto visit(const K& key, F&& fun) -> size_t { PrivateData* d = const_cast(cget_data()); @@ -1923,16 +1973,34 @@ namespace seq size_t hash = d->hash_key((key)); return d->at(index_from_hash(hash)).visit_hash(hash, key, std::forward(fun)); } + template + SEQ_CONCURRENT_INLINE auto visit_hash(size_t hash, const K& key, F&& fun) -> size_t + { + PrivateData* d = const_cast(cget_data()); + if SEQ_UNLIKELY (!d) + return 0; + return d->at(index_from_hash(hash)).visit_hash(hash, key, std::forward(fun)); + } template SEQ_CONCURRENT_INLINE bool contains(const K& key) const { return visit(key, [](const auto&) { return false; }); } template + SEQ_CONCURRENT_INLINE bool contains_hash(size_t hash, const K& key) const + { + return visit_hash(hash, key, [](const auto&) { return false; }); + } + template SEQ_CONCURRENT_INLINE auto count(const K& key) const -> size_t { return contains(key); } + template + SEQ_CONCURRENT_INLINE auto count_hash(size_t hash, const K& key) const -> size_t + { + return contains_hash(hash, key); + } template SEQ_CONCURRENT_INLINE auto erase(const K& key, F&& fun) -> size_t { diff --git a/seq/internal/hash_impl.hpp b/seq/internal/hash_impl.hpp index 4f526dbb..645df8e5 100644 --- a/seq/internal/hash_impl.hpp +++ b/seq/internal/hash_impl.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/seq/internal/hash_utils.hpp b/seq/internal/hash_utils.hpp index 73553547..7d0d2b6c 100644 --- a/seq/internal/hash_utils.hpp +++ b/seq/internal/hash_utils.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/seq/internal/radix_tree.hpp b/seq/internal/radix_tree.hpp index 3c129f21..4ce4f364 100644 --- a/seq/internal/radix_tree.hpp +++ b/seq/internal/radix_tree.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,10 +26,10 @@ #define SEQ_RADIX_TREE_HPP #include "../bits.hpp" -#include "../tagged_pointer.hpp" #include "../hash.hpp" #include "../flat_map.hpp" #include "../type_traits.hpp" +#include "tagged_pointer.hpp" #include "simd.hpp" namespace seq @@ -61,7 +61,7 @@ namespace seq struct ExtractKeyResultType { using rtype = decltype(ExtractKey{}(std::declval())); - using type = typename std::decay::type; + using type = std::decay_t; }; template @@ -330,6 +330,62 @@ namespace seq } }; + /// @brief Hash class for time_point + template + struct RadixHasher> : RadixHasher{}.time_since_epoch().count())> + { + using time_point_type = std::chrono::time_point; + using integral_type = decltype(time_point_type{}.time_since_epoch().count()); + + SEQ_ALWAYS_INLINE RadixHasher(integral_type v = 0) noexcept + : RadixHasher{ to_uint(v) } + { + } + template + SEQ_ALWAYS_INLINE auto hash(U k) const noexcept + { + return RadixHasher{ static_cast(k).time_since_epoch().count() }; + } + template + static SEQ_ALWAYS_INLINE bool less(const U& l, const V& r) noexcept + { + return static_cast(l) < static_cast(r); + } + template + SEQ_ALWAYS_INLINE bool equal(const U& l, const V& r) const noexcept + { + return static_cast(l) == static_cast(r); + } + }; + + /// @brief Hash class for duration + template + struct RadixHasher> : RadixHasher + { + using duration_type = std::chrono::duration; + using integral_type = Rep; + + SEQ_ALWAYS_INLINE RadixHasher(integral_type v = 0) noexcept + : RadixHasher{ to_uint(v) } + { + } + template + SEQ_ALWAYS_INLINE auto hash(U k) const noexcept + { + return RadixHasher{ std::chrono::duration_cast(k).count() }; + } + template + static SEQ_ALWAYS_INLINE bool less(const U& l, const V& r) noexcept + { + return std::chrono::duration_cast(l) < std::chrono::duration_cast(r); + } + template + SEQ_ALWAYS_INLINE bool equal(const U& l, const V& r) const noexcept + { + return std::chrono::duration_cast(l) == std::chrono::duration_cast(r); + } + }; + /// @brief Hash class for any kind of array of primitive type (std::string, std::wstring, std::vector, std::array...) template struct RadixHasher::value, void>::type> @@ -363,9 +419,9 @@ namespace seq // One byte: string uint64_t hash = 0; if (size >= byte_offset + 8u) - memcpy(&hash, data + byte_offset, 8u); + memcpy((void*)&hash, data + byte_offset, 8u); else if (byte_offset < size) - memcpy(&hash, data + byte_offset, static_cast(size - byte_offset)); + memcpy((void*)&hash, data + byte_offset, static_cast(size - byte_offset)); return static_cast((swap_b(hash) << bit_offset) >> (64u - count)); } else { @@ -374,9 +430,9 @@ namespace seq byte_offset = (byte_offset) % sizeof(type); if (size >= char_offset + 8U / sizeof(type)) - memcpy(&hash, data + char_offset, 8U); + memcpy((void*)&hash, data + char_offset, 8U); else if (char_offset < size) - memcpy(&hash, data + char_offset, static_cast((size - char_offset) * sizeof(type))); + memcpy((void*)&hash, data + char_offset, static_cast((size - char_offset) * sizeof(type))); // That's easier in big endian as all the bytes // are already in the right order... @@ -581,9 +637,7 @@ namespace seq } RadixHasher() {} - SEQ_ALWAYS_INLINE RadixHasher(const T& val) { - TupleInfo::build_hash(data, val); - } + SEQ_ALWAYS_INLINE RadixHasher(const T& val) { TupleInfo::build_hash(data, val); } SEQ_ALWAYS_INLINE constexpr auto get_size() const noexcept -> size_t { return max_bits; } @@ -602,9 +656,9 @@ namespace seq size_t bit_offset = (shift) & 7U; uint64_t hash = 0; if (size >= byte_offset + 8u) - memcpy(&hash, data + byte_offset, 8u); + memcpy((void*)&hash, data + byte_offset, 8u); else if (byte_offset < size) - memcpy(&hash, data + byte_offset, static_cast(size - byte_offset)); + memcpy((void*)&hash, data + byte_offset, std::min(8u,static_cast(size - byte_offset))); return static_cast((swap_b(hash) << bit_offset) >> (64u - count)); } } @@ -638,6 +692,7 @@ namespace seq using this_type = RadixHasherUnordered; static constexpr bool is_transparent = hash_is_transparent::value; static constexpr bool prefix_search = false; + static constexpr size_t bit_step = 1;//TEST instead of 2 SEQ_ALWAYS_INLINE RadixHasherUnordered(size_t val = 0, const Hash& h = Hash(), const Equal& eq = Equal()) noexcept : RadixHasher{ val } @@ -682,7 +737,7 @@ namespace seq return static_cast(v.emplace_no_check(std::forward(key), std::forward(args)...).first); } template - static T* emplace(T* dst, K&& key, Args&&... args) + static SEQ_ALWAYS_INLINE T* emplace(T* dst, K&& key, Args&&... args) { return new (dst) T(std::forward(key), std::forward(args)...); } @@ -705,14 +760,14 @@ namespace seq .first); } template - static T* emplace(T* dst, K&& key, Args&&... args) + static SEQ_ALWAYS_INLINE T* emplace(T* dst, K&& key, Args&&... args) { return new (dst) T(std::piecewise_construct, std::forward_as_tuple(std::forward(key)), std::forward_as_tuple(std::forward(args)...)); } }; template - static inline SizeType compute_lower_bound(const T* vals, SizeType size, const K& key) + static SEQ_ALWAYS_INLINE SizeType compute_lower_bound(const T* vals, SizeType size, const K& key) { return lower_bound(vals, size, key, [](const auto& l, const auto& r) { return Hasher::less(ExtractKey{}(l), ExtractKey{}(r)); }).first; } @@ -720,7 +775,7 @@ namespace seq /// @brief Copy count elements from src to dst while destroying elements in src /// In case of exception, destroy all elements in in src and dst. template - static inline void copy_destroy(U* dst, U* src, unsigned count) + static void copy_destroy(U* dst, U* src, unsigned count) { if constexpr (is_relocatable::value) memcpy(static_cast(dst), static_cast(src), sizeof(U) * count); @@ -748,7 +803,7 @@ namespace seq /// @brief Insert element at src position while moving elements to the right at dst. /// Basic exception guarantee. template - static void SEQ_ALWAYS_INLINE insert_move_right(U* dst, U* src, unsigned count, Policy, Args&&... args) + static void insert_move_right(U* dst, U* src, unsigned count, Policy, Args&&... args) { // Move src to the right // In case of exception, values are in undefined state, but no new value created (basic exception guarantee) @@ -807,51 +862,36 @@ namespace seq std::uint64_t tmp = (word & 0x7F7F7F7F7F7F7F7FULL) + 0x7F7F7F7F7F7F7F7FULL; return ~(tmp | word | 0x7F7F7F7F7F7F7F7FULL); } -#ifdef __SSE3__ - static SEQ_ALWAYS_INLINE unsigned movemask32(const unsigned char* hashs, unsigned char th) noexcept +#ifdef __SSE2__ + static SEQ_ALWAYS_INLINE unsigned movemask16(const unsigned char* hashs, unsigned char th) noexcept { - auto left = static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_loadu_si128(reinterpret_cast(hashs)), _mm_set1_epi8(static_cast(th))))); - auto right = - static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_loadu_si128(reinterpret_cast(hashs + 16)), _mm_set1_epi8(static_cast(th))))); - return static_cast(left) | (static_cast(right) << 16); + return static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_loadu_si128(reinterpret_cast(hashs)), _mm_set1_epi8(static_cast(th))))); } #endif /// @brief Swiss table like find using AVX2, SSE3, or 8 byte movemask. - template + template static SEQ_ALWAYS_INLINE unsigned find_value(const Equal& eq, const T* values, const unsigned char* ths, unsigned size, unsigned char th, unsigned* insert_pos, const U& val) { + if constexpr (UseLowerBound) { *insert_pos = size; if (Less{}(ExtractKey{}(values[size - 1]), val)) return static_cast(-1); } -#ifdef __SSE3__ - if (cpu_features().HAS_SSE3) { - unsigned size32 = size & ~31u; - for (unsigned i = 0; i < size32; i += 32) { - if (unsigned found = movemask32(ths + i, th)) { - do { - unsigned pos = bit_scan_forward_32(found); - if (eq(ExtractKey{}(values[i + pos]), val)) - return i + pos; - found = found & ~(1U << pos); - } while (found); - } - } - if (unsigned rem = size - size32) { - if (unsigned found = movemask32(ths + size32, th) & ((1U << rem) - 1U)) { - do { - unsigned pos = bit_scan_forward_32(found); - if (eq(ExtractKey{}(values[size32 + pos]), val)) - return size32 + pos; - found = found & ~(1U << pos); - } while (found); - } +#if defined(__SSE2__) + + for (unsigned i = 0; i < size; i += 16) { + if (unsigned found = movemask16(ths + i, th) & ((i + 16 > size) ? ((1U << (size & 15u)) - 1U) : 0xFFFFFFFFu)) { + do { + unsigned pos = bit_scan_forward_32(found); + if (eq(ExtractKey{}(values[i + pos]), val)) + return i + pos; + found = found & ~(1U << pos); + } while (found); } } - else -#endif +#else { unsigned count = size & ~7U; uint64_t _th; @@ -875,6 +915,7 @@ namespace seq } } } +#endif if constexpr (UseLowerBound) *insert_pos = compute_lower_bound(values, size, val); @@ -915,8 +956,8 @@ namespace seq static constexpr bool is_sorted = Sorted; // header size on 64 bits static constexpr unsigned header_size = sizeof(std::uint64_t); - // minimum capacity, depends on sizeof(T) to allow an AVX (32 bytes) load - static constexpr unsigned min_capacity = sizeof(T) == 1 ? 32 : sizeof(T) <= 3 ? 16 : sizeof(T) <= 8 ? 8 : sizeof(T) <= 16 ? 4 : 2; + // minimum capacity, depends on sizeof(T) to allow an SSE2 (16 bytes) load + static constexpr unsigned min_capacity = sizeof(T) == 1 ? 16 : sizeof(T) <= 3 ? 8 : sizeof(T) <= 8 ? 4 : sizeof(T) <= 16 ? 2 : 1; // maximum capacity (and size), lower for sorted elements static constexpr unsigned max_capacity = Sorted ? 64 : 96; @@ -925,7 +966,6 @@ namespace seq // returns the capacity for a given size static unsigned capacity_for_size(unsigned size) noexcept { - // return (size / min_capacity) * min_capacity + (size % min_capacity ? min_capacity : 0); if (size <= min_capacity) return min_capacity; unsigned bits = bit_scan_reverse_32(size); @@ -972,19 +1012,19 @@ namespace seq return std::pair(nullptr, 0); } unsigned insert_pos = static_cast(-1); - unsigned pos = find_value<(Sorted && EnsureSorted), ExtractKey, Equal, Less>(eq, values(), hashs(), count(), static_cast(th), &insert_pos, val); + unsigned pos = find_value(eq, values(), hashs(), count(), static_cast(th), &insert_pos, val); return std::pair(pos == static_cast(-1) ? nullptr : values() + pos, insert_pos); } // Returns value index, -1 if not found template SEQ_ALWAYS_INLINE unsigned find(const Equal& eq, size_t /*start_bit*/, std::uint8_t th, const K& key) const { - return find_value(eq, values(), hashs(), count(), th, nullptr, key); + return find_value(eq, values(), hashs(), count(), th, nullptr, key); } template SEQ_ALWAYS_INLINE unsigned find(const Equal& eq, std::uint8_t th, const K& key) const { - return find_value(eq, values(), hashs(), count(), th, nullptr, key); + return find_value(eq, values(), hashs(), count(), th, nullptr, key); } // Sort leaf @@ -999,8 +1039,9 @@ namespace seq std::pair switch_buffer(NodeAllocator& al, unsigned old_size, unsigned pos, std::uint8_t th, Policy, Args&&... args) { unsigned new_capacity = capacity_for_size(old_size + 1); + unsigned new_hash_count = hash_for_size(old_size + 1, new_capacity); // might throw, fine - LeafNode* n = al.allocate(hash_for_size(old_size + 1, new_capacity), new_capacity); + LeafNode* n = al.allocate(new_hash_count, new_capacity); *n->size() = this->count(); *n->capacity() = new_capacity; try { @@ -1072,18 +1113,31 @@ namespace seq template LeafNode* erase(NodeAllocator& al, unsigned pos) { - erase_pos(values(), pos, *size()); - erase_pos(hashs(), pos, *size()); - (*size())--; - - if (*size() == 0) { + unsigned s = *size(); + if (s == 1) { + values()->~T(); unsigned cap = capacity_for_size(1); al.deallocate(this, hash_for_size(1, cap), cap); return nullptr; } + if (Sorted) { + erase_pos(values(), pos, *size()); + erase_pos(hashs(), pos, *size()); + } + else { + // swap position + if (pos != s - 1) { + // Might throw, fine + values()[pos] = std::move(values()[s - 1]); + hashs()[pos] = hashs()[s - 1]; + hashs()[s - 1] = 0; + } + values()[s - 1].~T(); + } + (*size())--; - unsigned cap = capacity_for_size(*size()); - if (cap != *capacity()) { + if (*size() <= (*capacity()) / 2) { + unsigned cap = capacity_for_size(*size()); // might throw, fine LeafNode* n = al.allocate(hash_for_size(*size(), cap), cap); *n->size() = *this->size(); @@ -1291,7 +1345,7 @@ namespace seq }; /// @brief Class handling allocation/deallocation for leaves and directories - template + template class NodeAllocator : private Allocator { template @@ -1310,13 +1364,44 @@ namespace seq size_t size; // tree size directory* root; // root directory + /// @brief Returns an empty directory used to initialize a radix tree + static directory* get_null_dir() noexcept + { + struct null_dir + { + directory dir; + typename directory::child_ptr child; + }; + static null_dir inst; + return &inst.dir; + } + NodeAllocator(const Allocator& al) : Allocator(al) , size(0) - , root(nullptr) + , root(get_null_dir()) { } - ~NodeAllocator() {} + + ~NodeAllocator() noexcept { clear(); } + + void clear() noexcept + { + if (root != get_null_dir()) + directory::destroy(*this, root); + reset(); + } + void reset() noexcept + { + size = 0; + root = get_null_dir(); + } + + void swap_data(NodeAllocator& other) noexcept + { + std::swap(size, other.size); + std::swap(root, other.root); + } Allocator& get_allocator() noexcept { return *this; } const Allocator& get_allocator() const noexcept { return *this; } @@ -1382,12 +1467,11 @@ namespace seq /// @brief Allocate a directory for given bit length directory* allocate_dir(size_t hash_len) { - - RebindAlloc al = get_allocator(); size_t dir_size = 1ULL << hash_len; size_t bytes = sizeof(directory) + sizeof(child_ptr) * dir_size; size_t to_alloc = bytes / alloc_size + (bytes % alloc_size ? 1 : 0); + RebindAlloc al = get_allocator(); directory* dir = reinterpret_cast(al.allocate(to_alloc)); memset(static_cast(dir), 0, to_alloc * sizeof(std::uint64_t)); @@ -1397,6 +1481,7 @@ namespace seq /// @brief Deallocate directory void deallocate_dir(directory* dir) { + RebindAlloc al = get_allocator(); size_t bytes = sizeof(directory) + sizeof(child_ptr) * dir->size(); size_t to_dealloc = bytes / alloc_size + (bytes % alloc_size ? 1 : 0); @@ -1406,14 +1491,13 @@ namespace seq }; /// @brief Iterator class for radix trees - template + template class RadixConstIter { public: using child_ptr = typename Dir::child_ptr; using node = typename Dir::node; - Data* data; Dir* dir; size_t bit_pos; // bit position of the directory unsigned child; // node position in directory @@ -1513,8 +1597,12 @@ namespace seq { auto tmp = find_next(dir, child + 1, bit_pos); if SEQ_UNLIKELY (!tmp.dir) { - dir = nullptr; - child = 0; + // Find root + auto* root = dir; + while (root->parent) + root = root->parent; + dir = root; + child = root->size(); node_pos = 0; return *this; // end of iteration } @@ -1544,36 +1632,33 @@ namespace seq return *this; } - SEQ_ALWAYS_INLINE RadixConstIter(const Data* dt, const Dir* d, unsigned c, unsigned np, size_t bp) noexcept - : data(const_cast(dt)) - , dir(const_cast(d)) + SEQ_ALWAYS_INLINE RadixConstIter(const Dir* d, unsigned c, unsigned np, size_t bp) noexcept + : dir(const_cast(d)) , bit_pos(bp) , child(c) , node_pos(np) { } - SEQ_ALWAYS_INLINE RadixConstIter(const Data* dt = nullptr) noexcept // end - : data(const_cast(dt)) - , dir(nullptr) + SEQ_ALWAYS_INLINE RadixConstIter(const Dir* root) noexcept // end + : dir(const_cast(root)) , bit_pos(0) - , child(0) + , child(root->size()) , node_pos(0) { } - RadixConstIter(const RadixConstIter&) noexcept = default; - RadixConstIter& operator=(const RadixConstIter&) noexcept = default; + SEQ_ALWAYS_INLINE RadixConstIter(const RadixConstIter&) noexcept = default; + SEQ_ALWAYS_INLINE RadixConstIter& operator=(const RadixConstIter&) noexcept = default; bool is_vector() const noexcept { return dir->const_child(child).tag() == Dir::IsVector; } SEQ_ALWAYS_INLINE auto operator*() const noexcept -> reference { - SEQ_ASSERT_DEBUG(dir, "dereferencing null iterator"); + SEQ_ASSERT_DEBUG(dir && child < dir->size(), "dereferencing invalid iterator"); return const_cast(dir->const_child(child).tag() == Dir::IsVector ? to_vector()->at(node_pos) : to_node()->values()[node_pos]); } SEQ_ALWAYS_INLINE auto operator->() const noexcept -> pointer { return std::pointer_traits::pointer_to(**this); } SEQ_ALWAYS_INLINE auto operator++() noexcept -> RadixConstIter& { - SEQ_ASSERT_DEBUG(data, ""); ++node_pos; if (node_pos != (dir->const_child(child).tag() == Dir::IsVector ? to_vector()->size() : to_node()->count())) return *this; @@ -1588,10 +1673,10 @@ namespace seq } SEQ_ALWAYS_INLINE auto operator--() noexcept -> RadixConstIter& { - SEQ_ASSERT_DEBUG(data, ""); - if (!dir) { + if (child == dir->size()) { // end iterator: got to last element - auto tmp = find_prev(data->base.root, data->base.root->size(), 0); + SEQ_ASSERT_DEBUG(dir->size() != 0, ""); + auto tmp = find_prev(dir, dir->size(), 0); dir = tmp.dir; child = tmp.child; node_pos = dir->const_child(child).tag() == Dir::IsVector ? static_cast(to_vector()->size()) - 1 : to_node()->count() - 1; @@ -1609,16 +1694,8 @@ namespace seq --(*this); return _Tmp; } - SEQ_ALWAYS_INLINE bool operator==(const RadixConstIter& other) const noexcept - { - SEQ_ASSERT_DEBUG(data == other.data || !data || !other.data, "comparing iterators from different radix trees"); - return dir == other.dir && child == other.child && node_pos == other.node_pos; - } - SEQ_ALWAYS_INLINE bool operator!=(const RadixConstIter& other) const noexcept - { - SEQ_ASSERT_DEBUG(data == other.data || !data || !other.data, "comparing iterators from different radix trees"); - return dir != other.dir || child != other.child || node_pos != other.node_pos; - } + SEQ_ALWAYS_INLINE bool operator==(const RadixConstIter& other) const noexcept { return dir == other.dir && child == other.child && node_pos == other.node_pos; } + SEQ_ALWAYS_INLINE bool operator!=(const RadixConstIter& other) const noexcept { return dir != other.dir || child != other.child || node_pos != other.node_pos; } }; inline void check_vector_size(size_t size) @@ -1772,10 +1849,10 @@ namespace seq }; /// @brief Root of a radix tree - template - struct RootTree : public NodeAllocator + template + struct RootTree : public NodeAllocator { - using base_type = NodeAllocator; + using base_type = NodeAllocator; using directory = Directory; using node = Node; @@ -1792,9 +1869,7 @@ namespace seq /// @tparam Hash Hasher type, either Hasher or SortedHasher /// @tparam Extract extract key from value type template, class NodeType = LeafNode, unsigned MaxDepth = 16> - struct RadixTree - : public Hash - , public Allocator + struct RadixTree : public Hash { static constexpr unsigned start_arity = Hash::bit_step; static constexpr bool prefix_search = Hash::prefix_search; @@ -1808,7 +1883,7 @@ namespace seq using vector_type = VectorNode)>; using directory = Directory; using child_ptr = typename directory::child_ptr; - using root_type = RootTree; + using root_type = RootTree; using this_type = RadixTree; template @@ -1819,102 +1894,39 @@ namespace seq /// @brief Equal structure operating on keys using Equal = VectorEqual; - /// @brief Internal data - struct PrivateData - { - root_type base; - PrivateData(const Allocator& al, unsigned start_len = start_arity) - : base(al) - { - base.root = directory::make(base, start_len); - } - ~PrivateData() { directory::destroy(base, base.root); } - }; - - /// @brief Returns an empty directory used to initialize a radix tree - static directory* get_null_dir() noexcept - { - struct null_dir - { - directory dir; - typename directory::child_ptr child; - }; - static null_dir inst; - return &inst.dir; - } - - /// @brief Destroy internal data and reset root - void destroy_data() - { - if (!d_data) - return; - RebindAlloc al = get_allocator(); - destroy_ptr(d_data); - al.deallocate(d_data, 1); - d_data = nullptr; - d_root = get_null_dir(); - } - - /// @brief Allocate and construct internal data with given arity - void make_data(unsigned start_len = start_arity) - { - if (d_data) - return; - RebindAlloc al = get_allocator(); - PrivateData* d = al.allocate(1); - try { - construct_ptr(d, get_allocator(), start_len); - } - catch (...) { - al.deallocate(d, 1); - throw; - } - d_data = d; - d_root = d->base.root; - } - public: using value_type = T; using hash_type = Hash; using const_hash_ref = const Hash&; using extract_key_type = ExtractKey; - using const_iterator = RadixConstIter; + using const_iterator = RadixConstIter; using iterator = const_iterator; - PrivateData* d_data; - directory* d_root; + root_type d_base; // Constructors RadixTree(const Allocator& al = Allocator()) - : Allocator(al) - , d_data(nullptr) - , d_root(get_null_dir()) + : d_base(al) { } RadixTree(const Hash& h, const Allocator& al = Allocator()) : Hash(h) - , Allocator(al) - , d_data(nullptr) - , d_root(get_null_dir()) + , d_base(al) { } RadixTree(const RadixTree& other) : Hash(other) - , Allocator(copy_allocator(other.get_allocator())) - , d_data(nullptr) - , d_root(get_null_dir()) + , d_base(copy_allocator(other.get_allocator())) { if (other.size()) insert(other.begin(), other.end(), false); } RadixTree(const RadixTree& other, const Allocator& al) : Hash(other) - , Allocator(al) - , d_data(nullptr) - , d_root(get_null_dir()) + , d_base(al) { if (other.size()) insert(other.begin(), other.end(), false); @@ -1922,17 +1934,13 @@ namespace seq RadixTree(RadixTree&& other) noexcept(std::is_nothrow_move_constructible_v && std::is_nothrow_copy_constructible_v) : Hash(other) - , Allocator(std::move(other.get_allocator())) - , d_data(nullptr) - , d_root(get_null_dir()) + , d_base(std::move(other.get_allocator())) { swap(other, false); } RadixTree(RadixTree&& other, const Allocator& alloc) : Hash(other) - , Allocator(alloc) - , d_data(nullptr) - , d_root(get_null_dir()) + , d_base(alloc) { if (alloc == other.get_allocator()) swap(other, false); @@ -1942,9 +1950,7 @@ namespace seq template RadixTree(Iter first, Iter last, const Allocator& alloc = Allocator()) - : Allocator(alloc) - , d_data(nullptr) - , d_root(get_null_dir()) + : d_base(alloc) { insert(first, last); } @@ -1964,7 +1970,7 @@ namespace seq { if (std::addressof(other) != this) { clear(); - assign_allocator(*this, other); + assign_allocator(get_allocator(), other.get_allocator()); insert(other.begin(), other.end(), false); } return *this; @@ -2005,34 +2011,33 @@ namespace seq if (size() == 0) return end(); // Find the first valid value - auto tmp = const_iterator::find_next(d_data->base.root, 0, 0); - return const_iterator(d_data, tmp.dir, tmp.child, 0, tmp.bit_pos); + auto tmp = const_iterator::find_next(d_base.root, 0, 0); + return const_iterator(tmp.dir, tmp.child, 0, tmp.bit_pos); } - SEQ_ALWAYS_INLINE const_iterator end() const noexcept { return const_iterator(d_data); } + SEQ_ALWAYS_INLINE const_iterator end() const noexcept { return const_iterator(d_base.root); } SEQ_ALWAYS_INLINE const_iterator cbegin() const noexcept { return begin(); } SEQ_ALWAYS_INLINE const_iterator cend() const noexcept { return end(); } // Alloctor - SEQ_ALWAYS_INLINE Allocator& get_allocator() noexcept { return static_cast(*this); } - SEQ_ALWAYS_INLINE const Allocator& get_allocator() const noexcept { return static_cast(*this); } + SEQ_ALWAYS_INLINE Allocator& get_allocator() noexcept { return d_base.get_allocator(); } + SEQ_ALWAYS_INLINE const Allocator& get_allocator() const noexcept { return d_base.get_allocator(); } // Size functions - SEQ_ALWAYS_INLINE bool empty() const noexcept { return !d_data || d_data->base.size == 0; } - SEQ_ALWAYS_INLINE size_t size() const noexcept { return d_data ? d_data->base.size : 0; } + SEQ_ALWAYS_INLINE bool empty() const noexcept { return d_base.size == 0; } + SEQ_ALWAYS_INLINE size_t size() const noexcept { return d_base.size; } SEQ_ALWAYS_INLINE size_t max_size() const noexcept { return std::numeric_limits::max(); } /// @brief Destroy and deallocate all values, directories and nodes - void clear() { destroy_data(); } + void clear() noexcept { d_base.clear(); } /// @brief Swap 2 radix trees void swap(RadixTree& other, bool swap_alloc = true) noexcept(noexcept(swap_allocator(std::declval(), std::declval()))) { - std::swap(d_data, other.d_data); - std::swap(d_root, other.d_root); + d_base.swap_data(other.d_base); if (swap_alloc) - swap_allocator(*this, other); + swap_allocator(d_base.get_allocator(), other.d_base.get_allocator()); } /// @brief Reserve capcity ahead, only works for unsorted trees @@ -2047,8 +2052,9 @@ namespace seq // Update its root size capacity = (capacity / (node::max_capacity)); unsigned bits = bit_scan_reverse_64(capacity) + 1; - bits = std::min(bits, 26U); // maximum 26 bits for a directory - other.make_data(bits); + bits = std::min(bits, 31U); // maximum 31 bits for a directory + bits = bits & (~start_arity); + other.d_base.root = directory::make(other.d_base, bits); // Move all values inside the new tree for (auto it = begin(); it != end(); ++it) @@ -2076,7 +2082,7 @@ namespace seq if (size() == 0) return; - d_data->base.root->for_each_leaf([](directory* dir, unsigned pos) { + d_base.root->for_each_leaf([](directory* dir, unsigned pos) { auto child = dir->child(pos); if (child.tag() == directory::IsLeaf) { child.to_node()->template sort(Less{}); @@ -2102,7 +2108,7 @@ namespace seq directory* make_intermediate(directory* parent, unsigned hash_len, unsigned parent_pos) { - directory* intermediate = directory::make(d_data->base, hash_len); + directory* intermediate = directory::make(d_base, hash_len); intermediate->parent = parent; intermediate->parent_pos = parent_pos; parent->children()[parent_pos] = child_ptr(intermediate, directory::IsDir); @@ -2125,11 +2131,10 @@ namespace seq if SEQ_UNLIKELY (new_hash_len >= 32) // we are above maximum allowed size for a directory return nullptr; - + /* // Future release: use this version to soften the memory peak - /* directory* new_dir = nullptr; + directory* new_dir = nullptr; { - //TEST //Link all children of dir directory* first = static_cast(dir->children()[0].ptr()); directory* link = first; @@ -2144,11 +2149,11 @@ namespace seq auto prefix_len = dir->prefix_len; // Destroy dir before allocating new directory to avoid memory peak - directory::destroy(d_data->base, dir, false); + directory::destroy(d_base, dir, false); try { // might throw, fine - new_dir = directory::make(d_data->base, new_hash_len); + new_dir = directory::make(d_base, new_hash_len); // copy prefix length new_dir->prefix_len = prefix_len; // set parent, used by iterator::get_bit_pos @@ -2238,16 +2243,16 @@ namespace seq } } - directory::destroy(d_data->base, child, false); + directory::destroy(d_base, child, false); first = next; } } catch (...) { // to keep the basic exception guarantee, the simplest solution is just to clear the tree - directory::destroy(d_data->base, new_dir, true); + directory::destroy(d_base, new_dir, true); for (; i != size; ++i) { directory* next = first->parent; - directory::destroy(d_data->base, first, true); + directory::destroy(d_base, first, true); first = next; } clear(); @@ -2257,13 +2262,13 @@ namespace seq // reset parent new_dir->parent = nullptr; - }*/ + } + */ - // save internal value in order to reset it later to the new directory // might throw, fine - directory* new_dir = directory::make(d_data->base, new_hash_len); + directory* new_dir = directory::make(d_base, new_hash_len); // copy prefix length new_dir->prefix_len = dir->prefix_len; // set parent, used by iterator::get_bit_pos @@ -2330,9 +2335,9 @@ namespace seq ++new_dir->child_count; // update directory count - if SEQ_UNLIKELY(child->children()[j].tag() == directory::IsDir) { + if SEQ_UNLIKELY (child->children()[j].tag() == directory::IsDir) { new_dir->dir_count++; - directory* d = child->const_child(j).to_dir(); + directory* d = child->const_child(j).to_dir(); d->parent = new_dir; d->parent_pos = loc; } @@ -2342,12 +2347,12 @@ namespace seq } dir->children()[i] = child_ptr(); - directory::destroy(d_data->base, child, false); + directory::destroy(d_base, child, false); } } catch (...) { // to keep the basic exception guarantee, the simplest solution is just to clear the tree - directory::destroy(d_data->base, new_dir, true); + directory::destroy(d_base, new_dir, true); clear(); throw; @@ -2357,8 +2362,7 @@ namespace seq new_dir->parent = nullptr; // destroy old directory - directory::destroy(d_data->base, dir, false); - + directory::destroy(d_base, dir, false); // keep merging if possible while (new_dir->dir_count == new_dir->size()) { @@ -2387,7 +2391,7 @@ namespace seq merge_dir(parent_dir); } else - d_root = d_data->base.root = new_dir; + d_base.root = new_dir; return new_dir; } @@ -2477,7 +2481,7 @@ namespace seq const_iterator insert_in_vector(directory* dir, size_t bit_pos, node* child, unsigned pos, Policy, K&& key, Args&&... args) { // turn node into a vector and move values - vector_type* vec = d_data->base.make_vector(*this); + vector_type* vec = d_base.make_vector(*this); unsigned position = 0; try { T* vals = child->values(); @@ -2487,17 +2491,17 @@ namespace seq position = Policy::emplace_vector_no_check(*vec, std::forward(key), std::forward(args)...); } catch (...) { - d_data->base.destroy_vector(vec); + d_base.destroy_vector(vec); throw; } // destroy old child - node::destroy(d_data->base, child); + node::destroy(d_base, child); dir->children()[pos] = child_ptr(vec, directory::IsVector); - ++d_data->base.size; + ++d_base.size; - return const_iterator(d_data, dir, pos, position, bit_pos); + return const_iterator(dir, pos, position, bit_pos); } /// @brief Returns directory depth @@ -2533,7 +2537,7 @@ namespace seq return insert_in_vector(dir, hash_bits, child, pos, policy, std::forward(key), std::forward(args)...); // create new child directory, might throw, fine - directory* child_dir = directory::make(d_data->base, start_arity); + directory* child_dir = directory::make(d_base, start_arity); node* n = nullptr; try { @@ -2560,7 +2564,7 @@ namespace seq if (!child_dir->const_child(new_pos).full()) { // create node. If this throw, the new directory is destroyed (basic exception guarantee) - n = node::make(d_data->base, cth, EmplacePolicy{}, std::move(vals[i])).first; + n = node::make(d_base, cth, EmplacePolicy{}, std::move(vals[i])).first; child_dir->child(new_pos) = child_ptr(n, directory::IsLeaf); child_dir->child_count++; child_dir->first_valid_child = new_pos; @@ -2570,7 +2574,7 @@ namespace seq // if Sort is false, insert the value at the end of the leaf (only for sorted nodes) n = static_cast(child_dir->const_child(new_pos).ptr()); auto p = n->template insert( - d_data->base, hash_bits, Sort ? static_cast(-1) : n->count(), cth, EmplacePolicy{}, std::move(vals[i])); + d_base, hash_bits, Sort ? static_cast(-1) : n->count(), cth, EmplacePolicy{}, std::move(vals[i])); child_dir->child(new_pos) = child_ptr(n = p.first, directory::IsLeaf); } } @@ -2578,12 +2582,12 @@ namespace seq catch (...) { // In case of exception, just destroyed the newly created directory. // Some values might have been moved to it but, hey, this is basic exception guarantee only - directory::destroy(d_data->base, child_dir); + directory::destroy(d_base, child_dir); throw; } // destroy child node unused anymore - node::destroy(d_data->base, child); + node::destroy(d_base, child); // update parent dir child_dir->parent = dir; @@ -2593,9 +2597,10 @@ namespace seq // now, check if the current directory contains only directories, and merge it if possible if (dir->dir_count == dir->size()) { + if (merge_dir(dir, prev_hash_bits)) { // directory merging succeded, now insert the new value starting from the root - return this->insert_hash_with_tiny(d_data->base.root, 0, hash, th, policy, std::forward(key), std::forward(args)...).first; + return this->insert_hash_with_tiny(d_base.root, 0, hash, th, policy, std::forward(key), std::forward(args)...).first; } } @@ -2613,13 +2618,13 @@ namespace seq std::pair insert_null_node(directory* dir, size_t bit_pos, unsigned pos, unsigned th, Policy policy, K&& key, Args&&... args) { // child is empty: create a new leaf, might throw (fine) - auto p = node::make(d_data->base, th, policy, std::forward(key), std::forward(args)...); + auto p = node::make(d_base, th, policy, std::forward(key), std::forward(args)...); dir->child(pos) = child_ptr(p.first, directory::IsLeaf); dir->child_count++; dir->first_valid_child = pos; - ++d_data->base.size; + ++d_base.size; - return std::pair(const_iterator(d_data, dir, pos, 0, bit_pos), true); + return std::pair(const_iterator(dir, pos, 0, bit_pos), true); } /// @brief Insert new value in a vector node @@ -2630,9 +2635,9 @@ namespace seq vector_type* child = static_cast(dir->children()[pos].ptr()); auto found = Policy::emplace_vector(*child, std::forward(key), std::forward(args)...); if (!found.second) - return std::pair(const_iterator(d_data, dir, pos, static_cast(found.first), bit_pos), false); - ++d_data->base.size; - return std::pair(const_iterator(d_data, dir, pos, static_cast(found.first), bit_pos), true); + return std::pair(const_iterator(dir, pos, static_cast(found.first), bit_pos), false); + ++d_base.size; + return std::pair(const_iterator(dir, pos, static_cast(found.first), bit_pos), true); } /// @brief The value to insert does not follow a directory prefox -> create intermediate directory @@ -2646,7 +2651,7 @@ namespace seq // create intermediate directory with a new prefix length // might throw, fine - directory* new_dir = directory::make(d_data->base, start_arity); + directory* new_dir = directory::make(d_base, start_arity); d->parent->child(pos) = child_ptr(new_dir, directory::IsDir); new_dir->parent = d->parent; new_dir->parent_pos = pos; @@ -2738,7 +2743,7 @@ namespace seq if (found.first) // key already exists! - return std::pair(const_iterator(d_data, dir, pos, static_cast(found.first - child->values()), hash_bits), false); + return std::pair(const_iterator(dir, pos, static_cast(found.first - child->values()), hash_bits), false); // check if the node is full and needs to be rehashed if SEQ_UNLIKELY (child->full()) @@ -2747,23 +2752,23 @@ namespace seq // add to leaf auto p = child->template insert( - d_data->base, hash_bits, EnsureSorted ? found.second : child->count(), th, policy, std::forward(key), std::forward(args)...); + d_base, hash_bits, EnsureSorted ? found.second : child->count(), th, policy, std::forward(key), std::forward(args)...); // update first_valid_child dir->first_valid_child = pos; // increment size - ++d_data->base.size; + ++d_base.size; // update child at pos dir->child(pos) = child_ptr(p.first, directory::IsLeaf); - return std::pair(const_iterator(d_data, dir, pos, p.second, hash_bits), true); + return std::pair(const_iterator(dir, pos, p.second, hash_bits), true); } /// @brief Main key insertion process, starting from dir at hash_bits position. template - SEQ_ALWAYS_INLINE std::pair + std::pair insert_hash_with_tiny(directory* dir, size_t hash_bits, const_hash_ref hash, std::uint8_t th, Policy p, K&& key, Args&&... args) { static constexpr bool Sort = EnsureSorted && node::is_sorted; @@ -2787,9 +2792,15 @@ namespace seq pos = hash.n_bits(hash_bits, dir->hash_len); } - if (dir->const_child(pos).tag() == directory::IsNull) + if (dir->const_child(pos).tag() == directory::IsNull) { + if SEQ_UNLIKELY (dir->hash_len == 0) { + // empty root dir + d_base.root = directory::make(d_base, start_arity); + return insert_hash_with_tiny(d_base.root, 0, hash, th, p, std::forward(key), std::forward(args)...); + } // child is empty: create a new node return insert_null_node(dir, hash_bits, pos, th, p, std::forward(key), std::forward(args)...); + } else if (dir->const_child(pos).tag() == directory::IsVector) // insert in vector node @@ -2804,12 +2815,8 @@ namespace seq template SEQ_ALWAYS_INLINE std::pair emplace_hash(const_hash_ref hash, Policy p, K&& key, Args&&... args) { - if SEQ_UNLIKELY (!d_data) - // initialize root - make_data(); - // insert - return insert_hash_with_tiny(d_data->base.root, 0, hash, hash.tiny_hash(), p, std::forward(key), std::forward(args)...); + return insert_hash_with_tiny(d_base.root, 0, hash, hash.tiny_hash(), p, std::forward(key), std::forward(args)...); } /// @brief Construct emplace and insert @@ -2823,7 +2830,6 @@ namespace seq template SEQ_ALWAYS_INLINE std::pair emplace_hash_hint(const_iterator hint, const_hash_ref h, Policy p, K&& key, Args&&... args) { - SEQ_ASSERT_DEBUG(hint.data == d_data || !hint.data, ""); // Insert value and try to reuse the previous position information. // This makes sorted insertion (ascending or desending order) way faster. @@ -2833,15 +2839,11 @@ namespace seq // In addition, we only do that for fixed length keys // to avoid potential cache misses in check_prefix(). - if SEQ_UNLIKELY (!d_data) - // initialize root - make_data(); - // compute tiny hash std::uint8_t th = h.tiny_hash(); if (variable_length || !is_sorted || hint == end() || hint.is_vector() || !check_prefix(h, hash_key(hint.dir->any_child()), 0, hint.bit_pos)) { // Reset position to insert from the root directory - hint.dir = d_data->base.root; + hint.dir = d_base.root; hint.bit_pos = 0; } @@ -2909,8 +2911,8 @@ namespace seq if (!d->parent) { // Root dir // Create a new root of half the size, might throw, fine - auto* new_root = directory::make(d_data->base, d->hash_len - start_arity); - d_data->base.root = d_root = new_root; + auto* new_root = directory::make(d_base, d->hash_len - start_arity); + d_base.root = new_root; } // The remaining does not throw @@ -2930,7 +2932,7 @@ namespace seq directory* parent = p->parent; unsigned parent_pos = p->parent_pos; - directory::destroy(d_data->base, p, false); + directory::destroy(d_base, p, false); parent->children()[parent_pos] = child_ptr(); parent->child_count--; parent->dir_count--; @@ -2950,7 +2952,7 @@ namespace seq count += dir->child(pos).to_vector()->size(); }); - this->d_data->base.size -= count; + this->d_base.size -= count; return count; } @@ -2993,7 +2995,7 @@ namespace seq // Insert all values except the moved one if (it.dir != dir || it.child != pos || it.node_pos != i) this->insert_hash_with_tiny( - d_data->base.root, 0, hash_key(node->values()[i]), node->hashs()[i], EmplacePolicy{}, std::move(node->values()[i])); + d_base.root, 0, hash_key(node->values()[i]), node->hashs()[i], EmplacePolicy{}, std::move(node->values()[i])); } } else { @@ -3014,13 +3016,13 @@ namespace seq catch (...) { // Destroy the directory and recompute ends // to leave the tree in a valid state - directory::destroy(d_data->base, d); + directory::destroy(d_base, d); if (value) value->~T(); throw; } - directory::destroy(d_data->base, d); + directory::destroy(d_base, d); return res; } @@ -3049,7 +3051,7 @@ namespace seq d->child_count--; } else { - --d_data->base.size; + --d_base.size; if (it.node_pos == v->size()) return next; return it; @@ -3058,19 +3060,19 @@ namespace seq else { // erase in leaf node node* n = d->child(dpos).to_node(); - n = n->erase(d_data->base, it.node_pos); + n = n->erase(d_base, it.node_pos); d->children()[dpos] = child_ptr(n, n ? d->child(dpos).tag() : 0); if (!n) d->child_count--; else { - --d_data->base.size; + --d_base.size; if (it.node_pos == n->count()) return next; return it; } } - --d_data->base.size; + --d_base.size; // If we reach that point, the directory might be empty @@ -3090,7 +3092,7 @@ namespace seq directory* parent = d->parent; unsigned parent_pos = d->parent_pos; - directory::destroy(d_data->base, d, false); + directory::destroy(d_base, d, false); parent->children()[parent_pos] = child_ptr(); parent->child_count--; parent->dir_count--; @@ -3122,12 +3124,13 @@ namespace seq /// @brief Find key in vector node template - const_iterator find_in_vector(const directory* d, size_t bit_pos, unsigned pos, const vector_type* vec, const U& key) const + SEQ_NOINLINE(const_iterator) + find_in_vector(const directory* d, size_t bit_pos, unsigned pos, const vector_type* vec, const U& key) const { size_t found = vec->find(key); if (found == vec->size()) return end(); - return const_iterator(d_data, d, pos, static_cast(found), bit_pos); + return const_iterator(d, pos, static_cast(found), bit_pos); } template @@ -3145,13 +3148,11 @@ namespace seq SEQ_ALWAYS_INLINE const_iterator find_hash(const_hash_ref hash, const U& key) const { // start directory - const directory* d = d_root; - - // tiny hash - unsigned th = hash.tiny_hash(); + const directory* d = d_base.root; // start position within start directory size_t bit_pos = 0; + unsigned th = hash.tiny_hash(); unsigned pos = hash.n_bits(bit_pos, d->hash_len); // walk through the tree as long as we keep finding directories @@ -3162,11 +3163,13 @@ namespace seq d = d->children()[pos].to_dir(); // check directory prefix - if (prefix_search && d->prefix_len) { - if (!check_prefix(hash, bit_pos, d)) - return cend(); - else - bit_pos += d->prefix_len; + if constexpr (prefix_search) { + if (d->prefix_len) { + if (!check_prefix(hash, bit_pos, d)) + return cend(); + else + bit_pos += d->prefix_len; + } } // compute new position within current directory @@ -3179,7 +3182,7 @@ namespace seq case directory::IsLeaf: th = d->children()[pos].to_node()->template find(Equal{}, bit_pos, static_cast(th), key); if (th != static_cast(-1)) - return const_iterator(d_data, d, pos, th, bit_pos); + return const_iterator(d, pos, th, bit_pos); return cend(); } } @@ -3191,7 +3194,7 @@ namespace seq SEQ_ALWAYS_INLINE const T* find_ptr_hash(const_hash_ref hash, const U& key) const { // start directory - const directory* d = d_root; + const directory* d = d_base.root; // tiny hash unsigned th = hash.tiny_hash(); // start position within start directory @@ -3206,11 +3209,13 @@ namespace seq d = d->children()[pos].to_dir(); // check directory prefix - if (prefix_search && d->prefix_len) { - if (!check_prefix(hash, bit_pos, d)) - return nullptr; - else - bit_pos += d->prefix_len; + if constexpr (prefix_search) { + if (d->prefix_len) { + if (!check_prefix(hash, bit_pos, d)) + return nullptr; + else + bit_pos += d->prefix_len; + } } // compute new position within current directory @@ -3220,11 +3225,13 @@ namespace seq case directory::IsVector: return find_in_vector_ptr(d, bit_pos, pos, d->children()[pos].to_vector(), key); - case directory::IsLeaf: - th = d->children()[pos].to_node()->template find(Equal{}, bit_pos, static_cast(th), key); + case directory::IsLeaf: { + auto node = d->children()[pos].to_node(); + th = node->template find(Equal{}, bit_pos, static_cast(th), key); if (th != static_cast(-1)) - return d->children()[pos].to_node()->values() + th; + return node->values() + th; return nullptr; + } } } // not found: the node is null or the key is not inside @@ -3248,15 +3255,16 @@ namespace seq const vector_type* v = d->children()[pos].to_vector(); unsigned p = static_cast(v->lower_bound(key)); if (p != v->size()) - return const_iterator(d_data, d, pos, p, bit_pos); - return ++const_iterator(d_data, d, pos, p - 1, bit_pos); + return const_iterator(d, pos, p, bit_pos); + return ++const_iterator(d, pos, p - 1, bit_pos); } /// @brief Find key based on its hash value - SEQ_ALWAYS_INLINE const_iterator lower_bound_hash(const_hash_ref hash, const key_type& key) const + template + SEQ_ALWAYS_INLINE const_iterator lower_bound_hash(const_hash_ref hash, const U& key) const { // start directory - const directory* d = d_root; + const directory* d = d_base.root; // tiny hash unsigned th = hash.tiny_hash(); // start position within start directory @@ -3274,12 +3282,14 @@ namespace seq // check directory prefix if (d->prefix_len) { if (!check_prefix(hash, bit_pos, d)) { - bool less = Less{}(ExtractKey{}(d->any_child()), key); + bool less = Less{}(ExtractKey{}(d->any_child()), ExtractKey{}(key)); auto tmp = const_iterator::find_next( less ? d->parent : d, less ? d->parent_pos + 1 : 0, less ? bit_pos - d->parent->hash_len : bit_pos + d->prefix_len); // Note: could be end() iterator - return const_iterator(d_data, tmp.dir, tmp.child, 0, tmp.bit_pos); + if (!tmp.dir) + return end(); + return const_iterator(tmp.dir, tmp.child, 0, tmp.bit_pos); } else bit_pos += d->prefix_len; @@ -3294,19 +3304,21 @@ namespace seq auto tmp = const_iterator::find_next(d, pos + 1, bit_pos); // Note: could be end() iterator - return const_iterator(d_data, tmp.dir, tmp.child, 0, tmp.bit_pos); + if (!tmp.dir) + return end(); + return const_iterator(tmp.dir, tmp.child, 0, tmp.bit_pos); } // at this point, the position within current directory hold either a standard leaf or a vector if SEQ_UNLIKELY (d->children()[pos].tag() == directory::IsVector) - return lower_bound_in_vector(d, pos, bit_pos, key); + return lower_bound_in_vector(d, pos, bit_pos, ExtractKey{}(key)); const node* n = d->children()[pos].to_node(); unsigned p = n->template lower_bound(bit_pos, th, key); if (p != n->count()) - return const_iterator(d_data, d, pos, p, bit_pos); - return ++const_iterator(d_data, d, pos, p - 1, bit_pos); + return const_iterator(d, pos, p, bit_pos); + return ++const_iterator(d, pos, p - 1, bit_pos); } template SEQ_ALWAYS_INLINE const_iterator lower_bound(const U& k) const @@ -3314,10 +3326,11 @@ namespace seq return lower_bound_hash(hash_key(k), k); } - SEQ_ALWAYS_INLINE const_iterator upper_bound_hash(const_hash_ref hash, const key_type& key) const + template + SEQ_ALWAYS_INLINE const_iterator upper_bound_hash(const_hash_ref hash, const U& key) const { const_iterator it = lower_bound_hash(hash, key); - if (it != end() && ExtractKey{}(*it) == key) + if (it != end() && ExtractKey{}(*it) == ExtractKey{}(key)) ++it; return it; } @@ -3327,13 +3340,21 @@ namespace seq return upper_bound_hash(hash_key(k), k); } + template + bool key_equals(const key_type& prefix, const K& other) const noexcept + { + if (other.size() < prefix.size()) + return false; + return memcmp(prefix.data(), other.data(), prefix.size()) == 0; + } + /// @brief Find key based on its hash value SEQ_ALWAYS_INLINE const_iterator prefix_hash(const_hash_ref hash, const key_type& key) const { static_assert(prefix_search, "prefix function is only available for variable length keys!"); auto it = lower_bound_hash(hash, key); - if (it != end() && ExtractKey{}(*it).find(key.data(), 0, key.size()) == 0) + if (it != end() && key_equals(key, ExtractKey{}(*it))) return it; return end(); } @@ -3343,52 +3364,16 @@ namespace seq return prefix_hash(hash_key(k), k); } - class const_prefix_iterator - { - const_iterator it; - key_type prefix; - - public: - using iterator_category = std::forward_iterator_tag; - using value_type = T; - using difference_type = std::ptrdiff_t; - using const_pointer = const value_type*; - using const_reference = const value_type&; - using pointer = value_type*; - using reference = value_type&; - - SEQ_ALWAYS_INLINE const_prefix_iterator(const const_iterator& i = const_iterator(), const key_type& pr = key_type()) noexcept - : it(i) - , prefix(pr) - { - } - const_prefix_iterator(const const_prefix_iterator&) = default; - const_prefix_iterator& operator=(const const_prefix_iterator&) = default; - - SEQ_ALWAYS_INLINE auto operator*() const noexcept -> reference { return *it; } - SEQ_ALWAYS_INLINE auto operator->() const noexcept -> pointer { return std::pointer_traits::pointer_to(**this); } - SEQ_ALWAYS_INLINE auto operator++() noexcept -> const_prefix_iterator& - { - ++it; - if (it != const_iterator(it.data) && ExtractKey{}(*it).find(prefix.data(), 0, prefix.size()) != 0) - it = const_iterator(it.data); // end - return *this; - } - SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> const_prefix_iterator - { - const_prefix_iterator _Tmp = *this; - ++(*this); - return _Tmp; - } - SEQ_ALWAYS_INLINE bool operator==(const const_prefix_iterator& other) const noexcept { return it == other.it; } - SEQ_ALWAYS_INLINE bool operator!=(const const_prefix_iterator& other) const noexcept { return it != other.it; } - }; - - template - SEQ_ALWAYS_INLINE auto prefix_range(const U& k) const -> std::pair + SEQ_ALWAYS_INLINE auto prefix_range(const key_type& k) const -> std::pair { auto it = prefix(k); - return std::make_pair(const_prefix_iterator(it, extract_key_type{}(k)), const_prefix_iterator(end())); + auto prefix_end = it; + while (prefix_end != end()) { + ++prefix_end; + if (prefix_end != end() && !key_equals(k, ExtractKey{}(*prefix_end))) + break; + } + return {it, prefix_end}; } }; } diff --git a/seq/internal/simd.hpp b/seq/internal/simd.hpp index 4b3aa653..07086226 100644 --- a/seq/internal/simd.hpp +++ b/seq/internal/simd.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/seq/tagged_pointer.hpp b/seq/internal/tagged_pointer.hpp similarity index 99% rename from seq/tagged_pointer.hpp rename to seq/internal/tagged_pointer.hpp index 2a982c82..61183ff6 100644 --- a/seq/tagged_pointer.hpp +++ b/seq/internal/tagged_pointer.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,7 +33,7 @@ #include -#include "bits.hpp" +#include "../bits.hpp" namespace seq { diff --git a/seq/utils.hpp b/seq/internal/utils.hpp similarity index 95% rename from seq/utils.hpp rename to seq/internal/utils.hpp index df3fb5e8..5e14dc0e 100644 --- a/seq/utils.hpp +++ b/seq/internal/utils.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,8 +29,8 @@ #include -#include "bits.hpp" -#include "type_traits.hpp" +#include "../bits.hpp" +#include "../type_traits.hpp" namespace seq { @@ -358,34 +358,7 @@ namespace seq new (p) T(std::forward(args)...); } - /// @brief Iterator range class, provides a valid range object from a start and end iterator. - template - class iterator_range - { - Iter d_begin; - Iter d_end; - - public: - using iterator = Iter; - using value_type = typename std::iterator_traits::value_type; - using reference = typename std::iterator_traits::reference; - using pointer = typename std::iterator_traits::pointer; - using difference_type = typename std::iterator_traits::difference_type; - using size_type = difference_type; - - iterator_range(const Iter& b = Iter(), const Iter& e = Iter()) - : d_begin(b) - , d_end(e) - { - } - iterator_range(const iterator_range&) = default; - iterator_range(iterator_range&&) noexcept = default; - iterator_range& operator=(const iterator_range&) = default; - iterator_range& operator=(iterator_range&&) = default; - - Iter begin() const { return d_begin; } - Iter end() const { return d_end; } - }; + namespace detail { @@ -541,8 +514,29 @@ namespace seq { return ResizeHelperDirect{ v }; } + + template + using RebindAllocator = typename std::allocator_traits::template rebind_alloc; + } + + + /// @brief Allocate count elements of type T using provided allocator + template + T* allocate_from(const Alloc& al, size_t count = 1) + { + detail::RebindAllocator alloc{ al }; + return alloc.allocate(count); + } + /// @brief Deallocate count elements using provided allocator + template + void deallocate_from(const Alloc& al, T* p, size_t count = 1) noexcept + { + detail::RebindAllocator alloc{ al }; + alloc.deallocate(p, count); } + + /// @brief Copy allocator for container copy constructor template auto copy_allocator(const Allocator& alloc) noexcept(std::is_nothrow_copy_constructible_v) -> Allocator @@ -598,6 +592,9 @@ namespace seq static constexpr bool value = std::allocator_traits::propagate_on_container_move_assignment::type && !is_always_equal::value; }; + + + } // end namespace seq #endif diff --git a/seq/charconv.hpp b/seq/legacy/charconv.hpp similarity index 99% rename from seq/charconv.hpp rename to seq/legacy/charconv.hpp index ef2c0d3e..fb192de1 100644 --- a/seq/charconv.hpp +++ b/seq/legacy/charconv.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -114,8 +114,8 @@ In additional to seq::std_input_stream, charconv module provides the similar seq #include #include -#include "bits.hpp" -#include "tiny_string.hpp" +#include "../bits.hpp" +#include "../tiny_string.hpp" #ifdef min #undef min diff --git a/seq/format.hpp b/seq/legacy/format.hpp similarity index 98% rename from seq/format.hpp rename to seq/legacy/format.hpp index 42583de0..198ee907 100644 --- a/seq/format.hpp +++ b/seq/legacy/format.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -476,10 +476,10 @@ Above example compiled with gcc 10.1.0 (-O3) for msys2 on Windows 10 on a Intel( #include #include -#include "tiny_string.hpp" -#include "type_traits.hpp" +#include "../tiny_string.hpp" +#include "../type_traits.hpp" #include "charconv.hpp" -#include "utils.hpp" // for iterator_range +#include "../internal/utils.hpp" // for iterator_range #undef min #undef max @@ -496,6 +496,36 @@ namespace seq // forward declaration template class ostream_format; + + + /// @brief Iterator range class, provides a valid range object from a start and end iterator. + template + class iterator_range + { + Iter d_begin; + Iter d_end; + + public: + using iterator = Iter; + using value_type = typename std::iterator_traits::value_type; + using reference = typename std::iterator_traits::reference; + using pointer = typename std::iterator_traits::pointer; + using difference_type = typename std::iterator_traits::difference_type; + using size_type = difference_type; + + iterator_range(const Iter& b = Iter(), const Iter& e = Iter()) + : d_begin(b) + , d_end(e) + { + } + iterator_range(const iterator_range&) = default; + iterator_range(iterator_range&&) noexcept = default; + iterator_range& operator=(const iterator_range&) = default; + iterator_range& operator=(iterator_range&&) = default; + + Iter begin() const { return d_begin; } + Iter end() const { return d_end; } + }; namespace detail { diff --git a/seq/lock.hpp b/seq/lock.hpp index 7a9cfb4e..ab3d961a 100644 --- a/seq/lock.hpp +++ b/seq/lock.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -126,7 +126,7 @@ namespace seq } SEQ_ALWAYS_INLINE bool try_lock(lock_type& expect) { - if SEQ_UNLIKELY(!d_lock.compare_exchange_strong(expect, write, std::memory_order_acq_rel)) { + if SEQ_UNLIKELY (!d_lock.compare_exchange_strong(expect, write, std::memory_order_acq_rel)) { return failed_lock(expect); } return true; @@ -146,8 +146,9 @@ namespace seq SEQ_ALWAYS_INLINE void lock() { lock_type expect = 0; - while SEQ_UNLIKELY(!try_lock(expect)) - yield(); + while + SEQ_UNLIKELY(!try_lock(expect)) + yield(); } SEQ_ALWAYS_INLINE void unlock() { @@ -156,8 +157,9 @@ namespace seq } SEQ_ALWAYS_INLINE void lock_shared() { - while SEQ_UNLIKELY(!try_lock_shared()) - yield(); + while + SEQ_UNLIKELY(!try_lock_shared()) + yield(); } SEQ_ALWAYS_INLINE void unlock_shared() { @@ -172,7 +174,7 @@ namespace seq lock_type expect = 0; return d_lock.compare_exchange_strong(expect, write, std::memory_order_acq_rel); } - SEQ_ALWAYS_INLINE bool try_lock_fast() { return try_lock();} + SEQ_ALWAYS_INLINE bool try_lock_fast() { return try_lock(); } SEQ_ALWAYS_INLINE bool try_lock_shared() { // This version might be slightly slower in some situations (low concurrency). @@ -184,7 +186,7 @@ namespace seq else { // Version based on fetch_add if (!(d_lock.load(std::memory_order_relaxed) & (need_lock | write))) { - if SEQ_LIKELY(!(d_lock.fetch_add(read, std::memory_order_acquire) & (need_lock | write))) + if SEQ_LIKELY (!(d_lock.fetch_add(read, std::memory_order_acquire) & (need_lock | write))) return true; d_lock.fetch_add(-read, std::memory_order_release); } @@ -193,6 +195,30 @@ namespace seq } SEQ_ALWAYS_INLINE bool is_locked() const noexcept { return d_lock.load(std::memory_order_relaxed) != 0; } SEQ_ALWAYS_INLINE bool is_locked_shared() const noexcept { return d_lock.load(std::memory_order_relaxed) & write; } + + SEQ_ALWAYS_INLINE void upgrade() noexcept + { + for (;;) { + if (d_lock.load(std::memory_order_relaxed) == read) { + lock_type l = read; + if (d_lock.compare_exchange_strong(l, write)) + return; + } + yield(); + } + } + + SEQ_ALWAYS_INLINE void downgrade() noexcept + { + for (;;) { + if (d_lock.load(std::memory_order_relaxed) == write) { + lock_type l = write; + if (d_lock.compare_exchange_strong(l, read)) + return; + } + yield(); + } + } }; using shared_spinlock = shared_spinner<>; @@ -218,6 +244,9 @@ namespace seq void lock_shared() noexcept {} bool try_lock_shared() noexcept { return true; } void unlock_shared() noexcept {} + + void upgrade() noexcept {} + void downgrade() noexcept {} }; } diff --git a/seq/net_sort.hpp b/seq/net_sort.hpp new file mode 100644 index 00000000..2afdd0b7 --- /dev/null +++ b/seq/net_sort.hpp @@ -0,0 +1,1346 @@ +/** + * MIT License + * + * Copyright (c) 2026 Victor Moncada + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef SEQ_NET_SORT_HPP +#define SEQ_NET_SORT_HPP + +#include "bits.hpp" +#include "type_traits.hpp" + +#include +#include +#include +#include +#include + +#define SEQ_ALGO_ASSERT_DEBUG(condition, msg) SEQ_ASSERT_DEBUG(condition, msg) + +namespace seq +{ + namespace algo_detail + { + // Unspecified length + static constexpr unsigned Unspecified = (unsigned)-1; + + /// @brief Iterator wrapper for bidirectional iterator + template + class IterWrapper + { + template + SEQ_ALWAYS_INLINE void increment_iter(Iter& it, Diff d) + { + if constexpr (std::is_same_v) + it += d; + else + std::advance(it, d); + } + + public: + using value_type = typename std::iterator_traits::value_type; + using iterator_category = typename std::iterator_traits::iterator_category; + using difference_type = typename std::iterator_traits::difference_type; + using reference = typename std::iterator_traits::reference; + using pointer = typename std::iterator_traits::pointer; + + Iter iter; + ptrdiff_t pos; + + SEQ_ALWAYS_INLINE IterWrapper(Iter it = Iter(), std::ptrdiff_t p = 0) noexcept + : iter(it) + , pos(p) + { + } + SEQ_ALWAYS_INLINE auto operator*() const noexcept -> reference { return (*this->iter); } + SEQ_ALWAYS_INLINE auto operator++() noexcept -> IterWrapper& + { + ++iter; + ++pos; + return *this; + } + SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> IterWrapper + { + IterWrapper it = *this; + ++(*this); + return it; + } + SEQ_ALWAYS_INLINE auto operator--() noexcept -> IterWrapper& + { + --iter; + --pos; + return *this; + } + SEQ_ALWAYS_INLINE auto operator--(int) noexcept -> IterWrapper + { + IterWrapper it = *this; + --(*this); + return it; + } + template + SEQ_ALWAYS_INLINE auto operator+=(Diff diff) noexcept -> IterWrapper& + { + pos += static_cast(diff); + increment_iter(iter, static_cast(diff)); + return *this; + } + template + SEQ_ALWAYS_INLINE auto operator-=(Diff diff) noexcept -> IterWrapper& + { + pos -= static_cast(diff); + increment_iter(iter, -static_cast(diff)); + return *this; + } + }; + template + SEQ_ALWAYS_INLINE auto operator+(const IterWrapper& it, typename IterWrapper::difference_type diff) noexcept + { + IterWrapper res = it; + res += diff; + return res; + } + template + SEQ_ALWAYS_INLINE auto operator-(const IterWrapper& it, typename IterWrapper::difference_type diff) noexcept + { + IterWrapper res = it; + res -= diff; + return res; + } + template + SEQ_ALWAYS_INLINE auto operator-(const IterWrapper& it1, const IterWrapper& it2) noexcept + { + return it1.pos - it2.pos; + } + template + SEQ_ALWAYS_INLINE auto operator==(const IterWrapper& it1, const IterWrapper& it2) noexcept + { + return it1.iter == it2.iter; + } + template + SEQ_ALWAYS_INLINE auto operator!=(const IterWrapper& it1, const IterWrapper& it2) noexcept + { + return it1.iter != it2.iter; + } + + // noexcept specifier for inplace merge/sort + template + struct NothrowSort + { + // Check if sorting given type with given comparator does not throw + static constexpr bool value = std::is_nothrow_move_assignable_v && std::is_nothrow_move_constructible_v && std::is_nothrow_constructible_v && + noexcept(std::declval()(std::declval(), std::declval())); + }; + // noexcept specifier for inplace merge/sort + template + struct NothrowSortIter : public NothrowSort::value_type, Cmp> + { + }; + + // Interpret element as bitfield for relocatable types + template + struct Bitfield + { + alignas(T) char data[sizeof(T)]; + }; + template + static SEQ_ALWAYS_INLINE Bitfield& as_bits(T& ref) noexcept + { + return (Bitfield&)ref; + } + template + static SEQ_ALWAYS_INLINE Bitfield& as_bits(T&& ref) noexcept + { + return (Bitfield&)ref; + } + template + static SEQ_ALWAYS_INLINE T& as_type(Bitfield& d) + { + return (T&)d; + } + + // similar to std::next, but use ++ operator when no distance is specified + template + static SEQ_ALWAYS_INLINE auto iter_next(Iter it) noexcept + { + return ++it; + } + template + static SEQ_ALWAYS_INLINE auto iter_next(Iter it, Diff d) noexcept + { + return it + d; + } + + // similar to std::prev, but use -- operator when no distance is specified + template + static SEQ_ALWAYS_INLINE auto iter_prev(Iter it) noexcept + { + return --it; + } + template + static SEQ_ALWAYS_INLINE auto iter_prev(Iter it, Diff d) noexcept + { + return it - d; + } + + template + static SEQ_ALWAYS_INLINE auto unwrap_iter(Iter it) noexcept + { + return it; + } + template + static SEQ_ALWAYS_INLINE auto unwrap_iter(std::move_iterator it) noexcept + { + return unwrap_iter(it.base()); + } + template + static SEQ_ALWAYS_INLINE auto unwrap_iter(std::reverse_iterator it) noexcept + { + return unwrap_iter(it.base()); + } + + // compare iterators for equality without triggering compile error + template + static SEQ_ALWAYS_INLINE bool iter_equal(Iter1 it1, Iter2 it2) noexcept + { + using type1 = decltype(unwrap_iter(it1)); + using type2 = decltype(unwrap_iter(it2)); + if constexpr (std::is_same_v) + return unwrap_iter(it1) == unwrap_iter(it2); + else + return false; + } + + // similar to std::distance with an overload for IterWrapper that supports subtracting 2 iterators + template + static SEQ_ALWAYS_INLINE auto iter_distance(Iter first, Iter last) noexcept + { + return std::distance(first, last); + } + template + static SEQ_ALWAYS_INLINE auto iter_distance(const IterWrapper& first, const IterWrapper& last) noexcept + { + return last - first; + } + + template + static SEQ_ALWAYS_INLINE auto wrap_iter(Iter it, ptrdiff_t d = 0) + { + if constexpr (is_random_access::value) { + (void)d; + return it; + } + else + return IterWrapper(it, d); + } + + template + static void + merge_inplace_left(Iter f0, size_t n0, Iter f1, size_t n1, Iter& f0_0, size_t& n0_0, Iter& f0_1, size_t& n0_1, Iter& f1_0, size_t& n1_0, Iter& f1_1, size_t& n1_1, Cmp r) noexcept( + NothrowSortIter::value) + { + // Subroutine of inplace_merge_n + SEQ_ALGO_ASSERT_DEBUG((size_t)iter_distance(f0, f1) == n0, ""); + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(f0, iter_next(f0, n0), r) && std::is_sorted(f1, iter_next(f1, n1), r), ""); + SEQ_ALGO_ASSERT_DEBUG(n0 > 0, ""); + SEQ_ALGO_ASSERT_DEBUG(n1 > 0, ""); + + f0_0 = f0; + n0_0 = n0 >> 1; + f0_1 = f0; + f0_1 = iter_next(f0_1, n0_0); + f1_1 = std::lower_bound(f1, iter_next(f1, n1), *f0_1, r); + f1_0 = std::rotate(f0_1, f1, f1_1); + n0_1 = iter_distance(f0_1, f1_0); + ++f1_0; + n1_0 = (n0 - n0_0) - 1; + n1_1 = n1 - n0_1; + } + + template + static void + merge_inplace_right(Iter f0, size_t n0, Iter f1, size_t n1, Iter& f0_0, size_t& n0_0, Iter& f0_1, size_t& n0_1, Iter& f1_0, size_t& n1_0, Iter& f1_1, size_t& n1_1, Cmp r) noexcept( + NothrowSortIter::value) + { + // Subroutine of inplace_merge_n + SEQ_ALGO_ASSERT_DEBUG((size_t)iter_distance(f0, f1) == n0, ""); + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(f0, iter_next(f0, n0), r) && std::is_sorted(f1, iter_next(f1, n1), r), ""); + SEQ_ALGO_ASSERT_DEBUG(n0 > 0, ""); + SEQ_ALGO_ASSERT_DEBUG(n1 > 0, ""); + + f0_0 = f0; + n0_1 = n1 >> 1; + f1_1 = f1; + f1_1 = iter_next(f1_1, n0_1); + f0_1 = std::upper_bound(f0, iter_next(f0, n0), *f1_1, r); + ++f1_1; + f1_0 = std::rotate(f0_1, f1, f1_1); + n0_0 = iter_distance(f0_0, f0_1); + n1_0 = n0 - n0_0; + n1_1 = (n1 - n0_1) - 1; + } + + template + static SEQ_ALWAYS_INLINE Out copy_internal(Iter begin, Iter end, Out out) + { + // direct std::copy call + return std::copy((begin), (end), (out)); + } + template + static SEQ_ALWAYS_INLINE Out copy_internal(std::move_iterator begin, std::move_iterator end, Out out) + { + // Let the compiler decide to use memmove if necesary + return std::move((begin.base()), (end.base()), (out)); + } + + template + static Out merge_forward(Iter1 first1, Iter1 end1, Iter2 first2, Iter2 end2, Out out, Cmp c) + { + // Merge 2 range forward + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(first1, end1, c), ""); + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(first2, end2, c), ""); + SEQ_DEBUG_ONLY(Out dst = out;) + + if constexpr (is_random_access::value && is_random_access::value) { + + // Check for unbalanced merge + auto s1 = iter_distance(first1, end1); + auto s2 = iter_distance(first2, end2); + + if (s1 * 32 < s2) { + // Left is way smaller than right + for (; first1 != end1; ++first1) { + if (first2 != end2) { + while (first1 != end1 && !c(*first2, *first1)) + *out++ = (*first1++); + if (first1 == end1) + break; + + auto found = std::lower_bound(first2, end2, *first1, c); + out = copy_internal(first2, found, out); + first2 = found; + } + *out++ = (*first1); + } + goto end; + } + else if (s2 * 32 < s1) { + // Right is way smaller than left + for (; first2 != end2; ++first2) { + if (first1 != end1) { + while (first2 != end2 && c(*first2, *first1)) + *out++ = (*first2++); + if (first2 == end2) + break; + + auto found = std::upper_bound(first1, end1, *first2, c); + out = copy_internal(first1, found, out); + first1 = found; + } + *out++ = (*first2); + } + goto end; + } + } + + // More efficient merge than std::merge (usually) + + while (first2 != end2) { + while (first1 != end1 && !c(*first2, *first1)) { + + *out = (*first1); + ++out; + ++first1; + } + + if (first1 == end1) + break; + + *out = (*first2); + ++out; + ++first2; + + while (first2 != end2 && c(*first2, *first1)) { + + *out = (*first2); + ++out; + ++first2; + } + + *out = (*first1); + ++out; + ++first1; + } + + end: + out = copy_internal(first1, end1, out); + if (Overlap && iter_equal(first2, out)) { + // The last range is already inplace: + // just advance output iterator if not + // a std::reverse_iterator (used by + // merge_backward which do not use the result). + if constexpr (!is_reverse_iterator::value) + out = iter_next(out, iter_distance(first2, end2)); + } + else + out = copy_internal(first2, end2, out); + + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(dst, out, c), ""); + return out; + } + + template + static SEQ_ALWAYS_INLINE std::pair merge_tails(Iter* first, Iter* second, Out& out_left, Out& out_right, Cmp c) + { + // Merge tails and advance + bool left_order = c(*first[1], *first[0]); + bool right_order = !c(*second[1], *second[0]); + *out_left = std::move(*first[left_order]); + *out_right = std::move(*second[right_order]); + ++out_left; + --out_right; + first[1] += left_order; + first[0] += !left_order; + second[1] -= right_order; + second[0] -= !right_order; + return { left_order, right_order }; + } + + template + static SEQ_ALWAYS_INLINE void finish_bidirectional_merge(Iter* first, Iter* second, Out out_left, Cmp c) + { + bool finish_left = (second[0] < first[0]); + bool finish_right = (second[1] < first[1]); + + if (!finish_left && !finish_right) { + merge_forward( + std::make_move_iterator(first[0]), std::make_move_iterator(++second[0]), std::make_move_iterator(first[1]), std::make_move_iterator(++second[1]), out_left, c); + } + else if (finish_left) + std::move((first[1]), (++second[1]), (out_left)); + else if (finish_right) + std::move((first[0]), (++second[0]), (out_left)); + } + + template + static Out merge_move_bidirectional(Iter first1, Iter last1, Iter first2, Iter last2, Out out, Cmp c, Out* out_end = nullptr) noexcept(NothrowSortIter::value) + { + using T = typename std::iterator_traits::value_type; + + // Merge 2 sorted ranges to given output. + // Uses the fastest available method: standard forward merge + // or branchless merge from both ends for random access iterators and relocatable types. + + if constexpr (is_random_access::value && is_relocatable::value) { + + // Branchless merge from both ends + // Only truly faster with trivial comparison function, + // which is usually the case for relocatable types. + + Out out_left = out; + Iter first[2] = { first1, first2 }; + Iter second[2] = { iter_prev(last1), iter_prev(last2) }; + + if constexpr (Count != Unspecified) { + Out res = out_end ? *out_end : iter_next(out, Count * 2); + Out out_right = iter_prev(res); + + while (first[0] < second[0] && first[1] < second[1]) + merge_tails(first, second, out_left, out_right, c); + + finish_bidirectional_merge(first, second, out_left, c); + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(out, res, c), ""); + return res; + } + else { + + ptrdiff_t dist1 = iter_distance(first1, last1); + ptrdiff_t dist2 = iter_distance(first2, last2); + + Out res = out_end ? *out_end : iter_next(out, dist1 + dist2); + Out out_right = iter_prev(res); + + // Unbalanced merge from both ends. + // For the first part of the merge (1/16 of the smallest range), + // check if the order is pseud random. If not, finish with + // merge_forward(). + + const ptrdiff_t iter_count = std::min(dist1, dist2); + const ptrdiff_t stop = iter_count / 16; + ptrdiff_t order = 0; + ptrdiff_t count = 0; + + std::pair prev_order = { true, true }; + + if (first[0] < second[0] && first[1] < second[1]) { + + prev_order = merge_tails(first, second, out_left, out_right, c); + + while (count < stop && first[0] < second[0] && first[1] < second[1]) { + auto ord = merge_tails(first, second, out_left, out_right, c); + order += (ptrdiff_t)(ord.first == prev_order.first) + (ptrdiff_t)(ord.second == prev_order.second); + prev_order = ord; + ++count; + } + if (order <= stop + stop / 2) { + // Balanced merging: keep using bidirectional merge + while (first[0] < second[0] && first[1] < second[1]) + merge_tails(first, second, out_left, out_right, c); + } + + // Finish with merge_forward + finish_bidirectional_merge(first, second, out_left, c); + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(out, res, c), ""); + return res; + } + } + } + // Standard forward merge + return merge_forward(std::make_move_iterator(first1), std::make_move_iterator(last1), std::make_move_iterator(first2), std::make_move_iterator(last2), out, c); + } + + template + static SEQ_ALWAYS_INLINE void merge_backward(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Out out_end, Cmp c) noexcept(NothrowSortIter::value) + { + // Merge backward implemented in terms of merge_forward + merge_forward(std::make_reverse_iterator(last2), + std::make_reverse_iterator(first2), + std::make_reverse_iterator(last1), + std::make_reverse_iterator(first1), + std::make_reverse_iterator(out_end), + [c](const auto& a, const auto& b) { return c(b, a); }); + } + + template + static SEQ_ALWAYS_INLINE void merge_with_buffer(Iter first, size_t n0, Iter middle, size_t n1, Iter e1, Cmp r, B buffer) noexcept(NothrowSortIter::value) + { + // Inplace merge 2 ranges using provided buffer. + // Moves as few elements as possible to the temporary buffer. + if (n0 <= n1) { + auto blast = std::move((first), (middle), buffer); + merge_forward(std::make_move_iterator(buffer), std::make_move_iterator(blast), std::make_move_iterator(middle), std::make_move_iterator(e1), first, r); + } + else { + auto last = e1; + auto blast = std::move((middle), (last), buffer); + merge_backward(std::make_move_iterator(first), std::make_move_iterator(middle), std::make_move_iterator(buffer), std::make_move_iterator(blast), last, r); + } + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(first, iter_next(middle, n1), r), ""); + } + + template + static void rotate_one_right(Iter first, Iter mid, Iter last) + { + // Exchanges the range [first, mid) with [mid, last) + // pre: distance(mid, last) is 1 + using type = typename std::iterator_traits::value_type; + type tmp(std::move(*mid)); + std::move_backward((first), (mid), (last)); + *first = std::move(tmp); + } + + template + static void merge_adaptive_n(Iter f0, size_t n0, Iter f1, size_t n1, Iter e1, Cmp r, B buffer) noexcept(NothrowSortIter::value) + { + // Inplace merge with buffer, first published by Dudzin'sky and Dydek in 1981 IPL 12(1):5-8 + // Implementation from: https://www.jmeiners.com/efficient-programming-with-components/15_merge_inplace.html + + SEQ_ALGO_ASSERT_DEBUG((size_t)iter_distance(f0, f1) == n0, ""); + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(f0, iter_next(f0, n0), r), ""); + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(f1, iter_next(f1, n1), r), ""); + + if constexpr (FirstChecks) { + // Perform the firsts, easy checks + + if (!n0 || !n1 || !r(*f1, *iter_prev(f1))) + // One of the 2 ranges is empty, or already sorted + return; + + if (r(*iter_prev(e1), *f0)) { + // Simple rotation needed + std::rotate((f0), (f1), (e1)); + return; + } + + // The following checks come from msvc STL implementation + // and help a LOT in some situations + + // Increment f0 as long as it is smaller than first value of second range + for (;;) { + if (f0 == f1) + // We reach the end of first range: already in order + return; + if (r(*f1, *f0)) + break; + ++f0; + --n0; + } + + auto highest = iter_prev(f1); + do { + --e1; + --n1; + if (f1 == e1) { // rotate only element remaining in right partition to the beginning, without allocating + rotate_one_right(f0, f1, ++e1); + return; + } + } while (!r(*e1, *highest)); // found that *highest goes in *e1's position + + ++e1; + ++n1; + } + + if (n0 <= buffer.size || n1 <= buffer.size) + // We have enough buffer: merge + return merge_with_buffer((f0), n0, (f1), n1, (e1), r, (buffer.first)); + + // Rotate left or right range + Iter f0_0, f0_1, f1_0, f1_1; + size_t n0_0, n0_1, n1_0, n1_1; + if (n0 < n1) + merge_inplace_left(f0, n0, f1, n1, f0_0, n0_0, f0_1, n0_1, f1_0, n1_0, f1_1, n1_1, r); + else + merge_inplace_right(f0, n0, f1, n1, f0_0, n0_0, f0_1, n0_1, f1_0, n1_0, f1_1, n1_1, r); + + // Recurse on each range + merge_adaptive_n(f0_0, n0_0, f0_1, n0_1, iter_next(f0_1, n0_1), r, buffer); + merge_adaptive_n(f1_0, n1_0, f1_1, n1_1, iter_next(f1_1, n1_1), r, buffer); + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(f0, e1), ""); + } + + template + Iter insertion_sort_n(Iter begin, unsigned count, Cmp l) noexcept(NothrowSortIter::value) + { + // Standard in-place insertion sort working on bidirectional iterators, + // but using a number of values to sort instead of an end iterator. + + using T = typename std::iterator_traits::value_type; + if SEQ_UNLIKELY (count < 2) + return count == 0 ? begin : iter_next(begin); + + auto cur = begin; + auto prev = cur++; + for (; count > 1; --count) { + if (l(*cur, *prev)) { + auto sift = cur; + T tmp = std::move(*sift); + do { + *sift = std::move(*prev); + --sift; + } while (sift != begin && l(tmp, *(--prev))); + *sift = std::move(tmp); + } + prev = cur++; + } + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(begin, cur, l), ""); + return cur; + } + + template + static void reverse_sort(Iter begin, Iter end, Cmp l) noexcept(NothrowSortIter::value) + { + // Inplace reverse range. + // The range must be sorted in descending order. + // Stable version of std::reverse. + + if (begin == end) + return; + Iter start = begin; + Iter prev = begin++; + while (begin != end) { + // Loop through non equal values + while (l(*begin, *prev)) { + prev = begin++; + if SEQ_UNLIKELY (begin == end) + goto reverse_full; + } + // Find full equal range and reverse it + Iter start_equal = prev++; + ++begin; + while (begin != end && !l(*begin, *prev)) + prev = begin++; + std::reverse((start_equal), (begin)); + } + + reverse_full: + // Reverse the full sequence. + // Equal ranges will get back their natural orders. + std::reverse((start), (end)); + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(start, end, [l](const auto& a, const auto& b) { return l(a, b); }), ""); + } + + template + static void ping_pong_merge_4(Iter it0, Iter it1, Iter it2, Iter it3, Iter it4, Cmp c, Storage tmp) noexcept(NothrowSortIter::value) + { + // Ping pong merge 4 sorted ranges using provided buffer. + + if ((size_t)(it4 - it0) <= tmp.size) { + const bool s0 = !c(*it1, *iter_prev(it1)); + const bool s1 = !c(*it2, *iter_prev(it2)); + const bool s2 = !c(*it3, *iter_prev(it3)); + if (s0 && s1 && s2) + return; + + decltype(tmp.first) middle, end; + if (!s0) + middle = merge_move_bidirectional((it0), (it1), (it1), (it2), tmp.first, c); + else { + auto dst = std::move((it0), (it1), tmp.first); + middle = std::move((it1), (it2), dst); + } + if (!s2) + end = merge_move_bidirectional((it2), (it3), (it3), (it4), middle, c); + else { + auto dst = std::move((it2), (it3), middle); + end = std::move((it3), (it4), dst); + } + if (c(*middle, *iter_prev(middle))) + merge_move_bidirectional(tmp.first, (middle), (middle), (end), it0, c, &it4); + else { + auto dst = std::move((tmp.first), (middle), it0); + std::move((middle), (end), dst); + } + } + else { + merge_adaptive_n(it0, it1 - it0, it1, it2 - it1, it2, c, tmp); + merge_adaptive_n(it2, it3 - it2, it3, it4 - it3, it4, c, tmp); + merge_adaptive_n(it0, it2 - it0, it2, it4 - it2, it4, c, tmp); + } + } + template + static void ping_pong_merge_3(Iter it0, Iter it1, Iter it2, Iter it3, Cmp c, Storage tmp) noexcept(NothrowSortIter::value) + { + // Ping pong merge 3 sorted ranges using provided buffer. + + if ((size_t)(it2 - it0) <= tmp.size) { + const bool s0 = !c(*it1, *iter_prev(it1)); + const bool s1 = !c(*it2, *iter_prev(it2)); + if (s0 && s1) + return; + + auto middle = tmp.first; + if (!s0) + middle = merge_move_bidirectional((it0), (it1), (it1), (it2), tmp.first, c); + else { + auto dst = std::move((it0), (it1), tmp.first); + middle = std::move((it1), (it2), dst); + } + if (c(*it2, *iter_prev(middle))) + merge_forward(std::make_move_iterator(tmp.first), std::make_move_iterator(middle), std::make_move_iterator(it2), std::make_move_iterator(it3), it0, c); + else + std::move((tmp.first), (middle), it0); + } + else { + merge_adaptive_n(it0, it1 - it0, it1, it2 - it1, it2, c, tmp); + merge_adaptive_n(it0, it2 - it0, it2, it3 - it2, it3, c, tmp); + } + } + + template::pointer> + static void merge_sorted_runs_with_buffer(Iter* iters, size_t start, size_t last, Cmp cmp, Buffer buf) noexcept(NothrowSortIter::value) + { + // Inplace merge already sorted ranges represented by an array of iterators. + // Supports bidirectional iterators. + // Internally calls merge_adaptive_n with provided buffer. + + auto size = last - start; + + if (size > 4) { + auto quarter = size / 4; + auto half = quarter * 2; + auto quarter3 = quarter * 3; + // merge each 4 ranges + merge_sorted_runs_with_buffer(iters, start, start + quarter, cmp, buf); + merge_sorted_runs_with_buffer(iters, start + quarter, start + half, cmp, buf); + merge_sorted_runs_with_buffer(iters, start + half, start + quarter3, cmp, buf); + merge_sorted_runs_with_buffer(iters, start + quarter3, last, cmp, buf); + return ping_pong_merge_4(iters[start], iters[start + quarter], iters[start + half], iters[start + quarter3], iters[last], cmp, buf); + } + + switch (size) { + case 4: + ping_pong_merge_4(iters[start], iters[start + 1], iters[start + 2], iters[start + 3], iters[start + 4], cmp, buf); + break; + case 3: + ping_pong_merge_3(iters[start], iters[start + 1], iters[start + 2], iters[start + 3], cmp, buf); + break; + case 2: + merge_adaptive_n(iters[start], iters[start + 1] - iters[start], iters[start + 1], iters[last] - iters[start + 1], iters[last], cmp, buf); + break; + default: + break; + } + } + + template + static SEQ_ALWAYS_INLINE void swap_branchless(T&& a, T&& b, bool b_is_less) noexcept + { + // Swap elements based on is_less. + // Uses branchless swap for relocatable types. + + if constexpr (is_relocatable::value) { + auto tmp = as_bits(a); + as_bits(a) = b_is_less ? as_bits(b) : tmp; + as_bits(b) = b_is_less ? tmp : as_bits(b); + } + else { + using std::swap; + if (b_is_less) + swap(a, b); + } + } + +#define CHECK_2(l, r) swap_branchless(a##l, a##r, cmp(a##r, a##l)) +#define CHECK_2_NO_OVERLAPP(a, b, c, d) \ + CHECK_2(a, b); \ + CHECK_2(c, d) +#define CHECK_4_NO_OVERLAPP(a, b, c, d, e, f, g, h) \ + CHECK_2_NO_OVERLAPP(a, b, c, d); \ + CHECK_2_NO_OVERLAPP(e, f, g, h) + + template + static SEQ_ALWAYS_INLINE void network_sort_8(T&& a0, T&& a1, T&& a2, T&& a3, T&& a4, T&& a5, T&& a6, T&& a7, Cmp cmp) + { + CHECK_4_NO_OVERLAPP(0, 1, 2, 3, 4, 5, 6, 7); + CHECK_4_NO_OVERLAPP(0, 2, 1, 3, 4, 6, 5, 7); + CHECK_4_NO_OVERLAPP(0, 4, 1, 5, 2, 6, 3, 7); + CHECK_2_NO_OVERLAPP(2, 4, 3, 5); + CHECK_2_NO_OVERLAPP(1, 4, 3, 6); + CHECK_2_NO_OVERLAPP(1, 2, 3, 4); + CHECK_2(5, 6); + } + +#undef CHECK_2 +#undef CHECK_2_NO_OVERLAPP +#undef CHECK_4_NO_OVERLAPP + + template + static SEQ_ALWAYS_INLINE Iter atom_sort_8(Iter vals, unsigned count, Cmp cmp) noexcept(NothrowSortIter::value) + { + // Sort up to 8 values + if constexpr (is_random_access::value && N == 8) { + if (count == 8) { + // Sort 8 values using a sorting netork + network_sort_8(vals[0], vals[1], vals[2], vals[3], vals[4], vals[5], vals[6], vals[7], cmp); + return vals + 8; + } + } + + return insertion_sort_n(vals, N == Unspecified ? count : N, cmp); + } + + template + static Out atom_sort_64(Iter& first, Out out, Cmp c) noexcept(NothrowSortIter::value) + { + // Sort 64 values to output + + auto it0 = atom_sort_8<8>(first, 8, c); + auto it1 = atom_sort_8<8>(it0, 8, c); + auto it2 = atom_sort_8<8>(it1, 8, c); + auto it3 = atom_sort_8<8>(it2, 8, c); + auto it4 = atom_sort_8<8>(it3, 8, c); + auto it5 = atom_sort_8<8>(it4, 8, c); + auto it6 = atom_sort_8<8>(it5, 8, c); + auto it7 = atom_sort_8<8>(it6, 8, c); + + auto o0 = merge_move_bidirectional<8>(first, it0, it0, it1, out, c); + auto o1 = merge_move_bidirectional<8>(it1, it2, it2, it3, o0, c); + auto o2 = merge_move_bidirectional<8>(it3, it4, it4, it5, o1, c); + auto o3 = merge_move_bidirectional<8>(it5, it6, it6, it7, o2, c); + auto d0 = merge_move_bidirectional<16>(out, o0, o0, o1, first, c); + auto d1 = merge_move_bidirectional<16>(o1, o2, o2, o3, d0, c); + auto r = merge_move_bidirectional<32>(first, d0, d0, d1, out, c); + first = d1; + return r; + } + + template + static Iter sort_128(Iter vals, unsigned count, Cmp c, Buffer buf) noexcept(NothrowSortIter::value) + { + // Sort inplace up to 128 values using provided buffer + + if (count == 128 && buf.size >= 128) { + auto src = (vals); + auto it0 = atom_sort_64((src), (buf.first), c); + auto it1 = atom_sort_64((src), (it0), c); + return merge_move_bidirectional<64>((buf.first), (it0), (it0), (it1), vals, c); + } + else { + // Buffer too small or less than 128 elements + Iter iters[17] = { vals }; + unsigned cnt = 1; + unsigned rem = count; + Iter src = vals; + while (rem) { + auto p = atom_sort_8(src, std::min(rem, 8u), c); + rem -= (unsigned)(p - src); + iters[cnt++] = src = p; + } + merge_sorted_runs_with_buffer(iters, 0, cnt - 1, c, buf); + return src; + } + } + + template + static std::pair try_wave_sort(T begin, size_t size, size_t min_dist, Cmp c, Buffer buf) noexcept(NothrowSortIter::value) + { + // Attempt to sort the range [begin,end). + // Find consecutive sorted runs (ascending or descending), up to IterCount-1. + // Stop when reaching end iterator or runs limit. + // Supports bidirectional iterators. + + if SEQ_UNLIKELY (size == 0) + return { begin, 0 }; + + T start = begin; + T prev = begin; + T iters[IterCount] = { begin++ }; + bool ascending[IterCount]; + unsigned cnt = 1; + size_t dist = 1; + + if (size == 1) + return { begin, 1 }; + + ascending[0] = !c(*begin, *prev); + prev = begin; + ++begin; + ++dist; + + for (; dist != size; ++begin, ++prev, ++dist) { + // Find consecutive ascending or descending runs + const bool as = ascending[cnt - 1]; + if (as) { + for (; dist != size && !c(*begin, *prev); ++dist) + prev = begin++; + } + else { + for (; dist != size && !c(*prev, *begin); ++dist) + prev = begin++; + } + + // Stop before adding the start of a new sorted range, + // or the last range will have a size of 1... + if (cnt == (IterCount - 1)) + break; + + ascending[cnt] = !as; + iters[cnt++] = begin; + if (cnt > 1 && begin == iter_next(iters[cnt - 2])) { + ascending[cnt - 2] = !as; + --cnt; + } + if SEQ_UNLIKELY (dist == size) + break; + } + + if (dist < min_dist) + // We were not able to sort up to min_dist elements, + // returns the start iterator to notify this failure. + return { start, 0 }; + + // add last iterator + if (iters[cnt - 1u] != begin) + iters[cnt++] = begin; + + // reverse descending ranges + for (size_t i = 0; i < cnt - 1; ++i) { + if (!ascending[i]) + // stable reverse + reverse_sort(iters[i], iters[i + 1], c); + } + + // inplace merge runs + merge_sorted_runs_with_buffer(iters, 0u, cnt - 1u, c, buf); + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(start, begin, c), ""); + return { begin, dist }; + } + + template + static void generic_merge_sort_internal(Iter begin, Iter end, size_t size, Cmp l, Fn sort_sub_range, Buff buf, size_t min_size = 0) noexcept(NothrowSortIter::value) + { + // Generic merge sort that uses a custom sort function for small chunks. + // Supports bidirectional iterators. + + size_t remaining = size; + Iter start = begin; + Iter last_start = begin; + size_t new_min_size = std::numeric_limits::max(); + + do { + size_t cnt = 1; + size_t cum_dist = 0; + Iter iters[MaxIters] = { begin }; + + do { + // Sort any number of elements (up to remaining). + // sort_sub_range must return a std::pair. + std::pair r; + if (min_size == 0) + // First pass: sort input chunk + r = sort_sub_range(begin, remaining, l); + else { + // Next passes : identify sorted range + if (remaining >= min_size) { + // We need to identify the sorted range with is_sorted_until() + + // Increment it by min_size -1, starting from begin or from end + auto it = begin; + + if constexpr (!is_random_access::value) { + size_t d = (size_t)iter_distance(begin, end); + if (min_size > d / 2) + it = iter_prev(end, d - min_size + 1); + } + if (it == begin) + it = iter_next(begin, min_size - 1); + + auto p = std::is_sorted_until((it), (end), l); + r = { p, min_size - 1 + (size_t)iter_distance(it, p) }; + } + else { + // No need to call is_sorted_until(), we know + // the sorted range goes to the end. + r = { end, remaining }; + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(begin, r.first, l), ""); + } + } + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(begin, r.first, l), ""); + + // Store new start position + // and update remaining elements + + iters[cnt++] = begin = r.first; + remaining -= r.second; + cum_dist += r.second; + + } while (remaining != 0 && cnt < MaxIters); + + if (remaining == 0 && cnt > 1 && iters[cnt - 1] == iters[cnt - 2]) { + // Special case when remaining is 0: the last iterator might be equal to the previous one. + // We must remove it to avoid range of size 0 + --cnt; + } + + // Merge all sorted runs + merge_sorted_runs_with_buffer(iters, 0u, cnt - 1u, l, buf); + + // Update new min size if this is NOT the last chunk + if (remaining || new_min_size == std::numeric_limits::max()) { + new_min_size = std::min(new_min_size, cum_dist); + } + + last_start = iters[0]; + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(iters[0], iters[cnt - 1u], l), ""); + + } while (remaining != 0); + + // Compute end iterator during the first pass + if (min_size == 0) + end = begin; + + if (last_start != start) { + // We need a next pass if the number of iterators was not enough + return generic_merge_sort_internal(start, end, size, l, sort_sub_range, buf, new_min_size); + } + } + + template + static void merge_sort_internal(Iter begin, size_t size, Cmp l, Buffer buf) noexcept(NothrowSortIter::value) + { + // Bottom-up merge sort. + // Small chunks of up to 128 elements are sorted using insertion sort or sorting network and ping-pong merge. + // If possible, use wave sort on longer runs. + // Supports bidirectional iterators. + + if (size < 128) { + auto r = try_wave_sort<5>(begin, size, size, l, buf); + if (r.first == begin) + // Failed + sort_128(begin, (unsigned)size, l, buf); + return; + } + + generic_merge_sort_internal<65>( + begin, + begin, + size, + l, + [&](Iter b, size_t remaining, auto l) { + // Try wave sort first, as it might consume a lot more + // than the default 128 elements (possibly the whole sequence) + auto r = try_wave_sort<5>(b, remaining, std::min(remaining, (size_t)128u), l, buf); + if (r.first != b) { + // Success, retrieve new start position and number of sorted elements + return r; + } + + // Failure, use sort_128() + unsigned cnt = (unsigned)std::min(remaining, (size_t)128u); + auto it = sort_128(b, cnt, l, buf); + return std::make_pair(it, (size_t)cnt); + }, + buf); + + SEQ_ALGO_ASSERT_DEBUG(std::is_sorted(begin, iter_next(begin, size), l), ""); + } + + // Create the internal buffer used by net_sort() + template + static size_t sort_buffer_size(const Buffer& buf, size_t count) noexcept + { + if (buf.size == 0) + return (count); + if (buf.size == std::numeric_limits::max() - 1) + return (count / 8u); + if (buf.size == std::numeric_limits::max() - 2) + return (count / 32u); + return (count / 128u); + } + + } // end algo_detail + + /// @brief Buffer object used by net_sort(), net_sort_size() and inplace_merge() + template + struct buffer + { + Iter first; + size_t size = 0; + }; + + /// @brief Default sort buffer size, uses input size/2 bytes + static constexpr buffer default_buffer{ nullptr, 0 }; + + /// @brief Medium sort buffer size, uses input size/16 bytes + static constexpr buffer medium_buffer{ nullptr, std::numeric_limits::max() - 1u }; + + /// @brief Small sort buffer size, uses input size/64 bytes + static constexpr buffer small_buffer{ nullptr, std::numeric_limits::max() - 2u }; + + /// @brief Tiny sort buffer size, uses input size/256 bytes + static constexpr buffer tiny_buffer{ nullptr, std::numeric_limits::max() - 3u }; + + /// @brief Null buffer, uses (slow) bufferless merge sort + static constexpr buffer null_buffer{ nullptr, 0 }; + + /// @brief Stable merge algorithm similar to std::merge. + /// + /// This algorithm is usually more efficient than regular + /// std::merge, at least on msvc. It provides a better + /// handling of consecutive ordered values, and has a + /// special case for unbalanced merging (one range is + /// way smaller than the other). + /// + template> + SEQ_ALWAYS_INLINE Out merge(Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, Out out, Cmp c = Cmp()) + { + return algo_detail::merge_forward(first1, last1, first2, last2, out, c); + } + + /// @brief Inplace stable merge algorithm similar to std::inplace_merge. + /// + /// The main difference with std::inplace_merge is that this version + /// uses a user provided buffer for merging. This function only allocate + /// memory if provided buffer is one of 'default_buffer', 'medium_buffer', + /// 'small_buffer' or 'tiny_buffer'. 0 sized buffer are supported (but way slower). + /// + template, class Buffer = buffer> + void inplace_merge(Iter first, Iter middle, Iter last, Cmp c = Cmp(), Buffer buf = Buffer()) + { + using namespace algo_detail; + using Key = typename std::iterator_traits::value_type; + + if (middle == first || middle == last) + return; + + // Perform the firsts, easy checks + if (!c(*middle, *std::prev(middle))) + // One of the 2 ranges is empty, or already sorted + return; + + if (c(*std::prev(last), *first)) { + // Simple rotation needed + std::rotate(first, middle, last); + return; + } + + // The following checks come from msvc STL implementation + // and help a LOT in some situations + + // Increment first as long as it is smaller than first value of second range + for (;;) { + if (first == middle) + // We reach the end of first range: already in order + return; + if (c(*middle, *first)) + break; + ++first; + } + + auto highest = iter_prev(middle); + do { + --last; + if (middle == last) { // rotate only element remaining in right partition to the beginning, without allocating + rotate_one_right(first, middle, ++last); + return; + } + } while (!c(*last, *highest)); // found that *highest goes in *last's position + + ++last; + + // Now go through merge_adaptive_n + + auto s1 = (size_t)std::distance(first, middle); + auto s2 = (size_t)std::distance(middle, last); + + if constexpr (std::is_same_v, Buffer>) { + // Compute buffer size + size_t min_size = std::min(s1, s2); + size_t buf_size = sort_buffer_size(buf, min_size); + std::vector buf_(buf_size); + return merge_adaptive_n(wrap_iter(first), s1, wrap_iter(middle, s1), s2, wrap_iter(last, s1 + s2), c, buffer{ buf_.data(), buf_.size() }); + } + else + // Use provided buffer + return merge_adaptive_n(wrap_iter(first), s1, wrap_iter(middle, s1), s2, wrap_iter(last, s1 + s2), c, buf); + } + + /// @brief Inplace merge count-1 consecutive sorted ranges. + /// + /// The first argument is an array of count iterators representing count-1 consecutive sorted ranges. + /// + /// This function only allocate memory if provided buffer is one of 'default_buffer', 'medium_buffer', + /// 'small_buffer' or 'tiny_buffer'. 0 sized buffer are supported (but way slower). + /// + template, class Buffer = buffer> + void inplace_merge(Iter* iters, size_t count, Cmp c = Cmp(), Buffer buf = Buffer()) + { + using namespace algo_detail; + using Key = typename std::iterator_traits::value_type; + + if (count <= 2) + // Nothing to do + return; + + if constexpr (std::is_same_v, Buffer>) { + // Compute buffer size + size_t buf_size = sort_buffer_size(buf, std::distance(iters[0], iters[count - 1]) / 2); + std::vector buf_(buf_size); + merge_sorted_runs_with_buffer(iters, 0, count - 1, c, buffer{ buf_.data(), buf_.size() }); + } + else + // Use provided buffer + merge_sorted_runs_with_buffer(iters, 0, count - 1, c, buf); + } + + /// @brief Reverse a range already sorted in descending order while preserving stability. + /// + /// The full range is reversed except equal values which order is preserved. + /// + template> + SEQ_ALWAYS_INLINE void reverse_descending(Iter first, Iter last, Cmp c = Cmp()) + { + algo_detail::reverse_sort(first, last, c); + } + + /// @brief Stable merge sort algorithm using an external buffer. + /// + /// net sort is a merge sort algorithm with the following specificities: + /// + /// - Bottom-up merging instead of the more traditional top-down approach, + /// - Small blocks of 8 elements are sorted using a sorting network, + /// - Bidirectional merging is used for relocatable types, + /// - Ping-pong merge is used to merge 4 sorted ranges, + /// - Can work without allocating memory through a (potentially empty) user provided buffer, + /// - Works on bidirectional iterators. + /// + /// If provided buffer is one of 'seq::default_buffer', 'seq::medium_buffer', 'seq::small_buffer' + /// or 'seq::tiny_buffer', this function will try to allocate memory. + /// + /// From my tests on multiple input types, net_sort() is always faster than std::stable_sort(). + /// + /// net_sort_size() and net_sort() work on bidirectional iterators. + /// Using net_sort_size() instead of net_sort() is faster when the range size is already known. + /// + /// All credits to scandum (https://github.com/scandum) for its quadsort algorithm from which + /// I took several ideas (bidirectional merge and ping-pong merge). + /// + template, class Buffer = buffer> + void net_sort_size(Iter begin, size_t size, Cmp cmp = Cmp(), Buffer buf = Buffer()) + { + using namespace algo_detail; + using Key = typename std::iterator_traits::value_type; + using Cat = typename std::iterator_traits::iterator_category; + + static_assert(!std::is_same_v, "unsupported iterator category"); + static_assert(!std::is_same_v, "unsupported iterator category"); + static_assert(!std::is_same_v, "unsupported iterator category"); + + if (size < 16) { + insertion_sort_n(begin, (unsigned)size, cmp); + return; + } + + if constexpr (std::is_same_v, Buffer>) { + // Compute buffer size + size_t buf_size = sort_buffer_size(buf, size / 2); + if (buf_size < 16) + buf_size = 16; + std::vector buf_(buf_size); + return merge_sort_internal(wrap_iter(begin, 0), size, cmp, seq::buffer{ buf_.data(), buf_.size() }); + } + else { + // Use provided buffer + return merge_sort_internal(wrap_iter(begin, 0), size, cmp, buf); + } + } + + /// @brief Stable merge sort algorithm using an external buffer. + /// + /// net sort is a merge sort algorithm with the following specificities: + /// + /// - Bottom-up merging instead of the more traditional top-down approach, + /// - Small blocks of 8 elements are sorted using a sorting network, + /// - Bidirectional merging is used for relocatable types, + /// - Ping-pong merge is used to merge 4 sorted ranges, + /// - Can work without allocating memory through a (potentially null) user provided buffer, + /// - Works on bidirectional iterators. + /// + /// If provided buffer is one of 'seq::default_buffer', 'seq::medium_buffer', 'seq::small_buffer' + /// or 'seq::tiny_buffer', this function will try to allocate memory. + /// + /// From my tests on multiple input types, net_sort() is always faster than std::stable_sort(). + /// + /// net_sort_size() and net_sort() work on bidirectional iterators. + /// Using net_sort_size() instead of net_sort() is faster when the range size is already known. + /// + /// Full credits to scandum (https://github.com/scandum) for its quadsort algorithm from which + /// I took several ideas (bidirectional merge and ping-pong merge). + /// + template, class Buffer = buffer> + SEQ_ALWAYS_INLINE void net_sort(Iter begin, Iter end, Cmp cmp = Cmp(), Buffer buffer = Buffer()) + { + net_sort_size(begin, std::distance(begin, end), cmp, buffer); + } + +} // end namespace seq + +#endif diff --git a/seq/ordered_map.hpp b/seq/ordered_map.hpp index 7297ed9a..8eff7844 100644 --- a/seq/ordered_map.hpp +++ b/seq/ordered_map.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,8 +28,9 @@ /** @file */ #include "internal/hash_utils.hpp" +#include "internal/utils.hpp" + #include "sequence.hpp" -#include "utils.hpp" #include "hash.hpp" namespace seq @@ -188,7 +189,7 @@ namespace seq struct InsertPolicy { template - static auto emplace(Seq& s, K&& key, Args&&... args) + static SEQ_ALWAYS_INLINE auto emplace(Seq& s, K&& key, Args&&... args) { if constexpr (Loc == Front) return s.emplace_front_iter(std::forward(key), std::forward(args)...); @@ -202,7 +203,7 @@ namespace seq struct TryInsertPolicy { template - static auto emplace(Seq& s, K&& key, Args&&... args) -> typename Seq::iterator + static SEQ_ALWAYS_INLINE auto emplace(Seq& s, K&& key, Args&&... args) -> typename Seq::iterator { if constexpr (Loc == Front) return s.emplace_front_iter( @@ -337,7 +338,6 @@ namespace seq d_buckets[index] = node_type(node_type::small_hash(hash), dist, first.as_uint()); // Insert node - // if (dist) start_insert(d_buckets, new_hash_mask, index, dist, n); } @@ -424,12 +424,11 @@ namespace seq } else { if (dist > static_cast(d_max_dist)) - d_max_dist = static_cast(dist); // = (dist > node_type::max_distance) ? node_type::max_distance : dist; + d_max_dist = static_cast(dist); node_type n = node_type(static_cast(h), static_cast(dist), it.as_uint()); std::swap(n, d_buckets[index]); - if (dist) - start_insert(d_buckets, bsize, index, static_cast(dist), n); + start_insert(d_buckets, bsize, index, static_cast(dist), n); } ++it; } @@ -536,7 +535,6 @@ namespace seq noexcept(std::declval().swap(std::declval())) && noexcept(std::declval().swap(std::declval()))) { if (this != std::addressof(other)) { - std::swap(static_cast(*this), static_cast(other)); std::swap(d_buckets, other.d_buckets); std::swap(d_hash_mask, other.d_hash_mask); std::swap(d_hash_len, other.d_hash_len); @@ -639,7 +637,7 @@ namespace seq { // Key lookup - const bool robin_hood = d_max_dist < (node_type::max_distance); + const dist_type robin_hood = d_max_dist < (node_type::max_distance); const tiny_hash h = node_type::small_hash(hash); const auto* it = d_buckets + mask_hash(hash, d_hash_mask, d_hash_len); const auto* end = d_buckets + d_hash_mask; @@ -647,20 +645,18 @@ namespace seq // Combination of linear and robin-hood probing in the same loop. // An empty node has a distance of -1 and break the probe chain. - // A tombstone has a value 127 and never break the probe chain + // A tombstone has a value of 127 and never break the probe chain // (it is superior to node_type::max_distance which is 126, and dist is never incremented when tombstones are present: linear situation) // A tombstone value is never checked as its tiny hash is 0 (which is invalid). - if SEQ_UNLIKELY(!robin_hood) - check_hash_operation(); + check_hash_operation(); while (!(dist > it->distance())) { // Check for equality (first the hash part and then the key itself). if (h == it->hash() && (*this)(extract_key::key(sequence_node_value(*it)), key)) return const_iterator(it->node(), it->pos()); it = it == end ? d_buckets : it + 1; - if SEQ_LIKELY(robin_hood) - ++dist; + dist += robin_hood; } // Failed lookup return d_seq.end(); @@ -727,7 +723,6 @@ namespace seq // Move nodes based on distance (robin hood hashing), only if computed distance is not null node_type n = *it; *it = node_type(h, dist, tmp_it.as_uint()); - // if (dist ) start_insert(d_buckets, d_hash_mask, static_cast(it - d_buckets), dist, n); return std::pair(tmp_it, true); @@ -842,6 +837,7 @@ namespace seq return d_seq.erase(it); // return res; } auto erase(const_iterator it) -> iterator { return erase_hash(hash_key(extract_key::key(*it)), it); } + auto erase(iterator it) -> iterator { return erase_hash(hash_key(extract_key::key(*it)), it); } template auto erase(const K& key) -> size_t @@ -1075,7 +1071,7 @@ namespace seq /// @param hash hash function to use /// @param equal comparison function to use for all key comparisons of this container /// @param alloc allocator to use for all memory allocations of this container - template + template::value, int> = 0> ordered_set(InputIt first, InputIt last, const Hash& hash = Hash(), const key_equal& equal = key_equal(), const Allocator& alloc = Allocator()) : base_type(hash, equal, alloc) { @@ -1088,7 +1084,7 @@ namespace seq /// @param first the range to copy the elements from /// @param last the range to copy the elements from /// @param alloc allocator to use for all memory allocations of this container - template + template::value, int> = 0> ordered_set(InputIt first, InputIt last, const Allocator& alloc) : ordered_set(first, last, Hash(), key_equal(), alloc) { @@ -1574,18 +1570,18 @@ namespace seq { } - template + template::value, int> = 0> ordered_map(InputIt first, InputIt last, const Hash& hash = Hash(), const key_equal& equal = key_equal(), const Allocator& alloc = Allocator()) : base_type(hash, equal, alloc) { insert(first, last); } - template + template::value, int> = 0> ordered_map(InputIt first, InputIt last, const Allocator& alloc) : ordered_map(first, last, Hash(), key_equal(), alloc) { } - template + template::value, int> = 0> ordered_map(InputIt first, InputIt last, const Hash& hash, const Allocator& alloc) : ordered_map(first, last, hash, key_equal(), alloc) { @@ -1957,6 +1953,7 @@ namespace seq SEQ_ALWAYS_INLINE auto operator[](const Key& k) -> T& { return try_emplace(k).first->second; } SEQ_ALWAYS_INLINE auto operator[](Key&& k) -> T& { return try_emplace(std::move(k)).first->second; } + SEQ_ALWAYS_INLINE auto erase(iterator pos) -> iterator { return this->base_type::erase(pos); } SEQ_ALWAYS_INLINE auto erase(const_iterator pos) -> iterator { return this->base_type::erase(pos); } SEQ_ALWAYS_INLINE auto erase(const Key& key) -> size_type { return this->base_type::erase(key); } template::value && has_is_transparent::value>::type* = nullptr> diff --git a/seq/radix_hash_map.hpp b/seq/radix_hash_map.hpp index ecf36a8e..0f113b9f 100644 --- a/seq/radix_hash_map.hpp +++ b/seq/radix_hash_map.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,7 +32,7 @@ */ #include "internal/radix_tree.hpp" -#include "utils.hpp" +#include "internal/utils.hpp" namespace seq { @@ -145,7 +145,7 @@ namespace seq /// @param hash hash function to use /// @param equal comparison function to use for all key comparisons of this container /// @param alloc allocator to use for all memory allocations of this container - template + template::value, int> = 0> radix_hash_set(InputIt first, InputIt last, const Hash& hash = Hash(), const key_equal& eq = key_equal(), const Allocator& alloc = Allocator()) : d_tree(radix_hash(0, hash, eq), alloc) { @@ -157,7 +157,7 @@ namespace seq /// @param first the range to copy the elements from /// @param last the range to copy the elements from /// @param alloc allocator to use for all memory allocations of this container - template + template::value, int> = 0> radix_hash_set(InputIt first, InputIt last, const Allocator& alloc) : d_tree(first, last, alloc) { @@ -169,7 +169,7 @@ namespace seq /// @param last the range to copy the elements from /// @param hash hash function to use /// @param alloc allocator to use for all memory allocations of this container - template + template::value, int> = 0> radix_hash_set(InputIt first, InputIt last, const Hash& hash, const Allocator& alloc) : d_tree(radix_hash(0, hash), alloc) { @@ -628,19 +628,19 @@ namespace seq { } - template + template::value, int> = 0> radix_hash_map(InputIt first, InputIt last, const Hash& hash = Hash(), const key_equal& eq = key_equal(), const Allocator& alloc = Allocator()) : d_tree(radix_hash(0, hash, eq), alloc) { insert(first, last); } - template + template::value, int> = 0> radix_hash_map(InputIt first, InputIt last, const Allocator& alloc) : d_tree(alloc) { insert(first, last); } - template + template::value, int> = 0> radix_hash_map(InputIt first, InputIt last, const Hash& hash, const Allocator& alloc) : radix_hash_map(first, last, hash, key_equal(), alloc) { @@ -836,6 +836,7 @@ namespace seq SEQ_ALWAYS_INLINE auto operator[](const Key& key) -> T& { return try_emplace(key).first->second; } SEQ_ALWAYS_INLINE auto operator[](Key&& key) -> T& { return try_emplace(std::move(key)).first->second; } + SEQ_ALWAYS_INLINE auto erase(iterator pos) -> iterator { return d_tree.erase(pos.iter); } SEQ_ALWAYS_INLINE auto erase(const_iterator pos) -> iterator { return d_tree.erase(pos.iter); } SEQ_ALWAYS_INLINE auto erase(const Key& key) -> size_type { return d_tree.erase(key); } template::value && has_is_transparent::value>::type* = nullptr> diff --git a/seq/radix_map.hpp b/seq/radix_map.hpp index e4ef92c1..11c5ecd6 100644 --- a/seq/radix_map.hpp +++ b/seq/radix_map.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,120 +32,153 @@ */ #include "internal/radix_tree.hpp" -#include "utils.hpp" +#include "internal/utils.hpp" namespace seq { - - /// @brief Radix based sorted container using Variable Arity Radix Tree (VART). Same interface as std::set. - /// @tparam Key key type - /// @tparam ExtractKey Functor extracting a suitable key for the radix tree - /// @tparam Allocator allocator - template> - class radix_set + namespace detail { - using radix_key_type = typename radix_detail::ExtractKeyResultType::type; - struct Extract + template + class radix_const_iterator { - SEQ_ALWAYS_INLINE const radix_key_type& operator()(const radix_key_type& p) const noexcept { return p; } - template - SEQ_ALWAYS_INLINE auto operator()(const U& p) const noexcept - { - return ExtractKey{}(p); - } - }; - - using Policy = detail::BuildValue; - using radix_tree_type = radix_detail::RadixTree, Extract, Allocator, radix_detail::LeafNode>; - radix_tree_type d_tree; + protected: + Iter d_iter; - public: - /// @brief const iterator class - struct const_iterator - { - using iter_type = typename radix_tree_type::const_iterator; - using value_type = Key; + public: + using value_type = typename Iter::value_type; using iterator_category = std::bidirectional_iterator_tag; using size_type = size_t; using difference_type = std::ptrdiff_t; using pointer = const value_type*; using reference = const value_type&; - iter_type iter; + using const_pointer = const value_type*; + using const_reference = const value_type&; - const_iterator() {} - const_iterator(const iter_type& it) - : iter(it) - { - } - const_iterator(iter_type&& it) - : iter(std::move(it)) + const Iter& base_iter() const noexcept { return d_iter; } + + radix_const_iterator() noexcept = default; + radix_const_iterator(const radix_const_iterator&) noexcept = default; + radix_const_iterator(const Iter& it) noexcept + : d_iter(it) { } - SEQ_ALWAYS_INLINE auto operator++() noexcept -> const_iterator& + SEQ_ALWAYS_INLINE auto operator++() noexcept -> radix_const_iterator& { - ++iter; + ++d_iter; return *this; } - SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> const_iterator + SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> radix_const_iterator { - const_iterator _Tmp = *this; + radix_const_iterator _Tmp = *this; ++(*this); return _Tmp; } - SEQ_ALWAYS_INLINE auto operator--() noexcept -> const_iterator& + SEQ_ALWAYS_INLINE auto operator--() noexcept -> radix_const_iterator& { - --iter; + --d_iter; return *this; } - SEQ_ALWAYS_INLINE auto operator--(int) noexcept -> const_iterator + SEQ_ALWAYS_INLINE auto operator--(int) noexcept -> radix_const_iterator { - const_iterator _Tmp = *this; + radix_const_iterator _Tmp = *this; --(*this); return _Tmp; } - SEQ_ALWAYS_INLINE auto operator*() const noexcept -> reference { return *iter; } + SEQ_ALWAYS_INLINE auto operator*() const noexcept -> reference { return reinterpret_cast(*d_iter); } SEQ_ALWAYS_INLINE auto operator->() const noexcept -> pointer { return std::pointer_traits::pointer_to(**this); } - SEQ_ALWAYS_INLINE auto operator==(const const_iterator& it) const noexcept -> bool { return iter == it.iter; } - SEQ_ALWAYS_INLINE auto operator!=(const const_iterator& it) const noexcept -> bool { return iter != it.iter; } + SEQ_ALWAYS_INLINE friend bool operator==(const radix_const_iterator& l, const radix_const_iterator& r) noexcept { return l.d_iter == r.d_iter; } + SEQ_ALWAYS_INLINE friend bool operator!=(const radix_const_iterator& l, const radix_const_iterator& r) noexcept { return l.d_iter != r.d_iter; } + }; - using iterator = const_iterator; - /// @brief const iterator class for prefix search - struct const_prefix_iterator + template + bool check_bit_pos(const radix_const_iterator& it) { - using iter_type = typename radix_tree_type::const_prefix_iterator; - using value_type = Key; - using iterator_category = std::forward_iterator_tag; + // For tests only + return it.base_iter().bit_pos == it.base_iter().get_bit_pos(it.base_iter().dir); + } + + template + struct radix_iterator : radix_const_iterator + { + using base_type = radix_const_iterator; + using value_type = typename Iter::value_type; + ; + using iterator_category = std::bidirectional_iterator_tag; using size_type = size_t; using difference_type = std::ptrdiff_t; - using pointer = const value_type*; - using reference = const value_type&; - iter_type iter; + using pointer = value_type*; + using reference = value_type&; + using const_pointer = const value_type*; + using const_reference = const value_type&; - const_prefix_iterator() {} - const_prefix_iterator(const iter_type& it) - : iter(it) + radix_iterator() noexcept = default; + radix_iterator(const base_type& other) noexcept + : base_type(other) { } - SEQ_ALWAYS_INLINE auto operator++() noexcept -> const_prefix_iterator& + radix_iterator(const Iter& it) noexcept + : base_type(it) { - ++iter; + } + SEQ_ALWAYS_INLINE auto operator++() noexcept -> radix_iterator& + { + ++this->d_iter; return *this; } - SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> const_prefix_iterator + SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> radix_iterator { - const_prefix_iterator _Tmp = *this; + radix_iterator ret = *this; ++(*this); - return _Tmp; + return ret; } - SEQ_ALWAYS_INLINE auto operator*() const noexcept -> reference { return *iter; } - SEQ_ALWAYS_INLINE auto operator->() const noexcept -> pointer { return std::pointer_traits::pointer_to(**this); } + SEQ_ALWAYS_INLINE auto operator--() noexcept -> radix_iterator& + { + --this->d_iter; + return *this; + } + SEQ_ALWAYS_INLINE auto operator--(int) noexcept -> radix_iterator + { + radix_iterator ret = *this; + --(*this); + return ret; + } + SEQ_ALWAYS_INLINE auto operator*() noexcept -> reference { return const_cast(base_type::operator*()); } + SEQ_ALWAYS_INLINE auto operator->() noexcept -> pointer { return std::pointer_traits::pointer_to(**this); } + SEQ_ALWAYS_INLINE auto operator*() const noexcept -> reference { return const_cast(base_type::operator*()); } + SEQ_ALWAYS_INLINE auto operator->() const noexcept -> const_pointer { return std::pointer_traits::pointer_to(**this); } + }; + + } - SEQ_ALWAYS_INLINE auto operator==(const const_prefix_iterator& it) const noexcept -> bool { return iter == it.iter; } - SEQ_ALWAYS_INLINE auto operator!=(const const_prefix_iterator& it) const noexcept -> bool { return iter != it.iter; } + /// @brief Radix based sorted container using Variable Arity Radix Tree (VART). Same interface as std::set. + /// @tparam Key key type + /// @tparam ExtractKey Functor extracting a suitable key for the radix tree + /// @tparam Allocator allocator + template> + class radix_set + { + using radix_key_type = typename radix_detail::ExtractKeyResultType::type; + struct Extract + { + SEQ_ALWAYS_INLINE const radix_key_type& operator()(const radix_key_type& p) const noexcept { return p; } + template + SEQ_ALWAYS_INLINE auto operator()(const U& p) const noexcept + { + return ExtractKey{}(p); + } }; - using prefix_iterator = const_prefix_iterator; + + using Policy = detail::BuildValue; + using radix_tree_type = radix_detail::RadixTree, Extract, Allocator, radix_detail::LeafNode>; + using tree_iterator = typename radix_tree_type::iterator; + + radix_tree_type d_tree; + + public: + using const_iterator = detail::radix_const_iterator; + using iterator = detail::radix_iterator; using key_type = Key; using value_type = Key; @@ -177,7 +210,7 @@ namespace seq /// @param first the range to copy the elements from /// @param last the range to copy the elements from /// @param alloc allocator to use for all memory allocations of this container - template + template::value, int> = 0> radix_set(InputIt first, InputIt last, const Allocator& alloc = Allocator()) : radix_set(alloc) { @@ -278,13 +311,13 @@ namespace seq template SEQ_ALWAYS_INLINE auto emplace_hint(const_iterator hint, Args&&... args) -> iterator { - return d_tree.emplace_hint(hint.iter, Policy::make(std::forward(args)...)); + return d_tree.emplace_hint(hint.base_iter(), Policy::make(std::forward(args)...)); } /// @brief Same as std::set::insert() - SEQ_ALWAYS_INLINE auto insert(const_iterator hint, const value_type& value) -> iterator { return d_tree.emplace_hint(hint.iter, value); } + SEQ_ALWAYS_INLINE auto insert(const_iterator hint, const value_type& value) -> iterator { return d_tree.emplace_hint(hint.base_iter(), value); } /// @brief Same as std::set::insert() - SEQ_ALWAYS_INLINE auto insert(const_iterator hint, value_type&& value) -> iterator { return d_tree.emplace_hint(hint.iter, std::move(value)); } + SEQ_ALWAYS_INLINE auto insert(const_iterator hint, value_type&& value) -> iterator { return d_tree.emplace_hint(hint.base_iter(), std::move(value)); } /// @brief Inserts elements from range [first, last). If multiple elements in the range have keys that compare equivalent, /// it is unspecified which element is inserted, except if template parameter Stable is true. @@ -309,9 +342,10 @@ namespace seq } /// @brief Erase element at given location. - SEQ_ALWAYS_INLINE auto erase(const_iterator pos) -> iterator { return d_tree.erase(pos.iter); } + SEQ_ALWAYS_INLINE auto erase(iterator pos) -> iterator { return d_tree.erase(pos.base_iter()); } + SEQ_ALWAYS_INLINE auto erase(const_iterator pos) -> iterator { return d_tree.erase(pos.base_iter()); } /// @brief Erase elements in the range [first, last) - SEQ_ALWAYS_INLINE auto erase(const_iterator first, const_iterator last) -> iterator { return d_tree.erase(first.iter, last.iter); } + SEQ_ALWAYS_INLINE auto erase(const_iterator first, const_iterator last) -> iterator { return d_tree.erase(first.base_iter(), last.base_iter()); } /// @brief Removes the element (if one exists) with key that compares equivalent to the value key. /// @param key key value of the elements to remove @@ -330,12 +364,22 @@ namespace seq { return d_tree.find(key); } + template + SEQ_ALWAYS_INLINE auto find(const K& key) -> iterator + { + return d_tree.find(key); + } template SEQ_ALWAYS_INLINE auto find_ptr(const K& key) const -> const value_type* { return d_tree.find_ptr(key); } + template + SEQ_ALWAYS_INLINE auto find_ptr(const K& key) -> value_type* + { + return (value_type*)d_tree.find_ptr(key); + } /// @brief Returns an iterator pointing to the first element that compares not less (i.e. greater or equal) to the value key. template @@ -343,6 +387,11 @@ namespace seq { return d_tree.lower_bound(key); } + template + SEQ_ALWAYS_INLINE auto lower_bound(const K& key) -> iterator + { + return d_tree.lower_bound(key); + } /// @brief Returns an iterator pointing to the first element that compares greater to the value key. template @@ -350,18 +399,34 @@ namespace seq { return d_tree.upper_bound(key); } + template + SEQ_ALWAYS_INLINE auto upper_bound(const K& key) -> iterator + { + return d_tree.upper_bound(key); + } template SEQ_ALWAYS_INLINE auto prefix(const K& key) const -> const_iterator { return d_tree.prefix(key); } + template + SEQ_ALWAYS_INLINE auto prefix(const K& key) -> iterator + { + return d_tree.prefix(key); + } template - SEQ_ALWAYS_INLINE auto prefix_range(const K& key) const -> std::pair + SEQ_ALWAYS_INLINE auto prefix_range(const K& key) const -> std::pair { - auto p = d_tree.prefix_range(key); - return { p.first, p.second }; + auto r = d_tree.prefix_range(key); + return { (r.first), (r.second) }; + } + template + SEQ_ALWAYS_INLINE auto prefix_range(const K& key) -> std::pair + { + auto r = d_tree.prefix_range(key); + return { (r.first), (r.second) }; } /// @brief Checks if there is an element with key that compares equivalent to the value key. @@ -382,7 +447,16 @@ namespace seq /// The range is defined by two iterators, one pointing to the first element that is not less than key and another pointing to the first element greater than key. /// Alternatively, the first iterator may be obtained with lower_bound(), and the second with upper_bound(). template - SEQ_ALWAYS_INLINE auto equal_range(const K& key) const -> std::pair + SEQ_ALWAYS_INLINE auto equal_range(const K& key) const -> std::pair + { + auto it = d_tree.find(key); + if (it == d_tree.end()) + return std::pair(d_tree.end(), d_tree.end()); + auto start = it++; + return std::pair(start, it); + } + template + SEQ_ALWAYS_INLINE auto equal_range(const K& key) -> std::pair { auto it = d_tree.find(key); if (it == d_tree.end()) @@ -494,195 +568,11 @@ namespace seq using Policy = detail::BuildValue, true>; using radix_tree_type = radix_detail::RadixTree, radix_detail::RadixHasher, Extract, Allocator>; + using tree_iterator = typename radix_tree_type::const_iterator; + radix_tree_type d_tree; public: - struct const_iterator - { - using iter_type = typename radix_tree_type::const_iterator; - using value_type = std::pair; - using iterator_category = std::bidirectional_iterator_tag; - using size_type = size_t; - using difference_type = std::ptrdiff_t; - using pointer = const value_type*; - using reference = const value_type&; - using const_pointer = const value_type*; - using const_reference = const value_type&; - iter_type iter; - - const_iterator() {} - const_iterator(const iter_type& it) - : iter(it) - { - } - const_iterator(iter_type&& it) - : iter(std::move(it)) - { - } - SEQ_ALWAYS_INLINE auto operator++() noexcept -> const_iterator& - { - ++iter; - return *this; - } - SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> const_iterator - { - const_iterator _Tmp = *this; - ++(*this); - return _Tmp; - } - SEQ_ALWAYS_INLINE auto operator--() noexcept -> const_iterator& - { - --iter; - return *this; - } - SEQ_ALWAYS_INLINE auto operator--(int) noexcept -> const_iterator - { - const_iterator _Tmp = *this; - --(*this); - return _Tmp; - } - SEQ_ALWAYS_INLINE auto operator*() const noexcept -> reference { return reinterpret_cast(*iter); } - SEQ_ALWAYS_INLINE auto operator->() const noexcept -> pointer { return std::pointer_traits::pointer_to(**this); } - - SEQ_ALWAYS_INLINE auto operator==(const const_iterator& it) const noexcept -> bool { return iter == it.iter; } - SEQ_ALWAYS_INLINE auto operator!=(const const_iterator& it) const noexcept -> bool { return iter != it.iter; } - }; - - struct iterator : public const_iterator - { - using iter_type = typename radix_tree_type::const_iterator; - using value_type = std::pair; - using iterator_category = std::bidirectional_iterator_tag; - using size_type = size_t; - using difference_type = std::ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - using const_pointer = const value_type*; - using const_reference = const value_type&; - - iterator() - : const_iterator() - { - } - iterator(const iter_type& it) - : const_iterator(it) - { - } - iterator(iter_type&& it) - : const_iterator(std::move(it)) - { - } - iterator(const const_iterator& it) - : const_iterator(it) - { - } - SEQ_ALWAYS_INLINE auto operator++() noexcept -> iterator& - { - ++this->iter; - return *this; - } - SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> iterator - { - iterator _Tmp = *this; - ++(*this); - return _Tmp; - } - SEQ_ALWAYS_INLINE auto operator--() noexcept -> iterator& - { - --this->iter; - return *this; - } - SEQ_ALWAYS_INLINE auto operator--(int) noexcept -> iterator - { - iterator _Tmp = *this; - --(*this); - return _Tmp; - } - SEQ_ALWAYS_INLINE auto operator*() noexcept -> reference { return reinterpret_cast(const_cast&>(*this->iter)); } - SEQ_ALWAYS_INLINE auto operator->() noexcept -> pointer { return std::pointer_traits::pointer_to(**this); } - SEQ_ALWAYS_INLINE auto operator*() const noexcept -> const_reference { return reinterpret_cast(*this->iter); } - SEQ_ALWAYS_INLINE auto operator->() const noexcept -> const_pointer { return std::pointer_traits::pointer_to(**this); } - - SEQ_ALWAYS_INLINE auto operator==(const const_iterator& it) const noexcept -> bool { return this->iter == it.iter; } - SEQ_ALWAYS_INLINE auto operator!=(const const_iterator& it) const noexcept -> bool { return this->iter != it.iter; } - }; - - /// @brief const iterator class for prefix search - struct const_prefix_iterator - { - using iter_type = typename radix_tree_type::const_prefix_iterator; - using value_type = std::pair; - using iterator_category = std::forward_iterator_tag; - using size_type = size_t; - using difference_type = std::ptrdiff_t; - using pointer = const value_type*; - using reference = const value_type&; - using const_pointer = const value_type*; - using const_reference = const value_type&; - iter_type iter; - - const_prefix_iterator() {} - const_prefix_iterator(const iter_type& it) - : iter(it) - { - } - SEQ_ALWAYS_INLINE auto operator++() noexcept -> const_prefix_iterator& - { - ++iter; - return *this; - } - SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> const_prefix_iterator - { - const_prefix_iterator _Tmp = *this; - ++(*this); - return _Tmp; - } - SEQ_ALWAYS_INLINE auto operator*() const noexcept -> reference { return *iter; } - SEQ_ALWAYS_INLINE auto operator->() const noexcept -> pointer { return std::pointer_traits::pointer_to(**this); } - - SEQ_ALWAYS_INLINE auto operator==(const const_prefix_iterator& it) const noexcept -> bool { return iter == it.iter; } - SEQ_ALWAYS_INLINE auto operator!=(const const_prefix_iterator& it) const noexcept -> bool { return iter != it.iter; } - }; - - struct prefix_iterator : public const_prefix_iterator - { - using iter_type = typename radix_tree_type::const_prefix_iterator; - using value_type = std::pair; - using iterator_category = std::forward_iterator_tag; - using size_type = size_t; - using difference_type = std::ptrdiff_t; - using pointer = value_type*; - using reference = value_type&; - using const_pointer = const value_type*; - using const_reference = const value_type&; - - prefix_iterator() - : const_prefix_iterator() - { - } - prefix_iterator(const iter_type& it) - : const_prefix_iterator(it) - { - } - SEQ_ALWAYS_INLINE auto operator++() noexcept -> prefix_iterator& - { - ++this->iter; - return *this; - } - SEQ_ALWAYS_INLINE auto operator++(int) noexcept -> prefix_iterator - { - prefix_iterator _Tmp = *this; - ++(*this); - return _Tmp; - } - SEQ_ALWAYS_INLINE auto operator*() noexcept -> reference { return reinterpret_cast(const_cast&>(*this->iter)); } - SEQ_ALWAYS_INLINE auto operator->() noexcept -> pointer { return std::pointer_traits::pointer_to(**this); } - SEQ_ALWAYS_INLINE auto operator*() const noexcept -> const_reference { return reinterpret_cast(*this->iter); } - SEQ_ALWAYS_INLINE auto operator->() const noexcept -> const_pointer { return std::pointer_traits::pointer_to(**this); } - SEQ_ALWAYS_INLINE auto operator==(const const_prefix_iterator& it) const noexcept -> bool { return this->iter == it.iter; } - SEQ_ALWAYS_INLINE auto operator!=(const const_prefix_iterator& it) const noexcept -> bool { return this->iter != it.iter; } - }; - using key_type = Key; using mapped_type = T; using value_type = std::pair; @@ -695,6 +585,8 @@ namespace seq using const_reference = const value_type&; using pointer = typename std::allocator_traits::pointer; using const_pointer = typename std::allocator_traits::const_pointer; + using const_iterator = detail::radix_const_iterator; + using iterator = detail::radix_iterator; using reverse_iterator = std::reverse_iterator; using const_reverse_iterator = std::reverse_iterator; @@ -703,16 +595,11 @@ namespace seq : d_tree(alloc) { } - template + template::value, int> = 0> radix_map(InputIt first, InputIt last, const Allocator& alloc = Allocator()) : d_tree(first, last, alloc) { } - template - radix_map(const radix_map& other) - : d_tree(other.d_tree) - { - } radix_map(const radix_map& other, const Allocator& alloc) : d_tree(other.d_tree, alloc) @@ -767,12 +654,12 @@ namespace seq { return d_tree.emplace(Policy::make(std::forward

(value))); } - SEQ_ALWAYS_INLINE auto insert(const_iterator hint, const value_type& value) -> iterator { return d_tree.emplace_hint(hint.iter, value); } - SEQ_ALWAYS_INLINE auto insert(const_iterator hint, value_type&& value) -> iterator { return d_tree.emplace_hint(hint.iter, std::move(value)); } + SEQ_ALWAYS_INLINE auto insert(const_iterator hint, const value_type& value) -> iterator { return d_tree.emplace_hint(hint.base_iter(), value); } + SEQ_ALWAYS_INLINE auto insert(const_iterator hint, value_type&& value) -> iterator { return d_tree.emplace_hint(hint.base_iter(), std::move(value)); } template, int>::type = 0> auto insert(const_iterator hint, P&& value) -> iterator { - return d_tree.emplace_hint(hint.iter, Policy::make(std::forward

(value))); + return d_tree.emplace_hint(hint.base_iter(), Policy::make(std::forward

(value))); } template @@ -783,7 +670,7 @@ namespace seq template SEQ_ALWAYS_INLINE auto emplace_hint(const_iterator hint, Args&&... args) -> iterator { - return d_tree.emplace_hint(hint.iter, Policy::make(std::forward(args)...)); + return d_tree.emplace_hint(hint.base_iter(), Policy::make(std::forward(args)...)); } template @@ -799,12 +686,12 @@ namespace seq template SEQ_ALWAYS_INLINE auto try_emplace(const_iterator hint, const Key& k, Args&&... args) -> iterator { - return d_tree.try_emplace_hint(hint.iter, k, std::forward(args)...).first; + return d_tree.try_emplace_hint(hint.base_iter(), k, std::forward(args)...).first; } template SEQ_ALWAYS_INLINE auto try_emplace(const_iterator hint, Key&& k, Args&&... args) -> iterator { - return d_tree.try_emplace_hint(hint.iter, std::move(k), std::forward(args)...).first; + return d_tree.try_emplace_hint(hint.base_iter(), std::move(k), std::forward(args)...).first; } template @@ -827,7 +714,7 @@ namespace seq template SEQ_ALWAYS_INLINE auto insert_or_assign(const_iterator hint, const Key& k, M&& obj) -> iterator { - auto inserted = d_tree.try_emplace_hint(hint.iter, k, std::forward(obj)); + auto inserted = d_tree.try_emplace_hint(hint.base_iter(), k, std::forward(obj)); if (!inserted.second) inserted.first->second = std::forward(obj); return inserted.first; @@ -835,7 +722,7 @@ namespace seq template SEQ_ALWAYS_INLINE auto insert_or_assign(const_iterator hint, Key&& k, M&& obj) -> iterator { - auto inserted = d_tree.try_emplace_hint(hint.iter, std::move(k), std::forward(obj)); + auto inserted = d_tree.try_emplace_hint(hint.base_iter(), std::move(k), std::forward(obj)); if (!inserted.second) inserted.first->second = std::forward(obj); return inserted.first; @@ -872,9 +759,9 @@ namespace seq d_tree.assign(first, last); } - SEQ_ALWAYS_INLINE auto erase(iterator pos) -> iterator { return d_tree.erase(pos.iter); } - SEQ_ALWAYS_INLINE auto erase(const_iterator pos) -> iterator { return d_tree.erase(pos.iter); } - SEQ_ALWAYS_INLINE auto erase(const_iterator first, const_iterator last) -> iterator { return d_tree.erase(first.iter, last.iter); } + SEQ_ALWAYS_INLINE auto erase(iterator pos) -> iterator { return d_tree.erase(pos.base_iter()); } + SEQ_ALWAYS_INLINE auto erase(const_iterator pos) -> iterator { return d_tree.erase(pos.base_iter()); } + SEQ_ALWAYS_INLINE auto erase(const_iterator first, const_iterator last) -> iterator { return d_tree.erase(first.base_iter(), last.base_iter()); } template SEQ_ALWAYS_INLINE auto erase(const K& key) -> size_type @@ -941,13 +828,13 @@ namespace seq } template - SEQ_ALWAYS_INLINE auto prefix_range(const K& x) -> std::pair + SEQ_ALWAYS_INLINE auto prefix_range(const K& x) -> std::pair { auto p = d_tree.prefix_range(x); return { p.first, p.second }; } template - SEQ_ALWAYS_INLINE auto prefix_range(const K& x) const -> std::pair + SEQ_ALWAYS_INLINE auto prefix_range(const K& x) const -> std::pair { auto p = d_tree.prefix_range(x); return { p.first, p.second }; diff --git a/seq/seq_config.hpp b/seq/seq_config.hpp index 29a6a750..0172062f 100644 --- a/seq/seq_config.hpp +++ b/seq/seq_config.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/seq/sequence.hpp b/seq/sequence.hpp index c5250054..e50240fd 100644 --- a/seq/sequence.hpp +++ b/seq/sequence.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,8 +31,8 @@ #include #include "type_traits.hpp" -#include "utils.hpp" -#include "algorithm.hpp" +#include "internal/utils.hpp" +#include "net_sort.hpp" namespace seq { @@ -122,18 +122,18 @@ namespace seq // Standard list_chunk allocator for OptimizeForMemory // Keep track of the list_chunk number for fast memory footprint template, bool Align64 = false> - struct std_alloc : private Allocator + struct ChunkAlloc : private Allocator { template using rebind_alloc = typename std::allocator_traits::template rebind_alloc; size_t chunks = 0; - std_alloc(const Allocator& alloc) noexcept(std::is_nothrow_copy_constructible_v) + ChunkAlloc(const Allocator& alloc) noexcept(std::is_nothrow_copy_constructible_v) : Allocator(alloc) , chunks(0) { } - std_alloc(size_t /*unused*/, const Allocator& alloc) noexcept + ChunkAlloc(size_t /*unused*/, const Allocator& alloc) noexcept : Allocator(alloc) , chunks(0) { @@ -754,7 +754,7 @@ namespace seq template using rebind_alloc = typename std::allocator_traits::template rebind_alloc; using chunk_type = detail::list_chunk; - using layout_manager = detail::std_alloc; + using layout_manager = detail::ChunkAlloc; static constexpr std::uint64_t count = detail::list_chunk::count; static constexpr std::uint64_t count1 = detail::list_chunk::count - 1; static constexpr std::uint64_t count_bits = detail::list_chunk::count_bits; @@ -1340,19 +1340,12 @@ namespace seq // Sequence object internal data Data* d_data; - auto make_data(const Allocator& al) -> Data* - { - rebind_alloc a = al; - Data* d = a.allocate(1); - construct_ptr(d, al); - return d; - } + auto make_data(const Allocator& al) -> Data* { return new (allocate_from(al)) Data(al); } void destroy_data(Data* d) { if (d) { - rebind_alloc a = get_allocator(); - destroy_ptr(d); - a.deallocate(d, 1); + d->~Data(); + deallocate_from(get_allocator(), d); } } @@ -1444,7 +1437,7 @@ namespace seq /// @param first begin iterator /// @param last end iterator /// @param al allocator object - template + template::value, int> = 0> sequence(Iter first, Iter last, const Allocator& al = Allocator()) : Allocator(al) , d_data(make_data(al)) @@ -2124,7 +2117,6 @@ namespace seq /// @param first iterator to the element to remove /// @param last iterator to the element to remove /// @return Iterator following the last removed element - /// This function performs in O(1). /// Iterators and references to the erased elements are invalidated. /// Iterators and references to other elements in the sequence remain valid. auto erase(const_iterator first, const_iterator last) noexcept -> iterator @@ -2174,9 +2166,8 @@ namespace seq return res; } - /// @brief Sort the sequence. + /// @brief Sort the sequence inplace and in a stable way. /// The sequence is sorted using the std::less comparator. - /// sort() relies on the pdqsort implementation from Orson Peters, and should be as fast as sorting a std::deque. /// This invalidates all iterators and references. void sort() { sort(std::less()); } diff --git a/seq/tiered_vector.hpp b/seq/tiered_vector.hpp index c1b298ad..4bdec5ee 100644 --- a/seq/tiered_vector.hpp +++ b/seq/tiered_vector.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -25,12 +25,11 @@ #ifndef SEQ_TIERED_VECTOR_HPP #define SEQ_TIERED_VECTOR_HPP +#include "internal/utils.hpp" #include "devector.hpp" -#include "utils.hpp" #include "tiny_string.hpp" -#include "algorithm.hpp" +#include "net_sort.hpp" -#include #include #include @@ -51,6 +50,7 @@ namespace seq using cbuffer_pos = int; // index type within circular buffer, must be signed + template struct deque_const_iterator { @@ -69,13 +69,13 @@ namespace seq SEQ_ALWAYS_INLINE std::pair update(size_type p) const noexcept { - if (mgr) + if SEQ_LIKELY(mgr) return mgr->indexes(p); return { 0, 0 }; } - std::pair update() noexcept { return update(pos); } + SEQ_ALWAYS_INLINE std::pair update() noexcept { return update(pos); } - SEQ_ALWAYS_INLINE deque_const_iterator() noexcept {} + SEQ_ALWAYS_INLINE deque_const_iterator() noexcept = default; SEQ_ALWAYS_INLINE explicit deque_const_iterator(const bucket_manager* d) noexcept : mgr(const_cast(d)) , pos(d ? d->size() : 0) @@ -97,13 +97,11 @@ namespace seq } // any pos deque_const_iterator(const deque_const_iterator&) noexcept = default; - deque_const_iterator(deque_const_iterator&&) noexcept = default; deque_const_iterator& operator=(const deque_const_iterator&) noexcept = default; - deque_const_iterator& operator=(deque_const_iterator&&) noexcept = default; SEQ_ALWAYS_INLINE auto absolutePos() const noexcept -> size_type { return pos; } - void setPos(size_t new_pos) noexcept + SEQ_ALWAYS_INLINE void setPos(size_t new_pos) noexcept { SEQ_ASSERT_DEBUG(new_pos <= static_cast(mgr->d_size), "invalid iterator position"); pos = new_pos; @@ -362,7 +360,7 @@ namespace seq chunk_type* node = nullptr; int pos = 0; - tvector_ra_iterator() noexcept {} + tvector_ra_iterator() noexcept = default; SEQ_ALWAYS_INLINE tvector_ra_iterator(const BucketMgr* d) noexcept // begin : data(const_cast(d)) , node(const_cast(d->d_buckets.data())) @@ -551,13 +549,11 @@ namespace seq // Element access. SEQ_ALWAYS_INLINE auto operator[](cbuffer_pos index) noexcept -> T& { - // SEQ_ASSERT_DEBUG(index >= 0, "Invalid index"); SEQ_ASSERT_DEBUG(!(index >= max_size() && begin == 0), "Invalid index"); return buffer()[((begin + index) & (max_size1))]; } SEQ_ALWAYS_INLINE auto operator[](cbuffer_pos index) const noexcept -> const T& { - // SEQ_ASSERT_DEBUG(index >= 0, "Invalid index"); SEQ_ASSERT_DEBUG(!(index >= max_size() && begin == 0), "Invalid index"); return buffer()[((begin + index) & (max_size1))]; } @@ -1722,8 +1718,8 @@ namespace seq } // Returns true is tiered_vector size is even - SEQ_ALWAYS_INLINE auto isPow2Size() const noexcept -> bool { return isPow2Size(d_size); } - SEQ_ALWAYS_INLINE auto isPow2Size(size_t s) const noexcept -> bool { return ((s - 1) & s) == 0; } + SEQ_ALWAYS_INLINE auto is_pow_2() const noexcept -> bool { return is_pow_2(d_size); } + SEQ_ALWAYS_INLINE auto is_pow_2(size_t s) const noexcept -> bool { return ((s - 1) & s) == 0; } // Returns allocator SEQ_ALWAYS_INLINE auto get_allocator() const noexcept -> const Allocator& { return *this; } SEQ_ALWAYS_INLINE auto get_allocator() noexcept -> Allocator& { return *this; } @@ -1793,8 +1789,9 @@ namespace seq return d_buckets.front()->front(); } - SEQ_ALWAYS_INLINE std::pair indexes(size_type pos) + SEQ_ALWAYS_INLINE std::pair indexes(size_type pos) const noexcept { + SEQ_ASSERT_DEBUG(d_buckets.size() > 0, "empty container"); const auto front_size = static_cast(d_buckets.front()->size); return { static_cast((pos + (static_cast(d_bucket_size) - front_size)) >> static_cast(d_bucket_size_bits)), static_cast((pos - (pos < front_size ? 0 : front_size)) & static_cast(d_bucket_size1)) }; @@ -1803,19 +1800,13 @@ namespace seq // Returns element at global position SEQ_ALWAYS_INLINE auto at(size_type pos) noexcept -> T& { - SEQ_ASSERT_DEBUG(d_buckets.size() > 0, "empty container"); - const auto front_size = static_cast(d_buckets.front()->size); - const auto bucket = (pos + (static_cast(d_bucket_size) - front_size)) >> static_cast(d_bucket_size_bits); - const auto index = (pos - (pos < front_size ? 0 : front_size)) & static_cast(d_bucket_size1); - return (*d_buckets[bucket].bucket)[static_cast(index)]; + const auto p = indexes(pos); + return (*d_buckets[p.first].bucket)[p.second]; } SEQ_ALWAYS_INLINE auto at(size_type pos) const noexcept -> const T& { - SEQ_ASSERT_DEBUG(d_buckets.size() > 0, "empty container"); - const auto front_size = static_cast(d_buckets.front()->size); - const auto bucket = (pos + (static_cast(d_bucket_size) - front_size)) >> static_cast(d_bucket_size_bits); - const auto index = (pos - (pos < front_size ? 0 : front_size)) & static_cast(d_bucket_size1); - return (*d_buckets[bucket].bucket)[static_cast(index)]; + const auto p = indexes(pos); + return (*d_buckets[p.first].bucket)[p.second]; } // Create bucket at the back @@ -1886,7 +1877,7 @@ namespace seq bucket = create_back_bucket(); } T* ptr = bucket->emplace_back(std::forward(args)...); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets.back().update(); d_size++; return *ptr; @@ -1906,7 +1897,7 @@ namespace seq bucket = create_front_bucket(); T* res = bucket->emplace_front(std::forward(args)...); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets.front().update(); d_size++; return *res; @@ -1924,7 +1915,7 @@ namespace seq // All functions might throw, fine res = create_back_bucket()->emplace_back(std::move(d_buckets[0]->insert_pop_back(index, std::forward(args)...))); - if (StoreBackValues) { + if constexpr (StoreBackValues) { d_buckets.back().update(); d_buckets[0].update(); } @@ -1932,7 +1923,7 @@ namespace seq else { // Might throw, fine res = d_buckets[0].bucket->emplace(index, std::forward(args)...); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets.front().update(); } return res; @@ -1951,7 +1942,7 @@ namespace seq // We insert into the first bucket which is not full // Might throw, fine res = bucket(bucket_index)->emplace(index, std::forward(args)...); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[bucket_index].update(); } else if SEQ_UNLIKELY (bucket_index == 0) { @@ -1967,7 +1958,7 @@ namespace seq this->remove_front_bucket(); throw; } - if (StoreBackValues) { + if constexpr (StoreBackValues) { d_buckets[0].update(); d_buckets[1].update(); } @@ -1975,19 +1966,17 @@ namespace seq else { // we are going to loose value at index 0, save it size_t bindex = bucket_index; - // T tmp = std::move((*bucket(bindex))[0]); - // bucket(bucket_index)->insert(index, value); - + // Might throw, fine T tmp = bucket(bucket_index)->insert_pop_front(index, std::forward(args)...); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[bucket_index].update(); // propagate down to left side while (--bindex > 0) { // Might throw, fine bucket(bindex)->push_back_pop_front(tmp); // No discard - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[bindex].update(); } // first bucket @@ -1995,12 +1984,12 @@ namespace seq if (!bucket->isFull()) { // Might throw, fine bucket->emplace_back(std::move(tmp)); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[0].update(); } else { bucket->push_back_pop_front(tmp); // No discard - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[0].update(); bucket = create_front_bucket(); try { @@ -2011,7 +2000,7 @@ namespace seq this->remove_front_bucket(); throw; } - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[0].update(); } res = &at(pos); @@ -2026,14 +2015,14 @@ namespace seq SEQ_ASSERT_DEBUG(bucket_index == 0 || bucket_index == d_buckets.size() - 1, "Corrupted tiered_vector structure"); // Might throw, fine res = bucket(bucket_index)->emplace(index, std::forward(args)...); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[bucket_index].update(); } else if SEQ_UNLIKELY (bucket_index == d_buckets.size() - 1) { // Inserting into last (full) bucket // Might throw, fine T tmp = bucket(bucket_index)->insert_pop_back(index, std::forward(args)...); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[bucket_index].update(); try { res = create_back_bucket()->emplace_back(std::move(tmp)); @@ -2044,7 +2033,7 @@ namespace seq remove_back_bucket(); throw; } - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets.back().update(); } else { @@ -2052,7 +2041,7 @@ namespace seq size_type bindex = bucket_index; // Might throw, fine T tmp = bucket(bindex)->insert_pop_back(index, std::forward(args)...); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[bindex].update(); // Propagate to right buckets with successive push_front @@ -2073,11 +2062,11 @@ namespace seq else { // Might throw, fine bucket->push_front_pop_back(tmp); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets.back().update(); bucket = create_back_bucket(); bucket->emplace_front(std::move(tmp)); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets.back().update(); } res = &at(pos); @@ -2135,7 +2124,7 @@ namespace seq d_buckets.back()->pop_back(); if SEQ_UNLIKELY (d_buckets.back()->size == 0 && d_buckets.size() > 1) remove_back_bucket(); - else if (StoreBackValues) + else if constexpr (StoreBackValues) d_buckets.back().update(); --d_size; } @@ -2148,48 +2137,43 @@ namespace seq if SEQ_UNLIKELY (bucket->size == 0 && d_buckets.size() > 1) { remove_front_bucket(); } - else if (StoreBackValues) + else if constexpr (StoreBackValues) d_buckets.front().update(); --d_size; } - void erase_extremity(size_type pos) noexcept - { - if (pos == 0) - pop_front(); - else - pop_back(); - } - // Erase anywhere void erase(size_type pos) { SEQ_ASSERT_DEBUG(pos < d_size, "tiered_vector: erase at invalid position"); SEQ_ASSERT_DEBUG(d_size > 0, "tiered_vector: erase element on an empty tiered_vector"); - if SEQ_UNLIKELY (pos == 0 || pos == d_size - 1) - return erase_extremity(pos); - erase_middle(pos); + if (pos == 0) + pop_front(); + else if (pos == d_size - 1) + pop_back(); + else + erase_middle(pos); } SEQ_ALWAYS_INLINE void erase_left(size_type bucket_index, int index) { // shift left values T tmp = std::move((d_buckets.front().bucket)->back()); d_buckets.front().bucket->pop_back(); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets.front().update(); for (size_type i = 1; i < bucket_index; ++i) { // Might throw, fine bucket(i)->push_front_pop_back(tmp); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[i].update(); } // Might throw, fine bucket(bucket_index)->erase_push_front(index, std::move(tmp)); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[bucket_index].update(); if (d_buckets.front()->size == 0 && d_buckets.size() > 1) { remove_front_bucket(); @@ -2201,20 +2185,20 @@ namespace seq // Might throw, fine bucket(bucket_index)->erase_push_back(index, std::move(bucket(bucket_index + 1)->front())); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[bucket_index].update(); for (size_type i = bucket_index + 1; i < d_buckets.size() - 1; ++i) { // Might throw, fine bucket(i)->push_back_pop_front(bucket(i + 1)->front()); - if (StoreBackValues) + if constexpr (StoreBackValues) d_buckets[i].update(); } d_buckets.back()->pop_front(); if (d_buckets.back()->size == 0 && d_buckets.size() > 1) { remove_back_bucket(); } - else if (StoreBackValues) + else if constexpr (StoreBackValues) d_buckets.back().update(); } void erase_middle(size_type pos) @@ -2233,7 +2217,7 @@ namespace seq if (bucket->size == 0) { remove_front_bucket(); } - else if (StoreBackValues) + else if constexpr (StoreBackValues) d_buckets.front().update(); } else if (bucket_index == d_buckets.size() - 1) { @@ -2245,7 +2229,7 @@ namespace seq if (bucket->size == 0 && d_buckets.size() > 1) { remove_back_bucket(); } - else if (StoreBackValues) + else if constexpr (StoreBackValues) d_buckets.back().update(); } else if (pos < d_size / 2) { @@ -2928,7 +2912,7 @@ namespace seq /// @param first first iterator of the range /// @param last last iterator of the range /// @param alloc allocator object - template + template::value,int> = 0> tiered_vector(Iter first, Iter last, const Allocator& alloc = Allocator()) : Allocator(alloc) , d_manager(make_manager(alloc, min_block_size, alloc)) diff --git a/seq/timer.hpp b/seq/timer.hpp index 03164f85..7370132e 100644 --- a/seq/timer.hpp +++ b/seq/timer.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,6 +34,7 @@ #include #endif #include +#include #include "bits.hpp" diff --git a/seq/tiny_string.hpp b/seq/tiny_string.hpp index b380671b..0a5237c0 100644 --- a/seq/tiny_string.hpp +++ b/seq/tiny_string.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -37,7 +37,7 @@ #include "hash.hpp" #include "type_traits.hpp" -#include "utils.hpp" +#include "internal/utils.hpp" #ifdef min #undef min @@ -380,7 +380,6 @@ namespace seq size_t common = detail::count_approximate_common_bytes(in + 1, s + 1, in + n); if (common == (n - 1) * sizeof(Char)) return static_cast(in - data); - // in += common +1; ++in; } return npos; @@ -1184,7 +1183,7 @@ namespace seq } } /// @brief Construct by copying the range [first,last) - template + template::value, int> = 0> tiny_string(Iter first, Iter last, const Allocator& al = Allocator()) : d_data(al) { @@ -3147,8 +3146,8 @@ namespace seq class hasher> { public: - using is_transparent = int; - using is_avalanching = int; + using is_transparent = std::true_type; + using is_avalanching = std::true_type; SEQ_STR_INLINE_STRONG auto operator()(const seq::tiny_string& str) const noexcept -> size_t { return seq::hash_bytes_komihash((str.data()), str.size() * sizeof(Char)); @@ -3178,8 +3177,8 @@ namespace seq class hasher> { public: - using is_transparent = int; - using is_avalanching = int; + using is_transparent = std::true_type; + using is_avalanching = std::true_type; template SEQ_STR_INLINE_STRONG auto operator()(const seq::tiny_string& str) const noexcept -> size_t @@ -3225,8 +3224,8 @@ namespace std class hash> { public: - using is_transparent = int; - using is_avalanching = int; + using is_transparent = std::true_type; + using is_avalanching = std::true_type; SEQ_STR_INLINE_STRONG auto operator()(const seq::tiny_string& str) const noexcept -> size_t { return seq::hash_bytes_komihash(str.data(), str.size() * sizeof(Char)); diff --git a/seq/type_traits.hpp b/seq/type_traits.hpp index de4ab821..a06c7705 100644 --- a/seq/type_traits.hpp +++ b/seq/type_traits.hpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -165,42 +165,23 @@ namespace seq // Specilizations for unique_ptr, shared_ptr and pair template - struct is_relocatable> + struct is_relocatable> : std::bool_constant::value> { - static constexpr bool value = is_relocatable::value; }; template - struct is_relocatable> + struct is_relocatable> : std::true_type { - static constexpr bool value = true; }; template - struct is_relocatable> + struct is_relocatable> : std::bool_constant::value && is_relocatable::value> { - static constexpr bool value = is_relocatable::value && is_relocatable::value; }; template - struct is_relocatable> + struct is_relocatable> : std::true_type { - static constexpr bool value = true; }; - - namespace detail - { - template::value> - struct is_relocatable_tupe - { - using type = typename std::tuple_element::value - Pos, Tuple>::type; - static constexpr bool value = seq::is_relocatable::value && is_relocatable_tupe::value; - }; - template - struct is_relocatable_tupe - { - static constexpr bool value = true; - }; - } template - struct is_relocatable> : detail::is_relocatable_tupe> + struct is_relocatable> : std::bool_constant<(is_relocatable_v && ...)> { }; @@ -302,6 +283,17 @@ namespace seq static constexpr bool value = has_iterator::value && has_value_type::value; }; + /// @brief Check if givern type is an iterator + template + struct is_iterator : std::false_type + { + }; + + template + struct is_iterator::iterator_category>> : std::true_type + { + }; + template struct has_is_transparent : std::false_type { diff --git a/seq_config.hpp.in b/seq_config.hpp.in index 6a26a8f7..69ad9d0b 100644 --- a/seq_config.hpp.in +++ b/seq_config.hpp.in @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_seq/CMakeLists.txt b/tests/test_seq/CMakeLists.txt index 7608fad9..f1e17974 100644 --- a/tests/test_seq/CMakeLists.txt +++ b/tests/test_seq/CMakeLists.txt @@ -9,8 +9,6 @@ enable_testing() create_test_sourcelist (Tests seq_tests.cpp test_any.cpp - test_charconv.cpp - test_format.cpp test_devector.cpp test_flat_map.cpp test_ordered_map.cpp @@ -21,6 +19,7 @@ create_test_sourcelist (Tests test_tiny_string.cpp test_all_maps.cpp test_concurrent_map.cpp +test_concurrent_queue.cpp test_algorithm.cpp ) diff --git a/tests/test_seq/test_algorithm.cpp b/tests/test_seq/test_algorithm.cpp index 9abb4d2b..457ab9e7 100644 --- a/tests/test_seq/test_algorithm.cpp +++ b/tests/test_seq/test_algorithm.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include "testing.hpp" #include #include diff --git a/tests/test_seq/test_any.cpp b/tests/test_seq/test_any.cpp index 145bfe6c..5c833043 100644 --- a/tests/test_seq/test_any.cpp +++ b/tests/test_seq/test_any.cpp @@ -10,8 +10,8 @@ #include "seq/any.hpp" #include "seq/tiny_string.hpp" #include "seq/hash.hpp" -#include "seq/testing.hpp" -#include "seq/format.hpp" +#include "testing.hpp" +#include "seq/legacy/format.hpp" #include "seq/ordered_map.hpp" namespace seq @@ -629,7 +629,7 @@ static void test_hold_any() SEQ_TEST(set.find(2.5) != set.end()); // use heterogeneous lookup SEQ_TEST(set.find("hello") != set.end()); // use heterogeneous lookup SEQ_TEST(set.find(tstring("world")) != set.end()); // use heterogeneous lookup - SEQ_TEST(set.find("ok") == set.end()); //"ok" not found has we compare 2 const char* -> pointer comparison, not string comparison + SEQ_TEST(set.find("ok") != set.end()); //ensure const char* to const char* results in string comparison SEQ_TEST(set.find("no") == set.end()); // failed lookup } { @@ -671,7 +671,6 @@ static void test_hold_any() int test_any(int, char*[]) { - SEQ_TEST_MODULE_RETURN(any, 1, test_hold_any()); return 0; } diff --git a/tests/test_seq/test_charconv.cpp b/tests/test_seq/test_charconv.cpp deleted file mode 100644 index 90962bc3..00000000 --- a/tests/test_seq/test_charconv.cpp +++ /dev/null @@ -1,238 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Victor Moncada - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include "seq/charconv.hpp" -#include "seq/utils.hpp" -#include "seq/testing.hpp" - -inline double promote(float val) { return static_cast(val); } -inline double promote(double val) { return val; } -inline long double promote(long double val) { return val; } - -template -int __float_to_string_seq(T val, seq::chars_format fmt, int prec, char *dst, char * end) -{ - using namespace seq; - int size = static_cast(seq::to_chars(dst, end, val, fmt, prec).ptr - dst); - dst[size] = 0; - return size; -} -template -int __float_to_string_printf(T val, seq::chars_format fmt, int prec, char* dst, char* end) -{ - using namespace seq; - std::ostringstream oss; - if (fmt == seq::general) { - if(std::is_same::value) - oss <<"%." << prec << "Lg"; - else - oss << "%." << prec << "g" ; - } - else if (fmt == seq::scientific) { - if (std::is_same::value) - oss << "%." << prec << "Le" ; - else - oss << "%." << prec << "e" ; - } - else { - if (std::is_same::value) - oss << "%." << prec<< "Lf" ; - else - oss << "%." << prec<< "f"; - } - - snprintf(dst,static_cast(end-dst), oss.str().c_str(), promote(val)); - return static_cast(strlen(dst)); -} - -template -bool __test_read_val(const char* src) -{ - using namespace seq; - std::istringstream iss(src); - T v; - iss >> v; - if (!iss) { - if (strcmp(src, "inf") == 0) return true; - if (strcmp(src, "-inf") == 0) return true; - if (strcmp(src, "nan") == 0) return true; - return false; - } - return iss.peek() == EOF; -} - -template -int exponent(T v) -{ - T exp = std::log10(std::abs(v)); - exp = std::floor(exp); - return static_cast(exp); -} - -template -bool __test_equal(const char* s1, const char* s2, seq::chars_format fmt, int prec) -{ - using namespace seq; - int l1 = static_cast(strlen(s1)); - //int l2 = (int)strlen(s2); - - if (strcmp(s1, s2) == 0) - return true; - - //check for inf and nan - T v1; - if (from_chars(s1, s1 + l1, v1).ec != std::errc()) - return false; - T v2; - { - std::istringstream iss(s2); - iss >> v2; - } - - T saved1 = v1; - T saved2 = v2; - - //equal values: no need to go further - if (v1 == v2) - return true; - if (std::isnan(v1) && std::isnan(v2)) - return true; - - - int exp1 = exponent(v1); - int exp2 = exponent(v2); - if (fmt == seq::fixed) { - // for fixed specifier, harder to compare, for now just check the exponents - return exp1 == exp2; - } - v1 *= static_cast(std::pow(static_cast(10), static_cast(-exp1))); - v2 *= static_cast(std::pow(static_cast(10), static_cast(-exp1))); - - int p = prec; - if (p > 14) p = 14; - if (std::is_same::value && p > 6) - p = 6; - if (p > 0) --p; - T error = static_cast(std::pow(static_cast(10.), static_cast(-p))) *4; - T diff = std::abs(v1 - v2); - if (diff <= error) - return true; - - std::cout << std::setprecision(prec + 6) << "read vals: " << saved1 << " and " << saved2 << std::endl; - std::cout << std::setprecision(prec + 6) << "normalized: " << v1 << " and " << v2 << std::endl; - - std::cout << std::setprecision(prec + 6) << "diff is " << diff << " and max error is " << error << std::endl; - return false; - - /*int p = prec; - if (p > 15) p = 15; - if (std::is_same::value && p > 6) - p = 6; - if (p > 0) --p; - T error = (T)std::pow((T)10., (T)-p); - T pow = std::log10(std::abs(v1)); - T mul = std::pow((T)10., pow < 0 ? std::ceil(pow) : std::floor(pow)); - error *= mul*4; - - T diff = std::abs(v1 - v2); - if (diff <= error) - return true; - - std::cout << std::setprecision(prec+6)<< "diff is " << diff << " and max error is " << error << std::endl; - return false;*/ -} - - - - - -template -void test_to_chars(unsigned count, seq::chars_format fmt, int p) -{ - using namespace seq; - std::cout << "test charconv for " << count << " random " << typeid(T).name() << " with precision " << p << " and type " << (fmt == seq::scientific ? "scientific" : (fmt == seq::fixed ? "fixed" : "general")) << std::endl; - - random_float_genertor rgn; - std::vector vals; - for (unsigned i = 0; i < count; ++i) - vals.push_back(rgn()); - - char dst1[1000]; - char dst2[1000]; - - for (unsigned i = 0; i < count; ++i) - { - - - __float_to_string_seq(vals[i], fmt, p, dst1, dst1 + sizeof(dst1)); - __float_to_string_printf(vals[i], fmt, p, dst2, dst2 + sizeof(dst2)); - - SEQ_TEST(__test_read_val(dst1)); - SEQ_TEST(__test_read_val(dst2)); - - try { - - SEQ_TEST(__test_equal(dst1, dst2,fmt,p)); - } - catch (...) - { - //SEQ_TEST(__test_equal(dst1, dst2,fmt, p)); - std::cout << "error while parsing " << dst1 << " (seq) and " << dst2 << " (printf) for value " << std::setprecision(static_cast(p+6)) << vals[i]<< std::endl; - throw; - } - - } -} - - -inline void test_charconv(unsigned count = 100000, int max_precision = 50) -{ - - for (int p = 0; p < max_precision; ++p) - test_to_chars(count, seq::general, p); - for (int p = 0; p < max_precision; ++p) - test_to_chars(count, seq::scientific, p); - - - for (int p = 0; p < max_precision; ++p) - test_to_chars(count, seq::general, p); - for (int p = 0; p < max_precision; ++p) - test_to_chars(count, seq::scientific, p); - - for (int p = 0; p < max_precision; ++p) - test_to_chars(count, seq::general, p); - for (int p = 0; p < max_precision; ++p) - test_to_chars(count, seq::scientific, p); - -} - - -SEQ_PROTOTYPE(int test_charconv(int , char*[])) -{ - SEQ_TEST_MODULE_RETURN(charconv, 1, test_charconv(10000, 30)); - return 0; -} diff --git a/tests/test_seq/test_concurrent_map.cpp b/tests/test_seq/test_concurrent_map.cpp index 42dd58a1..270fc538 100644 --- a/tests/test_seq/test_concurrent_map.cpp +++ b/tests/test_seq/test_concurrent_map.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "seq/testing.hpp" +#include "testing.hpp" #include "seq/concurrent_map.hpp" #include "seq/tiny_string.hpp" diff --git a/tests/test_seq/test_concurrent_queue.cpp b/tests/test_seq/test_concurrent_queue.cpp new file mode 100644 index 00000000..82f4c835 --- /dev/null +++ b/tests/test_seq/test_concurrent_queue.cpp @@ -0,0 +1,146 @@ + /* + * Copyright (c) 2026 Victor Moncada + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "testing.hpp" +#include "seq/concurrent_queue.hpp" +#include +#include +#include +#include + +template +void test_empty_queue(Queue& q) +{ + SEQ_TEST(q.size() == 0); + SEQ_TEST(q.empty()); + SEQ_TEST(std::distance(q.begin(), q.end()) == 0); + SEQ_TEST(std::distance(q.cbegin(), q.cend()) == 0); + SEQ_TEST(std::distance(q.rbegin(), q.rend()) == 0); + SEQ_TEST(std::distance(q.crbegin(), q.crend()) == 0); + } + +template +void queue_thread(Queue& q, const std::vector & vec, std::atomic & start) +{ + thread_local int loc = 0; + std::uniform_int_distribution uniform_dist(0, (int)vec.size()); + std::mt19937 e2((unsigned)(uintptr_t)&loc); + while (!start) + ; + + for (size_t i = 0; i < vec.size(); ++i) { + auto v = uniform_dist(e2); + if (v % 2 == 0) + q.push(vec[i]); + else + q.pop(); + } +} + +template +static void test_queue(const std::vector& vals, const Al& al) +{ + using queue_type = seq::concurrent_queue; + + { + queue_type q(al); + test_empty_queue(q); + } + { + queue_type q(100000, al); + test_empty_queue(q); + } + { + queue_type q(al); + for (auto& v : vals) + q.push(v); + SEQ_TEST((size_t)std::distance(q.begin(), q.end()) == vals.size()); + SEQ_TEST((size_t)std::distance(q.cbegin(), q.cend()) == vals.size()); + SEQ_TEST((size_t)std::distance(q.rbegin(), q.rend()) == vals.size()); + SEQ_TEST((size_t)std::distance(q.crbegin(), q.crend()) == vals.size()); + for (auto& v: vals){ + (void)v; + q.pop(); + } + test_empty_queue(q); + for (auto& v : vals) + q.emplace(v); + for (auto& v : vals) { + T unused; + q.pop(unused); + SEQ_TEST(unused == v); + } + q.shrink_to_fit(); + test_empty_queue(q); + } + { + queue_type q(al); + for (auto& v : vals) + q.push(v); + q.clear(); + test_empty_queue(q); + } + { + queue_type q(al); + for (auto& v : vals) + q.push(v); + } + + queue_type q(al); + + std::vector threads(16); + std::atomic start{ false }; + for (size_t i = 0; i < threads.size(); ++i) + threads[i] = std::thread([&]() { queue_thread(q, vals, start); }); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + start.store(true); + for (size_t i = 0; i < threads.size(); ++i) + threads[i].join(); + +} + +#include "tests.hpp" + +struct Int +{ + +}; + +SEQ_PROTOTYPE(int test_concurrent_queue(int, char*[])) +{ + + std::vector> vals(1000000); + for (size_t i = 0; i < vals.size(); ++i) + vals[i] = TestDestroy ((double)i); + + CountAlloc> al; + auto prev = TestDestroy::count(); + test_queue(vals, al); + auto current = TestDestroy::count(); + SEQ_TEST(current == prev); + SEQ_TEST(get_alloc_bytes(al) == 0); + + return 0; + +} \ No newline at end of file diff --git a/tests/test_seq/test_devector.cpp b/tests/test_seq/test_devector.cpp index 9593c226..87b55b96 100644 --- a/tests/test_seq/test_devector.cpp +++ b/tests/test_seq/test_devector.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "seq/testing.hpp" +#include "testing.hpp" #include "seq/devector.hpp" #include "tests.hpp" #include diff --git a/tests/test_seq/test_flat_map.cpp b/tests/test_seq/test_flat_map.cpp index b3b74e53..e3841faa 100644 --- a/tests/test_seq/test_flat_map.cpp +++ b/tests/test_seq/test_flat_map.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -27,7 +27,7 @@ #include #include "seq/flat_map.hpp" -#include "seq/testing.hpp" +#include "testing.hpp" #include "tests.hpp" #include diff --git a/tests/test_seq/test_format.cpp b/tests/test_seq/test_format.cpp deleted file mode 100644 index 793d6b41..00000000 --- a/tests/test_seq/test_format.cpp +++ /dev/null @@ -1,307 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Victor Moncada - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include - -namespace seq -{ - // Specialization of ostream_format for std::pair - - template - class ostream_format, S> : public base_ostream_format, ostream_format, S>> - { - using base_type = base_ostream_format, ostream_format, S>>; - - public: - ostream_format() - : base_type() - { - } - ostream_format(const std::pair& v) - : base_type(v) - { - } - - // The specialization must provide this member: - - template - size_t to_string(Out& out) const - { - using Char = typename character_type::type; - size_t prev = out.size(); - - out.push_back((Char)'('); - // Format the first member of the pair using the internal numeric format - fmt(this->value().first).set_numeric_format(this->numeric_fmt()).append(out); - out.push_back((Char)','); - out.push_back((Char)' '); - // Format the second member of the pair using the internal numeric format - fmt(this->value().second).set_numeric_format(this->numeric_fmt()).append(out); - out.push_back((Char)')'); - - return out.size() - prev; - } - }; -} - -inline void test_format() -{ - - using namespace seq; - - const double PI = 3.14159265358979323846; - - { - // Test formatting single values - - SEQ_TEST_TO_OSTREAM("3.14159", fmt(PI)); // default double formatting - SEQ_TEST_TO_OSTREAM("3.141593E+00", fmt(PI, 'E')); // scientific notation, equivalent to fmt(PI).format('E') or fmt(PI).t('E') - SEQ_TEST_TO_OSTREAM("3.14159265359E+00", fmt(PI, 'E').precision(12)); // scientific notation with maximum precision, equivalent to fmt(PI).t('E').precision(12) or fmt(PI).t('E').p(12) - SEQ_TEST_TO_OSTREAM("3,14159", fmt(PI).dot(',')); // change dot, equivalent to fmt(PI).d(',') - SEQ_TEST_TO_OSTREAM("---3.14159", fmt(PI).right(10).fill('-')); // align to the right and pad with '-', equivalent to fmt(PI).r(10).f('-') - SEQ_TEST_TO_OSTREAM("3.14159---", fmt(PI).left(10).fill('-')); // align to the left and pad with '-', equivalent to fmt(PI).l(10).f('-') - SEQ_TEST_TO_OSTREAM("-3.14159--", fmt(PI).center(10).fill('-')); // align to the center and pad with '-', equivalent to fmt(PI).c(10).f('-') - SEQ_TEST_TO_OSTREAM("0x1E240", fmt(123456).base(16).hex_prefix().upper()); // hexadecimal upper case with '0x' prefix. equivalent to fmt(123456).b(16).h().u() or hex(123456).h().u() - SEQ_TEST_TO_OSTREAM("**hello***", fmt("hello").c(10).f('*')); // center string and pad with '*', equivalent to fmt("hello").center(10).fill('*') - SEQ_TEST_TO_OSTREAM("ell", fmt("hello").c(3).f('*')); // center and truncate string - - // Direct string conversion - std::string str = fmt(PI); - // Direct string conversion using .str() - std::string str2 = "PI value is " + fmt(PI).str(); - SEQ_TEST_TO_OSTREAM("PI value is 3.14159", str2); - } - - { - // Test formatting single values with shortcuts - - SEQ_TEST_TO_OSTREAM("u", ch('u')); // equivalent to fmt('u').as_char() or fmt('u').c() - SEQ_TEST_TO_OSTREAM("1.2e+00", e(1.2)); // equivalent to fmt(1.2,'e') or fmt(1.2).format('e') or fmt(1.2).t('e') - SEQ_TEST_TO_OSTREAM("1.2E+00", E(1.2)); // equivalent to fmt(1.2,'E') or fmt(1.2).format('E') or fmt(1.2).t('E') - SEQ_TEST_TO_OSTREAM("1.2", f(1.2)); // equivalent to fmt(1.2,'f') or fmt(1.2).format('f') or fmt(1.2).t('f') - SEQ_TEST_TO_OSTREAM("1.2", F(1.2)); // equivalent to fmt(1.2,'F') or fmt(1.2).format('F') or fmt(1.2).t('F') - SEQ_TEST_TO_OSTREAM("1.2", g(1.2)); // equivalent to fmt(1.2,'g') or fmt(1.2).format('g') or fmt(1.2).t('g') - SEQ_TEST_TO_OSTREAM("1.2", G(1.2)); // equivalent to fmt(1.2,'G') or fmt(1.2).format('G') or fmt(1.2).t('G') - SEQ_TEST_TO_OSTREAM("100", fmt(100)); // - SEQ_TEST_TO_OSTREAM("64", hex(100)); // equivalent to fmt(100).base(16) or fmt(100).b(16) - SEQ_TEST_TO_OSTREAM("144", oct(100)); // equivalent to fmt(100).base(8) or fmt(100).b(8) - SEQ_TEST_TO_OSTREAM("1100100", bin(100)); // equivalent to fmt(100).base(2) or fmt(100).b(2) - } - - { - // Nested formatting - SEQ_TEST_TO_OSTREAM("-----#####**surrounded text***#####-----", fmt(fmt(fmt(fmt("surrounded text")).c(20).f('*')).c(30).f('#')).c(40).f('-')); - } - - { - // Test formatting multiple values - - // Direct stream - SEQ_TEST_TO_OSTREAM("The answer is 43 ...", fmt("The answer is ", 43, " ...")); - // Direct stream with nested formatting - SEQ_TEST_TO_OSTREAM("...Or it could be 4.33e+01 ", fmt("...Or it could be", fmt(43.3, 'e').c(10))); - - // Reuse a formatting object built without arguments - auto f = fmt(); - SEQ_TEST_TO_OSTREAM("1 + 2.2 = 3.2", f(1, " + ", 2.2, " = ", 3.2)); - - // Reuse a formatting object and use seq::null to only update some arguments - auto f2 = fmt(int(), " + ", fmt().format('g'), " = ", fmt().format('e')); - SEQ_TEST_TO_OSTREAM("1 + 2.2 = 3.2e+00", f2(1, null, 2.2, null, 3.2)); - - // Convert to string or tstring - std::string s1 = f2(1, null, 2.2, null, 3.2); // equivalent to s1 = f2(1, null, 2.2, null, 3.2).str(); - tstring s2 = f2(1, null, 2.2, null, 3.2); // equivalent to s2 = f2(1, null, 2.2, null, 3.2).str(); - SEQ_TEST(s1 == "1 + 2.2 = 3.2e+00"); - SEQ_TEST(s2 == "1 + 2.2 = 3.2e+00"); - - // Append to string - s2 += ", repeat-> "; - f2(1, null, 2.2, null, 3.2).append(s2); // append formatted result to s2 - SEQ_TEST_TO_OSTREAM("1 + 2.2 = 3.2e+00, repeat-> 1 + 2.2 = 3.2e+00", s2); - - // Modify formatting object using get() and/or set() - f2.set<0>(fmt().base(16).h().u()); // reset the formatting object at position 0 - f2.get<2>().format('e'); // modifiy the formatting object at position 2 - SEQ_TEST_TO_OSTREAM("0x1 + 2.2e+00 = 3.2e+00", f2(1, null, 2.2, null, 3.2)); - - // Use positional argument - SEQ_TEST_TO_OSTREAM("0x1 + 2.2e+00 = 3.2e+00", f2(pos<0, 2, 4>(), 1, 2.2, 3.2)); // provided arguments are used for positions 0, 2 and 4 - - // Positional directly in the fmt call - auto f3 = fmt(pos<0, 2, 4>(), int(), " + ", seq::g(), " = ", seq::e()); - SEQ_TEST_TO_OSTREAM("1 + 2.2 = 3.2e+00", f3(1, 2.2, 3.2)); - - // Building tables - - // header/trailer format, 2 columns of width 20 centered, separated by a '|' - auto header = fmt(pos<1, 3>(), "|", seq::str().c(20), "|", seq::str().c(20), "|"); - // line format, 2 columns of width 20 centered, separated by a '|' - auto line = fmt(pos<1, 3>(), "|", seq::fmt().c(20), "|", seq::fmt().c(20), "|"); - // write table - SEQ_TEST_TO_OSTREAM("| Header 1 | Header 2 |", header("Header 1", "Header 2")); - SEQ_TEST_TO_OSTREAM("| 1.1 | 2.2 |", line(1.1, 2.2)); - SEQ_TEST_TO_OSTREAM("| 3.3 | 4.4 |", line(3.3, 4.4)); - SEQ_TEST_TO_OSTREAM("| Trailer 1 | Trailer 2 |", header("Trailer 1", "Trailer 2")); - } - - { - // Print to std::ostream - SEQ_TEST_TO_OSTREAM("1.12346", seq::fmt(1.123456789, 'g')); - - // Convert to string - std::string str = seq::fmt(1.123456789, 'g').str(); - SEQ_TEST(str == "1.12346"); - - // Append to an existing string - std::string str2; - seq::fmt(1.123456789, 'g').append(str2); - SEQ_TEST(str2 == "1.12346"); - - // write to buffer (to_chars(char*) returns past-the-end pointer) - char dst[100]; - *seq::fmt(1.123456789, 'g').to_chars(dst) = 0; - SEQ_TEST(dst == std::string("1.12346")); - - // write to buffer with maximum size (to_chars(char*,size_t) returns a pair of past-the-end pointer and size without truncation) - char dst2[100]; - *seq::fmt(1.123456789, 'g').to_chars(dst2, sizeof(dst2)).first = 0; - SEQ_TEST(dst2 == std::string("1.12346")); - } - - { - // Formatting custom types - SEQ_TEST_TO_OSTREAM("Print a pair of float: (1.2, 3.4)", fmt("Print a pair of float: ", std::make_pair(1.2f, 3.4f))); - - // Formatting custom types with custom format - SEQ_TEST_TO_OSTREAM("Print a pair of double: (1.2e+00, 3.4e+00)", fmt("Print a pair of double: ", fmt(std::make_pair(1.2, 3.4)).format('e'))); - - // Formatting custom types with custom format and alignment - SEQ_TEST_TO_OSTREAM("Print a pair of double centered: ******(1.2e+00, 3.4e+00)******", fmt("Print a pair of double centered: ", fmt(std::make_pair(1.2, 3.4)).t('e').c(30).f('*'))); - } - - { - // Create table. Make sure this compile without warning. - // Example used within format documentation. - - using namespace seq; - - // Build the line format. Use join() to add a '|' character in between columns. Use _a() to format anything with the supplied width modifiers. - // The first column is left aligned with a width of 20 characters. The 3 remaining columns are centered on a 15 characters width. - auto line = join("|", _a().l(20), _a().c(15), _a().c(15), _a().c(15), ""); - // Slot argument passed to line object. Displays a time measurement as unsigned integer followed by " ms" string - auto slot = fmt(_u(), "ms"); - // Another slot argument passed to line object. Displays a time measurment as unsigned integer followed by " ms" string, and a memory measurement as unsigned integer followed by " MO" - // string. - auto slot2 = fmt(_u(), "ms / ", _u(), "MO"); - - // Output table header using the 'line' format object - std::cout << line("Operation type", "std::vector", "std::deque", "std::list") << std::endl; - - // Output the separator between table header and actual table content (something like ----|----|----|). - // Use reset() to clear the line content and set the fill character to '-' - std::cout << line.reset('-') << std::endl; - - // Reset the fill character to ' ' (blank space) - line.reset(' '); - - // Containers to benchmark - std::vector vec; - std::deque deq; - std::list lst; - - // Benchmark back insertion for std::vector, std::deque, std::list. - // We measure the time spent and the program memory footprint afterward. - - reset_memory_usage(); - tick(); - for (size_t i = 0; i < 10000000; ++i) - vec.push_back(i); - std::uint64_t tvec = tock_ms(); // measure elapsed time - std::uint64_t mvec = get_memory_usage() / 1000000; // measure program memory usage - - reset_memory_usage(); - tick(); - for (size_t i = 0; i < 10000000; ++i) - deq.push_back(i); - std::uint64_t tdeq = tock_ms(); - std::uint64_t mdeq = get_memory_usage() / 1000000; // measure program memory usage - - reset_memory_usage(); - tick(); - for (size_t i = 0; i < 10000000; ++i) - lst.push_back(i); - std::uint64_t tlst = tock_ms(); - std::uint64_t mlst = get_memory_usage() / 1000000; - - // Output measurments using the 'line' format object. - // Note that in this situation, we must use the operator*() of the slot objects to create copies. - // Indeed, passing values to the slot will modify it and return a reference. By calling slot2(...) several times - // in the same instruction, the line object will only receive the last set values in slot2 (depending on function evaluation order). - std::cout << line("push_back", // type of operation (left aligned on 20 characters) - *slot2(tvec, mvec), // time and memory (centered on 15 characters) - *slot2(tdeq, mdeq), // time and memory (centered on 15 characters) - *slot2(tlst, mlst) // time and memory (centered on 15 characters) - ) - << std::endl; - - // Benchmark iteration - - tick(); - for (auto it = vec.begin(); it != vec.end(); ++it) - SEQ_TEST(*it != 10000000); // Use SEQ_TEST to make sure the compiler wont 'optimize' the loop (and remove it) - tvec = tock_ms(); - - tick(); - for (auto it = deq.begin(); it != deq.end(); ++it) - SEQ_TEST(*it != 10000000); - tdeq = tock_ms(); - - tick(); - for (auto it = lst.begin(); it != lst.end(); ++it) - SEQ_TEST(*it != 10000000); - tlst = tock_ms(); - - // Output measurments - std::cout << line("iterate", // type of operation (left aligned on 20 characters) - *slot(tvec), // time (centered on 15 characters) - *slot(tdeq), // time (centered on 15 characters) - *slot(tlst) // time (centered on 15 characters) - ) - << std::endl; - } -} - -SEQ_PROTOTYPE(int test_format(int, char*[])) -{ - - SEQ_TEST_MODULE_RETURN(format, 1, test_format()); - return 0; -} diff --git a/tests/test_seq/test_ordered_map.cpp b/tests/test_seq/test_ordered_map.cpp index 54a660df..29b8647f 100644 --- a/tests/test_seq/test_ordered_map.cpp +++ b/tests/test_seq/test_ordered_map.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -23,7 +23,7 @@ */ -#include "seq/testing.hpp" +#include "testing.hpp" #include "seq/ordered_map.hpp" #include "seq/tiny_string.hpp" #include "tests.hpp" diff --git a/tests/test_seq/test_radix_hash_map.cpp b/tests/test_seq/test_radix_hash_map.cpp index d76a5c99..cc186571 100644 --- a/tests/test_seq/test_radix_hash_map.cpp +++ b/tests/test_seq/test_radix_hash_map.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "seq/testing.hpp" +#include "testing.hpp" #include "seq/radix_hash_map.hpp" #include "seq/tiny_string.hpp" diff --git a/tests/test_seq/test_radix_tree.cpp b/tests/test_seq/test_radix_tree.cpp index 5b638c1c..fb5ab4a4 100644 --- a/tests/test_seq/test_radix_tree.cpp +++ b/tests/test_seq/test_radix_tree.cpp @@ -1,7 +1,7 @@ /** * MIT License * - * Copyright (c) 2025 Victor Moncada + * Copyright (c) 2026 Victor Moncada * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,15 +28,16 @@ #endif #include "seq/radix_map.hpp" -#include "seq/testing.hpp" +#include "testing.hpp" #include "seq/tiny_string.hpp" +#include "tests.hpp" #include #include #include #include -#include "tests.hpp" + template using RebindAlloc = typename std::allocator_traits::template rebind_alloc; @@ -307,7 +308,7 @@ inline void test_radix_set_common() for (size_t i = 1000 * mul; i < 2000 * mul; i += 5) { auto it = set.lower_bound(i); // test bit position - SEQ_TEST(it.iter.bit_pos == it.iter.get_bit_pos(it.iter.dir)); + SEQ_TEST(seq::detail::check_bit_pos(it)); // test iterator validity SEQ_TEST(it != set.end() && *it == i); } @@ -332,7 +333,7 @@ inline void test_radix_set_common() // test bit position if (it != set.end()) - SEQ_TEST(it.iter.bit_pos == it.iter.get_bit_pos(it.iter.dir)); + SEQ_TEST(seq::detail::check_bit_pos(it)); if (i < 1000 * mul) { // values < first value @@ -354,6 +355,51 @@ inline void test_radix_set_common() } } } + + { + // Test prefix search + seq::radix_set set; + set.insert("this is ok"); + set.insert("this is not ok"); + set.insert("this is right"); + set.insert("this is not right"); + set.insert("this was a test"); + set.insert("test"); + set.insert("unused"); + + auto range = set.prefix_range("this"); + SEQ_TEST(std::distance(range.first, range.second) == 5); + range = set.prefix_range("this is"); + SEQ_TEST(std::distance(range.first, range.second) == 4); + range = set.prefix_range("t"); + SEQ_TEST(std::distance(range.first, range.second) == 6); + range = set.prefix_range("u"); + SEQ_TEST(std::distance(range.first, range.second) == 1); + range = set.prefix_range("a"); + SEQ_TEST(std::distance(range.first, range.second) == 0); + } + { + // Test prefix search + seq::radix_map set; + set.insert({ "this is ok", 0 }); + set.insert({ "this is not ok", 0 }); + set.insert({ "this is right", 0 }); + set.insert({ "this is not right", 0 }); + set.insert({ "this was a test", 0 }); + set.insert({ "test", 0 }); + set.insert({ "unused", 0 }); + + auto range = set.prefix_range("this"); + SEQ_TEST(std::distance(range.first, range.second) == 5); + range = set.prefix_range("this is"); + SEQ_TEST(std::distance(range.first, range.second) == 4); + range = set.prefix_range("t"); + SEQ_TEST(std::distance(range.first, range.second) == 6); + range = set.prefix_range("u"); + SEQ_TEST(std::distance(range.first, range.second) == 1); + range = set.prefix_range("a"); + SEQ_TEST(std::distance(range.first, range.second) == 0); + } } inline void test_radix_map_common() @@ -475,7 +521,7 @@ inline void test_radix_map_common() for (size_t i = 1000 * mul; i < 2000 * mul; i += 5) { auto it = set.lower_bound(i); // test bit position - SEQ_TEST(it.iter.bit_pos == it.iter.get_bit_pos(it.iter.dir)); + SEQ_TEST(seq::detail::check_bit_pos(it)); // test iterator validity SEQ_TEST(it != set.end() && it->first == i); } @@ -486,7 +532,7 @@ inline void test_radix_map_common() // test bit position if (it != set.end()) - SEQ_TEST(it.iter.bit_pos == it.iter.get_bit_pos(it.iter.dir)); + SEQ_TEST(seq::detail::check_bit_pos(it)); if (i < 1000 * mul) { // values < first value @@ -1295,8 +1341,87 @@ static void test_issue() + + + +using schema_t = std::tuple; + +struct extractor_t +{ + schema_t operator()(const schema_t* tuple) const { return schema_t{ std::get<1>(*tuple), std::get<0>(*tuple) }; } +}; + +void show_all(seq::radix_map::iterator begin, seq::radix_map::iterator end) +{ + for (auto it = begin; it != end; ++it) { + std::cout << "Tuple: (" << std::get<0>(*(it->first)) << "," << std::get<1>(*(it->first)) << ") -> " << it->second << std::endl; + } +} +static void test_issue2() +{ + seq::radix_map tuples; + extractor_t my_extractor; + schema_t my_tuple = schema_t{ 0, 1 }; + tuples.insert(std::make_pair(&my_tuple, 1)); + // is stored in order (1,0) because schema_t* is given => applying extractor_t to get inner_key + std::cout << "All tuples in the index:" << std::endl; + for (auto it : tuples) { + std::cout << "Tuple: (" << std::get<0>(*(it.first)) << "," << std::get<1>(*(it.first)) << ") -> " << it.second << std::endl; + } + std::cout << "===============================" << std::endl; + + // Works CORRECT: schema_t is given => equal to radix_map::radix_key_type + // => no extractor applied => no result found (as only the inner key (1,0) is inserted) + std::cout << "Variant 0:" << std::endl; + seq::radix_map::iterator lower_0 = tuples.lower_bound(my_tuple); + seq::radix_map::iterator upper_0 = tuples.upper_bound(my_tuple); + show_all(lower_0, upper_0); + std::cout << "===============================" << std::endl; + + // Works CORRECT: schema_t is given => equal to radix_map::radix_key_type + // => no extractor applied => but we applied the constructor before so the tuple is reordered to (1,0) => result found + std::cout << "Variant 1:" << std::endl; + schema_t to_test_tuple = my_extractor(&my_tuple); + std::cout << "To test tuple: (" << std::get<0>(to_test_tuple) << "," << std::get<1>(to_test_tuple) << ")" << std::endl; + seq::radix_map::iterator lower_1 = tuples.lower_bound(to_test_tuple); + seq::radix_map::iterator upper_1 = tuples.upper_bound(to_test_tuple); + show_all(lower_1, upper_1); + std::cout << "===============================" << std::endl; + + // !Does not compile.! + // Expected: schema_t* is given => equal to radix_map::Key template argument + // => given extractor should be applied => seeking for (1,0) => result found + std::cout << "Variant 2:" << std::endl; + seq::radix_map::iterator lower_2 = tuples.lower_bound(&my_tuple); + seq::radix_map::iterator upper_2 = tuples.upper_bound(&my_tuple); + show_all(lower_2, upper_2); + std::cout << "===============================" << std::endl; + +} + +int test_chrono() +{ + using time = decltype(std::chrono::steady_clock::now()); + + seq::radix_set