From d8b26b7e51885f041f824cecbf83affd1114ffde Mon Sep 17 00:00:00 2001 From: tzcnt Date: Thu, 25 Jun 2026 08:35:08 -0700 Subject: [PATCH 1/2] default executor spins 4 -> 0 --- include/tmc/detail/init_params.hpp | 39 ++++++++++++------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/include/tmc/detail/init_params.hpp b/include/tmc/detail/init_params.hpp index b06ff292..a5e4ae66 100644 --- a/include/tmc/detail/init_params.hpp +++ b/include/tmc/detail/init_params.hpp @@ -29,17 +29,15 @@ namespace detail { struct InitParams { size_t priority_count = 0; size_t thread_count = 0; - size_t spins = 4; + size_t spins = 0; work_stealing_strategy strategy = work_stealing_strategy::HIERARCHY_MATRIX; std::vector thread_occupancy = {}; std::function thread_init_hook = nullptr; - std::function thread_teardown_hook = - nullptr; + std::function thread_teardown_hook = nullptr; std::function thread_post_run_hook = nullptr; #ifdef TMC_USE_HWLOC std::vector partitions = {}; - tmc::topology::thread_pinning_level pin = - tmc::topology::thread_pinning_level::GROUP; + tmc::topology::thread_pinning_level pin = tmc::topology::thread_pinning_level::GROUP; tmc::topology::thread_packing_strategy pack = tmc::topology::thread_packing_strategy::PACK; @@ -52,13 +50,11 @@ struct InitParams { std::vector priority_ranges = {}; TMC_DECL void add_partition(tmc::topology::topology_filter const& Filter); - TMC_DECL void - set_thread_pinning_level(tmc::topology::thread_pinning_level Pin); - TMC_DECL void - set_thread_packing_strategy(tmc::topology::thread_packing_strategy Pack); + TMC_DECL void set_thread_pinning_level(tmc::topology::thread_pinning_level Pin); + TMC_DECL void set_thread_packing_strategy(tmc::topology::thread_packing_strategy Pack); TMC_DECL void set_thread_occupancy( - float ThreadOccupancy, tmc::topology::cpu_kind::value CpuKinds = - tmc::topology::cpu_kind::PERFORMANCE + float ThreadOccupancy, + tmc::topology::cpu_kind::value CpuKinds = tmc::topology::cpu_kind::PERFORMANCE ); #endif @@ -69,23 +65,18 @@ struct InitParams { TMC_DECL void set_thread_count(size_t ThreadCount); - TMC_DECL void - set_thread_post_run_hook(std::function const& Hook); + TMC_DECL void set_thread_post_run_hook(std::function const& Hook); TMC_DECL void set_thread_init_hook(std::function const& Hook); - TMC_DECL void - set_thread_teardown_hook(std::function const& Hook); + TMC_DECL void set_thread_teardown_hook(std::function const& Hook); - TMC_DECL void set_thread_post_run_hook( - std::function const& Hook - ); - TMC_DECL void set_thread_init_hook( - std::function const& Hook - ); - TMC_DECL void set_thread_teardown_hook( - std::function const& Hook - ); + TMC_DECL void + set_thread_post_run_hook(std::function const& Hook); + TMC_DECL void + set_thread_init_hook(std::function const& Hook); + TMC_DECL void + set_thread_teardown_hook(std::function const& Hook); TMC_DECL void set_spins(size_t Spins); From be3e9c04b93841d1470890825e829e42996a781d Mon Sep 17 00:00:00 2001 From: tzcnt Date: Thu, 25 Jun 2026 08:57:24 -0700 Subject: [PATCH 2/2] update comments with new default value --- include/tmc/ex_cpu.hpp | 71 +++++++++++++++------------------------ include/tmc/ex_cpu_st.hpp | 45 +++++++++---------------- 2 files changed, 44 insertions(+), 72 deletions(-) diff --git a/include/tmc/ex_cpu.hpp b/include/tmc/ex_cpu.hpp index 144c465c..8b7220c1 100644 --- a/include/tmc/ex_cpu.hpp +++ b/include/tmc/ex_cpu.hpp @@ -38,9 +38,8 @@ class ex_cpu { private: struct alignas(TMC_CACHE_LINE_SIZE) ThreadState { std::atomic yield_priority; // check to yield to a higher prio task - std::atomic - sleep_wait; // futex waker for this thread - tmc::detail::qu_inbox* inbox; // shared with group + std::atomic sleep_wait; // futex waker for this thread + tmc::detail::qu_inbox* inbox; // shared with group size_t group_size; TMC_DISABLE_WARNING_PADDED_BEGIN }; @@ -68,8 +67,7 @@ class ex_cpu { std::vector threads_by_priority_bitset; // TODO maybe shrink this by 1? prio 0 tasks cannot yield - tmc::detail::atomic_bitmap* - task_stopper_bitsets; // array of size PRIORITY_COUNT + tmc::detail::atomic_bitmap* task_stopper_bitsets; // array of size PRIORITY_COUNT ThreadState* thread_states; // array of size thread_count() @@ -130,8 +128,8 @@ class ex_cpu { // Returns a lambda closure that is executed on a worker thread TMC_DECL auto make_worker( - tmc::topology::thread_info Info, size_t PriorityRangeBegin, - size_t PriorityRangeEnd, ex_cpu::cldq_t** StealOrder, + tmc::topology::thread_info Info, size_t PriorityRangeBegin, size_t PriorityRangeEnd, + ex_cpu::cldq_t** StealOrder, std::atomic& InitThreadsBarrier, // will be nullptr if hwloc is not enabled tmc::detail::hwloc_unique_bitmap& CpuSet, @@ -142,14 +140,13 @@ class ex_cpu { // returns true if no tasks were found (caller should wait on cv) // returns false if thread stop requested (caller should exit) TMC_FORCE_INLINE inline bool try_run_some( - std::stop_token& ThreadStopToken, const size_t Slot, - const size_t PriorityRangeBegin, const size_t PriorityRangeEnd, - size_t& PrevPriority, bool& Spinning + std::stop_token& ThreadStopToken, const size_t Slot, const size_t PriorityRangeBegin, + const size_t PriorityRangeEnd, size_t& PrevPriority, bool& Spinning ); TMC_DECL void run_one( - tmc::work_item& Item, const size_t Slot, const size_t Prio, - size_t& PrevPriority, bool& Spinning + tmc::work_item& Item, const size_t Slot, const size_t Prio, size_t& PrevPriority, + bool& Spinning ); TMC_DECL std::coroutine_handle<> @@ -182,8 +179,8 @@ class ex_cpu { /// P-cores). It can be called multiple times to set different occupancies for /// different CPU kinds. TMC_DECL ex_cpu& set_thread_occupancy( - float ThreadOccupancy, tmc::topology::cpu_kind::value CpuKinds = - tmc::topology::cpu_kind::PERFORMANCE + float ThreadOccupancy, + tmc::topology::cpu_kind::value CpuKinds = tmc::topology::cpu_kind::PERFORMANCE ); /// Requires `TMC_USE_HWLOC`. @@ -210,8 +207,7 @@ class ex_cpu { /// Requires `TMC_USE_HWLOC`. /// Builder func to specify whether threads should be pinned/bound to /// specific cores, groups, or NUMA nodes. The default is GROUP. - TMC_DECL ex_cpu& - set_thread_pinning_level(tmc::topology::thread_pinning_level Level); + TMC_DECL ex_cpu& set_thread_pinning_level(tmc::topology::thread_pinning_level Level); /// Requires `TMC_USE_HWLOC`. /// Builder func to configure how threads should be allocated when the thread @@ -230,18 +226,16 @@ class ex_cpu { /// Builder func to set a hook that will be invoked before destruction of each /// thread owned by this executor, and passed information about this thread. /// This overload requires `TMC_USE_HWLOC`. - TMC_DECL ex_cpu& set_thread_teardown_hook( - std::function Hook - ); + TMC_DECL ex_cpu& + set_thread_teardown_hook(std::function Hook); /// Builder func to set a hook that will be invoked by each worker thread /// after it finishes running a batch of tasks, before entering the /// spinning/sleeping phase. If the hook returns true, the worker will /// immediately re-enter the run loop to check for more work. /// This overload requires `TMC_USE_HWLOC`. - TMC_DECL ex_cpu& set_thread_post_run_hook( - std::function Hook - ); + TMC_DECL ex_cpu& + set_thread_post_run_hook(std::function Hook); #endif /// Builder func to set the number of threads before calling `init()`. /// The maximum allowed value is equal to the number of bits on your @@ -290,13 +284,12 @@ class ex_cpu { /// Builder func to set the number of times that a thread worker will spin /// looking for new work when all queues appear to be empty before suspending /// the thread. Each spin is an asm("pause") followed by re-checking all - /// queues. The default is 4. + /// queues. The default is 0. TMC_DECL ex_cpu& set_spins(size_t Spins); /// Builder func to configure the work-stealing strategy used internally by /// this executor. The default is `HIERARCHY_MATRIX`. - TMC_DECL ex_cpu& - set_work_stealing_strategy(tmc::work_stealing_strategy Strategy); + TMC_DECL ex_cpu& set_work_stealing_strategy(tmc::work_stealing_strategy Strategy); /// Initializes the executor. If you want to customize the behavior, call the /// `set_X()` functions before calling `init()`. By default, uses hwloc to @@ -329,8 +322,7 @@ class ex_cpu { /// /// Rather than calling this directly, it is recommended to use the /// `tmc::post()` free function template. - TMC_DECL void - post(work_item&& Item, size_t Priority = 0, size_t ThreadHint = NO_HINT); + TMC_DECL void post(work_item&& Item, size_t Priority = 0, size_t ThreadHint = NO_HINT); /// Returns a pointer to the type erased `ex_any` version of this executor. /// This object shares a lifetime with this executor, and can be used for @@ -345,12 +337,10 @@ class ex_cpu { /// Rather than calling this directly, it is recommended to use the /// `tmc::post_bulk()` free function template. template - void post_bulk( - It&& Items, size_t Count, size_t Priority = 0, size_t ThreadHint = NO_HINT - ) { + void + post_bulk(It&& Items, size_t Count, size_t Priority = 0, size_t ThreadHint = NO_HINT) { clamp_priority(Priority); - bool fromExecThread = - tmc::detail::this_thread::executor() == &type_erased_this; + bool fromExecThread = tmc::detail::this_thread::executor() == &type_erased_this; bool allowedPriority = fromExecThread && threads_by_priority_bitset[Priority].test_bit(current_thread_index()); @@ -360,8 +350,7 @@ class ex_cpu { } if (ThreadHint < thread_count() && // Check allowed priority of the target thread, not the current thread - threads_by_priority_bitset[Priority].test_bit(ThreadHint)) - [[unlikely]] { + threads_by_priority_bitset[Priority].test_bit(ThreadHint)) [[unlikely]] { size_t enqueuedCount = thread_states[ThreadHint].inbox->try_push_bulk( static_cast(Items), Count, Priority ); @@ -375,8 +364,7 @@ class ex_cpu { if (Count > 0) [[likely]] { if (allowedPriority) [[likely]] { // Push to this thread's Chase-Lev deque for this priority. - cldq_t** producers = - static_cast(tmc::detail::this_thread::producers()); + cldq_t** producers = static_cast(tmc::detail::this_thread::producers()); cldq_t* myDeque = producers[q_ws[Priority].tlsArrayOffset]; myDeque->post_bulk(static_cast(Items), Count); } else { @@ -392,14 +380,12 @@ class ex_cpu { namespace detail { template <> struct executor_traits { - static TMC_DECL void post( - tmc::ex_cpu& ex, tmc::work_item&& Item, size_t Priority, size_t ThreadHint - ); + static TMC_DECL void + post(tmc::ex_cpu& ex, tmc::work_item&& Item, size_t Priority, size_t ThreadHint); template static inline void post_bulk( - tmc::ex_cpu& ex, It&& Items, size_t Count, size_t Priority, - size_t ThreadHint + tmc::ex_cpu& ex, It&& Items, size_t Count, size_t Priority, size_t ThreadHint ) { ex.post_bulk(static_cast(Items), Count, Priority, ThreadHint); } @@ -424,8 +410,7 @@ inline ex_cpu g_ex_cpu; constexpr ex_cpu& cpu_executor() { return tmc::detail::g_ex_cpu; } namespace detail { TMC_DECL tmc::task client_main_awaiter( - tmc::task ClientMainTask, - std::atomic* ExitCode_out + tmc::task ClientMainTask, std::atomic* ExitCode_out ); } diff --git a/include/tmc/ex_cpu_st.hpp b/include/tmc/ex_cpu_st.hpp index 1d84ff04..95bbb2c7 100644 --- a/include/tmc/ex_cpu_st.hpp +++ b/include/tmc/ex_cpu_st.hpp @@ -46,8 +46,7 @@ class ex_cpu_st { using task_queue_t = tmc::detail::qu_mpsc_blocking; tmc::detail::tiny_vec work_queues; // size() == PRIORITY_COUNT - tmc::detail::tiny_vec - private_work; // size() == PRIORITY_COUNT + tmc::detail::tiny_vec private_work; // size() == PRIORITY_COUNT // stop_source for the single worker thread std::stop_source thread_stopper; size_t spins; @@ -93,18 +92,15 @@ class ex_cpu_st { // will be nullptr if hwloc is not enabled void* Topology, // will be nullptr if hwloc is not enabled - tmc::detail::hwloc_unique_bitmap& CpuSet, - tmc::topology::cpu_kind::value Kind + tmc::detail::hwloc_unique_bitmap& CpuSet, tmc::topology::cpu_kind::value Kind ); // returns true if no tasks were found (caller should wait on cv) // returns false if thread stop requested (caller should exit) - TMC_DECL bool try_run_some( - std::stop_token& ThreadStopToken, size_t& PrevPriority, bool* DidSleep - ); + TMC_DECL bool + try_run_some(std::stop_token& ThreadStopToken, size_t& PrevPriority, bool* DidSleep); - TMC_DECL void - run_one(tmc::work_item& Item, const size_t Prio, size_t& PrevPriority); + TMC_DECL void run_one(tmc::work_item& Item, const size_t Prio, size_t& PrevPriority); TMC_DECL std::coroutine_handle<> dispatch(std::coroutine_handle<> Outer, size_t Priority); @@ -147,8 +143,7 @@ class ex_cpu_st { /// after it finishes running a batch of tasks, before entering the /// spinning/sleeping phase. If the hook returns true, the worker will /// immediately re-enter the run loop to check for more work. - TMC_DECL ex_cpu_st& - set_thread_post_run_hook(std::function Hook); + TMC_DECL ex_cpu_st& set_thread_post_run_hook(std::function Hook); /// Builder func to set a hook that will be invoked at the startup of the /// executor thread, and passed the ordinal index of the thread (which is @@ -158,13 +153,12 @@ class ex_cpu_st { /// Builder func to set a hook that will be invoked before destruction of each /// thread owned by this executor, and passed the ordinal index of the thread /// (which is always 0, since this is a single-threaded executor). - TMC_DECL ex_cpu_st& - set_thread_teardown_hook(std::function Hook); + TMC_DECL ex_cpu_st& set_thread_teardown_hook(std::function Hook); /// Builder func to set the number of times that a thread worker will spin /// looking for new work when all queues appear to be empty before suspending /// the thread. Each spin is an asm("pause") followed by re-checking all - /// queues. The default is 4. + /// queues. The default is 0. TMC_DECL ex_cpu_st& set_spins(size_t Spins); /// Initializes the executor. If you want to customize the behavior, call the @@ -195,8 +189,7 @@ class ex_cpu_st { /// /// Rather than calling this directly, it is recommended to use the /// `tmc::post()` free function template. - TMC_DECL void - post(work_item&& Item, size_t Priority = 0, size_t ThreadHint = NO_HINT); + TMC_DECL void post(work_item&& Item, size_t Priority = 0, size_t ThreadHint = NO_HINT); /// Returns a pointer to the type erased `ex_any` version of this executor. /// This object shares a lifetime with this executor, and can be used for @@ -211,12 +204,10 @@ class ex_cpu_st { /// Rather than calling this directly, it is recommended to use the /// `tmc::post_bulk()` free function template. template - void post_bulk( - It&& Items, size_t Count, size_t Priority = 0, size_t ThreadHint = NO_HINT - ) { + void + post_bulk(It&& Items, size_t Count, size_t Priority = 0, size_t ThreadHint = NO_HINT) { clamp_priority(Priority); - bool fromExecThread = - tmc::detail::this_thread::executor() == &type_erased_this; + bool fromExecThread = tmc::detail::this_thread::executor() == &type_erased_this; if (Count > 0) [[likely]] { // A zero ThreadHint indicates that reschedule() was called. In that // case we should use the external queue to force FIFO ordering. @@ -224,8 +215,7 @@ class ex_cpu_st { private_work[Priority].push_back_bulk(static_cast(Items), Count); request_yield(Priority); } else { - bool didWake = - work_queues[Priority].post_bulk(static_cast(Items), Count); + bool didWake = work_queues[Priority].post_bulk(static_cast(Items), Count); if (!didWake) { request_yield(Priority); } @@ -236,15 +226,12 @@ class ex_cpu_st { namespace detail { template <> struct executor_traits { - static TMC_DECL void post( - tmc::ex_cpu_st& ex, tmc::work_item&& Item, size_t Priority, - size_t ThreadHint - ); + static TMC_DECL void + post(tmc::ex_cpu_st& ex, tmc::work_item&& Item, size_t Priority, size_t ThreadHint); template static inline void post_bulk( - tmc::ex_cpu_st& ex, It&& Items, size_t Count, size_t Priority, - size_t ThreadHint + tmc::ex_cpu_st& ex, It&& Items, size_t Count, size_t Priority, size_t ThreadHint ) { ex.post_bulk(static_cast(Items), Count, Priority, ThreadHint); }