Skip to content

reduce default executor spins from 4 to 0 #262

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tzcnt merged 2 commits into from
Jun 25, 2026
Merged

reduce default executor spins from 4 to 0 #262

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d8b26b7
default executor spins 4 -> 0
tzcnt Jun 25, 2026
be3e9c0
update comments with new default value
tzcnt Jun 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 15 additions & 24 deletions include/tmc/detail/init_params.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,15 @@ namespace detail {
struct InitParams {
size_t priority_count = 0;
size_t thread_count = 0;
size_t spins = 4;
size_t spins = 0;
work_stealing_strategy strategy = work_stealing_strategy::HIERARCHY_MATRIX;
std::vector<float> thread_occupancy = {};
std::function<void(tmc::topology::thread_info)> thread_init_hook = nullptr;
std::function<void(tmc::topology::thread_info)> thread_teardown_hook =
nullptr;
std::function<void(tmc::topology::thread_info)> thread_teardown_hook = nullptr;
std::function<bool(tmc::topology::thread_info)> thread_post_run_hook = nullptr;
#ifdef TMC_USE_HWLOC
std::vector<tmc::topology::topology_filter> partitions = {};
tmc::topology::thread_pinning_level pin =
tmc::topology::thread_pinning_level::GROUP;
tmc::topology::thread_pinning_level pin = tmc::topology::thread_pinning_level::GROUP;
tmc::topology::thread_packing_strategy pack =
tmc::topology::thread_packing_strategy::PACK;

Expand All @@ -52,13 +50,11 @@ struct InitParams {
std::vector<PriorityRange> priority_ranges = {};

TMC_DECL void add_partition(tmc::topology::topology_filter const& Filter);
TMC_DECL void
set_thread_pinning_level(tmc::topology::thread_pinning_level Pin);
TMC_DECL void
set_thread_packing_strategy(tmc::topology::thread_packing_strategy Pack);
TMC_DECL void set_thread_pinning_level(tmc::topology::thread_pinning_level Pin);
TMC_DECL void set_thread_packing_strategy(tmc::topology::thread_packing_strategy Pack);
TMC_DECL void set_thread_occupancy(
float ThreadOccupancy, tmc::topology::cpu_kind::value CpuKinds =
tmc::topology::cpu_kind::PERFORMANCE
float ThreadOccupancy,
tmc::topology::cpu_kind::value CpuKinds = tmc::topology::cpu_kind::PERFORMANCE
);

#endif
Expand All @@ -69,23 +65,18 @@ struct InitParams {

TMC_DECL void set_thread_count(size_t ThreadCount);

TMC_DECL void
set_thread_post_run_hook(std::function<bool(size_t)> const& Hook);
TMC_DECL void set_thread_post_run_hook(std::function<bool(size_t)> const& Hook);

TMC_DECL void set_thread_init_hook(std::function<void(size_t)> const& Hook);

TMC_DECL void
set_thread_teardown_hook(std::function<void(size_t)> const& Hook);
TMC_DECL void set_thread_teardown_hook(std::function<void(size_t)> const& Hook);

TMC_DECL void set_thread_post_run_hook(
std::function<bool(tmc::topology::thread_info)> const& Hook
);
TMC_DECL void set_thread_init_hook(
std::function<void(tmc::topology::thread_info)> const& Hook
);
TMC_DECL void set_thread_teardown_hook(
std::function<void(tmc::topology::thread_info)> const& Hook
);
TMC_DECL void
set_thread_post_run_hook(std::function<bool(tmc::topology::thread_info)> const& Hook);
TMC_DECL void
set_thread_init_hook(std::function<void(tmc::topology::thread_info)> const& Hook);
TMC_DECL void
set_thread_teardown_hook(std::function<void(tmc::topology::thread_info)> const& Hook);

TMC_DECL void set_spins(size_t Spins);

Expand Down
71 changes: 28 additions & 43 deletions include/tmc/ex_cpu.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,8 @@ class ex_cpu {
private:
struct alignas(TMC_CACHE_LINE_SIZE) ThreadState {
std::atomic<size_t> yield_priority; // check to yield to a higher prio task
std::atomic<tmc::detail::atomic_waker_t>
sleep_wait; // futex waker for this thread
tmc::detail::qu_inbox<tmc::work_item, 4096>* inbox; // shared with group
std::atomic<tmc::detail::atomic_waker_t> sleep_wait; // futex waker for this thread
tmc::detail::qu_inbox<tmc::work_item, 4096>* inbox; // shared with group
size_t group_size;
TMC_DISABLE_WARNING_PADDED_BEGIN
};
Expand Down Expand Up @@ -68,8 +67,7 @@ class ex_cpu {
std::vector<tmc::detail::bitmap> threads_by_priority_bitset;

// TODO maybe shrink this by 1? prio 0 tasks cannot yield
tmc::detail::atomic_bitmap*
task_stopper_bitsets; // array of size PRIORITY_COUNT
tmc::detail::atomic_bitmap* task_stopper_bitsets; // array of size PRIORITY_COUNT

ThreadState* thread_states; // array of size thread_count()

Expand Down Expand Up @@ -130,8 +128,8 @@ class ex_cpu {

// Returns a lambda closure that is executed on a worker thread
TMC_DECL auto make_worker(
tmc::topology::thread_info Info, size_t PriorityRangeBegin,
size_t PriorityRangeEnd, ex_cpu::cldq_t** StealOrder,
tmc::topology::thread_info Info, size_t PriorityRangeBegin, size_t PriorityRangeEnd,
ex_cpu::cldq_t** StealOrder,
std::atomic<tmc::detail::atomic_wait_t>& InitThreadsBarrier,
// will be nullptr if hwloc is not enabled
tmc::detail::hwloc_unique_bitmap& CpuSet,
Expand All @@ -142,14 +140,13 @@ class ex_cpu {
// returns true if no tasks were found (caller should wait on cv)
// returns false if thread stop requested (caller should exit)
TMC_FORCE_INLINE inline bool try_run_some(
std::stop_token& ThreadStopToken, const size_t Slot,
const size_t PriorityRangeBegin, const size_t PriorityRangeEnd,
size_t& PrevPriority, bool& Spinning
std::stop_token& ThreadStopToken, const size_t Slot, const size_t PriorityRangeBegin,
const size_t PriorityRangeEnd, size_t& PrevPriority, bool& Spinning
);

TMC_DECL void run_one(
tmc::work_item& Item, const size_t Slot, const size_t Prio,
size_t& PrevPriority, bool& Spinning
tmc::work_item& Item, const size_t Slot, const size_t Prio, size_t& PrevPriority,
bool& Spinning
);

TMC_DECL std::coroutine_handle<>
Expand Down Expand Up @@ -182,8 +179,8 @@ class ex_cpu {
/// P-cores). It can be called multiple times to set different occupancies for
/// different CPU kinds.
TMC_DECL ex_cpu& set_thread_occupancy(
float ThreadOccupancy, tmc::topology::cpu_kind::value CpuKinds =
tmc::topology::cpu_kind::PERFORMANCE
float ThreadOccupancy,
tmc::topology::cpu_kind::value CpuKinds = tmc::topology::cpu_kind::PERFORMANCE
);

/// Requires `TMC_USE_HWLOC`.
Expand All @@ -210,8 +207,7 @@ class ex_cpu {
/// Requires `TMC_USE_HWLOC`.
/// Builder func to specify whether threads should be pinned/bound to
/// specific cores, groups, or NUMA nodes. The default is GROUP.
TMC_DECL ex_cpu&
set_thread_pinning_level(tmc::topology::thread_pinning_level Level);
TMC_DECL ex_cpu& set_thread_pinning_level(tmc::topology::thread_pinning_level Level);

/// Requires `TMC_USE_HWLOC`.
/// Builder func to configure how threads should be allocated when the thread
Expand All @@ -230,18 +226,16 @@ class ex_cpu {
/// Builder func to set a hook that will be invoked before destruction of each
/// thread owned by this executor, and passed information about this thread.
/// This overload requires `TMC_USE_HWLOC`.
TMC_DECL ex_cpu& set_thread_teardown_hook(
std::function<void(tmc::topology::thread_info)> Hook
);
TMC_DECL ex_cpu&
set_thread_teardown_hook(std::function<void(tmc::topology::thread_info)> Hook);

/// Builder func to set a hook that will be invoked by each worker thread
/// after it finishes running a batch of tasks, before entering the
/// spinning/sleeping phase. If the hook returns true, the worker will
/// immediately re-enter the run loop to check for more work.
/// This overload requires `TMC_USE_HWLOC`.
TMC_DECL ex_cpu& set_thread_post_run_hook(
std::function<bool(tmc::topology::thread_info)> Hook
);
TMC_DECL ex_cpu&
set_thread_post_run_hook(std::function<bool(tmc::topology::thread_info)> Hook);
#endif
/// Builder func to set the number of threads before calling `init()`.
/// The maximum allowed value is equal to the number of bits on your
Expand Down Expand Up @@ -290,13 +284,12 @@ class ex_cpu {
/// Builder func to set the number of times that a thread worker will spin
/// looking for new work when all queues appear to be empty before suspending
/// the thread. Each spin is an asm("pause") followed by re-checking all
/// queues. The default is 4.
/// queues. The default is 0.
TMC_DECL ex_cpu& set_spins(size_t Spins);

/// Builder func to configure the work-stealing strategy used internally by
/// this executor. The default is `HIERARCHY_MATRIX`.
TMC_DECL ex_cpu&
set_work_stealing_strategy(tmc::work_stealing_strategy Strategy);
TMC_DECL ex_cpu& set_work_stealing_strategy(tmc::work_stealing_strategy Strategy);

/// Initializes the executor. If you want to customize the behavior, call the
/// `set_X()` functions before calling `init()`. By default, uses hwloc to
Expand Down Expand Up @@ -329,8 +322,7 @@ class ex_cpu {
///
/// Rather than calling this directly, it is recommended to use the
/// `tmc::post()` free function template.
TMC_DECL void
post(work_item&& Item, size_t Priority = 0, size_t ThreadHint = NO_HINT);
TMC_DECL void post(work_item&& Item, size_t Priority = 0, size_t ThreadHint = NO_HINT);

/// Returns a pointer to the type erased `ex_any` version of this executor.
/// This object shares a lifetime with this executor, and can be used for
Expand All @@ -345,12 +337,10 @@ class ex_cpu {
/// Rather than calling this directly, it is recommended to use the
/// `tmc::post_bulk()` free function template.
template <typename It>
void post_bulk(
It&& Items, size_t Count, size_t Priority = 0, size_t ThreadHint = NO_HINT
) {
void
post_bulk(It&& Items, size_t Count, size_t Priority = 0, size_t ThreadHint = NO_HINT) {
clamp_priority(Priority);
bool fromExecThread =
tmc::detail::this_thread::executor() == &type_erased_this;
bool fromExecThread = tmc::detail::this_thread::executor() == &type_erased_this;
bool allowedPriority =
fromExecThread &&
threads_by_priority_bitset[Priority].test_bit(current_thread_index());
Expand All @@ -360,8 +350,7 @@ class ex_cpu {
}
if (ThreadHint < thread_count() &&
// Check allowed priority of the target thread, not the current thread
threads_by_priority_bitset[Priority].test_bit(ThreadHint))
[[unlikely]] {
threads_by_priority_bitset[Priority].test_bit(ThreadHint)) [[unlikely]] {
size_t enqueuedCount = thread_states[ThreadHint].inbox->try_push_bulk(
static_cast<It&&>(Items), Count, Priority
);
Expand All @@ -375,8 +364,7 @@ class ex_cpu {
if (Count > 0) [[likely]] {
if (allowedPriority) [[likely]] {
// Push to this thread's Chase-Lev deque for this priority.
cldq_t** producers =
static_cast<cldq_t**>(tmc::detail::this_thread::producers());
cldq_t** producers = static_cast<cldq_t**>(tmc::detail::this_thread::producers());
cldq_t* myDeque = producers[q_ws[Priority].tlsArrayOffset];
myDeque->post_bulk(static_cast<It&&>(Items), Count);
} else {
Expand All @@ -392,14 +380,12 @@ class ex_cpu {

namespace detail {
template <> struct executor_traits<tmc::ex_cpu> {
static TMC_DECL void post(
tmc::ex_cpu& ex, tmc::work_item&& Item, size_t Priority, size_t ThreadHint
);
static TMC_DECL void
post(tmc::ex_cpu& ex, tmc::work_item&& Item, size_t Priority, size_t ThreadHint);

template <typename It>
static inline void post_bulk(
tmc::ex_cpu& ex, It&& Items, size_t Count, size_t Priority,
size_t ThreadHint
tmc::ex_cpu& ex, It&& Items, size_t Count, size_t Priority, size_t ThreadHint
) {
ex.post_bulk(static_cast<It&&>(Items), Count, Priority, ThreadHint);
}
Expand All @@ -424,8 +410,7 @@ inline ex_cpu g_ex_cpu;
constexpr ex_cpu& cpu_executor() { return tmc::detail::g_ex_cpu; }
namespace detail {
TMC_DECL tmc::task<void> client_main_awaiter(
tmc::task<int> ClientMainTask,
std::atomic<tmc::detail::atomic_wait_t>* ExitCode_out
tmc::task<int> ClientMainTask, std::atomic<tmc::detail::atomic_wait_t>* ExitCode_out
);
}

Expand Down
45 changes: 16 additions & 29 deletions include/tmc/ex_cpu_st.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,7 @@ class ex_cpu_st {
using task_queue_t = tmc::detail::qu_mpsc_blocking<work_item, qu_cfg>;
tmc::detail::tiny_vec<task_queue_t> work_queues; // size() == PRIORITY_COUNT

tmc::detail::tiny_vec<tmc::detail::tiny_stack>
private_work; // size() == PRIORITY_COUNT
tmc::detail::tiny_vec<tmc::detail::tiny_stack> private_work; // size() == PRIORITY_COUNT
// stop_source for the single worker thread
std::stop_source thread_stopper;
size_t spins;
Expand Down Expand Up @@ -93,18 +92,15 @@ class ex_cpu_st {
// will be nullptr if hwloc is not enabled
void* Topology,
// will be nullptr if hwloc is not enabled
tmc::detail::hwloc_unique_bitmap& CpuSet,
tmc::topology::cpu_kind::value Kind
tmc::detail::hwloc_unique_bitmap& CpuSet, tmc::topology::cpu_kind::value Kind
);

// returns true if no tasks were found (caller should wait on cv)
// returns false if thread stop requested (caller should exit)
TMC_DECL bool try_run_some(
std::stop_token& ThreadStopToken, size_t& PrevPriority, bool* DidSleep
);
TMC_DECL bool
try_run_some(std::stop_token& ThreadStopToken, size_t& PrevPriority, bool* DidSleep);

TMC_DECL void
run_one(tmc::work_item& Item, const size_t Prio, size_t& PrevPriority);
TMC_DECL void run_one(tmc::work_item& Item, const size_t Prio, size_t& PrevPriority);

TMC_DECL std::coroutine_handle<>
dispatch(std::coroutine_handle<> Outer, size_t Priority);
Expand Down Expand Up @@ -147,8 +143,7 @@ class ex_cpu_st {
/// after it finishes running a batch of tasks, before entering the
/// spinning/sleeping phase. If the hook returns true, the worker will
/// immediately re-enter the run loop to check for more work.
TMC_DECL ex_cpu_st&
set_thread_post_run_hook(std::function<bool(size_t)> Hook);
TMC_DECL ex_cpu_st& set_thread_post_run_hook(std::function<bool(size_t)> Hook);

/// Builder func to set a hook that will be invoked at the startup of the
/// executor thread, and passed the ordinal index of the thread (which is
Expand All @@ -158,13 +153,12 @@ class ex_cpu_st {
/// Builder func to set a hook that will be invoked before destruction of each
/// thread owned by this executor, and passed the ordinal index of the thread
/// (which is always 0, since this is a single-threaded executor).
TMC_DECL ex_cpu_st&
set_thread_teardown_hook(std::function<void(size_t)> Hook);
TMC_DECL ex_cpu_st& set_thread_teardown_hook(std::function<void(size_t)> Hook);

/// Builder func to set the number of times that a thread worker will spin
/// looking for new work when all queues appear to be empty before suspending
/// the thread. Each spin is an asm("pause") followed by re-checking all
/// queues. The default is 4.
/// queues. The default is 0.
TMC_DECL ex_cpu_st& set_spins(size_t Spins);

/// Initializes the executor. If you want to customize the behavior, call the
Expand Down Expand Up @@ -195,8 +189,7 @@ class ex_cpu_st {
///
/// Rather than calling this directly, it is recommended to use the
/// `tmc::post()` free function template.
TMC_DECL void
post(work_item&& Item, size_t Priority = 0, size_t ThreadHint = NO_HINT);
TMC_DECL void post(work_item&& Item, size_t Priority = 0, size_t ThreadHint = NO_HINT);

/// Returns a pointer to the type erased `ex_any` version of this executor.
/// This object shares a lifetime with this executor, and can be used for
Expand All @@ -211,21 +204,18 @@ class ex_cpu_st {
/// Rather than calling this directly, it is recommended to use the
/// `tmc::post_bulk()` free function template.
template <typename It>
void post_bulk(
It&& Items, size_t Count, size_t Priority = 0, size_t ThreadHint = NO_HINT
) {
void
post_bulk(It&& Items, size_t Count, size_t Priority = 0, size_t ThreadHint = NO_HINT) {
clamp_priority(Priority);
bool fromExecThread =
tmc::detail::this_thread::executor() == &type_erased_this;
bool fromExecThread = tmc::detail::this_thread::executor() == &type_erased_this;
if (Count > 0) [[likely]] {
// A zero ThreadHint indicates that reschedule() was called. In that
// case we should use the external queue to force FIFO ordering.
if (fromExecThread && ThreadHint != 0) [[likely]] {
private_work[Priority].push_back_bulk(static_cast<It&&>(Items), Count);
request_yield(Priority);
} else {
bool didWake =
work_queues[Priority].post_bulk(static_cast<It&&>(Items), Count);
bool didWake = work_queues[Priority].post_bulk(static_cast<It&&>(Items), Count);
if (!didWake) {
request_yield(Priority);
}
Expand All @@ -236,15 +226,12 @@ class ex_cpu_st {

namespace detail {
template <> struct executor_traits<tmc::ex_cpu_st> {
static TMC_DECL void post(
tmc::ex_cpu_st& ex, tmc::work_item&& Item, size_t Priority,
size_t ThreadHint
);
static TMC_DECL void
post(tmc::ex_cpu_st& ex, tmc::work_item&& Item, size_t Priority, size_t ThreadHint);

template <typename It>
static inline void post_bulk(
tmc::ex_cpu_st& ex, It&& Items, size_t Count, size_t Priority,
size_t ThreadHint
tmc::ex_cpu_st& ex, It&& Items, size_t Count, size_t Priority, size_t ThreadHint
) {
ex.post_bulk(static_cast<It&&>(Items), Count, Priority, ThreadHint);
}
Expand Down
Loading