diff --git a/include/tmc/channel.hpp b/include/tmc/channel.hpp index eab70ed3..a2ac2fd1 100644 --- a/include/tmc/channel.hpp +++ b/include/tmc/channel.hpp @@ -28,6 +28,7 @@ #include "tmc/detail/compat.hpp" #include "tmc/detail/concepts_awaitable.hpp" #include "tmc/detail/qu_storage.hpp" +#include "tmc/detail/timer_compat.hpp" #include "tmc/detail/tiny_lock.hpp" #include "tmc/ex_any.hpp" #include "tmc/task.hpp" diff --git a/include/tmc/detail/compat.hpp b/include/tmc/detail/compat.hpp index 4bc10d8a..c633ffb5 100644 --- a/include/tmc/detail/compat.hpp +++ b/include/tmc/detail/compat.hpp @@ -42,7 +42,7 @@ #ifdef __has_cpp_attribute -#if __has_cpp_attribute(clang::coro_await_elidable_argument) && \ +#if __has_cpp_attribute(clang::coro_await_elidable) && \ __has_cpp_attribute(clang::coro_await_elidable_argument) #define TMC_CORO_AWAIT_ELIDABLE [[clang::coro_await_elidable]] #define TMC_CORO_AWAIT_ELIDABLE_ARGUMENT [[clang::coro_await_elidable_argument]] @@ -62,8 +62,8 @@ #define TMC_SIZED_DEALLOCATION 0 #endif -#if defined(__x86_64__) || defined(_M_AMD64) || defined(i386) || \ - defined(__i386__) || defined(__i386) || defined(_M_IX86) +#if defined(__x86_64__) || defined(_M_AMD64) || defined(i386) || defined(__i386__) || \ + defined(__i386) || defined(_M_IX86) #ifdef _MSC_VER #include #else @@ -71,40 +71,26 @@ #endif #define TMC_CPU_X86 #define TMC_CPU_PAUSE _mm_pause -static inline size_t TMC_CPU_TIMESTAMP() noexcept { - return static_cast(__rdtsc()); -} -// Assume a 3.5GHz CPU if we can't get the value (on x86). -// Yes, this is hacky. Getting the real RDTSC freq requires -// waiting for another time source (system timer) and then dividing by -// that duration. This takes real time and would have to be done on -// startup. Using a 3.5GHz default means that slower processors will appear to -// be running faster, and vice versa. For the current usage of this (the -// clustering threshold in tmc::channel) this seems like reasonable behavior -// anyway. -static inline const size_t TMC_CPU_FREQ = 3500000000; -#elif defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(__aarch64__) || defined(__ARM_ACLE) +#elif defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) +#if defined(__aarch64__) || defined(_M_ARM64) || \ + (defined(__ARM_ARCH_PROFILE) && __ARM_ARCH_PROFILE == 'A') || \ + (defined(__ARM_ARCH_PROFILE) && __ARM_ARCH_PROFILE == 'R') #define TMC_CPU_ARM +#endif +#ifdef _MSC_VER +#include +#endif static inline void TMC_CPU_PAUSE() noexcept { - // Clang defines __yield intrinsic, but GCC doesn't, so we use asm + // Clang defines __yield intrinsic, but GCC doesn't, so we use asm where we + // know the ARM yield instruction is available. +#if defined(__aarch64__) || defined(_M_ARM64) || (defined(__ARM_ARCH) && __ARM_ARCH >= 7) +#if defined(_MSC_VER) + __yield(); +#else asm volatile("yield"); +#endif +#endif } -// Read the ARM "Virtual Counter" register. -// This ticks at a frequency independent of the processor frequency. -// https://developer.arm.com/documentation/ddi0406/cb/System-Level-Architecture/The-Generic-Timer/About-the-Generic-Timer/The-virtual-counter?lang=en -static inline size_t TMC_CPU_TIMESTAMP() noexcept { - size_t count; - asm volatile("mrs %0, cntvct_el0; " : "=r"(count)::"memory"); - return count; -} -// Read the ARM "Virtual Counter" frequency. -static inline size_t TMC_ARM_CPU_FREQ() noexcept { - size_t freq; - asm volatile("mrs %0, cntfrq_el0; isb; " : "=r"(freq)::"memory"); - return freq; -} -static inline const size_t TMC_CPU_FREQ = TMC_ARM_CPU_FREQ(); #elif defined(__loongarch__) && defined(__LP64__) // Use some barrier instructions to generate a delay, as there's // no instruction dedicated for the delay in spinlock :(. @@ -112,31 +98,7 @@ static inline void TMC_CPU_PAUSE() noexcept { for (int i = 0; i < 32; i++) asm volatile("ibar 0"); } -// Read the LoongArch stable counter -static inline size_t TMC_CPU_TIMESTAMP() noexcept { - size_t count; - asm volatile("rdtime.d %0, $zero" : "=r"(count)); - return count; -} -// Calculate the LoongArch stable counter frequency -static inline size_t TMC_LOONGARCH_CPU_FREQ() noexcept { - size_t cc_freq; - asm ("cpucfg %0, %1" : "=r"(cc_freq) : "r"(0x4)); - - size_t cc_mul_div; - asm ("cpucfg %0, %1" : "=r"(cc_mul_div) : "r"(0x5)); - - size_t cc_mul = cc_mul_div & 0xffff; - size_t cc_div = (cc_mul_div >> 16) & 0xffff; - - return cc_freq * cc_mul / cc_div; -} - -static inline const size_t TMC_CPU_FREQ = TMC_LOONGARCH_CPU_FREQ(); #elif defined(__riscv) -#if defined(__linux__) -#include -#endif #define TMC_CPU_RISCV static inline void TMC_CPU_PAUSE() noexcept { #if defined(__riscv_zihintpause) @@ -145,41 +107,9 @@ static inline void TMC_CPU_PAUSE() noexcept { asm volatile("nop" ::: "memory"); #endif } -static inline size_t TMC_RISCV_READ_TIMEBASE_FREQ() noexcept { -#if defined(__linux__) - const char* paths[] = { - "/proc/device-tree/cpus/timebase-frequency", - "/sys/firmware/devicetree/base/cpus/timebase-frequency", - }; - for (const char* path : paths) { - auto* file = std::fopen(path, "rb"); - if (file == nullptr) { - continue; - } - unsigned char bytes[8] = {}; - const auto size = std::fread(bytes, 1, sizeof(bytes), file); - std::fclose(file); - if (size == 4 || size == 8) { - size_t result = 0; - for (size_t i = 0; i != size; ++i) { - result = (result << 8) | bytes[i]; - } - if (result != 0) { - return result; - } - } - } -#endif - // This is only used for heuristics if the real RISC-V timebase is unavailable. - // Use the 2GHz clock frequency of SG2042-class server chips as a default. - return 2000000000; -} -static inline size_t TMC_CPU_TIMESTAMP() noexcept { - size_t count; - asm volatile("rdtime %0" : "=r"(count)); - return count; -} -static inline const size_t TMC_CPU_FREQ = TMC_RISCV_READ_TIMEBASE_FREQ(); +#else +// Fallback for unknown architectures +static inline void TMC_CPU_PAUSE() noexcept {} #endif // clang-format tries to collapse the pragmas into one line... diff --git a/include/tmc/detail/timer_compat.hpp b/include/tmc/detail/timer_compat.hpp new file mode 100644 index 00000000..a929785b --- /dev/null +++ b/include/tmc/detail/timer_compat.hpp @@ -0,0 +1,188 @@ +// Copyright (c) 2023-2026 Logan McDougall +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +// CPU timestamp-counter facilities (TMC_CPU_TIMESTAMP / TMC_CPU_FREQ) used by +// tmc::channel for load-based clustering. Split out of compat.hpp because +// channel.hpp is the only consumer. The architecture #if/#elif chain below +// mirrors the one in compat.hpp; when adding support for a new architecture, +// update both. + +#include +#include // IWYU pragma: keep + +#if defined(__x86_64__) || defined(_M_AMD64) || defined(i386) || defined(__i386__) || \ + defined(__i386) || defined(_M_IX86) +#ifdef _MSC_VER +#include +#else +#include +#endif +static inline size_t TMC_CPU_TIMESTAMP() noexcept { + return static_cast(__rdtsc()); +} +// Assume a 3.5GHz CPU if we can't get the value (on x86). +// Yes, this is hacky. Getting the real RDTSC freq requires +// waiting for another time source (system timer) and then dividing by +// that duration. This takes real time and would have to be done on +// startup. Using a 3.5GHz default means that slower processors will appear to +// be running faster, and vice versa. For the current usage of this (the +// clustering threshold in tmc::channel) this seems like reasonable behavior +// anyway. +static inline const size_t TMC_CPU_FREQ = 3500000000; +#elif defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) +#ifdef _MSC_VER +#include +#endif +#if defined(_MSC_VER) && defined(_M_ARM64) +#define TMC_ARM64_SYSREG(op0, op1, crn, crm, op2) \ + (((op0 & 1) << 14) | ((op1 & 7) << 11) | ((crn & 15) << 7) | ((crm & 15) << 3) | \ + ((op2 & 7) << 0)) +static inline size_t TMC_CPU_TIMESTAMP() noexcept { + return static_cast(_ReadStatusReg(TMC_ARM64_SYSREG(3, 3, 14, 0, 2))); +} +static inline size_t TMC_ARM_CPU_FREQ() noexcept { + return static_cast(_ReadStatusReg(TMC_ARM64_SYSREG(3, 3, 14, 0, 0))); +} +#undef TMC_ARM64_SYSREG +#elif defined(_MSC_VER) && defined(_M_ARM) +static inline size_t TMC_CPU_TIMESTAMP() noexcept { + return static_cast(_MoveFromCoprocessor64(15, 1, 14)); +} +static inline size_t TMC_ARM_CPU_FREQ() noexcept { + return static_cast(_MoveFromCoprocessor(15, 0, 14, 0, 0)); +} +#elif defined(__aarch64__) || defined(_M_ARM64) +// AArch64: the generic timer is read through dedicated system registers. +// Read the ARM "Virtual Counter" register. +// This ticks at a frequency independent of the processor frequency. +// https://developer.arm.com/documentation/ddi0406/cb/System-Level-Architecture/The-Generic-Timer/About-the-Generic-Timer/The-virtual-counter?lang=en +static inline size_t TMC_CPU_TIMESTAMP() noexcept { + size_t count; + asm volatile("mrs %0, cntvct_el0; " : "=r"(count)::"memory"); + return count; +} +// Read the ARM "Virtual Counter" frequency. +static inline size_t TMC_ARM_CPU_FREQ() noexcept { + size_t freq; + asm volatile("mrs %0, cntfrq_el0; isb; " : "=r"(freq)::"memory"); + return freq; +} +#elif defined(__ARM_ARCH) && __ARM_ARCH >= 7 && defined(__ARM_ARCH_PROFILE) && \ + (__ARM_ARCH_PROFILE == 'A' || __ARM_ARCH_PROFILE == 'R') +#if defined(__linux__) +#include +#include +#endif +// AArch32 (e.g. armhf): the AArch64 `cntvct_el0` / `cntfrq_el0` system +// registers don't exist here. The ARMv7-A Generic Timer exposes the same +// counters through CP15 coprocessor accesses instead. +// https://developer.arm.com/documentation/ddi0406/cb/System-Level-Architecture/The-Generic-Timer/Register-descriptions?lang=en +// The Generic Timer is optional on ARMv7-A, and user-mode access is controlled +// by the OS. On Linux, HWCAP_EVTSTRM indicates that the kernel configured the +// architected timer event stream; if that is absent, fall back to inert timing +// rather than trapping on an undefined/privileged CP15 access. +static inline bool TMC_ARM_HAS_GENERIC_TIMER_IMPL() noexcept { +#if defined(__linux__) && defined(HWCAP_EVTSTRM) + return (getauxval(AT_HWCAP) & HWCAP_EVTSTRM) != 0; +#else + return false; +#endif +} +static inline const bool TMC_ARM_HAS_GENERIC_TIMER = TMC_ARM_HAS_GENERIC_TIMER_IMPL(); +// Read the 64-bit "Virtual Counter" (CNTVCT) via MRRC into a register pair. +// size_t is 32-bit here, so we return only the low word - matching the 32-bit +// x86 path, which likewise truncates __rdtsc(). The counter ticks at a +// frequency independent of the processor frequency. +static inline size_t TMC_CPU_TIMESTAMP() noexcept { + if (!TMC_ARM_HAS_GENERIC_TIMER) { + return 0; + } + uint32_t lo, hi; + asm volatile("mrrc p15, 1, %0, %1, c14" : "=r"(lo), "=r"(hi)::"memory"); + (void)hi; + return lo; +} +// Read the "Virtual Counter" frequency (CNTFRQ, a 32-bit register) via MRC. +static inline size_t TMC_ARM_CPU_FREQ() noexcept { + if (!TMC_ARM_HAS_GENERIC_TIMER) { + return 1000000000; + } + uint32_t freq; + asm volatile("mrc p15, 0, %0, c14, c0, 0" : "=r"(freq)::"memory"); + return freq; +} +#else +static inline size_t TMC_CPU_TIMESTAMP() noexcept { return 0; } +static inline size_t TMC_ARM_CPU_FREQ() noexcept { return 1000000000; } +#endif +static inline const size_t TMC_CPU_FREQ = TMC_ARM_CPU_FREQ(); +#elif defined(__loongarch__) && defined(__LP64__) +// Read the LoongArch stable counter +static inline size_t TMC_CPU_TIMESTAMP() noexcept { + size_t count; + asm volatile("rdtime.d %0, $zero" : "=r"(count)); + return count; +} +// Calculate the LoongArch stable counter frequency +static inline size_t TMC_LOONGARCH_CPU_FREQ() noexcept { + size_t cc_freq; + asm("cpucfg %0, %1" : "=r"(cc_freq) : "r"(0x4)); + + size_t cc_mul_div; + asm("cpucfg %0, %1" : "=r"(cc_mul_div) : "r"(0x5)); + + size_t cc_mul = cc_mul_div & 0xffff; + size_t cc_div = (cc_mul_div >> 16) & 0xffff; + + return cc_freq * cc_mul / cc_div; +} + +static inline const size_t TMC_CPU_FREQ = TMC_LOONGARCH_CPU_FREQ(); +#elif defined(__riscv) +#if defined(__linux__) +#include +#endif +static inline size_t TMC_RISCV_READ_TIMEBASE_FREQ() noexcept { +#if defined(__linux__) + const char* paths[] = { + "/proc/device-tree/cpus/timebase-frequency", + "/sys/firmware/devicetree/base/cpus/timebase-frequency", + }; + for (const char* path : paths) { + auto* file = std::fopen(path, "rb"); + if (file == nullptr) { + continue; + } + unsigned char bytes[8] = {}; + const auto size = std::fread(bytes, 1, sizeof(bytes), file); + std::fclose(file); + if (size == 4 || size == 8) { + size_t result = 0; + for (size_t i = 0; i != size; ++i) { + result = (result << 8) | bytes[i]; + } + if (result != 0) { + return result; + } + } + } +#endif + // This is only used for heuristics if the real RISC-V timebase is unavailable. + // Use the 2GHz clock frequency of SG2042-class server chips as a default. + return 2000000000; +} +static inline size_t TMC_CPU_TIMESTAMP() noexcept { + size_t count; + asm volatile("rdtime %0" : "=r"(count)); + return count; +} +static inline const size_t TMC_CPU_FREQ = TMC_RISCV_READ_TIMEBASE_FREQ(); +#else +// Fallback for unknown architectures +static inline size_t TMC_CPU_TIMESTAMP() noexcept { return 0; } +static inline const size_t TMC_CPU_FREQ = 1000000000; +#endif