From 4b0e0ad2032bddd492ce161937c42be463a7202f Mon Sep 17 00:00:00 2001 From: henderkes Date: Fri, 12 Jun 2026 16:03:43 +0700 Subject: [PATCH] TSRM: make CG, EG, SCNG and AG compile-time offsets saves an offset load every time they're accessed --- TSRM/TSRM.c | 39 ++++++++++++++++++++++++++++++++++++++ TSRM/TSRM.h | 7 +++++++ Zend/zend.c | 9 ++++++--- Zend/zend_alloc.c | 5 +++-- Zend/zend_globals.h | 9 +++++++++ Zend/zend_globals_macros.h | 6 +++--- Zend/zend_portability.h | 2 +- main/main.c | 6 ++++++ 8 files changed, 74 insertions(+), 9 deletions(-) diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c index e99993204b6f..7cf7ae7732d5 100644 --- a/TSRM/TSRM.c +++ b/TSRM/TSRM.c @@ -36,6 +36,15 @@ struct _tsrm_tls_entry { tsrm_tls_entry *next; }; +#if defined(__cplusplus) || defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 202311L) +# define TSRM_STATIC_ASSERT(c, m) static_assert((c), m) +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define TSRM_STATIC_ASSERT(c, m) _Static_assert((c), m) +#else +# define TSRM_STATIC_ASSERT(c, m) +#endif +TSRM_STATIC_ASSERT(TSRM_ALIGNED_SIZE(sizeof(tsrm_tls_entry)) == TSRM_FAST_RESERVED_BASE, + "TSRM_FAST_RESERVED_BASE must equal the tsrm_tls_entry header size"); typedef struct { size_t size; @@ -319,6 +328,14 @@ TSRM_API void tsrm_reserve(size_t size) }/*}}}*/ +/* Reserve the front of the fast space for fixed-offset fast resources, so the + * bump-allocated ones (ts_allocate_fast_id) are placed after it. */ +TSRM_API void tsrm_reserve_fast_front(size_t size) +{/*{{{*/ + tsrm_reserved_pos = TSRM_ALIGNED_SIZE(size); +}/*}}}*/ + + /* allocates a new fast thread-safe-resource id */ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor) {/*{{{*/ @@ -368,6 +385,28 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz return *rsrc_id; }/*}}}*/ + +/* Allocate a fast resource id at a fixed, compile-time-constant offset. + * Like tsrm_reserve(), assumes single-threaded startup. */ +TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, size_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor) +{/*{{{*/ + size_t saved_pos = tsrm_reserved_pos; + + if (fixed_offset - TSRM_FAST_RESERVED_BASE > tsrm_reserved_size) { + TSRM_ERROR((TSRM_ERROR_LEVEL_ERROR, "Unable to allocate space for fixed fast resource")); + *rsrc_id = 0; + *offset = 0; + return 0; + } + tsrm_reserved_pos = fixed_offset - TSRM_FAST_RESERVED_BASE; + ts_allocate_fast_id(rsrc_id, offset, size, ctor, dtor); + /* keep ordinary (bump-allocated) fast resources past the high-water mark */ + if (tsrm_reserved_pos < saved_pos) { + tsrm_reserved_pos = saved_pos; + } + return *rsrc_id; +}/*}}}*/ + static void set_thread_local_storage_resource_to(tsrm_tls_entry *thread_resource) { tsrm_tls_set(thread_resource); diff --git a/TSRM/TSRM.h b/TSRM/TSRM.h index ea13552c8374..fa7f56904073 100644 --- a/TSRM/TSRM.h +++ b/TSRM/TSRM.h @@ -75,6 +75,8 @@ typedef void (*ts_allocate_dtor)(void *); #define THREAD_HASH_OF(thr,ts) (unsigned long)thr%(unsigned long)ts +#define TSRM_FAST_RESERVED_BASE TSRM_ALIGNED_SIZE(4 * sizeof(void *)) + #ifdef __cplusplus extern "C" { #endif @@ -94,6 +96,11 @@ TSRM_API ts_rsrc_id ts_allocate_id(ts_rsrc_id *rsrc_id, size_t size, ts_allocate TSRM_API void tsrm_reserve(size_t size); TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor); +/* Fast resources at caller-chosen, compile-time-constant offsets. The fixed + * prefix must be reserved after tsrm_reserve() and before any fast id. */ +TSRM_API void tsrm_reserve_fast_front(size_t size); +TSRM_API ts_rsrc_id ts_allocate_fast_id_at(ts_rsrc_id *rsrc_id, size_t *offset, size_t fixed_offset, size_t size, ts_allocate_ctor ctor, ts_allocate_dtor dtor); + /* fetches the requested resource for the current thread */ TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id); #define ts_resource(id) ts_resource_ex(id, NULL) diff --git a/Zend/zend.c b/Zend/zend.c index f16b1a30dbbc..9411b92a2018 100644 --- a/Zend/zend.c +++ b/Zend/zend.c @@ -1019,9 +1019,12 @@ void zend_startup(zend_utility_functions *utility_functions) /* {{{ */ zend_init_rsrc_list_dtors(); #ifdef ZTS - ts_allocate_fast_id(&compiler_globals_id, &compiler_globals_offset, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor); - ts_allocate_fast_id(&executor_globals_id, &executor_globals_offset, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor); - ts_allocate_fast_id(&language_scanner_globals_id, &language_scanner_globals_offset, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL); + ts_allocate_fast_id_at(&compiler_globals_id, &compiler_globals_offset, ZEND_CG_OFFSET, sizeof(zend_compiler_globals), (ts_allocate_ctor) compiler_globals_ctor, (ts_allocate_dtor) compiler_globals_dtor); + ts_allocate_fast_id_at(&executor_globals_id, &executor_globals_offset, ZEND_EG_OFFSET, sizeof(zend_executor_globals), (ts_allocate_ctor) executor_globals_ctor, (ts_allocate_dtor) executor_globals_dtor); + ts_allocate_fast_id_at(&language_scanner_globals_id, &language_scanner_globals_offset, ZEND_SCNG_OFFSET, sizeof(zend_php_scanner_globals), (ts_allocate_ctor) php_scanner_globals_ctor, NULL); + ZEND_ASSERT(compiler_globals_offset == ZEND_CG_OFFSET); + ZEND_ASSERT(executor_globals_offset == ZEND_EG_OFFSET); + ZEND_ASSERT(language_scanner_globals_offset == ZEND_SCNG_OFFSET); ts_allocate_fast_id(&ini_scanner_globals_id, &ini_scanner_globals_offset, sizeof(zend_ini_scanner_globals), (ts_allocate_ctor) ini_scanner_globals_ctor, NULL); compiler_globals = ts_resource(compiler_globals_id); executor_globals = ts_resource(executor_globals_id); diff --git a/Zend/zend_alloc.c b/Zend/zend_alloc.c index 942a8b8e1309..7a4469826054 100644 --- a/Zend/zend_alloc.c +++ b/Zend/zend_alloc.c @@ -2614,7 +2614,7 @@ typedef struct _zend_alloc_globals { #ifdef ZTS static int alloc_globals_id; static size_t alloc_globals_offset; -# define AG(v) ZEND_TSRMG_FAST(alloc_globals_offset, zend_alloc_globals *, v) +# define AG(v) ZEND_TSRMG_FAST(ZEND_AG_OFFSET, zend_alloc_globals *, v) #else # define AG(v) (alloc_globals.v) static zend_alloc_globals alloc_globals; @@ -3335,7 +3335,8 @@ ZEND_API void start_memory_manager(void) # endif #endif #ifdef ZTS - ts_allocate_fast_id(&alloc_globals_id, &alloc_globals_offset, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor); + ts_allocate_fast_id_at(&alloc_globals_id, &alloc_globals_offset, ZEND_AG_OFFSET, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor); + ZEND_ASSERT(alloc_globals_offset == ZEND_AG_OFFSET); #else alloc_globals_ctor(&alloc_globals); #endif diff --git a/Zend/zend_globals.h b/Zend/zend_globals.h index 8257df32e831..5ab6df774386 100644 --- a/Zend/zend_globals.h +++ b/Zend/zend_globals.h @@ -329,6 +329,15 @@ struct _zend_executor_globals { void *reserved[ZEND_MAX_RESERVED_RESOURCES]; }; +#ifdef ZTS +/* Fixed compile-time offsets of the hot globals in the TSRM fast space. + * AG is last, sizeof(zend_alloc_globals) isn't header-visible. */ +# define ZEND_CG_OFFSET (TSRM_FAST_RESERVED_BASE) +# define ZEND_EG_OFFSET (ZEND_CG_OFFSET + TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals))) +# define ZEND_SCNG_OFFSET (ZEND_EG_OFFSET + TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals))) +# define ZEND_AG_OFFSET (ZEND_SCNG_OFFSET + TSRM_ALIGNED_SIZE(sizeof(zend_php_scanner_globals))) +#endif + #define EG_FLAGS_INITIAL (0) #define EG_FLAGS_IN_SHUTDOWN (1<<0) #define EG_FLAGS_OBJECT_STORE_NO_REUSE (1<<1) diff --git a/Zend/zend_globals_macros.h b/Zend/zend_globals_macros.h index bde10a0989d1..2d2948e50a86 100644 --- a/Zend/zend_globals_macros.h +++ b/Zend/zend_globals_macros.h @@ -30,7 +30,7 @@ BEGIN_EXTERN_C() /* Compiler */ #ifdef ZTS -# define CG(v) ZEND_TSRMG_FAST(compiler_globals_offset, zend_compiler_globals *, v) +# define CG(v) ZEND_TSRMG_FAST(ZEND_CG_OFFSET, zend_compiler_globals *, v) #else # define CG(v) (compiler_globals.v) extern ZEND_API struct _zend_compiler_globals compiler_globals; @@ -40,7 +40,7 @@ ZEND_API int zendparse(void); /* Executor */ #ifdef ZTS -# define EG(v) ZEND_TSRMG_FAST(executor_globals_offset, zend_executor_globals *, v) +# define EG(v) ZEND_TSRMG_FAST(ZEND_EG_OFFSET, zend_executor_globals *, v) #else # define EG(v) (executor_globals.v) extern ZEND_API zend_executor_globals executor_globals; @@ -48,7 +48,7 @@ extern ZEND_API zend_executor_globals executor_globals; /* Language Scanner */ #ifdef ZTS -# define LANG_SCNG(v) ZEND_TSRMG_FAST(language_scanner_globals_offset, zend_php_scanner_globals *, v) +# define LANG_SCNG(v) ZEND_TSRMG_FAST(ZEND_SCNG_OFFSET, zend_php_scanner_globals *, v) extern ZEND_API ts_rsrc_id language_scanner_globals_id; extern ZEND_API size_t language_scanner_globals_offset; #else diff --git a/Zend/zend_portability.h b/Zend/zend_portability.h index ccad24682fdb..33ea4230b6a4 100644 --- a/Zend/zend_portability.h +++ b/Zend/zend_portability.h @@ -862,7 +862,7 @@ extern "C++" { /** @deprecated */ #define ZEND_CGG_DIAGNOSTIC_IGNORED_END ZEND_DIAGNOSTIC_IGNORED_END -#if defined(__cplusplus) +#if defined(__cplusplus) || defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 202311L) # define ZEND_STATIC_ASSERT(c, m) static_assert((c), m) #elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ # define ZEND_STATIC_ASSERT(c, m) _Static_assert((c), m) diff --git a/main/main.c b/main/main.c index 6bda55ac8746..c1a09b5b3e6a 100644 --- a/main/main.c +++ b/main/main.c @@ -2842,6 +2842,12 @@ PHPAPI bool php_tsrm_startup_ex(int expected_threads) { bool ret = tsrm_startup(expected_threads, 1, 0, NULL); php_reserve_tsrm_memory(); + /* Must reserve exactly the prefix laid out by ZEND_*_OFFSET (zend_globals.h). */ + tsrm_reserve_fast_front( + TSRM_ALIGNED_SIZE(sizeof(zend_compiler_globals)) + + TSRM_ALIGNED_SIZE(sizeof(zend_executor_globals)) + + TSRM_ALIGNED_SIZE(sizeof(zend_php_scanner_globals)) + + TSRM_ALIGNED_SIZE(zend_mm_globals_size())); // AG size, exposed through function call (void)ts_resource(0); return ret; }