From 581b8a8b72bb751e3a6cc3ab61f4e46fd036301e Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 11:12:36 +0000 Subject: [PATCH 01/11] Add bimodal heap init distribution weights table Integer weights table encoding the bimodal object population shape observed in the lobsters benchmark. Two Gaussian modes: IMEMO peak at pool 0 and class/hash peak at pool 2. --- gc/default/default.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/gc/default/default.c b/gc/default/default.c index 1099d6e0dc11e5..c76ddc4e4f97ac 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -201,6 +201,31 @@ typedef struct ractor_newobj_cache { rb_ractor_newobj_heap_cache_t heap_caches[HEAP_COUNT]; } rb_ractor_newobj_cache_t; +/* + * Bimodal page distribution weights for heap initialization. + * + * Two Gaussian modes fitted to lobsters benchmark object populations: + * Mode 1: IMEMO/string peak at pool 0 (sigma=0.7) + * Mode 2: class/hash peak at pool 2 (sigma=0.4, weight=0.65) + * + * Raw: G(i,0,0.7) + 0.65*G(i,2,0.4), scaled by 10000 and rounded. + * The shape encodes the bimodal object population of a typical Ruby app: + * Pool 0 (40B): IMEMOs — call caches, method entries, cref chains + * Pool 2 (160B): CLASSes, HASHes, ICLASSes — framework infrastructure + * Pool 1 (80B): Valley — short strings, too large for IMEMOs, too small for classes + */ +static const unsigned int gc_heap_init_weights[HEAP_COUNT] = { + 10000, 3890, 6669, 287, 0 +}; +#define GC_HEAP_INIT_WEIGHT_SUM 20846 + +#ifndef GC_HEAP_INIT_TOTAL_PAGES +#define GC_HEAP_INIT_TOTAL_PAGES 195 /* ~3.0 MiB at 64 KiB/page */ +#endif +#ifndef GC_HEAP_INIT_FLOOR_PAGES +#define GC_HEAP_INIT_FLOOR_PAGES 6 +#endif + typedef struct { size_t heap_init_slots[HEAP_COUNT]; size_t heap_free_slots; From 4b7e4fa5ad60330ab2cdfac0af2ef2fa37439e23 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 11:17:22 +0000 Subject: [PATCH 02/11] Fix GC_HEAP_INIT_TOTAL_PAGES comment: 64 KiB pages give ~12 MiB not 3 MiB --- gc/default/default.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gc/default/default.c b/gc/default/default.c index c76ddc4e4f97ac..bf85c095fb07fd 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -220,7 +220,7 @@ static const unsigned int gc_heap_init_weights[HEAP_COUNT] = { #define GC_HEAP_INIT_WEIGHT_SUM 20846 #ifndef GC_HEAP_INIT_TOTAL_PAGES -#define GC_HEAP_INIT_TOTAL_PAGES 195 /* ~3.0 MiB at 64 KiB/page */ +#define GC_HEAP_INIT_TOTAL_PAGES 195 /* ~12 MiB at 64 KiB/page */ #endif #ifndef GC_HEAP_INIT_FLOOR_PAGES #define GC_HEAP_INIT_FLOOR_PAGES 6 From 569738f4dae14c2807316174228d9916a377a2fb Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 11:19:22 +0000 Subject: [PATCH 03/11] Add gc_heap_compute_init_slots for bimodal distribution Converts total page budget and floor pages into per-pool slot counts using the bimodal weights table. No behavioral change yet. --- gc/default/default.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/gc/default/default.c b/gc/default/default.c index bf85c095fb07fd..d72b9a50954df3 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -1605,6 +1605,20 @@ rb_gc_impl_get_measure_total_time(void *objspace_ptr) return objspace->flags.measure_gc; } +static void +gc_heap_compute_init_slots(size_t *init_slots, size_t total_pages, size_t floor_pages) +{ + size_t budget = total_pages - floor_pages * HEAP_COUNT; + + for (int i = 0; i < HEAP_COUNT; i++) { + size_t pages = floor_pages + + (budget * gc_heap_init_weights[i] + GC_HEAP_INIT_WEIGHT_SUM / 2) + / GC_HEAP_INIT_WEIGHT_SUM; + size_t slot_size = (size_t)((1 << i) * BASE_SLOT_SIZE); + init_slots[i] = pages * (HEAP_PAGE_SIZE / slot_size); + } +} + static size_t minimum_slots_for_heap(rb_objspace_t *objspace, rb_heap_t *heap) { From 861aed97bc3cecc9d54cedc3952120b6943d1a48 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 11:25:19 +0000 Subject: [PATCH 04/11] Guard gc_heap_compute_init_slots against underflow Clamp floor_total so misconfigured env vars (floor*HEAP_COUNT > total_pages) degrade to floor_pages per pool rather than wrapping to near SIZE_MAX. Add comment on intentional slot-count overcount. --- gc/default/default.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gc/default/default.c b/gc/default/default.c index d72b9a50954df3..1f125c546435c9 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -1608,13 +1608,17 @@ rb_gc_impl_get_measure_total_time(void *objspace_ptr) static void gc_heap_compute_init_slots(size_t *init_slots, size_t total_pages, size_t floor_pages) { - size_t budget = total_pages - floor_pages * HEAP_COUNT; + size_t floor_total = floor_pages * HEAP_COUNT; + if (floor_total > total_pages) floor_total = total_pages; + size_t budget = total_pages - floor_total; for (int i = 0; i < HEAP_COUNT; i++) { size_t pages = floor_pages + (budget * gc_heap_init_weights[i] + GC_HEAP_INIT_WEIGHT_SUM / 2) / GC_HEAP_INIT_WEIGHT_SUM; size_t slot_size = (size_t)((1 << i) * BASE_SLOT_SIZE); + /* Intentionally ignores page header alignment overhead; see heap_add_page. + * A slight overcount means at most one extra page allocated per pool. */ init_slots[i] = pages * (HEAP_PAGE_SIZE / slot_size); } } From 453e72e94ff7f4135677b5dbfb7d763f22d6559a Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 11:40:35 +0000 Subject: [PATCH 05/11] Use bimodal distribution for heap init slot defaults Replace uniform GC_HEAP_INIT_SLOTS=10000 per pool with proportional allocation from a bimodal page budget. Total RSS budget unchanged at ~12 MiB (195 pages at 64 KiB). Pools 0 and 2 get the majority of pages, matching observed IMEMO and class/hash populations. --- gc/default/default.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index 1f125c546435c9..2be6a20e1ea1f9 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -9582,11 +9582,10 @@ rb_gc_impl_objspace_init(void *objspace_ptr) #if RGENGC_ESTIMATE_OLDMALLOC objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min; #endif - /* Set size pools allocatable pages. */ - for (int i = 0; i < HEAP_COUNT; i++) { - /* Set the default value of heap_init_slots. */ - gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS; - } + /* Compute per-pool init slots from the bimodal page distribution. */ + gc_heap_compute_init_slots(gc_params.heap_init_slots, + GC_HEAP_INIT_TOTAL_PAGES, + GC_HEAP_INIT_FLOOR_PAGES); init_mark_stack(&objspace->mark_stack); From 31537fcf0c47e9d3bb4124090da2cd520364c593 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 11:46:26 +0000 Subject: [PATCH 06/11] Remove unused GC_HEAP_INIT_SLOTS macro The uniform default is replaced by gc_heap_compute_init_slots. The static initializer now uses { 0 } since objspace_init overwrites all entries via the bimodal distribution. --- gc/default/default.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index 2be6a20e1ea1f9..c9fee95575c2d9 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -109,9 +109,6 @@ # define RUBY_DEBUG_LOG(...) #endif -#ifndef GC_HEAP_INIT_SLOTS -#define GC_HEAP_INIT_SLOTS 10000 -#endif #ifndef GC_HEAP_FREE_SLOTS #define GC_HEAP_FREE_SLOTS 4096 #endif @@ -248,7 +245,7 @@ typedef struct { } ruby_gc_params_t; static ruby_gc_params_t gc_params = { - { GC_HEAP_INIT_SLOTS }, + { 0 }, /* set by gc_heap_compute_init_slots in rb_gc_impl_objspace_init */ GC_HEAP_FREE_SLOTS, GC_HEAP_GROWTH_FACTOR, GC_HEAP_GROWTH_MAX_SLOTS, From 0debd0c676ebfb97a580355057b512fa8f243e78 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 11:56:46 +0000 Subject: [PATCH 07/11] Add RUBY_GC_HEAP_INIT_TOTAL_PAGES/FLOOR_PAGES env vars Users can scale the bimodal distribution up or down without changing the shape. Per-pool RUBY_GC_HEAP_N_INIT_SLOTS still overrides individual pools. --- gc/default/default.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/gc/default/default.c b/gc/default/default.c index c9fee95575c2d9..e70e6a4ae12f9a 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -7966,6 +7966,14 @@ get_envparam_double(const char *name, double *default_value, double lower_bound, * where R is this factor and * N is the number of old objects just after last full GC. * + * * RUBY_GC_HEAP_INIT_TOTAL_PAGES (new) + * - Total page budget for initial heap allocation across all pools. + * Pages are distributed proportionally using a bimodal curve. + * Default: 195 (~12 MiB at 64 KiB/page). + * * RUBY_GC_HEAP_INIT_FLOOR_PAGES (new) + * - Minimum pages per pool. Ensures no pool starts empty. + * Default: 6. + * * * obsolete * * RUBY_FREE_MIN -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1) * * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1) @@ -7988,6 +7996,19 @@ rb_gc_impl_set_params(void *objspace_ptr) /* ok */ } + /* RUBY_GC_HEAP_INIT_TOTAL_PAGES / RUBY_GC_HEAP_INIT_FLOOR_PAGES: + * Recompute the bimodal init slot distribution if either is set. */ + { + size_t total_pages = GC_HEAP_INIT_TOTAL_PAGES; + size_t floor_pages = GC_HEAP_INIT_FLOOR_PAGES; + int recompute = 0; + recompute |= get_envparam_size("RUBY_GC_HEAP_INIT_TOTAL_PAGES", &total_pages, 0); + recompute |= get_envparam_size("RUBY_GC_HEAP_INIT_FLOOR_PAGES", &floor_pages, 0); + if (recompute) { + gc_heap_compute_init_slots(gc_params.heap_init_slots, total_pages, floor_pages); + } + } + for (int i = 0; i < HEAP_COUNT; i++) { char env_key[sizeof("RUBY_GC_HEAP_" "_INIT_SLOTS") + DECIMAL_SIZE_OF_BITS(sizeof(int) * CHAR_BIT)]; snprintf(env_key, sizeof(env_key), "RUBY_GC_HEAP_%d_INIT_SLOTS", i); From 9ff314648f8bd6c84369d1776fec6209b3ffeb2b Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 12:01:39 +0000 Subject: [PATCH 08/11] Add tests for bimodal heap init distribution Three tests: - Verify bimodal shape (pool 0 > pool 4, pool 2 > pool 4) - Verify RUBY_GC_HEAP_INIT_TOTAL_PAGES scaling - Verify per-pool env vars override bimodal defaults --- test/ruby/test_gc.rb | 58 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 60f04f8e10cf11..488dbfc888a605 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -553,6 +553,64 @@ def test_gc_parameter_init_slots RUBY end + def test_bimodal_heap_init_distribution + # The bimodal distribution gives pool 0 ~139k init slots (vs old uniform 10k). + # Prove it by filling pool 0 to 50k without triggering GC. + assert_separately([{}, "-W0"], __FILE__, __LINE__, <<~RUBY, timeout: 60) + gc_count = GC.stat(:count) + + # Fill pool 0 to 50,000 slots. Under the old uniform 10k default this + # would trigger GC; under the bimodal distribution (init_slots ~139k) + # it should not. + capa = (GC.stat_heap(0, :slot_size) - GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD] - (2 * RbConfig::SIZEOF["void*"])) / RbConfig::SIZEOF["void*"] + while GC.stat_heap(0, :heap_eden_slots) < 50_000 + Array.new(capa) + end + + assert_equal gc_count, GC.stat(:count), + "Filling pool 0 to 50k should not trigger GC (bimodal init_slots ~139k)" + RUBY + end + + def test_heap_init_total_pages_env + # RUBY_GC_HEAP_INIT_TOTAL_PAGES=400 raises pool 0 init_slots from ~139k to ~301k. + # Prove it by filling pool 0 to 200k without triggering GC. + env = { "RUBY_GC_HEAP_INIT_TOTAL_PAGES" => "400" } + + assert_separately([env, "-W0"], __FILE__, __LINE__, <<~RUBY, timeout: 60) + gc_count = GC.stat(:count) + + # Target 200k is above the default (~139k) but below the scaled (~301k). + # This should pass only because the env var raised the init_slots target. + capa = (GC.stat_heap(0, :slot_size) - GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD] - (2 * RbConfig::SIZEOF["void*"])) / RbConfig::SIZEOF["void*"] + while GC.stat_heap(0, :heap_eden_slots) < 200_000 + Array.new(capa) + end + + assert_equal gc_count, GC.stat(:count), + "Filling pool 0 to 200k should not trigger GC with total_pages=400" + RUBY + end + + def test_per_pool_init_slots_overrides_bimodal + # Per-pool env vars should override the bimodal defaults + env = { "RUBY_GC_HEAP_4_INIT_SLOTS" => "50000" } + + assert_separately([env, "-W0"], __FILE__, __LINE__, <<~RUBY, timeout: 60) + # Pool 4 normally gets very few init slots (~600). + # With override to 50000, it should have at least that many. + gc_count = GC.stat(:count) + + capa = (GC.stat_heap(4, :slot_size) - GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD] - (2 * RbConfig::SIZEOF["void*"])) / RbConfig::SIZEOF["void*"] + while GC.stat_heap(4, :heap_eden_slots) < 50000 + Array.new(capa) + end + + assert_equal gc_count, GC.stat(:count), + "Filling pool 4 to 50000 should not trigger GC when RUBY_GC_HEAP_4_INIT_SLOTS=50000" + RUBY + end + def test_profiler_enabled GC::Profiler.enable assert_equal(true, GC::Profiler.enabled?) From 35a8883020822c6dfb48f7b42a27649d0f8f341a Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 13:31:19 +0000 Subject: [PATCH 09/11] Remove misleading MiB comment from GC_HEAP_INIT_TOTAL_PAGES The page count is a budget for lazy allocation, not a fixed RSS cost. --- gc/default/default.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index e70e6a4ae12f9a..5f729c100b2505 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -217,7 +217,7 @@ static const unsigned int gc_heap_init_weights[HEAP_COUNT] = { #define GC_HEAP_INIT_WEIGHT_SUM 20846 #ifndef GC_HEAP_INIT_TOTAL_PAGES -#define GC_HEAP_INIT_TOTAL_PAGES 195 /* ~12 MiB at 64 KiB/page */ +#define GC_HEAP_INIT_TOTAL_PAGES 195 #endif #ifndef GC_HEAP_INIT_FLOOR_PAGES #define GC_HEAP_INIT_FLOOR_PAGES 6 @@ -7969,7 +7969,7 @@ get_envparam_double(const char *name, double *default_value, double lower_bound, * * RUBY_GC_HEAP_INIT_TOTAL_PAGES (new) * - Total page budget for initial heap allocation across all pools. * Pages are distributed proportionally using a bimodal curve. - * Default: 195 (~12 MiB at 64 KiB/page). + * Default: 195. * * RUBY_GC_HEAP_INIT_FLOOR_PAGES (new) * - Minimum pages per pool. Ensures no pool starts empty. * Default: 6. From 53f12b9475c979cdb4934baea842cff64a091b16 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 15:11:29 +0000 Subject: [PATCH 10/11] Seed allocatable_slots from init_slots, remove force-allocation bypass heap_prepare previously force-allocated pages outside the GC budget when total_slots < init_slots. With bimodal init_slots giving pool 0 ~139k slots, this meant up to 85 pages allocated invisibly to the GC, breaking the invariant that free_slots + allocatable_slots predicts when GC triggers. Instead, seed objspace->heap_pages.allocatable_slots with the sum of all init_slots at startup. Pages are now allocated through normal budget accounting. The init_slots floor is still enforced by gc_sweep_finish_heap and gc_marks_finish for shrinkage prevention. --- gc/default/default.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index 5f729c100b2505..94c213bd664709 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -2125,13 +2125,6 @@ heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap) { GC_ASSERT(heap->free_pages == NULL); - if (heap->total_slots < gc_params.heap_init_slots[heap - heaps] && - heap->sweeping_page == NULL) { - heap_page_allocate_and_initialize_force(objspace, heap); - GC_ASSERT(heap->free_pages != NULL); - return; - } - /* Continue incremental marking or lazy sweeping, if in any of those steps. */ gc_continue(objspace, heap); @@ -8016,6 +8009,12 @@ rb_gc_impl_set_params(void *objspace_ptr) get_envparam_size(env_key, &gc_params.heap_init_slots[i], 0); } + /* Re-seed allocation budget from (possibly overridden) init_slots. */ + objspace->heap_pages.allocatable_slots = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + objspace->heap_pages.allocatable_slots += gc_params.heap_init_slots[i]; + } + get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE); get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_SLOTS", &gc_params.growth_max_slots, 0); get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio, @@ -9605,6 +9604,12 @@ rb_gc_impl_objspace_init(void *objspace_ptr) GC_HEAP_INIT_TOTAL_PAGES, GC_HEAP_INIT_FLOOR_PAGES); + /* Seed the allocation budget so heaps can grow to their init_slots + * targets through normal page allocation. */ + for (int i = 0; i < HEAP_COUNT; i++) { + objspace->heap_pages.allocatable_slots += gc_params.heap_init_slots[i]; + } + init_mark_stack(&objspace->mark_stack); objspace->profile.invoke_time = getrusage_time(); From 3c9c960eca3ee6b1fd4117a656be1cc66aedbfac Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 26 Feb 2026 20:09:42 +0000 Subject: [PATCH 11/11] Filter hidden refs from ObjectSpace.each_object ObjectSpace.each_object already skips hidden objects directly, but it could still yield visible container objects that hold hidden internals. In that case, calling methods like Hash#inspect can raise NotImplementedError for a hidden T_ARRAY value. Add a lightweight direct-reference check for Array and Hash entries and skip containers that contain hidden/internal objects. This keeps hidden internals from leaking through enumeration and fixes iteration patterns that call inspect while traversing object space. This was exposed because the changes to the heap init changes the number of GC's that get run on ruby startup, leaving internal objects created during interpreter boot still in the heap until the first GC is run. --- gc.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/gc.c b/gc.c index 1af2ba41e7bf73..f03182665220f2 100644 --- a/gc.c +++ b/gc.c @@ -1692,6 +1692,48 @@ struct os_each_struct { VALUE of; }; +struct os_obj_contains_internal_ref_data { + bool found; +}; + +static int +os_obj_hash_contains_internal_ref_i(VALUE key, VALUE value, VALUE arg) +{ + struct os_obj_contains_internal_ref_data *data = (void *)arg; + + if ((!SPECIAL_CONST_P(key) && internal_object_p(key)) || + (!SPECIAL_CONST_P(value) && internal_object_p(value))) { + data->found = true; + return ST_STOP; + } + + return ST_CONTINUE; +} + +static bool +os_obj_contains_internal_ref_p(VALUE obj) +{ + switch (BUILTIN_TYPE(obj)) { + case T_ARRAY: + for (long i = 0; i < RARRAY_LEN(obj); i++) { + VALUE e = RARRAY_AREF(obj, i); + if (!SPECIAL_CONST_P(e) && internal_object_p(e)) { + return true; + } + } + return false; + + case T_HASH: { + struct os_obj_contains_internal_ref_data data = { false }; + rb_hash_foreach(obj, os_obj_hash_contains_internal_ref_i, (VALUE)&data); + return data.found; + } + + default: + return false; + } +} + static int os_obj_of_i(void *vstart, void *vend, size_t stride, void *data) { @@ -1699,13 +1741,12 @@ os_obj_of_i(void *vstart, void *vend, size_t stride, void *data) VALUE v = (VALUE)vstart; for (; v != (VALUE)vend; v += stride) { - if (!internal_object_p(v)) { - if (!oes->of || rb_obj_is_kind_of(v, oes->of)) { - if (!rb_multi_ractor_p() || rb_ractor_shareable_p(v)) { - rb_yield(v); - oes->num++; - } - } + if (internal_object_p(v)) continue; + if (oes->of && !rb_obj_is_kind_of(v, oes->of)) continue; + if (os_obj_contains_internal_ref_p(v)) continue; + if (!rb_multi_ractor_p() || rb_ractor_shareable_p(v)) { + rb_yield(v); + oes->num++; } }