Optimize garbage collection with a generational GC

doc/src/memory-management.md

@@ -922,3 +922,40 @@ match binaries, as with the case of refc binaries on the process heap.
 #### Deletion
 
 Once all terms have been copied from the old heap to the new heap, and once the MSO list has been swept for unreachable references, the old heap is simply discarded via the `free` function.
+
+### Generational Garbage Collection
+
+The garbage collection described above is a *full sweep*: every live term is copied from the old heap to the new heap and the entire old heap is freed. While correct, this can be expensive for processes with large heaps, because long-lived data that has already survived previous collections must be copied again each time.
+
+AtomVM implements *generational* (or *minor*) garbage collection to reduce this cost, using the same approach as BEAM. The key observation is that most terms die young: they are allocated, used briefly, and become garbage. Terms that have survived at least one collection are likely to survive many more. Generational GC exploits this by dividing the heap into two generations:
+
+* **Young generation**: recently allocated terms, between the *high water mark* and the current heap pointer.
+* **Old (mature) generation**: terms that have survived at least one minor collection, stored in a separate old heap.
+
+#### High Water Mark
+
+After each garbage collection, the heap pointer position is recorded as the *high water mark*. On the next collection, terms allocated below the high water mark (i.e., terms that existed at the time of the previous collection) are considered mature. Terms allocated above the high water mark are young.
+
+#### Minor Collection
+
+During a minor collection:
+
+1. A new young heap is allocated.
+2. Mature terms (below the high water mark) are *promoted*: copied to the old heap rather than the new young heap.
+3. Young terms that are still reachable are copied to the new young heap.
+4. Both the new young heap and the newly promoted old region are scanned for references, since promoted terms may reference young terms and vice versa.
+5. Only the young MSO list is swept; the old MSO list is preserved.
+6. The previous heap is freed, but the old heap persists across minor collections.
+
+Because the old heap is not scanned for garbage during a minor collection, the cost is proportional to the size of the young generation rather than the entire heap.
+
+#### When Full vs. Minor Collection Occurs
+
+AtomVM keeps a counter (`gc_count`) of how many minor collections have occurred since the last full sweep. A full sweep is forced when:
+
+* The process has never been garbage collected (no high water mark exists).
+* `gc_count` reaches the `fullsweep_after` threshold.
+* The old heap does not have enough space to accommodate promoted terms.
+* A `MEMORY_FORCE_SHRINK` request is made (e.g., via `erlang:garbage_collect/0`).
+
+The `fullsweep_after` value can be set per-process via [`spawn_opt`](./programmers-guide.md#spawning-processes) or [`erlang:process_flag/2`](./apidocs/erlang/estdlib/erlang.md#process_flag2). The default value is 65535, meaning full sweeps are infrequent under normal operation. Setting it to `0` disables generational collection entirely, forcing a full sweep on every garbage collection event.

doc/src/programmers-guide.md

@@ -365,6 +365,7 @@ The [options](./apidocs/erlang/estdlib/erlang.md#spawn_option) argument is a pro
 |-----|------------|---------------|-------------|
 | `min_heap_size` | `non_neg_integer()` | none | Minimum heap size of the process.  The heap will shrink no smaller than this size. |
 | `max_heap_size` | `non_neg_integer()` | unbounded | Maximum heap size of the process.  The heap will grow no larger than this size. |
+| `fullsweep_after` | `non_neg_integer()` | 65535 | Maximum number of [minor garbage collections](./memory-management.md#generational-garbage-collection) before a full sweep is forced.  Set to `0` to disable generational garbage collection. |
 | `link` | `boolean()` | `false` | Whether to link the spawned process to the spawning process. |
 | `monitor` | `boolean()` | `false` | Whether to link the spawning process should monitor the spawned process. |
 | `atomvm_heap_growth` | `bounded_free \| minimum \| fibonacci` | `bounded_free` | [Strategy](./memory-management.md#heap-growth-strategies) to grow the heap of the process. |

libs/estdlib/src/erlang.erl

@@ -177,6 +177,7 @@
 -type spawn_option() ::
     {min_heap_size, pos_integer()}
     | {max_heap_size, pos_integer()}
+    | {fullsweep_after, non_neg_integer()}
     | {atomvm_heap_growth, atomvm_heap_growth_strategy()}
     | link
     | monitor.
@@ -1308,7 +1309,9 @@ group_leader(_Leader, _Pid) ->
 %%
 %% @end
 %%-----------------------------------------------------------------------------
--spec process_flag(Flag :: trap_exit, Value :: boolean()) -> pid().
+-spec process_flag
+    (trap_exit, boolean()) -> boolean();
+    (fullsweep_after, non_neg_integer()) -> non_neg_integer().
 process_flag(_Flag, _Value) ->
     erlang:nif_error(undefined).
 

libs/jit/src/jit_aarch64.erl

@@ -164,25 +164,25 @@
     | {maybe_free_aarch64_register(), '&', non_neg_integer(), '!=', integer()}
     | {{free, aarch64_register()}, '==', {free, aarch64_register()}}.
 
-% ctx->e is 0x28
-% ctx->x is 0x30
+% ctx->e is 0x50
+% ctx->x is 0x58
 -define(WORD_SIZE, 8).
 -define(CTX_REG, r0).
 -define(JITSTATE_REG, r1).
 -define(NATIVE_INTERFACE_REG, r2).
--define(Y_REGS, {?CTX_REG, 16#28}).
--define(X_REG(N), {?CTX_REG, 16#30 + (N * ?WORD_SIZE)}).
--define(CP, {?CTX_REG, 16#B8}).
--define(FP_REGS, {?CTX_REG, 16#C0}).
+-define(Y_REGS, {?CTX_REG, 16#50}).
+-define(X_REG(N), {?CTX_REG, 16#58 + (N * ?WORD_SIZE)}).
+-define(CP, {?CTX_REG, 16#E0}).
+-define(FP_REGS, {?CTX_REG, 16#E8}).
 -define(FP_REG_OFFSET(State, F),
     (F *
         case (State)#state.variant band ?JIT_VARIANT_FLOAT32 of
             0 -> 8;
             _ -> 4
         end)
 ).
--define(BS, {?CTX_REG, 16#C8}).
--define(BS_OFFSET, {?CTX_REG, 16#D0}).
+-define(BS, {?CTX_REG, 16#F0}).
+-define(BS_OFFSET, {?CTX_REG, 16#F8}).
 -define(JITSTATE_MODULE, {?JITSTATE_REG, 0}).
 -define(JITSTATE_CONTINUATION, {?JITSTATE_REG, 16#8}).
 -define(JITSTATE_REDUCTIONCOUNT, {?JITSTATE_REG, 16#10}).

libs/jit/src/jit_armv6m.erl

@@ -166,15 +166,15 @@
     | {{free, armv6m_register()}, '==', {free, armv6m_register()}}.
 
 % ctx->e is 0x28
-% ctx->x is 0x30
+% ctx->x is 0x2C
 -define(CTX_REG, r0).
 -define(NATIVE_INTERFACE_REG, r2).
--define(Y_REGS, {?CTX_REG, 16#14}).
--define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}).
--define(CP, {?CTX_REG, 16#5C}).
--define(FP_REGS, {?CTX_REG, 16#60}).
--define(BS, {?CTX_REG, 16#64}).
--define(BS_OFFSET, {?CTX_REG, 16#68}).
+-define(Y_REGS, {?CTX_REG, 16#28}).
+-define(X_REG(N), {?CTX_REG, 16#2C + (N * 4)}).
+-define(CP, {?CTX_REG, 16#70}).
+-define(FP_REGS, {?CTX_REG, 16#74}).
+-define(BS, {?CTX_REG, 16#78}).
+-define(BS_OFFSET, {?CTX_REG, 16#7C}).
 % JITSTATE is on stack, accessed via stack offset
 % These macros now expect a register that contains the jit_state pointer
 -define(JITSTATE_MODULE(Reg), {Reg, 0}).

libs/jit/src/jit_riscv32.erl

@@ -195,16 +195,16 @@
     | {{free, riscv32_register()}, '==', {free, riscv32_register()}}.
 
 % Context offsets (32-bit architecture)
-% ctx->e is 0x14
-% ctx->x is 0x18
+% ctx->e is 0x28
+% ctx->x is 0x2C
 -define(CTX_REG, a0).
 -define(NATIVE_INTERFACE_REG, a2).
--define(Y_REGS, {?CTX_REG, 16#14}).
--define(X_REG(N), {?CTX_REG, 16#18 + (N * 4)}).
--define(CP, {?CTX_REG, 16#5C}).
--define(FP_REGS, {?CTX_REG, 16#60}).
--define(BS, {?CTX_REG, 16#64}).
--define(BS_OFFSET, {?CTX_REG, 16#68}).
+-define(Y_REGS, {?CTX_REG, 16#28}).
+-define(X_REG(N), {?CTX_REG, 16#2C + (N * 4)}).
+-define(CP, {?CTX_REG, 16#70}).
+-define(FP_REGS, {?CTX_REG, 16#74}).
+-define(BS, {?CTX_REG, 16#78}).
+-define(BS_OFFSET, {?CTX_REG, 16#7C}).
 % JITSTATE is in a1 register (no prolog, following aarch64 model)
 -define(JITSTATE_REG, a1).
 % Return address register (like LR in AArch64)

libs/jit/src/jit_x86_64.erl

@@ -150,28 +150,28 @@
 -define(WORD_SIZE, 8).
 
 % Following offsets are verified with static asserts in jit.c
-% ctx->e is 0x28
-% ctx->x is 0x30
-% ctx->cp is 0xB8
-% ctx->fr is 0xC0
-% ctx->bs is 0xC8
-% ctx->bs_offset is 0xD0
+% ctx->e is 0x50
+% ctx->x is 0x58
+% ctx->cp is 0xE0
+% ctx->fr is 0xE8
+% ctx->bs is 0xF0
+% ctx->bs_offset is 0xF8
 -define(CTX_REG, rdi).
 -define(JITSTATE_REG, rsi).
 -define(NATIVE_INTERFACE_REG, rdx).
--define(Y_REGS, {16#28, ?CTX_REG}).
--define(X_REG(N), {16#30 + (N * ?WORD_SIZE), ?CTX_REG}).
--define(CP, {16#B8, ?CTX_REG}).
--define(FP_REGS, {16#C0, ?CTX_REG}).
+-define(Y_REGS, {16#50, ?CTX_REG}).
+-define(X_REG(N), {16#58 + (N * ?WORD_SIZE), ?CTX_REG}).
+-define(CP, {16#E0, ?CTX_REG}).
+-define(FP_REGS, {16#E8, ?CTX_REG}).
 -define(FP_REG_OFFSET(State, F),
     (F *
         case (State)#state.variant band ?JIT_VARIANT_FLOAT32 of
             0 -> 8;
             _ -> 4
         end)
 ).
--define(BS, {16#C8, ?CTX_REG}).
--define(BS_OFFSET, {16#D0, ?CTX_REG}).
+-define(BS, {16#F0, ?CTX_REG}).
+-define(BS_OFFSET, {16#F8, ?CTX_REG}).
 -define(JITSTATE_MODULE, {0, ?JITSTATE_REG}).
 -define(JITSTATE_CONTINUATION, {16#8, ?JITSTATE_REG}).
 -define(JITSTATE_REMAINING_REDUCTIONS, {16#10, ?JITSTATE_REG}).

src/libAtomVM/context.c

@@ -81,6 +81,8 @@ Context *context_new(GlobalContext *glb)
     ctx->min_heap_size = 0;
     ctx->max_heap_size = 0;
     ctx->heap_growth_strategy = BoundedFreeHeapGrowth;
+    ctx->fullsweep_after = 65535;
+    ctx->gc_count = 0;
     ctx->has_min_heap_size = 0;
     ctx->has_max_heap_size = 0;
 
@@ -136,6 +138,14 @@ Context *context_new(GlobalContext *glb)
 
 void context_destroy(Context *ctx)
 {
+    // If the process was never scheduled (still in Spawning state),
+    // it is still in the waiting_processes list and must be removed.
+    if (ctx->flags & Spawning) {
+        SMP_SPINLOCK_LOCK(&ctx->global->processes_spinlock);
+        list_remove(&ctx->processes_list_head);
+        SMP_SPINLOCK_UNLOCK(&ctx->global->processes_spinlock);
+    }
+
     // Another process can get an access to our mailbox until this point.
     struct ListHead *processes_table_list = synclist_wrlock(&ctx->global->processes_table);
     UNUSED(processes_table_list);
@@ -525,6 +535,7 @@ bool context_get_process_info(Context *ctx, term *out, size_t *term_size, term a
         case MESSAGE_QUEUE_LEN_ATOM:
         case REGISTERED_NAME_ATOM:
         case MEMORY_ATOM:
+        case FULLSWEEP_AFTER_ATOM:
             ret_size = TUPLE_SIZE(2);
             break;
         case LINKS_ATOM: {
@@ -675,6 +686,12 @@ bool context_get_process_info(Context *ctx, term *out, size_t *term_size, term a
             break;
         }
 
+        case FULLSWEEP_AFTER_ATOM: {
+            term_put_tuple_element(ret, 0, FULLSWEEP_AFTER_ATOM);
+            term_put_tuple_element(ret, 1, term_from_int(ctx->fullsweep_after));
+            break;
+        }
+
         case CURRENT_STACKTRACE_ATOM: {
             term_put_tuple_element(ret, 0, CURRENT_STACKTRACE_ATOM);
             // FIXME: since it's not possible how to build stacktrace here with the current API,
@@ -1209,6 +1226,14 @@ COLD_FUNC void context_dump(Context *ctx)
         ct++;
     }
 
+    fprintf(stderr, "\n\nHeap\n----\n");
+    fprintf(stderr, "young heap: %zu words\n", (size_t) (ctx->heap.heap_end - ctx->heap.heap_start));
+    if (ctx->heap.old_heap_start) {
+        fprintf(stderr, "old heap: %zu words (used: %zu)\n",
+            (size_t) (ctx->heap.old_heap_end - ctx->heap.old_heap_start),
+            (size_t) (ctx->heap.old_heap_ptr - ctx->heap.old_heap_start));
+    }
+
     fprintf(stderr, "\n\nMailbox\n-------\n");
     mailbox_crashdump(ctx);
 

src/libAtomVM/context.h

@@ -113,6 +113,8 @@ struct Context
     size_t min_heap_size;
     size_t max_heap_size;
     enum HeapGrowthStrategy heap_growth_strategy;
+    unsigned int fullsweep_after;
+    unsigned int gc_count;
 
     // saved state when scheduled out
     Module *saved_module;

src/libAtomVM/defaultatoms.def

@@ -207,6 +207,7 @@ X(EMU_ATOM, "\x3", "emu")
 X(JIT_ATOM, "\x3", "jit")
 X(EMU_FLAVOR_ATOM, "\xA", "emu_flavor")
 X(CODE_SERVER_ATOM, "\xB", "code_server")
+X(FULLSWEEP_AFTER_ATOM, "\xF", "fullsweep_after")
 X(LOAD_ATOM, "\x4", "load")
 X(JIT_X86_64_ATOM, "\xA", "jit_x86_64")
 X(JIT_AARCH64_ATOM, "\xB", "jit_aarch64")

src/libAtomVM/erl_nif_priv.h

@@ -63,6 +63,11 @@ static inline void erl_nif_env_partial_init_from_globalcontext(ErlNifEnv *env, G
     env->heap.heap_start = NULL;
     env->heap.heap_ptr = NULL;
     env->heap.heap_end = NULL;
+    env->heap.high_water_mark = NULL;
+    env->heap.old_heap_start = NULL;
+    env->heap.old_heap_ptr = NULL;
+    env->heap.old_heap_end = NULL;
+    env->heap.old_mso_list = term_nil();
     env->stack_pointer = NULL;
     env->x[0] = term_nil();
     env->x[1] = term_nil();
@@ -76,6 +81,11 @@ static inline void erl_nif_env_partial_init_from_resource(ErlNifEnv *env, void *
     env->heap.heap_start = NULL;
     env->heap.heap_ptr = NULL;
     env->heap.heap_end = NULL;
+    env->heap.high_water_mark = NULL;
+    env->heap.old_heap_start = NULL;
+    env->heap.old_heap_ptr = NULL;
+    env->heap.old_heap_end = NULL;
+    env->heap.old_mso_list = term_nil();
     env->stack_pointer = NULL;
     env->x[0] = term_nil();
     env->x[1] = term_nil();

src/libAtomVM/jit.c

@@ -68,23 +68,23 @@ _Static_assert(
 
 // Verify offsets in jit_x86_64.erl
 #if JIT_ARCH_TARGET == JIT_ARCH_X86_64 || JIT_ARCH_TARGET == JIT_ARCH_AARCH64
-_Static_assert(offsetof(Context, e) == 0x28, "ctx->e is 0x28 in jit/src/jit_{aarch64,x86_64}.erl");
-_Static_assert(offsetof(Context, x) == 0x30, "ctx->x is 0x30 in jit/src/jit_{aarch64,x86_64}.erl");
-_Static_assert(offsetof(Context, cp) == 0xB8, "ctx->cp is 0xB8 in jit/src/jit_{aarch64,x86_64}.erl");
-_Static_assert(offsetof(Context, fr) == 0xC0, "ctx->fr is 0xC0 in jit/src/jit_{aarch64,x86_64}.erl");
-_Static_assert(offsetof(Context, bs) == 0xC8, "ctx->bs is 0xC8 in jit/src/jit_{aarch64,x86_64}.erl");
-_Static_assert(offsetof(Context, bs_offset) == 0xD0, "ctx->bs_offset is 0xD0 in jit/src/jit_{aarch64,x86_64}.erl");
+_Static_assert(offsetof(Context, e) == 0x50, "ctx->e is 0x50 in jit/src/jit_{aarch64,x86_64}.erl");
+_Static_assert(offsetof(Context, x) == 0x58, "ctx->x is 0x58 in jit/src/jit_{aarch64,x86_64}.erl");
+_Static_assert(offsetof(Context, cp) == 0xE0, "ctx->cp is 0xE0 in jit/src/jit_{aarch64,x86_64}.erl");
+_Static_assert(offsetof(Context, fr) == 0xE8, "ctx->fr is 0xE8 in jit/src/jit_{aarch64,x86_64}.erl");
+_Static_assert(offsetof(Context, bs) == 0xF0, "ctx->bs is 0xF0 in jit/src/jit_{aarch64,x86_64}.erl");
+_Static_assert(offsetof(Context, bs_offset) == 0xF8, "ctx->bs_offset is 0xF8 in jit/src/jit_{aarch64,x86_64}.erl");
 
 _Static_assert(offsetof(JITState, module) == 0x0, "jit_state->module is 0x0 in jit/src/jit_{aarch64,x86_64}.erl");
 _Static_assert(offsetof(JITState, continuation) == 0x8, "jit_state->continuation is 0x8 in jit/src/jit_{aarch64,x86_64}.erl");
 _Static_assert(offsetof(JITState, remaining_reductions) == 0x10, "jit_state->remaining_reductions is 0x10 in jit/src/jit_{aarch64,x86_64}.erl");
 #elif JIT_ARCH_TARGET == JIT_ARCH_ARMV6M
-_Static_assert(offsetof(Context, e) == 0x14, "ctx->e is 0x14 in jit/src/jit_armv6m.erl");
-_Static_assert(offsetof(Context, x) == 0x18, "ctx->x is 0x18 in jit/src/jit_armv6m.erl");
-_Static_assert(offsetof(Context, cp) == 0x5C, "ctx->cp is 0x5C in jit/src/jit_armv6m.erl");
-_Static_assert(offsetof(Context, fr) == 0x60, "ctx->fr is 0x60 in jit/src/jit_armv6m.erl");
-_Static_assert(offsetof(Context, bs) == 0x64, "ctx->bs is 0x64 in jit/src/jit_armv6m.erl");
-_Static_assert(offsetof(Context, bs_offset) == 0x68, "ctx->bs_offset is 0x68 in jit/src/jit_armv6m.erl");
+_Static_assert(offsetof(Context, e) == 0x28, "ctx->e is 0x28 in jit/src/jit_armv6m.erl");
+_Static_assert(offsetof(Context, x) == 0x2C, "ctx->x is 0x2C in jit/src/jit_armv6m.erl");
+_Static_assert(offsetof(Context, cp) == 0x70, "ctx->cp is 0x70 in jit/src/jit_armv6m.erl");
+_Static_assert(offsetof(Context, fr) == 0x74, "ctx->fr is 0x74 in jit/src/jit_armv6m.erl");
+_Static_assert(offsetof(Context, bs) == 0x78, "ctx->bs is 0x78 in jit/src/jit_armv6m.erl");
+_Static_assert(offsetof(Context, bs_offset) == 0x7C, "ctx->bs_offset is 0x7C in jit/src/jit_armv6m.erl");
 
 _Static_assert(offsetof(JITState, module) == 0x0, "jit_state->module is 0x0 in jit/src/jit_armv6m.erl");
 _Static_assert(offsetof(JITState, continuation) == 0x4, "jit_state->continuation is 0x4 in jit/src/jit_armv6m.erl");
@@ -93,12 +93,12 @@ _Static_assert(offsetof(JITState, remaining_reductions) == 0x8, "jit_state->rema
 _Static_assert(sizeof(size_t) == 4, "size_t is expected to be 32 bits");
 
 #elif JIT_ARCH_TARGET == JIT_ARCH_RISCV32
-_Static_assert(offsetof(Context, e) == 0x14, "ctx->e is 0x14 in jit/src/jit_riscv32.erl");
-_Static_assert(offsetof(Context, x) == 0x18, "ctx->x is 0x18 in jit/src/jit_riscv32.erl");
-_Static_assert(offsetof(Context, cp) == 0x5C, "ctx->cp is 0x5C in jit/src/jit_riscv32.erl");
-_Static_assert(offsetof(Context, fr) == 0x60, "ctx->fr is 0x60 in jit/src/jit_riscv32.erl");
-_Static_assert(offsetof(Context, bs) == 0x64, "ctx->bs is 0x64 in jit/src/jit_riscv32.erl");
-_Static_assert(offsetof(Context, bs_offset) == 0x68, "ctx->bs_offset is 0x68 in jit/src/jit_riscv32.erl");
+_Static_assert(offsetof(Context, e) == 0x28, "ctx->e is 0x28 in jit/src/jit_riscv32.erl");
+_Static_assert(offsetof(Context, x) == 0x2C, "ctx->x is 0x2C in jit/src/jit_riscv32.erl");
+_Static_assert(offsetof(Context, cp) == 0x70, "ctx->cp is 0x70 in jit/src/jit_riscv32.erl");
+_Static_assert(offsetof(Context, fr) == 0x74, "ctx->fr is 0x74 in jit/src/jit_riscv32.erl");
+_Static_assert(offsetof(Context, bs) == 0x78, "ctx->bs is 0x78 in jit/src/jit_riscv32.erl");
+_Static_assert(offsetof(Context, bs_offset) == 0x7C, "ctx->bs_offset is 0x7C in jit/src/jit_riscv32.erl");
 
 _Static_assert(offsetof(JITState, module) == 0x0, "jit_state->module is 0x0 in jit/src/jit_riscv32.erl");
 _Static_assert(offsetof(JITState, continuation) == 0x4, "jit_state->continuation is 0x4 in jit/src/jit_riscv32.erl");