From f2b96f88b0d2f447a5d65b1fdd098db4be6ad445 Mon Sep 17 00:00:00 2001 From: Esity Date: Wed, 17 Jun 2026 17:36:05 -0500 Subject: [PATCH 1/2] fix: trace store incremental dirty tracking to eliminate GVL stall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit snapshot_dirty_state was serializing ALL traces to JSON on every flush just to diff and find which ones changed. With thousands of traces this monopolized the Ruby GVL for seconds, starving every other thread and pegging a single core at 100%. Replace the boolean @traces_dirty flag with a per-trace @dirty_trace_ids Set. Now flush only serializes traces that were actually modified — O(changed) instead of O(total). Also eliminates the redundant full-collection serialize on boot (load_from_local). --- .../agentic/memory/trace/helpers/store.rb | 68 +++++++++++++------ 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/lib/legion/extensions/agentic/memory/trace/helpers/store.rb b/lib/legion/extensions/agentic/memory/trace/helpers/store.rb index a0477df..86e7955 100644 --- a/lib/legion/extensions/agentic/memory/trace/helpers/store.rb +++ b/lib/legion/extensions/agentic/memory/trace/helpers/store.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'json' +require 'set' module Legion module Extensions @@ -22,7 +23,8 @@ def initialize(partition_id: nil) @traces = {} @associations = Hash.new { |h, k| h[k] = Hash.new(0) } @partition_id = partition_id || resolve_partition_id - @traces_dirty = false + @dirty_trace_ids = Set.new + @deleted_trace_ids = Set.new @associations_dirty = false @persisted_trace_rows = {} load_from_local @@ -32,7 +34,7 @@ def store(trace) persisted_trace = Helpers::Trace.normalize_trace_affect(trace) persisted_trace[:partition_id] ||= @partition_id @mutex.synchronize do - @traces_dirty = true if @traces[persisted_trace[:trace_id]] != persisted_trace + @dirty_trace_ids << persisted_trace[:trace_id] @traces[persisted_trace[:trace_id]] = persisted_trace end persisted_trace[:trace_id] @@ -50,7 +52,10 @@ def delete(trace_id) removed_links = @associations.delete(trace_id) @associations.each_value { |links| links.delete(trace_id) } - @traces_dirty = true if removed_trace + if removed_trace + @dirty_trace_ids.delete(trace_id) + @deleted_trace_ids << trace_id + end @associations_dirty = true if removed_trace || removed_links end end @@ -92,8 +97,10 @@ def record_coactivation(trace_id_a, trace_id_b) @associations_dirty = true threshold = Helpers::Trace::COACTIVATION_THRESHOLD - @traces_dirty = true if @associations[trace_id_a][trace_id_b] >= threshold && - link_traces(trace_id_a, trace_id_b) + if @associations[trace_id_a][trace_id_b] >= threshold && link_traces(trace_id_a, trace_id_b) + @dirty_trace_ids << trace_id_a + @dirty_trace_ids << trace_id_b + end end end @@ -128,7 +135,8 @@ def restore_traces(traces) @mutex.synchronize do @traces = snapshot @associations = Hash.new { |h, k| h[k] = Hash.new(0) } - @traces_dirty = true + @dirty_trace_ids = Set.new(@traces.keys) + @deleted_trace_ids = Set.new @associations_dirty = true end flush @@ -183,27 +191,41 @@ def save_to_local end def snapshot_dirty_state - traces_snapshot, associations_snapshot, trace_rows_snapshot, trace_changes, associations_dirty = @mutex.synchronize do - ts = @traces.transform_values(&:dup) + @mutex.synchronize do + dirty_ids = @dirty_trace_ids.to_a + deleted_ids = @deleted_trace_ids.to_a + associations_dirty = @associations_dirty + + return nil if dirty_ids.empty? && deleted_ids.empty? && !associations_dirty + + dirty_rows = dirty_ids.each_with_object({}) do |trace_id, h| + trace = @traces[trace_id] + h[trace_id] = serialize_trace_for_db(trace) if trace + end + + trace_rows_snapshot = @persisted_trace_rows.merge(dirty_rows) + deleted_ids.each { |id| trace_rows_snapshot.delete(id) } + + traces_snapshot = @traces.transform_values(&:dup) as = @associations.each_with_object({}) { |(tid, targets), memo| memo[tid] = targets.dup } - trs = ts.transform_values { |trace| serialize_trace_for_db(trace) } - changed_trace_ids = trs.each_key.reject { |trace_id| trs[trace_id] == @persisted_trace_rows[trace_id] } - trace_changes = { dirty: @traces_dirty || changed_trace_ids.any?, changed_ids: changed_trace_ids } - [ts, as, trs, trace_changes, @associations_dirty] - end - return nil unless trace_changes[:dirty] || associations_dirty + trace_changes = { dirty: true, changed_ids: dirty_ids, deleted_ids: deleted_ids } - [traces_snapshot, associations_snapshot, trace_rows_snapshot, trace_changes, associations_dirty] + [traces_snapshot, as, trace_rows_snapshot, trace_changes, associations_dirty] + end end def persist_dirty_traces(db, trace_rows_snapshot, trace_changes, stale_ids) - return unless trace_changes[:dirty] || !stale_ids.empty? + changed_ids = trace_changes[:changed_ids] || [] + deleted_ids = trace_changes[:deleted_ids] || [] + all_removals = (stale_ids + deleted_ids).uniq + return if changed_ids.empty? && all_removals.empty? ds = db[:memory_traces] - trace_changes[:changed_ids].each do |trace_id| - ds.insert_conflict(:replace).insert(trace_rows_snapshot.fetch(trace_id)) + changed_ids.each do |trace_id| + row = trace_rows_snapshot[trace_id] + ds.insert_conflict(:replace).insert(row) if row end - db[:memory_traces].where(trace_id: stale_ids).delete unless stale_ids.empty? + db[:memory_traces].where(trace_id: all_removals).delete unless all_removals.empty? end def persist_dirty_associations(db, associations_snapshot, scoped_trace_ids, memory_trace_ids, stale_ids, dirty) @@ -232,7 +254,8 @@ def persist_dirty_associations(db, associations_snapshot, scoped_trace_ids, memo def clear_dirty_flags(trace_rows_snapshot) @mutex.synchronize do - @traces_dirty = false + @dirty_trace_ids.clear + @deleted_trace_ids.clear @associations_dirty = false @persisted_trace_rows = trace_rows_snapshot end @@ -246,12 +269,13 @@ def load_from_local db[:memory_traces].where(partition_id: @partition_id).each do |row| @traces[row[:trace_id]] = deserialize_trace_from_db(row) + @persisted_trace_rows[row[:trace_id]] = row.dup end load_local_associations(db) - @persisted_trace_rows = @traces.transform_values { |trace| serialize_trace_for_db(trace) } - @traces_dirty = false + @dirty_trace_ids = Set.new + @deleted_trace_ids = Set.new @associations_dirty = false end From 6470cc7b47009a2b013931297556a771aca37e03 Mon Sep 17 00:00:00 2001 From: Esity Date: Wed, 17 Jun 2026 17:40:37 -0500 Subject: [PATCH 2/2] bump v0.1.40; add changelog; remove redundant require 'set' --- CHANGELOG.md | 7 +++++++ .../extensions/agentic/memory/trace/helpers/store.rb | 1 - lib/legion/extensions/agentic/memory/version.rb | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a358556..6ea79fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.1.40] - 2026-06-17 +### Fixed +- **Critical performance fix:** Trace store `snapshot_dirty_state` no longer serializes ALL traces to JSON on every flush — replaced boolean `@traces_dirty` flag with per-trace `@dirty_trace_ids` Set so only modified traces are serialized (O(changed) instead of O(total)) +- `load_from_local` no longer re-serializes every trace on boot — stores raw DB rows directly in `@persisted_trace_rows` +- `persist_dirty_traces` now handles explicit deletion tracking via `@deleted_trace_ids` Set +- Eliminates 97%+ single-core CPU usage caused by `decay_cycle` triggering full-store JSON serialization every 60 seconds + ## [0.1.39] - 2026-06-01 ### Fixed - `ErrorTracer` now guards against infinite recursion — if downstream trace storage triggers error/fatal logging, the `tracing?` thread-local flag prevents re-entry diff --git a/lib/legion/extensions/agentic/memory/trace/helpers/store.rb b/lib/legion/extensions/agentic/memory/trace/helpers/store.rb index 86e7955..1f1b52c 100644 --- a/lib/legion/extensions/agentic/memory/trace/helpers/store.rb +++ b/lib/legion/extensions/agentic/memory/trace/helpers/store.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'json' -require 'set' module Legion module Extensions diff --git a/lib/legion/extensions/agentic/memory/version.rb b/lib/legion/extensions/agentic/memory/version.rb index 02c13b3..7eb99b9 100644 --- a/lib/legion/extensions/agentic/memory/version.rb +++ b/lib/legion/extensions/agentic/memory/version.rb @@ -4,7 +4,7 @@ module Legion module Extensions module Agentic module Memory - VERSION = '0.1.39' + VERSION = '0.1.40' end end end