diff --git a/CHANGELOG.md b/CHANGELOG.md index a358556..6ea79fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.1.40] - 2026-06-17 +### Fixed +- **Critical performance fix:** Trace store `snapshot_dirty_state` no longer serializes ALL traces to JSON on every flush — replaced boolean `@traces_dirty` flag with per-trace `@dirty_trace_ids` Set so only modified traces are serialized (O(changed) instead of O(total)) +- `load_from_local` no longer re-serializes every trace on boot — stores raw DB rows directly in `@persisted_trace_rows` +- `persist_dirty_traces` now handles explicit deletion tracking via `@deleted_trace_ids` Set +- Eliminates 97%+ single-core CPU usage caused by `decay_cycle` triggering full-store JSON serialization every 60 seconds + ## [0.1.39] - 2026-06-01 ### Fixed - `ErrorTracer` now guards against infinite recursion — if downstream trace storage triggers error/fatal logging, the `tracing?` thread-local flag prevents re-entry diff --git a/lib/legion/extensions/agentic/memory/trace/helpers/store.rb b/lib/legion/extensions/agentic/memory/trace/helpers/store.rb index a0477df..1f1b52c 100644 --- a/lib/legion/extensions/agentic/memory/trace/helpers/store.rb +++ b/lib/legion/extensions/agentic/memory/trace/helpers/store.rb @@ -22,7 +22,8 @@ def initialize(partition_id: nil) @traces = {} @associations = Hash.new { |h, k| h[k] = Hash.new(0) } @partition_id = partition_id || resolve_partition_id - @traces_dirty = false + @dirty_trace_ids = Set.new + @deleted_trace_ids = Set.new @associations_dirty = false @persisted_trace_rows = {} load_from_local @@ -32,7 +33,7 @@ def store(trace) persisted_trace = Helpers::Trace.normalize_trace_affect(trace) persisted_trace[:partition_id] ||= @partition_id @mutex.synchronize do - @traces_dirty = true if @traces[persisted_trace[:trace_id]] != persisted_trace + @dirty_trace_ids << persisted_trace[:trace_id] @traces[persisted_trace[:trace_id]] = persisted_trace end persisted_trace[:trace_id] @@ -50,7 +51,10 @@ def delete(trace_id) removed_links = @associations.delete(trace_id) @associations.each_value { |links| links.delete(trace_id) } - @traces_dirty = true if removed_trace + if removed_trace + @dirty_trace_ids.delete(trace_id) + @deleted_trace_ids << trace_id + end @associations_dirty = true if removed_trace || removed_links end end @@ -92,8 +96,10 @@ def record_coactivation(trace_id_a, trace_id_b) @associations_dirty = true threshold = Helpers::Trace::COACTIVATION_THRESHOLD - @traces_dirty = true if @associations[trace_id_a][trace_id_b] >= threshold && - link_traces(trace_id_a, trace_id_b) + if @associations[trace_id_a][trace_id_b] >= threshold && link_traces(trace_id_a, trace_id_b) + @dirty_trace_ids << trace_id_a + @dirty_trace_ids << trace_id_b + end end end @@ -128,7 +134,8 @@ def restore_traces(traces) @mutex.synchronize do @traces = snapshot @associations = Hash.new { |h, k| h[k] = Hash.new(0) } - @traces_dirty = true + @dirty_trace_ids = Set.new(@traces.keys) + @deleted_trace_ids = Set.new @associations_dirty = true end flush @@ -183,27 +190,41 @@ def save_to_local end def snapshot_dirty_state - traces_snapshot, associations_snapshot, trace_rows_snapshot, trace_changes, associations_dirty = @mutex.synchronize do - ts = @traces.transform_values(&:dup) + @mutex.synchronize do + dirty_ids = @dirty_trace_ids.to_a + deleted_ids = @deleted_trace_ids.to_a + associations_dirty = @associations_dirty + + return nil if dirty_ids.empty? && deleted_ids.empty? && !associations_dirty + + dirty_rows = dirty_ids.each_with_object({}) do |trace_id, h| + trace = @traces[trace_id] + h[trace_id] = serialize_trace_for_db(trace) if trace + end + + trace_rows_snapshot = @persisted_trace_rows.merge(dirty_rows) + deleted_ids.each { |id| trace_rows_snapshot.delete(id) } + + traces_snapshot = @traces.transform_values(&:dup) as = @associations.each_with_object({}) { |(tid, targets), memo| memo[tid] = targets.dup } - trs = ts.transform_values { |trace| serialize_trace_for_db(trace) } - changed_trace_ids = trs.each_key.reject { |trace_id| trs[trace_id] == @persisted_trace_rows[trace_id] } - trace_changes = { dirty: @traces_dirty || changed_trace_ids.any?, changed_ids: changed_trace_ids } - [ts, as, trs, trace_changes, @associations_dirty] - end - return nil unless trace_changes[:dirty] || associations_dirty + trace_changes = { dirty: true, changed_ids: dirty_ids, deleted_ids: deleted_ids } - [traces_snapshot, associations_snapshot, trace_rows_snapshot, trace_changes, associations_dirty] + [traces_snapshot, as, trace_rows_snapshot, trace_changes, associations_dirty] + end end def persist_dirty_traces(db, trace_rows_snapshot, trace_changes, stale_ids) - return unless trace_changes[:dirty] || !stale_ids.empty? + changed_ids = trace_changes[:changed_ids] || [] + deleted_ids = trace_changes[:deleted_ids] || [] + all_removals = (stale_ids + deleted_ids).uniq + return if changed_ids.empty? && all_removals.empty? ds = db[:memory_traces] - trace_changes[:changed_ids].each do |trace_id| - ds.insert_conflict(:replace).insert(trace_rows_snapshot.fetch(trace_id)) + changed_ids.each do |trace_id| + row = trace_rows_snapshot[trace_id] + ds.insert_conflict(:replace).insert(row) if row end - db[:memory_traces].where(trace_id: stale_ids).delete unless stale_ids.empty? + db[:memory_traces].where(trace_id: all_removals).delete unless all_removals.empty? end def persist_dirty_associations(db, associations_snapshot, scoped_trace_ids, memory_trace_ids, stale_ids, dirty) @@ -232,7 +253,8 @@ def persist_dirty_associations(db, associations_snapshot, scoped_trace_ids, memo def clear_dirty_flags(trace_rows_snapshot) @mutex.synchronize do - @traces_dirty = false + @dirty_trace_ids.clear + @deleted_trace_ids.clear @associations_dirty = false @persisted_trace_rows = trace_rows_snapshot end @@ -246,12 +268,13 @@ def load_from_local db[:memory_traces].where(partition_id: @partition_id).each do |row| @traces[row[:trace_id]] = deserialize_trace_from_db(row) + @persisted_trace_rows[row[:trace_id]] = row.dup end load_local_associations(db) - @persisted_trace_rows = @traces.transform_values { |trace| serialize_trace_for_db(trace) } - @traces_dirty = false + @dirty_trace_ids = Set.new + @deleted_trace_ids = Set.new @associations_dirty = false end diff --git a/lib/legion/extensions/agentic/memory/version.rb b/lib/legion/extensions/agentic/memory/version.rb index 02c13b3..7eb99b9 100644 --- a/lib/legion/extensions/agentic/memory/version.rb +++ b/lib/legion/extensions/agentic/memory/version.rb @@ -4,7 +4,7 @@ module Legion module Extensions module Agentic module Memory - VERSION = '0.1.39' + VERSION = '0.1.40' end end end