diff --git a/eval/dataset_4_discord_gateway_cascade/expected.json b/eval/dataset_4_discord_gateway_cascade/expected.json new file mode 100644 index 0000000..ec55275 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/expected.json @@ -0,0 +1,79 @@ +{ + "query": "why are users getting disconnected from the gateway", + "expected_clarification": { + "should_clarify": false, + "rationale": "'gateway' is a registered service and the time window (2026-06-10 02:14–02:48) is unambiguous in the logs." + }, + "expected_answer": { + "incident_window": { + "start": "2026-06-10T02:14:00Z", + "end": "2026-06-10T02:48:00Z" + }, + "affected_services": ["read-states-svc", "gateway", "message-svc", "cdn-edge"], + "trigger_event": { + "service": "read-states-svc", + "timestamp_approx": "2026-06-10T02:14:52Z", + "log_line_must_contain": "Cassandra timeout" + }, + "propagation_chain_must_include_in_order": [ + { + "service": "read-states-svc", + "ts_approx": "2026-06-10T02:14:52Z", + "what_must_mention": "Cassandra / BATCH APPLY / OperationTimedOutException" + }, + { + "service": "gateway", + "ts_approx": "2026-06-10T02:16:15Z", + "what_must_mention": "read_states / mark_read / timeout / disconnect" + }, + { + "service": "gateway", + "ts_approx": "2026-06-10T02:17:45Z", + "what_must_mention": "reconnect storm" + }, + { + "service": "message-svc", + "ts_approx": "2026-06-10T02:19:05Z", + "what_must_mention": "thread pool / saturated / 512" + }, + { + "service": "cdn-edge", + "ts_approx": "2026-06-10T02:16:44Z", + "what_must_mention": "502" + } + ], + "root_cause_must_mention": [ + "Cassandra", + "read-states", + "timeout", + "reconnect", + "pool" + ], + "ruled_out_hypotheses_must_include": [ + { + "hypothesis_about": "presence-svc", + "rationale": "presence-svc shows elevated activity caused by the gateway reconnect storm, not a cause of it — no Cassandra errors, no connection pool exhaustion" + } + ], + "assumptions_must_echo": [ + "'gateway' refers to the WebSocket gateway service" + ], + "confidence": "high", + "gaps_must_be_empty_or_minor": true + }, + "common_failure_modes_to_grade_against": [ + "Names gateway as root cause (it is the dominant symptom, not the trigger). Should be downgraded.", + "Misses read-states-svc entirely because it was not mentioned in the query.", + "Includes presence-svc in affected_services without noting it is a downstream symptom of reconnects.", + "Fails to link Cassandra partition leader re-election → read_states timeouts → WebSocket disconnects → reconnect storm chain.", + "Skips cdn-edge 502s as evidence of the gateway impact radius.", + "Does not identify OperationTimedOutException as the trigger mechanism." + ], + "what_this_dataset_tests": [ + "Multi-hop causal chain: 4 services deep from trigger (read-states) to visible symptom (cdn-edge 502).", + "Cross-service auto-sweep: read-states-svc errors must surface even though only 'gateway' was queried.", + "Red-herring elimination: presence-svc elevated but is a symptom of the reconnect storm, not a cause.", + "Infrastructure-level root cause: Cassandra partition leader re-election, not application code.", + "Reconnect amplification pattern: disconnects → reconnects → message-svc saturation loop." + ] +} diff --git a/eval/dataset_4_discord_gateway_cascade/live/cdn-edge.log b/eval/dataset_4_discord_gateway_cascade/live/cdn-edge.log new file mode 100644 index 0000000..cb568e3 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/live/cdn-edge.log @@ -0,0 +1,30 @@ +2026-06-10T02:10:00Z [INFO] CDN edge node cdn-edge-01 started (PoP: us-east-1) +2026-06-10T02:10:01Z [INFO] Origin backend: gateway-cluster.internal (4 instances, health checked) +2026-06-10T02:11:00Z [INFO] Health check ok — origin P99: 18ms, cache hit rate: 96% +2026-06-10T02:12:00Z [INFO] Health check ok — origin P99: 19ms, cache hit rate: 96% +2026-06-10T02:13:00Z [INFO] Health check ok — origin P99: 21ms, cache hit rate: 96% +2026-06-10T02:14:00Z [INFO] Health check ok — origin P99: 20ms, cache hit rate: 96% +2026-06-10T02:15:00Z [INFO] Health check ok — origin P99: 148ms, cache hit rate: 96% +2026-06-10T02:16:00Z [INFO] Health check ok — origin P99: 892ms, cache hit rate: 95% +2026-06-10T02:16:44Z [ERROR] Origin 502: gateway-01.internal /api/v9/channels/8821/messages returned 502 Bad Gateway +2026-06-10T02:16:46Z [ERROR] Origin 502: gateway-02.internal /api/v9/channels/4491/messages returned 502 Bad Gateway +2026-06-10T02:16:50Z [ERROR] Origin 502: gateway-01.internal /gateway?v=9&encoding=json returned 502 Bad Gateway +2026-06-10T02:16:55Z [ERROR] Origin 502: gateway-03.internal /api/v9/users/@me/channels returned 502 Bad Gateway +2026-06-10T02:16:58Z [ERROR] Origin 502 rate: 24% of requests to gateway cluster +2026-06-10T02:17:10Z [ERROR] Origin 502 rate: 41% of requests to gateway cluster +2026-06-10T02:17:20Z [ERROR] Origin 502 rate: 58% of requests to gateway cluster — circuit breaker threshold approaching +2026-06-10T02:17:30Z [WARNING] Serving stale cache for /api/v9/channels/* (TTL extended to 300s due to origin degradation) +2026-06-10T02:18:00Z [ERROR] Origin 502 rate: 62% of requests — gateway cluster under severe load +2026-06-10T02:18:30Z [WARNING] Edge cache miss rate elevated: 34% (baseline 4%) — reconnecting clients bypass cache with fresh IDENTIFY requests +2026-06-10T02:19:00Z [ERROR] Origin 502 rate: 71% — gateway message-svc pool exhausted, fanout requests failing +2026-06-10T02:20:00Z [ERROR] Origin 502 rate: 69% +2026-06-10T02:22:00Z [ERROR] Origin 502 rate: 65% +2026-06-10T02:25:00Z [ERROR] Origin 502 rate: 58% +2026-06-10T02:30:00Z [WARNING] Origin 502 rate: 44% — slowly improving +2026-06-10T02:35:00Z [WARNING] Origin 502 rate: 28% +2026-06-10T02:38:00Z [WARNING] Origin 502 rate: 12% — gateway recovering +2026-06-10T02:40:00Z [WARNING] Origin 502 rate: 4% +2026-06-10T02:42:10Z [INFO] Origin 502 rate dropped to 0 — gateway cluster healthy +2026-06-10T02:43:00Z [INFO] Health check ok — origin P99: 22ms, cache hit rate: 94% +2026-06-10T02:45:00Z [INFO] Health check ok — origin P99: 20ms, cache hit rate: 95% +2026-06-10T02:48:00Z [INFO] Edge health nominal — origin P99: 18ms, cache hit rate: 96% diff --git a/eval/dataset_4_discord_gateway_cascade/live/gateway.log b/eval/dataset_4_discord_gateway_cascade/live/gateway.log new file mode 100644 index 0000000..76087f4 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/live/gateway.log @@ -0,0 +1,46 @@ +2026-06-10T02:10:00Z [INFO] Gateway started, version 8.14.2 +2026-06-10T02:10:01Z [INFO] WebSocket listener bound on :443 (TLS termination via envoy) +2026-06-10T02:10:02Z [INFO] Connected clients: 84312 +2026-06-10T02:11:00Z [INFO] Health check ok — connected clients: 85102, read_states P99: 8ms +2026-06-10T02:12:00Z [INFO] Health check ok — connected clients: 85318, read_states P99: 9ms +2026-06-10T02:13:00Z [INFO] Health check ok — connected clients: 85541, read_states P99: 11ms +2026-06-10T02:14:00Z [INFO] Health check ok — connected clients: 85749, read_states P99: 10ms +2026-06-10T02:15:00Z [INFO] Health check ok — connected clients: 86001, read_states P99: 312ms +2026-06-10T02:15:30Z [WARNING] read_states mark_read API P99 latency: 3214ms (threshold 200ms) — upstream degraded +2026-06-10T02:15:45Z [WARNING] read_states mark_read API P99 latency: 5800ms — requests queuing +2026-06-10T02:16:00Z [WARNING] mark_read upstream timeout rate: 12% of requests timing out +2026-06-10T02:16:15Z [ERROR] read_states mark_read timeout for session_id=sess_4f9a2b after 5000ms +2026-06-10T02:16:18Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_4f9a2b user_id=u_9f2a1c +2026-06-10T02:16:22Z [ERROR] read_states mark_read timeout for session_id=sess_7c1e3d after 5000ms +2026-06-10T02:16:24Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_7c1e3d user_id=u_3b8e2f +2026-06-10T02:16:30Z [ERROR] mark_read upstream timeout rate: 68% of requests timing out +2026-06-10T02:16:45Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_2a9c4e user_id=u_f7a1d9 +2026-06-10T02:16:50Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_b3f1c2 user_id=u_e2c4a1 +2026-06-10T02:17:00Z [ERROR] Bulk disconnect event: 842 sessions dropped in 60s (threshold 50) +2026-06-10T02:17:10Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_9d2e1f user_id=u_a4b9c8 +2026-06-10T02:17:20Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_c8f2b1 user_id=u_d1e7f3 +2026-06-10T02:17:30Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_4a1d9c user_id=u_b2c8e4 +2026-06-10T02:17:45Z [ERROR] Client reconnect storm: 4821 sessions reconnecting in 30s window (threshold 500) +2026-06-10T02:17:50Z [ERROR] Reconnect authentication flood: 4821 IDENTIFY payloads queued +2026-06-10T02:18:00Z [WARNING] message-svc event fanout queue depth: 31204 pending (baseline 400) +2026-06-10T02:18:10Z [ERROR] HTTP client pool to read-states-svc under pressure: 180/200 in use +2026-06-10T02:18:30Z [ERROR] HTTP client pool to message-svc under pressure: 190/200 in use +2026-06-10T02:19:00Z [ERROR] Reconnect storm continuing: 3201 sessions reconnecting in 30s window +2026-06-10T02:19:20Z [ERROR] HTTP client pool to message-svc EXHAUSTED (200/200 in use) — new fanout requests queuing +2026-06-10T02:19:35Z [ERROR] Event fanout timeout for session_id=sess_f1a2b3: no message-svc slot available after 2000ms +2026-06-10T02:19:40Z [ERROR] Event fanout timeout for session_id=sess_d4e5f6: no message-svc slot available after 2000ms +2026-06-10T02:20:00Z [ERROR] Connected clients: 71204 (-14797 from peak) — active disconnects ongoing +2026-06-10T02:21:00Z [ERROR] Connected clients: 68021 (-17980 from peak) — read_states upstream still timing out +2026-06-10T02:23:00Z [ERROR] Connected clients: 62441 — read_states upstream still timing out +2026-06-10T02:25:00Z [ERROR] Connected clients: 61208 — read_states upstream still timing out +2026-06-10T02:30:00Z [ERROR] Connected clients: 60112 — sustained degradation +2026-06-10T02:35:00Z [WARNING] Connected clients: 61804 — slight recovery, reconnects stabilising +2026-06-10T02:38:00Z [INFO] read_states mark_read API latency normalized: P99=45ms (upstream recovered) +2026-06-10T02:38:05Z [INFO] mark_read upstream timeout rate: 0.2% (nominal) +2026-06-10T02:38:10Z [INFO] Client reconnect rate dropping: 180 sessions reconnecting in 30s window +2026-06-10T02:38:30Z [INFO] HTTP client pool to read-states-svc recovered: 14/200 in use +2026-06-10T02:39:00Z [INFO] HTTP client pool to message-svc recovered: 8/200 in use +2026-06-10T02:41:00Z [INFO] Reconnect storm subsided: 41 sessions reconnecting in 30s window +2026-06-10T02:43:00Z [INFO] Connected clients: 83201 — recovery in progress +2026-06-10T02:46:00Z [INFO] Health check ok — connected clients: 84892, read_states P99: 12ms +2026-06-10T02:48:00Z [INFO] Connected clients: 85104 — fully recovered diff --git a/eval/dataset_4_discord_gateway_cascade/live/message-svc.log b/eval/dataset_4_discord_gateway_cascade/live/message-svc.log new file mode 100644 index 0000000..3589754 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/live/message-svc.log @@ -0,0 +1,30 @@ +2026-06-10T02:10:00Z [INFO] MessageService started, version 5.2.8 +2026-06-10T02:10:01Z [INFO] Worker thread pool initialized: 512 threads +2026-06-10T02:10:02Z [INFO] Connected to gateway cluster (load balanced, 12 gateway instances) +2026-06-10T02:11:00Z [INFO] Health check ok — inbound rate: 848 req/s, queue depth: 312 +2026-06-10T02:12:00Z [INFO] Health check ok — inbound rate: 852 req/s, queue depth: 289 +2026-06-10T02:13:00Z [INFO] Health check ok — inbound rate: 861 req/s, queue depth: 301 +2026-06-10T02:14:00Z [INFO] Health check ok — inbound rate: 874 req/s, queue depth: 320 +2026-06-10T02:17:00Z [INFO] Health check ok — inbound rate: 1240 req/s, queue depth: 4812 +2026-06-10T02:18:00Z [INFO] Health check ok — inbound rate: 4480 req/s, queue depth: 24312 +2026-06-10T02:18:10Z [WARNING] Inbound connection rate from gateway: 12400 req/s (baseline 850 req/s) — gateway reconnect storm detected +2026-06-10T02:18:20Z [WARNING] Worker thread pool utilization: 448/512 (87%) — approaching saturation +2026-06-10T02:18:35Z [WARNING] Fanout queue depth: 48120 pending events (threshold 10000) +2026-06-10T02:18:50Z [WARNING] Worker thread pool utilization: 501/512 (98%) — critical +2026-06-10T02:19:05Z [ERROR] Worker thread pool saturated: 512/512 threads in use — new requests rejected +2026-06-10T02:19:10Z [ERROR] Message fanout rejected for channel_id=ch_9f2e1b: no worker threads available +2026-06-10T02:19:15Z [ERROR] Message fanout rejected for channel_id=ch_4a8c2d: no worker threads available +2026-06-10T02:19:20Z [ERROR] Message fanout rejected for channel_id=ch_1b7e3f: no worker threads available +2026-06-10T02:19:30Z [ERROR] Message fanout timeout for channel_id=ch_9f2e1c: upstream slow (2000ms deadline exceeded) +2026-06-10T02:19:45Z [ERROR] Message fanout timeout for channel_id=ch_6b3a4e: upstream slow (2000ms deadline exceeded) +2026-06-10T02:20:00Z [ERROR] Worker thread pool saturated: 512/512 threads in use — queue depth 84210 +2026-06-10T02:22:00Z [ERROR] Worker thread pool saturated: 512/512 threads in use — queue depth 91044 +2026-06-10T02:25:00Z [ERROR] Worker thread pool saturated: 512/512 threads in use — queue depth 88302 +2026-06-10T02:30:00Z [WARNING] Worker thread pool utilization: 498/512 — still elevated, reconnect storm persisting +2026-06-10T02:35:00Z [WARNING] Worker thread pool utilization: 420/512 — improving +2026-06-10T02:38:15Z [INFO] Inbound rate from gateway normalizing: 1840 req/s +2026-06-10T02:38:20Z [INFO] Worker thread pool: 248/512 in use — pressure releasing +2026-06-10T02:38:30Z [INFO] Fanout queue draining: 12401 pending +2026-06-10T02:39:00Z [INFO] Health check ok — inbound rate: 1120 req/s, queue depth: 4802 +2026-06-10T02:41:00Z [INFO] Health check ok — inbound rate: 882 req/s, queue depth: 388 +2026-06-10T02:42:00Z [INFO] Health check ok — inbound rate: 861 req/s, queue depth: 291 — nominal diff --git a/eval/dataset_4_discord_gateway_cascade/live/presence-svc.log b/eval/dataset_4_discord_gateway_cascade/live/presence-svc.log new file mode 100644 index 0000000..9543287 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/live/presence-svc.log @@ -0,0 +1,22 @@ +2026-06-10T02:10:00Z [INFO] PresenceService started, version 2.4.1 +2026-06-10T02:10:01Z [INFO] Connected to presence store (Redis cluster, 6 shards) +2026-06-10T02:11:00Z [INFO] Health check ok — presence updates: 480/s, Redis P99: 3ms +2026-06-10T02:12:00Z [INFO] Health check ok — presence updates: 492/s, Redis P99: 3ms +2026-06-10T02:13:00Z [INFO] Health check ok — presence updates: 501/s, Redis P99: 4ms +2026-06-10T02:14:00Z [INFO] Health check ok — presence updates: 488/s, Redis P99: 3ms +2026-06-10T02:15:00Z [INFO] Health check ok — presence updates: 495/s, Redis P99: 3ms +2026-06-10T02:17:00Z [WARNING] Presence update flood: 4200 state changes/s (threshold 500/s) — consistent with large client reconnect event +2026-06-10T02:17:10Z [WARNING] Inbound OFFLINE events: 4821/30s (clients disconnecting from gateway) +2026-06-10T02:17:30Z [WARNING] Inbound ONLINE events: 3901/30s (clients reconnecting to gateway) +2026-06-10T02:18:00Z [WARNING] Presence update queue depth: 18240 — processing backlog +2026-06-10T02:18:30Z [WARNING] Redis pipeline latency P99: 42ms (threshold 10ms) — write contention under flood +2026-06-10T02:19:00Z [WARNING] Presence update flood: 3800 state changes/s — sustained +2026-06-10T02:19:22Z [WARNING] Rate-limiting presence updates: dropping 18% of state changes to protect Redis cluster +2026-06-10T02:20:00Z [WARNING] Presence update flood: 3200 state changes/s +2026-06-10T02:22:00Z [WARNING] Presence update flood: 2800 state changes/s +2026-06-10T02:25:00Z [WARNING] Presence update flood: 2400 state changes/s — slowly subsiding +2026-06-10T02:30:00Z [WARNING] Presence update rate: 1200 state changes/s +2026-06-10T02:35:00Z [INFO] Health check ok — presence updates: 820/s, Redis P99: 5ms +2026-06-10T02:38:00Z [INFO] Presence update rate returning to normal: 540/s +2026-06-10T02:41:10Z [INFO] Health check ok — presence updates: 498/s, Redis P99: 3ms — nominal +2026-06-10T02:42:00Z [INFO] Health check ok — presence updates: 491/s, Redis P99: 3ms diff --git a/eval/dataset_4_discord_gateway_cascade/live/read-states-svc.log b/eval/dataset_4_discord_gateway_cascade/live/read-states-svc.log new file mode 100644 index 0000000..db8ed4d --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/live/read-states-svc.log @@ -0,0 +1,36 @@ +2026-06-10T02:10:00Z [INFO] Service started, version 3.8.1 (Cassandra: 4.1.3) +2026-06-10T02:10:01Z [INFO] Connected to Cassandra cluster (nodes: 10.0.1.6, 10.0.1.7, 10.0.1.8) +2026-06-10T02:10:02Z [INFO] Keyspace read_states: replication_factor=3, consistency=QUORUM +2026-06-10T02:11:00Z [INFO] Health check ok — batch write latency P99: 12ms +2026-06-10T02:12:00Z [INFO] Health check ok — batch write latency P99: 14ms +2026-06-10T02:13:00Z [INFO] Health check ok — batch write latency P99: 11ms +2026-06-10T02:14:00Z [INFO] Processing 142 read-state updates (user mark-reads, last-viewed) +2026-06-10T02:14:30Z [INFO] Health check ok — batch write latency P99: 18ms +2026-06-10T02:14:48Z [WARNING] Cassandra BATCH APPLY latency spike: 2847ms (threshold 500ms), coordinator=10.0.1.8 +2026-06-10T02:14:52Z [WARNING] Cassandra BATCH APPLY latency spike: 4103ms (threshold 500ms), coordinator=10.0.1.8 +2026-06-10T02:15:00Z [ERROR] Cassandra timeout on BATCH APPLY to read_states keyspace: OperationTimedOutException (coordinator=10.0.1.8, consistency=QUORUM, received=1, required=2) +2026-06-10T02:15:01Z [ERROR] Failed to persist read-state for user_id=u_9f2a1c, channel_id=ch_8821: write timeout +2026-06-10T02:15:02Z [ERROR] Cassandra timeout on BATCH APPLY to read_states keyspace: OperationTimedOutException (coordinator=10.0.1.8, consistency=QUORUM, received=1, required=2) +2026-06-10T02:15:03Z [ERROR] Failed to persist read-state for user_id=u_c3b8e2, channel_id=ch_4491: write timeout +2026-06-10T02:15:05Z [WARNING] Cassandra node 10.0.1.8 not responding to gossip — possible partition leader election in progress +2026-06-10T02:15:12Z [ERROR] Cassandra timeout on BATCH APPLY to read_states keyspace: OperationTimedOutException (coordinator=10.0.1.8, consistency=QUORUM, received=1, required=2) +2026-06-10T02:15:30Z [ERROR] Cassandra connection pool to coordinator 10.0.1.8 degraded: 28/50 connections healthy +2026-06-10T02:16:00Z [ERROR] Cassandra timeout on BATCH APPLY: coordinator=10.0.1.8 unreachable, retrying with 10.0.1.7 +2026-06-10T02:16:15Z [ERROR] All coordinators timeout: QUORUM write requires 2 nodes but only 1 acknowledged (coordinator=10.0.1.7) +2026-06-10T02:17:00Z [ERROR] Cassandra write queue depth: 8421 pending BATCH APPLY operations +2026-06-10T02:17:30Z [ERROR] Cassandra write queue depth: 18204 pending BATCH APPLY operations — queue growing faster than drain rate +2026-06-10T02:18:00Z [ERROR] read_states write queue overflow: dropping oldest 5000 pending writes to prevent OOM +2026-06-10T02:20:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:22:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:25:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:28:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:30:00Z [WARNING] Cassandra node 10.0.1.8 gossip re-established — partition leader election in progress +2026-06-10T02:33:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:36:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:38:30Z [INFO] Cassandra coordinator re-elected, new partition leader: 10.0.1.9 +2026-06-10T02:38:35Z [INFO] Cassandra QUORUM writes resumed — batch write latency P99: 28ms +2026-06-10T02:38:40Z [INFO] Processing 41821 queued read-state updates (backlog drain started) +2026-06-10T02:39:00Z [INFO] Health check ok — batch write latency P99: 22ms +2026-06-10T02:39:30Z [INFO] Backlog drain progress: 12400/41821 updates processed +2026-06-10T02:41:00Z [INFO] Backlog drain complete — all pending read-state updates written +2026-06-10T02:42:00Z [INFO] Health check ok — batch write latency P99: 13ms diff --git a/eval/dataset_4_discord_gateway_cascade/logs/cdn-edge.log b/eval/dataset_4_discord_gateway_cascade/logs/cdn-edge.log new file mode 100644 index 0000000..cb568e3 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/logs/cdn-edge.log @@ -0,0 +1,30 @@ +2026-06-10T02:10:00Z [INFO] CDN edge node cdn-edge-01 started (PoP: us-east-1) +2026-06-10T02:10:01Z [INFO] Origin backend: gateway-cluster.internal (4 instances, health checked) +2026-06-10T02:11:00Z [INFO] Health check ok — origin P99: 18ms, cache hit rate: 96% +2026-06-10T02:12:00Z [INFO] Health check ok — origin P99: 19ms, cache hit rate: 96% +2026-06-10T02:13:00Z [INFO] Health check ok — origin P99: 21ms, cache hit rate: 96% +2026-06-10T02:14:00Z [INFO] Health check ok — origin P99: 20ms, cache hit rate: 96% +2026-06-10T02:15:00Z [INFO] Health check ok — origin P99: 148ms, cache hit rate: 96% +2026-06-10T02:16:00Z [INFO] Health check ok — origin P99: 892ms, cache hit rate: 95% +2026-06-10T02:16:44Z [ERROR] Origin 502: gateway-01.internal /api/v9/channels/8821/messages returned 502 Bad Gateway +2026-06-10T02:16:46Z [ERROR] Origin 502: gateway-02.internal /api/v9/channels/4491/messages returned 502 Bad Gateway +2026-06-10T02:16:50Z [ERROR] Origin 502: gateway-01.internal /gateway?v=9&encoding=json returned 502 Bad Gateway +2026-06-10T02:16:55Z [ERROR] Origin 502: gateway-03.internal /api/v9/users/@me/channels returned 502 Bad Gateway +2026-06-10T02:16:58Z [ERROR] Origin 502 rate: 24% of requests to gateway cluster +2026-06-10T02:17:10Z [ERROR] Origin 502 rate: 41% of requests to gateway cluster +2026-06-10T02:17:20Z [ERROR] Origin 502 rate: 58% of requests to gateway cluster — circuit breaker threshold approaching +2026-06-10T02:17:30Z [WARNING] Serving stale cache for /api/v9/channels/* (TTL extended to 300s due to origin degradation) +2026-06-10T02:18:00Z [ERROR] Origin 502 rate: 62% of requests — gateway cluster under severe load +2026-06-10T02:18:30Z [WARNING] Edge cache miss rate elevated: 34% (baseline 4%) — reconnecting clients bypass cache with fresh IDENTIFY requests +2026-06-10T02:19:00Z [ERROR] Origin 502 rate: 71% — gateway message-svc pool exhausted, fanout requests failing +2026-06-10T02:20:00Z [ERROR] Origin 502 rate: 69% +2026-06-10T02:22:00Z [ERROR] Origin 502 rate: 65% +2026-06-10T02:25:00Z [ERROR] Origin 502 rate: 58% +2026-06-10T02:30:00Z [WARNING] Origin 502 rate: 44% — slowly improving +2026-06-10T02:35:00Z [WARNING] Origin 502 rate: 28% +2026-06-10T02:38:00Z [WARNING] Origin 502 rate: 12% — gateway recovering +2026-06-10T02:40:00Z [WARNING] Origin 502 rate: 4% +2026-06-10T02:42:10Z [INFO] Origin 502 rate dropped to 0 — gateway cluster healthy +2026-06-10T02:43:00Z [INFO] Health check ok — origin P99: 22ms, cache hit rate: 94% +2026-06-10T02:45:00Z [INFO] Health check ok — origin P99: 20ms, cache hit rate: 95% +2026-06-10T02:48:00Z [INFO] Edge health nominal — origin P99: 18ms, cache hit rate: 96% diff --git a/eval/dataset_4_discord_gateway_cascade/logs/gateway.log b/eval/dataset_4_discord_gateway_cascade/logs/gateway.log new file mode 100644 index 0000000..76087f4 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/logs/gateway.log @@ -0,0 +1,46 @@ +2026-06-10T02:10:00Z [INFO] Gateway started, version 8.14.2 +2026-06-10T02:10:01Z [INFO] WebSocket listener bound on :443 (TLS termination via envoy) +2026-06-10T02:10:02Z [INFO] Connected clients: 84312 +2026-06-10T02:11:00Z [INFO] Health check ok — connected clients: 85102, read_states P99: 8ms +2026-06-10T02:12:00Z [INFO] Health check ok — connected clients: 85318, read_states P99: 9ms +2026-06-10T02:13:00Z [INFO] Health check ok — connected clients: 85541, read_states P99: 11ms +2026-06-10T02:14:00Z [INFO] Health check ok — connected clients: 85749, read_states P99: 10ms +2026-06-10T02:15:00Z [INFO] Health check ok — connected clients: 86001, read_states P99: 312ms +2026-06-10T02:15:30Z [WARNING] read_states mark_read API P99 latency: 3214ms (threshold 200ms) — upstream degraded +2026-06-10T02:15:45Z [WARNING] read_states mark_read API P99 latency: 5800ms — requests queuing +2026-06-10T02:16:00Z [WARNING] mark_read upstream timeout rate: 12% of requests timing out +2026-06-10T02:16:15Z [ERROR] read_states mark_read timeout for session_id=sess_4f9a2b after 5000ms +2026-06-10T02:16:18Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_4f9a2b user_id=u_9f2a1c +2026-06-10T02:16:22Z [ERROR] read_states mark_read timeout for session_id=sess_7c1e3d after 5000ms +2026-06-10T02:16:24Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_7c1e3d user_id=u_3b8e2f +2026-06-10T02:16:30Z [ERROR] mark_read upstream timeout rate: 68% of requests timing out +2026-06-10T02:16:45Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_2a9c4e user_id=u_f7a1d9 +2026-06-10T02:16:50Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_b3f1c2 user_id=u_e2c4a1 +2026-06-10T02:17:00Z [ERROR] Bulk disconnect event: 842 sessions dropped in 60s (threshold 50) +2026-06-10T02:17:10Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_9d2e1f user_id=u_a4b9c8 +2026-06-10T02:17:20Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_c8f2b1 user_id=u_d1e7f3 +2026-06-10T02:17:30Z [ERROR] WebSocket session disconnect: upstream read_states timeout session_id=sess_4a1d9c user_id=u_b2c8e4 +2026-06-10T02:17:45Z [ERROR] Client reconnect storm: 4821 sessions reconnecting in 30s window (threshold 500) +2026-06-10T02:17:50Z [ERROR] Reconnect authentication flood: 4821 IDENTIFY payloads queued +2026-06-10T02:18:00Z [WARNING] message-svc event fanout queue depth: 31204 pending (baseline 400) +2026-06-10T02:18:10Z [ERROR] HTTP client pool to read-states-svc under pressure: 180/200 in use +2026-06-10T02:18:30Z [ERROR] HTTP client pool to message-svc under pressure: 190/200 in use +2026-06-10T02:19:00Z [ERROR] Reconnect storm continuing: 3201 sessions reconnecting in 30s window +2026-06-10T02:19:20Z [ERROR] HTTP client pool to message-svc EXHAUSTED (200/200 in use) — new fanout requests queuing +2026-06-10T02:19:35Z [ERROR] Event fanout timeout for session_id=sess_f1a2b3: no message-svc slot available after 2000ms +2026-06-10T02:19:40Z [ERROR] Event fanout timeout for session_id=sess_d4e5f6: no message-svc slot available after 2000ms +2026-06-10T02:20:00Z [ERROR] Connected clients: 71204 (-14797 from peak) — active disconnects ongoing +2026-06-10T02:21:00Z [ERROR] Connected clients: 68021 (-17980 from peak) — read_states upstream still timing out +2026-06-10T02:23:00Z [ERROR] Connected clients: 62441 — read_states upstream still timing out +2026-06-10T02:25:00Z [ERROR] Connected clients: 61208 — read_states upstream still timing out +2026-06-10T02:30:00Z [ERROR] Connected clients: 60112 — sustained degradation +2026-06-10T02:35:00Z [WARNING] Connected clients: 61804 — slight recovery, reconnects stabilising +2026-06-10T02:38:00Z [INFO] read_states mark_read API latency normalized: P99=45ms (upstream recovered) +2026-06-10T02:38:05Z [INFO] mark_read upstream timeout rate: 0.2% (nominal) +2026-06-10T02:38:10Z [INFO] Client reconnect rate dropping: 180 sessions reconnecting in 30s window +2026-06-10T02:38:30Z [INFO] HTTP client pool to read-states-svc recovered: 14/200 in use +2026-06-10T02:39:00Z [INFO] HTTP client pool to message-svc recovered: 8/200 in use +2026-06-10T02:41:00Z [INFO] Reconnect storm subsided: 41 sessions reconnecting in 30s window +2026-06-10T02:43:00Z [INFO] Connected clients: 83201 — recovery in progress +2026-06-10T02:46:00Z [INFO] Health check ok — connected clients: 84892, read_states P99: 12ms +2026-06-10T02:48:00Z [INFO] Connected clients: 85104 — fully recovered diff --git a/eval/dataset_4_discord_gateway_cascade/logs/message-svc.log b/eval/dataset_4_discord_gateway_cascade/logs/message-svc.log new file mode 100644 index 0000000..3589754 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/logs/message-svc.log @@ -0,0 +1,30 @@ +2026-06-10T02:10:00Z [INFO] MessageService started, version 5.2.8 +2026-06-10T02:10:01Z [INFO] Worker thread pool initialized: 512 threads +2026-06-10T02:10:02Z [INFO] Connected to gateway cluster (load balanced, 12 gateway instances) +2026-06-10T02:11:00Z [INFO] Health check ok — inbound rate: 848 req/s, queue depth: 312 +2026-06-10T02:12:00Z [INFO] Health check ok — inbound rate: 852 req/s, queue depth: 289 +2026-06-10T02:13:00Z [INFO] Health check ok — inbound rate: 861 req/s, queue depth: 301 +2026-06-10T02:14:00Z [INFO] Health check ok — inbound rate: 874 req/s, queue depth: 320 +2026-06-10T02:17:00Z [INFO] Health check ok — inbound rate: 1240 req/s, queue depth: 4812 +2026-06-10T02:18:00Z [INFO] Health check ok — inbound rate: 4480 req/s, queue depth: 24312 +2026-06-10T02:18:10Z [WARNING] Inbound connection rate from gateway: 12400 req/s (baseline 850 req/s) — gateway reconnect storm detected +2026-06-10T02:18:20Z [WARNING] Worker thread pool utilization: 448/512 (87%) — approaching saturation +2026-06-10T02:18:35Z [WARNING] Fanout queue depth: 48120 pending events (threshold 10000) +2026-06-10T02:18:50Z [WARNING] Worker thread pool utilization: 501/512 (98%) — critical +2026-06-10T02:19:05Z [ERROR] Worker thread pool saturated: 512/512 threads in use — new requests rejected +2026-06-10T02:19:10Z [ERROR] Message fanout rejected for channel_id=ch_9f2e1b: no worker threads available +2026-06-10T02:19:15Z [ERROR] Message fanout rejected for channel_id=ch_4a8c2d: no worker threads available +2026-06-10T02:19:20Z [ERROR] Message fanout rejected for channel_id=ch_1b7e3f: no worker threads available +2026-06-10T02:19:30Z [ERROR] Message fanout timeout for channel_id=ch_9f2e1c: upstream slow (2000ms deadline exceeded) +2026-06-10T02:19:45Z [ERROR] Message fanout timeout for channel_id=ch_6b3a4e: upstream slow (2000ms deadline exceeded) +2026-06-10T02:20:00Z [ERROR] Worker thread pool saturated: 512/512 threads in use — queue depth 84210 +2026-06-10T02:22:00Z [ERROR] Worker thread pool saturated: 512/512 threads in use — queue depth 91044 +2026-06-10T02:25:00Z [ERROR] Worker thread pool saturated: 512/512 threads in use — queue depth 88302 +2026-06-10T02:30:00Z [WARNING] Worker thread pool utilization: 498/512 — still elevated, reconnect storm persisting +2026-06-10T02:35:00Z [WARNING] Worker thread pool utilization: 420/512 — improving +2026-06-10T02:38:15Z [INFO] Inbound rate from gateway normalizing: 1840 req/s +2026-06-10T02:38:20Z [INFO] Worker thread pool: 248/512 in use — pressure releasing +2026-06-10T02:38:30Z [INFO] Fanout queue draining: 12401 pending +2026-06-10T02:39:00Z [INFO] Health check ok — inbound rate: 1120 req/s, queue depth: 4802 +2026-06-10T02:41:00Z [INFO] Health check ok — inbound rate: 882 req/s, queue depth: 388 +2026-06-10T02:42:00Z [INFO] Health check ok — inbound rate: 861 req/s, queue depth: 291 — nominal diff --git a/eval/dataset_4_discord_gateway_cascade/logs/presence-svc.log b/eval/dataset_4_discord_gateway_cascade/logs/presence-svc.log new file mode 100644 index 0000000..9543287 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/logs/presence-svc.log @@ -0,0 +1,22 @@ +2026-06-10T02:10:00Z [INFO] PresenceService started, version 2.4.1 +2026-06-10T02:10:01Z [INFO] Connected to presence store (Redis cluster, 6 shards) +2026-06-10T02:11:00Z [INFO] Health check ok — presence updates: 480/s, Redis P99: 3ms +2026-06-10T02:12:00Z [INFO] Health check ok — presence updates: 492/s, Redis P99: 3ms +2026-06-10T02:13:00Z [INFO] Health check ok — presence updates: 501/s, Redis P99: 4ms +2026-06-10T02:14:00Z [INFO] Health check ok — presence updates: 488/s, Redis P99: 3ms +2026-06-10T02:15:00Z [INFO] Health check ok — presence updates: 495/s, Redis P99: 3ms +2026-06-10T02:17:00Z [WARNING] Presence update flood: 4200 state changes/s (threshold 500/s) — consistent with large client reconnect event +2026-06-10T02:17:10Z [WARNING] Inbound OFFLINE events: 4821/30s (clients disconnecting from gateway) +2026-06-10T02:17:30Z [WARNING] Inbound ONLINE events: 3901/30s (clients reconnecting to gateway) +2026-06-10T02:18:00Z [WARNING] Presence update queue depth: 18240 — processing backlog +2026-06-10T02:18:30Z [WARNING] Redis pipeline latency P99: 42ms (threshold 10ms) — write contention under flood +2026-06-10T02:19:00Z [WARNING] Presence update flood: 3800 state changes/s — sustained +2026-06-10T02:19:22Z [WARNING] Rate-limiting presence updates: dropping 18% of state changes to protect Redis cluster +2026-06-10T02:20:00Z [WARNING] Presence update flood: 3200 state changes/s +2026-06-10T02:22:00Z [WARNING] Presence update flood: 2800 state changes/s +2026-06-10T02:25:00Z [WARNING] Presence update flood: 2400 state changes/s — slowly subsiding +2026-06-10T02:30:00Z [WARNING] Presence update rate: 1200 state changes/s +2026-06-10T02:35:00Z [INFO] Health check ok — presence updates: 820/s, Redis P99: 5ms +2026-06-10T02:38:00Z [INFO] Presence update rate returning to normal: 540/s +2026-06-10T02:41:10Z [INFO] Health check ok — presence updates: 498/s, Redis P99: 3ms — nominal +2026-06-10T02:42:00Z [INFO] Health check ok — presence updates: 491/s, Redis P99: 3ms diff --git a/eval/dataset_4_discord_gateway_cascade/logs/read-states-svc.log b/eval/dataset_4_discord_gateway_cascade/logs/read-states-svc.log new file mode 100644 index 0000000..db8ed4d --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/logs/read-states-svc.log @@ -0,0 +1,36 @@ +2026-06-10T02:10:00Z [INFO] Service started, version 3.8.1 (Cassandra: 4.1.3) +2026-06-10T02:10:01Z [INFO] Connected to Cassandra cluster (nodes: 10.0.1.6, 10.0.1.7, 10.0.1.8) +2026-06-10T02:10:02Z [INFO] Keyspace read_states: replication_factor=3, consistency=QUORUM +2026-06-10T02:11:00Z [INFO] Health check ok — batch write latency P99: 12ms +2026-06-10T02:12:00Z [INFO] Health check ok — batch write latency P99: 14ms +2026-06-10T02:13:00Z [INFO] Health check ok — batch write latency P99: 11ms +2026-06-10T02:14:00Z [INFO] Processing 142 read-state updates (user mark-reads, last-viewed) +2026-06-10T02:14:30Z [INFO] Health check ok — batch write latency P99: 18ms +2026-06-10T02:14:48Z [WARNING] Cassandra BATCH APPLY latency spike: 2847ms (threshold 500ms), coordinator=10.0.1.8 +2026-06-10T02:14:52Z [WARNING] Cassandra BATCH APPLY latency spike: 4103ms (threshold 500ms), coordinator=10.0.1.8 +2026-06-10T02:15:00Z [ERROR] Cassandra timeout on BATCH APPLY to read_states keyspace: OperationTimedOutException (coordinator=10.0.1.8, consistency=QUORUM, received=1, required=2) +2026-06-10T02:15:01Z [ERROR] Failed to persist read-state for user_id=u_9f2a1c, channel_id=ch_8821: write timeout +2026-06-10T02:15:02Z [ERROR] Cassandra timeout on BATCH APPLY to read_states keyspace: OperationTimedOutException (coordinator=10.0.1.8, consistency=QUORUM, received=1, required=2) +2026-06-10T02:15:03Z [ERROR] Failed to persist read-state for user_id=u_c3b8e2, channel_id=ch_4491: write timeout +2026-06-10T02:15:05Z [WARNING] Cassandra node 10.0.1.8 not responding to gossip — possible partition leader election in progress +2026-06-10T02:15:12Z [ERROR] Cassandra timeout on BATCH APPLY to read_states keyspace: OperationTimedOutException (coordinator=10.0.1.8, consistency=QUORUM, received=1, required=2) +2026-06-10T02:15:30Z [ERROR] Cassandra connection pool to coordinator 10.0.1.8 degraded: 28/50 connections healthy +2026-06-10T02:16:00Z [ERROR] Cassandra timeout on BATCH APPLY: coordinator=10.0.1.8 unreachable, retrying with 10.0.1.7 +2026-06-10T02:16:15Z [ERROR] All coordinators timeout: QUORUM write requires 2 nodes but only 1 acknowledged (coordinator=10.0.1.7) +2026-06-10T02:17:00Z [ERROR] Cassandra write queue depth: 8421 pending BATCH APPLY operations +2026-06-10T02:17:30Z [ERROR] Cassandra write queue depth: 18204 pending BATCH APPLY operations — queue growing faster than drain rate +2026-06-10T02:18:00Z [ERROR] read_states write queue overflow: dropping oldest 5000 pending writes to prevent OOM +2026-06-10T02:20:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:22:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:25:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:28:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:30:00Z [WARNING] Cassandra node 10.0.1.8 gossip re-established — partition leader election in progress +2026-06-10T02:33:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:36:00Z [ERROR] Cassandra timeout on BATCH APPLY: OperationTimedOutException (received=1, required=2) +2026-06-10T02:38:30Z [INFO] Cassandra coordinator re-elected, new partition leader: 10.0.1.9 +2026-06-10T02:38:35Z [INFO] Cassandra QUORUM writes resumed — batch write latency P99: 28ms +2026-06-10T02:38:40Z [INFO] Processing 41821 queued read-state updates (backlog drain started) +2026-06-10T02:39:00Z [INFO] Health check ok — batch write latency P99: 22ms +2026-06-10T02:39:30Z [INFO] Backlog drain progress: 12400/41821 updates processed +2026-06-10T02:41:00Z [INFO] Backlog drain complete — all pending read-state updates written +2026-06-10T02:42:00Z [INFO] Health check ok — batch write latency P99: 13ms diff --git a/eval/dataset_4_discord_gateway_cascade/seed.py b/eval/dataset_4_discord_gateway_cascade/seed.py new file mode 100644 index 0000000..ae20346 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/seed.py @@ -0,0 +1,49 @@ +"""Seed dataset_4_discord_gateway_cascade. + +Ingests 5 service logs from a Discord-style gateway cascade outage. +Root cause: Cassandra partition leader re-election in read-states-svc causes +a reconnect storm in gateway, which saturates message-svc's thread pool and +triggers 502s on cdn-edge. presence-svc is a red herring (elevated only due +to the reconnect flood, not a causal factor). +""" +from __future__ import annotations +import asyncio +from pathlib import Path +from repi.core.container import get_container + +LOGS_DIR = Path(__file__).parent / "logs" + +SERVICES = { + "read-states-svc.log": "read-states-svc", + "gateway.log": "gateway", + "message-svc.log": "message-svc", + "presence-svc.log": "presence-svc", + "cdn-edge.log": "cdn-edge", +} + + +async def main() -> int: + container = get_container() + total = 0 + async with container.get_session() as session: + from sqlalchemy import text + await session.execute(text("TRUNCATE TABLE log_chunks RESTART IDENTITY CASCADE")) + await session.commit() + + ingestor = container.get_ingestor(session) + for filename, service in SERVICES.items(): + path = LOGS_DIR / filename + if not path.exists(): + print(f" SKIP {service}: {path} not found") + continue + content = path.read_text() + count = (await ingestor.ingest(content, source_service=service, source_env="eval")).chunk_count + print(f" {service:20s} {count:4d} chunks") + total += count + + print(f"\nSeeded {total} chunks for dataset_4_discord_gateway_cascade") + return total + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/eval/dataset_4_discord_gateway_cascade/simulate_live.py b/eval/dataset_4_discord_gateway_cascade/simulate_live.py new file mode 100644 index 0000000..2f291d9 --- /dev/null +++ b/eval/dataset_4_discord_gateway_cascade/simulate_live.py @@ -0,0 +1,180 @@ +"""Replay the Discord gateway cascade logs incrementally. + +Simulates real-time log growth so the repi worker can pick up new bytes via +watchers as the incident unfolds. Writes to a separate live/ directory so the +static eval logs stay untouched. + +Usage: + # 1. Start the API and worker in separate terminals: + # uv run repi serve + # uv run python -m repi.worker + + # 2. Register watchers (once) — copy the printed curl commands: + uv run python eval/dataset_4_discord_gateway_cascade/simulate_live.py --register + + # 3. Replay logs (in a third terminal): + uv run python eval/dataset_4_discord_gateway_cascade/simulate_live.py + + # Options: + # --live-dir /tmp/repi-discord Output directory (default: eval/.../live/) + # --batch-seconds 60 How many log-time seconds per batch (default: 60) + # --sleep 5 Real seconds to sleep between batches (default: 5) + # --register Print curl commands for watcher registration then exit +""" +from __future__ import annotations +import argparse +import os +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +LOGS_DIR = Path(__file__).parent / "logs" +DEFAULT_LIVE_DIR = Path(__file__).parent / "live" + +SERVICES = [ + ("read-states-svc.log", "read-states-svc"), + ("gateway.log", "gateway"), + ("message-svc.log", "message-svc"), + ("presence-svc.log", "presence-svc"), + ("cdn-edge.log", "cdn-edge"), +] + +API_BASE = os.environ.get("REPI_API", "http://localhost:8000") + + +def _parse_ts(line: str) -> Optional[datetime]: + """Extract the leading ISO8601 timestamp from a log line.""" + try: + ts_str = line.split(" ")[0] + return datetime.fromisoformat(ts_str.replace("Z", "+00:00")) + except Exception: + return None + + +def _load_lines(filename: str) -> list[tuple[datetime, str]]: + path = LOGS_DIR / filename + if not path.exists(): + print(f" [warn] {path} not found — skipping", file=sys.stderr) + return [] + entries: list[tuple[datetime, str]] = [] + for raw in path.read_text().splitlines(): + line = raw.rstrip() + if not line: + continue + ts = _parse_ts(line) + if ts: + entries.append((ts, line)) + return entries + + +def _print_watcher_commands(live_dir: Path) -> None: + print("\nRegister these watchers via the repi API:\n") + for filename, service in SERVICES: + target = live_dir / filename + print( + f"curl -s -X POST {API_BASE}/watchers \\\n" + f" -H 'Content-Type: application/json' \\\n" + f" -d '{{\"service_name\":\"{service}\",\"watch_path\":\"{target}\",\"env\":\"live-demo\"}}'" + ) + print() + print(f"Live log files will be written to: {live_dir}\n") + + +def run_replay(live_dir: Path, batch_seconds: int, sleep_secs: float) -> None: + live_dir.mkdir(parents=True, exist_ok=True) + + # Collect all entries from all services, sort by timestamp. + all_entries: list[tuple[datetime, str, Path]] = [] + for filename, _ in SERVICES: + target = live_dir / filename + # Truncate (or create) the live file so watcher offset resets. + target.write_text("") + for ts, line in _load_lines(filename): + all_entries.append((ts, line, target)) + + all_entries.sort(key=lambda x: x[0]) + + if not all_entries: + print("No log entries found — check LOGS_DIR.", file=sys.stderr) + sys.exit(1) + + start_ts = all_entries[0][0] + end_ts = all_entries[-1][0] + total_log_seconds = (end_ts - start_ts).total_seconds() + batches = int(total_log_seconds / batch_seconds) + 1 + + print( + f"Replaying {len(all_entries)} log lines across {len(SERVICES)} services\n" + f" Log window : {start_ts.isoformat()} → {end_ts.isoformat()}\n" + f" Batch size : {batch_seconds}s of log-time per batch\n" + f" Sleep : {sleep_secs}s between batches\n" + f" Batches : ~{batches}\n" + f" Live dir : {live_dir}\n" + f" Total wall : ~{int(batches * sleep_secs)}s\n" + ) + print("─" * 60) + + bucket_start = start_ts + written_total = 0 + batch_num = 0 + + while True: + bucket_end_ts = bucket_start.timestamp() + batch_seconds + batch: list[tuple[datetime, str, Path]] = [] + remaining: list[tuple[datetime, str, Path]] = [] + + for entry in all_entries: + if entry[0].timestamp() < bucket_end_ts: + batch.append(entry) + else: + remaining.append(entry) + + if batch: + batch_num += 1 + # Group by file to do one append per file per batch. + by_file: dict[Path, list[str]] = {} + for ts, line, target in batch: + by_file.setdefault(target, []).append(line) + + now_wall = datetime.now(timezone.utc).strftime("%H:%M:%S") + log_time = batch[0][0].strftime("%H:%M:%S") + print(f"[{now_wall}] batch {batch_num:3d} log-time {log_time} +{len(batch):3d} lines", end="") + + for target, lines in by_file.items(): + with open(target, "a") as f: + f.write("\n".join(lines) + "\n") + print(f" {target.stem}({len(lines)})", end="") + + print() + written_total += len(batch) + all_entries = remaining + + if not all_entries: + print(f"\nDone — wrote {written_total} lines total.") + break + + bucket_start = datetime.fromtimestamp(bucket_end_ts, tz=timezone.utc) + time.sleep(sleep_secs) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Replay Discord cascade logs for live watcher demo") + parser.add_argument("--live-dir", default=str(DEFAULT_LIVE_DIR), help="Output directory for live log files") + parser.add_argument("--batch-seconds", type=int, default=60, help="Log-time seconds per batch (default 60)") + parser.add_argument("--sleep", type=float, default=5.0, help="Real seconds between batches (default 5)") + parser.add_argument("--register", action="store_true", help="Print watcher curl commands and exit") + args = parser.parse_args() + + live_dir = Path(args.live_dir).resolve() + + if args.register: + _print_watcher_commands(live_dir) + return + + run_replay(live_dir, batch_seconds=args.batch_seconds, sleep_secs=args.sleep) + + +if __name__ == "__main__": + main() diff --git a/eval/ragas_datasets/ragas_discord_cascade/__init__.py b/eval/ragas_datasets/ragas_discord_cascade/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eval/ragas_datasets/ragas_discord_cascade/ground_truth.json b/eval/ragas_datasets/ragas_discord_cascade/ground_truth.json new file mode 100644 index 0000000..37956b2 --- /dev/null +++ b/eval/ragas_datasets/ragas_discord_cascade/ground_truth.json @@ -0,0 +1,42 @@ +{ + "dataset_name": "ragas_discord_cascade", + "description": "Tests retrieval across a 5-service cascade. The query surface (gateway disconnects, 502s) is far from the root cause (Cassandra timeouts in read-states-svc). Also includes a red-herring service (presence-svc) that shows real activity but is downstream of the storm, not causal.", + "questions": [ + { + "question": "why are users getting disconnected from the gateway", + "ground_truth_answer": "Cassandra partition leader re-election in read-states-svc caused OperationTimedOutException on all BATCH APPLY writes. Gateway timed out waiting for mark_read responses, disconnecting sessions and triggering a reconnect storm of 4821 sessions in 30 seconds. This exhausted gateway's HTTP client pool to message-svc (200/200 in use), leading to event fanout timeouts and sustained client disconnects.", + "ground_truth_services": ["read-states-svc", "gateway", "message-svc"], + "ground_truth_keywords": ["Cassandra timeout", "OperationTimedOutException", "mark_read", "session disconnect", "reconnect storm", "EXHAUSTED"] + }, + { + "question": "Cassandra timeouts read states service", + "ground_truth_answer": "read-states-svc experienced OperationTimedOutException on BATCH APPLY operations to the read_states keyspace when Cassandra coordinator 10.0.1.8 became unreachable during a partition leader election. QUORUM consistency required 2 node acknowledgements but only 1 was available, causing all write operations to time out. The write queue grew to 18204 pending BATCH APPLY operations before the leader re-elected to 10.0.1.9.", + "ground_truth_services": ["read-states-svc"], + "ground_truth_keywords": ["Cassandra", "BATCH APPLY", "OperationTimedOutException", "QUORUM", "partition leader", "coordinator=10.0.1.8", "10.0.1.9"] + }, + { + "question": "why did cdn edge start returning 502 errors", + "ground_truth_answer": "cdn-edge began returning 502 Bad Gateway errors when the gateway cluster became overloaded by a client reconnect storm caused by Cassandra timeouts in read-states-svc. Origin 502 rate peaked at 71% as gateway's message-svc HTTP client pool was exhausted. cdn-edge extended cache TTLs as a fallback and observed elevated cache miss rates (34%, baseline 4%) from clients bypassing cache with reconnect requests.", + "ground_truth_services": ["cdn-edge", "gateway"], + "ground_truth_keywords": ["502", "Bad Gateway", "gateway-01.internal", "71%", "cache miss rate", "34%"] + }, + { + "question": "message service thread pool saturated", + "ground_truth_answer": "message-svc thread pool was saturated (512/512 threads in use) due to a reconnect storm from gateway. When Cassandra timeouts in read-states-svc caused gateway to disconnect 4821 sessions in 30 seconds, all reconnecting clients sent IDENTIFY payloads that flooded message-svc with 12400 req/s (baseline 850 req/s). Message fanout requests were rejected and timed out, with queue depth reaching 91044 at peak.", + "ground_truth_services": ["message-svc", "gateway"], + "ground_truth_keywords": ["512/512", "saturated", "12400 req/s", "reconnect storm", "fanout rejected", "queue depth 91044"] + }, + { + "question": "what triggered the client reconnect storm", + "ground_truth_answer": "The reconnect storm was triggered by Cassandra write timeouts in read-states-svc causing gateway to time out on mark_read API calls and disconnect WebSocket sessions. When read-states-svc's Cassandra coordinator (10.0.1.8) stopped responding during a partition leader election, gateway's mark_read P99 latency spiked to 5800ms. Sessions were disconnected after 5000ms timeouts, producing a bulk disconnect event of 842 sessions in 60 seconds, followed by 4821 sessions reconnecting in 30 seconds.", + "ground_truth_services": ["gateway", "read-states-svc"], + "ground_truth_keywords": ["read_states timeout", "Cassandra", "partition leader", "mark_read", "5800ms", "4821 sessions", "bulk disconnect"] + }, + { + "question": "was presence service the cause of the outage", + "ground_truth_answer": "No. presence-svc was a downstream symptom of the gateway reconnect storm, not a cause. It experienced a presence update flood (4200 state changes/s vs 500/s baseline) because 4821 clients disconnected and reconnected from gateway, generating OFFLINE and ONLINE state changes. presence-svc backed its state in Redis (not Cassandra), had no connection pool exhaustion, and its elevated activity resolved automatically once the gateway reconnect storm subsided. The root cause was Cassandra timeouts in read-states-svc.", + "ground_truth_services": ["presence-svc"], + "ground_truth_keywords": ["OFFLINE events", "ONLINE events", "Redis", "rate-limiting", "downstream", "reconnect storm"] + } + ] +} diff --git a/eval/ragas_datasets/ragas_discord_cascade/seed.py b/eval/ragas_datasets/ragas_discord_cascade/seed.py new file mode 100644 index 0000000..e68e4ef --- /dev/null +++ b/eval/ragas_datasets/ragas_discord_cascade/seed.py @@ -0,0 +1,51 @@ +"""Seed the ragas_discord_cascade dataset. + +Reuses log files from dataset_4_discord_gateway_cascade. Ingests all 5 +services so the retrieval pipeline must surface read-states-svc evidence +even when the query only mentions gateway or disconnects. +""" +from __future__ import annotations +import asyncio +from pathlib import Path +from repi.core.container import get_container + +LOGS_DIR = ( + Path(__file__).parent.parent.parent + / "dataset_4_discord_gateway_cascade" + / "logs" +) + +SERVICES = { + "read-states-svc.log": "read-states-svc", + "gateway.log": "gateway", + "message-svc.log": "message-svc", + "presence-svc.log": "presence-svc", + "cdn-edge.log": "cdn-edge", +} + + +async def main() -> int: + container = get_container() + total = 0 + async with container.get_session() as session: + from sqlalchemy import text + await session.execute(text("TRUNCATE TABLE log_chunks RESTART IDENTITY CASCADE")) + await session.commit() + + ingestor = container.get_ingestor(session) + for filename, service in SERVICES.items(): + path = LOGS_DIR / filename + if not path.exists(): + print(f" SKIP {service}: {path} not found") + continue + content = path.read_text() + count = (await ingestor.ingest(content, source_service=service, source_env="eval")).chunk_count + print(f" {service:20s} {count:4d} chunks") + total += count + + print(f"\nSeeded {total} chunks for ragas_discord_cascade") + return total + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/eval/ragas_eval.py b/eval/ragas_eval.py index 7a66d5d..b1214ae 100644 --- a/eval/ragas_eval.py +++ b/eval/ragas_eval.py @@ -70,6 +70,12 @@ "seed_module": "eval.ragas_datasets.ragas_cross_service.seed", "ground_truth_path": ROOT / "eval/ragas_datasets/ragas_cross_service/ground_truth.json", }, + { + "name": "ragas_discord_cascade", + "category": "agent", + "seed_module": "eval.ragas_datasets.ragas_discord_cascade.seed", + "ground_truth_path": ROOT / "eval/ragas_datasets/ragas_discord_cascade/ground_truth.json", + }, ] diff --git a/eval/run_evals.py b/eval/run_evals.py index c0616ba..6715097 100644 --- a/eval/run_evals.py +++ b/eval/run_evals.py @@ -41,6 +41,11 @@ "seed_module": "eval.dataset_3_jwt_key_rotation_noise.seed", "expected_path": ROOT / "eval/dataset_3_jwt_key_rotation_noise/expected.json", }, + { + "name": "dataset_4_discord_gateway_cascade", + "seed_module": "eval.dataset_4_discord_gateway_cascade.seed", + "expected_path": ROOT / "eval/dataset_4_discord_gateway_cascade/expected.json", + }, ] # ─── CLI arg parsing ──────────────────────────────────────────────────────── @@ -123,7 +128,7 @@ def create_judge(args: dict, mut_provider_name: str) -> LLMJudge: preferences = [ ("openai", settings.OPENAI_API_KEY, lambda k: OpenAIProvider(api_key=k, model="gpt-4o")), ("anthropic", settings.ANTHROPIC_API_KEY, lambda k: AnthropicProvider(api_key=k, model="claude-3-5-sonnet-20240620")), - ("gemini", settings.GEMINI_API_KEY, lambda k: GeminiProvider(api_key=k, model="gemini-1.5-pro")), + ("gemini", settings.GEMINI_API_KEY, lambda k: GeminiProvider(api_key=k, model="gemini-2.0-flash")), ("mistral", settings.MISTRAL_API_KEY, lambda k: MistralProvider(api_key=k, model="mistral-large-latest")), ] for name, key, factory in preferences: diff --git a/repi/core/config.py b/repi/core/config.py index e96ad84..9f4a823 100644 --- a/repi/core/config.py +++ b/repi/core/config.py @@ -58,12 +58,16 @@ class Settings(BaseSettings): WATCHER_CONFIG_REFRESH_SECS: int = 30 - # "fastembed" (ONNX Runtime, ~50 MB) or "torch" via sentence-transformers - # (~790 MB). Vectors are byte-identical; the choice is image size / RSS. - # Additional options: "nomic" (768d), "bge" (384d). - EMBEDDING_BACKEND: str = "fastembed" - - # "paradedb" (BM25 via pg_search) or "pg" (PostgreSQL tsvector). + # "bge" (BAAI/bge-small-en-v1.5, 384d, ONNX) — default, best retrieval quality. + # "fastembed" (all-MiniLM-L6-v2, 384d, ONNX) — lighter, same dimension. + # "nomic" (nomic-embed-text-v1.5, 768d) — highest quality, needs schema migration. + # "torch" — sentence-transformers CPU path, ~790 MB. + # All 384d backends produce byte-compatible vectors; switching between them + # requires re-ingesting existing chunks (embeddings are not interchangeable). + EMBEDDING_BACKEND: str = "bge" + + # "paradedb" (BM25 via pg_search) — default, recommended. + # "pg" (PostgreSQL tsvector) — fallback if pg_search extension is unavailable. FTS_BACKEND: str = "paradedb" ENABLE_LEVEL_BOOST: bool = True diff --git a/repi/llm/adapters.py b/repi/llm/adapters.py index 2f1f382..da4a924 100644 --- a/repi/llm/adapters.py +++ b/repi/llm/adapters.py @@ -197,7 +197,7 @@ def model_name(self) -> str: return self._model class GeminiProvider(LLMProvider): - def __init__(self, api_key: str, model: str = "gemini-1.5-pro"): + def __init__(self, api_key: str, model: str = "gemini-2.0-flash"): self._api_key = api_key self._model = model self._url_tmpl = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={key}"