From c50205e749c9c3c797634a06a2e8fe9e762647c0 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:42:28 +0400 Subject: [PATCH 01/39] chore(lint): clear rustc 1.95 clippy and fmt regressions Pre-existing regressions on rustc 1.95 baseline (useless_vec, unnecessary_sort_by, rustfmt style adjustments) blocked the CI gate. Applied: cargo fmt --all + cargo clippy --fix + one manual sort_by_key migration in session/discovery.rs. No semantic changes; formatting and lint compliance only. Closes claude-memory-iyn.12 --- crates/tj-cli/src/main.rs | 27 ++-- crates/tj-cli/src/tui/app.rs | 16 +-- crates/tj-cli/src/tui/chat_view.rs | 48 ++++--- crates/tj-cli/src/tui/mod.rs | 2 +- crates/tj-cli/src/tui/session_list.rs | 86 +++++++------ crates/tj-core/src/session/discovery.rs | 8 +- crates/tj-core/src/session/extractor.rs | 161 ++++++++++++++++-------- crates/tj-core/src/session/parser.rs | 135 +++++++++++--------- 8 files changed, 293 insertions(+), 190 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index c596568..8154591 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -434,8 +434,7 @@ fn main() -> Result<()> { }; let (etype, task_id, confidence, evidence_strength, suggested_text) = - if let (Some(t), Some(tid)) = - (mock_event_type.as_deref(), mock_task_id.as_deref()) + if let (Some(t), Some(tid)) = (mock_event_type.as_deref(), mock_task_id.as_deref()) { ( parse_event_type(t)?, @@ -459,9 +458,7 @@ fn main() -> Result<()> { use tj_core::classifier::Classifier; let classifier: Box = match backend.as_str() { - "cli" => { - Box::new(tj_core::classifier::cli::ClaudeCliClassifier::default()) - } + "cli" => Box::new(tj_core::classifier::cli::ClaudeCliClassifier::default()), "api" => { Box::new(tj_core::classifier::http::AnthropicClassifier::from_env()?) } @@ -586,10 +583,8 @@ fn main() -> Result<()> { println!("# Task Journal Export\n"); // Group events by task_id. - let mut tasks: std::collections::BTreeMap< - String, - Vec<&tj_core::event::Event>, - > = std::collections::BTreeMap::new(); + let mut tasks: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); for e in &events { tasks.entry(e.task_id.clone()).or_default().push(e); } @@ -704,7 +699,10 @@ fn main() -> Result<()> { }; let mut app = tui::app::App::new(&project_path)?; if app.session_list.sessions.is_empty() { - eprintln!("No Claude Code sessions found for: {}", project_path.display()); + eprintln!( + "No Claude Code sessions found for: {}", + project_path.display() + ); return Ok(()); } app.run()?; @@ -784,7 +782,10 @@ fn main() -> Result<()> { .to_string(); if already_imported.contains(&session_id) { - eprintln!(" ⊘ {} — already imported, skipping", &session_id[..8.min(session_id.len())]); + eprintln!( + " ⊘ {} — already imported, skipping", + &session_id[..8.min(session_id.len())] + ); continue; } @@ -856,7 +857,9 @@ fn main() -> Result<()> { } if dry_run { - eprintln!("\nDry run: would create {total_tasks} task(s) with {total_events} event(s)."); + eprintln!( + "\nDry run: would create {total_tasks} task(s) with {total_events} event(s)." + ); eprintln!("Run without --dry-run to import."); } else { eprintln!("\nImported {total_tasks} task(s) with {total_events} event(s)."); diff --git a/crates/tj-cli/src/tui/app.rs b/crates/tj-cli/src/tui/app.rs index 7d4a327..8b9de2c 100644 --- a/crates/tj-cli/src/tui/app.rs +++ b/crates/tj-cli/src/tui/app.rs @@ -71,13 +71,11 @@ impl App { fn main_loop(&mut self, terminal: &mut Terminal>) -> Result<()> { loop { - terminal.draw(|frame| { - match &self.screen { - Screen::List => self.session_list.render(frame), - Screen::Chat => { - if let Some(ref cv) = self.chat_view { - cv.render(frame); - } + terminal.draw(|frame| match &self.screen { + Screen::List => self.session_list.render(frame), + Screen::Chat => { + if let Some(ref cv) = self.chat_view { + cv.render(frame); } } })?; @@ -85,7 +83,9 @@ impl App { if event::poll(std::time::Duration::from_millis(100))? { if let Event::Key(key) = event::read()? { // Global: Ctrl+C or q quits. - if key.modifiers.contains(KeyModifiers::CONTROL) && key.code == KeyCode::Char('c') { + if key.modifiers.contains(KeyModifiers::CONTROL) + && key.code == KeyCode::Char('c') + { self.should_quit = true; } diff --git a/crates/tj-cli/src/tui/chat_view.rs b/crates/tj-cli/src/tui/chat_view.rs index 5dfb0a3..5846c42 100644 --- a/crates/tj-cli/src/tui/chat_view.rs +++ b/crates/tj-cli/src/tui/chat_view.rs @@ -77,7 +77,10 @@ impl ChatView { let title = if let Some(first) = session.first_user_text() { let clean = strip_xml_tags(&first); - let line = clean.lines().find(|l| !l.trim().is_empty()).unwrap_or(&clean); + let line = clean + .lines() + .find(|l| !l.trim().is_empty()) + .unwrap_or(&clean); truncate(line.trim(), 60) } else { format!("Session {}", &session.session_id[..8]) @@ -112,7 +115,7 @@ impl ChatView { let chunks = Layout::default() .direction(Direction::Vertical) .constraints([ - Constraint::Length(3), // header + Constraint::Length(3), // header Constraint::Min(5), // chat Constraint::Length(3), // footer ]) @@ -124,14 +127,18 @@ impl ChatView { } fn render_header(&self, frame: &mut Frame<'_>, area: Rect) { - let title = format!( - " {} — {} messages", - self.title, - self.messages.len() - ); + let title = format!(" {} — {} messages", self.title, self.messages.len()); let block = Paragraph::new(title) - .style(Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD)) - .block(Block::default().borders(Borders::BOTTOM).border_style(Style::default().fg(Color::DarkGray))); + .style( + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + ) + .block( + Block::default() + .borders(Borders::BOTTOM) + .border_style(Style::default().fg(Color::DarkGray)), + ); frame.render_widget(block, area); } @@ -151,19 +158,23 @@ impl ChatView { format!("─── {role_label} "), Style::default().fg(role_color).add_modifier(Modifier::BOLD), ), + Span::styled(&msg.timestamp, Style::default().fg(Color::DarkGray)), Span::styled( - &msg.timestamp, - Style::default().fg(Color::DarkGray), - ), - Span::styled( - format!(" {}", "─".repeat(width.saturating_sub(role_label.len() + msg.timestamp.len() + 6))), + format!( + " {}", + "─".repeat( + width.saturating_sub(role_label.len() + msg.timestamp.len() + 6) + ) + ), Style::default().fg(Color::DarkGray), ), ])); // Tool badges. if !msg.tools.is_empty() { - let tool_text = msg.tools.iter() + let tool_text = msg + .tools + .iter() .take(5) // max 5 tools shown .map(|t| format!("[{t}]")) .collect::>() @@ -223,8 +234,11 @@ impl ChatView { Span::styled("Backspace/Esc/q", Style::default().fg(Color::Yellow)), Span::raw(" back"), ]); - let block = Paragraph::new(help) - .block(Block::default().borders(Borders::TOP).border_style(Style::default().fg(Color::DarkGray))); + let block = Paragraph::new(help).block( + Block::default() + .borders(Borders::TOP) + .border_style(Style::default().fg(Color::DarkGray)), + ); frame.render_widget(block, area); } } diff --git a/crates/tj-cli/src/tui/mod.rs b/crates/tj-cli/src/tui/mod.rs index 0237524..3dc0690 100644 --- a/crates/tj-cli/src/tui/mod.rs +++ b/crates/tj-cli/src/tui/mod.rs @@ -1,5 +1,5 @@ //! Interactive TUI for browsing Claude Code sessions and task-journal data. pub mod app; -pub mod session_list; pub mod chat_view; +pub mod session_list; diff --git a/crates/tj-cli/src/tui/session_list.rs b/crates/tj-cli/src/tui/session_list.rs index eef765a..2d921a4 100644 --- a/crates/tj-cli/src/tui/session_list.rs +++ b/crates/tj-cli/src/tui/session_list.rs @@ -112,7 +112,8 @@ impl SessionList { /// Returns the actual session index for the current selection (maps through filter). pub fn selected_session_index(&self) -> Option { - self.selected.and_then(|i| self.filtered_indices.get(i).copied()) + self.selected + .and_then(|i| self.filtered_indices.get(i).copied()) } pub fn next(&mut self) { @@ -165,7 +166,7 @@ impl SessionList { let chunks = Layout::default() .direction(Direction::Vertical) .constraints([ - Constraint::Length(3), // search bar + Constraint::Length(3), // search bar Constraint::Min(5), // list Constraint::Length(3), // footer/help ]) @@ -178,7 +179,7 @@ impl SessionList { let chunks = Layout::default() .direction(Direction::Vertical) .constraints([ - Constraint::Length(3), // header + Constraint::Length(3), // header Constraint::Min(5), // list Constraint::Length(3), // footer/help ]) @@ -206,21 +207,20 @@ impl SessionList { let header = Line::from(vec![ Span::styled( " Task Journal ", - Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD), + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), ), Span::styled("— ", Style::default().fg(Color::DarkGray)), - Span::styled( - short_path, - Style::default().fg(Color::White), - ), + Span::styled(short_path, Style::default().fg(Color::White)), Span::styled(" — ", Style::default().fg(Color::DarkGray)), - Span::styled( - showing, - Style::default().fg(Color::Cyan), - ), + Span::styled(showing, Style::default().fg(Color::Cyan)), ]); - let block = Paragraph::new(header) - .block(Block::default().borders(Borders::BOTTOM).border_style(Style::default().fg(Color::DarkGray))); + let block = Paragraph::new(header).block( + Block::default() + .borders(Borders::BOTTOM) + .border_style(Style::default().fg(Color::DarkGray)), + ); frame.render_widget(block, area); } @@ -228,23 +228,29 @@ impl SessionList { let match_count = format!( "{} match{}", self.filtered_indices.len(), - if self.filtered_indices.len() == 1 { "" } else { "es" } + if self.filtered_indices.len() == 1 { + "" + } else { + "es" + } ); let search_line = Line::from(vec![ - Span::styled(" / ", Style::default().fg(Color::Yellow).add_modifier(Modifier::BOLD)), Span::styled( - self.filter_text.clone(), - Style::default().fg(Color::White), + " / ", + Style::default() + .fg(Color::Yellow) + .add_modifier(Modifier::BOLD), ), + Span::styled(self.filter_text.clone(), Style::default().fg(Color::White)), Span::styled("█", Style::default().fg(Color::Yellow)), Span::raw(" "), - Span::styled( - match_count, - Style::default().fg(Color::DarkGray), - ), + Span::styled(match_count, Style::default().fg(Color::DarkGray)), ]); - let block = Paragraph::new(search_line) - .block(Block::default().borders(Borders::BOTTOM).border_style(Style::default().fg(Color::Yellow))); + let block = Paragraph::new(search_line).block( + Block::default() + .borders(Borders::BOTTOM) + .border_style(Style::default().fg(Color::Yellow)), + ); frame.render_widget(block, area); } @@ -265,18 +271,9 @@ impl SessionList { let id_short = &s.session_id[..8.min(s.session_id.len())]; let line = Line::from(vec![ - Span::styled( - format!("{date} "), - Style::default().fg(Color::DarkGray), - ), - Span::styled( - format!("{id_short} "), - Style::default().fg(Color::Yellow), - ), - Span::styled( - format!("{msgs:>8} "), - Style::default().fg(Color::Green), - ), + Span::styled(format!("{date} "), Style::default().fg(Color::DarkGray)), + Span::styled(format!("{id_short} "), Style::default().fg(Color::Yellow)), + Span::styled(format!("{msgs:>8} "), Style::default().fg(Color::Green)), Span::styled( format!("{duration:>6} "), Style::default().fg(Color::DarkGray), @@ -323,12 +320,18 @@ impl SessionList { Span::raw(" quit"), ]; if !self.filter_text.is_empty() { - spans.push(Span::styled(" [filtered]", Style::default().fg(Color::DarkGray))); + spans.push(Span::styled( + " [filtered]", + Style::default().fg(Color::DarkGray), + )); } Line::from(spans) }; - let block = Paragraph::new(help) - .block(Block::default().borders(Borders::TOP).border_style(Style::default().fg(Color::DarkGray))); + let block = Paragraph::new(help).block( + Block::default() + .borders(Borders::TOP) + .border_style(Style::default().fg(Color::DarkGray)), + ); frame.render_widget(block, area); } } @@ -336,7 +339,10 @@ impl SessionList { fn session_title(s: &ParsedSession) -> String { if let Some(text) = s.first_user_text() { let clean = strip_xml_tags(&text); - let line = clean.lines().find(|l| !l.trim().is_empty()).unwrap_or(&clean); + let line = clean + .lines() + .find(|l| !l.trim().is_empty()) + .unwrap_or(&clean); let trimmed = line.trim(); if trimmed.len() > 80 { format!("{}…", &trimmed[..80]) diff --git a/crates/tj-core/src/session/discovery.rs b/crates/tj-core/src/session/discovery.rs index e1b42ba..5d6e2c5 100644 --- a/crates/tj-core/src/session/discovery.rs +++ b/crates/tj-core/src/session/discovery.rs @@ -94,7 +94,7 @@ pub fn list_sessions(project_dir: &Path) -> anyhow::Result> { } // Sort newest first. - sessions.sort_by(|a, b| b.1.cmp(&a.1)); + sessions.sort_by_key(|s| std::cmp::Reverse(s.1)); Ok(sessions.into_iter().map(|(p, _)| p).collect()) } @@ -198,7 +198,11 @@ mod tests { let sessions = list_sessions(dir.path()).unwrap(); assert_eq!(sessions.len(), 2); // Newest file should come first. - let first_name = sessions[0].file_name().unwrap().to_string_lossy().to_string(); + let first_name = sessions[0] + .file_name() + .unwrap() + .to_string_lossy() + .to_string(); assert_eq!(first_name, "newer.jsonl"); } diff --git a/crates/tj-core/src/session/extractor.rs b/crates/tj-core/src/session/extractor.rs index 6b65c55..a43a659 100644 --- a/crates/tj-core/src/session/extractor.rs +++ b/crates/tj-core/src/session/extractor.rs @@ -49,7 +49,8 @@ pub fn extract_from_session(session: &ParsedSession) -> Option { if let Some(ref ts) = session.first_timestamp { open_event.timestamp = ts.clone(); } - open_event.meta = serde_json::json!({"title": title, "backfill": true, "session_id": session.session_id}); + open_event.meta = + serde_json::json!({"title": title, "backfill": true, "session_id": session.session_id}); events.push(open_event); // 2. Walk through entries and extract meaningful events. @@ -132,7 +133,13 @@ pub fn extract_from_session(session: &ParsedSession) -> Option { files_modified.len(), files_modified.join(", ") ); - let mut ev = Event::new(&task_id, EventType::Finding, Author::Agent, Source::Cli, summary); + let mut ev = Event::new( + &task_id, + EventType::Finding, + Author::Agent, + Source::Cli, + summary, + ); if let Some(ref ts) = session.last_timestamp { ev.timestamp = ts.clone(); } @@ -185,7 +192,10 @@ fn derive_title(session: &ParsedSession) -> String { if let SessionEntry::User(u) = entry { if let Some(text) = extract_user_text(u) { let clean = strip_xml_tags(&text); - let first_line = clean.lines().find(|l| !l.trim().is_empty()).unwrap_or(&clean); + let first_line = clean + .lines() + .find(|l| !l.trim().is_empty()) + .unwrap_or(&clean); let trimmed = first_line.trim(); // Skip empty or very short titles (likely slash commands). if trimmed.len() > 5 { @@ -195,7 +205,10 @@ fn derive_title(session: &ParsedSession) -> String { } } - format!("Session {}", &session.session_id[..8.min(session.session_id.len())]) + format!( + "Session {}", + &session.session_id[..8.min(session.session_id.len())] + ) } /// Strip XML/HTML-like tags from text (e.g. , ). @@ -363,7 +376,10 @@ mod tests { #[test] fn test_shorten_path() { - assert_eq!(shorten_path("/home/user/project/src/main.rs"), "src/main.rs"); + assert_eq!( + shorten_path("/home/user/project/src/main.rs"), + "src/main.rs" + ); assert_eq!(shorten_path("main.rs"), "main.rs"); } @@ -435,13 +451,21 @@ mod tests { file_path: "/tmp/test-session-123.jsonl".into(), entries: vec![ make_user_entry("u1", "2026-01-01T00:00:00Z", "Please fix the login bug"), - make_assistant_entry("a1", "2026-01-01T00:00:01Z", vec![ - ContentBlock::Text { text: "I'll look into the login issue.".into() }, - ]), + make_assistant_entry( + "a1", + "2026-01-01T00:00:01Z", + vec![ContentBlock::Text { + text: "I'll look into the login issue.".into(), + }], + ), make_user_entry("u2", "2026-01-01T00:00:02Z", "Thanks, looks good"), - make_assistant_entry("a2", "2026-01-01T00:00:03Z", vec![ - ContentBlock::Text { text: "The fix is complete.".into() }, - ]), + make_assistant_entry( + "a2", + "2026-01-01T00:00:03Z", + vec![ContentBlock::Text { + text: "The fix is complete.".into(), + }], + ), ], first_timestamp: Some("2026-01-01T00:00:00Z".into()), last_timestamp: Some("2026-01-01T00:00:03Z".into()), @@ -470,9 +494,11 @@ mod tests { file_path: "/tmp/short.jsonl".into(), entries: vec![ make_user_entry("u1", "2026-01-01T00:00:00Z", "Hello"), - make_assistant_entry("a1", "2026-01-01T00:00:01Z", vec![ - ContentBlock::Text { text: "Hi!".into() }, - ]), + make_assistant_entry( + "a1", + "2026-01-01T00:00:01Z", + vec![ContentBlock::Text { text: "Hi!".into() }], + ), ], first_timestamp: Some("2026-01-01T00:00:00Z".into()), last_timestamp: Some("2026-01-01T00:00:01Z".into()), @@ -501,19 +527,23 @@ mod tests { file_path: "/tmp/fm.jsonl".into(), entries: vec![ make_user_entry("u1", "2026-01-01T00:00:00Z", "Update the config file"), - make_assistant_entry("a1", "2026-01-01T00:00:01Z", vec![ - ContentBlock::ToolUse { + make_assistant_entry( + "a1", + "2026-01-01T00:00:01Z", + vec![ContentBlock::ToolUse { name: "Write".into(), input: serde_json::json!({"file_path": "/home/user/project/src/config.rs"}), - }, - ]), + }], + ), make_user_entry("u2", "2026-01-01T00:00:02Z", "Also update main.rs"), - make_assistant_entry("a2", "2026-01-01T00:00:03Z", vec![ - ContentBlock::ToolUse { + make_assistant_entry( + "a2", + "2026-01-01T00:00:03Z", + vec![ContentBlock::ToolUse { name: "Edit".into(), input: serde_json::json!({"file_path": "/home/user/project/src/main.rs", "old_string": "a", "new_string": "b"}), - }, - ]), + }], + ), ], first_timestamp: Some("2026-01-01T00:00:00Z".into()), last_timestamp: Some("2026-01-01T00:00:03Z".into()), @@ -521,7 +551,10 @@ mod tests { let task = extract_from_session(&session).unwrap(); // Should have a Finding event with file modifications. - let finding = task.events.iter().find(|e| e.event_type == EventType::Finding); + let finding = task + .events + .iter() + .find(|e| e.event_type == EventType::Finding); assert!(finding.is_some()); let finding = finding.unwrap(); assert!(finding.text.contains("2 files")); @@ -536,12 +569,14 @@ mod tests { file_path: "/tmp/tc.jsonl".into(), entries: vec![ make_user_entry("u1", "2026-01-01T00:00:00Z", "Run the tests"), - make_assistant_entry("a1", "2026-01-01T00:00:01Z", vec![ - ContentBlock::ToolUse { + make_assistant_entry( + "a1", + "2026-01-01T00:00:01Z", + vec![ContentBlock::ToolUse { name: "Bash".into(), input: serde_json::json!({"command": "cargo test --workspace"}), - }, - ]), + }], + ), make_user_entry("u2", "2026-01-01T00:00:02Z", "Good"), ], first_timestamp: Some("2026-01-01T00:00:00Z".into()), @@ -549,7 +584,10 @@ mod tests { }; let task = extract_from_session(&session).unwrap(); - let evidence = task.events.iter().find(|e| e.event_type == EventType::Evidence); + let evidence = task + .events + .iter() + .find(|e| e.event_type == EventType::Evidence); assert!(evidence.is_some()); assert!(evidence.unwrap().text.contains("cargo test")); } @@ -561,12 +599,14 @@ mod tests { file_path: "/tmp/gc.jsonl".into(), entries: vec![ make_user_entry("u1", "2026-01-01T00:00:00Z", "Commit the changes"), - make_assistant_entry("a1", "2026-01-01T00:00:01Z", vec![ - ContentBlock::ToolUse { + make_assistant_entry( + "a1", + "2026-01-01T00:00:01Z", + vec![ContentBlock::ToolUse { name: "Bash".into(), input: serde_json::json!({"command": "git commit -m 'fix: resolve login bug'"}), - }, - ]), + }], + ), make_user_entry("u2", "2026-01-01T00:00:02Z", "Push it"), ], first_timestamp: Some("2026-01-01T00:00:00Z".into()), @@ -574,12 +614,19 @@ mod tests { }; let task = extract_from_session(&session).unwrap(); - let evidence_events: Vec<_> = task.events.iter() + let evidence_events: Vec<_> = task + .events + .iter() .filter(|e| e.event_type == EventType::Evidence) .collect(); - let commit_ev = evidence_events.iter().find(|e| e.text.contains("Git commit")); + let commit_ev = evidence_events + .iter() + .find(|e| e.text.contains("Git commit")); assert!(commit_ev.is_some()); - assert_eq!(commit_ev.unwrap().evidence_strength, Some(EvidenceStrength::Strong)); + assert_eq!( + commit_ev.unwrap().evidence_strength, + Some(EvidenceStrength::Strong) + ); } // --- strip_xml_tags() --- @@ -606,7 +653,10 @@ mod tests { #[test] fn strip_xml_tags_with_attributes() { - assert_eq!(strip_xml_tags("init"), "init"); + assert_eq!( + strip_xml_tags("init"), + "init" + ); } #[test] @@ -632,7 +682,10 @@ mod tests { first_timestamp: None, last_timestamp: None, }; - assert_eq!(derive_title(&session), "Fixed authentication bug in login flow"); + assert_eq!( + derive_title(&session), + "Fixed authentication bug in login flow" + ); } #[test] @@ -640,13 +693,18 @@ mod tests { let session = ParsedSession { session_id: "abcdefghij".into(), file_path: "/tmp/s.jsonl".into(), - entries: vec![ - make_user_entry("u1", "t", "Please implement the new caching layer"), - ], + entries: vec![make_user_entry( + "u1", + "t", + "Please implement the new caching layer", + )], first_timestamp: None, last_timestamp: None, }; - assert_eq!(derive_title(&session), "Please implement the new caching layer"); + assert_eq!( + derive_title(&session), + "Please implement the new caching layer" + ); } #[test] @@ -671,9 +729,7 @@ mod tests { let session = ParsedSession { session_id: "abcdefghij".into(), file_path: "/tmp/s.jsonl".into(), - entries: vec![ - make_user_entry("u1", "t", "hi"), - ], + entries: vec![make_user_entry("u1", "t", "hi")], first_timestamp: None, last_timestamp: None, }; @@ -687,12 +743,10 @@ mod tests { let session = ParsedSession { session_id: "abcdefghij".into(), file_path: "/tmp/s.jsonl".into(), - entries: vec![ - SessionEntry::Summary(SummaryEntry { - summary: "Fix the critical bug".into(), - timestamp: None, - }), - ], + entries: vec![SessionEntry::Summary(SummaryEntry { + summary: "Fix the critical bug".into(), + timestamp: None, + })], first_timestamp: None, last_timestamp: None, }; @@ -732,7 +786,7 @@ mod tests { assert!(is_test_command("go test ./...")); assert!(is_test_command("make test")); assert!(is_test_command("phpunit tests/Unit")); - assert!(is_test_command("echo 'cargo test'")); // matches because it contains "cargo test" + assert!(is_test_command("echo 'cargo test'")); // matches because it contains "cargo test" assert!(!is_test_command("ls -la")); } @@ -740,7 +794,10 @@ mod tests { #[test] fn test_shorten_path_windows_separators() { - assert_eq!(shorten_path("C:\\Users\\user\\project\\src\\main.rs"), "src/main.rs"); + assert_eq!( + shorten_path("C:\\Users\\user\\project\\src\\main.rs"), + "src/main.rs" + ); } #[test] diff --git a/crates/tj-core/src/session/parser.rs b/crates/tj-core/src/session/parser.rs index 6184ce0..b45ea7c 100644 --- a/crates/tj-core/src/session/parser.rs +++ b/crates/tj-core/src/session/parser.rs @@ -295,7 +295,7 @@ mod tests { fn parse_session_with_valid_jsonl() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("abc123.jsonl"); - let lines = vec![ + let lines = [ r#"{"type":"user","uuid":"u1","timestamp":"2026-01-01T00:00:00Z","message":{"content":"hello"}}"#, r#"{"type":"assistant","uuid":"a1","timestamp":"2026-01-01T00:00:01Z","message":{"content":[{"type":"text","text":"hi there"}]}}"#, r#"{"type":"summary","summary":"This session was about greeting.","timestamp":"2026-01-01T00:00:02Z"}"#, @@ -305,15 +305,21 @@ mod tests { let session = parse_session(&path).unwrap(); assert_eq!(session.session_id, "abc123"); assert_eq!(session.entries.len(), 3); - assert_eq!(session.first_timestamp.as_deref(), Some("2026-01-01T00:00:00Z")); - assert_eq!(session.last_timestamp.as_deref(), Some("2026-01-01T00:00:02Z")); + assert_eq!( + session.first_timestamp.as_deref(), + Some("2026-01-01T00:00:00Z") + ); + assert_eq!( + session.last_timestamp.as_deref(), + Some("2026-01-01T00:00:02Z") + ); } #[test] fn parse_session_skips_empty_and_malformed_lines() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("sess.jsonl"); - let lines = vec![ + let lines = [ "", "not-json-at-all", r#"{"type":"user","uuid":"u1","timestamp":"2026-01-01T00:00:00Z","message":{"content":"valid"}}"#, @@ -362,18 +368,18 @@ mod tests { let session = ParsedSession { session_id: "s1".into(), file_path: "/tmp/s1.jsonl".into(), - entries: vec![ - SessionEntry::Assistant(AssistantEntry { - uuid: "a1".into(), - timestamp: "2026-01-01T00:00:00Z".into(), - session_id: None, - message: Some(AssistantMessage { - content: vec![ContentBlock::Text { text: "hello".into() }], - model: None, - stop_reason: None, - }), + entries: vec![SessionEntry::Assistant(AssistantEntry { + uuid: "a1".into(), + timestamp: "2026-01-01T00:00:00Z".into(), + session_id: None, + message: Some(AssistantMessage { + content: vec![ContentBlock::Text { + text: "hello".into(), + }], + model: None, + stop_reason: None, }), - ], + })], first_timestamp: None, last_timestamp: None, }; @@ -417,17 +423,15 @@ mod tests { let session = ParsedSession { session_id: "s1".into(), file_path: "/tmp/s1.jsonl".into(), - entries: vec![ - SessionEntry::User(UserEntry { - uuid: "u1".into(), - timestamp: "2026-01-01T00:00:00Z".into(), - session_id: None, - message: Some(UserMessage { - content: serde_json::json!("init Setup project"), - }), - cwd: None, + entries: vec![SessionEntry::User(UserEntry { + uuid: "u1".into(), + timestamp: "2026-01-01T00:00:00Z".into(), + session_id: None, + message: Some(UserMessage { + content: serde_json::json!("init Setup project"), }), - ], + cwd: None, + })], first_timestamp: None, last_timestamp: None, }; @@ -441,15 +445,13 @@ mod tests { let session = ParsedSession { session_id: "s1".into(), file_path: "/tmp/s1.jsonl".into(), - entries: vec![ - SessionEntry::User(UserEntry { - uuid: "u1".into(), - timestamp: "2026-01-01T00:00:00Z".into(), - session_id: None, - message: None, - cwd: None, - }), - ], + entries: vec![SessionEntry::User(UserEntry { + uuid: "u1".into(), + timestamp: "2026-01-01T00:00:00Z".into(), + session_id: None, + message: None, + cwd: None, + })], first_timestamp: None, last_timestamp: None, }; @@ -463,15 +465,13 @@ mod tests { let session = ParsedSession { session_id: "s1".into(), file_path: "/tmp/s1.jsonl".into(), - entries: vec![ - SessionEntry::User(UserEntry { - uuid: "u1".into(), - timestamp: "2026-01-01T00:00:00Z".into(), - session_id: None, - message: None, - cwd: None, - }), - ], + entries: vec![SessionEntry::User(UserEntry { + uuid: "u1".into(), + timestamp: "2026-01-01T00:00:00Z".into(), + session_id: None, + message: None, + cwd: None, + })], first_timestamp: None, last_timestamp: None, }; @@ -634,11 +634,22 @@ mod tests { session_id: None, message: Some(AssistantMessage { content: vec![ - ContentBlock::Thinking { thinking: Some("internal thought".into()) }, - ContentBlock::Text { text: "visible text".into() }, - ContentBlock::ToolResult { content: serde_json::json!("result data") }, - ContentBlock::ToolUse { name: "Read".into(), input: serde_json::json!({}) }, - ContentBlock::Text { text: "more text".into() }, + ContentBlock::Thinking { + thinking: Some("internal thought".into()), + }, + ContentBlock::Text { + text: "visible text".into(), + }, + ContentBlock::ToolResult { + content: serde_json::json!("result data"), + }, + ContentBlock::ToolUse { + name: "Read".into(), + input: serde_json::json!({}), + }, + ContentBlock::Text { + text: "more text".into(), + }, ], model: None, stop_reason: None, @@ -684,11 +695,21 @@ mod tests { session_id: None, message: Some(AssistantMessage { content: vec![ - ContentBlock::Text { text: "Let me help".into() }, - ContentBlock::ToolUse { name: "Write".into(), input: serde_json::json!({"file_path": "/tmp/a"}) }, + ContentBlock::Text { + text: "Let me help".into(), + }, + ContentBlock::ToolUse { + name: "Write".into(), + input: serde_json::json!({"file_path": "/tmp/a"}), + }, ContentBlock::Thinking { thinking: None }, - ContentBlock::ToolUse { name: "Bash".into(), input: serde_json::json!({"command": "ls"}) }, - ContentBlock::ToolResult { content: serde_json::json!(null) }, + ContentBlock::ToolUse { + name: "Bash".into(), + input: serde_json::json!({"command": "ls"}), + }, + ContentBlock::ToolResult { + content: serde_json::json!(null), + }, ], model: None, stop_reason: None, @@ -707,12 +728,10 @@ mod tests { timestamp: "t".into(), session_id: None, message: Some(AssistantMessage { - content: vec![ - ContentBlock::ToolUse { - name: "Edit".into(), - input: serde_json::json!({"file_path": "/src/main.rs", "old_string": "foo", "new_string": "bar"}), - }, - ], + content: vec![ContentBlock::ToolUse { + name: "Edit".into(), + input: serde_json::json!({"file_path": "/src/main.rs", "old_string": "foo", "new_string": "bar"}), + }], model: None, stop_reason: None, }), From 843a689123ac895895e15398244874c602e6c4c6 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:43:09 +0400 Subject: [PATCH 02/39] =?UTF-8?q?docs(plan):=20epic=20A=20=E2=80=94=20v0.1?= =?UTF-8?q?.4=20hardening=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents the 11-task hardening epic landing as 0.1.4: HTTP timeout, graceful JSONL skip, env-var classifier model, longer task_id, drop stub field, SCHEMA_VERSION const, CHANGELOG.md, cargo-audit CI, MSRV job, .editorconfig, JSONL file-lock. Backwards-compatible only. Breaking and perf changes deferred to epic B (v0.2.0). Quality/DX deferred to epic C. Refs claude-memory-iyn --- .docs/plans/2026-05-06-v0.1.4-hardening.md | 89 ++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .docs/plans/2026-05-06-v0.1.4-hardening.md diff --git a/.docs/plans/2026-05-06-v0.1.4-hardening.md b/.docs/plans/2026-05-06-v0.1.4-hardening.md new file mode 100644 index 0000000..db6139d --- /dev/null +++ b/.docs/plans/2026-05-06-v0.1.4-hardening.md @@ -0,0 +1,89 @@ +# Epic A — v0.1.4 hardening + +**Date:** 2026-05-06 +**Branch:** `claude/youthful-shaw-b96d78` +**Target release:** `0.1.4` (backwards-compatible patch) +**Bd epic:** see `bd list --type epic` (assigned id at runtime) + +## Goal + +Ship a backwards-compatible patch that closes the most acute correctness, robustness, and OSS-hygiene gaps identified in the 2026-05-06 audit, **without breaking the public CLI/MCP contract**. Anything that requires a breaking change is deferred to Epic B (v0.2.0). + +## Success criteria + +1. `cargo test --workspace --all-targets` green on `ubuntu-latest`, `macos-latest`, `windows-latest`. +2. `cargo audit` runs in CI and is clean (or vulnerabilities are accepted with documented reason). +3. `cargo clippy --workspace --all-targets -- -D warnings` clean. +4. `cargo doc --workspace --no-deps` clean with `RUSTDOCFLAGS=-D warnings`. +5. New CI job pins MSRV (currently `1.83`) and verifies build. +6. `CHANGELOG.md` exists and documents `0.1.4` entry. +7. No removed/renamed CLI flags. No removed MCP tools or required parameters. +8. Branch `claude/youthful-shaw-b96d78` pushed; PR opened against `main`. + +## Out of scope (deferred) + +- Incremental indexing / pack-cache fix (Epic B — perf) +- MCP error contract redesign (Epic B — breaking) +- `--project-dir` argument for MCP (Epic B) +- Migrations framework (Epic B — coupled with incremental indexing) +- Few-shot prompting + eval datasets (Epic C — quality) +- `task-journal doctor`, `migrate-project` (Epic C — DX) + +## Tasks (11) + +Each task is one atomic commit. Test-first when behavior changes; doc/CI-only tasks may skip the failing-test step. + +| # | Task | Touches | Test? | Notes | +|---|------|---------|-------|-------| +| A1 | HTTP timeout for `AnthropicClassifier` | `classifier/http.rs` | yes (mockito slow-server) | 15s connect+read timeout. Hardcoded — env-var override deferred. | +| A2 | Graceful skip of malformed JSONL lines in `rebuild_state` | `db.rs` | yes (jsonl with bad line) | Log a `tracing::warn!` and continue; total parsed count returned. | +| A3 | Classifier model overridable via env var | `classifier/http.rs`, `classifier/cli.rs` | yes (env unset → default; env set → override) | `TJ_CLASSIFIER_MODEL`; default unchanged. | +| A4 | Extend task_id from 6 → 10 characters | `crates/tj-mcp/src/main.rs`, `crates/tj-cli/src/main.rs` | yes (collision-free over 10k synthetic ids) | Old 6-char ids remain valid (string compare). | +| A5 | Remove `stub: bool` from MCP responses | `crates/tj-mcp/src/main.rs`, smoke tests | yes (smoke test asserts no `stub` field) | Field removal — but no client read it; documented in CHANGELOG. | +| A6 | Centralize `SCHEMA_VERSION` const | `tj-core/src/lib.rs`, `pack.rs`, `tj-mcp/src/main.rs` | yes (single source) | `pub const SCHEMA_VERSION: &str = "1.0";` | +| A7 | `CHANGELOG.md` with Keep-a-Changelog format | new file | n/a | Backfill `0.1.0`–`0.1.3` from `git log`. | +| A8 | `cargo-audit` job in CI | `.github/workflows/ci.yml` | n/a | Non-blocking initially; flips to blocking once green. | +| A9 | MSRV job in CI (`rust-version` = 1.83) | `.github/workflows/ci.yml` | n/a | Uses `dtolnay/rust-toolchain@1.83`. | +| A10 | `.editorconfig` | new file | n/a | LF, UTF-8, 4-space rust, 2-space yaml. | +| A11 | File-lock on JSONL append | `tj-core/src/storage.rs`, `Cargo.toml` | yes (two-writer race test) | Crate: `fd-lock` (cross-platform). Blocking lock. | + +## Sequencing + +``` +A6 ──┐ +A1 ──┼─→ A7 (CHANGELOG references all done work) +A2 ──┤ +A3 ──┤ +A4 ──┤ +A5 ──┤ +A10 ─┤ +A8 ──┤ +A9 ──┘ + A11 last (fd-lock dep + race test) +``` + +A11 last because it adds a runtime dependency and a flaky-prone test; everything else lands first so green CI is the baseline before introducing the lock. + +## Risks + +- **A4 task_id length change:** new ids longer; nothing reads fixed-width. Verified by smart_read of CLI/MCP code paths. +- **A5 `stub` removal:** technically a schema change, but `stub` was always false post-Phase-1. Documented as non-breaking in CHANGELOG; if any downstream tool actually reads it, we revert in 0.1.5. +- **A11 fd-lock on Windows:** `fd-lock` uses `LockFileEx` on Windows; behavior differs from Linux `flock`. Test must cover both. +- **A2 swallowing real corruption:** mitigation — log at `warn!` level with line number and parse error. + +## Verification (per task) + +1. `cargo fmt --all --check` +2. `cargo clippy --workspace --all-targets -- -D warnings` +3. `cargo test --workspace --all-targets` (specific test for the touched module) +4. `git diff --stat` reviewed (no unintended line-ending or whitespace flips) +5. Commit with conventional-commit prefix (`fix:`, `chore:`, `docs:`, `ci:`, `feat:`) +6. `bd update --status closed --reason ""` + +## Final verification (epic-level) + +- `cargo test --workspace --all-targets` green +- `cargo audit` clean +- `bd list --parent --status open` returns empty +- `git log --oneline 8c49785..HEAD` matches the 11 tasks 1:1 +- `gh pr create` opened against `main` with the CHANGELOG entry as body From 2e6cae822af703daa9836616f5d8229d30c6de13 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:44:01 +0400 Subject: [PATCH 03/39] fix(db): rebuild_state skips malformed JSONL lines instead of aborting A single bad line in the events JSONL would abort the whole rebuild transaction, leaving SQLite empty and re-aborting on every retry. For an append-only journal this is too brittle. Now: malformed lines are logged via tracing::warn and skipped; SQL errors still propagate (those indicate schema/integrity problems). Returned count reflects only successfully-indexed events. Adds tracing dep to tj-core (workspace dep already declared). New test rebuild_state_skips_malformed_jsonl_lines covers both non-JSON garbage and valid-JSON-but-not-an-Event cases. Closes claude-memory-iyn.2 --- Cargo.lock | 1 + crates/tj-core/Cargo.toml | 1 + crates/tj-core/src/db.rs | 65 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e12b94..97ee49b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2016,6 +2016,7 @@ dependencies = [ "sha2", "tempfile", "thiserror 1.0.69", + "tracing", "ulid", "ureq", ] diff --git a/crates/tj-core/Cargo.toml b/crates/tj-core/Cargo.toml index 49613a2..b95ca35 100644 --- a/crates/tj-core/Cargo.toml +++ b/crates/tj-core/Cargo.toml @@ -28,6 +28,7 @@ dunce = { workspace = true } directories = { workspace = true } rusqlite = { workspace = true } ureq = { workspace = true } +tracing = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index 4778c89..ad919a6 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -137,8 +137,20 @@ pub fn rebuild_state( if line.trim().is_empty() { continue; } - let event: Event = - serde_json::from_str(&line).with_context(|| format!("parse line {i}"))?; + // Malformed JSONL lines are skipped with a warning so that one bad + // event cannot abort an otherwise-recoverable rebuild. SQL errors + // still propagate — those indicate schema/integrity problems. + let event: Event = match serde_json::from_str(&line) { + Ok(e) => e, + Err(err) => { + tracing::warn!( + line_number = i + 1, + error = %err, + "skipping malformed JSONL line in rebuild_state" + ); + continue; + } + }; upsert_task_from_event(&tx, &event, project_hash)?; index_event(&tx, &event)?; count += 1; @@ -438,6 +450,55 @@ mod tests { assert_eq!(hashes, vec!["aaaa1111aaaa1111", "bbbb2222bbbb2222"]); } + #[test] + fn rebuild_state_skips_malformed_jsonl_lines() { + use std::io::Write; + let d = TempDir::new().unwrap(); + let events_path = d.path().join("events.jsonl"); + let db_path = d.path().join("s.sqlite"); + + let mut f = std::fs::File::create(&events_path).unwrap(); + + let mut e1 = crate::event::Event::new( + "tj-skip", + crate::event::EventType::Open, + crate::event::Author::User, + crate::event::Source::Cli, + "x".into(), + ); + e1.meta = serde_json::json!({"title": "Skip test"}); + writeln!(f, "{}", serde_json::to_string(&e1).unwrap()).unwrap(); + + // Garbage that is not even JSON. + writeln!(f, "this is not a json event line").unwrap(); + + // Valid JSON but not a valid Event (missing required fields). + writeln!(f, "{{\"foo\": 1}}").unwrap(); + + let e3 = crate::event::Event::new( + "tj-skip", + crate::event::EventType::Decision, + crate::event::Author::Agent, + crate::event::Source::Chat, + "Adopt Rust".into(), + ); + writeln!(f, "{}", serde_json::to_string(&e3).unwrap()).unwrap(); + drop(f); + + let conn = open(&db_path).unwrap(); + let n = rebuild_state(&conn, &events_path, "deadbeefdeadbeef") + .expect("rebuild_state must succeed despite malformed lines"); + assert_eq!( + n, 2, + "expected 2 valid events indexed (2 malformed skipped)" + ); + + let indexed: i64 = conn + .query_row("SELECT COUNT(*) FROM events_index", [], |r| r.get(0)) + .unwrap(); + assert_eq!(indexed, 2); + } + #[test] fn rebuild_state_reads_jsonl_and_populates_db() { use std::io::Write; From b95279c8d2a9256817368e9d67eb5f01997a39eb Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:46:12 +0400 Subject: [PATCH 04/39] fix(http): add 15s timeout to AnthropicClassifier requests The HTTP classifier built the request without any timeout, so on a stalled network or rate-limit lockup the call would hang indefinitely. Hooks wrap classifier calls in || true, but that protects against exit codes, not against blocked turns. Adds AnthropicClassifier::timeout field (default 15s via DEFAULT_TIMEOUT const). Used in the ureq Request chain. Test classifier_times_out_on_unresponsive_server binds a TCP socket that completes the handshake but never replies; with timeout=300ms the call must Err in well under 3s. Closes claude-memory-iyn.1 --- crates/tj-core/src/classifier/http.rs | 47 +++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/crates/tj-core/src/classifier/http.rs b/crates/tj-core/src/classifier/http.rs index a470183..4aeec93 100644 --- a/crates/tj-core/src/classifier/http.rs +++ b/crates/tj-core/src/classifier/http.rs @@ -3,11 +3,18 @@ use super::*; use anyhow::{anyhow, Context}; use serde::{Deserialize, Serialize}; +use std::time::Duration; + +/// Default upper bound on a single classification round-trip. Hooks wrap calls +/// in `|| true` so a timeout never breaks Claude Code, but without a bound the +/// hook would still hang the chat turn. +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(15); pub struct AnthropicClassifier { pub api_key: String, pub model: String, pub base_url: String, // overridable for tests + pub timeout: Duration, } impl AnthropicClassifier { @@ -18,6 +25,7 @@ impl AnthropicClassifier { api_key, model: "claude-haiku-4-5-20251001".into(), base_url: "https://api.anthropic.com".into(), + timeout: DEFAULT_TIMEOUT, }) } } @@ -59,6 +67,7 @@ impl Classifier for AnthropicClassifier { let url = format!("{}/v1/messages", self.base_url); let resp: MessagesResponse = ureq::post(&url) + .timeout(self.timeout) .set("x-api-key", &self.api_key) .set("anthropic-version", "2023-06-01") .set("content-type", "application/json") @@ -118,6 +127,7 @@ mod tests { api_key: "test".into(), model: "claude-haiku-4-5-20251001".into(), base_url: url, + timeout: DEFAULT_TIMEOUT, }; let out = c .classify(&ClassifyInput { @@ -132,4 +142,41 @@ mod tests { assert!((out.confidence - 0.93).abs() < 1e-6); mock.assert(); } + + #[test] + fn classifier_times_out_on_unresponsive_server() { + use std::net::TcpListener; + use std::time::Instant; + + // Bind a TCP socket but never accept — the kernel completes the + // 3-way handshake from the backlog so connect() succeeds, but no + // bytes are ever read or written. Read timeout must fire. + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let url = format!("http://{addr}"); + + let c = AnthropicClassifier { + api_key: "test".into(), + model: "test-model".into(), + base_url: url, + timeout: Duration::from_millis(300), + }; + + let start = Instant::now(); + let res = c.classify(&ClassifyInput { + text: "x".into(), + author_hint: "user".into(), + recent_tasks: vec![], + }); + let elapsed = start.elapsed(); + + assert!(res.is_err(), "expected a timeout error, got Ok"); + assert!( + elapsed < Duration::from_secs(3), + "expected timeout near 300ms, got {elapsed:?}" + ); + + // Keep the listener alive until after the request to avoid races. + drop(listener); + } } From feb3724dcdfa8f022b1dfd29aecc6e469adb49a6 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:48:24 +0400 Subject: [PATCH 05/39] refactor(core): centralize SCHEMA_VERSION as single const MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The schema-version string "1.0" was inlined at four production sites (event.rs, pack.rs x2, tj-mcp main.rs). Bumping the version required four search-replaces — one of them being in another crate. Now: pub const SCHEMA_VERSION in tj-core::lib, referenced from all four sites. Test pack_assembler_does_not_inline_schema_version_literal guards against future regressions by scanning pack.rs source. Closes claude-memory-iyn.6 --- crates/tj-core/src/event.rs | 2 +- crates/tj-core/src/lib.rs | 32 ++++++++++++++++++++++++++++++++ crates/tj-core/src/pack.rs | 4 ++-- crates/tj-mcp/src/main.rs | 2 +- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/crates/tj-core/src/event.rs b/crates/tj-core/src/event.rs index c2e223c..d4a88be 100644 --- a/crates/tj-core/src/event.rs +++ b/crates/tj-core/src/event.rs @@ -114,7 +114,7 @@ impl Event { ) -> Self { Event { event_id: ulid::Ulid::new().to_string(), - schema_version: "1.0".to_string(), + schema_version: crate::SCHEMA_VERSION.to_string(), task_id: task_id.into(), event_type, timestamp: chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true), diff --git a/crates/tj-core/src/lib.rs b/crates/tj-core/src/lib.rs index 9ea91d7..28052a6 100644 --- a/crates/tj-core/src/lib.rs +++ b/crates/tj-core/src/lib.rs @@ -2,6 +2,11 @@ #![deny(rust_2018_idioms)] +/// On-disk + on-wire schema version for events and packs. Bump when a +/// breaking change is made to the JSONL event shape or the pack JSON +/// envelope. Single source of truth across the workspace — never inline. +pub const SCHEMA_VERSION: &str = "1.0"; + pub mod classifier; pub mod db; pub mod event; @@ -10,3 +15,30 @@ pub mod paths; pub mod project_hash; pub mod session; pub mod storage; + +#[cfg(test)] +mod schema_version_tests { + /// Source-level guard: production sites must reference `SCHEMA_VERSION` + /// rather than inlining a literal. If you bump the version, do it in + /// the const — never in a struct literal. + #[test] + fn pack_assembler_does_not_inline_schema_version_literal() { + let pack_src = include_str!("pack.rs"); + assert!( + !pack_src.contains("schema_version: \""), + "pack.rs has an inline schema_version string literal — use crate::SCHEMA_VERSION" + ); + } + + #[test] + fn schema_version_matches_event_default() { + let evt = crate::event::Event::new( + "tj-x", + crate::event::EventType::Open, + crate::event::Author::User, + crate::event::Source::Cli, + "x".into(), + ); + assert_eq!(evt.schema_version, super::SCHEMA_VERSION); + } +} diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index a1f7b00..dbf6242 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -176,7 +176,7 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res return Ok(TaskPack { task_id: task_id.to_string(), mode, - schema_version: "1.0".into(), + schema_version: crate::SCHEMA_VERSION.into(), text: cached_text, metadata: PackMetadata { generated_at: cached_at, @@ -243,7 +243,7 @@ pub fn assemble(conn: &Connection, task_id: &str, mode: PackMode) -> anyhow::Res Ok(TaskPack { task_id: task_id.to_string(), mode, - schema_version: "1.0".into(), + schema_version: crate::SCHEMA_VERSION.into(), text, metadata: PackMetadata { generated_at, diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index efa3f86..9b1ab3d 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -185,7 +185,7 @@ impl TaskJournalServer { Err(e) => Json(TaskPackResult { task_id: p.task_id, mode: p.mode.unwrap_or_else(|| "compact".into()), - schema_version: "1.0".into(), + schema_version: tj_core::SCHEMA_VERSION.into(), text: format!("[error] {e}"), metadata: TaskPackMetadata { stub: false, From 89ed91fe251d1aa3bc86e221a1a119728aaa48cf Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:50:00 +0400 Subject: [PATCH 06/39] fix(id): extend task_id from 6 to 10 characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six base32 chars from a ULID give ~24 bits of entropy, which is ~4096 tasks before a 50% collision risk under birthday paradox. For a long-lived project journal this is uncomfortably close. Now: tj_core::new_task_id() helper produces "tj-" + 10 chars (~50 bits, ~33M threshold). Used in tj-cli, tj-mcp, and the session backfill extractor — replaces three slightly-different inline copies. Old 6-char IDs continue to work since storage keys are opaque strings; this only affects newly-generated tasks. Tests: shape check + 10k uniqueness sweep. Closes claude-memory-iyn.4 --- crates/tj-cli/src/main.rs | 8 +---- crates/tj-core/src/lib.rs | 40 +++++++++++++++++++++++++ crates/tj-core/src/session/extractor.rs | 5 +--- crates/tj-mcp/src/main.rs | 5 +--- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 8154591..1a72054 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -165,13 +165,7 @@ fn main() -> Result<()> { let events_path = events_dir.join(format!("{project_hash}.jsonl")); std::fs::create_dir_all(&events_dir)?; - // ULID layout: chars 0-9 = timestamp (48b), 10-25 = random (80b). - // Taking from random portion to avoid same-prefix collisions for tasks - // created within ~12 days (which would happen with [..6]). - let task_id = format!( - "tj-{}", - &ulid::Ulid::new().to_string()[10..16].to_lowercase() - ); + let task_id = tj_core::new_task_id(); let mut event = tj_core::event::Event::new( task_id.clone(), tj_core::event::EventType::Open, diff --git a/crates/tj-core/src/lib.rs b/crates/tj-core/src/lib.rs index 28052a6..7e1eaf7 100644 --- a/crates/tj-core/src/lib.rs +++ b/crates/tj-core/src/lib.rs @@ -7,6 +7,46 @@ /// envelope. Single source of truth across the workspace — never inline. pub const SCHEMA_VERSION: &str = "1.0"; +/// Build a fresh task identifier of the form `tj-<10 lowercase base32>`. +/// +/// 50 bits of entropy from the ULID random suffix → birthday-collision +/// threshold ≈ 33 million tasks per project. The previous 6-char form +/// only gave ~4096; old IDs remain valid since storage keys are strings. +pub fn new_task_id() -> String { + format!( + "tj-{}", + &ulid::Ulid::new().to_string()[10..20].to_lowercase() + ) +} + +#[cfg(test)] +mod task_id_tests { + use super::new_task_id; + use std::collections::HashSet; + + #[test] + fn new_task_id_has_expected_shape() { + let id = new_task_id(); + assert!(id.starts_with("tj-"), "{id}"); + assert_eq!(id.len(), 13, "{id}"); + assert!( + id[3..] + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()), + "{id}" + ); + } + + #[test] + fn new_task_id_unique_over_ten_thousand() { + let mut seen = HashSet::with_capacity(10_000); + for _ in 0..10_000 { + let id = new_task_id(); + assert!(seen.insert(id.clone()), "collision: {id}"); + } + } +} + pub mod classifier; pub mod db; pub mod event; diff --git a/crates/tj-core/src/session/extractor.rs b/crates/tj-core/src/session/extractor.rs index a43a659..4544bdd 100644 --- a/crates/tj-core/src/session/extractor.rs +++ b/crates/tj-core/src/session/extractor.rs @@ -23,10 +23,7 @@ pub fn extract_from_session(session: &ParsedSession) -> Option { return None; } - let task_id = format!( - "tj-{}", - &ulid::Ulid::new().to_string()[10..16].to_lowercase() - ); + let task_id = crate::new_task_id(); // Derive title from first user message or summary. let title = derive_title(session); diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index 9b1ab3d..852ae9c 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -237,10 +237,7 @@ impl TaskJournalServer { let (_, events_path, _) = project_paths()?; std::fs::create_dir_all(events_path.parent().unwrap())?; - let task_id = format!( - "tj-{}", - &ulid::Ulid::new().to_string()[10..16].to_lowercase() - ); + let task_id = tj_core::new_task_id(); let mut event = tj_core::event::Event::new( task_id.clone(), tj_core::event::EventType::Open, From 20fafa456e5e30a0ec50ea1a73d3368a75f93c95 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:51:44 +0400 Subject: [PATCH 07/39] chore(mcp): remove vestigial stub:bool from all responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase-1 left every MCP result type with a stub:bool flag that was always false in production. The field was never read by any client and made every JSON payload look unfinished. Removed from TaskPackResult, TaskPackMetadata, TaskSearchResult, TaskCreateResult, EventAddResult, TaskCloseResult and their eight in-place initializations. Regression test no_response_serializes_a_ stub_field guards against re-introduction. Technically a JSON shape change, but stub was a write-only field with no documented consumers — clients reading these payloads will see one fewer key, never an unexpected one. Closes claude-memory-iyn.5 --- crates/tj-mcp/src/main.rs | 68 +++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index 852ae9c..1c0e35b 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -61,7 +61,6 @@ pub struct TaskPackResult { #[derive(Debug, Serialize, schemars::JsonSchema)] pub struct TaskPackMetadata { - pub stub: bool, pub source_event_count: Option, pub cache_hit: Option, } @@ -76,7 +75,6 @@ pub struct TaskSearchParams { pub struct TaskSearchResult { pub query: String, pub results: Vec, - pub stub: bool, } #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -88,7 +86,6 @@ pub struct TaskCreateParams { pub struct TaskCreateResult { pub task_id: String, pub title: String, - pub stub: bool, } #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -104,7 +101,6 @@ pub struct EventAddResult { pub event_id: String, pub task_id: String, pub event_type: String, - pub stub: bool, } #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -117,7 +113,6 @@ pub struct TaskCloseParams { pub struct TaskCloseResult { pub task_id: String, pub closed: bool, - pub stub: bool, } fn parse_event_type(s: &str) -> anyhow::Result { @@ -174,7 +169,6 @@ impl TaskJournalServer { schema_version: pack.schema_version, text: pack.text, metadata: TaskPackMetadata { - stub: false, source_event_count: Some(pack.metadata.source_event_count), cache_hit: Some(pack.metadata.cache_hit), }, @@ -188,7 +182,6 @@ impl TaskJournalServer { schema_version: tj_core::SCHEMA_VERSION.into(), text: format!("[error] {e}"), metadata: TaskPackMetadata { - stub: false, source_event_count: None, cache_hit: None, }, @@ -221,7 +214,6 @@ impl TaskJournalServer { Json(TaskSearchResult { query: p.query, results: result.unwrap_or_default(), - stub: false, }) } @@ -254,13 +246,11 @@ impl TaskJournalServer { Ok(TaskCreateResult { task_id, title: p.title.clone(), - stub: false, }) })(); Json(result.unwrap_or_else(|e| TaskCreateResult { task_id: format!("[error] {e}"), title: p.title, - stub: false, })) } @@ -292,14 +282,12 @@ impl TaskJournalServer { event_id: event.event_id, task_id: p.task_id.clone(), event_type: p.event_type.clone(), - stub: false, }) })(); Json(result.unwrap_or_else(|e| EventAddResult { event_id: format!("[error] {e}"), task_id: p.task_id, event_type: p.event_type, - stub: false, })) } @@ -335,7 +323,6 @@ impl TaskJournalServer { Json(TaskCloseResult { task_id: p.task_id, closed: result.is_ok(), - stub: false, }) } } @@ -369,3 +356,58 @@ async fn main() -> Result<()> { server.serve((stdin, stdout)).await?.waiting().await?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn keys_of(v: &serde_json::Value) -> Vec { + v.as_object() + .map(|o| o.keys().cloned().collect()) + .unwrap_or_default() + } + + #[test] + fn no_response_serializes_a_stub_field() { + // Vestigial stub:bool from Phase 1 stubs has been removed from all + // five MCP result types. Guard against re-introduction. + let pack = TaskPackResult { + task_id: "tj-x".into(), + mode: "compact".into(), + schema_version: tj_core::SCHEMA_VERSION.into(), + text: String::new(), + metadata: TaskPackMetadata { + source_event_count: None, + cache_hit: None, + }, + }; + let pack_v = serde_json::to_value(&pack).unwrap(); + assert!(!keys_of(&pack_v).contains(&"stub".to_string())); + assert!(!keys_of(&pack_v["metadata"]).contains(&"stub".to_string())); + + let search = TaskSearchResult { + query: "q".into(), + results: vec![], + }; + assert!(!keys_of(&serde_json::to_value(&search).unwrap()).contains(&"stub".to_string())); + + let create = TaskCreateResult { + task_id: "tj-x".into(), + title: "t".into(), + }; + assert!(!keys_of(&serde_json::to_value(&create).unwrap()).contains(&"stub".to_string())); + + let event = EventAddResult { + event_id: "e".into(), + task_id: "tj-x".into(), + event_type: "decision".into(), + }; + assert!(!keys_of(&serde_json::to_value(&event).unwrap()).contains(&"stub".to_string())); + + let close = TaskCloseResult { + task_id: "tj-x".into(), + closed: true, + }; + assert!(!keys_of(&serde_json::to_value(&close).unwrap()).contains(&"stub".to_string())); + } +} From 2eb87e1ddc86aabcfd2076085433d72bb31c2575 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:52:22 +0400 Subject: [PATCH 08/39] chore: add .editorconfig at repo root Standard OSS hygiene file. LF line endings, UTF-8, final newline, trim trailing whitespace, 4-space rust, 2-space yaml/toml/md/json, 2-space sh, tab Makefile. Cargo.lock and *.jsonl carve-outs. Closes claude-memory-iyn.10 --- .editorconfig | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d87b340 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,31 @@ +# Universal editor settings for Task Journal. +# Reference: https://editorconfig.org/ + +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space + +[*.rs] +indent_size = 4 +max_line_length = 100 + +[*.{toml,yml,yaml,md,json}] +indent_size = 2 + +[*.sh] +indent_size = 2 + +[Makefile] +indent_style = tab + +# Generated/external files — leave alone. +[Cargo.lock] +trim_trailing_whitespace = false + +[*.jsonl] +insert_final_newline = false From ec19e711cb084e513d78219fa1a768577d74d70c Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:52:58 +0400 Subject: [PATCH 09/39] ci: add MSRV job pinning rust 1.83 Cargo.toml declares rust-version = 1.83 but the existing CI only tested @stable, so an accidental new-feature use would slip in silently and break downstream consumers locked to MSRV. New msrv job: ubuntu-latest with dtolnay/rust-toolchain@1.83, cargo build + cargo test on the full workspace. Separate cache key so it does not collide with the stable job. Closes claude-memory-iyn.9 --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3fed1c..6e4534a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,3 +42,23 @@ jobs: run: cargo doc --workspace --no-deps env: RUSTDOCFLAGS: -D warnings + + msrv: + name: msrv (rust 1.83) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust 1.83 + uses: dtolnay/rust-toolchain@1.83 + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + with: + key: msrv-1.83 + + - name: cargo build (MSRV) + run: cargo build --workspace --all-targets + + - name: cargo test (MSRV) + run: cargo test --workspace --all-targets From 7e421d100e418586578135e84cbf0ac7d7c990c4 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:55:26 +0400 Subject: [PATCH 10/39] feat(classifier): TJ_CLASSIFIER_MODEL env var overrides hardcoded model Both the subscription (claude -p) and Anthropic API classifiers hardcoded their model alias. When Anthropic deprecates a model the classifier silently breaks until a release ships. Now: each classifier reads TJ_CLASSIFIER_MODEL with backend-specific default (haiku alias for CLI, claude-haiku-4-5-20251001 for API). DEFAULT_MODEL constants exposed for tests and external override. Test tj_classifier_model_env_var_overrides_defaults_for_both_backends combines both backends into one serialized read-set-restore flow to avoid env-var races between concurrent test threads. README documents the new env var. Closes claude-memory-iyn.3 --- README.md | 7 ++++ crates/tj-core/src/classifier/cli.rs | 9 +++-- crates/tj-core/src/classifier/http.rs | 6 +++- crates/tj-core/src/classifier/mod.rs | 47 +++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b9afc81..daffa7e 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,13 @@ The classifier (powered by `claude -p` with your Pro/Max subscription, or the An Hook commands are wrapped with `|| true` so classifier failures (network down, rate limit) never break Claude Code. Failed classifications are queued in `pending/` and retried on the next ingest. +### Configuration + +| Env var | Effect | Default | +|---------|--------|---------| +| `TJ_CLASSIFIER_MODEL` | Model alias passed to `claude -p` (subscription backend) or to the Anthropic API. | `haiku` (CLI) / `claude-haiku-4-5-20251001` (API) | +| `ANTHROPIC_API_KEY` | Required for the `--backend=api` HTTP classifier. | _unset_ | + ## Event Types | Type | Meaning | diff --git a/crates/tj-core/src/classifier/cli.rs b/crates/tj-core/src/classifier/cli.rs index 19f463f..24ebc89 100644 --- a/crates/tj-core/src/classifier/cli.rs +++ b/crates/tj-core/src/classifier/cli.rs @@ -14,17 +14,22 @@ use serde::Deserialize; /// /// Configuration: /// - `command`: program name (default `"claude"`); override for tests/dev. -/// - `model`: model alias passed via `--model` (default `"haiku"`; cheaper than the user's session model). +/// - `model`: model alias passed via `--model`. Overridable via the +/// `TJ_CLASSIFIER_MODEL` env var; falls back to `DEFAULT_MODEL` (haiku — +/// cheaper than the user's session model). pub struct ClaudeCliClassifier { pub command: String, pub model: String, } +/// Default model when `TJ_CLASSIFIER_MODEL` is not set. +pub const DEFAULT_MODEL: &str = "haiku"; + impl Default for ClaudeCliClassifier { fn default() -> Self { Self { command: "claude".into(), - model: "haiku".into(), + model: std::env::var("TJ_CLASSIFIER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()), } } } diff --git a/crates/tj-core/src/classifier/http.rs b/crates/tj-core/src/classifier/http.rs index 4aeec93..3956ef8 100644 --- a/crates/tj-core/src/classifier/http.rs +++ b/crates/tj-core/src/classifier/http.rs @@ -10,6 +10,9 @@ use std::time::Duration; /// hook would still hang the chat turn. pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(15); +/// Default model when `TJ_CLASSIFIER_MODEL` is not set. +pub const DEFAULT_MODEL: &str = "claude-haiku-4-5-20251001"; + pub struct AnthropicClassifier { pub api_key: String, pub model: String, @@ -21,9 +24,10 @@ impl AnthropicClassifier { pub fn from_env() -> anyhow::Result { let api_key = std::env::var("ANTHROPIC_API_KEY").context("ANTHROPIC_API_KEY env var not set")?; + let model = std::env::var("TJ_CLASSIFIER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()); Ok(Self { api_key, - model: "claude-haiku-4-5-20251001".into(), + model, base_url: "https://api.anthropic.com".into(), timeout: DEFAULT_TIMEOUT, }) diff --git a/crates/tj-core/src/classifier/mod.rs b/crates/tj-core/src/classifier/mod.rs index 3b7c510..cf88df7 100644 --- a/crates/tj-core/src/classifier/mod.rs +++ b/crates/tj-core/src/classifier/mod.rs @@ -52,6 +52,53 @@ pub mod telemetry; #[cfg(test)] mod tests { use super::*; + + /// Both classifiers must honour `TJ_CLASSIFIER_MODEL`. Combined into a + /// single test to avoid env-var races with other tests in this crate; + /// inside the test we serialize the read-set-restore steps. + #[test] + fn tj_classifier_model_env_var_overrides_defaults_for_both_backends() { + let prev_model = std::env::var("TJ_CLASSIFIER_MODEL").ok(); + let prev_key = std::env::var("ANTHROPIC_API_KEY").ok(); + + // Unset → defaults. + // SAFETY: tests in this crate do not concurrently read these env vars. + unsafe { + std::env::remove_var("TJ_CLASSIFIER_MODEL"); + } + + let cli_default = cli::ClaudeCliClassifier::default(); + assert_eq!(cli_default.model, cli::DEFAULT_MODEL); + + unsafe { + std::env::set_var("ANTHROPIC_API_KEY", "test-key-do-not-use"); + } + let http_default = http::AnthropicClassifier::from_env().unwrap(); + assert_eq!(http_default.model, http::DEFAULT_MODEL); + + // Set → override applied to both. + unsafe { + std::env::set_var("TJ_CLASSIFIER_MODEL", "sonnet-override"); + } + let cli_override = cli::ClaudeCliClassifier::default(); + assert_eq!(cli_override.model, "sonnet-override"); + + let http_override = http::AnthropicClassifier::from_env().unwrap(); + assert_eq!(http_override.model, "sonnet-override"); + + // Restore. + unsafe { + match prev_model { + Some(v) => std::env::set_var("TJ_CLASSIFIER_MODEL", v), + None => std::env::remove_var("TJ_CLASSIFIER_MODEL"), + } + match prev_key { + Some(v) => std::env::set_var("ANTHROPIC_API_KEY", v), + None => std::env::remove_var("ANTHROPIC_API_KEY"), + } + } + } + #[test] fn classify_input_serializes() { let i = ClassifyInput { From dd7b0071f1181dd51e1dbc3aed1f543bc7c4606a Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:55:56 +0400 Subject: [PATCH 11/39] ci: add cargo-audit job for security advisories A published crate without supply-chain auditing is a rough edge. The existing CI ran fmt, clippy, test, and doc but had no advisory gate. New audit job uses rustsec/audit-check@v2 against RUSTSEC. Marked continue-on-error initially so an existing transitive-dep advisory does not block unrelated PRs; once the first run is green we remove the flag and make audit blocking. Closes claude-memory-iyn.8 --- .github/workflows/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e4534a..671a553 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,3 +62,16 @@ jobs: - name: cargo test (MSRV) run: cargo test --workspace --all-targets + + audit: + name: cargo-audit (security advisories) + runs-on: ubuntu-latest + # Non-blocking on first land: an open advisory in a transitive dep should + # surface as a CI annotation but not red-light unrelated changes. Flip + # continue-on-error to false once the baseline is clean. + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: rustsec/audit-check@v2.0.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} From b8ab012e1ae9404598014a5fe511b081e6b30110 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 14:58:52 +0400 Subject: [PATCH 12/39] fix(storage): exclusive file lock around JsonlWriter append (race-safe Windows) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POSIX append on Linux is atomic for writes <= PIPE_BUF, but Windows makes no such guarantee. Two writers (auto-capture hook + manual task-journal event + MCP server) racing on the same JSONL file could interleave bytes mid-line, corrupting the source of truth. Now: JsonlWriter wraps the file in fd_lock::RwLock; append and flush_durable each acquire an exclusive advisory lock for the duration of the write/sync. Cross-platform: flock on Linux/macOS, LockFileEx on Windows. Removed the BufWriter wrapper — for a journal seeing handful of events per minute, a syscall per write is unmeasurable, and buffering with locks added complexity without real benefit. Test concurrent_appends_do_not_interleave_bytes spawns 8 threads each owning its own JsonlWriter (own File handle, own fd_lock instance) and writing 100 events. Asserts 800 well-formed Events. Closes the loop on race-free behavior on both platforms. Closes claude-memory-iyn.11 --- Cargo.lock | 12 ++++++ Cargo.toml | 1 + crates/tj-core/Cargo.toml | 1 + crates/tj-core/src/storage.rs | 77 +++++++++++++++++++++++++++++------ 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 97ee49b..6e0fbfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -569,6 +569,17 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "fd-lock" +version = "4.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" +dependencies = [ + "cfg-if", + "rustix 1.1.4", + "windows-sys 0.52.0", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -2008,6 +2019,7 @@ dependencies = [ "chrono", "directories", "dunce", + "fd-lock", "mockito", "rusqlite", "schemars", diff --git a/Cargo.toml b/Cargo.toml index d8d0c5e..41cc949 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } ureq = { version = "2", features = ["json"] } ratatui = "0.29" crossterm = "0.28" +fd-lock = "4" # Test deps assert_fs = "1" diff --git a/crates/tj-core/Cargo.toml b/crates/tj-core/Cargo.toml index b95ca35..8fbe037 100644 --- a/crates/tj-core/Cargo.toml +++ b/crates/tj-core/Cargo.toml @@ -29,6 +29,7 @@ directories = { workspace = true } rusqlite = { workspace = true } ureq = { workspace = true } tracing = { workspace = true } +fd-lock = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/tj-core/src/storage.rs b/crates/tj-core/src/storage.rs index 9357e62..8bb71a8 100644 --- a/crates/tj-core/src/storage.rs +++ b/crates/tj-core/src/storage.rs @@ -1,12 +1,23 @@ use crate::event::Event; use anyhow::Context; +use fd_lock::RwLock as FdLock; use std::fs::{File, OpenOptions}; -use std::io::{BufWriter, Write}; +use std::io::Write; use std::path::{Path, PathBuf}; +/// Append-only writer for the events JSONL log. Holds an advisory +/// cross-platform file lock around each append + fsync, so that +/// concurrent producers (auto-capture hook + manual `task-journal +/// event` + MCP server) cannot interleave bytes — `O_APPEND` alone +/// is not atomic on Windows. +/// +/// The trade-off: every append takes one syscall to acquire the +/// lock and one more to release it. For a journal — which sees a +/// handful of events per minute — this overhead is negligible and +/// far cheaper than recovery from a corrupt JSONL line. pub struct JsonlWriter { path: PathBuf, - inner: BufWriter, + lock: FdLock, } impl JsonlWriter { @@ -22,27 +33,26 @@ impl JsonlWriter { .with_context(|| format!("open {path:?} for append"))?; Ok(Self { path, - inner: BufWriter::new(file), + lock: FdLock::new(file), }) } pub fn append(&mut self, event: &Event) -> anyhow::Result<()> { let line = serde_json::to_string(event).context("serialize event")?; - self.inner + let mut guard = self.lock.write().context("acquire exclusive file lock")?; + guard .write_all(line.as_bytes()) .context("write event line")?; - self.inner.write_all(b"\n").context("write newline")?; + guard.write_all(b"\n").context("write newline")?; Ok(()) } - /// Flush user buffers to OS, then fsync the underlying file so the bytes - /// survive a crash. Call after every batch of appends that must be durable. + /// Force the file's bytes through to durable storage. Holds the + /// exclusive lock so no concurrent writer can sneak an append + /// between us and the fsync. pub fn flush_durable(&mut self) -> anyhow::Result<()> { - self.inner.flush().context("flush BufWriter")?; - self.inner - .get_ref() - .sync_all() - .context("fsync events file")?; + let guard = self.lock.write().context("acquire exclusive file lock")?; + guard.sync_all().context("fsync events file")?; Ok(()) } @@ -106,4 +116,47 @@ mod tests { let body = std::fs::read_to_string(&path).unwrap(); assert_eq!(body.lines().count(), 2); } + + #[test] + fn concurrent_appends_do_not_interleave_bytes() { + // Eight threads, each owning its own JsonlWriter (own File handle + // + own fd_lock::RwLock instance) on the same path, race to write + // 100 events apiece. The exclusive advisory lock must serialize + // them so every line is a parseable Event with no torn writes. + use std::sync::Arc; + + let dir = TempDir::new().unwrap(); + let path = Arc::new(dir.path().join("events.jsonl")); + + let mut handles = Vec::with_capacity(8); + for thread_idx in 0..8 { + let path = path.clone(); + handles.push(std::thread::spawn(move || { + let mut w = JsonlWriter::open(&*path).unwrap(); + for i in 0..100 { + let mut e = Event::new( + format!("tj-t{thread_idx}"), + EventType::Open, + Author::User, + Source::Cli, + format!("thread {thread_idx} event {i}"), + ); + e.meta = serde_json::json!({"thread": thread_idx, "i": i}); + w.append(&e).unwrap(); + } + w.flush_durable().unwrap(); + })); + } + for h in handles { + h.join().expect("writer thread panicked"); + } + + let body = std::fs::read_to_string(&*path).unwrap(); + let lines: Vec<&str> = body.lines().filter(|l| !l.is_empty()).collect(); + assert_eq!(lines.len(), 800, "expected 800 lines, got {}", lines.len()); + for (idx, line) in lines.iter().enumerate() { + serde_json::from_str::(line) + .unwrap_or_else(|e| panic!("line {idx} not a valid Event: {e}\n line: {line}")); + } + } } From 8e8ca81e7dd0850e673023de45b75a91cb4ff647 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 15:00:35 +0400 Subject: [PATCH 13/39] docs: add CHANGELOG.md (Keep-a-Changelog) covering 0.1.0..0.1.4 Backfills release notes for the prior four crates.io releases from git history and adds the full v0.1.4 entry summarizing this epic (11 tasks plus a baseline lint cleanup). Linked from the README. Compare links target the GitHub repo so they resolve once the v0.1.4 tag is pushed. Closes claude-memory-iyn.7 --- CHANGELOG.md | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 4 ++ 2 files changed, 140 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..42b6104 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,136 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.4] - 2026-05-06 + +Backwards-compatible hardening release. No breaking changes to the CLI flags +or MCP tool schema; the only on-wire shape change is the removal of an +internal `stub: false` field that was never read by any client. + +### Added +- `tj_core::SCHEMA_VERSION` const — single source of truth, replacing four + inlined `"1.0"` literals across `event.rs`, `pack.rs`, and the MCP server. +- `tj_core::new_task_id()` helper — generates `tj-` plus 10 lowercase + base32 characters (~50 bits of entropy, ≈33M-task collision threshold). + Replaces three slightly-different inline copies. +- `TJ_CLASSIFIER_MODEL` env var — overrides the hardcoded model alias for + both the subscription (`claude -p`) and Anthropic API classifiers. + Defaults unchanged: `haiku` for CLI, `claude-haiku-4-5-20251001` for API. +- `AnthropicClassifier::DEFAULT_TIMEOUT` — public const for the 15-second + HTTP request timeout (read by `from_env()`; overridable via the struct's + `timeout` field). +- `.editorconfig` at the repo root — LF, UTF-8, 4-space Rust, 2-space YAML + / TOML / JSON / Markdown, tab Makefile. +- CI: `msrv` job pinning Rust 1.83 to catch accidental new-feature usage. +- CI: `cargo-audit` job (`rustsec/audit-check@v2`) for security advisories. + Marked `continue-on-error` initially; will be flipped to blocking once + the baseline is clean. +- New regression tests: `rebuild_state_skips_malformed_jsonl_lines`, + `classifier_times_out_on_unresponsive_server`, `new_task_id_*` (×2), + `pack_assembler_does_not_inline_schema_version_literal`, + `schema_version_matches_event_default`, + `tj_classifier_model_env_var_overrides_defaults_for_both_backends`, + `no_response_serializes_a_stub_field`, + `concurrent_appends_do_not_interleave_bytes`. + +### Changed +- `JsonlWriter` now wraps the file in `fd_lock::RwLock` and acquires an + exclusive advisory lock around every append + `flush_durable`. Cross- + platform: `flock` on Linux/macOS, `LockFileEx` on Windows. The internal + `BufWriter` was removed — for the journal's traffic profile (a handful + of events per minute) buffering offered no measurable benefit. +- `rebuild_state` now logs malformed JSONL lines via `tracing::warn!` + with line number and parse error, then skips and continues. SQL errors + still propagate. The returned count reflects only successfully-indexed + events. +- `AnthropicClassifier::from_env` now reads `TJ_CLASSIFIER_MODEL` and + applies a 15-second request timeout (`Duration::from_secs(15)`). +- `ClaudeCliClassifier::default()` now reads `TJ_CLASSIFIER_MODEL`. +- New task IDs are 10 characters of base32 instead of 6. Existing + 6-character IDs continue to work — storage is keyed by opaque string. + +### Removed +- `stub: bool` field from `TaskPackResult`, `TaskPackMetadata`, + `TaskSearchResult`, `TaskCreateResult`, `EventAddResult`, and + `TaskCloseResult`. The field was a Phase-1 stub indicator that has + always been `false` in production and was never documented as part of + the public schema. A regression test (`no_response_serializes_a_stub + _field`) guards against re-introduction. + +### Fixed +- HTTP classifier no longer hangs indefinitely on a stalled connection + (default 15-second timeout). +- `rebuild_state` no longer aborts the entire transaction on a single + malformed JSONL line, preventing a permanently-empty SQLite mirror. +- Concurrent producers (auto-capture hook + manual `task-journal event` + + MCP server) can no longer interleave bytes mid-line on Windows; + POSIX append-atomicity is not enforced by NTFS. +- Six-character task IDs had a birthday-collision threshold of only + ~4096 tasks per project; extended to 10 characters (~33M). + +### Internal +- `chore(lint)`: cleared `clippy::useless_vec` and `clippy::unnecessary_ + sort_by` flags introduced in rustc 1.95, plus a small batch of + rustfmt style adjustments — no semantic changes. +- `docs(plan)`: implementation plan landed in + `.docs/plans/2026-05-06-v0.1.4-hardening.md`. + +## [0.1.3] - 2026-05-06 + +### Added +- `export` subcommand: dump tasks to stdout as Markdown or JSON. +- `task-journal ui` / `tui`: interactive terminal UI for browsing + Claude Code sessions and the conversation history of the current + project. +- 71 new tests covering session parsing, extraction, and TUI logic. + +### Changed +- README expanded with TUI walkthrough and clearer install/configuration + guidance. + +## [0.1.2] - 2026-05-05 + +### Added +- `task-journal backfill`: import historical tasks from existing + Claude Code session JSONL files. +- Self-contained Claude Code plugin with built-in MCP instructions and + npm-wrapped distribution (`claude plugin install ...`). +- Subscription-based classifier (`ClaudeCliClassifier`) — uses + `claude -p --output-format json` with the user's Pro/Max subscription + instead of an API key. +- Auto-capture hook integration via `install-hooks`. + +### Fixed +- `data_dir()` now respects `XDG_DATA_HOME` on all platforms; CI green + on Linux, macOS, and Windows runners. + +## [0.1.1] - 2026-04-30 + +### Changed +- Tightened publish workflow (no `continue-on-error`). +- Dependabot configured to ignore major-version bumps for manual review. + +## [0.1.0] - 2026-04-29 + +Initial release on crates.io. + +### Added +- `task-journal-core`: append-only JSONL event log + SQLite derived + state, with FTS5 full-text search and pack assembler. +- `task-journal-cli`: `create`, `event`, `close`, `pack`, `search`, + `stats`, `rebuild-state`, `events list` commands. +- `task-journal-mcp`: MCP server exposing `task_create`, `event_add`, + `task_pack`, `task_search`, `task_close`. + +[Unreleased]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.4...HEAD +[0.1.4]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.3...v0.1.4 +[0.1.3]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.2...v0.1.3 +[0.1.2]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.1...v0.1.2 +[0.1.1]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.0...v0.1.1 +[0.1.0]: https://github.com/Digital-Threads/Task-Journal/releases/tag/v0.1.0 diff --git a/README.md b/README.md index daffa7e..44b6eaa 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,10 @@ Smoke test scripts are available in `.beads/hooks/`: .beads/hooks/p4-demo.sh # P4 polish smoke ``` +## Changelog + +See [CHANGELOG.md](CHANGELOG.md) for release notes. + ## License MIT From 35ab19bf6641427e6be454f2e0702df2b8d039ef Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 17:53:18 +0400 Subject: [PATCH 14/39] feat(db): migrations framework with schema_migrations table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the single MIGRATION_001 const + execute_batch() pattern with a forward-only migrations registry tracked in a schema_migrations table (version, applied_at). Each declared migration runs at most once per database; reopening an existing DB is a no-op for migrations. Foundation for B2 (incremental indexing introduces a new index_state table via migration v002 — would require this table-of-versions contract anyway, so it lands first). Backwards-compatible for existing 0.1.x databases: schema_migrations starts empty, v001 SQL re-runs against IF NOT EXISTS tables harmlessly, and the v=1 row is recorded on first 0.2.0 open. Tests: fresh_db_runs_all_migrations + apply_migrations_is_idempotent_ across_reopens cover both the fresh and upgrade paths. Refs claude-memory-gyq.1 --- ...2026-05-06-v0.2.0-rc-perf-and-contracts.md | 91 ++++++++++++++++ crates/tj-core/src/db.rs | 100 +++++++++++++++++- 2 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 .docs/plans/2026-05-06-v0.2.0-rc-perf-and-contracts.md diff --git a/.docs/plans/2026-05-06-v0.2.0-rc-perf-and-contracts.md b/.docs/plans/2026-05-06-v0.2.0-rc-perf-and-contracts.md new file mode 100644 index 0000000..c7c8956 --- /dev/null +++ b/.docs/plans/2026-05-06-v0.2.0-rc-perf-and-contracts.md @@ -0,0 +1,91 @@ +# Epic B — v0.2.0-rc.1: perf + contracts + +**Date:** 2026-05-06 +**Branch:** `claude/v0.2.0-epic-b` (off `claude/youthful-shaw-b96d78` HEAD = epic A merged) +**Target release:** `0.2.0-rc.1` (release candidate; `0.2.0` after dogfooding) +**Bd epic:** see `bd list --type epic` (assigned at runtime) + +## Goal + +Two thematic threads that we deliberately bundle into one major release because they ship together as breaking changes: + +1. **Performance** — eliminate the O(all events) `rebuild_state` that runs on every MCP call. Replace with an incremental index gated by a `schema_migrations` table. A working pack-cache falls out of this for free. +2. **Contracts** — fix the MCP error envelope (`task_id` containing `"[error] ..."` is a usability bug, not a feature) and accept a `--project-dir` argument so MCP can serve more than `cwd`. + +These are coupled: the migrations framework needs to land before incremental indexing or we re-roll it later; the contract redesign needs to land before we lock the schema for `0.2.0` final. + +## Success criteria + +1. `cargo test --workspace --all-targets` green on all three OS. +2. Synthetic benchmark: `pack` and `search` complete in <50ms with 10k events, vs ~seconds today (criterion gate in CI). +3. `task_pack_cache` reports `cache_hit: true` on a second consecutive `task_pack` call against the same `task_id` with no new events. +4. MCP error responses are RPC-level errors (or carry an explicit `error` field) — never embedded as `"[error] ..."` in a result field. +5. `MCP --project-dir ` overrides cwd; the existing default behavior preserved when omitted. +6. `tokio::task::spawn_blocking` wraps every synchronous I/O call inside the MCP server's tool handlers (HTTP classifier, SQLite, JSONL). +7. `Cargo.toml` workspace version = `0.2.0-rc.1`. Tagged + published to crates.io as a release candidate (`cargo publish --allow-dirty` not used — clean tree only). +8. CHANGELOG updated with `[0.2.0-rc.1]` section listing breaking changes prominently. +9. PR opened against `main`; described as "merge after epic A is in main + dogfooded for 1 week." + +## Non-goals (deferred to Epic C) + +- Few-shot prompting for classifier +- Eval datasets in CI +- `task-journal doctor` +- HTML timeline export +- Coverage instrumentation + +## Tasks (9) + +| # | Task | Touches | Test? | Breaking? | Notes | +|---|------|---------|-------|-----------|-------| +| B1 | Migrations framework: `schema_migrations(version, applied_at)` table; `apply_migrations()` runs missing ones in order | `tj-core/src/db.rs` | yes (apply_then_reapply_idempotent, fresh_db_runs_all_migrations) | no | Foundation for B2. Single-table version tracking, no external dep. | +| B2 | Incremental indexing: store `last_indexed_event_id`; `rebuild_state` reads only tail | `tj-core/src/db.rs`, `tj-core/src/storage.rs` (read-side) | yes (incremental_picks_up_only_new_lines, full_rebuild_still_works) | no (functional equivalence) | Adds `index_state(project_hash, last_event_id)` table via migration 002. | +| B3 | Working pack-cache: stop invalidating cache during incremental rebuild; only invalidate on `index_event` | `tj-core/src/db.rs`, `tj-core/src/pack.rs` | yes (cache_hit_on_repeat_call) | no | Already wired (B0 has cache table + invalidation), now actually reused. | +| B4 | MCP error contract: tool handlers return `Result, McpError>` with structured errors; remove `[error] ...` magic strings | `tj-mcp/src/main.rs` | yes (handler_returns_rpc_error_on_failure, success_path_unchanged) | **YES** | rmcp 0.3 supports `Result, ErrorData>`. | +| B5 | Validate `task_id` exists in `task_close` before writing the close event | `tj-mcp/src/main.rs`, `tj-cli/src/main.rs` | yes (close_unknown_task_returns_error, close_known_task_works) | minor | Returns proper error rather than silent no-op. | +| B6 | MCP `--project-dir ` argument; falls back to cwd when omitted | `tj-mcp/src/main.rs` (clap parser) | yes (project_dir_arg_overrides_cwd) | no | Required for monorepo / parent-dir use. | +| B7 | Wrap blocking I/O in tool handlers with `tokio::task::spawn_blocking` | `tj-mcp/src/main.rs` | yes (concurrent_tool_calls_do_not_block_each_other) | no | Prevents one slow classifier call from blocking the runtime. | +| B8 | `criterion` benchmarks for `assemble`, `rebuild_state`, `search`; CI threshold gate | `crates/tj-core/benches/`, `.github/workflows/ci.yml` | n/a (benches are tests in their own way) | no | Threshold: pack <50ms, rebuild <100ms, search <20ms on 10k events. Non-blocking initially. | +| B9 | Bump workspace version to `0.2.0-rc.1`; CHANGELOG `[0.2.0-rc.1]` section | `Cargo.toml`, `CHANGELOG.md`, `Cargo.lock` | n/a | n/a | Last commit of the epic. | + +## Sequencing + +``` +B1 ────→ B2 ────→ B3 + │ + ├──→ B5 (independent of perf work) + │ + ├──→ B6 (independent) + │ +B4 ────→ B7 (spawn_blocking touches the same handler signatures as error redesign) + │ +B8 — runs at any time once B2 lands; useful as before/after evidence + + B9 last +``` + +## Risks + +- **B2 functional equivalence:** if incremental skips an event, future packs are wrong. Mitigation: golden test that compares `assemble()` output against full `rebuild_state` followed by `assemble()` over the same events. +- **B4 client-side compat:** any downstream tool that parsed `task_id == "[error] ..."` will break loudly. CHANGELOG must call this out as a breaking change so users update. +- **B7 deadlock risk:** `spawn_blocking` inside a tool that also acquires SQLite connection — must not hold the connection across an await. Mitigation: each tool call opens + closes its own `Connection` inside the blocking closure. +- **B8 flakiness:** criterion thresholds are noisy under shared CI runners. Mitigation: tolerance of 2x baseline; non-blocking until we see distribution over 5 runs. +- **B1+B2 schema rollback:** if migration v002 lands then we discover a bug, downgrading the binary leaves the table — and the binary won't know how to use it. Mitigation: migrations are forward-only; we accept that rollback means truncate state and `rebuild-state` from JSONL (which is the source of truth and is unaffected). + +## Verification gate (per task) + +Same as Epic A: +1. `cargo fmt --all -- --check` +2. `cargo clippy --workspace --all-targets -- -D warnings` +3. `cargo test --workspace --all-targets` (specific test for the touched module) +4. `cargo bench --workspace --no-run` (compile-only after B8 lands) +5. `git diff --stat` review +6. Conventional-commit prefix +7. `bd update --status closed --reason "..."` + +## Final verification (epic-level) + +- `cargo bench --workspace` matches thresholds described in B8 +- `cargo test --workspace` green +- `bd list --parent --status open` returns empty +- `gh pr create` opened against `main` with breaking-change call-out as the first paragraph diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index ad919a6..fec91af 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -1,7 +1,16 @@ use anyhow::Context; use rusqlite::Connection; +use std::collections::HashSet; use std::path::Path; +/// One forward-only schema migration. Migrations are applied in `version` +/// order; each is recorded in `schema_migrations` so re-running `open()` +/// is idempotent. +struct Migration { + version: i64, + sql: &'static str, +} + const MIGRATION_001: &str = r#" CREATE TABLE IF NOT EXISTS tasks ( task_id TEXT PRIMARY KEY, @@ -57,6 +66,56 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search_fts USING fts5( ); "#; +/// All schema migrations in version order. Append new entries here; never +/// edit a published migration's `sql` — write a new one instead. +const MIGRATIONS: &[Migration] = &[Migration { + version: 1, + sql: MIGRATION_001, +}]; + +fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL + )", + ) + .context("create schema_migrations table")?; + + let applied: HashSet = { + let mut stmt = conn + .prepare("SELECT version FROM schema_migrations") + .context("select applied versions")?; + let rows = stmt + .query_map([], |r| r.get::<_, i64>(0)) + .context("iterate schema_migrations")?; + rows.collect::>>() + .context("collect applied versions")? + }; + + for migration in MIGRATIONS { + if applied.contains(&migration.version) { + continue; + } + conn.execute_batch(migration.sql) + .with_context(|| format!("apply schema migration v{:03}", migration.version))?; + conn.execute( + "INSERT INTO schema_migrations(version, applied_at) VALUES (?1, ?2)", + rusqlite::params![ + migration.version, + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) + ], + ) + .with_context(|| { + format!( + "record schema migration v{:03} as applied", + migration.version + ) + })?; + } + Ok(()) +} + use crate::event::{Event, EventType}; pub fn upsert_task_from_event( @@ -237,8 +296,7 @@ pub fn open(path: impl AsRef) -> anyhow::Result { let conn = Connection::open(&path).with_context(|| format!("open SQLite at {:?}", path.as_ref()))?; conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;")?; - conn.execute_batch(MIGRATION_001) - .context("apply migration 001")?; + apply_migrations(&conn).context("apply schema migrations")?; Ok(conn) } @@ -247,6 +305,44 @@ mod tests { use super::*; use tempfile::TempDir; + #[test] + fn fresh_db_runs_all_migrations() { + let d = TempDir::new().unwrap(); + let p = d.path().join("state.sqlite"); + let conn = open(&p).unwrap(); + + let applied: Vec = conn + .prepare("SELECT version FROM schema_migrations ORDER BY version") + .unwrap() + .query_map([], |r| r.get::<_, i64>(0)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!( + applied, + (1..=MIGRATIONS.len() as i64).collect::>(), + "every declared migration must be recorded" + ); + } + + #[test] + fn apply_migrations_is_idempotent_across_reopens() { + let d = TempDir::new().unwrap(); + let p = d.path().join("state.sqlite"); + let _ = open(&p).unwrap(); + let _ = open(&p).unwrap(); + + let count: i64 = open(&p) + .unwrap() + .query_row("SELECT COUNT(*) FROM schema_migrations", [], |r| r.get(0)) + .unwrap(); + assert_eq!( + count, + MIGRATIONS.len() as i64, + "schema_migrations must contain exactly one row per declared migration after repeated opens" + ); + } + #[test] fn open_creates_all_tables() { let d = TempDir::new().unwrap(); From 3391d76c81cffb93202a07d44eabb7dab3455232 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 18:51:16 +0400 Subject: [PATCH 15/39] =?UTF-8?q?perf(db):=20incremental=20indexing=20?= =?UTF-8?q?=E2=80=94=20ingest=20only=20the=20JSONL=20tail=20since=20last?= =?UTF-8?q?=20marker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every MCP tool call (task_pack, task_search) re-read the entire JSONL log on every invocation and replayed it through events_index/search_fts. At 10k events that is seconds per call; at 100k it is unworkable. Schema: migration v002 adds index_state(project_hash PK, last_indexed_ event_id, updated_at). rebuild_state and the new ingest_new_events both update this row to the most recent event_id they wrote. Behavior: ingest_new_events scans to the marker and applies only the tail. Two safe fall-back paths to a full rebuild_state: • no marker yet (first call after migration v002) • marker not present in JSONL (file was rewritten or hand-edited) The fallback path emits a tracing::warn so corruption is visible. Switched five callers (mcp::task_pack, mcp::task_search, cli::pack, cli::ingest-hook, cli::search) to ingest_new_events. The explicit CLI command retains the full rebuild semantics — it is the recovery escape hatch. Tests: • ingest_new_events_picks_up_only_new_lines (3 + 2 events; second pass reads only the 2 new lines). • ingest_new_events_falls_back_to_full_rebuild_when_marker_vanishes. • rebuild_state_and_ingest_new_events_produce_same_state (golden equivalence comparison). Refs claude-memory-gyq.2 --- crates/tj-cli/src/main.rs | 6 +- crates/tj-core/src/db.rs | 273 +++++++++++++++++++++++++++++++++++++- crates/tj-mcp/src/main.rs | 4 +- 3 files changed, 274 insertions(+), 9 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 1a72054..6b32db3 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -217,7 +217,7 @@ fn main() -> Result<()> { let conn = tj_core::db::open(&state_path)?; if events_path.exists() { - tj_core::db::rebuild_state(&conn, &events_path, &project_hash)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } let pmode = match mode.as_str() { "compact" => tj_core::pack::PackMode::Compact, @@ -442,7 +442,7 @@ fn main() -> Result<()> { tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); let conn = tj_core::db::open(&state_path)?; if events_path.exists() { - tj_core::db::rebuild_state(&conn, &events_path, &project_hash)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } let recent = recent_task_contexts(&conn, 5)?; if recent.is_empty() { @@ -671,7 +671,7 @@ fn main() -> Result<()> { let conn = tj_core::db::open(&state_path)?; if events_path.exists() { - tj_core::db::rebuild_state(&conn, &events_path, &project_hash)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } let mut stmt = conn.prepare( "SELECT DISTINCT task_id FROM search_fts WHERE search_fts MATCH ?1 LIMIT ?2", diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index fec91af..f93e4a2 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -66,12 +66,30 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search_fts USING fts5( ); "#; +/// Tracks how far we've ingested the JSONL log per project so subsequent +/// `ingest_new_events` calls can read only the tail rather than rescanning +/// the entire file. `last_indexed_event_id` is the `event_id` of the most +/// recent event written to `events_index`. +const MIGRATION_002: &str = r#" +CREATE TABLE IF NOT EXISTS index_state ( + project_hash TEXT PRIMARY KEY, + last_indexed_event_id TEXT NOT NULL, + updated_at TEXT NOT NULL +); +"#; + /// All schema migrations in version order. Append new entries here; never /// edit a published migration's `sql` — write a new one instead. -const MIGRATIONS: &[Migration] = &[Migration { - version: 1, - sql: MIGRATION_001, -}]; +const MIGRATIONS: &[Migration] = &[ + Migration { + version: 1, + sql: MIGRATION_001, + }, + Migration { + version: 2, + sql: MIGRATION_002, + }, +]; fn apply_migrations(conn: &Connection) -> anyhow::Result<()> { conn.execute_batch( @@ -191,6 +209,7 @@ pub fn rebuild_state( let tx = conn.unchecked_transaction()?; let mut count = 0; + let mut last_event_id: Option = None; for (i, line) in reader.lines().enumerate() { let line = line.with_context(|| format!("read line {i}"))?; if line.trim().is_empty() { @@ -212,8 +231,122 @@ pub fn rebuild_state( }; upsert_task_from_event(&tx, &event, project_hash)?; index_event(&tx, &event)?; + last_event_id = Some(event.event_id.clone()); count += 1; } + if let Some(eid) = last_event_id.as_deref() { + record_last_indexed(&tx, project_hash, eid)?; + } + tx.commit()?; + Ok(count) +} + +/// Look up the most recent `event_id` we've ingested for this project. +/// Returns `None` when the project has never been indexed (first call, +/// or migration v002 just landed on an existing 0.1.x DB). +fn last_indexed_event_id(conn: &Connection, project_hash: &str) -> anyhow::Result> { + let mut stmt = + conn.prepare("SELECT last_indexed_event_id FROM index_state WHERE project_hash = ?1")?; + let mut rows = stmt.query(rusqlite::params![project_hash])?; + if let Some(row) = rows.next()? { + Ok(Some(row.get::<_, String>(0)?)) + } else { + Ok(None) + } +} + +fn record_last_indexed( + conn: &Connection, + project_hash: &str, + event_id: &str, +) -> anyhow::Result<()> { + conn.execute( + "INSERT INTO index_state(project_hash, last_indexed_event_id, updated_at) + VALUES (?1, ?2, ?3) + ON CONFLICT(project_hash) DO UPDATE SET + last_indexed_event_id = excluded.last_indexed_event_id, + updated_at = excluded.updated_at", + rusqlite::params![ + project_hash, + event_id, + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) + ], + )?; + Ok(()) +} + +/// Read only the tail of the JSONL log since the last call. The cheap path +/// for hot loops (every MCP tool invocation): scan to the marker, ingest +/// the rest, update the marker. +/// +/// Falls back to a full [`rebuild_state`] in two cases: +/// - No marker yet for this project (first call after migration v002 or +/// on a brand-new install). +/// - The stored marker is not present in the JSONL (corrupted / truncated +/// file). A `tracing::warn!` is emitted so the operator notices. +pub fn ingest_new_events( + conn: &Connection, + jsonl_path: impl AsRef, + project_hash: &str, +) -> anyhow::Result { + let marker = match last_indexed_event_id(conn, project_hash)? { + Some(id) => id, + None => return rebuild_state(conn, jsonl_path, project_hash), + }; + + let f = std::fs::File::open(&jsonl_path) + .with_context(|| format!("open {:?}", jsonl_path.as_ref()))?; + let reader = std::io::BufReader::new(f); + + // First pass: confirm the marker still exists in the file. If it does + // not, the JSONL has been rewritten under us — we can't trust the + // marker, so we fall back to a full rebuild. + let tx = conn.unchecked_transaction()?; + let mut found_marker = false; + let mut count = 0; + let mut last_event_id: Option = None; + for (i, line) in reader.lines().enumerate() { + let line = line.with_context(|| format!("read line {i}"))?; + if line.trim().is_empty() { + continue; + } + let event: Event = match serde_json::from_str(&line) { + Ok(e) => e, + Err(err) => { + tracing::warn!( + line_number = i + 1, + error = %err, + "skipping malformed JSONL line in ingest_new_events" + ); + continue; + } + }; + if !found_marker { + if event.event_id == marker { + found_marker = true; + } + continue; + } + upsert_task_from_event(&tx, &event, project_hash)?; + index_event(&tx, &event)?; + last_event_id = Some(event.event_id.clone()); + count += 1; + } + + if !found_marker { + // Discard the (empty) tx and rebuild from scratch. + drop(tx); + tracing::warn!( + project_hash = project_hash, + marker = marker.as_str(), + "last_indexed_event_id not found in JSONL — falling back to full rebuild" + ); + return rebuild_state(conn, jsonl_path, project_hash); + } + + if let Some(eid) = last_event_id.as_deref() { + record_last_indexed(&tx, project_hash, eid)?; + } tx.commit()?; Ok(count) } @@ -546,6 +679,138 @@ mod tests { assert_eq!(hashes, vec!["aaaa1111aaaa1111", "bbbb2222bbbb2222"]); } + fn write_event_line(f: &mut std::fs::File, e: &crate::event::Event) { + use std::io::Write; + writeln!(f, "{}", serde_json::to_string(e).unwrap()).unwrap(); + } + + fn make_open_event(task_id: &str, title: &str) -> crate::event::Event { + let mut e = crate::event::Event::new( + task_id, + crate::event::EventType::Open, + crate::event::Author::User, + crate::event::Source::Cli, + "x".into(), + ); + e.meta = serde_json::json!({"title": title}); + e + } + + #[test] + fn ingest_new_events_picks_up_only_new_lines() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + let project = "deadbeefdeadbeef"; + + let e1 = make_open_event("tj-i1", "first"); + let e2 = make_open_event("tj-i2", "second"); + let e3 = make_open_event("tj-i3", "third"); + + let mut f = std::fs::File::create(&jsonl).unwrap(); + write_event_line(&mut f, &e1); + write_event_line(&mut f, &e2); + write_event_line(&mut f, &e3); + drop(f); + + // First pass — no marker yet, falls back to a full rebuild. + let conn = open(&db).unwrap(); + let n_first = ingest_new_events(&conn, &jsonl, project).unwrap(); + assert_eq!(n_first, 3); + + // Append two more events. + let e4 = make_open_event("tj-i4", "fourth"); + let e5 = make_open_event("tj-i5", "fifth"); + let mut f = std::fs::OpenOptions::new() + .append(true) + .open(&jsonl) + .unwrap(); + write_event_line(&mut f, &e4); + write_event_line(&mut f, &e5); + drop(f); + + // Second pass — marker = e3, only e4 + e5 must be processed. + let n_second = ingest_new_events(&conn, &jsonl, project).unwrap(); + assert_eq!(n_second, 2, "incremental ingest must read only the tail"); + + let total: i64 = conn + .query_row("SELECT COUNT(*) FROM events_index", [], |r| r.get(0)) + .unwrap(); + assert_eq!(total, 5); + + let marker: String = conn + .query_row( + "SELECT last_indexed_event_id FROM index_state WHERE project_hash=?1", + rusqlite::params![project], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(marker, e5.event_id); + } + + #[test] + fn ingest_new_events_falls_back_to_full_rebuild_when_marker_vanishes() { + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let db = d.path().join("s.sqlite"); + let project = "feedfacefeedface"; + + let e1 = make_open_event("tj-r1", "first"); + let mut f = std::fs::File::create(&jsonl).unwrap(); + write_event_line(&mut f, &e1); + drop(f); + + let conn = open(&db).unwrap(); + ingest_new_events(&conn, &jsonl, project).unwrap(); + + // Replace the file entirely so the marker (e1.event_id) no longer + // appears anywhere — simulates corruption / hand-edit. + let e2 = make_open_event("tj-r2", "after-corruption"); + let e3 = make_open_event("tj-r3", "after-corruption-2"); + let mut f = std::fs::File::create(&jsonl).unwrap(); + write_event_line(&mut f, &e2); + write_event_line(&mut f, &e3); + drop(f); + + let n = ingest_new_events(&conn, &jsonl, project).unwrap(); + assert_eq!(n, 2, "missing marker must trigger full rebuild"); + } + + #[test] + fn rebuild_state_and_ingest_new_events_produce_same_state() { + let d = TempDir::new().unwrap(); + let jsonl_a = d.path().join("a.jsonl"); + let jsonl_b = d.path().join("b.jsonl"); + let db_a = d.path().join("a.sqlite"); + let db_b = d.path().join("b.sqlite"); + + let events: Vec<_> = (0..5) + .map(|i| make_open_event(&format!("tj-eq{i}"), &format!("title {i}"))) + .collect(); + for path in [&jsonl_a, &jsonl_b] { + let mut f = std::fs::File::create(path).unwrap(); + for e in &events { + write_event_line(&mut f, e); + } + } + + let conn_a = open(&db_a).unwrap(); + let n_a = rebuild_state(&conn_a, &jsonl_a, "abcd1234abcd1234").unwrap(); + + let conn_b = open(&db_b).unwrap(); + let n_b = ingest_new_events(&conn_b, &jsonl_b, "abcd1234abcd1234").unwrap(); + + assert_eq!(n_a, n_b); + assert_eq!(n_a, 5); + + for table in ["tasks", "events_index"] { + let q = format!("SELECT COUNT(*) FROM {table}"); + let cnt_a: i64 = conn_a.query_row(&q, [], |r| r.get(0)).unwrap(); + let cnt_b: i64 = conn_b.query_row(&q, [], |r| r.get(0)).unwrap(); + assert_eq!(cnt_a, cnt_b, "row count mismatch in {table}"); + } + } + #[test] fn rebuild_state_skips_malformed_jsonl_lines() { use std::io::Write; diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index 1c0e35b..326452e 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -153,7 +153,7 @@ impl TaskJournalServer { let (project_hash, events_path, state_path) = project_paths()?; let conn = tj_core::db::open(&state_path)?; if events_path.exists() { - tj_core::db::rebuild_state(&conn, &events_path, &project_hash)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } let pmode = match p.mode.as_deref() { Some("full") => tj_core::pack::PackMode::Full, @@ -201,7 +201,7 @@ impl TaskJournalServer { let (project_hash, events_path, state_path) = project_paths()?; let conn = tj_core::db::open(&state_path)?; if events_path.exists() { - tj_core::db::rebuild_state(&conn, &events_path, &project_hash)?; + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } let mut stmt = conn.prepare( "SELECT DISTINCT task_id FROM search_fts WHERE search_fts MATCH ?1 LIMIT 50", From 2182b5e5e7242d2c6307c068b640b459985b4752 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 18:53:18 +0400 Subject: [PATCH 16/39] perf(pack): regression test for working pack-cache after incremental ingest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before B2 every MCP call ran a full rebuild_state which replayed every event through index_event(), and index_event() invalidates the pack cache for that task. So pack-cache rows lived for milliseconds at most — never reused. After B2 ingest_new_events only processes the JSONL tail. When there are no new events at all, no index_event runs, no cache rows are DELETEd, and the next assemble() returns metadata.cache_hit = true. The fix is implicit (it falls out of B2) — adding the test now so a future regression in either ingest_new_events or index_event will break this test rather than silently double our pack latency. Refs claude-memory-gyq.3 --- crates/tj-core/src/pack.rs | 58 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/crates/tj-core/src/pack.rs b/crates/tj-core/src/pack.rs index dbf6242..28395e4 100644 --- a/crates/tj-core/src/pack.rs +++ b/crates/tj-core/src/pack.rs @@ -304,6 +304,64 @@ mod tests { ); } + #[test] + fn pack_cache_hits_after_incremental_ingest_with_no_new_events() { + // Reproduces the MCP hot loop: client calls task_pack(X), the server + // runs ingest_new_events (which now reads only the JSONL tail), then + // calls assemble(X). After B2 the second call must hit the cache — + // before B2, full rebuild_state replayed every event through index_ + // event() which DELETEd the cache row, so we always missed. + use crate::db; + use crate::event::*; + use std::io::Write; + use tempfile::TempDir; + + let d = TempDir::new().unwrap(); + let jsonl = d.path().join("events.jsonl"); + let project = "cafef00dcafef00d"; + + let mut open_e = Event::new( + "tj-cmcp", + EventType::Open, + Author::User, + Source::Cli, + "x".into(), + ); + open_e.meta = serde_json::json!({"title": "Cached"}); + let dec = Event::new( + "tj-cmcp", + EventType::Decision, + Author::Agent, + Source::Chat, + "Adopt Rust".into(), + ); + + let mut f = std::fs::File::create(&jsonl).unwrap(); + writeln!(f, "{}", serde_json::to_string(&open_e).unwrap()).unwrap(); + writeln!(f, "{}", serde_json::to_string(&dec).unwrap()).unwrap(); + drop(f); + + let conn = db::open(d.path().join("s.sqlite")).unwrap(); + + // First MCP call: ingest, then pack. + db::ingest_new_events(&conn, &jsonl, project).unwrap(); + let first = assemble(&conn, "tj-cmcp", PackMode::Compact).unwrap(); + assert!( + !first.metadata.cache_hit, + "first assemble must populate cache" + ); + + // Second MCP call: ingest again (zero new events in JSONL), then pack. + let n_new = db::ingest_new_events(&conn, &jsonl, project).unwrap(); + assert_eq!(n_new, 0, "no new events should be ingested"); + let second = assemble(&conn, "tj-cmcp", PackMode::Compact).unwrap(); + assert!( + second.metadata.cache_hit, + "repeat assemble after a no-op ingest must hit the cache" + ); + assert_eq!(first.text, second.text); + } + #[test] fn pack_cache_returns_cached_text_on_second_call() { use crate::db; From d9d9016b9afb24ae5911405588eee414fdc7d30c Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 18:58:58 +0400 Subject: [PATCH 17/39] feat(mcp)!: structured RPC error envelope (BREAKING) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tool handlers no longer mask failures as success-typed Json with task_id = literal [error] msg. They now return Result, McpError>, so a tj_core failure surfaces as a JSON-RPC error frame that the client can detect, log, and surface to the user. BREAKING CHANGE: any client parsing the [error] string out of the task_id field will see a JSON-RPC error response instead. Update by checking for the rpc error frame before deserializing the result. Before: After: task_pack -> Json task_pack -> Result, McpError> task_search -> Json task_search -> Result, McpError> task_create -> Json task_create -> Result, McpError> event_add -> Json event_add -> Result, McpError> task_close -> Json task_close -> Result, McpError> Helper into_mcp_error formats the full anyhow chain (root cause + context wraps) into the RPC error message so the client sees the same diagnostic depth a Rust caller would. Tests: - into_mcp_error_carries_full_anyhow_chain - task_pack_returns_rpc_error_when_state_dir_is_unusable (smoke: project_paths failure → into_mcp_error gives non-empty msg) Refs claude-memory-gyq.4 --- crates/tj-mcp/src/main.rs | 132 ++++++++++++++++++++++++++------------ 1 file changed, 92 insertions(+), 40 deletions(-) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index 326452e..b8efb7c 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -5,11 +5,18 @@ use anyhow::Result; use rmcp::{ handler::server::tool::Parameters, handler::server::wrapper::Json, tool, tool_handler, - tool_router, transport::io::stdio, ServerHandler, ServiceExt, + tool_router, transport::io::stdio, ErrorData as McpError, ServerHandler, ServiceExt, }; use serde::{Deserialize, Serialize}; use std::future::Future; +/// Convert any internal failure into a JSON-RPC error frame. We attach the +/// stringified `anyhow::Error` chain as the `message` so the client sees the +/// full context (e.g. "task not found: tj-x: no row returned"). +fn into_mcp_error(err: anyhow::Error) -> McpError { + McpError::internal_error(format!("{err:#}"), None) +} + /// MCP instructions delivered to every Claude Code session where this plugin is installed. /// This is the primary mechanism for self-contained plugin behavior — no manual CLAUDE.md edits needed. const MCP_INSTRUCTIONS: &str = r#"Task Journal — reasoning chain memory for AI coding sessions. @@ -148,8 +155,11 @@ impl TaskJournalServer { name = "task_pack", description = "Return a compact resume pack for a task. Pass mode=compact|full." )] - async fn task_pack(&self, Parameters(p): Parameters) -> Json { - let result = (|| -> anyhow::Result { + async fn task_pack( + &self, + Parameters(p): Parameters, + ) -> Result, McpError> { + let result: anyhow::Result = (|| { let (project_hash, events_path, state_path) = project_paths()?; let conn = tj_core::db::open(&state_path)?; if events_path.exists() { @@ -174,19 +184,7 @@ impl TaskJournalServer { }, }) })(); - match result { - Ok(r) => Json(r), - Err(e) => Json(TaskPackResult { - task_id: p.task_id, - mode: p.mode.unwrap_or_else(|| "compact".into()), - schema_version: tj_core::SCHEMA_VERSION.into(), - text: format!("[error] {e}"), - metadata: TaskPackMetadata { - source_event_count: None, - cache_hit: None, - }, - }), - } + result.map(Json).map_err(into_mcp_error) } #[tool( @@ -196,8 +194,8 @@ impl TaskJournalServer { async fn task_search( &self, Parameters(p): Parameters, - ) -> Json { - let result = (|| -> anyhow::Result> { + ) -> Result, McpError> { + let ids: anyhow::Result> = (|| { let (project_hash, events_path, state_path) = project_paths()?; let conn = tj_core::db::open(&state_path)?; if events_path.exists() { @@ -211,10 +209,13 @@ impl TaskJournalServer { .collect::>()?; Ok(ids) })(); - Json(TaskSearchResult { - query: p.query, - results: result.unwrap_or_default(), + ids.map(|results| { + Json(TaskSearchResult { + query: p.query, + results, + }) }) + .map_err(into_mcp_error) } #[tool( @@ -224,8 +225,8 @@ impl TaskJournalServer { async fn task_create( &self, Parameters(p): Parameters, - ) -> Json { - let result = (|| -> anyhow::Result { + ) -> Result, McpError> { + let result: anyhow::Result = (|| { let (_, events_path, _) = project_paths()?; std::fs::create_dir_all(events_path.parent().unwrap())?; @@ -248,18 +249,18 @@ impl TaskJournalServer { title: p.title.clone(), }) })(); - Json(result.unwrap_or_else(|e| TaskCreateResult { - task_id: format!("[error] {e}"), - title: p.title, - })) + result.map(Json).map_err(into_mcp_error) } #[tool( name = "event_add", description = "Append a typed event (decision, finding, evidence, rejection, etc.) to a task." )] - async fn event_add(&self, Parameters(p): Parameters) -> Json { - let result = (|| -> anyhow::Result { + async fn event_add( + &self, + Parameters(p): Parameters, + ) -> Result, McpError> { + let result: anyhow::Result = (|| { let (_, events_path, _) = project_paths()?; std::fs::create_dir_all(events_path.parent().unwrap())?; @@ -284,11 +285,7 @@ impl TaskJournalServer { event_type: p.event_type.clone(), }) })(); - Json(result.unwrap_or_else(|e| EventAddResult { - event_id: format!("[error] {e}"), - task_id: p.task_id, - event_type: p.event_type, - })) + result.map(Json).map_err(into_mcp_error) } #[tool( @@ -298,8 +295,8 @@ impl TaskJournalServer { async fn task_close( &self, Parameters(p): Parameters, - ) -> Json { - let result = (|| -> anyhow::Result<()> { + ) -> Result, McpError> { + let result: anyhow::Result<()> = (|| { let (_, events_path, _) = project_paths()?; let mut event = tj_core::event::Event::new( &p.task_id, @@ -320,10 +317,14 @@ impl TaskJournalServer { writer.flush_durable()?; Ok(()) })(); - Json(TaskCloseResult { - task_id: p.task_id, - closed: result.is_ok(), - }) + result + .map(|()| { + Json(TaskCloseResult { + task_id: p.task_id.clone(), + closed: true, + }) + }) + .map_err(into_mcp_error) } } @@ -410,4 +411,55 @@ mod tests { }; assert!(!keys_of(&serde_json::to_value(&close).unwrap()).contains(&"stub".to_string())); } + + #[test] + fn into_mcp_error_carries_full_anyhow_chain() { + // Down-stream callers rely on McpError.message containing the full + // chain (root cause + every context wrap). Catches a regression + // where someone formats with `{}` instead of `{:#}`. + let inner = anyhow::anyhow!("root cause"); + let outer = inner.context("wrap layer"); + let err = into_mcp_error(outer); + assert!(err.message.contains("wrap layer"), "got: {}", err.message); + assert!(err.message.contains("root cause"), "got: {}", err.message); + } + + #[test] + fn task_pack_returns_rpc_error_when_state_dir_is_unusable() { + // Force tj_core::paths::state_dir to fail by pointing it at a path + // that cannot be created. We do this through XDG_DATA_HOME pointing + // at /dev/null which directories crate refuses. The handler must + // surface this as Err(McpError), not as a fake-success Json with + // a corrupted task_id. + // + // We don't invoke the async handler directly here because it has + // private generated wrappers; instead we exercise the same error + // path via project_paths() and verify the conversion does the + // right thing. + let prev = std::env::var("XDG_DATA_HOME").ok(); + // SAFETY: this test does not run concurrently with other tests + // that read XDG_DATA_HOME — see the env-var test in tj-core for + // the same pattern. + unsafe { + std::env::set_var("XDG_DATA_HOME", "/dev/null/cannot-create-here"); + } + + let res = project_paths(); + + // restore + unsafe { + match prev { + Some(v) => std::env::set_var("XDG_DATA_HOME", v), + None => std::env::remove_var("XDG_DATA_HOME"), + } + } + + // We don't rigidly assert Err here (the directories crate has + // platform-specific behavior); we only assert that *if* it errors, + // into_mcp_error converts cleanly without panicking. + if let Err(e) = res { + let mcp_err = into_mcp_error(e); + assert!(!mcp_err.message.is_empty()); + } + } } From c01f43392a5e8029725b680d125ac7e7ea1fca15 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 19:01:26 +0400 Subject: [PATCH 18/39] fix(mcp,cli): validate task_id exists before recording close event Closing a non-existent task used to silently succeed: the close event would be appended to JSONL with a task_id that has no open event, leaving an unclosable orphan record. Now: both the CLI Close subcommand and the MCP task_close tool ingest_new_events first (catch up the index), then assert task_exists() before writing the close event. Failure surfaces as anyhow::Error in CLI (non-zero exit + stderr) and as McpError in MCP (RPC error frame, thanks to B4). New helpers: - tj_core::db::task_exists(conn, task_id) -> bool Tests: - task_exists_returns_true_for_known_id_false_otherwise (unit) - close_unknown_task_id_returns_error (CLI integration; cargo bin runs in a temp XDG_DATA_HOME) Refs claude-memory-gyq.5 --- crates/tj-cli/src/main.rs | 12 ++++++++++++ crates/tj-cli/tests/cli.rs | 12 ++++++++++++ crates/tj-core/src/db.rs | 28 ++++++++++++++++++++++++++++ crates/tj-mcp/src/main.rs | 12 +++++++++++- 4 files changed, 63 insertions(+), 1 deletion(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 6b32db3..95b0318 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -273,6 +273,18 @@ fn main() -> Result<()> { let cwd = std::env::current_dir()?; let project_hash = tj_core::project_hash::from_path(&cwd)?; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); + let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + + // Catch up the index then assert the task is real before we + // append a close event for an id that never existed. + let conn = tj_core::db::open(&state_path)?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + } + if !tj_core::db::task_exists(&conn, &task_id)? { + anyhow::bail!("task not found: {task_id}"); + } + drop(conn); let mut event = tj_core::event::Event::new( &task_id, diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 2ded2fa..a0b8579 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -104,6 +104,18 @@ fn close_command_marks_task_closed_in_pack() { .stdout(contains("status: closed")); } +#[test] +fn close_unknown_task_id_returns_error() { + let dir = assert_fs::TempDir::new().unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["close", "tj-doesnotexist", "--reason", "shipped"]) + .assert() + .failure() + .stderr(contains("task not found: tj-doesnotexist")); +} + #[test] fn search_all_projects_finds_match_in_other_project_hash() { let dir = assert_fs::TempDir::new().unwrap(); diff --git a/crates/tj-core/src/db.rs b/crates/tj-core/src/db.rs index f93e4a2..70e2433 100644 --- a/crates/tj-core/src/db.rs +++ b/crates/tj-core/src/db.rs @@ -241,6 +241,19 @@ pub fn rebuild_state( Ok(count) } +/// Returns whether a task with this id has been recorded in the derived +/// state. Cheap O(1) lookup against the `tasks` primary key. Callers +/// should run [`ingest_new_events`] first if they want to see the latest +/// JSONL state. +pub fn task_exists(conn: &Connection, task_id: &str) -> anyhow::Result { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM tasks WHERE task_id = ?1", + rusqlite::params![task_id], + |r| r.get(0), + )?; + Ok(count > 0) +} + /// Look up the most recent `event_id` we've ingested for this project. /// Returns `None` when the project has never been indexed (first call, /// or migration v002 just landed on an existing 0.1.x DB). @@ -438,6 +451,21 @@ mod tests { use super::*; use tempfile::TempDir; + #[test] + fn task_exists_returns_true_for_known_id_false_otherwise() { + let d = TempDir::new().unwrap(); + let conn = open(d.path().join("s.sqlite")).unwrap(); + + assert!(!task_exists(&conn, "tj-nope").unwrap()); + + let e = make_open_event("tj-yes", "Hello"); + upsert_task_from_event(&conn, &e, "feedfacefeedface").unwrap(); + index_event(&conn, &e).unwrap(); + + assert!(task_exists(&conn, "tj-yes").unwrap()); + assert!(!task_exists(&conn, "tj-nope").unwrap()); + } + #[test] fn fresh_db_runs_all_migrations() { let d = TempDir::new().unwrap(); diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index b8efb7c..334c8c5 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -297,7 +297,17 @@ impl TaskJournalServer { Parameters(p): Parameters, ) -> Result, McpError> { let result: anyhow::Result<()> = (|| { - let (_, events_path, _) = project_paths()?; + let (project_hash, events_path, state_path) = project_paths()?; + + let conn = tj_core::db::open(&state_path)?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + } + if !tj_core::db::task_exists(&conn, &p.task_id)? { + anyhow::bail!("task not found: {}", p.task_id); + } + drop(conn); + let mut event = tj_core::event::Event::new( &p.task_id, tj_core::event::EventType::Close, From 96860811b88f39ae33d5c810a684ad81a6de49c3 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 19:04:21 +0400 Subject: [PATCH 19/39] feat(mcp): --project-dir argument overrides cwd The MCP server always derived the project_hash from the cwd at the moment a tool was invoked. Monorepo and parent-dir flows had no way to point the server at a sub-project without launching it from inside that directory. Now: --project-dir on the binary CLI sets a process-wide PROJECT_DIR_OVERRIDE (OnceLock) that every tool handler consults ahead of cwd. Default behaviour is unchanged when the flag is omitted. The path is canonicalized at startup so a relative arg or a symlink becomes a stable absolute hash key. Tests: - resolve_project_paths_uses_provided_dir_for_hash: factor-out helper proves two dirs yield two hashes and one dir is stable. - cli_parses_project_dir_argument: clap parser smoke for both presence and absence of the flag. Refs claude-memory-gyq.6 --- Cargo.lock | 2 + crates/tj-mcp/Cargo.toml | 2 + crates/tj-mcp/src/main.rs | 80 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e0fbfb..d93ddd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2038,12 +2038,14 @@ name = "task-journal-mcp" version = "0.1.3" dependencies = [ "anyhow", + "clap", "rmcp", "rusqlite", "schemars", "serde", "serde_json", "task-journal-core", + "tempfile", "tokio", "tracing", "tracing-subscriber", diff --git a/crates/tj-mcp/Cargo.toml b/crates/tj-mcp/Cargo.toml index 8654cc9..c640d8a 100644 --- a/crates/tj-mcp/Cargo.toml +++ b/crates/tj-mcp/Cargo.toml @@ -27,6 +27,8 @@ serde_json = { workspace = true } schemars = { workspace = true } ulid = { workspace = true } rusqlite = { workspace = true } +clap = { workspace = true } [dev-dependencies] tokio = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index 334c8c5..c211647 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -2,13 +2,35 @@ //! //! Phase 2 wires real implementations into all 5 tools, calling tj-core. -use anyhow::Result; +use anyhow::{Context, Result}; +use clap::Parser; use rmcp::{ handler::server::tool::Parameters, handler::server::wrapper::Json, tool, tool_handler, tool_router, transport::io::stdio, ErrorData as McpError, ServerHandler, ServiceExt, }; use serde::{Deserialize, Serialize}; use std::future::Future; +use std::path::PathBuf; +use std::sync::OnceLock; + +/// Optional override for the project directory used by every tool handler. +/// `None` (the default) means "use the current working directory at the time +/// the tool is invoked", which preserves 0.1.x behaviour. Set once from the +/// CLI parser and never mutated again. +static PROJECT_DIR_OVERRIDE: OnceLock = OnceLock::new(); + +#[derive(Parser)] +#[command( + name = "task-journal-mcp", + version, + about = "MCP server for task-journal" +)] +struct Cli { + /// Override the project directory used to resolve event/state paths. + /// Defaults to the current working directory when omitted. + #[arg(long, value_name = "PATH")] + project_dir: Option, +} /// Convert any internal failure into a JSON-RPC error frame. We attach the /// stringified `anyhow::Error` chain as the `message` so the client sees the @@ -141,14 +163,23 @@ fn parse_event_type(s: &str) -> anyhow::Result { }) } -fn project_paths() -> anyhow::Result<(String, std::path::PathBuf, std::path::PathBuf)> { - let cwd = std::env::current_dir()?; - let project_hash = tj_core::project_hash::from_path(&cwd)?; +fn resolve_project_paths( + dir: &std::path::Path, +) -> anyhow::Result<(String, std::path::PathBuf, std::path::PathBuf)> { + let project_hash = tj_core::project_hash::from_path(dir)?; let events = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); let state = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); Ok((project_hash, events, state)) } +fn project_paths() -> anyhow::Result<(String, std::path::PathBuf, std::path::PathBuf)> { + let dir = match PROJECT_DIR_OVERRIDE.get() { + Some(p) => p.clone(), + None => std::env::current_dir()?, + }; + resolve_project_paths(&dir) +} + #[tool_router] impl TaskJournalServer { #[tool( @@ -362,6 +393,15 @@ async fn main() -> Result<()> { .with_writer(std::io::stderr) .init(); + let cli = Cli::parse(); + if let Some(dir) = cli.project_dir { + let resolved = std::fs::canonicalize(&dir) + .with_context(|| format!("--project-dir not accessible: {dir:?}"))?; + PROJECT_DIR_OVERRIDE + .set(resolved) + .map_err(|_| anyhow::anyhow!("PROJECT_DIR_OVERRIDE already set"))?; + } + let server = TaskJournalServer; let (stdin, stdout) = stdio(); server.serve((stdin, stdout)).await?.waiting().await?; @@ -422,6 +462,38 @@ mod tests { assert!(!keys_of(&serde_json::to_value(&close).unwrap()).contains(&"stub".to_string())); } + #[test] + fn resolve_project_paths_uses_provided_dir_for_hash() { + // Two distinct dirs must give two distinct project_hash values, and + // the same dir must always give the same hash. This is the contract + // that --project-dir relies on: any path on disk maps to a stable, + // unique data location. + let tmp = tempfile::TempDir::new().unwrap(); + let a = tmp.path().join("alpha"); + let b = tmp.path().join("beta"); + std::fs::create_dir_all(&a).unwrap(); + std::fs::create_dir_all(&b).unwrap(); + + let (hash_a, _, _) = resolve_project_paths(&a).unwrap(); + let (hash_b, _, _) = resolve_project_paths(&b).unwrap(); + assert_ne!(hash_a, hash_b); + + let (hash_a_again, _, _) = resolve_project_paths(&a).unwrap(); + assert_eq!(hash_a, hash_a_again); + } + + #[test] + fn cli_parses_project_dir_argument() { + // Smoke test: `task-journal-mcp --project-dir /tmp/foo` parses and + // populates the field. We do not actually launch the server here — + // that needs a real stdio peer. + let cli = Cli::try_parse_from(["task-journal-mcp", "--project-dir", "/tmp/foo"]).unwrap(); + assert_eq!(cli.project_dir, Some(std::path::PathBuf::from("/tmp/foo"))); + + let cli = Cli::try_parse_from(["task-journal-mcp"]).unwrap(); + assert!(cli.project_dir.is_none()); + } + #[test] fn into_mcp_error_carries_full_anyhow_chain() { // Down-stream callers rely on McpError.message containing the full From 571aebe8fac98ccc060d410520dac89074bd9bc4 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 19:06:45 +0400 Subject: [PATCH 20/39] perf(mcp): wrap blocking I/O in tokio::task::spawn_blocking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tokio runtime hosts a small thread pool sized to the number of CPU cores. Synchronous SQLite + JSONL + filesystem work directly in async fn handlers monopolised that thread for the duration of each tool call, so two concurrent client requests serialised even on a multicore box. Now: every tool body is moved into a closure passed to tokio::task::spawn_blocking via a small run_blocking() helper that also collapses JoinError + anyhow::Error into McpError. Inside the closure we still own + open + drop SQLite connections normally — crucially never holding a Connection across an await, since rusqlite::Connection is Send but not Sync. The classifier-aware tools never directly call HTTP from the MCP server (only the CLI does), so the synchronous ureq stays on the blocking pool for free. Test run_blocking_executes_two_tasks_concurrently: tokio::join! two 200ms sleep_in_blocking calls and assert wall clock < 350ms. Refs claude-memory-gyq.7 --- crates/tj-mcp/src/main.rs | 96 ++++++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 27 deletions(-) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index c211647..4187e9b 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -39,6 +39,21 @@ fn into_mcp_error(err: anyhow::Error) -> McpError { McpError::internal_error(format!("{err:#}"), None) } +/// Run synchronous I/O on the tokio blocking pool. Without this, every tool +/// handler would do SQLite + JSONL work directly on the executor thread +/// and a slow operation in one tool would stall every other concurrent +/// request — defeats the point of using an async runtime at all. +async fn run_blocking(f: F) -> Result +where + F: FnOnce() -> anyhow::Result + Send + 'static, + T: Send + 'static, +{ + let join_result = tokio::task::spawn_blocking(f) + .await + .map_err(|e| McpError::internal_error(format!("blocking task panicked: {e}"), None))?; + join_result.map_err(into_mcp_error) +} + /// MCP instructions delivered to every Claude Code session where this plugin is installed. /// This is the primary mechanism for self-contained plugin behavior — no manual CLAUDE.md edits needed. const MCP_INSTRUCTIONS: &str = r#"Task Journal — reasoning chain memory for AI coding sessions. @@ -190,7 +205,7 @@ impl TaskJournalServer { &self, Parameters(p): Parameters, ) -> Result, McpError> { - let result: anyhow::Result = (|| { + run_blocking(move || { let (project_hash, events_path, state_path) = project_paths()?; let conn = tj_core::db::open(&state_path)?; if events_path.exists() { @@ -214,8 +229,9 @@ impl TaskJournalServer { cache_hit: Some(pack.metadata.cache_hit), }, }) - })(); - result.map(Json).map_err(into_mcp_error) + }) + .await + .map(Json) } #[tool( @@ -226,7 +242,8 @@ impl TaskJournalServer { &self, Parameters(p): Parameters, ) -> Result, McpError> { - let ids: anyhow::Result> = (|| { + let query = p.query.clone(); + let results = run_blocking(move || { let (project_hash, events_path, state_path) = project_paths()?; let conn = tj_core::db::open(&state_path)?; if events_path.exists() { @@ -239,14 +256,9 @@ impl TaskJournalServer { .query_map(rusqlite::params![p.query], |r| r.get::<_, String>(0))? .collect::>()?; Ok(ids) - })(); - ids.map(|results| { - Json(TaskSearchResult { - query: p.query, - results, - }) }) - .map_err(into_mcp_error) + .await?; + Ok(Json(TaskSearchResult { query, results })) } #[tool( @@ -257,7 +269,7 @@ impl TaskJournalServer { &self, Parameters(p): Parameters, ) -> Result, McpError> { - let result: anyhow::Result = (|| { + run_blocking(move || { let (_, events_path, _) = project_paths()?; std::fs::create_dir_all(events_path.parent().unwrap())?; @@ -279,8 +291,9 @@ impl TaskJournalServer { task_id, title: p.title.clone(), }) - })(); - result.map(Json).map_err(into_mcp_error) + }) + .await + .map(Json) } #[tool( @@ -291,7 +304,7 @@ impl TaskJournalServer { &self, Parameters(p): Parameters, ) -> Result, McpError> { - let result: anyhow::Result = (|| { + run_blocking(move || { let (_, events_path, _) = project_paths()?; std::fs::create_dir_all(events_path.parent().unwrap())?; @@ -315,8 +328,9 @@ impl TaskJournalServer { task_id: p.task_id.clone(), event_type: p.event_type.clone(), }) - })(); - result.map(Json).map_err(into_mcp_error) + }) + .await + .map(Json) } #[tool( @@ -327,7 +341,8 @@ impl TaskJournalServer { &self, Parameters(p): Parameters, ) -> Result, McpError> { - let result: anyhow::Result<()> = (|| { + let task_id = p.task_id.clone(); + run_blocking(move || { let (project_hash, events_path, state_path) = project_paths()?; let conn = tj_core::db::open(&state_path)?; @@ -357,15 +372,12 @@ impl TaskJournalServer { writer.append(&event)?; writer.flush_durable()?; Ok(()) - })(); - result - .map(|()| { - Json(TaskCloseResult { - task_id: p.task_id.clone(), - closed: true, - }) - }) - .map_err(into_mcp_error) + }) + .await?; + Ok(Json(TaskCloseResult { + task_id, + closed: true, + })) } } @@ -482,6 +494,36 @@ mod tests { assert_eq!(hash_a, hash_a_again); } + #[tokio::test] + async fn run_blocking_executes_two_tasks_concurrently() { + use std::time::{Duration, Instant}; + + // Two tasks each sleep ~200ms. If run_blocking handed work to the + // tokio blocking pool they overlap (~200ms wall-clock). If we ever + // regress to running the closure inline on the executor thread, + // tokio::join! still wakes both futures but only one progresses at + // a time and total wall-clock approaches 400ms. + let start = Instant::now(); + let (a, b) = tokio::join!( + run_blocking(|| { + std::thread::sleep(Duration::from_millis(200)); + Ok::<_, anyhow::Error>(1u32) + }), + run_blocking(|| { + std::thread::sleep(Duration::from_millis(200)); + Ok::<_, anyhow::Error>(2u32) + }), + ); + let elapsed = start.elapsed(); + + assert_eq!(a.unwrap(), 1); + assert_eq!(b.unwrap(), 2); + assert!( + elapsed < Duration::from_millis(350), + "blocking tasks must overlap on the blocking pool — got {elapsed:?}" + ); + } + #[test] fn cli_parses_project_dir_argument() { // Smoke test: `task-journal-mcp --project-dir /tmp/foo` parses and From c2d639252b64b1fa35138f7072afa43405c05ed4 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 19:09:52 +0400 Subject: [PATCH 21/39] perf(bench): criterion benches for rebuild_state, pack assemble, FTS search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We claim B2 made hot paths O(new) instead of O(all), but every claim without a number is a wish. Adds a criterion harness that exercises the three paths the MCP server walks on every tool call. Three benches, two sizes each (1k and 10k events spread across 100 synthetic tasks): - rebuild_state — full-rebuild baseline (the cost we used to pay on every MCP call before B2) - pack_assemble_cold — invalidates cache then recomputes - search_fts — FTS5 MATCH lookup Wired into CI as a separate benches-compile job that runs cargo bench --no-run; full timing runs are best done locally on a quiet box, not on shared GitHub runners. Threshold gates (B2 promised <50ms pack / <100ms rebuild on 10k) are deferred until a real CI box exists or five baselines are collected. Refs claude-memory-gyq.8 --- .github/workflows/ci.yml | 12 ++ Cargo.lock | 197 +++++++++++++++++++++++++++- Cargo.toml | 1 + crates/tj-core/Cargo.toml | 5 + crates/tj-core/benches/hot_paths.rs | 115 ++++++++++++++++ 5 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 crates/tj-core/benches/hot_paths.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 671a553..fbd8957 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,3 +75,15 @@ jobs: - uses: rustsec/audit-check@v2.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} + + benches-compile: + name: criterion benches (compile only) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + key: benches + - name: cargo bench --no-run + run: cargo bench --workspace --no-run diff --git a/Cargo.lock b/Cargo.lock index d93ddd1..bb17fe3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,6 +44,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -202,6 +208,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "castaway" version = "0.2.4" @@ -241,6 +253,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.1" @@ -334,6 +373,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -384,6 +459,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -803,6 +884,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -844,6 +936,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "http" version = "1.4.0" @@ -1113,12 +1211,32 @@ dependencies = [ "syn", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -1324,6 +1442,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "option-ext" version = "0.2.0" @@ -1377,6 +1501,34 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -1506,7 +1658,7 @@ dependencies = [ "crossterm", "indoc", "instability", - "itertools", + "itertools 0.13.0", "lru", "paste", "strum", @@ -1515,6 +1667,26 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2017,6 +2189,7 @@ version = "0.1.3" dependencies = [ "anyhow", "chrono", + "criterion", "directories", "dunce", "fd-lock", @@ -2130,6 +2303,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.52.1" @@ -2266,7 +2449,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" dependencies = [ - "itertools", + "itertools 0.13.0", "unicode-segmentation", "unicode-width 0.1.14", ] @@ -2477,6 +2660,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.97" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 41cc949..0ebeb34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,3 +45,4 @@ assert_fs = "1" predicates = "3" tempfile = "3" mockito = "1" +criterion = "0.5" diff --git a/crates/tj-core/Cargo.toml b/crates/tj-core/Cargo.toml index 8fbe037..c11030d 100644 --- a/crates/tj-core/Cargo.toml +++ b/crates/tj-core/Cargo.toml @@ -34,3 +34,8 @@ fd-lock = { workspace = true } [dev-dependencies] tempfile = { workspace = true } mockito = { workspace = true } +criterion = { workspace = true } + +[[bench]] +name = "hot_paths" +harness = false diff --git a/crates/tj-core/benches/hot_paths.rs b/crates/tj-core/benches/hot_paths.rs new file mode 100644 index 0000000..b6d35bf --- /dev/null +++ b/crates/tj-core/benches/hot_paths.rs @@ -0,0 +1,115 @@ +//! Criterion benchmarks for the hottest paths the MCP server walks every +//! tool call: rebuild_state, ingest_new_events, pack::assemble, FTS search. +//! +//! These exist to (a) put numbers on the B2 incremental-indexing win and +//! (b) catch regressions before they ship. CI runs `cargo bench --no-run` +//! so the harness must compile; full runs happen locally on a quiet box +//! or in a dedicated bench job. + +use std::io::Write; + +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use tempfile::TempDir; +use tj_core::{ + db, + event::{Author, Event, EventType, Source}, + pack, +}; + +const PROJECT_HASH: &str = "deadbeefdeadbeef"; + +/// Materialize an N-event JSONL file spread across 100 distinct tasks. +fn synthetic_jsonl(n: usize) -> (TempDir, std::path::PathBuf, std::path::PathBuf) { + let dir = TempDir::new().unwrap(); + let jsonl = dir.path().join("events.jsonl"); + let sqlite = dir.path().join("s.sqlite"); + let mut f = std::fs::File::create(&jsonl).unwrap(); + for i in 0..n { + let task_id = format!("tj-b{:03}", i % 100); + let kind = match i % 4 { + 0 => EventType::Open, + 1 => EventType::Decision, + 2 => EventType::Finding, + _ => EventType::Evidence, + }; + let mut e = Event::new( + &task_id, + kind, + Author::User, + Source::Cli, + format!("event {i} for task {task_id}"), + ); + if matches!(kind, EventType::Open) { + e.meta = serde_json::json!({"title": format!("Task {}", i % 100)}); + } + writeln!(f, "{}", serde_json::to_string(&e).unwrap()).unwrap(); + } + drop(f); + (dir, jsonl, sqlite) +} + +fn bench_rebuild_state(c: &mut Criterion) { + let mut group = c.benchmark_group("rebuild_state"); + for &n in &[1_000usize, 10_000] { + group.bench_function(format!("{n}_events"), |b| { + b.iter_batched( + || synthetic_jsonl(n), + |(_dir, jsonl, sqlite)| { + let conn = db::open(&sqlite).unwrap(); + db::rebuild_state(&conn, &jsonl, PROJECT_HASH).unwrap(); + }, + BatchSize::PerIteration, + ); + }); + } + group.finish(); +} + +fn bench_pack_assemble_cold(c: &mut Criterion) { + let mut group = c.benchmark_group("pack_assemble_cold"); + for &n in &[1_000usize, 10_000] { + let (_dir, jsonl, sqlite) = synthetic_jsonl(n); + let conn = db::open(&sqlite).unwrap(); + db::rebuild_state(&conn, &jsonl, PROJECT_HASH).unwrap(); + group.bench_function(format!("{n}_events"), |b| { + b.iter(|| { + // Invalidate the cache so each iteration is a cold compute. + conn.execute("DELETE FROM task_pack_cache", []).unwrap(); + pack::assemble(&conn, "tj-b000", pack::PackMode::Compact).unwrap(); + }); + }); + } + group.finish(); +} + +fn bench_search_fts(c: &mut Criterion) { + let mut group = c.benchmark_group("search_fts"); + for &n in &[1_000usize, 10_000] { + let (_dir, jsonl, sqlite) = synthetic_jsonl(n); + let conn = db::open(&sqlite).unwrap(); + db::rebuild_state(&conn, &jsonl, PROJECT_HASH).unwrap(); + group.bench_function(format!("{n}_events"), |b| { + b.iter(|| { + let mut stmt = conn + .prepare( + "SELECT DISTINCT task_id FROM search_fts WHERE search_fts MATCH ?1 LIMIT 50", + ) + .unwrap(); + let _: Vec = stmt + .query_map(rusqlite::params!["event"], |r| r.get::<_, String>(0)) + .unwrap() + .collect::>() + .unwrap(); + }); + }); + } + group.finish(); +} + +criterion_group!( + benches, + bench_rebuild_state, + bench_pack_assemble_cold, + bench_search_fts +); +criterion_main!(benches); From 0a6bf5c7cd3bc29bc780bd94e5bb0761c0129d2b Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Wed, 6 May 2026 19:12:32 +0400 Subject: [PATCH 22/39] release: bump workspace version to 0.2.0-rc.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last commit of epic B. Workspace version 0.1.3 -> 0.2.0-rc.1. Inner crate dependency declarations updated to match (tj-cli and tj-mcp both depend on tj-core). CHANGELOG.md gets a [0.2.0-rc.1] - 2026-05-06 section with the breaking change (MCP error contract) called out first, then Added / Changed / Performance subsections summarising the eight feature commits in this epic. After dogfooding, 0.2.0 will be cut without further code changes — the rc tag is the gating signal that we want feedback on the new contract before it hits stable. Closes claude-memory-gyq.9 --- CHANGELOG.md | 68 +++++++++++++++++++++++++++++++++++++++- Cargo.lock | 6 ++-- Cargo.toml | 2 +- crates/tj-cli/Cargo.toml | 2 +- crates/tj-mcp/Cargo.toml | 2 +- 5 files changed, 73 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42b6104..472d7b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,71 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.0-rc.1] - 2026-05-06 + +> **Release candidate.** Major version bump because the MCP error +> contract changed shape (see _BREAKING_ below). After dogfooding +> for a week the matching `0.2.0` will be cut without further code +> changes. + +### BREAKING + +- **MCP error contract.** Tool handlers (`task_pack`, `task_search`, + `task_create`, `event_add`, `task_close`) no longer mask failures + as success-typed JSON with `task_id = "[error] msg"`. They now + return JSON-RPC error frames (rmcp `ErrorData`) carrying the full + `anyhow` chain in the `message` field. Any client that was parsing + `"[error]"` out of the result must switch to detecting the rpc + error envelope first. + +### Added +- `tj_core::db::ingest_new_events` — incremental indexing that reads + only the JSONL tail since the last marker. Two safe fallbacks to + full `rebuild_state`: no marker yet, or marker missing in file. +- `tj_core::db::task_exists` — O(1) lookup against `tasks` PK. +- Migration v002: `index_state(project_hash, last_indexed_event_id, + updated_at)` table, plus a forward-only migrations registry tracked + in `schema_migrations(version, applied_at)`. +- MCP `--project-dir ` argument — overrides the cwd-derived + project hash. Path is canonicalized at startup. +- `criterion` benchmarks for `rebuild_state`, `pack_assemble_cold`, + and FTS `search` at 1k and 10k events. CI `benches-compile` job + guards the harness. +- New regression tests: + `fresh_db_runs_all_migrations`, `apply_migrations_is_idempotent_ + across_reopens`, `task_exists_returns_true_for_known_id_false_ + otherwise`, `ingest_new_events_picks_up_only_new_lines`, + `ingest_new_events_falls_back_to_full_rebuild_when_marker_vanishes`, + `rebuild_state_and_ingest_new_events_produce_same_state`, + `pack_cache_hits_after_incremental_ingest_with_no_new_events`, + `into_mcp_error_carries_full_anyhow_chain`, + `resolve_project_paths_uses_provided_dir_for_hash`, + `cli_parses_project_dir_argument`, + `run_blocking_executes_two_tasks_concurrently`, + `close_unknown_task_id_returns_error` (CLI integration). + +### Changed +- Every MCP tool handler now offloads its synchronous I/O to the + tokio blocking pool via `tokio::task::spawn_blocking`. Concurrent + client requests no longer serialise behind one slow operation. +- `rebuild_state` writes the `last_indexed_event_id` marker on + completion so subsequent `ingest_new_events` calls can pick up + from the tail. +- CLI `Close` and MCP `task_close` validate that `task_id` exists + in the `tasks` table before appending a close event. Closing an + unknown id used to silently succeed; now it returns an error + (CLI: non-zero exit + stderr; MCP: rpc error frame). +- Workspace version `0.1.3` → `0.2.0-rc.1`. + +### Performance +- `task_pack`, `task_search`, and the auto-capture hook used to + re-read the entire JSONL log on every invocation through + `rebuild_state`. They now use `ingest_new_events` and only + process events newer than the last marker. The pack-cache, which + was wiped on every `index_event` call during full rebuild, is now + reused naturally — a no-op ingest yields `cache_hit: true` on the + next `assemble`. + ## [0.1.4] - 2026-05-06 Backwards-compatible hardening release. No breaking changes to the CLI flags @@ -128,7 +193,8 @@ Initial release on crates.io. - `task-journal-mcp`: MCP server exposing `task_create`, `event_add`, `task_pack`, `task_search`, `task_close`. -[Unreleased]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.4...HEAD +[Unreleased]: https://github.com/Digital-Threads/Task-Journal/compare/v0.2.0-rc.1...HEAD +[0.2.0-rc.1]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.4...v0.2.0-rc.1 [0.1.4]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.3...v0.1.4 [0.1.3]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.2...v0.1.3 [0.1.2]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.1...v0.1.2 diff --git a/Cargo.lock b/Cargo.lock index bb17fe3..8a6506b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2165,7 +2165,7 @@ dependencies = [ [[package]] name = "task-journal-cli" -version = "0.1.3" +version = "0.2.0-rc.1" dependencies = [ "anyhow", "assert_cmd", @@ -2185,7 +2185,7 @@ dependencies = [ [[package]] name = "task-journal-core" -version = "0.1.3" +version = "0.2.0-rc.1" dependencies = [ "anyhow", "chrono", @@ -2208,7 +2208,7 @@ dependencies = [ [[package]] name = "task-journal-mcp" -version = "0.1.3" +version = "0.2.0-rc.1" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index 0ebeb34..6737a28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [workspace.package] -version = "0.1.3" +version = "0.2.0-rc.1" edition = "2021" rust-version = "1.83" license = "MIT" diff --git a/crates/tj-cli/Cargo.toml b/crates/tj-cli/Cargo.toml index dea04ae..3616a9e 100644 --- a/crates/tj-cli/Cargo.toml +++ b/crates/tj-cli/Cargo.toml @@ -16,7 +16,7 @@ name = "task-journal" path = "src/main.rs" [dependencies] -tj-core = { package = "task-journal-core", version = "0.1.3", path = "../tj-core" } +tj-core = { package = "task-journal-core", version = "0.2.0-rc.1", path = "../tj-core" } anyhow = { workspace = true } clap = { workspace = true } tracing = { workspace = true } diff --git a/crates/tj-mcp/Cargo.toml b/crates/tj-mcp/Cargo.toml index c640d8a..97c2348 100644 --- a/crates/tj-mcp/Cargo.toml +++ b/crates/tj-mcp/Cargo.toml @@ -16,7 +16,7 @@ name = "task-journal-mcp" path = "src/main.rs" [dependencies] -tj-core = { package = "task-journal-core", version = "0.1.2", path = "../tj-core" } +tj-core = { package = "task-journal-core", version = "0.2.0-rc.1", path = "../tj-core" } anyhow = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } From eb1e8b8600cc76a36a8e6de36e52624c62925818 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 09:52:19 +0400 Subject: [PATCH 23/39] =?UTF-8?q?chore:=20OSS=20hygiene=20=E2=80=94=20CONT?= =?UTF-8?q?RIBUTING,=20CoC,=20issue=20and=20PR=20templates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standard scaffolding so new contributors find the rules without asking. Five files: - CONTRIBUTING.md (one-thing-per-PR, conventional commits, CI gate expectations, what I will not merge) - CODE_OF_CONDUCT.md (Contributor Covenant 2.1 reference) - .github/ISSUE_TEMPLATE/bug.md, feature.md, question.md - .github/PULL_REQUEST_TEMPLATE.md (matches CONTRIBUTING checklist) Plan landed in .docs/plans/2026-05-06-v0.2.0-epic-c-quality.md (epic C scope) — committed in the same change because it covers all eight C sub-tasks rather than just this one. README links to CONTRIBUTING / CoC / issue templates from a new Contributing section. Refs claude-memory-1yc.1 --- .../plans/2026-05-06-v0.2.0-epic-c-quality.md | 84 +++++++++++++++++++ .github/ISSUE_TEMPLATE/bug.md | 36 ++++++++ .github/ISSUE_TEMPLATE/feature.md | 28 +++++++ .github/ISSUE_TEMPLATE/question.md | 23 +++++ .github/PULL_REQUEST_TEMPLATE.md | 28 +++++++ CODE_OF_CONDUCT.md | 30 +++++++ CONTRIBUTING.md | 61 ++++++++++++++ README.md | 7 ++ 8 files changed, 297 insertions(+) create mode 100644 .docs/plans/2026-05-06-v0.2.0-epic-c-quality.md create mode 100644 .github/ISSUE_TEMPLATE/bug.md create mode 100644 .github/ISSUE_TEMPLATE/feature.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md diff --git a/.docs/plans/2026-05-06-v0.2.0-epic-c-quality.md b/.docs/plans/2026-05-06-v0.2.0-epic-c-quality.md new file mode 100644 index 0000000..863fa4b --- /dev/null +++ b/.docs/plans/2026-05-06-v0.2.0-epic-c-quality.md @@ -0,0 +1,84 @@ +# Epic C — v0.2.0: quality, DX, community polish + +**Date:** 2026-05-06 +**Branch:** `claude/v0.2.0-epic-c` (off `claude/v0.2.0-epic-b` HEAD) +**Target release:** `0.2.0` final (after epic B's `rc.1` is dogfooded and this PR merges) +**Bd epic:** `claude-memory-1yc` + +## Goal + +Three thematic threads, deliberately bundled because none alone deserves a major version but together they raise the project from "works" to "feels finished": + +1. **Classifier quality** — make the auto-capture hook actually trust-worthy with few-shot prompting and a regression-gated accuracy floor. +2. **User-facing DX** — `doctor` (diagnostic), `migrate-project` (path moved), HTML timeline (PR review). +3. **Community / coverage / Windows** — OSS hygiene files, llvm-cov badge, Windows test parity for the CLI classifier. + +## Success criteria + +1. `cargo test --workspace --all-targets` green on three OS — including the previously-skipped `cfg(unix)`-only classifier tests. +2. New `tests/classifier_eval.rs` runs against a checked-in labeled dataset and enforces an accuracy floor; CI fails when the floor is broken. +3. `task-journal doctor` exits 0 on a healthy install and emits a machine-readable summary that flags missing `claude` CLI / unwritable data dirs / unknown migrations. +4. `task-journal migrate-project --from --to ` re-keys the JSONL + SQLite + metrics for the new project hash; round-trips through `task_pack`. +5. `task-journal export --format html --task ` emits a self-contained HTML timeline. +6. Coverage report: `cargo llvm-cov --workspace` runs in CI and uploads to Codecov; README badge reflects the status. +7. `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`, `.github/ISSUE_TEMPLATE/*`, `.github/PULL_REQUEST_TEMPLATE.md` exist and link from README. +8. PR opened against `main` (after `0.2.0-rc.1` is in main). + +## Non-goals (deferred) + +- Opt-in telemetry endpoint (requires hosted backend — separate decision). +- C/C++/server-side LSP integration. +- Multi-language classifier prompts. + +## Tasks (8) + +| # | Task | Touches | Test? | Notes | +|---|------|---------|-------|-------| +| C1 | OSS hygiene files: `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md`, issue + PR templates | repo root + `.github/` | n/a | Standard OSS scaffolding; not blocking other work. | +| C2 | `cargo-llvm-cov` job in CI + Codecov upload + README badge | `.github/workflows/ci.yml`, `README.md` | n/a | Non-blocking initially; flip threshold to blocking after 5 baselines. | +| C3 | Windows-compatible tests for `ClaudeCliClassifier` (currently `cfg(all(test, unix))`) | `crates/tj-core/src/classifier/cli.rs` | yes (port the two existing fake-claude tests to use `.cmd`/`.bat` shim on Windows) | Closes the platform gap noticed in the audit. | +| C4 | `task-journal doctor` command | `tj-cli/src/main.rs`, possibly small `tj-core::diagnostics` mod | yes (CLI integration test) | Checks: claude bin in PATH, data dirs writable, schema_migrations matches expected, last_indexed_event_id consistent. | +| C5 | `task-journal migrate-project --from --to ` | `tj-cli/src/main.rs`, `tj-core::project_hash`, fs ops | yes | Renames `.jsonl`, `.sqlite`, `.jsonl` in metrics, etc. | +| C6 | `task-journal export --format html [--task ]` | `tj-cli/src/main.rs` (existing `export` command), new tiny `html_timeline` helper | yes | Self-contained: inline CSS, no external assets. | +| C7 | Few-shot prompting in classifier | `tj-core/src/classifier/prompt.rs` | yes (prompt contains 6 examples; size still bounded < 64KB) | 2 examples per harder pair: hypothesis vs finding, finding vs evidence, decision vs hypothesis. | +| C8 | Classifier eval dataset + accuracy gate | `tj-core/tests/classifier_eval.rs`, `tj-core/tests/fixtures/classifier_eval.jsonl` | yes (eval test enforces ≥ 70% baseline) | Hand-label ~30 chunks; uses `MockClassifier` + golden expected outputs to keep deterministic; real-classifier path stays opt-in via env var so CI does not need API access. | + +## Sequencing + +``` +C1 ─┐ +C2 ─┤ +C3 ─┼─→ (independent) +C4 ─┤ +C5 ─┤ +C6 ─┘ + +C7 ─→ C8 (eval validates the new prompt against the dataset) +``` + +C1/C2/C3 can land in any order. C4/C5/C6 are independent CLI features. C7 unlocks C8 (the eval dataset is the way to *measure* that few-shot improved precision rather than degraded it). + +## Risks + +- **C7 prompt regression:** few-shot can over-fit examples and degrade on out-of-distribution chunks. Mitigation: eval set in C8 covers boundary cases (`hypothesis-not-finding`, etc). +- **C8 false confidence:** ≥70% on 30 examples is a noisy estimate. Mitigation: ratchet floor up only after collecting 100+ labeled examples in dogfooding. +- **C5 destructive migration:** if `--from` and `--to` resolve to the same hash (symlink, case-insensitive FS), we'd corrupt data. Mitigation: refuse when `from_hash == to_hash`; require `--force` to overwrite an existing destination. +- **C3 Windows shim:** rewriting the fake-claude test in PowerShell vs `.cmd` vs Python — pick `.cmd` for minimal surface; some Windows tests skip on lack of `cmd.exe` is acceptable. + +## Verification gate (per task) + +Same as Epic A/B: +1. `cargo fmt --all -- --check` +2. `cargo clippy --workspace --all-targets -- -D warnings` +3. `cargo test --workspace --all-targets` +4. `git diff --stat` review +5. Conventional-commit prefix +6. `bd close --reason "..."` + +## Final verification (epic-level) + +- All 8 sub-tasks closed in bd +- `cargo bench --workspace --no-run` clean +- `cargo llvm-cov --workspace --summary-only` reports a number +- `task-journal doctor` runs locally and prints the diagnostics +- PR body lists which features changed user-facing CLI surface diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md new file mode 100644 index 0000000..c31df18 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -0,0 +1,36 @@ +--- +name: Bug report +about: Something is broken or behaves unexpectedly +title: "" +labels: bug +--- + +## What happened + + + +## How to reproduce + +```bash +# The exact command(s) you ran or MCP tool call(s). +``` + +## Expected vs. actual + +- Expected: +- Actual: + +## Environment + +- `task-journal --version`: +- `rustc --version`: +- OS / WSL distribution: +- Output of `task-journal doctor --json` (if relevant): + +```text + +``` + +## Anything else + + diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md new file mode 100644 index 0000000..b427642 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.md @@ -0,0 +1,28 @@ +--- +name: Feature request +about: Suggest a new capability or improvement +title: "" +labels: feature +--- + +## What problem are you trying to solve + + + +## Proposal + + + +## Alternatives considered + + + +## Scope check + +- [ ] This stays in the project's stated scope ("reasoning-chain memory + for AI coding sessions"). If you're not sure, that's fine — the + maintainer will discuss it on the issue. + +## Anything else + + diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..0613bb6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,23 @@ +--- +name: Question +about: How do I do X with task-journal? +title: "" +labels: question +--- + +## What are you trying to do + + + +## What you've tried + + + +## Where you got stuck + + + +## Environment + +- `task-journal --version`: +- OS / WSL distribution: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..6db9be5 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,28 @@ +## Summary + + + +## Type of change + +- [ ] Bug fix (non-breaking) +- [ ] Feature (non-breaking) +- [ ] Breaking change (CLI flag, MCP tool shape, on-disk format) +- [ ] Refactor / chore / docs / CI only + +## Test plan + +- [ ] Failing test added before the fix (bug) or describing the new + behavior (feature) +- [ ] `cargo fmt --all -- --check` +- [ ] `cargo clippy --workspace --all-targets -- -D warnings` +- [ ] `cargo test --workspace --all-targets` +- [ ] `cargo doc --workspace --no-deps` with `RUSTDOCFLAGS=-D warnings` + +## CHANGELOG + +- [ ] Added an entry under `## [Unreleased]` (Added / Changed / Removed + / Fixed / BREAKING) — or this PR doesn't affect users. + +## Related issues + + diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..765ffd4 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,30 @@ +# Code of Conduct + +This project follows the [Contributor Covenant v2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). + +## Summary + +In short: be respectful, focus on the work, assume good faith. Personal +attacks, sustained disruption, and harassment are not welcome here. + +## Scope + +This Code of Conduct applies to all project spaces — issues, pull +requests, discussions, the codebase itself (commit messages, code +comments), and any direct communication that references the project. + +## Reporting + +If you experience or witness behavior that violates this Code, please +report it to the maintainer at **shahinyanm@gmail.com**. Reports are +handled confidentially. + +The maintainer will review the report, ask follow-up questions if +needed, and decide on a response. Possible responses include private +clarification, a public warning, a temporary suspension from project +spaces, or a permanent ban — proportional to the violation. + +## Attribution + +The full text and enforcement guidelines are at +[contributor-covenant.org/version/2/1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..936589c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,61 @@ +# Contributing to Task Journal + +Thanks for your interest. Task Journal is a small Rust workspace; the +contribution loop is short by design. + +## Before you start + +- Read [README.md](README.md) for what the project is for. +- Read [CHANGELOG.md](CHANGELOG.md) for the current direction. +- Search [open issues](https://github.com/Digital-Threads/Task-Journal/issues) + before filing a duplicate. + +## Development setup + +```bash +git clone https://github.com/Digital-Threads/Task-Journal +cd Task-Journal +cargo test --workspace +``` + +Minimum supported Rust version: see `rust-version` in [Cargo.toml](Cargo.toml). + +## What I look for in a PR + +1. **One thing per PR.** Bug fix or feature, not both. Refactors get their + own PR. If you find a side issue while working, open a separate issue. +2. **A failing test before the fix.** For bugs, the test should reproduce + the bug at HEAD (red) and pass with your change (green). For features, + the test should describe the new behavior. +3. **Conventional commit prefix.** `fix:` / `feat:` / `chore:` / `docs:` / + `perf:` / `refactor:` / `test:` / `ci:`. Add `!` for breaking changes. +4. **CI green.** That means `cargo fmt --all -- --check`, + `cargo clippy --workspace --all-targets -- -D warnings`, + `cargo test --workspace --all-targets`, `cargo doc --workspace --no-deps` + with `RUSTDOCFLAGS=-D warnings`. +5. **CHANGELOG entry** if your change affects users (CLI flag, MCP tool, + on-disk format, public API). One line under the relevant `## [Unreleased]` + subsection (Added / Changed / Removed / Fixed / BREAKING). + +## What I won't merge + +- Cosmetic-only refactors of code that's been stable and tested. +- New abstractions without a second concrete user. +- Features that move the project away from "reasoning-chain memory for + AI coding sessions" — please open an issue first to discuss scope. + +## Reporting bugs + +Use the bug template under [`.github/ISSUE_TEMPLATE/`](.github/ISSUE_TEMPLATE/). +The most useful bug reports include: + +- The exact command you ran (or MCP tool call). +- The output you got vs. what you expected. +- `task-journal --version` and `rustc --version`. +- The contents of `task-journal doctor --json` if the bug looks like an + installation/environment problem. + +## License + +By contributing you agree your work is licensed under the MIT License +(see [LICENSE](LICENSE)). diff --git a/README.md b/README.md index 44b6eaa..da428dd 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,13 @@ Smoke test scripts are available in `.beads/hooks/`: See [CHANGELOG.md](CHANGELOG.md) for release notes. +## Contributing + +Pull requests are welcome — please read [CONTRIBUTING.md](CONTRIBUTING.md) +first. Filing bugs and feature requests goes through the +[issue templates](.github/ISSUE_TEMPLATE/). All participation is governed +by the [Code of Conduct](CODE_OF_CONDUCT.md). + ## License MIT From 7acf918a12f0ba0e762f4b6bd8df462447978f70 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 09:53:19 +0400 Subject: [PATCH 24/39] ci: cargo-llvm-cov coverage job + Codecov upload + README badge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a coverage workflow job that runs cargo llvm-cov --workspace --lcov, then uploads via codecov-action@v4. Marked continue-on-error: true on first land — once we collect 5 baselines and agree a floor the gate flips to blocking. CODECOV_TOKEN is read from GitHub secrets if present; for public repos Codecov v4 falls back to anonymous uploads, so the job is useful even before the secret is configured. README gets the codecov badge alongside the existing crates.io / CI / License badges. Refs claude-memory-1yc.2 --- .github/workflows/ci.yml | 26 ++++++++++++++++++++++++++ README.md | 1 + 2 files changed, 27 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbd8957..9c32ca1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,3 +87,29 @@ jobs: key: benches - name: cargo bench --no-run run: cargo bench --workspace --no-run + + coverage: + name: coverage (llvm-cov) + runs-on: ubuntu-latest + # Non-blocking on first land — coverage gates make sense once we have + # 5+ baselines and a target floor agreed. Flip continue-on-error to + # false once that lands. + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview + - uses: taiki-e/install-action@cargo-llvm-cov + - uses: Swatinem/rust-cache@v2 + with: + key: coverage + - name: cargo llvm-cov (lcov) + run: cargo llvm-cov --workspace --lcov --output-path lcov.info + - name: Upload to Codecov + uses: codecov/codecov-action@v4 + with: + files: lcov.info + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/README.md b/README.md index da428dd..46c6efa 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Crates.io](https://img.shields.io/crates/v/task-journal-cli.svg)](https://crates.io/crates/task-journal-cli) [![CI](https://github.com/Digital-Threads/Task-Journal/workflows/CI/badge.svg)](https://github.com/Digital-Threads/Task-Journal/actions/workflows/ci.yml) +[![codecov](https://codecov.io/gh/Digital-Threads/Task-Journal/branch/main/graph/badge.svg)](https://codecov.io/gh/Digital-Threads/Task-Journal) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) Reasoning chain memory for AI coding sessions. From 78f1017922b633ebb9433c703b22e96c9ece80ae Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 09:55:48 +0400 Subject: [PATCH 25/39] test(classifier): cross-platform fake-claude shim, drop cfg(unix) gate The two ClaudeCliClassifier tests were gated cfg(all(test, unix)) and silently skipped on Windows CI. Closes that platform gap. The shim now writes the JSON envelope to a file and executes a tiny script that prints it back: cat "PATH" on Unix (.sh + chmod 0755), type "PATH" on Windows (.cmd batch). The type/cat form avoids the notoriously fragile cmd-batch escaping of the envelope JSON. Result: classifier_parses_cli_envelope_and_returns_classified_output and classifier_surfaces_not_logged_in_with_friendly_hint now run on all three matrix OS in CI. Refs claude-memory-1yc.3 --- crates/tj-core/src/classifier/cli.rs | 44 ++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/crates/tj-core/src/classifier/cli.rs b/crates/tj-core/src/classifier/cli.rs index 24ebc89..f06558b 100644 --- a/crates/tj-core/src/classifier/cli.rs +++ b/crates/tj-core/src/classifier/cli.rs @@ -92,24 +92,42 @@ impl Classifier for ClaudeCliClassifier { } } -// Tests use a `#!/bin/bash` shim to fake the `claude` CLI; gating to Unix -// so Windows clippy/build doesn't see the imports/helper as unused. -#[cfg(all(test, unix))] +// Tests use a tiny shell/.cmd shim to fake the `claude` CLI. Cross-platform +// strategy: write the JSON envelope to a file, then a one-liner script that +// `cat`s (Unix) or `type`s (Windows) it back. The `type` form sidesteps cmd +// .exe escaping pain for the JSON payload's quotes. +#[cfg(test)] mod tests { use super::*; use crate::event::EventType; - use std::os::unix::fs::PermissionsExt; - /// Build a fake `claude` script that prints a canned `--output-format json` envelope. - /// Returns the path so we can point ClaudeCliClassifier at it. + /// Build a fake `claude` shim that prints a canned `--output-format json` + /// envelope. Returns the path so we can point ClaudeCliClassifier at it. fn fake_claude(dir: &std::path::Path, envelope: &str) -> std::path::PathBuf { - let path = dir.join("fake-claude"); - let script = format!("#!/bin/bash\ncat <<'EOF'\n{envelope}\nEOF\n"); - std::fs::write(&path, script).unwrap(); - let mut perms = std::fs::metadata(&path).unwrap().permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&path, perms).unwrap(); - path + let json_path = dir.join("fake-claude-output.json"); + std::fs::write(&json_path, envelope).unwrap(); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let path = dir.join("fake-claude.sh"); + let script = format!("#!/bin/sh\ncat \"{}\"\n", json_path.to_string_lossy()); + std::fs::write(&path, script).unwrap(); + let mut perms = std::fs::metadata(&path).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&path, perms).unwrap(); + path + } + #[cfg(windows)] + { + let path = dir.join("fake-claude.cmd"); + // `type "PATH"` outputs file content verbatim; double quotes + // handle spaces, and JSON's special chars stay literal because + // type does not interpret content as commands. + let script = format!("@echo off\r\ntype \"{}\"\r\n", json_path.to_string_lossy()); + std::fs::write(&path, script).unwrap(); + path + } } #[test] From 837daabfdde0a81dac2b29c0250e805fbed77a44 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 10:00:01 +0400 Subject: [PATCH 26/39] feat(cli): task-journal doctor diagnostics command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-check command for users debugging install issues. Reports five groups of facts: 1. claude binary in PATH (with version) — required for the subscription-mode classifier 2. data dir + events/state/metrics sub-dir paths and writability 3. known projects on this machine (count of state-dir SQLite stems) 4. schema migrations applied for the current cwd project (if any) 5. an issues[] list of human-readable problems Exits 0 when issues is empty, 1 otherwise. Default output is human- readable; --json switches to a stable machine-parseable shape. CLI integration tests: - doctor_exits_zero_on_fresh_install (no events/state files yet) - doctor_json_output_is_parseable_and_lists_paths Refs claude-memory-1yc.4 --- Cargo.lock | 1 + crates/tj-cli/Cargo.toml | 1 + crates/tj-cli/src/main.rs | 191 +++++++++++++++++++++++++++++++++++++ crates/tj-cli/tests/cli.rs | 31 ++++++ 4 files changed, 224 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 8a6506b..26f6a2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2176,6 +2176,7 @@ dependencies = [ "predicates", "ratatui", "rusqlite", + "serde", "serde_json", "task-journal-core", "tracing", diff --git a/crates/tj-cli/Cargo.toml b/crates/tj-cli/Cargo.toml index 3616a9e..a20d189 100644 --- a/crates/tj-cli/Cargo.toml +++ b/crates/tj-cli/Cargo.toml @@ -21,6 +21,7 @@ anyhow = { workspace = true } clap = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } ulid = { workspace = true } rusqlite = { workspace = true } diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 95b0318..22639a9 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -1,8 +1,181 @@ use anyhow::Result; use clap::{Parser, Subcommand}; +use serde::Serialize; +use std::path::PathBuf; +use std::process::Command as PCommand; mod tui; +/// Diagnostic snapshot returned by `task-journal doctor`. Fields are +/// stable enough for scripting against `--json`. `issues` is the empty +/// list when everything looks healthy. +#[derive(Serialize)] +struct DoctorReport { + task_journal_version: &'static str, + claude_in_path: bool, + claude_version: Option, + data_dir: PathBuf, + events_dir: PathBuf, + state_dir: PathBuf, + metrics_dir: PathBuf, + events_dir_writable: bool, + state_dir_writable: bool, + metrics_dir_writable: bool, + known_projects: Vec, + schema_versions_applied: Vec, + issues: Vec, +} + +impl DoctorReport { + fn print_human(&self) { + println!("task-journal doctor"); + println!(" version {}", self.task_journal_version); + println!( + " claude binary {}", + if self.claude_in_path { + self.claude_version + .clone() + .unwrap_or_else(|| "found (version unknown)".into()) + } else { + "NOT FOUND in PATH".into() + } + ); + println!(" data dir {}", self.data_dir.display()); + println!( + " events dir {} ({})", + self.events_dir.display(), + if self.events_dir_writable { + "writable" + } else { + "NOT writable" + } + ); + println!( + " state dir {} ({})", + self.state_dir.display(), + if self.state_dir_writable { + "writable" + } else { + "NOT writable" + } + ); + println!( + " metrics dir {} ({})", + self.metrics_dir.display(), + if self.metrics_dir_writable { + "writable" + } else { + "NOT writable" + } + ); + println!(" known projects {}", self.known_projects.len()); + if !self.schema_versions_applied.is_empty() { + let v: Vec = self + .schema_versions_applied + .iter() + .map(|n| format!("v{n:03}")) + .collect(); + println!(" schema (current) {}", v.join(", ")); + } + if self.issues.is_empty() { + println!("\n✓ all checks passed"); + } else { + println!("\n✗ {} issue(s):", self.issues.len()); + for i in &self.issues { + println!(" - {i}"); + } + } + } +} + +fn dir_writable(dir: &std::path::Path) -> bool { + if std::fs::create_dir_all(dir).is_err() { + return false; + } + let probe = dir.join(".tj-doctor-write-probe"); + let r = std::fs::write(&probe, b"ok").is_ok(); + let _ = std::fs::remove_file(&probe); + r +} + +fn run_doctor() -> Result { + let mut issues: Vec = Vec::new(); + + // 1. claude binary in PATH + let claude_check = PCommand::new("claude").arg("--version").output(); + let (claude_in_path, claude_version) = match claude_check { + Ok(out) if out.status.success() => { + let v = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (true, Some(v)) + } + Ok(_) | Err(_) => { + issues.push( + "claude CLI not found on PATH — auto-capture hooks will fall back to API \ + backend (set ANTHROPIC_API_KEY) or fail silently" + .into(), + ); + (false, None) + } + }; + + // 2. data dir + sub-dir writability + let data_dir = tj_core::paths::data_dir()?; + let events_dir = tj_core::paths::events_dir()?; + let state_dir = tj_core::paths::state_dir()?; + let metrics_dir = tj_core::paths::metrics_dir()?; + let events_dir_writable = dir_writable(&events_dir); + let state_dir_writable = dir_writable(&state_dir); + let metrics_dir_writable = dir_writable(&metrics_dir); + if !events_dir_writable { + issues.push(format!("events dir not writable: {}", events_dir.display())); + } + if !state_dir_writable { + issues.push(format!("state dir not writable: {}", state_dir.display())); + } + if !metrics_dir_writable { + issues.push(format!( + "metrics dir not writable: {}", + metrics_dir.display() + )); + } + + // 3. known projects (from state dir SQLite stems) + let known_projects = tj_core::db::list_all_projects(&state_dir).unwrap_or_default(); + + // 4. schema versions for the current cwd's project (if any). + let schema_versions_applied = (|| -> Result> { + let cwd = std::env::current_dir()?; + let project_hash = tj_core::project_hash::from_path(&cwd)?; + let state_path = state_dir.join(format!("{project_hash}.sqlite")); + if !state_path.exists() { + return Ok(Vec::new()); + } + let conn = tj_core::db::open(&state_path)?; + let mut stmt = conn.prepare("SELECT version FROM schema_migrations ORDER BY version")?; + let v: Vec = stmt + .query_map([], |r| r.get::<_, i64>(0))? + .collect::>()?; + Ok(v) + })() + .unwrap_or_default(); + + Ok(DoctorReport { + task_journal_version: env!("CARGO_PKG_VERSION"), + claude_in_path, + claude_version, + data_dir, + events_dir, + state_dir, + metrics_dir, + events_dir_writable, + state_dir_writable, + metrics_dir_writable, + known_projects, + schema_versions_applied, + issues, + }) +} + #[derive(Parser)] #[command(name = "task-journal", version, about = "Task Journal CLI", long_about = None)] struct Cli { @@ -120,6 +293,13 @@ enum Commands { #[arg(long)] project: Option, }, + /// Self-check the install: claude binary, data dirs, known projects, + /// schema migrations. Exits 0 when all checks pass; 1 otherwise. + Doctor { + /// Emit a machine-readable JSON report instead of human text. + #[arg(long)] + json: bool, + }, /// Hook entry point: ingest a chat chunk through the classifier. IngestHook { /// Hook kind: UserPromptSubmit | PostToolUse | Stop | SessionStart. @@ -411,6 +591,17 @@ fn main() -> Result<()> { println!(" confirmed ratio: {ratio:.1}%"); } } + Commands::Doctor { json } => { + let report = run_doctor()?; + if json { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + report.print_human(); + } + if !report.issues.is_empty() { + std::process::exit(1); + } + } Commands::IngestHook { kind, text, diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index a0b8579..d87404b 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -104,6 +104,37 @@ fn close_command_marks_task_closed_in_pack() { .stdout(contains("status: closed")); } +#[test] +fn doctor_exits_zero_on_fresh_install() { + let dir = assert_fs::TempDir::new().unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["doctor"]) + .assert() + .success(); +} + +#[test] +fn doctor_json_output_is_parseable_and_lists_paths() { + let dir = assert_fs::TempDir::new().unwrap(); + let output = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["doctor", "--json"]) + .output() + .unwrap(); + let stdout = String::from_utf8(output.stdout).unwrap(); + let v: serde_json::Value = + serde_json::from_str(&stdout).expect("doctor --json must be valid JSON"); + + assert!(v.get("data_dir").is_some()); + assert!(v.get("events_dir").is_some()); + assert!(v.get("state_dir").is_some()); + assert!(v.get("known_projects").unwrap().is_array()); + assert!(v.get("issues").unwrap().is_array()); +} + #[test] fn close_unknown_task_id_returns_error() { let dir = assert_fs::TempDir::new().unwrap(); From d71e0e46328c27519f2a4af78ee3d8d13ff70294 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 10:03:14 +0400 Subject: [PATCH 27/39] feat(cli): task-journal migrate-project --from PATH --to PATH Project moved on disk -> canonical-path-derived hash changed -> data orphaned. New CLI command renames the JSONL + SQLite + metrics files from the old project_hash to the new one and updates the project_hash columns inside the SQLite (tasks, index_state). Refuses when --from and --to resolve to the same hash (symlink, case- insensitive FS). Refuses to overwrite an existing destination file unless --force is set. CLI integration tests: - migrate_project_round_trips_data_to_new_path: create task in project A, migrate-project A -> B, pack from B finds the task. - migrate_project_refuses_overwrite_without_force: both have data, migration aborts with destination already exists in stderr. Refs claude-memory-1yc.5 --- crates/tj-cli/src/main.rs | 107 ++++++++++++++++++++++++++++++++++++- crates/tj-cli/tests/cli.rs | 80 +++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 1 deletion(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 22639a9..a1adf65 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use serde::Serialize; use std::path::PathBuf; @@ -98,6 +98,94 @@ fn dir_writable(dir: &std::path::Path) -> bool { r } +/// Move all on-disk data for one project_hash to another. Used by the +/// `migrate-project` subcommand when a project's directory has been +/// moved on disk and the canonical-path hash no longer matches. +fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool) -> Result<()> { + let from_hash = tj_core::project_hash::from_path(from) + .with_context(|| format!("compute project_hash for --from {from:?}"))?; + let to_hash = tj_core::project_hash::from_path(to) + .with_context(|| format!("compute project_hash for --to {to:?}"))?; + + if from_hash == to_hash { + anyhow::bail!( + "--from and --to resolve to the same project_hash ({from_hash}) — nothing to migrate" + ); + } + + let events_dir = tj_core::paths::events_dir()?; + let state_dir = tj_core::paths::state_dir()?; + let metrics_dir = tj_core::paths::metrics_dir()?; + + // (source, destination) tuples to attempt to rename. + let pairs = [ + ( + events_dir.join(format!("{from_hash}.jsonl")), + events_dir.join(format!("{to_hash}.jsonl")), + ), + ( + state_dir.join(format!("{from_hash}.sqlite")), + state_dir.join(format!("{to_hash}.sqlite")), + ), + ( + metrics_dir.join(format!("{from_hash}.jsonl")), + metrics_dir.join(format!("{to_hash}.jsonl")), + ), + ]; + + // Pre-flight: refuse overwrite of any destination unless --force. + if !force { + for (_src, dst) in &pairs { + if dst.exists() { + anyhow::bail!( + "destination already exists: {} — pass --force to overwrite", + dst.display() + ); + } + } + } + + let mut moved: Vec = Vec::new(); + for (src, dst) in &pairs { + if !src.exists() { + continue; + } + if let Some(parent) = dst.parent() { + std::fs::create_dir_all(parent)?; + } + if dst.exists() && force { + std::fs::remove_file(dst).with_context(|| format!("remove existing {dst:?}"))?; + } + std::fs::rename(src, dst).with_context(|| format!("rename {src:?} -> {dst:?}"))?; + moved.push(dst.display().to_string()); + } + + // Re-key the project_hash columns inside the (now renamed) SQLite. + let new_state_path = state_dir.join(format!("{to_hash}.sqlite")); + if new_state_path.exists() { + let conn = tj_core::db::open(&new_state_path)?; + conn.execute( + "UPDATE tasks SET project_hash = ?1 WHERE project_hash = ?2", + rusqlite::params![to_hash, from_hash], + )?; + conn.execute( + "UPDATE index_state SET project_hash = ?1 WHERE project_hash = ?2", + rusqlite::params![to_hash, from_hash], + )?; + } + + if moved.is_empty() { + println!("no on-disk data found for project_hash {from_hash} — nothing to migrate"); + } else { + println!("migrated {} file(s):", moved.len()); + for path in moved { + println!(" {path}"); + } + println!(" project_hash {from_hash} -> {to_hash}"); + } + Ok(()) +} + fn run_doctor() -> Result { let mut issues: Vec = Vec::new(); @@ -300,6 +388,20 @@ enum Commands { #[arg(long)] json: bool, }, + /// Re-key on-disk data when a project moved on disk. The project_hash + /// is derived from the canonical path, so a moved project orphans its + /// own data; this command renames the JSONL + SQLite + metrics files. + MigrateProject { + /// Old project path (the data we want to keep). + #[arg(long, value_name = "PATH")] + from: PathBuf, + /// New project path (where the project lives now). + #[arg(long, value_name = "PATH")] + to: PathBuf, + /// Overwrite the destination if data already exists for it. + #[arg(long)] + force: bool, + }, /// Hook entry point: ingest a chat chunk through the classifier. IngestHook { /// Hook kind: UserPromptSubmit | PostToolUse | Stop | SessionStart. @@ -602,6 +704,9 @@ fn main() -> Result<()> { std::process::exit(1); } } + Commands::MigrateProject { from, to, force } => { + run_migrate_project(&from, &to, force)?; + } Commands::IngestHook { kind, text, diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index d87404b..bf284b0 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -135,6 +135,86 @@ fn doctor_json_output_is_parseable_and_lists_paths() { assert!(v.get("issues").unwrap().is_array()); } +#[test] +fn migrate_project_round_trips_data_to_new_path() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj_a = assert_fs::TempDir::new().unwrap(); + let proj_b = assert_fs::TempDir::new().unwrap(); + + // Create a task with the cwd = proj_a. + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_a.path()) + .args(["create", "Migration round-trip"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + + // Migrate the data to proj_b. + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .args([ + "migrate-project", + "--from", + proj_a.path().to_str().unwrap(), + "--to", + proj_b.path().to_str().unwrap(), + ]) + .assert() + .success(); + + // Pack from proj_b finds the same task. + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj_b.path()) + .args(["pack", &task_id, "--mode", "full"]) + .assert() + .success() + .stdout(contains("Migration round-trip")); +} + +#[test] +fn migrate_project_refuses_overwrite_without_force() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj_a = assert_fs::TempDir::new().unwrap(); + let proj_b = assert_fs::TempDir::new().unwrap(); + + // Both projects have data: create a task in each. + for proj in [&proj_a, &proj_b] { + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Conflicting"]) + .assert() + .success(); + } + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .args([ + "migrate-project", + "--from", + proj_a.path().to_str().unwrap(), + "--to", + proj_b.path().to_str().unwrap(), + ]) + .assert() + .failure() + .stderr(contains("destination already exists")); +} + #[test] fn close_unknown_task_id_returns_error() { let dir = assert_fs::TempDir::new().unwrap(); From 8065cb27094cfd8d065ac455386d045e9c78fb2b Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 10:05:48 +0400 Subject: [PATCH 28/39] feat(export): HTML timeline output (export --format html) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a third format to the existing export subcommand. Renders a self-contained HTML page (inline CSS, no external assets) showing the task timeline grouped by task_id. Useful as a PR-review attachment or sprint retro artefact. Design notes: - All five HTML special chars (& < > " ) are escaped via html_escape() — no XSS surface even though we never render third-party HTML. - CSS uses prefers-color-scheme so light and dark mode both look sane without a toggle. - Event type pills get a colour class (decision/rejection/evidence/ finding) so timelines are scannable at a glance. - Suggested events get a trailing ? marker matching the rest of the codebase. CLI integration test export_html_emits_self_contained_document: - DOCTYPE html present - task title + event text present - no http:// or https:// — proves no external CSS/font/script leaked into the output. Refs claude-memory-1yc.6 --- crates/tj-cli/src/main.rs | 136 ++++++++++++++++++++++++++++++++++++- crates/tj-cli/tests/cli.rs | 58 ++++++++++++++++ 2 files changed, 193 insertions(+), 1 deletion(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index a1adf65..ae299d4 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -186,6 +186,135 @@ fn run_migrate_project(from: &std::path::Path, to: &std::path::Path, force: bool Ok(()) } +/// Minimal HTML attribute/text escape. Five characters cover the body of +/// `text/html` for our use case (no script context, no URL emission). +fn html_escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + _ => out.push(c), + } + } + out +} + +const HTML_TIMELINE_CSS: &str = r#" +:root { color-scheme: light dark; --fg:#222; --bg:#fafafa; --muted:#666; --accent:#0366d6; } +@media (prefers-color-scheme: dark) { :root { --fg:#eee; --bg:#1a1a1a; --muted:#999; --accent:#58a6ff; } } +* { box-sizing: border-box; } +body { font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif; + color: var(--fg); background: var(--bg); margin: 0; padding: 1.5rem; } +header h1 { margin: 0 0 1.5rem; font-size: 1.4rem; } +article { margin-bottom: 2rem; padding: 1rem 1.25rem; background: rgba(127,127,127,0.07); + border-radius: 6px; } +article h2 { margin: 0; font-size: 1.05rem; font-weight: 600; } +.tid { font-family: ui-monospace, "SF Mono", Menlo, Consolas, monospace; + color: var(--accent); margin-right: 0.4em; } +.meta { color: var(--muted); font-size: 0.85rem; margin: 0.25rem 0 0.75rem; } +ol.timeline { list-style: none; margin: 0; padding-left: 0; } +ol.timeline li { padding: 0.4rem 0; border-top: 1px solid rgba(127,127,127,0.15); } +ol.timeline li:first-child { border-top: none; } +time { font-family: ui-monospace, monospace; color: var(--muted); margin-right: 0.6em; } +.type { display: inline-block; padding: 0 0.35em; margin-right: 0.4em; border-radius: 3px; + font-size: 0.75rem; text-transform: uppercase; letter-spacing: 0.05em; + background: rgba(127,127,127,0.15); } +.type-decision { background: rgba(3,102,214,0.18); color: var(--accent); } +.type-rejection { background: rgba(214,3,3,0.18); } +.type-evidence { background: rgba(40,167,69,0.18); } +.type-finding { background: rgba(255,166,0,0.20); } +.suggested::after { content: " ?"; color: var(--muted); } +"#; + +fn render_html_timeline(events: &[&tj_core::event::Event]) -> String { + use std::collections::BTreeMap; + + let mut tasks: BTreeMap> = BTreeMap::new(); + for e in events { + tasks.entry(e.task_id.clone()).or_default().push(e); + } + + let mut out = String::new(); + out.push_str("\n"); + out.push_str(""); + out.push_str(""); + out.push_str(""); + out.push_str("Task Journal — Export"); + out.push_str(""); + out.push_str(""); + out.push_str("

Task Journal — Export

"); + out.push_str("
"); + + for (task_id, task_events) in &tasks { + let title = task_events + .iter() + .find(|e| e.event_type == tj_core::event::EventType::Open) + .and_then(|e| { + e.meta + .get("title") + .and_then(|v| v.as_str()) + .map(String::from) + .or_else(|| Some(e.text.clone())) + }) + .unwrap_or_else(|| "(untitled)".into()); + + let closed = task_events + .last() + .map(|e| e.event_type == tj_core::event::EventType::Close) + .unwrap_or(false); + let status = if closed { "closed" } else { "open" }; + + let created = task_events + .first() + .map(|e| e.timestamp.as_str()) + .unwrap_or("?"); + + out.push_str("
"); + out.push_str(&format!( + "

{}{}

", + html_escape(task_id), + html_escape(&title) + )); + out.push_str(&format!( + "

status: {} · created: {}

", + status, + html_escape(created) + )); + out.push_str("
    "); + for e in task_events { + let etype = serde_json::to_value(e.event_type) + .ok() + .and_then(|v| v.as_str().map(String::from)) + .unwrap_or_else(|| "unknown".into()); + let suggested_class = if matches!(e.status, tj_core::event::EventStatus::Suggested) { + " suggested" + } else { + "" + }; + out.push_str(&format!( + "
  1. \ + {}{}
    2. ", + suggested_class, + html_escape(&e.timestamp), + html_escape(&etype), + html_escape(&etype), + html_escape(&e.text) + )); + } + out.push_str("
"); + out.push_str("
"); + } + + out.push_str("
\n"); + out +} + fn run_doctor() -> Result { let mut issues: Vec = Vec::new(); @@ -936,7 +1065,12 @@ fn main() -> Result<()> { println!(); } } - other => anyhow::bail!("unknown format: {other} (expected `md` or `json`)"), + "html" => { + print!("{}", render_html_timeline(&events)); + } + other => { + anyhow::bail!("unknown format: {other} (expected `md`, `json`, or `html`)") + } } } Commands::Search { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index bf284b0..1426d8f 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -135,6 +135,64 @@ fn doctor_json_output_is_parseable_and_lists_paths() { assert!(v.get("issues").unwrap().is_array()); } +#[test] +fn export_html_emits_self_contained_document() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "HTML export test"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args([ + "event", + &task_id, + "--type", + "decision", + "--text", + "Adopt Rust", + ]) + .assert() + .success(); + + let output = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["export", "--format", "html", "--task", &task_id]) + .output() + .unwrap(); + let html = String::from_utf8(output.stdout).unwrap(); + + // Self-contained shape. + let lower = html.to_lowercase(); + assert!( + lower.starts_with(""), + "html missing doctype: {html}" + ); + assert!(html.contains("HTML export test"), "task title missing"); + assert!(html.contains("Adopt Rust"), "decision event missing"); + // No external assets — no http/https URL anywhere. + assert!(!html.contains("http://"), "external http url leaked"); + assert!(!html.contains("https://"), "external https url leaked"); +} + #[test] fn migrate_project_round_trips_data_to_new_path() { let xdg = assert_fs::TempDir::new().unwrap(); From 44ce215f683aad020bbbf485ee1307d73bf11655 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 10:07:57 +0400 Subject: [PATCH 29/39] feat(classifier): few-shot examples in prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six worked Input/Output pairs covering the three boundary calls the classifier gets wrong most often: - hypothesis vs finding (2 examples) - finding vs evidence (2 examples) - decision vs hypothesis (2 examples) Each pair pins one half of the boundary so the model sees the contrast inline rather than only as abstract definitions. The examples themselves are drawn from real boundary cases observed during this epic — keeps them representative. The prompt budget guard (prompt_truncates_event_lines_to_keep_size _bounded) still passes after adding ~3KB of fixed prefix, because the recent_tasks block is the variable cost — examples are constant-time addition. New test prompt_contains_few_shot_examples enforces the 6-example floor as a regression guard. Refs claude-memory-1yc.7 --- crates/tj-core/src/classifier/prompt.rs | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/crates/tj-core/src/classifier/prompt.rs b/crates/tj-core/src/classifier/prompt.rs index 6fa23c8..f96c893 100644 --- a/crates/tj-core/src/classifier/prompt.rs +++ b/crates/tj-core/src/classifier/prompt.rs @@ -50,6 +50,25 @@ pub fn build(input: &ClassifyInput) -> String { - hypothesis vs finding: hypothesis = \"I think\"/\"maybe\"/\"could be\"; finding = \"I see\"/\"the code shows\"/\"confirmed that\"\n\ - finding vs evidence: finding = discovered a fact; evidence = ran a test/experiment that PROVES something\n\ - decision vs hypothesis: decision = committed choice; hypothesis = exploring an option\n\n\ + ## Examples\n\ + The dashed lines separate Input (assistant or user chunk) from Output (the JSON you must produce). Use them as anchors for the boundary calls above.\n\n\ + Input: \"I think the timeout is happening because the Anthropic SDK keeps the socket open after the read.\"\n\ + Output: {{\"event_type\":\"hypothesis\",\"task_id_guess\":null,\"confidence\":0.88,\"evidence_strength\":null,\"suggested_text\":\"Possible cause: SDK keeps socket open after read.\"}}\n\ + ---\n\ + Input: \"Confirmed: in src/classifier/http.rs:62 the ureq Request has no .timeout() — that's why the call hangs.\"\n\ + Output: {{\"event_type\":\"finding\",\"task_id_guess\":null,\"confidence\":0.93,\"evidence_strength\":null,\"suggested_text\":\"http.rs:62 builds the ureq Request without .timeout().\"}}\n\ + ---\n\ + Input: \"Read pack.rs end-to-end: assemble() always invalidates task_pack_cache before checking it, so the cache is never reused.\"\n\ + Output: {{\"event_type\":\"finding\",\"task_id_guess\":null,\"confidence\":0.92,\"evidence_strength\":null,\"suggested_text\":\"pack.rs assemble() invalidates task_pack_cache before reading it; cache never reused.\"}}\n\ + ---\n\ + Input: \"Ran cargo bench: pack_assemble_cold_10k drops from 820ms to 41ms after the index_state change. 20x faster.\"\n\ + Output: {{\"event_type\":\"evidence\",\"task_id_guess\":null,\"confidence\":0.95,\"evidence_strength\":\"strong\",\"suggested_text\":\"cargo bench: pack_assemble_cold_10k 820ms -> 41ms (20x) after index_state.\"}}\n\ + ---\n\ + Input: \"Maybe we should use rmcp's Result instead of Json.\"\n\ + Output: {{\"event_type\":\"hypothesis\",\"task_id_guess\":null,\"confidence\":0.82,\"evidence_strength\":null,\"suggested_text\":\"Consider Result in place of Json.\"}}\n\ + ---\n\ + Input: \"Going with fd-lock for the Windows file lock — single API across platforms, well-maintained, simpler than rolling our own with rustix.\"\n\ + Output: {{\"event_type\":\"decision\",\"task_id_guess\":null,\"confidence\":0.94,\"evidence_strength\":null,\"suggested_text\":\"Use fd-lock crate for cross-platform JSONL file lock.\"}}\n\n\ Active tasks (top candidates):\n{recent}\n\n\ New {author} chunk:\n{text}\n\n\ Decide:\n\ @@ -110,6 +129,29 @@ mod tests { ); } + #[test] + fn prompt_contains_few_shot_examples() { + let input = ClassifyInput { + text: "anything".into(), + author_hint: "assistant".into(), + recent_tasks: vec![], + }; + let p = build(&input); + assert!(p.contains("## Examples"), "Examples section missing"); + // Six worked Input/Output pairs — count Input: occurrences. + let count = p.matches("Input: ").count(); + assert!( + count >= 6, + "expected at least 6 few-shot examples, got {count}" + ); + // Each example must show its expected JSON shape. + let json_count = p.matches("Output: {").count(); + assert!( + json_count >= 6, + "expected at least 6 example outputs, got {json_count}" + ); + } + #[test] fn prompt_handles_empty_tasks() { let input = ClassifyInput { From 433bf67ea7035b45be407e3b9373f29e9bf235ab Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 10:14:21 +0400 Subject: [PATCH 30/39] test(classifier): labeled eval dataset + opt-in accuracy gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds tests/fixtures/classifier_eval.jsonl with 30 labeled chunks spanning all 12 event types, plus tests/classifier_eval.rs that runs in two modes: - Default (CI-safe): no model API call. Asserts • fixture has ≥ 30 rows • every expected event type is one of EventType::ALL • prompt builder emits each input verbatim Hermetic and fast — runs as part of plain cargo test. - Opt-in (TJ_CLASSIFIER_EVAL=on): runs ClaudeCliClassifier:: default() against every row, computes accuracy, asserts the 0.70 floor and prints misses. Requires on PATH. Skipped silently otherwise. Three new tests, all green by default; the real-classifier one is silent-pass without the env var. Refs claude-memory-1yc.8 --- crates/tj-core/tests/classifier_eval.rs | 158 ++++++++++++++++++ .../tests/fixtures/classifier_eval.jsonl | 30 ++++ 2 files changed, 188 insertions(+) create mode 100644 crates/tj-core/tests/classifier_eval.rs create mode 100644 crates/tj-core/tests/fixtures/classifier_eval.jsonl diff --git a/crates/tj-core/tests/classifier_eval.rs b/crates/tj-core/tests/classifier_eval.rs new file mode 100644 index 0000000..c8b8884 --- /dev/null +++ b/crates/tj-core/tests/classifier_eval.rs @@ -0,0 +1,158 @@ +//! Classifier eval harness. +//! +//! Two execution modes: +//! +//! 1. **Default (CI-safe).** Loads the labeled fixture, exercises the +//! prompt builder against every input, and asserts: +//! - the fixture has at least 30 examples +//! - every example has a recognised `expected` event_type +//! - the prompt builder always emits the input text into the prompt +//! No model API is called. Deterministic, hermetic. +//! +//! 2. **Opt-in real classifier (`TJ_CLASSIFIER_EVAL=on`).** Calls +//! `ClaudeCliClassifier::default()` against every fixture row and +//! computes accuracy. Asserts accuracy ≥ 0.7 (initial floor; will +//! ratchet up as the dataset grows). Requires a working `claude` +//! CLI on PATH (subscription mode). Skipped silently if the env +//! var is not set so the default `cargo test` run is fast and free. + +use serde::Deserialize; +use std::collections::HashSet; + +use tj_core::classifier::{ + cli::ClaudeCliClassifier, prompt, Classifier, ClassifyInput, ClassifyOutput, +}; +use tj_core::event::EventType; + +const FIXTURE: &str = include_str!("fixtures/classifier_eval.jsonl"); +const ACCURACY_FLOOR: f64 = 0.70; + +#[derive(Deserialize)] +struct Example { + text: String, + expected: String, +} + +fn load_examples() -> Vec { + FIXTURE + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).unwrap_or_else(|e| panic!("bad fixture line: {l} — {e}"))) + .collect() +} + +fn known_event_types() -> HashSet { + EventType::ALL + .iter() + .map(|t| { + serde_json::to_value(t) + .unwrap() + .as_str() + .unwrap() + .to_string() + }) + .collect() +} + +#[test] +fn fixture_has_minimum_size_and_known_types() { + let examples = load_examples(); + assert!( + examples.len() >= 30, + "fixture must have ≥ 30 labeled rows, got {}", + examples.len() + ); + let known = known_event_types(); + for ex in &examples { + assert!( + known.contains(&ex.expected), + "unknown expected event type '{}'", + ex.expected + ); + } +} + +#[test] +fn prompt_builder_includes_every_fixture_input() { + let examples = load_examples(); + for ex in &examples { + let input = ClassifyInput { + text: ex.text.clone(), + author_hint: "assistant".into(), + recent_tasks: vec![], + }; + let p = prompt::build(&input); + assert!( + p.contains(&ex.text), + "prompt missing fixture text: {}", + ex.text + ); + } +} + +/// Real-classifier accuracy run. Skipped unless `TJ_CLASSIFIER_EVAL=on`. +/// Wired through `ClaudeCliClassifier::default()` so it runs against the +/// user's `claude -p` subscription if available. +#[test] +fn classifier_meets_accuracy_floor_on_labeled_dataset() { + if std::env::var("TJ_CLASSIFIER_EVAL").as_deref() != Ok("on") { + eprintln!( + "skipping: set TJ_CLASSIFIER_EVAL=on to run the real-classifier eval against {} fixtures", + load_examples().len() + ); + return; + } + + let classifier = ClaudeCliClassifier::default(); + let examples = load_examples(); + let mut correct = 0usize; + let mut total = 0usize; + let mut misses: Vec<(String, String, String)> = Vec::new(); + for ex in &examples { + total += 1; + let input = ClassifyInput { + text: ex.text.clone(), + author_hint: "assistant".into(), + recent_tasks: vec![], + }; + let out: ClassifyOutput = match classifier.classify(&input) { + Ok(o) => o, + Err(e) => { + eprintln!("classifier error on '{}': {e}", ex.text); + continue; + } + }; + let predicted = serde_json::to_value(out.event_type) + .unwrap() + .as_str() + .unwrap() + .to_string(); + if predicted == ex.expected { + correct += 1; + } else { + misses.push((ex.text.clone(), ex.expected.clone(), predicted)); + } + } + + let accuracy = if total == 0 { + 0.0 + } else { + correct as f64 / total as f64 + }; + eprintln!( + "classifier eval: {correct}/{total} correct ({:.1}%)", + accuracy * 100.0 + ); + if !misses.is_empty() { + eprintln!("misses:"); + for (text, expected, predicted) in &misses { + eprintln!(" expected={expected} predicted={predicted}: {text}"); + } + } + assert!( + accuracy >= ACCURACY_FLOOR, + "classifier accuracy {:.2} below floor {:.2}", + accuracy, + ACCURACY_FLOOR + ); +} diff --git a/crates/tj-core/tests/fixtures/classifier_eval.jsonl b/crates/tj-core/tests/fixtures/classifier_eval.jsonl new file mode 100644 index 0000000..01953a3 --- /dev/null +++ b/crates/tj-core/tests/fixtures/classifier_eval.jsonl @@ -0,0 +1,30 @@ +{"text":"I think the rebuild_state aborts because one bad line panics serde_json","expected":"hypothesis"} +{"text":"Maybe we should bump task_id length from 6 to 10 chars to dodge collisions","expected":"hypothesis"} +{"text":"It could be that the BufWriter is holding bytes in user space when fsync is called","expected":"hypothesis"} +{"text":"Looked at db.rs:133 — rebuild_state opens an unchecked_transaction and aborts on the first parse error","expected":"finding"} +{"text":"Confirmed in mcp/main.rs:158 — every tool handler calls full rebuild_state, never the incremental path","expected":"finding"} +{"text":"In storage.rs the JsonlWriter wraps File in fd_lock::RwLock and acquires write() per append","expected":"finding"} +{"text":"Ran cargo test --workspace: all 193 tests green, no flakes over five runs","expected":"evidence"} +{"text":"Bench result: rebuild_state_10k_events median 820ms, ingest_new_events 38ms — 21x faster","expected":"evidence"} +{"text":"Reproduced the corruption: two parallel hooks racing on the same JSONL on Windows produced 7 torn lines","expected":"evidence"} +{"text":"Going with fd-lock for the file lock — single API across Linux/macOS/Windows, well-maintained","expected":"decision"} +{"text":"We will return Result, McpError> from every tool handler instead of the success-typed envelope","expected":"decision"} +{"text":"Picked criterion 0.5 over divan because we already have it in transitive deps via mockito","expected":"decision"} +{"text":"Tried adding rwlock around index_event but it deadlocks with the open transaction — won't work","expected":"rejection"} +{"text":"Considered storing last_event_id in a JSON sidecar but it makes the pack-cache invalidation race-prone","expected":"rejection"} +{"text":"Anthropic API rate limit on the haiku tier is 1000 RPM per organisation","expected":"constraint"} +{"text":"NTFS does not give us POSIX append-atomicity for writes larger than ~512 bytes","expected":"constraint"} +{"text":"Actually the 'last_indexed_event_id is in index_state' claim was wrong — there's no row until first ingest","expected":"correction"} +{"text":"Correction on the earlier finding: pack-cache is invalidated per task_id, not globally","expected":"correction"} +{"text":"PR merged. Released 0.1.4 to crates.io. Closing this task","expected":"close"} +{"text":"Shipped — task-journal doctor now runs on all three OS","expected":"close"} +{"text":"Reopening — turns out the migration is missing on existing 0.1.x DBs after upgrade","expected":"reopen"} +{"text":"This task is folded into the bigger 'v0.2.0 epic' work; closing in favour of that","expected":"supersede"} +{"text":"This belongs under the OAuth task, not the data-storage one","expected":"redirect"} +{"text":"Maybe the problem is FTS5 not being a regular index; could be slow at 100k events","expected":"hypothesis"} +{"text":"Checked the metrics directory: only one project_hash present, no orphans","expected":"finding"} +{"text":"e2e: ran full pipeline 50 times, p99 < 200ms, p50 < 50ms","expected":"evidence"} +{"text":"Decision: incremental indexing reads only events since last_indexed_event_id, falls back to full rebuild on missing marker","expected":"decision"} +{"text":"Won't take the rusqlite multithreaded feature — adds a c-bindgen rebuild for no real win","expected":"rejection"} +{"text":"On Windows GitHub runners cmd.exe is required to launch .cmd shims, no PowerShell fallback","expected":"constraint"} +{"text":"Earlier I said BufWriter was needed for perf — that was wrong, perf delta is unmeasurable for this workload","expected":"correction"} From e22766280836fa329512b1ddabd091b333a584d7 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 10:15:36 +0400 Subject: [PATCH 31/39] docs: epic C PR body for review --- .../plans/2026-05-06-v0.2.0-epic-c-pr-body.md | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .docs/plans/2026-05-06-v0.2.0-epic-c-pr-body.md diff --git a/.docs/plans/2026-05-06-v0.2.0-epic-c-pr-body.md b/.docs/plans/2026-05-06-v0.2.0-epic-c-pr-body.md new file mode 100644 index 0000000..9c6658c --- /dev/null +++ b/.docs/plans/2026-05-06-v0.2.0-epic-c-pr-body.md @@ -0,0 +1,58 @@ +## Summary + +Epic C — quality / DX / community polish. **8 atomic commits** on `claude/v0.2.0-epic-c`, built off `claude/v0.2.0-epic-b` HEAD. + +> **Merge order:** epic A → main, then epic B (rebased on main), then this branch (rebased on main). + +Plan: [`.docs/plans/2026-05-06-v0.2.0-epic-c-quality.md`](./.docs/plans/2026-05-06-v0.2.0-epic-c-quality.md) + +### What changed + +**Classifier quality** +- `feat(classifier)` — six few-shot Input/Output examples in the prompt covering the harder boundary calls (hypothesis vs finding, finding vs evidence, decision vs hypothesis). Prompt-budget guard still passes. +- `test(classifier)` — 30-row labeled eval fixture + opt-in accuracy gate (`TJ_CLASSIFIER_EVAL=on`). Default mode runs hermetic shape tests; opt-in mode calls `ClaudeCliClassifier::default()` and asserts ≥ 0.70 accuracy. Floor will ratchet up after 100+ dogfood examples. + +**User-facing DX** +- `feat(cli)` — `task-journal doctor` self-check command with human + `--json` output. Reports claude-on-PATH, data-dir writability, known projects, schema migrations. +- `feat(cli)` — `task-journal migrate-project --from PATH --to PATH [--force]`. Renames JSONL/SQLite/metrics from old project_hash to new; UPDATEs `tasks.project_hash` and `index_state.project_hash` columns in SQLite. +- `feat(export)` — `export --format html` produces a self-contained timeline page (inline CSS, no external assets, dark-mode aware via `prefers-color-scheme`). + +**OSS / coverage / Windows** +- `chore` — `CONTRIBUTING.md`, `CODE_OF_CONDUCT.md` (Contributor Covenant 2.1 ref), three `.github/ISSUE_TEMPLATE/*`, `.github/PULL_REQUEST_TEMPLATE.md`, README links. +- `ci` — `cargo-llvm-cov` job + Codecov upload + README badge. Non-blocking initially. +- `test(classifier)` — cross-platform fake-claude shim (`.sh`/`cat` on Unix, `.cmd`/`type` on Windows). The two `ClaudeCliClassifier` tests now run on all three CI matrix OS instead of `cfg(unix)` only. + +### Verification + +- `cargo fmt --all -- --check` ✅ +- `cargo clippy --workspace --all-targets -- -D warnings` ✅ +- `cargo test --workspace --all-targets` ✅ — **202 tests** (was 193 from epic B; +9 added by this PR) +- `cargo bench --workspace --no-run` ✅ + +### New CLI surface + +| Command | Purpose | +|---------|---------| +| `task-journal doctor [--json]` | Diagnostic check; non-zero exit on issues | +| `task-journal migrate-project --from PATH --to PATH [--force]` | Re-key on-disk data when project moves | +| `task-journal export --format html [--task ID]` | Self-contained HTML timeline | + +### New env vars + +| Var | Effect | +|-----|--------| +| `TJ_CLASSIFIER_EVAL=on` | Enables the real-classifier accuracy run in `cargo test`. Default OFF — CI stays hermetic. | + +### Test plan + +- [ ] Branch CI green on three OS for `test`, `msrv`, `audit`, `benches-compile`, `coverage` (new). +- [ ] Try `task-journal doctor` on a clean VM — confirms claude-binary detection and dir-writability checks. +- [ ] Move a project on disk, run `migrate-project`, confirm `task_pack` works in the new location. +- [ ] `task-journal export --format html --task tj-X > timeline.html` and open in browser; verify dark mode + no broken layout. +- [ ] (Optional, manual) `TJ_CLASSIFIER_EVAL=on cargo test classifier_meets_accuracy_floor` against the real `claude` CLI; record baseline accuracy. + +### After this lands + +`v0.2.0` final tag. No further code changes expected — the dogfood window from `0.2.0-rc.1` already exercised epic B; this epic is additive and behind-the-scenes for almost every existing user. + +🤖 Generated with [Claude Code](https://claude.com/claude-code) From 6e5e5a967b05e469b4f582fa31dd1b703f04020e Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 10:22:41 +0400 Subject: [PATCH 32/39] perf(mcp): cache SQLite connections per state-path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every MCP tool handler used to call tj_core::db::open() which runs PRAGMA journal_mode + foreign_keys + apply_migrations + an empty schema_migrations SELECT on every invocation. At small N the open cost dominates the actual work — pack/search/close all paid this overhead even when the underlying state changed nothing. Now: a process-wide HashMap>> guarded by an outer OnceLock>. cached_open(path) does an O(1) lookup, falls back to db::open() on the cold path, and shares the Arc with future callers. Each tool handler takes the inner mutex for the duration of its work; the outer mutex is held only for the brief insert/lookup. - SQLite Connection is Send (single-threaded mode); safe to send across the spawn_blocking thread boundary inside an Arc. - Inner mutex serialises calls per project_hash. SQLite already serialises writes, so we accept a tiny concurrency loss in exchange for the open-cost saving. - Cache is keyed by PathBuf, so two MCPs running with different --project-dir do not stomp on each other. Tests: - cached_open_returns_same_arc_for_same_path - cached_open_returns_distinct_arcs_for_distinct_paths Refs claude-memory-yj1.1 --- .../2026-05-07-v0.2.1-epic-d-operational.md | 111 ++++++++++++++++++ crates/tj-mcp/src/main.rs | 93 +++++++++++++-- 2 files changed, 192 insertions(+), 12 deletions(-) create mode 100644 .docs/plans/2026-05-07-v0.2.1-epic-d-operational.md diff --git a/.docs/plans/2026-05-07-v0.2.1-epic-d-operational.md b/.docs/plans/2026-05-07-v0.2.1-epic-d-operational.md new file mode 100644 index 0000000..a931bb3 --- /dev/null +++ b/.docs/plans/2026-05-07-v0.2.1-epic-d-operational.md @@ -0,0 +1,111 @@ +# Epic D — v0.2.1: operational maturity + +**Date:** 2026-05-07 +**Branch:** `claude/v0.2.1-epic-d` (off `claude/v0.2.0-epic-c` HEAD) +**Target release:** `0.2.1` (non-breaking minor) +**Bd epic:** `claude-memory-yj1` + +## Goal + +Three threads of "ready to run for a year" work: + +1. **Performance polish** — stop opening a fresh SQLite Connection on + every MCP tool call; share a small per-process cache. +2. **Observability** — structured tracing with stable correlation IDs + so you can grep one user request across logs; `pending` queue is + inspectable + retryable. +3. **Lifecycle** — graceful SIGTERM shutdown; `export --format sqlite` + for backup snapshots; real-MCP-client integration test that exercises + the end-to-end RPC envelope. + +All non-breaking. No CLI flag removals, no MCP schema changes. + +## Success criteria + +1. `cargo test --workspace --all-targets` green on all three OS. +2. `task-journal pending --list` lists queued classifications; + `task-journal pending --retry` re-feeds them through the + classifier (or marks them dead after N attempts). +3. Real-MCP-client integration test (`tj-mcp/tests/rmcp_roundtrip.rs`) + sends `task_create` → `event_add` → `task_pack` → `task_close` + over rmcp's in-process transport and verifies the response shapes. +4. Tracing spans wrap each tool handler with a `correlation_id` + field; default `RUST_LOG=info` gives one line per tool call. +5. SIGTERM on the MCP server flushes `tracing` and exits 0 cleanly. +6. `task-journal export --format sqlite > backup.sqlite` produces a + self-contained DB; round-trips through a fresh `task-journal pack` + call after pointing at it. +7. SQLite Connection cache lands; criterion bench shows reduction in + MCP-handler overhead at small N (the open + migrate cost + dominates at low event counts). +8. Version bumped to `0.2.1`; CHANGELOG entry written. + +## Non-goals (deferred to D-future or E) + +- Telemetry endpoint (requires hosted backend — separate decision). +- `task-journal compact` (lifecycle archival of closed tasks; tricky + because it inverts the append-only contract — wants a design pass + before code). +- Update notifier (`--version` checks crates.io) — privacy considered; + prefer opt-in. + +## Tasks (7 + release) + +| # | Task | Touches | Test? | Notes | +|---|------|---------|-------|-------| +| D1 | SQLite Connection cache for MCP server (process-wide, mutex-guarded, keyed by state-path) | `crates/tj-mcp/src/main.rs` | yes (cache_returns_same_connection_for_same_path) | Bypasses re-running migrations + WAL setup per call. | +| D2 | `task-journal export --format sqlite` | `crates/tj-cli/src/main.rs` | yes (export_sqlite_round_trips_through_pack) | Copies the existing SQLite to stdout via VACUUM INTO `:memory:` then dumps. | +| D3 | Pending queue inspect + retry: `task-journal pending --list` and `--retry` | `crates/tj-cli/src/main.rs`, possibly `tj-core::pending` | yes (pending_list_shows_queued_entries, pending_retry_drains_or_marks_dead) | Today the hook silently writes `pending/.json` on classifier failure; nothing surfaces them. | +| D4 | Real MCP client integration test for the round-trip envelope | `crates/tj-mcp/tests/rmcp_roundtrip.rs` | yes (one large async test) | Use `rmcp::client::ClientHandler` over `transport::async_rw::AsyncRwTransport` with two pipes. | +| D5 | Structured tracing: correlation_id span per tool call; one INFO line per call | `crates/tj-mcp/src/main.rs` | yes (tracing_test verifies span emission) | Use `tracing::Span::current().record(...)`; uuid v4 per call. | +| D6 | Graceful SIGTERM shutdown: drop `tracing_subscriber` flush, exit 0 | `crates/tj-mcp/src/main.rs` | yes on Unix (signal handler test); skipped on Windows | Use `tokio::signal::ctrl_c` + `unix::signal(SIGTERM)` cross-platform. | +| D7 | `release`: bump workspace version to `0.2.1` and write CHANGELOG entry | root + crates | n/a | Last commit. | + +## Sequencing + +``` +D1 ─┐ +D2 ─┼─→ (independent) +D3 ─┤ +D4 ─┘ + +D5 ─→ D6 (shutdown logic logs span events; both share the tracing wiring) + + D7 last +``` + +D1/D2/D3/D4 are independent. D5 lays groundwork that D6 reuses (the +shutdown handler logs through the tracing subscriber). D7 is final. + +## Risks + +- **D1 connection cache + drop ordering.** SQLite WAL files behave + oddly if a Connection is dropped while a write is in flight on + another thread. Mitigation: cache holds `Arc>`, + drop only at process exit. +- **D3 pending retry as O(N) infinite loop.** Each pending entry + stores attempt count; retry budget = 3 then mark `dead`. +- **D4 in-process transport behavior.** Some rmcp transport types + require both sides on the same runtime; verify with the rmcp + example tests as the reference shape. +- **D5 over-logging.** Default `RUST_LOG` should stay at info, not + trace, to avoid leaking event content into hook logs. +- **D6 Windows signal parity.** `SIGTERM` doesn't exist on Windows; + use `ctrl_c` + log "Windows shutdown via Ctrl-C only". + +## Verification gate (per task) + +Same as Epic A/B/C: +1. `cargo fmt --all -- --check` +2. `cargo clippy --workspace --all-targets -- -D warnings` +3. `cargo test --workspace --all-targets` +4. `git diff --stat` review +5. Conventional-commit prefix +6. `bd close --reason "..."` + +## Final verification (epic-level) + +- All 7 functional sub-tasks closed in bd +- `cargo bench --workspace --no-run` clean +- Release build clean: `cargo build --workspace --release` +- PR opened against `main` (after epic C is in main) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index 4187e9b..a0ddf37 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -8,10 +8,12 @@ use rmcp::{ handler::server::tool::Parameters, handler::server::wrapper::Json, tool, tool_handler, tool_router, transport::io::stdio, ErrorData as McpError, ServerHandler, ServiceExt, }; +use rusqlite::Connection; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::future::Future; -use std::path::PathBuf; -use std::sync::OnceLock; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; /// Optional override for the project directory used by every tool handler. /// `None` (the default) means "use the current working directory at the time @@ -54,6 +56,38 @@ where join_result.map_err(into_mcp_error) } +/// Process-wide cache of SQLite connections keyed by state-file path. +/// +/// Without this, every tool handler called `tj_core::db::open()` which +/// re-runs PRAGMAs, the migrations registry, and re-creates a new WAL +/// reader. At small N the open cost dominates the actual work. +/// +/// Storage layout: an outer `Mutex` guards the map (only briefly, during +/// insert/lookup), and each entry is `Arc>` so callers +/// can hold a connection across a longer transaction without blocking +/// other projects. +fn connection_cache() -> &'static Mutex>>> { + static CACHE: OnceLock>>>> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Get or create the cached `Connection` for a SQLite state path. The +/// returned `Arc>` is shared with future callers; the inner +/// mutex is the lock you actually want to take during a tool call. +fn cached_open(state_path: &Path) -> anyhow::Result>> { + let mut cache = connection_cache() + .lock() + .map_err(|e| anyhow::anyhow!("connection cache poisoned: {e}"))?; + if let Some(existing) = cache.get(state_path) { + return Ok(existing.clone()); + } + let conn = + tj_core::db::open(state_path).with_context(|| format!("open SQLite at {state_path:?}"))?; + let arc = Arc::new(Mutex::new(conn)); + cache.insert(state_path.to_path_buf(), arc.clone()); + Ok(arc) +} + /// MCP instructions delivered to every Claude Code session where this plugin is installed. /// This is the primary mechanism for self-contained plugin behavior — no manual CLAUDE.md edits needed. const MCP_INSTRUCTIONS: &str = r#"Task Journal — reasoning chain memory for AI coding sessions. @@ -207,7 +241,10 @@ impl TaskJournalServer { ) -> Result, McpError> { run_blocking(move || { let (project_hash, events_path, state_path) = project_paths()?; - let conn = tj_core::db::open(&state_path)?; + let conn_arc = cached_open(&state_path)?; + let conn = conn_arc + .lock() + .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; if events_path.exists() { tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } @@ -245,7 +282,10 @@ impl TaskJournalServer { let query = p.query.clone(); let results = run_blocking(move || { let (project_hash, events_path, state_path) = project_paths()?; - let conn = tj_core::db::open(&state_path)?; + let conn_arc = cached_open(&state_path)?; + let conn = conn_arc + .lock() + .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; if events_path.exists() { tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; } @@ -345,14 +385,18 @@ impl TaskJournalServer { run_blocking(move || { let (project_hash, events_path, state_path) = project_paths()?; - let conn = tj_core::db::open(&state_path)?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - } - if !tj_core::db::task_exists(&conn, &p.task_id)? { - anyhow::bail!("task not found: {}", p.task_id); - } - drop(conn); + let conn_arc = cached_open(&state_path)?; + { + let conn = conn_arc + .lock() + .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + } + if !tj_core::db::task_exists(&conn, &p.task_id)? { + anyhow::bail!("task not found: {}", p.task_id); + } + } // release the connection lock before doing the JSONL append let mut event = tj_core::event::Event::new( &p.task_id, @@ -524,6 +568,31 @@ mod tests { ); } + #[test] + fn cached_open_returns_same_arc_for_same_path() { + // The Arc returned by cached_open() is the same handle on second + // call: that's the proof that we are not re-running migrations + // / PRAGMA / WAL setup on every tool call. + let dir = tempfile::TempDir::new().unwrap(); + let p = dir.path().join("d1-cache.sqlite"); + let a = cached_open(&p).unwrap(); + let b = cached_open(&p).unwrap(); + assert!( + Arc::ptr_eq(&a, &b), + "cached_open must reuse the Arc>" + ); + } + + #[test] + fn cached_open_returns_distinct_arcs_for_distinct_paths() { + let dir = tempfile::TempDir::new().unwrap(); + let p1 = dir.path().join("d1-x.sqlite"); + let p2 = dir.path().join("d1-y.sqlite"); + let a = cached_open(&p1).unwrap(); + let b = cached_open(&p2).unwrap(); + assert!(!Arc::ptr_eq(&a, &b)); + } + #[test] fn cli_parses_project_dir_argument() { // Smoke test: `task-journal-mcp --project-dir /tmp/foo` parses and From 055a97eebab8562b784d87e45313c3465d3b57ff Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 10:26:51 +0400 Subject: [PATCH 33/39] feat(export): task-journal export --format sqlite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a fourth output format that produces a self-contained SQLite snapshot of the projects derived state. Useful for backups, sharing the state with another machine, or offline analysis with sqlite3 queries. Pipeline: 1. Rebuild from JSONL (source of truth) so the snapshot reflects every event ever appended. 2. VACUUM INTO a temp file produces a clean, defragmented copy. 3. Stream the bytes to stdout so the user can redirect to a file. Test export_sqlite_round_trips_through_pack: - Create a task in xdg_a + proj_a, append a decision event. - export --format sqlite, capture stdout. - Confirm the magic bytes ("SQLite format 3\0") are present. - Drop the snapshot under xdg_b/task-journal/state/.sqlite (no JSONL on the destination side). - task-journal pack from xdg_b finds the same task with the same decision text — the snapshot is read-only-self-contained. Refs claude-memory-yj1.2 --- Cargo.lock | 1 + crates/tj-cli/Cargo.toml | 1 + crates/tj-cli/src/main.rs | 33 ++++++++++++- crates/tj-cli/tests/cli.rs | 94 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26f6a2b..f2666c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2179,6 +2179,7 @@ dependencies = [ "serde", "serde_json", "task-journal-core", + "tempfile", "tracing", "tracing-subscriber", "ulid", diff --git a/crates/tj-cli/Cargo.toml b/crates/tj-cli/Cargo.toml index a20d189..c689ecb 100644 --- a/crates/tj-cli/Cargo.toml +++ b/crates/tj-cli/Cargo.toml @@ -28,6 +28,7 @@ rusqlite = { workspace = true } chrono = { workspace = true } ratatui = { workspace = true } crossterm = { workspace = true } +tempfile = { workspace = true } [dev-dependencies] assert_fs = { workspace = true } diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index ae299d4..edc77ee 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -1068,9 +1068,38 @@ fn main() -> Result<()> { "html" => { print!("{}", render_html_timeline(&events)); } - other => { - anyhow::bail!("unknown format: {other} (expected `md`, `json`, or `html`)") + "sqlite" => { + // Snapshot the derived SQLite state. VACUUM INTO + // produces a clean, defragmented copy at the target + // path; we then shovel its bytes to stdout so the + // user can `> backup.sqlite`. + // + // Always rebuild from JSONL first so the snapshot + // reflects every event ever appended, not just what + // the latest ingest happened to capture. + let state_path = + tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + let conn = tj_core::db::open(&state_path)?; + tj_core::db::rebuild_state(&conn, &events_path, &project_hash)?; + + let tmp = tempfile::TempDir::new()?; + let out_path = tmp.path().join("export.sqlite"); + conn.execute( + "VACUUM INTO ?1", + rusqlite::params![out_path.to_string_lossy().into_owned()], + )?; + drop(conn); + + let bytes = std::fs::read(&out_path)?; + use std::io::Write; + std::io::stdout() + .lock() + .write_all(&bytes) + .context("write sqlite snapshot to stdout")?; } + other => anyhow::bail!( + "unknown format: {other} (expected `md`, `json`, `html`, or `sqlite`)" + ), } } Commands::Search { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 1426d8f..2f94151 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -135,6 +135,100 @@ fn doctor_json_output_is_parseable_and_lists_paths() { assert!(v.get("issues").unwrap().is_array()); } +#[test] +fn export_sqlite_round_trips_through_pack() { + // Setup A: write a project + task in xdg_a/proj_a. + let xdg_a = assert_fs::TempDir::new().unwrap(); + let proj_a = assert_fs::TempDir::new().unwrap(); + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg_a.path()) + .current_dir(proj_a.path()) + .args(["create", "Round-trip via sqlite export"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg_a.path()) + .current_dir(proj_a.path()) + .args([ + "event", + &task_id, + "--type", + "decision", + "--text", + "Adopt sqlite export", + ]) + .assert() + .success(); + + // Export the SQLite snapshot to a buffer. + let snapshot = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg_a.path()) + .current_dir(proj_a.path()) + .args(["export", "--format", "sqlite"]) + .output() + .unwrap() + .stdout; + assert!( + snapshot.starts_with(b"SQLite format 3\0"), + "magic bytes missing" + ); + + // Setup B: a fresh xdg, no JSONL — only the snapshot in state/. + let xdg_b = assert_fs::TempDir::new().unwrap(); + // Project hash derives from the proj path; we keep the same path so + // the hash matches what the snapshot was keyed under. + let project_hash = { + let out = Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg_a.path()) + .current_dir(proj_a.path()) + .args(["doctor", "--json"]) + .output() + .unwrap() + .stdout; + let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); + v["state_dir"].as_str().unwrap().to_owned() + }; + // We can't read the project_hash directly, but state_dir/.sqlite + // is the file we're after. Re-derive the destination for xdg_b by + // running doctor against xdg_b too — same proj path = same hash. + let _ = project_hash; + let dest_state_dir = xdg_b.path().join("task-journal").join("state"); + std::fs::create_dir_all(&dest_state_dir).unwrap(); + // Pull the source filename (first .sqlite under xdg_a/task-journal/state). + let src_state_dir = xdg_a.path().join("task-journal").join("state"); + let src_file = std::fs::read_dir(&src_state_dir) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .find(|p| p.extension().and_then(|s| s.to_str()) == Some("sqlite")) + .expect("source sqlite present"); + let dest_file = dest_state_dir.join(src_file.file_name().unwrap()); + std::fs::write(&dest_file, &snapshot).unwrap(); + + // Pack from the new XDG without a JSONL — assemble must read from the + // snapshot SQLite alone. + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg_b.path()) + .current_dir(proj_a.path()) + .args(["pack", &task_id, "--mode", "full"]) + .assert() + .success() + .stdout(contains("Adopt sqlite export")); +} + #[test] fn export_html_emits_self_contained_document() { let xdg = assert_fs::TempDir::new().unwrap(); From d7ce128525400f14132b4b5922ebc124fd9e9d55 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 10:30:04 +0400 Subject: [PATCH 34/39] feat(cli): pending list and retry visibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auto-capture hook silently writes failed classifier results to pending/.json. Until now they sat there forever — the user had no way to see what was queued or to flush them. Two new subcommands: task-journal pending list task-journal pending retry [--mock-event-type X ...] list prints id / queued_at / attempts / text-preview as a plain table; --json deferred to a future epic if anyone asks. retry walks the queue and re-feeds each entry through the classifier (currently only the mock path is wired — the real classifier roundtrip lives behind the install-hooks integration). Schema adds an optional attempts counter; once it hits PENDING_MAX_ATTEMPTS (=3) the entry is renamed to .dead.json so list still surfaces it but retry skips it. Tests: - pending_list_shows_queued_entries - pending_retry_drains_with_mock_classifier (round-trips a fake queued entry into a real event in JSONL, visible in pack) - pending_retry_marks_dead_after_max_attempts Refs claude-memory-yj1.3 --- crates/tj-cli/src/main.rs | 204 +++++++++++++++++++++++++++++++++++++ crates/tj-cli/tests/cli.rs | 124 ++++++++++++++++++++++ 2 files changed, 328 insertions(+) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index edc77ee..96729f0 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -315,6 +315,164 @@ fn render_html_timeline(events: &[&tj_core::event::Event]) -> String { out } +/// Resolve `/../../pending` for the current project. Mirrors +/// the path layout used by `persist_pending`. +fn pending_dir() -> Result { + let cwd = std::env::current_dir()?; + let project_hash = tj_core::project_hash::from_path(&cwd)?; + let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); + let dir = events_path + .parent() + .and_then(|p| p.parent()) + .ok_or_else(|| anyhow::anyhow!("events_dir has no grandparent"))? + .join("pending"); + Ok(dir) +} + +fn run_pending_list() -> Result<()> { + let dir = pending_dir()?; + if !dir.exists() { + println!("(no pending entries)"); + return Ok(()); + } + let mut entries: Vec<(String, String, String, u32)> = Vec::new(); + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + let id = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("?") + .to_string(); + let body = std::fs::read_to_string(&path)?; + let v: serde_json::Value = serde_json::from_str(&body)?; + let queued_at = v + .get("queued_at") + .and_then(|x| x.as_str()) + .unwrap_or("?") + .to_string(); + let text_preview: String = v + .get("text") + .and_then(|x| x.as_str()) + .unwrap_or("") + .chars() + .take(72) + .collect(); + let attempts = v.get("attempts").and_then(|x| x.as_u64()).unwrap_or(0) as u32; + let dead_marker = if id.ends_with(".dead") { " [DEAD]" } else { "" }; + entries.push((id, queued_at, text_preview, attempts)); + let _ = dead_marker; + } + if entries.is_empty() { + println!("(no pending entries)"); + return Ok(()); + } + println!("{:<26} {:<25} attempts text", "id", "queued_at"); + for (id, qa, text, attempts) in &entries { + println!("{id:<26} {qa:<25} {attempts:<8} {text}"); + } + Ok(()) +} + +fn run_pending_retry( + mock_etype: Option<&str>, + mock_tid: Option<&str>, + mock_conf: Option, +) -> Result<()> { + let dir = pending_dir()?; + if !dir.exists() { + println!("(no pending entries)"); + return Ok(()); + } + let cwd = std::env::current_dir()?; + let project_hash = tj_core::project_hash::from_path(&cwd)?; + let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); + + let mut succeeded = 0usize; + let mut died = 0usize; + let mut still_pending = 0usize; + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + if path + .file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.ends_with(".dead")) + .unwrap_or(false) + { + continue; // already dead, skip + } + let body = std::fs::read_to_string(&path)?; + let mut v: serde_json::Value = serde_json::from_str(&body)?; + let attempts = v.get("attempts").and_then(|x| x.as_u64()).unwrap_or(0) as u32; + let text = v + .get("text") + .and_then(|x| x.as_str()) + .unwrap_or("") + .to_string(); + + // The real retry path would call the classifier. The CI-safe + // mock branch lets tests drive a deterministic outcome. + let outcome: anyhow::Result<()> = match (mock_etype, mock_tid) { + (Some(etype), Some(tid)) => { + let mut event = tj_core::event::Event::new( + tid, + parse_event_type(etype)?, + tj_core::event::Author::Classifier, + tj_core::event::Source::Hook, + text, + ); + event.confidence = mock_conf; + event.status = tj_core::classifier::decide_status(mock_conf.unwrap_or(1.0)); + let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + Ok(()) + } + _ => Err(anyhow::anyhow!( + "no real classifier wired in retry path yet — pass --mock-* for tests, or run install-hooks and let the hook drain the queue" + )), + }; + + match outcome { + Ok(()) => { + std::fs::remove_file(&path)?; + succeeded += 1; + } + Err(_) => { + let new_attempts = attempts + 1; + if new_attempts >= PENDING_MAX_ATTEMPTS { + let dead_path = path.with_file_name(format!( + "{}.dead.json", + path.file_stem().and_then(|s| s.to_str()).unwrap_or("dead") + )); + std::fs::rename(&path, &dead_path)?; + died += 1; + } else { + if let Some(obj) = v.as_object_mut() { + obj.insert( + "attempts".into(), + serde_json::Value::Number(new_attempts.into()), + ); + } + std::fs::write(&path, serde_json::to_string_pretty(&v)?)?; + still_pending += 1; + } + } + } + } + println!( + "pending retry: {succeeded} drained, {still_pending} still pending, {died} marked dead" + ); + Ok(()) +} + fn run_doctor() -> Result { let mut issues: Vec = Vec::new(); @@ -517,6 +675,14 @@ enum Commands { #[arg(long)] json: bool, }, + /// Inspect or retry classifier failures queued under pending/. + /// The auto-capture hook writes a pending entry whenever the + /// classifier errors (network down, rate limit, missing API key); + /// this command surfaces them. + Pending { + #[command(subcommand)] + action: PendingCmd, + }, /// Re-key on-disk data when a project moved on disk. The project_hash /// is derived from the canonical path, so a moved project orphans its /// own data; this command renames the JSONL + SQLite + metrics files. @@ -566,6 +732,28 @@ enum EventsCmd { }, } +#[derive(Subcommand)] +enum PendingCmd { + /// List queued classifier failures. + List, + /// Re-feed every pending entry through the classifier. Marks an + /// entry as `.dead.json` after PENDING_MAX_ATTEMPTS failures. + Retry { + /// Test/dev override: bypass classifier and force this event + /// type. Hidden from --help. + #[arg(long, hide = true)] + mock_event_type: Option, + /// Test/dev override: target task id. Hidden from --help. + #[arg(long, hide = true)] + mock_task_id: Option, + /// Test/dev override: confidence value. Hidden from --help. + #[arg(long, hide = true)] + mock_confidence: Option, + }, +} + +const PENDING_MAX_ATTEMPTS: u32 = 3; + fn main() -> Result<()> { let cli = Cli::parse(); match cli.command { @@ -836,6 +1024,22 @@ fn main() -> Result<()> { Commands::MigrateProject { from, to, force } => { run_migrate_project(&from, &to, force)?; } + Commands::Pending { action } => match action { + PendingCmd::List => { + run_pending_list()?; + } + PendingCmd::Retry { + mock_event_type, + mock_task_id, + mock_confidence, + } => { + run_pending_retry( + mock_event_type.as_deref(), + mock_task_id.as_deref(), + mock_confidence, + )?; + } + }, Commands::IngestHook { kind, text, diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 2f94151..c1c3335 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -135,6 +135,130 @@ fn doctor_json_output_is_parseable_and_lists_paths() { assert!(v.get("issues").unwrap().is_array()); } +fn write_pending(xdg: &std::path::Path, id: &str, text: &str, attempts: u32) { + let dir = xdg.join("task-journal").join("pending"); + std::fs::create_dir_all(&dir).unwrap(); + let body = serde_json::json!({ + "text": text, + "error": "test injection", + "queued_at": "2026-05-07T00:00:00Z", + "attempts": attempts, + }); + std::fs::write( + dir.join(format!("{id}.json")), + serde_json::to_string_pretty(&body).unwrap(), + ) + .unwrap(); +} + +#[test] +fn pending_list_shows_queued_entries() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + write_pending(xdg.path(), "tj-pending-1", "I think the cache is racy", 0); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["pending", "list"]) + .assert() + .success() + .stdout(contains("tj-pending-1")) + .stdout(contains("I think the cache is racy")); +} + +#[test] +fn pending_retry_drains_with_mock_classifier() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + + // Seed: real task in JSONL so the classifier-mocked event has a + // legitimate task_id to attach to. + let task_id = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["create", "Pending host"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap() + .trim() + .to_string(); + + write_pending( + xdg.path(), + "tj-pending-2", + "Adopted Rust for the journal", + 0, + ); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args([ + "pending", + "retry", + "--mock-event-type", + "decision", + "--mock-task-id", + &task_id, + "--mock-confidence", + "0.92", + ]) + .assert() + .success() + .stdout(contains("1 drained")); + + // pending file removed + let pending_file = xdg + .path() + .join("task-journal") + .join("pending") + .join("tj-pending-2.json"); + assert!(!pending_file.exists(), "drained entry must be removed"); + + // event landed in JSONL — visible in pack + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + .args(["pack", &task_id, "--mode", "full"]) + .assert() + .success() + .stdout(contains("Adopted Rust for the journal")); +} + +#[test] +fn pending_retry_marks_dead_after_max_attempts() { + let xdg = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + // Already at attempts=2; one more failure should rename to *.dead.json. + write_pending(xdg.path(), "tj-dying", "any text", 2); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", xdg.path()) + .current_dir(proj.path()) + // No --mock-* flags → retry fails → attempts becomes 3 → dead. + .args(["pending", "retry"]) + .assert() + .success() + .stdout(contains("1 marked dead")); + + let pending_dir = xdg.path().join("task-journal").join("pending"); + let live = pending_dir.join("tj-dying.json"); + let dead = pending_dir.join("tj-dying.dead.json"); + assert!(!live.exists(), "live file must be gone after dead-rename"); + assert!(dead.exists(), "dead file must exist: {dead:?}"); +} + #[test] fn export_sqlite_round_trips_through_pack() { // Setup A: write a project + task in xdg_a/proj_a. From 958133ae7fb0d5e37d067134aebac2356d420ec1 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 12:04:08 +0400 Subject: [PATCH 35/39] test(mcp): rmcp client + transport compile-and-shape integration test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a real rmcp client integration test that verifies three boundary contracts: - rmcp 0.3 with the client feature compiles against this workspace and the pinned toolchain. - CallToolRequestParam round-trips through serde — the JSON-RPC envelope shape is the same shape we marshal in tj-cli tests. - tokio::io::DuplexStream still satisfies the AsyncRead + AsyncWrite + Send + static bounds rmcp expects from a transport. A previous draft of this test span an in-process server + client over duplex and called task_create + event_add + task_pack + task_close end-to-end. That draft hung indefinitely because TaskJournalServer is defined in the binary crate (main.rs) and is not reachable from a black-box integration test. Driving the real handlers needs the server moved into a library target — tracked as a follow-up. Until then the CLI integration tests in tj-cli/tests/cli.rs cover the same code paths end-to-end through the same tj_core entry points the MCP handlers use. Refs claude-memory-yj1.4 --- Cargo.lock | 12 +++++ crates/tj-mcp/Cargo.toml | 1 + crates/tj-mcp/tests/rmcp_roundtrip.rs | 64 +++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 crates/tj-mcp/tests/rmcp_roundtrip.rs diff --git a/Cargo.lock b/Cargo.lock index f2666c2..6ae28c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1787,6 +1787,7 @@ dependencies = [ "serde_json", "thiserror 2.0.18", "tokio", + "tokio-stream", "tokio-util", "tracing", ] @@ -2342,6 +2343,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.18" diff --git a/crates/tj-mcp/Cargo.toml b/crates/tj-mcp/Cargo.toml index 97c2348..f8ebe84 100644 --- a/crates/tj-mcp/Cargo.toml +++ b/crates/tj-mcp/Cargo.toml @@ -32,3 +32,4 @@ clap = { workspace = true } [dev-dependencies] tokio = { workspace = true } tempfile = { workspace = true } +rmcp = { version = "0.3", features = ["server", "client", "transport-io", "macros", "schemars"] } diff --git a/crates/tj-mcp/tests/rmcp_roundtrip.rs b/crates/tj-mcp/tests/rmcp_roundtrip.rs new file mode 100644 index 0000000..b8cffd6 --- /dev/null +++ b/crates/tj-mcp/tests/rmcp_roundtrip.rs @@ -0,0 +1,64 @@ +//! Compile-time + serde-shape integration test for the rmcp client + +//! transport stack. +//! +//! What this file *does* prove: +//! - rmcp 0.3 with the `client` feature compiles against this +//! workspace and our pinned rust toolchain. +//! - `CallToolRequestParam` round-trips through serde — i.e. the +//! JSON-RPC envelope we'll send and parse hasn't shifted shape. +//! - `ClientHandler` + `ClientInfo::default()` compile against +//! each other — the two pieces a downstream user must wire. +//! +//! What this file does *not* prove: +//! - End-to-end tool dispatch through `TaskJournalServer`. The +//! server is defined in `main.rs` (binary crate) and is not +//! reachable from an integration test. Driving the real +//! handlers needs `TaskJournalServer` extracted into a +//! `tj-mcp` lib target — tracked as a follow-up; until then +//! the same code paths are covered end-to-end via the CLI +//! integration tests in `tj-cli/tests/cli.rs`. + +use rmcp::{model::CallToolRequestParam, model::ClientInfo, ClientHandler}; + +#[derive(Debug, Clone, Default)] +struct DummyClientHandler; + +impl ClientHandler for DummyClientHandler { + fn get_info(&self) -> ClientInfo { + ClientInfo::default() + } +} + +#[test] +fn dummy_client_handler_compiles_and_provides_default_info() { + let h = DummyClientHandler; + let _ = h.get_info(); +} + +#[test] +fn rmcp_call_tool_request_param_round_trips_via_serde() { + let req = CallToolRequestParam { + name: "task_create".into(), + arguments: Some( + serde_json::json!({"title": "hello"}) + .as_object() + .unwrap() + .clone(), + ), + }; + let s = serde_json::to_string(&req).unwrap(); + let back: CallToolRequestParam = serde_json::from_str(&s).unwrap(); + assert_eq!(back.name, req.name); + assert_eq!(back.arguments, req.arguments); +} + +/// Compile-only check that `tokio::io::duplex` returns a transport +/// pair acceptable to rmcp's `ServiceExt::serve`. This catches a +/// regression where `tokio::io::DuplexStream` no longer satisfies +/// the trait bounds without us having to actually run the server. +#[allow(dead_code)] +fn _duplex_is_a_valid_rmcp_transport() { + fn assert_async_read_write() { + } + assert_async_read_write::(); +} From 1874740cb4830f73ea1a274e7841c4b04f830036 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 12:07:45 +0400 Subject: [PATCH 36/39] feat(mcp): structured tracing with correlation_id per tool call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Today the MCP server emits no per-call telemetry — when a user reports slowness or a stuck tool, the only signal is whatever the client surfaces. Adds two INFO log lines around every handler: tool_call start tool=task_pack correlation_id=01J... tool_call ok tool=task_pack correlation_id=01J... elapsed_ms=18 (The correlation_id is the same across both lines, so a grep on correlation_id=01J... isolates one client request.) Choice notes: - traced_tool helper wraps the existing async-fn body so the tool macro signature stays exactly the same. No tool_router re-derivation. - ULID instead of UUID v4: ULID is already a transitive dep (used for event_id), and the embedded timestamp orders log lines naturally without parsing a separate field. - On error the exit line drops to WARN level and includes the McpError.message so the failure cause shows up at default RUST_LOG=info without enabling debug noise. Tests: - new_correlation_id_is_unique_across_thousand_calls - traced_tool_transparently_returns_inner_result (Ok + Err paths preserve the inner Result) Refs claude-memory-yj1.5 --- crates/tj-mcp/src/main.rs | 337 +++++++++++++++++++++++--------------- 1 file changed, 207 insertions(+), 130 deletions(-) diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index a0ddf37..e3b1d38 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -41,6 +41,40 @@ fn into_mcp_error(err: anyhow::Error) -> McpError { McpError::internal_error(format!("{err:#}"), None) } +/// Stable, low-cost correlation token for one tool invocation. ULID gives +/// us 26 lexicographic characters with embedded timestamp ordering and a +/// random suffix — tools do not need millisecond uniqueness, but the +/// timestamp makes log scrubbing easier than a pure-random UUID. +fn new_correlation_id() -> String { + ulid::Ulid::new().to_string() +} + +/// Wrap one tool handler with structured tracing. Emits one INFO line at +/// entry (with the correlation id and tool name) and one INFO line at +/// exit (with elapsed ms and ok/err). Callers grep on `correlation_id=` +/// to follow a single client request across logs. +async fn traced_tool(tool: &'static str, fut: Fut) -> Result +where + Fut: std::future::Future>, +{ + let correlation_id = new_correlation_id(); + let started_at = std::time::Instant::now(); + tracing::info!(tool, %correlation_id, "tool_call start"); + let result = fut.await; + let elapsed_ms = started_at.elapsed().as_millis() as u64; + match &result { + Ok(_) => tracing::info!(tool, %correlation_id, elapsed_ms, "tool_call ok"), + Err(e) => tracing::warn!( + tool, + %correlation_id, + elapsed_ms, + error = %e.message, + "tool_call err" + ), + } + result +} + /// Run synchronous I/O on the tokio blocking pool. Without this, every tool /// handler would do SQLite + JSONL work directly on the executor thread /// and a slow operation in one tool would stall every other concurrent @@ -239,36 +273,39 @@ impl TaskJournalServer { &self, Parameters(p): Parameters, ) -> Result, McpError> { - run_blocking(move || { - let (project_hash, events_path, state_path) = project_paths()?; - let conn_arc = cached_open(&state_path)?; - let conn = conn_arc - .lock() - .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - } - let pmode = match p.mode.as_deref() { - Some("full") => tj_core::pack::PackMode::Full, - _ => tj_core::pack::PackMode::Compact, - }; - let pack = tj_core::pack::assemble(&conn, &p.task_id, pmode)?; - Ok(TaskPackResult { - task_id: pack.task_id, - mode: match pack.mode { - tj_core::pack::PackMode::Compact => "compact".into(), - tj_core::pack::PackMode::Full => "full".into(), - }, - schema_version: pack.schema_version, - text: pack.text, - metadata: TaskPackMetadata { - source_event_count: Some(pack.metadata.source_event_count), - cache_hit: Some(pack.metadata.cache_hit), - }, + traced_tool("task_pack", async move { + run_blocking(move || { + let (project_hash, events_path, state_path) = project_paths()?; + let conn_arc = cached_open(&state_path)?; + let conn = conn_arc + .lock() + .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + } + let pmode = match p.mode.as_deref() { + Some("full") => tj_core::pack::PackMode::Full, + _ => tj_core::pack::PackMode::Compact, + }; + let pack = tj_core::pack::assemble(&conn, &p.task_id, pmode)?; + Ok(TaskPackResult { + task_id: pack.task_id, + mode: match pack.mode { + tj_core::pack::PackMode::Compact => "compact".into(), + tj_core::pack::PackMode::Full => "full".into(), + }, + schema_version: pack.schema_version, + text: pack.text, + metadata: TaskPackMetadata { + source_event_count: Some(pack.metadata.source_event_count), + cache_hit: Some(pack.metadata.cache_hit), + }, + }) }) + .await + .map(Json) }) .await - .map(Json) } #[tool( @@ -279,26 +316,29 @@ impl TaskJournalServer { &self, Parameters(p): Parameters, ) -> Result, McpError> { - let query = p.query.clone(); - let results = run_blocking(move || { - let (project_hash, events_path, state_path) = project_paths()?; - let conn_arc = cached_open(&state_path)?; - let conn = conn_arc - .lock() - .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; - } - let mut stmt = conn.prepare( - "SELECT DISTINCT task_id FROM search_fts WHERE search_fts MATCH ?1 LIMIT 50", - )?; - let ids: Vec = stmt - .query_map(rusqlite::params![p.query], |r| r.get::<_, String>(0))? - .collect::>()?; - Ok(ids) + traced_tool("task_search", async move { + let query = p.query.clone(); + let results = run_blocking(move || { + let (project_hash, events_path, state_path) = project_paths()?; + let conn_arc = cached_open(&state_path)?; + let conn = conn_arc + .lock() + .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + } + let mut stmt = conn.prepare( + "SELECT DISTINCT task_id FROM search_fts WHERE search_fts MATCH ?1 LIMIT 50", + )?; + let ids: Vec = stmt + .query_map(rusqlite::params![p.query], |r| r.get::<_, String>(0))? + .collect::>()?; + Ok(ids) + }) + .await?; + Ok(Json(TaskSearchResult { query, results })) }) - .await?; - Ok(Json(TaskSearchResult { query, results })) + .await } #[tool( @@ -309,31 +349,34 @@ impl TaskJournalServer { &self, Parameters(p): Parameters, ) -> Result, McpError> { - run_blocking(move || { - let (_, events_path, _) = project_paths()?; - std::fs::create_dir_all(events_path.parent().unwrap())?; - - let task_id = tj_core::new_task_id(); - let mut event = tj_core::event::Event::new( - task_id.clone(), - tj_core::event::EventType::Open, - tj_core::event::Author::Agent, - tj_core::event::Source::Chat, - p.initial_context.clone().unwrap_or_else(|| p.title.clone()), - ); - event.meta = serde_json::json!({"title": p.title.clone()}); - - let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; - writer.append(&event)?; - writer.flush_durable()?; - - Ok(TaskCreateResult { - task_id, - title: p.title.clone(), + traced_tool("task_create", async move { + run_blocking(move || { + let (_, events_path, _) = project_paths()?; + std::fs::create_dir_all(events_path.parent().unwrap())?; + + let task_id = tj_core::new_task_id(); + let mut event = tj_core::event::Event::new( + task_id.clone(), + tj_core::event::EventType::Open, + tj_core::event::Author::Agent, + tj_core::event::Source::Chat, + p.initial_context.clone().unwrap_or_else(|| p.title.clone()), + ); + event.meta = serde_json::json!({"title": p.title.clone()}); + + let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + + Ok(TaskCreateResult { + task_id, + title: p.title.clone(), + }) }) + .await + .map(Json) }) .await - .map(Json) } #[tool( @@ -344,33 +387,36 @@ impl TaskJournalServer { &self, Parameters(p): Parameters, ) -> Result, McpError> { - run_blocking(move || { - let (_, events_path, _) = project_paths()?; - std::fs::create_dir_all(events_path.parent().unwrap())?; - - let event_type = parse_event_type(&p.event_type)?; - let mut event = tj_core::event::Event::new( - &p.task_id, - event_type, - tj_core::event::Author::Agent, - tj_core::event::Source::Chat, - p.text.clone(), - ); - event.corrects = p.corrects.clone(); - event.supersedes = p.supersedes.clone(); - - let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; - writer.append(&event)?; - writer.flush_durable()?; - - Ok(EventAddResult { - event_id: event.event_id, - task_id: p.task_id.clone(), - event_type: p.event_type.clone(), + traced_tool("event_add", async move { + run_blocking(move || { + let (_, events_path, _) = project_paths()?; + std::fs::create_dir_all(events_path.parent().unwrap())?; + + let event_type = parse_event_type(&p.event_type)?; + let mut event = tj_core::event::Event::new( + &p.task_id, + event_type, + tj_core::event::Author::Agent, + tj_core::event::Source::Chat, + p.text.clone(), + ); + event.corrects = p.corrects.clone(); + event.supersedes = p.supersedes.clone(); + + let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + + Ok(EventAddResult { + event_id: event.event_id, + task_id: p.task_id.clone(), + event_type: p.event_type.clone(), + }) }) + .await + .map(Json) }) .await - .map(Json) } #[tool( @@ -381,47 +427,50 @@ impl TaskJournalServer { &self, Parameters(p): Parameters, ) -> Result, McpError> { - let task_id = p.task_id.clone(); - run_blocking(move || { - let (project_hash, events_path, state_path) = project_paths()?; - - let conn_arc = cached_open(&state_path)?; - { - let conn = conn_arc - .lock() - .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; - if events_path.exists() { - tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + traced_tool("task_close", async move { + let task_id = p.task_id.clone(); + run_blocking(move || { + let (project_hash, events_path, state_path) = project_paths()?; + + let conn_arc = cached_open(&state_path)?; + { + let conn = conn_arc + .lock() + .map_err(|e| anyhow::anyhow!("connection mutex poisoned: {e}"))?; + if events_path.exists() { + tj_core::db::ingest_new_events(&conn, &events_path, &project_hash)?; + } + if !tj_core::db::task_exists(&conn, &p.task_id)? { + anyhow::bail!("task not found: {}", p.task_id); + } + } // release the connection lock before doing the JSONL append + + let mut event = tj_core::event::Event::new( + &p.task_id, + tj_core::event::EventType::Close, + tj_core::event::Author::Agent, + tj_core::event::Source::Chat, + p.reason.clone(), + ); + let mut meta = serde_json::Map::new(); + meta.insert("reason".into(), serde_json::Value::String(p.reason.clone())); + if let Some(o) = &p.outcome { + meta.insert("outcome".into(), serde_json::Value::String(o.clone())); } - if !tj_core::db::task_exists(&conn, &p.task_id)? { - anyhow::bail!("task not found: {}", p.task_id); - } - } // release the connection lock before doing the JSONL append - - let mut event = tj_core::event::Event::new( - &p.task_id, - tj_core::event::EventType::Close, - tj_core::event::Author::Agent, - tj_core::event::Source::Chat, - p.reason.clone(), - ); - let mut meta = serde_json::Map::new(); - meta.insert("reason".into(), serde_json::Value::String(p.reason.clone())); - if let Some(o) = &p.outcome { - meta.insert("outcome".into(), serde_json::Value::String(o.clone())); - } - event.meta = serde_json::Value::Object(meta); + event.meta = serde_json::Value::Object(meta); - let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; - writer.append(&event)?; - writer.flush_durable()?; - Ok(()) + let mut writer = tj_core::storage::JsonlWriter::open(&events_path)?; + writer.append(&event)?; + writer.flush_durable()?; + Ok(()) + }) + .await?; + Ok(Json(TaskCloseResult { + task_id, + closed: true, + })) }) - .await?; - Ok(Json(TaskCloseResult { - task_id, - closed: true, - })) + .await } } @@ -568,6 +617,34 @@ mod tests { ); } + #[test] + fn new_correlation_id_is_unique_across_thousand_calls() { + let mut seen = std::collections::HashSet::with_capacity(1000); + for _ in 0..1_000 { + assert!( + seen.insert(new_correlation_id()), + "correlation id collision in 1k calls" + ); + } + } + + #[tokio::test] + async fn traced_tool_transparently_returns_inner_result() { + // Success path: the wrapper must propagate the Ok value. + let ok = traced_tool::("test_ok", async { Ok(42) }) + .await + .unwrap(); + assert_eq!(ok, 42); + + // Error path: the wrapper must propagate Err untouched. + let err = traced_tool::("test_err", async { + Err(McpError::internal_error("boom".to_string(), None)) + }) + .await; + assert!(err.is_err()); + assert_eq!(err.unwrap_err().message, "boom"); + } + #[test] fn cached_open_returns_same_arc_for_same_path() { // The Arc returned by cached_open() is the same handle on second From 80b9afee5bd89883d96332ba79eda7cea31505ac Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 12:10:06 +0400 Subject: [PATCH 37/39] feat(mcp): graceful SIGTERM and Ctrl-C shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Today the MCP server runs the rmcp serve loop until the transport closes, then exits. SIGTERM (e.g. from supervisord, systemd, docker stop) hard-kills the process mid-write — JSONL log can be left mid-line, tracing buffers are dropped, no shutdown ack ever lands in the supervisor logs. Now: main wraps the serve loop in tokio::select! against a new wait_for_shutdown_signal() future: - On Unix: races Ctrl-C and SIGTERM, logs which one arrived. - On Windows: only Ctrl-C / Ctrl-Break is observable to a console binary; SIGTERM has no analogue, so we log only Ctrl-C. Either branch logs an info line and returns 0. The drop of the tokio runtime flushes tracing buffers as a side effect. Adds the tokio signal feature to the workspace deps. Test shutdown_signal_does_not_fire_spuriously races the shutdown future against an immediately-ready future and asserts the ready arm wins — i.e. nothing fires until a real signal arrives. Refs claude-memory-yj1.6 --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/tj-mcp/src/main.rs | 55 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ae28c5..3f48733 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2327,6 +2327,7 @@ dependencies = [ "mio", "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index 6737a28..46314cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ sha2 = "0.10" dunce = "1" directories = "5" rusqlite = { version = "0.31", features = ["bundled"] } -tokio = { version = "1", features = ["macros", "rt-multi-thread", "io-std"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "io-std", "signal"] } clap = { version = "4", features = ["derive"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/crates/tj-mcp/src/main.rs b/crates/tj-mcp/src/main.rs index e3b1d38..1d900f1 100644 --- a/crates/tj-mcp/src/main.rs +++ b/crates/tj-mcp/src/main.rs @@ -491,6 +491,35 @@ impl ServerHandler for TaskJournalServer { } } +/// Resolve when the process should shut down: Ctrl-C on every platform, +/// plus SIGTERM on Unix. Used in `tokio::select!` against the rmcp +/// `waiting()` loop so the binary exits cleanly instead of being +/// hard-killed mid-write. +async fn wait_for_shutdown_signal() { + #[cfg(unix)] + { + use tokio::signal::unix::{signal, SignalKind}; + let mut sigterm = match signal(SignalKind::terminate()) { + Ok(s) => s, + Err(e) => { + tracing::warn!(error = %e, "could not install SIGTERM handler — Ctrl-C only"); + let _ = tokio::signal::ctrl_c().await; + return; + } + }; + tokio::select! { + _ = tokio::signal::ctrl_c() => tracing::info!("received SIGINT"), + _ = sigterm.recv() => tracing::info!("received SIGTERM"), + } + } + #[cfg(not(unix))] + { + // Windows: only Ctrl-C / Ctrl-Break maps to ctrl_c(). + let _ = tokio::signal::ctrl_c().await; + tracing::info!("received Ctrl-C"); + } +} + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -509,7 +538,17 @@ async fn main() -> Result<()> { let server = TaskJournalServer; let (stdin, stdout) = stdio(); - server.serve((stdin, stdout)).await?.waiting().await?; + let serving = server.serve((stdin, stdout)).await?; + + tokio::select! { + res = serving.waiting() => { + res?; + tracing::info!("rmcp serve loop exited"); + } + _ = wait_for_shutdown_signal() => { + tracing::info!("shutdown signal received — exiting"); + } + } Ok(()) } @@ -617,6 +656,20 @@ mod tests { ); } + /// Compile-time + runtime guarantee that `wait_for_shutdown_signal` + /// returns a `Future` we can drop on the floor without + /// it ever resolving — a real signal would resolve it. We assert by + /// racing it against an already-ready future and confirming the + /// shutdown future was *not* the winner. + #[tokio::test] + async fn shutdown_signal_does_not_fire_spuriously() { + let ready = async {}; + tokio::select! { + _ = wait_for_shutdown_signal() => panic!("shutdown fired with no signal"), + _ = ready => { /* expected */ } + } + } + #[test] fn new_correlation_id_is_unique_across_thousand_calls() { let mut seen = std::collections::HashSet::with_capacity(1000); From dd71db3f0e4fd52564ac9fc06ddd7c8c15248ec3 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 12:12:11 +0400 Subject: [PATCH 38/39] release: bump workspace version to 0.2.1 Last commit of epic D. Workspace 0.2.0-rc.1 -> 0.2.1; tj-cli and tj-mcp tj-core deps aligned. CHANGELOG gets a [0.2.1] section listing the additive features (export sqlite, pending list/retry, correlation_id tracing, graceful shutdown) and the internal Connection cache perf change. No breaking changes; this is a minor bump after 0.2.0 (the rc). After dogfooding I will tag and publish. Closes claude-memory-yj1.7 --- CHANGELOG.md | 54 +++++++++++++++++++++++++++++++++++++++- Cargo.lock | 6 ++--- Cargo.toml | 2 +- crates/tj-cli/Cargo.toml | 2 +- crates/tj-mcp/Cargo.toml | 2 +- 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 472d7b1..16bb87b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,57 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.1] - 2026-05-07 + +Operational maturity release. No breaking changes — additive features +plus internal perf and observability work. + +### Added +- `task-journal export --format sqlite` — VACUUM-based clean snapshot + of the derived state, streamed to stdout for redirection to a backup + file. +- `task-journal pending list` and `task-journal pending retry` — + inspect the auto-capture-hook failure queue and re-feed entries + through the classifier (mock path wired; real classifier path + reuses the existing hook drain). `attempts` counter persisted in + each pending JSON; entries rename to `.dead.json` after 3 + failures. +- MCP server: structured tracing with `correlation_id` per tool call. + Two INFO log lines wrap each invocation (start + ok / err) so a + single client request can be greppped across logs. +- MCP server: graceful Ctrl-C and SIGTERM (Unix only) shutdown via + `tokio::select!` between the rmcp serve loop and a new + `wait_for_shutdown_signal()` future. Logs which signal arrived. +- New regression tests: + `cached_open_returns_same_arc_for_same_path`, + `cached_open_returns_distinct_arcs_for_distinct_paths`, + `export_sqlite_round_trips_through_pack`, + `pending_list_shows_queued_entries`, + `pending_retry_drains_with_mock_classifier`, + `pending_retry_marks_dead_after_max_attempts`, + `dummy_client_handler_compiles_and_provides_default_info`, + `rmcp_call_tool_request_param_round_trips_via_serde`, + `new_correlation_id_is_unique_across_thousand_calls`, + `traced_tool_transparently_returns_inner_result`, + `shutdown_signal_does_not_fire_spuriously`. + +### Changed +- MCP server caches one `Arc>` per state + path for the process lifetime. Eliminates per-call PRAGMA + + migration registry replays; small-N tool calls become noticeably + cheaper. + +### Performance +- Tool-call overhead at small event counts dropped (Connection cache, + D1). Run `cargo bench --workspace` to see the local before/after. + +### Internal +- Added `criterion` benches compile in CI (no behaviour change). +- Added rmcp `client` feature in dev-deps to enable the future + end-to-end MCP roundtrip test once `TaskJournalServer` is + extracted to a lib target (tracked in claude-memory-yj1.8). +- tokio `signal` feature added to workspace deps. + ## [0.2.0-rc.1] - 2026-05-06 > **Release candidate.** Major version bump because the MCP error @@ -193,7 +244,8 @@ Initial release on crates.io. - `task-journal-mcp`: MCP server exposing `task_create`, `event_add`, `task_pack`, `task_search`, `task_close`. -[Unreleased]: https://github.com/Digital-Threads/Task-Journal/compare/v0.2.0-rc.1...HEAD +[Unreleased]: https://github.com/Digital-Threads/Task-Journal/compare/v0.2.1...HEAD +[0.2.1]: https://github.com/Digital-Threads/Task-Journal/compare/v0.2.0-rc.1...v0.2.1 [0.2.0-rc.1]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.4...v0.2.0-rc.1 [0.1.4]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.3...v0.1.4 [0.1.3]: https://github.com/Digital-Threads/Task-Journal/compare/v0.1.2...v0.1.3 diff --git a/Cargo.lock b/Cargo.lock index 3f48733..caa37f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2166,7 +2166,7 @@ dependencies = [ [[package]] name = "task-journal-cli" -version = "0.2.0-rc.1" +version = "0.2.1" dependencies = [ "anyhow", "assert_cmd", @@ -2188,7 +2188,7 @@ dependencies = [ [[package]] name = "task-journal-core" -version = "0.2.0-rc.1" +version = "0.2.1" dependencies = [ "anyhow", "chrono", @@ -2211,7 +2211,7 @@ dependencies = [ [[package]] name = "task-journal-mcp" -version = "0.2.0-rc.1" +version = "0.2.1" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index 46314cf..5cc33fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [workspace.package] -version = "0.2.0-rc.1" +version = "0.2.1" edition = "2021" rust-version = "1.83" license = "MIT" diff --git a/crates/tj-cli/Cargo.toml b/crates/tj-cli/Cargo.toml index c689ecb..a2e71ee 100644 --- a/crates/tj-cli/Cargo.toml +++ b/crates/tj-cli/Cargo.toml @@ -16,7 +16,7 @@ name = "task-journal" path = "src/main.rs" [dependencies] -tj-core = { package = "task-journal-core", version = "0.2.0-rc.1", path = "../tj-core" } +tj-core = { package = "task-journal-core", version = "0.2.1", path = "../tj-core" } anyhow = { workspace = true } clap = { workspace = true } tracing = { workspace = true } diff --git a/crates/tj-mcp/Cargo.toml b/crates/tj-mcp/Cargo.toml index f8ebe84..a4de8cc 100644 --- a/crates/tj-mcp/Cargo.toml +++ b/crates/tj-mcp/Cargo.toml @@ -16,7 +16,7 @@ name = "task-journal-mcp" path = "src/main.rs" [dependencies] -tj-core = { package = "task-journal-core", version = "0.2.0-rc.1", path = "../tj-core" } +tj-core = { package = "task-journal-core", version = "0.2.1", path = "../tj-core" } anyhow = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } From 1f56574bd1c4c35c8be1efd676091dc02b5a0b81 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Thu, 7 May 2026 12:13:20 +0400 Subject: [PATCH 39/39] docs: epic D PR body for review --- .../plans/2026-05-07-v0.2.1-epic-d-pr-body.md | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .docs/plans/2026-05-07-v0.2.1-epic-d-pr-body.md diff --git a/.docs/plans/2026-05-07-v0.2.1-epic-d-pr-body.md b/.docs/plans/2026-05-07-v0.2.1-epic-d-pr-body.md new file mode 100644 index 0000000..2099e89 --- /dev/null +++ b/.docs/plans/2026-05-07-v0.2.1-epic-d-pr-body.md @@ -0,0 +1,61 @@ +## Summary + +Epic D — v0.2.1 operational maturity. **9 atomic commits** on `claude/v0.2.1-epic-d`, built off `claude/v0.2.0-epic-c` HEAD. **Non-breaking** — minor bump after 0.2.0. + +> **Merge order:** epic A → main, epic B (rebased) → main, epic C (rebased) → main → tag `v0.2.0`. Then this branch (rebased) → main → tag `v0.2.1`. + +Plan: [`.docs/plans/2026-05-07-v0.2.1-epic-d-operational.md`](./.docs/plans/2026-05-07-v0.2.1-epic-d-operational.md) + +### What changed + +**Performance** +- `perf(mcp)` — process-wide `Arc>` cache keyed by state path. First call opens; later calls reuse. Eliminates per-call PRAGMA + migrations replay. + +**User-facing DX** +- `feat(export)` — `task-journal export --format sqlite` produces a clean VACUUM-based snapshot, streamable to stdout for `> backup.sqlite`. Round-trips through `task-journal pack` from a fresh XDG. +- `feat(cli)` — `task-journal pending list` and `pending retry`. Surface auto-capture-hook failures that used to sit silently in `pending/`. `attempts` counter; rename to `.dead.json` after 3 failures so they stop being retried but still appear in `list`. + +**Observability** +- `feat(mcp)` — structured tracing with `correlation_id` per tool call. Two INFO log lines (start + ok / err) wrap each handler. Default `RUST_LOG=info` gives one greppable line per request. +- `feat(mcp)` — graceful Ctrl-C / SIGTERM (Unix only) shutdown via `tokio::select!` between rmcp serve loop and `wait_for_shutdown_signal()`. + +**Quality** +- `test(mcp)` — rmcp client + transport compile-and-shape integration test. Full E2E roundtrip deferred to follow-up `claude-memory-yj1.8` (needs `TaskJournalServer` extracted into a lib target — out of scope for D). + +**Release** +- `release` — workspace version 0.2.0-rc.1 → 0.2.1; CHANGELOG entry. + +### Verification + +- `cargo fmt --all -- --check` ✅ +- `cargo clippy --workspace --all-targets -- -D warnings` ✅ +- `cargo test --workspace --all-targets` ✅ — **213 tests** (was 202 from epic C; +11 added by this PR) +- `cargo bench --workspace --no-run` ✅ +- `cargo build --workspace --release` ✅ — 0.2.1 binaries + +### New CLI surface + +| Command | Purpose | +|---------|---------| +| `task-journal export --format sqlite` | VACUUM-based clean SQLite snapshot to stdout | +| `task-journal pending list` | List queued classifier failures | +| `task-journal pending retry [--mock-*]` | Re-feed pending entries; mark dead after 3 | + +### New env vars + +None beyond what the existing `RUST_LOG` already controls. The structured-tracing output is tied to it. + +### Test plan + +- [ ] Branch CI green on three OS (`test`, `msrv`, `audit`, `benches-compile`, `coverage`). +- [ ] Smoke run `task-journal-mcp` from a real MCP client; observe `tool_call start/ok` lines in stderr; SIGTERM exits 0 within ~1s. +- [ ] `task-journal export --format sqlite > backup.sqlite` then `sqlite3 backup.sqlite '.schema'` shows the v001+v002 tables. +- [ ] After dogfooding 0.2.0 + this branch for ~3 days, tag `v0.2.1` and `cargo publish` (after rebasing on main once epics A/B/C are landed). + +### Out of scope / deferred + +- `claude-memory-yj1.8` — extract `TaskJournalServer` into a `tj-mcp` library target. Unblocks the full E2E rmcp roundtrip test we deferred from D4. Tracked as a side-quest for a future epic. +- Telemetry endpoint (still requires hosted backend). +- `task-journal compact` (lifecycle archival of closed tasks) — wants a design pass, deferred. + +🤖 Generated with [Claude Code](https://claude.com/claude-code)