From 82997be5aa5711eac23e9ba5a006e4c7f32cbeb3 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sat, 13 Jun 2026 15:57:38 +0400 Subject: [PATCH 1/6] =?UTF-8?q?feat(capture):=20adaptive=20self-tagging=20?= =?UTF-8?q?nudge=20(caveman=20pattern)=20=E2=80=94=20piece=201/5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reliable, free, non-blocking capture (epic claude-memory-isg, build order: self-tagging first). The UserPromptSubmit nudge is now adaptive: it always emits the base "record as you go" reminder, and — when the session has done substantial work (transcript ≥60KB) but logged few journal entries (<2 stamped with this session_id, counted from the JSONL tail) — it escalates with a ⚠ "log the key decisions NOW" line. Same proven mechanism as the active caveman mode: constant lightweight context injection every turn, never a blocking gate. - run_nudge / nudge_escalation_text (pure, unit-tested thresholds) / count_session_events_tail. is_terminal guard so manual runs don't block on stdin. No model, no spawn, no cost. Next pieces (same branch, one release at end): unified pluggable LLM backend (claude -p default / API / Agent SDK / Codex / Ollama-free), heuristic dropped, dream/consolidate routed through it, conventions → CLAUDE.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/tj-cli/src/main.rs | 114 ++++++++++++++++++++++++++++++++++---- 1 file changed, 104 insertions(+), 10 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index ddbfb72..041a4d9 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -3176,16 +3176,7 @@ fn main() -> Result<()> { print!("{}", run_statusline().unwrap_or_default()); } Commands::Nudge => { - // No model, no spawn, never touches the classifier — just prints a - // UserPromptSubmit additionalContext reminder so the agent keeps - // recording via the MCP tools deep into a session. - let env = serde_json::json!({ - "hookSpecificOutput": { - "hookEventName": "UserPromptSubmit", - "additionalContext": "📓 task-journal — record as you go: the moment you commit to a decision, rule an approach out, or verify a fact, call event_add (open or resume a task first). Don't batch it to the end. This memory only works if you log it now." - } - }); - print!("{env}"); + run_nudge()?; } Commands::RecallHook => { run_recall_hook()?; @@ -3809,6 +3800,94 @@ fn sync_global_memory(project_conn: &rusqlite::Connection, project_hash: &str) { } } +const NUDGE_BASE: &str = "📓 task-journal — record as you go: the moment you commit to a decision, rule an approach out, or verify a fact, call event_add (open or resume a task first). Don't batch it to the end. This memory only works if you log it now."; +/// A session whose transcript is at least this big counts as "substantial work". +const NUDGE_WORK_THRESHOLD: u64 = 60_000; +/// Below this many journal entries for a substantial session, escalate. +const NUDGE_MIN_EVENTS: usize = 2; + +/// The escalation line, or `None` when no escalation is warranted. Pure so the +/// thresholds are unit-testable without touching the filesystem. +fn nudge_escalation_text(work_bytes: u64, recorded: usize) -> Option { + if work_bytes < NUDGE_WORK_THRESHOLD || recorded >= NUDGE_MIN_EVENTS { + return None; + } + Some(format!( + "⚠ task-journal: this session has done substantial work but recorded only \ +{recorded} journal entr{} — log the key decisions, rejections, and findings NOW via \ +event_add before this reasoning is lost.", + if recorded == 1 { "y" } else { "ies" } + )) +} + +/// Count events in the tail of `path` stamped with `sid`. A tail scan keeps this +/// cheap even for a large journal; a session's events are always at the end. +fn count_session_events_tail(path: &std::path::Path, sid: &str, tail_lines: usize) -> usize { + let body = match std::fs::read_to_string(path) { + Ok(b) => b, + Err(_) => return 0, + }; + let lines: Vec<&str> = body.lines().collect(); + let start = lines.len().saturating_sub(tail_lines); + lines[start..] + .iter() + .filter(|l| { + serde_json::from_str::(l) + .ok() + .and_then(|e| { + e.get("meta") + .and_then(|m| m.get("session_id")) + .and_then(|s| s.as_str()) + .map(|s| s == sid) + }) + .unwrap_or(false) + }) + .count() +} + +/// Adaptive UserPromptSubmit nudge (caveman pattern, non-blocking, free): always +/// emit the base "record as you go" reminder, and — when the session has done +/// substantial work but logged little — escalate. All signals are cheap (a file +/// size + a tail scan); no model, never blocks the prompt. +fn run_nudge() -> anyhow::Result<()> { + let mut ctx = NUDGE_BASE.to_string(); + let escalation = (|| -> Option { + use std::io::{IsTerminal, Read}; + // Manual `task-journal nudge` in a terminal has no hook payload — don't + // block waiting on stdin. + if std::io::stdin().is_terminal() { + return None; + } + let mut buf = String::new(); + if std::io::stdin().read_to_string(&mut buf).is_err() || buf.trim().is_empty() { + return None; + } + let payload: serde_json::Value = serde_json::from_str(&buf).ok()?; + let sid = tj_core::session_id::live_session_id(Some(&payload))?; + let transcript = payload.get("transcript_path").and_then(|v| v.as_str())?; + let work_bytes = std::fs::metadata(transcript).map(|m| m.len()).unwrap_or(0); + let cwd = std::env::current_dir().ok()?; + let project_hash = tj_core::project_hash::from_path(&cwd).ok()?; + let events_path = tj_core::paths::events_dir() + .ok()? + .join(format!("{project_hash}.jsonl")); + let recorded = count_session_events_tail(&events_path, &sid, 400); + nudge_escalation_text(work_bytes, recorded) + })(); + if let Some(extra) = escalation { + ctx.push('\n'); + ctx.push_str(&extra); + } + let env = serde_json::json!({ + "hookSpecificOutput": { + "hookEventName": "UserPromptSubmit", + "additionalContext": ctx, + } + }); + print!("{env}"); + Ok(()) +} + /// Proactive recall injector (opt-in hook). Reads the UserPromptSubmit payload /// from stdin, keyword-searches the global index for relevant prior /// decisions/rejections/constraints across all projects, and emits a budgeted @@ -4943,6 +5022,21 @@ mod inline_tests { // declared before this module begins. use super::*; + #[test] + fn nudge_escalates_only_for_substantial_thin_sessions() { + // Small session → never escalate, regardless of capture. + assert!(nudge_escalation_text(1_000, 0).is_none()); + // Substantial session with enough capture → no escalation. + assert!(nudge_escalation_text(200_000, NUDGE_MIN_EVENTS).is_none()); + // Substantial session, thin capture → escalate. + let e = nudge_escalation_text(200_000, 0).expect("should escalate"); + assert!(e.contains("substantial work") && e.contains("event_add")); + // Singular grammar at exactly 1 entry. + assert!(nudge_escalation_text(200_000, 1) + .unwrap() + .contains("1 journal entry")); + } + #[test] fn flatten_transcript_tags_roles_in_order() { use tj_core::session::parser::parse_session; From 8fafef0a97a12d699915d9cfcb4289001574c491 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sat, 13 Jun 2026 16:03:44 +0400 Subject: [PATCH 2/6] =?UTF-8?q?feat(llm):=20pluggable=20backend=20(claude-?= =?UTF-8?q?p/anthropic/openai/ollama)=20+=20route=20consolidate=20?= =?UTF-8?q?=E2=80=94=20piece=202/5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A small LlmBackend trait + adapters so this public package grows new providers without touching callers (epic claude-memory-isg). Default claude-p (subscription, no key); pick others via TJ_BACKEND / --backend. - tj-core::llm: LlmBackend { complete, name }, backend_from_env(explicit) → Some(usable) / None(unavailable→skip) / Err(unknown name). Adapters: ClaudeCliBackend (default, run_claude_json), AnthropicBackend (API), and one OpenAiBackend covering OpenAI, Codex, and **Ollama** (free local — Ollama's OpenAI-compatible /v1) by pointing base_url + optional key. - consolidate refactored: heuristic-free already; summarize() now routes through llm::backend_from_env, drops its bespoke Anthropic client + claude-p path. `consolidate --backend `; default claude-p. - Tests: factory resolution (unknown errs, no-key None, ollama always, etc.), mockito for OpenAI + Anthropic adapters (6 llm); consolidate parse/build/skip (4); CLI consolidate forces TJ_BACKEND=anthropic against the mock. Green on default and --no-default-features. Next: route dream through llm + `complete` command; conventions → CLAUDE.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/tj-cli/src/main.rs | 33 +-- crates/tj-cli/tests/cli.rs | 1 + crates/tj-core/src/consolidate.rs | 207 +++-------------- crates/tj-core/src/lib.rs | 1 + crates/tj-core/src/llm.rs | 371 ++++++++++++++++++++++++++++++ 5 files changed, 418 insertions(+), 195 deletions(-) create mode 100644 crates/tj-core/src/llm.rs diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 041a4d9..0691bd4 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -654,13 +654,17 @@ enum Commands { Preferences, /// Distil this project's recurring decisions and constraints into durable /// semantic/procedural facts (Pillar C). MANUAL and opt-in — it makes ONE - /// direct Haiku API call per run (needs ANTHROPIC_API_KEY; ~1c/run) and is - /// never wired to a hook, so it can't spend automatically. Facts are stored - /// as events in a per-project "conventions" task and surface in ask/recall. + /// LLM call per run and is never wired to a hook, so it can't spend + /// automatically. Facts are stored as events in a per-project "conventions" + /// task and surface in ask/recall. Consolidate { /// Maximum number of facts to produce. #[arg(long, default_value_t = 8)] max_facts: usize, + /// LLM backend override: claude-p (default) | anthropic | openai | ollama. + /// Defaults to TJ_BACKEND, then claude-p (subscription, no API key). + #[arg(long)] + backend: Option, }, /// Render and print the resume pack for a task. Pack { @@ -1294,8 +1298,8 @@ fn main() -> Result<()> { } } } - Commands::Consolidate { max_facts } => { - run_consolidate(max_facts)?; + Commands::Consolidate { max_facts, backend } => { + run_consolidate(max_facts, backend.as_deref())?; } Commands::Event { task_id, @@ -3999,10 +4003,10 @@ fn emit_session_context(ctx: &str) { const CONSOLIDATE_TASK_TITLE: &str = "Project conventions (consolidated)"; /// Manual consolidation: read this project's recurring decisions/constraints, -/// distil them into durable facts via one direct Haiku API call, and store the -/// facts as events in a per-project conventions task. Skips cleanly (no spend) -/// when ANTHROPIC_API_KEY is absent. -fn run_consolidate(max_facts: usize) -> anyhow::Result<()> { +/// distil them into durable facts via one LLM call through the chosen backend, +/// and store the facts as events in a per-project conventions task. Skips +/// cleanly (no spend) when no backend is available. +fn run_consolidate(max_facts: usize, backend: Option<&str>) -> anyhow::Result<()> { let cwd = std::env::current_dir()?; let project_hash = tj_core::project_hash::from_path(&cwd)?; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); @@ -4021,19 +4025,20 @@ fn run_consolidate(max_facts: usize) -> anyhow::Result<()> { let texts: Vec = sources.iter().map(|(_, t)| t.clone()).collect(); let source_ids: Vec = sources.iter().map(|(id, _)| id.clone()).collect(); - let (backend, facts) = match tj_core::consolidate::summarize(&texts, max_facts)? { + let (backend_used, facts) = match tj_core::consolidate::summarize(&texts, max_facts, backend)? { Some(x) => x, None => { println!( - "skipped: no consolidation backend. Either set ANTHROPIC_API_KEY \ -(direct Haiku API, ~1c/run) or install Claude Code so `claude` is on PATH \ -(uses your subscription login, no API key needed)." + "skipped: no usable LLM backend. Default is `claude-p` (install Claude \ +Code so `claude` is on PATH — uses your subscription, no API key). Or pick \ +another via --backend / TJ_BACKEND: anthropic (ANTHROPIC_API_KEY), openai \ +(OPENAI_API_KEY), ollama (free, local)." ); return Ok(()); } }; eprintln!( - "consolidating {} high-signal event(s) via {backend} …", + "consolidating {} high-signal event(s) via {backend_used} …", texts.len() ); if facts.is_empty() { diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 829319c..c30251c 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -5270,6 +5270,7 @@ fn consolidate_writes_facts_to_conventions_task_and_dedups() { .unwrap() .current_dir(proj.path()) .env("XDG_DATA_HOME", xdg.path()) + .env("TJ_BACKEND", "anthropic") .env("ANTHROPIC_API_KEY", "test-key") .env("TJ_CONSOLIDATE_BASE_URL", server.url()) .env("TJ_EMBED", "hash") diff --git a/crates/tj-core/src/consolidate.rs b/crates/tj-core/src/consolidate.rs index c4b2d36..49d1375 100644 --- a/crates/tj-core/src/consolidate.rs +++ b/crates/tj-core/src/consolidate.rs @@ -1,24 +1,15 @@ //! Memory consolidation (Pillar C): distil a project's recurring decisions and -//! constraints into a handful of durable semantic/procedural facts with a -//! single LLM call. +//! constraints into a handful of durable semantic/procedural facts with a single +//! LLM call. //! -//! Two backends, picked by [`summarize`]: the **direct Anthropic Haiku API** -//! when `ANTHROPIC_API_KEY` is set (cheapest — only our ~7k-token prompt, -//! ~1c/run), otherwise the local **`claude -p`** binary (subscription auth, no -//! API key needed, but it boots the whole environment per call so it's -//! pricier). With neither, the caller skips cleanly — we never fall back to a -//! heuristic, which would manufacture low-trust "facts". +//! The call goes through the pluggable [`crate::llm`] backend — default +//! `claude-p` on your subscription (no API key), configurable to the Anthropic +//! API, any OpenAI-compatible provider (OpenAI / Codex), or a **free** local +//! Ollama. When no backend is available the caller skips cleanly; we never fall +//! back to a heuristic, which would manufacture low-trust "facts". //! -//! Either way this is a MANUAL command: one call per run, only when the user -//! asks, never wired to a hook — so it never resembles the per-prompt -//! classifier burn. - -use anyhow::{anyhow, Context}; -use serde::{Deserialize, Serialize}; -use std::time::Duration; - -/// Cheapest capable model for the summarisation step. -pub const DEFAULT_MODEL: &str = "claude-haiku-4-5-20251001"; +//! This is a MANUAL command: one call per run, only when the user asks, never on +//! a hook — so it never resembles the per-prompt classifier burn. /// A distilled fact and which tier it belongs to. #[derive(Debug, Clone, PartialEq, Eq)] @@ -29,113 +20,28 @@ pub struct ConsolidatedFact { pub text: String, } -/// Direct-API consolidator. -pub struct Consolidator { - pub api_key: String, - pub model: String, - pub base_url: String, - pub timeout: Duration, - pub max_facts: usize, -} - -impl Consolidator { - /// Build from the environment. Errors (so the caller can skip cleanly) when - /// `ANTHROPIC_API_KEY` is absent. Model overridable via `TJ_CONSOLIDATE_MODEL`. - pub fn from_env(max_facts: usize) -> anyhow::Result { - let api_key = std::env::var("ANTHROPIC_API_KEY").map_err(|_| { - anyhow!("consolidation needs ANTHROPIC_API_KEY for the direct Haiku API") - })?; - let model = std::env::var("TJ_CONSOLIDATE_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()); - // TJ_CONSOLIDATE_BASE_URL overrides the endpoint (used by tests to point - // at a local mock); production always hits the real Anthropic API. - let base_url = std::env::var("TJ_CONSOLIDATE_BASE_URL") - .unwrap_or_else(|_| "https://api.anthropic.com".into()); - Ok(Self { - api_key, - model, - base_url, - timeout: Duration::from_secs(60), - max_facts: max_facts.max(1), - }) - } - - /// Summarise the given event texts into durable facts. Empty input → no - /// call. Returns whatever facts the model produced (possibly none). - pub fn consolidate(&self, events: &[String]) -> anyhow::Result> { - if events.is_empty() { - return Ok(Vec::new()); - } - let prompt = build_prompt(events, self.max_facts); - let body = MessagesRequest { - model: &self.model, - max_tokens: 512, - messages: vec![MessageIn { - role: "user", - content: &prompt, - }], - }; - let url = format!("{}/v1/messages", self.base_url); - let resp: MessagesResponse = ureq::post(&url) - .timeout(self.timeout) - .set("x-api-key", &self.api_key) - .set("anthropic-version", "2023-06-01") - .set("content-type", "application/json") - .send_json(serde_json::to_value(&body)?) - .context("Anthropic API request failed")? - .into_json() - .context("decode Anthropic response")?; - let text = resp - .content - .iter() - .find(|b| b.kind == "text") - .map(|b| b.text.clone()) - .ok_or_else(|| anyhow!("no text content in response"))?; - Ok(parse_facts(&text)) - } -} - -/// Run whichever summarisation backend is available and return its label plus -/// the facts it produced. Order: (1) `ANTHROPIC_API_KEY` set → direct Haiku API -/// (cheapest, ~1c/run); (2) else `claude` on PATH → local `claude -p` -/// (subscription auth, no API key, heavier per-call boot); (3) else `Ok(None)`, -/// so the caller skips with a message — never a heuristic. -/// `TJ_CONSOLIDATE_BACKEND=none` forces the no-backend path (disable / tests). +/// Distil `events` into at most `max_facts` durable facts via the chosen +/// backend (`backend` overrides `TJ_BACKEND`; `None` uses the default chain). +/// Returns `(backend label, facts)`, or `None` when no backend is usable or +/// `TJ_CONSOLIDATE_BACKEND=none` forces a skip. pub fn summarize( events: &[String], max_facts: usize, + backend: Option<&str>, ) -> anyhow::Result)>> { if std::env::var("TJ_CONSOLIDATE_BACKEND").as_deref() == Ok("none") { return Ok(None); } - if std::env::var("ANTHROPIC_API_KEY").is_ok() { - let c = Consolidator::from_env(max_facts)?; - return Ok(Some(("haiku-api", c.consolidate(events)?))); - } - if crate::classifier::agent_sdk::claude_on_path() { - return Ok(Some(("claude -p", consolidate_via_cli(events, max_facts)?))); - } - Ok(None) -} - -/// Summarise via the local `claude -p` binary (subscription auth). Reuses the -/// classifier's command plumbing — including the recursion guard set by -/// `base_claude_command` — and unwraps the `--output-format json` envelope. -fn consolidate_via_cli( - events: &[String], - max_facts: usize, -) -> anyhow::Result> { + let llm = match crate::llm::backend_from_env(backend)? { + Some(b) => b, + None => return Ok(None), + }; if events.is_empty() { - return Ok(Vec::new()); + return Ok(Some((llm.name(), Vec::new()))); } let prompt = build_prompt(events, max_facts); - let model = std::env::var("TJ_CONSOLIDATE_MODEL") - .unwrap_or_else(|_| crate::classifier::agent_sdk::DEFAULT_MODEL.to_string()); - let text = crate::classifier::agent_sdk::run_claude_json( - &crate::classifier::agent_sdk::ClaudeBinaryStdinRunner, - &model, - &prompt, - )?; - Ok(parse_facts(&text)) + let text = llm.complete(&prompt, 512)?; + Ok(Some((llm.name(), parse_facts(&text)))) } /// The summarisation prompt. Deliberately strict: durable-only, fixed line @@ -184,29 +90,6 @@ pub fn parse_facts(text: &str) -> Vec { out } -#[derive(Serialize)] -struct MessagesRequest<'a> { - model: &'a str, - max_tokens: u32, - messages: Vec>, -} -#[derive(Serialize)] -struct MessageIn<'a> { - role: &'a str, - content: &'a str, -} -#[derive(Deserialize)] -struct MessagesResponse { - content: Vec, -} -#[derive(Deserialize)] -struct ContentBlock { - #[serde(rename = "type")] - kind: String, - #[serde(default)] - text: String, -} - #[cfg(test)] mod tests { use super::*; @@ -241,48 +124,10 @@ mod tests { } #[test] - fn consolidate_empty_input_makes_no_call() { - // base_url is unreachable; empty input must short-circuit before any - // request, so this must not error. - let c = Consolidator { - api_key: "x".into(), - model: "m".into(), - base_url: "http://127.0.0.1:1".into(), - timeout: Duration::from_millis(50), - max_facts: 5, - }; - assert!(c.consolidate(&[]).unwrap().is_empty()); - } - - #[test] - fn consolidate_calls_api_and_parses() { - let mut server = mockito::Server::new(); - let m = server - .mock("POST", "/v1/messages") - .with_status(200) - .with_header("content-type", "application/json") - .with_body( - serde_json::json!({ - "id": "msg", - "type": "message", - "role": "assistant", - "content": [{"type": "text", "text": "[semantic] Always use the ledger\n[procedural] TDD here"}] - }) - .to_string(), - ) - .create(); - - let c = Consolidator { - api_key: "test".into(), - model: "claude-haiku-4-5-20251001".into(), - base_url: server.url(), - timeout: Duration::from_secs(5), - max_facts: 5, - }; - let facts = c.consolidate(&["chose ledger".into()]).unwrap(); - m.assert(); - assert_eq!(facts.len(), 2); - assert_eq!(facts[0].tier, "semantic"); - assert_eq!(facts[1].tier, "procedural"); + fn summarize_skips_when_backend_forced_none() { + std::env::set_var("TJ_CONSOLIDATE_BACKEND", "none"); + let r = summarize(&["chose ledger".into()], 5, None).unwrap(); + std::env::remove_var("TJ_CONSOLIDATE_BACKEND"); + assert!(r.is_none()); } } diff --git a/crates/tj-core/src/lib.rs b/crates/tj-core/src/lib.rs index e13a3be..b5e46fc 100644 --- a/crates/tj-core/src/lib.rs +++ b/crates/tj-core/src/lib.rs @@ -57,6 +57,7 @@ pub mod embed; pub mod event; pub mod frontmatter; pub mod fts; +pub mod llm; pub mod memory; pub mod pack; pub mod paths; diff --git a/crates/tj-core/src/llm.rs b/crates/tj-core/src/llm.rs new file mode 100644 index 0000000..baacd0e --- /dev/null +++ b/crates/tj-core/src/llm.rs @@ -0,0 +1,371 @@ +//! Pluggable LLM backend for the journal's optional AI operations +//! (consolidation, dream backfill). One small trait, several adapters, picked by +//! name so this public package can grow new providers without touching callers. +//! +//! Default is **`claude-p`** — the local Claude CLI on your subscription, so the +//! out-of-the-box experience needs no API key. Override with `TJ_BACKEND` (env, +//! global) or a per-command `--backend`: +//! +//! - `claude-p` (default) — local `claude -p`, Haiku, subscription auth. +//! - `anthropic` — direct Anthropic API (`ANTHROPIC_API_KEY`). +//! - `openai` — any OpenAI-compatible chat API (`OPENAI_API_KEY`, +//! `TJ_OPENAI_BASE_URL`, `TJ_OPENAI_MODEL`). Covers OpenAI, Codex, and other +//! compatible providers by pointing the base URL. +//! - `ollama` — a local Ollama model (its OpenAI-compatible endpoint), **free**: +//! no key, no network beyond localhost. `TJ_OLLAMA_URL`, `TJ_OLLAMA_MODEL`. +//! +//! A backend that isn't usable (no key, no `claude` on PATH) yields `Ok(None)` +//! from [`backend_from_env`] so the caller skips cleanly — we never fabricate +//! output without a model. + +use anyhow::{anyhow, Context}; +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +/// One AI call: a prompt in, the model's text reply out. +pub trait LlmBackend: Send + Sync { + fn complete(&self, prompt: &str, max_tokens: u32) -> anyhow::Result; + /// Stable label for logs / provenance. + fn name(&self) -> &'static str; +} + +/// Resolve the backend from an explicit name (e.g. a `--backend` flag) or +/// `TJ_BACKEND`, defaulting to `claude-p`. Returns: +/// - `Ok(Some(_))` — a usable backend, +/// - `Ok(None)` — the chosen backend is unavailable (no key / no `claude`); the +/// caller should skip, +/// - `Err(_)` — an unknown backend name (a typo worth surfacing). +pub fn backend_from_env(explicit: Option<&str>) -> anyhow::Result>> { + let name = explicit + .map(str::to_string) + .or_else(|| std::env::var("TJ_BACKEND").ok()) + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "claude-p".to_string()); + + match name.trim() { + "claude-p" | "claude" | "agent-sdk" => { + if crate::classifier::agent_sdk::claude_on_path() { + Ok(Some(Box::new(ClaudeCliBackend::from_env()))) + } else { + Ok(None) + } + } + "anthropic" | "api" => match std::env::var("ANTHROPIC_API_KEY") { + Ok(key) if !key.is_empty() => Ok(Some(Box::new(AnthropicBackend::new(key)))), + _ => Ok(None), + }, + "openai" | "codex" => match std::env::var("OPENAI_API_KEY") { + Ok(key) if !key.is_empty() => Ok(Some(Box::new(OpenAiBackend::openai(key)))), + _ => Ok(None), + }, + "ollama" => Ok(Some(Box::new(OpenAiBackend::ollama()))), + other => Err(anyhow!( + "unknown backend '{other}' (expected: claude-p, anthropic, openai, ollama)" + )), + } +} + +// --------------------------------------------------------------------------- +// claude -p (default) — local CLI, subscription auth, no API key. +// --------------------------------------------------------------------------- + +pub struct ClaudeCliBackend { + model: String, +} + +impl ClaudeCliBackend { + pub fn from_env() -> Self { + let model = std::env::var("TJ_CONSOLIDATE_MODEL") + .unwrap_or_else(|_| crate::classifier::agent_sdk::DEFAULT_MODEL.to_string()); + Self { model } + } +} + +impl LlmBackend for ClaudeCliBackend { + fn complete(&self, prompt: &str, _max_tokens: u32) -> anyhow::Result { + crate::classifier::agent_sdk::run_claude_json( + &crate::classifier::agent_sdk::ClaudeBinaryStdinRunner, + &self.model, + prompt, + ) + } + fn name(&self) -> &'static str { + "claude-p" + } +} + +// --------------------------------------------------------------------------- +// Anthropic direct API. +// --------------------------------------------------------------------------- + +pub struct AnthropicBackend { + api_key: String, + model: String, + base_url: String, + timeout: Duration, +} + +impl AnthropicBackend { + pub fn new(api_key: String) -> Self { + let model = std::env::var("TJ_CONSOLIDATE_MODEL") + .unwrap_or_else(|_| "claude-haiku-4-5-20251001".to_string()); + let base_url = std::env::var("TJ_CONSOLIDATE_BASE_URL") + .unwrap_or_else(|_| "https://api.anthropic.com".to_string()); + Self { + api_key, + model, + base_url, + timeout: Duration::from_secs(60), + } + } +} + +#[derive(Serialize)] +struct AnthropicReq<'a> { + model: &'a str, + max_tokens: u32, + messages: Vec>, +} +#[derive(Serialize)] +struct AnthropicMsg<'a> { + role: &'a str, + content: &'a str, +} +#[derive(Deserialize)] +struct AnthropicResp { + content: Vec, +} +#[derive(Deserialize)] +struct AnthropicBlock { + #[serde(rename = "type")] + kind: String, + #[serde(default)] + text: String, +} + +impl LlmBackend for AnthropicBackend { + fn complete(&self, prompt: &str, max_tokens: u32) -> anyhow::Result { + let body = AnthropicReq { + model: &self.model, + max_tokens, + messages: vec![AnthropicMsg { + role: "user", + content: prompt, + }], + }; + let resp: AnthropicResp = ureq::post(&format!("{}/v1/messages", self.base_url)) + .timeout(self.timeout) + .set("x-api-key", &self.api_key) + .set("anthropic-version", "2023-06-01") + .set("content-type", "application/json") + .send_json(serde_json::to_value(&body)?) + .context("Anthropic API request failed")? + .into_json() + .context("decode Anthropic response")?; + resp.content + .iter() + .find(|b| b.kind == "text") + .map(|b| b.text.clone()) + .ok_or_else(|| anyhow!("no text content in Anthropic response")) + } + fn name(&self) -> &'static str { + "anthropic" + } +} + +// --------------------------------------------------------------------------- +// OpenAI-compatible — covers OpenAI, Codex, Ollama, and any compatible server. +// --------------------------------------------------------------------------- + +pub struct OpenAiBackend { + api_key: Option, + model: String, + base_url: String, + label: &'static str, + timeout: Duration, +} + +impl OpenAiBackend { + pub fn openai(api_key: String) -> Self { + Self { + api_key: Some(api_key), + model: std::env::var("TJ_OPENAI_MODEL").unwrap_or_else(|_| "gpt-4o-mini".to_string()), + base_url: std::env::var("TJ_OPENAI_BASE_URL") + .unwrap_or_else(|_| "https://api.openai.com".to_string()), + label: "openai", + timeout: Duration::from_secs(60), + } + } + + pub fn ollama() -> Self { + Self { + api_key: None, // local; no auth + model: std::env::var("TJ_OLLAMA_MODEL").unwrap_or_else(|_| "llama3.1".to_string()), + base_url: std::env::var("TJ_OLLAMA_URL") + .unwrap_or_else(|_| "http://localhost:11434".to_string()), + label: "ollama", + timeout: Duration::from_secs(120), + } + } +} + +#[derive(Serialize)] +struct OpenAiReq<'a> { + model: &'a str, + max_tokens: u32, + messages: Vec>, +} +#[derive(Deserialize)] +struct OpenAiResp { + choices: Vec, +} +#[derive(Deserialize)] +struct OpenAiChoice { + message: OpenAiMsg, +} +#[derive(Deserialize)] +struct OpenAiMsg { + #[serde(default)] + content: String, +} + +impl LlmBackend for OpenAiBackend { + fn complete(&self, prompt: &str, max_tokens: u32) -> anyhow::Result { + let body = OpenAiReq { + model: &self.model, + max_tokens, + messages: vec![AnthropicMsg { + role: "user", + content: prompt, + }], + }; + let mut req = ureq::post(&format!("{}/v1/chat/completions", self.base_url)) + .timeout(self.timeout) + .set("content-type", "application/json"); + if let Some(key) = &self.api_key { + req = req.set("authorization", &format!("Bearer {key}")); + } + let resp: OpenAiResp = req + .send_json(serde_json::to_value(&body)?) + .with_context(|| format!("{} request failed", self.label))? + .into_json() + .context("decode OpenAI-compatible response")?; + resp.choices + .into_iter() + .next() + .map(|c| c.message.content) + .ok_or_else(|| anyhow!("no choices in {} response", self.label)) + } + fn name(&self) -> &'static str { + self.label + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct EnvGuard(&'static str, Option); + impl EnvGuard { + fn set(k: &'static str, v: &str) -> Self { + let prev = std::env::var(k).ok(); + std::env::set_var(k, v); + Self(k, prev) + } + fn unset(k: &'static str) -> Self { + let prev = std::env::var(k).ok(); + std::env::remove_var(k); + Self(k, prev) + } + } + impl Drop for EnvGuard { + fn drop(&mut self) { + match &self.1 { + Some(v) => std::env::set_var(self.0, v), + None => std::env::remove_var(self.0), + } + } + } + + // Serialise env-touching tests (process-global env). + static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + #[test] + fn unknown_backend_errors() { + let _l = ENV_LOCK.lock().unwrap(); + assert!(backend_from_env(Some("nonsense")).is_err()); + } + + #[test] + fn anthropic_unavailable_without_key_is_none() { + let _l = ENV_LOCK.lock().unwrap(); + let _g = EnvGuard::unset("ANTHROPIC_API_KEY"); + assert!(backend_from_env(Some("anthropic")).unwrap().is_none()); + } + + #[test] + fn anthropic_with_key_resolves() { + let _l = ENV_LOCK.lock().unwrap(); + let _g = EnvGuard::set("ANTHROPIC_API_KEY", "k"); + let b = backend_from_env(Some("anthropic")).unwrap().unwrap(); + assert_eq!(b.name(), "anthropic"); + } + + #[test] + fn ollama_always_resolves_no_key() { + let _l = ENV_LOCK.lock().unwrap(); + let b = backend_from_env(Some("ollama")).unwrap().unwrap(); + assert_eq!(b.name(), "ollama"); + } + + #[test] + fn openai_calls_chat_completions_and_parses() { + let mut server = mockito::Server::new(); + let m = server + .mock("POST", "/v1/chat/completions") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + serde_json::json!({ + "choices": [{"message": {"role": "assistant", "content": "hello from openai"}}] + }) + .to_string(), + ) + .create(); + let b = OpenAiBackend { + api_key: Some("k".into()), + model: "gpt-4o-mini".into(), + base_url: server.url(), + label: "openai", + timeout: Duration::from_secs(5), + }; + let out = b.complete("hi", 64).unwrap(); + m.assert(); + assert_eq!(out, "hello from openai"); + } + + #[test] + fn anthropic_calls_messages_and_parses() { + let mut server = mockito::Server::new(); + let m = server + .mock("POST", "/v1/messages") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + serde_json::json!({ + "content": [{"type": "text", "text": "hello from anthropic"}] + }) + .to_string(), + ) + .create(); + let b = AnthropicBackend { + api_key: "k".into(), + model: "claude-haiku-4-5-20251001".into(), + base_url: server.url(), + timeout: Duration::from_secs(5), + }; + let out = b.complete("hi", 64).unwrap(); + m.assert(); + assert_eq!(out, "hello from anthropic"); + } +} From 6c8a7e186bc5e42e8c08577d971f3764760022f6 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sat, 13 Jun 2026 16:19:09 +0400 Subject: [PATCH 3/6] =?UTF-8?q?feat(capture):=20capture=20kill-switch=20?= =?UTF-8?q?=E2=80=94=20`capture=20on|off`=20marker=20no-ops=20ingest-hook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `.capture-disabled` marker in the data dir makes ingest-hook skip the realtime capture path (the read-only SessionStart resume still runs). Because the hook re-invokes the on-disk binary on every event, dropping the marker stops a stale auto-capture hook in an ALREADY-RUNNING session without a restart — env vars don't propagate into hook subprocesses, and a restart loses the session. - `task-journal capture off` writes the marker; `capture on` removes it. - ingest-hook checks it right after resolving the hook kind, before any capture, skipping everything except SessionStart resume. Motivated live: this long session still ran the pre-0.14.4 plugin's PostToolUse hook, which was recording Edit/MCP tool calls as junk "rejection" events; `capture off` silenced it immediately. Test: capture off → an auto-opening prompt records no task; capture on clears. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/tj-cli/src/main.rs | 36 ++++++++++++++++++++++++ crates/tj-cli/tests/cli.rs | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 0691bd4..213d1f1 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -652,6 +652,14 @@ enum Commands { }, /// List your stored user preferences. Preferences, + /// Turn realtime hook capture on or off via a `.capture-disabled` marker. + /// `off` no-ops the capture path of `ingest-hook` immediately — even in an + /// already-running session — without touching the read-only SessionStart + /// resume. Use it to silence a stale auto-capture hook. + Capture { + /// "on" (remove the marker) or "off" (write it). + state: String, + }, /// Distil this project's recurring decisions and constraints into durable /// semantic/procedural facts (Pillar C). MANUAL and opt-in — it makes ONE /// LLM call per run and is never wired to a hook, so it can't spend @@ -1298,6 +1306,21 @@ fn main() -> Result<()> { } } } + Commands::Capture { state } => { + let marker = tj_core::paths::data_dir()?.join(".capture-disabled"); + match state.trim().to_lowercase().as_str() { + "off" | "false" | "0" => { + std::fs::create_dir_all(marker.parent().unwrap())?; + std::fs::write(&marker, "")?; + println!("realtime capture OFF — the marker no-ops ingest-hook capture now (resume still works)."); + } + "on" | "true" | "1" => { + let _ = std::fs::remove_file(&marker); + println!("realtime capture ON."); + } + other => anyhow::bail!("expected `on` or `off`, got `{other}`"), + } + } Commands::Consolidate { max_facts, backend } => { run_consolidate(max_facts, backend.as_deref())?; } @@ -1942,6 +1965,19 @@ fn main() -> Result<()> { _ => parse_hook_stdin()?, }; + // Emergency capture kill-switch: a `.capture-disabled` marker in the + // data dir no-ops realtime capture (the read-only SessionStart + // resume still runs). Because the hook re-invokes this binary on + // every event, dropping the marker stops a stale auto-capture hook + // in an already-running session immediately — no restart needed. + if kind != "SessionStart" + && tj_core::paths::data_dir() + .map(|d| d.join(".capture-disabled").exists()) + .unwrap_or(false) + { + return Ok(()); + } + let cwd = std::env::current_dir()?; let project_hash = tj_core::project_hash::from_path(&cwd)?; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index c30251c..cefde28 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -5364,3 +5364,60 @@ fn consolidate_skips_without_api_key_and_spends_nothing() { .success() .stdout(contains("skipped")); } + +#[test] +fn capture_off_marker_no_ops_ingest_hook_capture() { + // `capture off` writes a marker that makes ingest-hook skip the capture + // path — so an auto-opening prompt records nothing. `capture on` clears it. + let dir = assert_fs::TempDir::new().unwrap(); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["capture", "off"]) + .assert() + .success() + .stdout(contains("OFF")); + + let payload = serde_json::json!({ + "hook_event_name": "UserPromptSubmit", + "session_id": "s-cap", + "transcript_path": "/tmp/x", + "cwd": "/tmp", + "prompt": "implement FIN-868 paygate fee dedup" + }) + .to_string(); + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .env("TJ_INGEST_SYNC", "1") + .args(["ingest-hook", "--backend", "hybrid"]) + .write_stdin(payload) + .assert() + .success(); + + let body = String::from_utf8( + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["search", "paygate"]) + .assert() + .success() + .get_output() + .stdout + .clone(), + ) + .unwrap(); + assert!( + !body.lines().any(|l| l.trim().starts_with("tj-")), + "capture off must no-op ingest-hook capture; got: {body:?}" + ); + + Command::cargo_bin("task-journal") + .unwrap() + .env("XDG_DATA_HOME", dir.path()) + .args(["capture", "on"]) + .assert() + .success() + .stdout(contains("ON")); +} From af92441986b2a898421b934ed6d6d50859a19eef Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sat, 13 Jun 2026 16:23:15 +0400 Subject: [PATCH 4/6] =?UTF-8?q?feat(dream):=20route=20through=20pluggable?= =?UTF-8?q?=20llm=20backend=20+=20`complete`=20command=20=E2=80=94=20piece?= =?UTF-8?q?=203/5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dream now gets the same backend choice as everything else, and there's a friendly per-task alias (epic claude-memory-isg). - tj-core::dream::llm_backend::LlmDreamBackend adapts any llm::LlmBackend into a DreamBackend: build the dream prompt → one completion → parse the JSON array of missed events (shared fence-stripping parser). - tj-cli: dream selection refactored into run_dream_op(), routed through llm::backend_from_env (default claude-p; --backend / TJ_BACKEND for anthropic / openai / ollama-free); skips cleanly when no backend is available. - New `task-journal complete [--dry-run] [--backend]` = dream scoped to a task's sessions — "make this task complete from its transcripts" in one obvious command. Free with `--backend ollama`. Tests: parse fence/empty + LlmDreamBackend over a fake llm (3 core); complete exits cleanly with no sessions (1 cli). Clean on default and --no-default-features. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/tj-cli/src/main.rs | 208 ++++++++++++++---------- crates/tj-cli/tests/cli.rs | 16 ++ crates/tj-core/src/dream/llm_backend.rs | 94 +++++++++++ crates/tj-core/src/dream/mod.rs | 1 + 4 files changed, 233 insertions(+), 86 deletions(-) create mode 100644 crates/tj-core/src/dream/llm_backend.rs diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index 213d1f1..e0dcadb 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -863,6 +863,23 @@ enum Commands { /// Cap sessions processed this run. #[arg(long)] limit: Option, + /// LLM backend override: claude-p (default) | anthropic | openai | ollama. + #[arg(long)] + backend: Option, + }, + /// Complete a task from its transcripts: re-read the sessions tied to a task + /// and append the decisions/findings the live capture missed. A friendly + /// alias for `dream --task `. MANUAL, one LLM call per session via the + /// chosen backend (free with `--backend ollama`). + Complete { + /// The task id to complete. + task: String, + /// Show scope without calling the model or writing anything. + #[arg(long)] + dry_run: bool, + /// LLM backend override: claude-p (default) | anthropic | openai | ollama. + #[arg(long)] + backend: Option, }, /// Export tasks as Markdown or JSON to stdout. Export { @@ -2744,93 +2761,17 @@ fn main() -> Result<()> { task, dry_run, limit, + backend, } => { - let cwd = std::env::current_dir()?; - let project_hash = tj_core::project_hash::from_path(&cwd)?; - let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); - let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); - let conn = tj_core::db::open(&state_path)?; - - // 1. Resolve session files in scope. - let project_dir = tj_core::session::discovery::find_project_dir(&cwd)?; - let Some(project_dir) = project_dir else { - println!("dream: no Claude Code sessions found for this project"); - return Ok(()); - }; - let session_paths = tj_core::session::discovery::list_sessions(&project_dir)?; - - let since_time = if let Some(days) = since { - Some( - std::time::SystemTime::now() - - std::time::Duration::from_secs((days.max(0) as u64) * 86_400), - ) - } else { - // Watermark → SystemTime. Absent watermark = all sessions. - match tj_core::dream::state::last_dream_at(&conn, &project_hash)? { - Some(ts) => chrono::DateTime::parse_from_rfc3339(&ts) - .ok() - .map(std::time::SystemTime::from), - None => None, - } - }; - - let scoped: Vec = session_paths - .into_iter() - .filter_map(|p| { - let mtime = std::fs::metadata(&p).ok()?.modified().ok()?; - Some(tj_core::dream::scope::SessionFile { path: p, mtime }) - }) - .collect(); - let in_scope = tj_core::dream::scope::in_scope(scoped, since_time, limit); - - // 2. Assemble (session_id, BackfillInput) per session. - let run_id = ulid::Ulid::new().to_string(); - let sessions = build_dream_inputs(&events_path, &in_scope, task.as_deref())?; - - // 3. Run. - let opts = tj_core::dream::DreamOptions { - project_hash: project_hash.clone(), - dry_run, - }; - if dry_run { - println!("dream (dry-run): {} session(s) in scope", sessions.len()); - return Ok(()); - } - // Prefer the subscription-native agent-sdk backend (local `claude` - // CLI, pinned to Haiku — cheap, no ANTHROPIC_API_KEY). Fall back to - // the Anthropic API backend only when no `claude` is on PATH. - let backend: Box = - match tj_core::dream::agent_sdk::ClaudeCliDreamBackend::from_env() { - Some(b) => { - eprintln!("dream: backend=agent-sdk (claude CLI, Haiku)"); - Box::new(b) - } - None => { - eprintln!( - "dream: no `claude` on PATH — backend=api (needs ANTHROPIC_API_KEY)" - ); - Box::new(tj_core::dream::http::AnthropicDreamBackend::from_env()?) - } - }; - let report = tj_core::dream::run_dream( - &conn, - &events_path, - &opts, - backend.as_ref(), - sessions, - &run_id, - )?; - - // 4. Advance watermark to now (only reached on success). - tj_core::dream::state::set_last_dream_at( - &conn, - &project_hash, - &chrono::Utc::now().to_rfc3339(), - )?; - println!( - "dream: {} session(s) processed, {} event(s) backfilled", - report.sessions_processed, report.events_backfilled - ); + run_dream_op(since, task, dry_run, limit, backend.as_deref())?; + } + Commands::Complete { + task, + dry_run, + backend, + } => { + // "complete this task" = dream scoped to that task's sessions. + run_dream_op(None, Some(task), dry_run, None, backend.as_deref())?; } Commands::Export { format, @@ -4038,6 +3979,101 @@ fn emit_session_context(ctx: &str) { const CONSOLIDATE_TASK_TITLE: &str = "Project conventions (consolidated)"; +/// Dream backfill, shared by `dream` and `complete`: re-read the in-scope +/// session transcripts and append the events the live capture missed, via the +/// chosen pluggable LLM backend. Skips cleanly when no backend is available. +fn run_dream_op( + since: Option, + task: Option, + dry_run: bool, + limit: Option, + backend: Option<&str>, +) -> anyhow::Result<()> { + let cwd = std::env::current_dir()?; + let project_hash = tj_core::project_hash::from_path(&cwd)?; + let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); + let state_path = tj_core::paths::state_dir()?.join(format!("{project_hash}.sqlite")); + let conn = tj_core::db::open(&state_path)?; + + // 1. Resolve session files in scope. + let project_dir = tj_core::session::discovery::find_project_dir(&cwd)?; + let Some(project_dir) = project_dir else { + println!("dream: no Claude Code sessions found for this project"); + return Ok(()); + }; + let session_paths = tj_core::session::discovery::list_sessions(&project_dir)?; + + let since_time = if let Some(days) = since { + Some( + std::time::SystemTime::now() + - std::time::Duration::from_secs((days.max(0) as u64) * 86_400), + ) + } else { + match tj_core::dream::state::last_dream_at(&conn, &project_hash)? { + Some(ts) => chrono::DateTime::parse_from_rfc3339(&ts) + .ok() + .map(std::time::SystemTime::from), + None => None, + } + }; + + let scoped: Vec = session_paths + .into_iter() + .filter_map(|p| { + let mtime = std::fs::metadata(&p).ok()?.modified().ok()?; + Some(tj_core::dream::scope::SessionFile { path: p, mtime }) + }) + .collect(); + let in_scope = tj_core::dream::scope::in_scope(scoped, since_time, limit); + + // 2. Assemble (session_id, BackfillInput) per session. + let run_id = ulid::Ulid::new().to_string(); + let sessions = build_dream_inputs(&events_path, &in_scope, task.as_deref())?; + + let opts = tj_core::dream::DreamOptions { + project_hash: project_hash.clone(), + dry_run, + }; + if dry_run { + println!("dream (dry-run): {} session(s) in scope", sessions.len()); + return Ok(()); + } + + // 3. Backend via the unified pluggable selector (default claude-p). + let llm = match tj_core::llm::backend_from_env(backend)? { + Some(l) => l, + None => { + println!( + "dream: no usable LLM backend. Default `claude-p` needs Claude Code on \ +PATH; or pick one via --backend / TJ_BACKEND: anthropic, openai, ollama (free, local)." + ); + return Ok(()); + } + }; + let dream_backend = tj_core::dream::llm_backend::LlmDreamBackend::new(llm); + eprintln!("dream: backend={}", dream_backend.backend_name()); + let report = tj_core::dream::run_dream( + &conn, + &events_path, + &opts, + &dream_backend, + sessions, + &run_id, + )?; + + // 4. Advance watermark to now (only reached on success). + tj_core::dream::state::set_last_dream_at( + &conn, + &project_hash, + &chrono::Utc::now().to_rfc3339(), + )?; + println!( + "dream: {} session(s) processed, {} event(s) backfilled", + report.sessions_processed, report.events_backfilled + ); + Ok(()) +} + /// Manual consolidation: read this project's recurring decisions/constraints, /// distil them into durable facts via one LLM call through the chosen backend, /// and store the facts as events in a per-project conventions task. Skips diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index cefde28..9e1024c 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -5421,3 +5421,19 @@ fn capture_off_marker_no_ops_ingest_hook_capture() { .success() .stdout(contains("ON")); } + +#[test] +fn complete_command_runs_and_skips_cleanly_without_sessions() { + // `complete ` is a friendly alias for `dream --task`; with no Claude + // Code sessions for the project it exits cleanly (no model call). + let dir = assert_fs::TempDir::new().unwrap(); + let proj = assert_fs::TempDir::new().unwrap(); + Command::cargo_bin("task-journal") + .unwrap() + .current_dir(proj.path()) + .env("XDG_DATA_HOME", dir.path()) + .args(["complete", "tj-x", "--dry-run"]) + .assert() + .success() + .stdout(contains("dream")); +} diff --git a/crates/tj-core/src/dream/llm_backend.rs b/crates/tj-core/src/dream/llm_backend.rs new file mode 100644 index 0000000..a31b55c --- /dev/null +++ b/crates/tj-core/src/dream/llm_backend.rs @@ -0,0 +1,94 @@ +//! Dream backfill over the unified pluggable [`crate::llm`] backend, so dream +//! gets the same provider choice as everything else (claude-p default, +//! Anthropic, OpenAI/Codex, free local Ollama) instead of its own bespoke +//! clients. + +use anyhow::Context; + +use crate::dream::backend::{BackfillEvent, BackfillInput, DreamBackend}; +use crate::llm::LlmBackend; + +/// Adapts any [`LlmBackend`] into a [`DreamBackend`]: build the dream prompt, +/// run one completion, parse the JSON array of missed events. +pub struct LlmDreamBackend { + llm: Box, +} + +impl LlmDreamBackend { + pub fn new(llm: Box) -> Self { + Self { llm } + } + + pub fn backend_name(&self) -> &'static str { + self.llm.name() + } +} + +impl DreamBackend for LlmDreamBackend { + fn backfill(&self, input: &BackfillInput) -> anyhow::Result> { + let prompt = crate::dream::prompt::build_prompt(input); + let text = self.llm.complete(&prompt, 1024)?; + parse_backfill_json(&text) + } +} + +/// Parse the model's reply (a JSON array of `BackfillEvent`, possibly wrapped in +/// a ```json fence) into events. +pub fn parse_backfill_json(text: &str) -> anyhow::Result> { + let json_str = text + .trim() + .trim_start_matches("```json") + .trim_start_matches("```") + .trim_end_matches("```") + .trim(); + serde_json::from_str(json_str) + .with_context(|| format!("dream JSON parse failed; got: {json_str}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event::EventType; + + #[test] + fn parse_strips_fence_and_decodes() { + let reply = "```json\n[{\"event_type\":\"decision\",\"task_id\":\"tj-1\",\ +\"text\":\"chose X\",\"timestamp\":\"2026-06-13T00:00:00Z\"}]\n```"; + let evs = parse_backfill_json(reply).unwrap(); + assert_eq!(evs.len(), 1); + assert_eq!(evs[0].event_type, EventType::Decision); + assert_eq!(evs[0].task_id, "tj-1"); + assert!(evs[0].text.contains("chose X")); + } + + #[test] + fn parse_empty_array() { + assert!(parse_backfill_json("[]").unwrap().is_empty()); + } + + #[test] + fn llm_dream_backend_runs_and_parses() { + struct FakeLlm; + impl LlmBackend for FakeLlm { + fn complete(&self, _prompt: &str, _max: u32) -> anyhow::Result { + Ok( + "[{\"event_type\":\"finding\",\"task_id\":\"tj-x\",\"text\":\"found it\",\ +\"timestamp\":\"2026-06-13T00:00:00Z\"}]" + .to_string(), + ) + } + fn name(&self) -> &'static str { + "fake" + } + } + let b = LlmDreamBackend::new(Box::new(FakeLlm)); + let input = BackfillInput { + tasks: vec![], + transcript: "x".into(), + }; + let evs = b.backfill(&input).unwrap(); + assert_eq!(evs.len(), 1); + assert_eq!(evs[0].text, "found it"); + assert_eq!(b.backend_name(), "fake"); + } +} diff --git a/crates/tj-core/src/dream/mod.rs b/crates/tj-core/src/dream/mod.rs index 96d5a9f..3924953 100644 --- a/crates/tj-core/src/dream/mod.rs +++ b/crates/tj-core/src/dream/mod.rs @@ -8,6 +8,7 @@ pub mod agent_sdk; pub mod backend; pub mod backfill; pub mod http; +pub mod llm_backend; pub mod prompt; pub mod scope; pub mod state; From b93cdf25db09a495e4af978db9dc6a627f634ae4 Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sat, 13 Jun 2026 16:26:13 +0400 Subject: [PATCH 5/6] =?UTF-8?q?feat(consolidate):=20promote=20conventions?= =?UTF-8?q?=20to=20always-on=20CLAUDE.md=20block=20=E2=80=94=20piece=205/5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the gap where auto-derived project conventions were only available on-demand (search/resume), never guaranteed like hand-written CLAUDE.md rules (epic claude-memory-isg). - tj-core::consolidate: render_conventions_block / upsert_conventions_block — a managed block delimited by `` that's inserted or regenerated in place, never touching hand-written content. - tj-cli: `consolidate --write-claude-md` writes this run's distilled conventions into ./CLAUDE.md so every future session sees them on the same guaranteed always-on path as your hand rules. Re-running regenerates the block (idempotent, no duplication). Tests: block append-then-replace keeps hand content + single block (core); CLI --write-claude-md produces the managed block with a fact (cli). Clean on default and --no-default-features. All functional pieces of the epic done; next is the single release. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/tj-cli/src/main.rs | 32 +++++++++++-- crates/tj-cli/tests/cli.rs | 19 ++++++++ crates/tj-core/src/consolidate.rs | 77 +++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 3 deletions(-) diff --git a/crates/tj-cli/src/main.rs b/crates/tj-cli/src/main.rs index e0dcadb..fc9639b 100644 --- a/crates/tj-cli/src/main.rs +++ b/crates/tj-cli/src/main.rs @@ -673,6 +673,10 @@ enum Commands { /// Defaults to TJ_BACKEND, then claude-p (subscription, no API key). #[arg(long)] backend: Option, + /// Also write the conventions into ./CLAUDE.md as a managed block, so + /// they're always-on for every session (regenerated on each run). + #[arg(long)] + write_claude_md: bool, }, /// Render and print the resume pack for a task. Pack { @@ -1338,8 +1342,12 @@ fn main() -> Result<()> { other => anyhow::bail!("expected `on` or `off`, got `{other}`"), } } - Commands::Consolidate { max_facts, backend } => { - run_consolidate(max_facts, backend.as_deref())?; + Commands::Consolidate { + max_facts, + backend, + write_claude_md, + } => { + run_consolidate(max_facts, backend.as_deref(), write_claude_md)?; } Commands::Event { task_id, @@ -4078,7 +4086,11 @@ PATH; or pick one via --backend / TJ_BACKEND: anthropic, openai, ollama (free, l /// distil them into durable facts via one LLM call through the chosen backend, /// and store the facts as events in a per-project conventions task. Skips /// cleanly (no spend) when no backend is available. -fn run_consolidate(max_facts: usize, backend: Option<&str>) -> anyhow::Result<()> { +fn run_consolidate( + max_facts: usize, + backend: Option<&str>, + write_claude_md: bool, +) -> anyhow::Result<()> { let cwd = std::env::current_dir()?; let project_hash = tj_core::project_hash::from_path(&cwd)?; let events_path = tj_core::paths::events_dir()?.join(format!("{project_hash}.jsonl")); @@ -4178,6 +4190,20 @@ another via --backend / TJ_BACKEND: anthropic (ANTHROPIC_API_KEY), openai \ println!( "consolidated {written} new fact(s) into task {task_id} (\"{CONSOLIDATE_TASK_TITLE}\")" ); + + // Promote to always-on: regenerate the managed conventions block in + // ./CLAUDE.md from this run's full set of facts. + if write_claude_md { + let path = cwd.join("CLAUDE.md"); + let existing = std::fs::read_to_string(&path).unwrap_or_default(); + let updated = tj_core::consolidate::upsert_conventions_block(&existing, &facts); + std::fs::write(&path, updated)?; + println!( + "wrote {} convention(s) into the managed block in {}", + facts.len(), + path.display() + ); + } Ok(()) } diff --git a/crates/tj-cli/tests/cli.rs b/crates/tj-cli/tests/cli.rs index 9e1024c..2409c64 100644 --- a/crates/tj-cli/tests/cli.rs +++ b/crates/tj-cli/tests/cli.rs @@ -5295,6 +5295,25 @@ fn consolidate_writes_facts_to_conventions_task_and_dedups() { ); mock.assert(); + // --write-claude-md promotes the conventions into a managed CLAUDE.md block. + Command::cargo_bin("task-journal") + .unwrap() + .current_dir(proj.path()) + .env("XDG_DATA_HOME", xdg.path()) + .env("TJ_BACKEND", "anthropic") + .env("ANTHROPIC_API_KEY", "test-key") + .env("TJ_CONSOLIDATE_BASE_URL", server.url()) + .env("TJ_EMBED", "hash") + .args(["consolidate", "--write-claude-md"]) + .assert() + .success(); + let claude_md = std::fs::read_to_string(proj.path().join("CLAUDE.md")).unwrap(); + assert!( + claude_md.contains("task-journal:conventions:start") + && claude_md.contains("idempotent ledger"), + "CLAUDE.md must hold the managed conventions block; got: {claude_md}" + ); + // The fact is now recallable. let recall = String::from_utf8( Command::cargo_bin("task-journal") diff --git a/crates/tj-core/src/consolidate.rs b/crates/tj-core/src/consolidate.rs index 49d1375..ffbe61c 100644 --- a/crates/tj-core/src/consolidate.rs +++ b/crates/tj-core/src/consolidate.rs @@ -90,10 +90,87 @@ pub fn parse_facts(text: &str) -> Vec { out } +// --------------------------------------------------------------------------- +// Promote conventions to always-on: a managed block in the project CLAUDE.md. +// --------------------------------------------------------------------------- + +const CONV_START: &str = ""; +const CONV_END: &str = ""; + +/// Render the consolidated facts as a managed CLAUDE.md block (delimited so it +/// can be regenerated without disturbing hand-written content). +pub fn render_conventions_block(facts: &[ConsolidatedFact]) -> String { + let mut s = String::from(CONV_START); + s.push_str( + "\n## Project conventions (auto-derived by task-journal)\n\ +_Regenerate with `task-journal consolidate --write-claude-md`. Lines between the \ +markers are overwritten — edit elsewhere._\n\n", + ); + for f in facts { + s.push_str(&format!("- ({}) {}\n", f.tier, f.text)); + } + s.push_str(CONV_END); + s +} + +/// Insert or replace the managed conventions block in `existing` CLAUDE.md text. +/// Replaces the block between the markers if present, else appends it. Never +/// touches anything outside the markers. +pub fn upsert_conventions_block(existing: &str, facts: &[ConsolidatedFact]) -> String { + let block = render_conventions_block(facts); + match (existing.find(CONV_START), existing.find(CONV_END)) { + (Some(start), Some(end_idx)) if end_idx >= start => { + let end = end_idx + CONV_END.len(); + format!("{}{}{}", &existing[..start], block, &existing[end..]) + } + _ => { + let mut out = existing.to_string(); + if !out.is_empty() { + if !out.ends_with('\n') { + out.push('\n'); + } + out.push('\n'); + } + out.push_str(&block); + out.push('\n'); + out + } + } +} + #[cfg(test)] mod tests { use super::*; + fn fact(tier: &str, text: &str) -> ConsolidatedFact { + ConsolidatedFact { + tier: tier.into(), + text: text.into(), + } + } + + #[test] + fn conventions_block_appends_then_replaces_idempotently() { + let facts = vec![fact("semantic", "always lock the DB for money")]; + // Append into existing hand-written content. + let v1 = upsert_conventions_block("# My project\n\nHand rules.\n", &facts); + assert!(v1.contains("# My project")); + assert!(v1.contains("always lock the DB")); + assert!(v1.contains(CONV_START) && v1.contains(CONV_END)); + + // Re-run with new facts → replaces the block, keeps hand content, no dup. + let facts2 = vec![fact("procedural", "PR into main, squash")]; + let v2 = upsert_conventions_block(&v1, &facts2); + assert!(v2.contains("# My project"), "hand content preserved"); + assert!(v2.contains("PR into main, squash")); + assert!(!v2.contains("always lock the DB"), "old facts replaced"); + assert_eq!( + v2.matches(CONV_START).count(), + 1, + "exactly one managed block" + ); + } + #[test] fn parse_facts_extracts_tagged_lines() { let reply = "[semantic] Refunds route through the idempotent ledger\n\ From 88a3d3a1c4b5cc7f9ee194591988a3744bef2ffa Mon Sep 17 00:00:00 2001 From: Mher Shahinyan Date: Sat, 13 Jun 2026 16:27:57 +0400 Subject: [PATCH 6/6] =?UTF-8?q?chore(release):=200.21.0=20=E2=80=94=20reli?= =?UTF-8?q?able+free=20capture,=20pluggable=20AI=20backend,=20complete,=20?= =?UTF-8?q?conventions=E2=86=92CLAUDE.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 41 +++++++++++++++++++++++++++++++ Cargo.lock | 6 ++--- Cargo.toml | 2 +- crates/tj-cli/Cargo.toml | 2 +- crates/tj-mcp/Cargo.toml | 2 +- plugin/.claude-plugin/plugin.json | 2 +- 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cb14a1..b626ab5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.21.0] - 2026-06-13 + +### Added +- **Reliable, free, non-blocking capture.** The UserPromptSubmit nudge is now + adaptive: it always reminds you to record, and escalates when a session has + done substantial work (large transcript) but logged few journal entries — + same constant-context-injection trick that keeps caveman mode reliable, never + a blocking gate. No model, no cost. +- **Capture kill-switch.** `task-journal capture off` writes a marker that + no-ops the realtime capture path of `ingest-hook` (the read-only SessionStart + resume still runs) — even in an already-running session, because the hook + re-invokes the on-disk binary each event. `capture on` clears it. Silences a + stale auto-capture hook without a restart. +- **Pluggable LLM backend.** A small `LlmBackend` trait + adapters so this + public package grows new providers without touching callers. Default + **`claude-p`** (your Claude subscription, no API key); pick others with + `TJ_BACKEND` or a per-command `--backend`: + - `anthropic` (`ANTHROPIC_API_KEY`), + - `openai` — any OpenAI-compatible API, covering OpenAI and Codex + (`OPENAI_API_KEY`, `TJ_OPENAI_BASE_URL`, `TJ_OPENAI_MODEL`), + - `ollama` — a **free** local model via Ollama's OpenAI-compatible endpoint + (`TJ_OLLAMA_URL`, `TJ_OLLAMA_MODEL`). + `consolidate` and `dream` both route through it; the heuristic fallback is + gone (AI-only — when no backend is available the command skips, never + fabricates). +- **`task-journal complete `** — "make this task complete from its + transcripts": re-reads the sessions tied to a task and appends the + decisions/findings the live capture missed. A friendly alias for + `dream --task`; one LLM call per session via the chosen backend (free with + `--backend ollama`). +- **Conventions → always-on.** `consolidate --write-claude-md` writes the + distilled project conventions into `./CLAUDE.md` as a managed, regenerable + block (``), so every session sees + them on the same guaranteed path as your hand-written rules. Re-running + regenerates the block in place without touching anything else. + +### Internal +- `tj-core::llm` (trait + claude-p/anthropic/openai/ollama adapters + + `backend_from_env`); `dream::llm_backend::LlmDreamBackend`; consolidate + conventions-block writer; adaptive-nudge escalation + capture marker. + ## [0.20.0] - 2026-06-13 ### Added diff --git a/Cargo.lock b/Cargo.lock index ccd4b02..f26e730 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2572,7 +2572,7 @@ dependencies = [ [[package]] name = "task-journal-cli" -version = "0.20.0" +version = "0.21.0" dependencies = [ "anyhow", "assert_cmd", @@ -2596,7 +2596,7 @@ dependencies = [ [[package]] name = "task-journal-core" -version = "0.20.0" +version = "0.21.0" dependencies = [ "anyhow", "chrono", @@ -2621,7 +2621,7 @@ dependencies = [ [[package]] name = "task-journal-mcp" -version = "0.20.0" +version = "0.21.0" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index c9190bf..1704417 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [workspace.package] -version = "0.20.0" +version = "0.21.0" edition = "2021" rust-version = "1.88" license = "MIT" diff --git a/crates/tj-cli/Cargo.toml b/crates/tj-cli/Cargo.toml index 494c6a8..677ced4 100644 --- a/crates/tj-cli/Cargo.toml +++ b/crates/tj-cli/Cargo.toml @@ -23,7 +23,7 @@ default = ["embed"] embed = ["tj-core/embed"] [dependencies] -tj-core = { package = "task-journal-core", version = "0.20.0", path = "../tj-core", default-features = false } +tj-core = { package = "task-journal-core", version = "0.21.0", path = "../tj-core", default-features = false } anyhow = { workspace = true } clap = { workspace = true } tracing = { workspace = true } diff --git a/crates/tj-mcp/Cargo.toml b/crates/tj-mcp/Cargo.toml index 46cadbe..42370f0 100644 --- a/crates/tj-mcp/Cargo.toml +++ b/crates/tj-mcp/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main.rs" [dependencies] # Lean: the MCP server doesn't embed yet, so it skips the model2vec backend. -tj-core = { package = "task-journal-core", version = "0.20.0", path = "../tj-core", default-features = false } +tj-core = { package = "task-journal-core", version = "0.21.0", path = "../tj-core", default-features = false } anyhow = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 2a66a31..e2537a3 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "task-journal", - "version": "0.20.0", + "version": "0.21.0", "description": "Append-only journal of AI-coding task reasoning chains: hypotheses, decisions, rejections, evidence. Renders compact resume packs so an agent can pick up a 2-week-old task with full context.", "author": { "name": "Mher Shahinyan"