From 7b7a7c9b7ecf67e6f22d31bd7e7ddc7db1033ca7 Mon Sep 17 00:00:00 2001 From: Ben Barber Date: Thu, 9 Jul 2026 11:04:28 -0400 Subject: [PATCH 1/2] fix(cli): clippy redundant_closure in cursor export --- crates/path-cli/src/cmd_export.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/crates/path-cli/src/cmd_export.rs b/crates/path-cli/src/cmd_export.rs index 9447f76e..2b6fc222 100644 --- a/crates/path-cli/src/cmd_export.rs +++ b/crates/path-cli/src/cmd_export.rs @@ -1405,9 +1405,9 @@ fn build_cursor_session( // Reuse the existing id when present, otherwise pre-create a // workspaceStorage entry so Cursor adopts ours on next open. let resolver = PathResolver::new(); - if let Ok(ensured) = resolver.ensure_workspace_storage_entry(&canonical, |path| { - stable_workspace_id_for(path) - }) { + if let Ok(ensured) = + resolver.ensure_workspace_storage_entry(&canonical, stable_workspace_id_for) + { projector = projector.with_workspace_id(ensured.id); if ensured.created { eprintln!( @@ -1535,13 +1535,8 @@ fn cursor_open_hints(workspace: &std::path::Path) -> Vec { } } - #[cfg(not(target_os = "emscripten"))] -fn upsert_cursor_kv( - tx: &rusqlite::Transaction<'_>, - key: &str, - value: &str, -) -> Result<()> { +fn upsert_cursor_kv(tx: &rusqlite::Transaction<'_>, key: &str, value: &str) -> Result<()> { tx.execute( "INSERT OR REPLACE INTO cursorDiskKV (key, value) VALUES (?1, ?2)", rusqlite::params![key, value], From d0fe974b4378dcb7212a1fc6c876f073ab708333 Mon Sep 17 00:00:00 2001 From: Ben Barber Date: Thu, 9 Jul 2026 11:14:28 -0400 Subject: [PATCH 2/2] =?UTF-8?q?feat(cli):=20add=20`path=20p=20cache=20sync?= =?UTF-8?q?`=20=E2=80=94=20incremental=20session=20ingestion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `path p cache sync [types…]` enumerates sessions across the installed agent harnesses (claude/gemini/codex/opencode/cursor/pi, or just the named ones) and derives into the cache only what is new or changed. The manifest at ~/.toolpath/sync.json maps artifact type → session id → {project?, cache_id, last_activity?, message_count, synced_at}; a session re-derives only when its fingerprint (last_activity + message_count) differs. Manifest writes are atomic (temp + rename, 0600) and checkpointed per type; sync overwrites what it re-derives (refresh semantics) and never deletes — the cache is an archive, not a mirror. Enumeration reuses cmd_share::gather_sessions; per-session derivation goes through new *_with variants of the cmd_import session helpers so listing and derivation share provider managers (and tests can inject fixture resolvers). Real-data numbers: 34 sessions / 129 MB of Claude JSONL → 5.7 s cold, 0.9 s incremental no-op. path-cli 0.15.0 → 0.16.0 (+ toolpath-cli shim bump). --- CHANGELOG.md | 31 ++ CLAUDE.md | 7 +- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 4 + crates/path-cli/Cargo.toml | 2 +- crates/path-cli/src/cmd_cache.rs | 10 + crates/path-cli/src/cmd_import.rs | 80 +++- crates/path-cli/src/cmd_share.rs | 15 +- crates/path-cli/src/lib.rs | 2 + crates/path-cli/src/sync.rs | 576 +++++++++++++++++++++++++++ crates/path-cli/tests/integration.rs | 102 +++++ crates/toolpath-cli/Cargo.toml | 4 +- site/_data/crates.json | 4 +- 14 files changed, 818 insertions(+), 23 deletions(-) create mode 100644 crates/path-cli/src/sync.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a38aed70..c3f3d521 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,37 @@ All notable changes to the Toolpath workspace are documented here. +## `path p cache sync` — incremental session ingestion — 2026-07-09 + +Adds `path p cache sync [types…]`, the first step toward a cache that fills +itself: it enumerates sessions across the installed agent harnesses and +derives into the cache only what is new or changed since the last sync, so +users no longer have to `p import` each session by hand. + +- **`path-cli`** (0.16.0): + - `path p cache sync [claude|gemini|codex|opencode|cursor|pi]…` syncs the + named artifact types; with no arguments it syncs every harness. + Enumeration reuses the `path share` aggregation, and per-session + derivation goes through the same provider managers, so listing and + derivation always agree on provider roots. + - The sync manifest at `~/.toolpath/sync.json` maps artifact type → + session id → `{project?, cache_id, last_activity?, message_count, + synced_at}`. A session is re-derived when its `last_activity` or + `message_count` differs from the record; otherwise it is skipped. + The manifest is written atomically (temp file + rename, `0600`) and + checkpointed after each type, so an interrupted first run keeps the + types it finished. + - Sync owns refresh semantics: it overwrites cache entries it + re-derives (no `--force` needed), while manual `p import` keeps its + error-on-hit default. Sessions deleted upstream keep both their cache + document and their manifest record — the cache is an archive, not a + mirror. + - Per-type summary on stderr (`claude 3 new, 1 updated, 240 unchanged`). + On a default run, harnesses with no sessions stay silent; explicit + types always report. Derivation failures warn and tally as `failed` + without aborting the run. +- **`toolpath-cli`** (0.16.0): version bump only (tracks `path-cli`). + ## Derive: resolve duplicate step ids — 2026-07-01 - **`toolpath-convo`** (0.11.1): `derive_path` now guarantees the derived diff --git a/CLAUDE.md b/CLAUDE.md index a404cc5e..f7327559 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -114,6 +114,8 @@ cargo run -p path-cli -- p export pathbase --input # Plumbing: manage the cache cargo run -p path-cli -- p cache ls cargo run -p path-cli -- p cache rm +cargo run -p path-cli -- p cache sync # ingest new/changed sessions from every harness +cargo run -p path-cli -- p cache sync claude codex # only these artifact types # Inspect / analyze cargo run -p path-cli -- p render dot --input doc.json @@ -150,7 +152,7 @@ cargo run -p path-cli -- auth logout and no deprecation shim. They all now live exclusively under `path p …`. -The **cache** at `~/.toolpath/documents/.json` is the single landing zone for every `import` (and for `import pathbase` downloads). Cache id is `-` — e.g. `claude-abc123`, `git-main`, `pathbase-alex-pathstash-path-pr-42` (Pathbase paths key on `--`, anon paths on `anon-pathstash-`). Files are `0600`, parent directory `0700`. `$TOOLPATH_CONFIG_DIR` overrides the root. Default behavior: error on cache hit; pass `--force` to overwrite. `--no-cache` sends the JSON to stdout for shell composition. +The **cache** at `~/.toolpath/documents/.json` is the single landing zone for every `import` (and for `import pathbase` downloads). Cache id is `-` — e.g. `claude-abc123`, `git-main`, `pathbase-alex-pathstash-path-pr-42` (Pathbase paths key on `--`, anon paths on `anon-pathstash-`). Files are `0600`, parent directory `0700`. `$TOOLPATH_CONFIG_DIR` overrides the root. Default behavior: error on cache hit; pass `--force` to overwrite. `--no-cache` sends the JSON to stdout for shell composition. `p cache sync` fills the cache incrementally from the installed agent harnesses (see "Things to know") and always overwrites what it re-derives. `path auth login` prints `/auth/cli`; the user opens it, logs in, and pastes the 8-character code back into the CLI. The CLI calls @@ -190,7 +192,7 @@ Tests live alongside the code (`#[cfg(test)] mod tests`), plus `path-cli` has in - `toolpath-cursor`: 78 unit + 8 integration round-trip + 1 real-DB sanity + 1 doc test (state.vscdb SQLite reader, bubble store + composer header parsing, content-addressed blob lookup, projector with full TOOL_TABLE coverage, JSONL transcript ingest in `examples/dump_fixture.rs`) - `toolpath-pi`: 133 unit + 26 integration + 5 doc tests (types, paths, error, reader, io, provider) - `toolpath-dot`: 30 unit + 2 doc tests (render, visual conventions, escaping) -- `path-cli`: 324 unit + 98 integration tests (import/export/cache, track sessions, merge, validate, roundtrip, render-md snapshots, deprecation aliases, pathbase HTTP mock-server tests, fzf-friendly TSV output, `path resume` orchestration with injectable `ExecStrategy`, `path query`/`path kind` jaq filters + kind-selector matching + step wrapping over a `$TOOLPATH_CONFIG_DIR` cache sandbox, streaming-planner recognition + streamed-output-equals-slurp equality checks). For an end-to-end check against a real Pathbase deployment, run `scripts/test-pathbase-live.sh ` — it does an anon round-trip in a sandboxed config dir and, if you're logged into that URL, an authed pathstash round-trip too. +- `path-cli`: 336 unit + 101 integration tests (import/export/cache, track sessions, merge, validate, roundtrip, render-md snapshots, deprecation aliases, pathbase HTTP mock-server tests, fzf-friendly TSV output, `path resume` orchestration with injectable `ExecStrategy`, `path query`/`path kind` jaq filters + kind-selector matching + step wrapping over a `$TOOLPATH_CONFIG_DIR` cache sandbox, streaming-planner recognition + streamed-output-equals-slurp equality checks, `p cache sync` incremental ingestion — manifest fingerprints, refresh-overwrite, per-type filtering, failure tallying — over resolver-injected provider fixtures). For an end-to-end check against a real Pathbase deployment, run `scripts/test-pathbase-live.sh ` — it does an anon round-trip in a sandboxed config dir and, if you're logged into that URL, an authed pathstash round-trip too. - `toolpath-cli`: 0 tests (it's a one-line `path_cli::run()` shim crate that exists only so `cargo install toolpath-cli` keeps installing the `path` binary) Validate example documents: `for f in examples/*.json; do cargo run -p path-cli -- p validate --input "$f"; done` @@ -259,3 +261,4 @@ Build the site after changes: `cd site && pnpm run build` (should produce 11 pag - `path share` is the one-shot equivalent of `path p import | path p export pathbase`. It probes installed agent harnesses (claude/gemini/codex/opencode/pi), aggregates their sessions into a single fzf picker, and ranks rows whose project (claude/gemini/pi) or recorded cwd (codex/opencode) canonicalizes to the current directory at the top. `--harness` narrows the picker to one provider; `--harness X --session Y` (and `--project P` for keyed providers) skips the picker entirely. Pathbase flags (`--url`, `--anon`, `--repo`, `--slug`, `--public`) match `path export pathbase`. By default the derived doc is written to the cache like `import` does; pass `--no-cache` to skip. - `path resume ` is the inverse of `path share`. It accepts a Pathbase URL, an `owner/repo/slug` shorthand, a local toolpath JSON file, or a cache id; resolves it (caching URL fetches under `~/.toolpath/documents/` unless `--no-cache`); validates that the document is a single agent-bearing `Path`; then opens an `fzf` harness picker (skipped with `--harness X`). The picker pre-selects the source harness inferred from `path.meta.source` (`claude-code`/`gemini-cli`/`codex`/`opencode`/`pi`) when it's installed. After picking, `path resume` projects the session into the harness's on-disk layout under the chosen working directory (default: shell cwd; override with `-C, --cwd P`) and `execvp`'s the harness's resume command (`claude -r ` / `gemini --resume ` / `codex resume ` / `opencode --session ` / `pi --session `). On Windows it spawns and waits, propagating the exit code. The exec is mockable via `cmd_resume::ExecStrategy` — production uses `RealExec`; integration tests use `RecordingExec` to capture the recipe without launching a real harness. - `path query` does not load the whole cache into memory when it can avoid it. `crates/path-cli/src/query/plan.rs` parses the jaq filter into jaq's own AST (`jaq_core::load::parse::Term`) and classifies it into a `Plan`: `PerFileStream` (`.[] | g` element-wise work — run per document, print as you go), `Decompose { reduce }` (algebraic aggregations — run the whole filter per file, concatenate the per-file outputs, then run a derived combine: `map`→`add` (array concat), top-N `sort_by(k)|.[:N]`→`add | sort_by(k)|.[:N]`, `length`→`add` over exact integer counts), or `Slurp` (the always-correct whole-array fallback). Recognition is conservative — a non-distributive prefix like `unique`/`group_by` slurps, and so do scalar `add` (float sums re-associate across per-file partials), `min`/`max` (`[] | min == null` poisons the merge), and any unrecognized tail — so **the planner never changes an answer** — `crates/path-cli/src/query/filter.rs` tests assert streamed output equals slurp byte-for-byte. `filter::execute` compiles the filter once (jaq's compiled `Filter` is fully owned, so it's reused across files) and drives the plan; `mod.rs::stream_files` yields one document's wrapped steps at a time. `TOOLPATH_QUERY_EXPLAIN=1` prints the chosen plan to stderr. No user-facing flag — it's automatic. Tie-break caveat: a streamed top-N matches slurp's *ranking*, but boundary ties may resolve to different specific rows. +- Cache sync: `path p cache sync [types…]` (`crates/path-cli/src/sync.rs`) incrementally ingests agent sessions into the cache — no args syncs all six harnesses. It enumerates via the `path share` aggregation (`cmd_share::gather_sessions`) and derives each session through the *same* provider managers (the `derive_*_session_with` helpers in `cmd_import.rs`), so listing and derivation agree on provider roots and tests can inject fixture resolvers. A session is re-derived only when its fingerprint (`last_activity` + `message_count`) differs from the manifest at `~/.toolpath/sync.json` (artifact type → session id → `{project?, cache_id, last_activity?, message_count, synced_at}`; atomic temp+rename writes, `0600`, checkpointed after each type). Sync always writes the cache with force — refresh semantics — and never deletes: sessions removed upstream keep their cache docs and manifest records (archive, not mirror). Derivation failures warn and tally, they don't abort. Caveats: enumeration costs the providers' metadata pass (full parse per JSONL session file — a stat-level fast path is the planned optimization before `path query` runs sync implicitly), and cursor composers without a workspace are skipped, same as `share`. diff --git a/Cargo.lock b/Cargo.lock index 982b6bfc..79cb4ba7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2441,7 +2441,7 @@ dependencies = [ [[package]] name = "path-cli" -version = "0.15.0" +version = "0.16.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index a8ffae2f..ebe7c222 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ toolpath-github = { version = "0.6.0", path = "crates/toolpath-github" } toolpath-dot = { version = "0.5.0", path = "crates/toolpath-dot" } toolpath-md = { version = "0.7.0", path = "crates/toolpath-md" } toolpath-pi = { version = "0.6.0", path = "crates/toolpath-pi" } -path-cli = { version = "0.15.0", path = "crates/path-cli" } +path-cli = { version = "0.16.0", path = "crates/path-cli" } pathbase-client = { version = "0.2.0", path = "crates/pathbase-client" } reqwest = { version = "0.13", default-features = false, features = ["blocking", "json", "rustls"] } diff --git a/README.md b/README.md index 69824357..963eebfd 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,10 @@ path p import opencode # List what's in the cache path p cache ls +# Ingest new/changed agent sessions into the cache (all harnesses, or named ones) +path p cache sync +path p cache sync claude codex + # Export a cached document back into a Claude Code session path p export claude --input claude- --project /path/to/resume diff --git a/crates/path-cli/Cargo.toml b/crates/path-cli/Cargo.toml index 8621989e..d73ecc87 100644 --- a/crates/path-cli/Cargo.toml +++ b/crates/path-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "path-cli" -version = "0.15.0" +version = "0.16.0" edition.workspace = true license.workspace = true repository = "https://github.com/empathic/toolpath" diff --git a/crates/path-cli/src/cmd_cache.rs b/crates/path-cli/src/cmd_cache.rs index e6a08c2d..88bfe854 100644 --- a/crates/path-cli/src/cmd_cache.rs +++ b/crates/path-cli/src/cmd_cache.rs @@ -23,12 +23,22 @@ pub enum CacheOp { /// Cache id (filename without `.json`) id: String, }, + /// Ingest agent sessions into the cache, deriving only what is new + /// or changed since the last sync (tracked in `$CONFIG_DIR/sync.json`) + #[cfg(not(target_os = "emscripten"))] + Sync { + /// Artifact types to sync (default: every agent harness) + #[arg(value_enum)] + types: Vec, + }, } pub fn run(op: CacheOp) -> Result<()> { match op { CacheOp::Ls => run_ls(), CacheOp::Rm { id } => run_rm(&id), + #[cfg(not(target_os = "emscripten"))] + CacheOp::Sync { types } => crate::sync::run(types), } } diff --git a/crates/path-cli/src/cmd_import.rs b/crates/path-cli/src/cmd_import.rs index 7dac6532..9b229965 100644 --- a/crates/path-cli/src/cmd_import.rs +++ b/crates/path-cli/src/cmd_import.rs @@ -473,7 +473,17 @@ fn derive_claude_with_manager( /// Used by `cmd_share` after its picker has resolved the pair; mirrors the /// `(Some(p), Some(s), _)` arm in [`derive_claude_with_manager`]. pub(crate) fn derive_claude_session(project: &str, session: &str) -> Result { - let manager = toolpath_claude::ClaudeConvo::new(); + derive_claude_session_with(&toolpath_claude::ClaudeConvo::new(), project, session) +} + +/// [`derive_claude_session`] against a caller-supplied manager, so sync +/// derives from the same roots it enumerated (and tests can inject a +/// fixture resolver). +pub(crate) fn derive_claude_session_with( + manager: &toolpath_claude::ClaudeConvo, + project: &str, + session: &str, +) -> Result { let cfg = toolpath_claude::derive::DeriveConfig { project_path: Some(project.to_string()), include_thinking: false, @@ -698,7 +708,21 @@ pub(crate) fn derive_gemini_session( session: &str, include_thinking: bool, ) -> Result { - let manager = toolpath_gemini::GeminiConvo::new(); + derive_gemini_session_with( + &toolpath_gemini::GeminiConvo::new(), + project, + session, + include_thinking, + ) +} + +/// [`derive_gemini_session`] against a caller-supplied manager. +pub(crate) fn derive_gemini_session_with( + manager: &toolpath_gemini::GeminiConvo, + project: &str, + session: &str, + include_thinking: bool, +) -> Result { let cfg = toolpath_gemini::derive::DeriveConfig { project_path: Some(project.to_string()), include_thinking, @@ -885,7 +909,14 @@ fn derive_codex(session: Option, all: bool) -> Result> { /// Derive a single Codex session given an explicit session id. pub(crate) fn derive_codex_session(session: &str) -> Result { - let manager = toolpath_codex::CodexConvo::new(); + derive_codex_session_with(&toolpath_codex::CodexConvo::new(), session) +} + +/// [`derive_codex_session`] against a caller-supplied manager. +pub(crate) fn derive_codex_session_with( + manager: &toolpath_codex::CodexConvo, + session: &str, +) -> Result { let config = toolpath_codex::derive::DeriveConfig { project_path: None }; let s = manager .read_session(session) @@ -1035,7 +1066,20 @@ pub(crate) fn derive_opencode_session( session: &str, no_snapshot_diffs: bool, ) -> Result { - let manager = toolpath_opencode::OpencodeConvo::new(); + derive_opencode_session_with( + &toolpath_opencode::OpencodeConvo::new(), + session, + no_snapshot_diffs, + ) +} + +/// [`derive_opencode_session`] against a caller-supplied manager. +#[cfg(not(target_os = "emscripten"))] +pub(crate) fn derive_opencode_session_with( + manager: &toolpath_opencode::OpencodeConvo, + session: &str, + no_snapshot_diffs: bool, +) -> Result { let config = toolpath_opencode::derive::DeriveConfig { no_snapshot_diffs, ..Default::default() @@ -1144,16 +1188,15 @@ fn derive_cursor( Ok(toolpath_cursor::derive_path(&s, &cfg)) }; - let workspace_filter = project.as_deref().map(|p| { - std::fs::canonicalize(p).unwrap_or_else(|_| PathBuf::from(p)) - }); + let workspace_filter = project + .as_deref() + .map(|p| std::fs::canonicalize(p).unwrap_or_else(|_| PathBuf::from(p))); let workspace_match = |m: &toolpath_cursor::CursorSessionMetadata| -> bool { match (&workspace_filter, &m.workspace_path) { (None, _) => true, (Some(_), None) => false, (Some(want), Some(have)) => { - let canonical = - std::fs::canonicalize(have).unwrap_or_else(|_| have.clone()); + let canonical = std::fs::canonicalize(have).unwrap_or_else(|_| have.clone()); &canonical == want } } @@ -1209,7 +1252,15 @@ fn derive_cursor( /// Derive a single cursor composer given an explicit composer id. #[cfg(not(target_os = "emscripten"))] pub(crate) fn derive_cursor_session(session: &str) -> Result { - let manager = toolpath_cursor::CursorConvo::new(); + derive_cursor_session_with(&toolpath_cursor::CursorConvo::new(), session) +} + +/// [`derive_cursor_session`] against a caller-supplied manager. +#[cfg(not(target_os = "emscripten"))] +pub(crate) fn derive_cursor_session_with( + manager: &toolpath_cursor::CursorConvo, + session: &str, +) -> Result { let s = manager .read_session(session) .map_err(|e| anyhow::anyhow!("{}", e))?; @@ -1410,6 +1461,15 @@ pub(crate) fn derive_pi_session( } else { toolpath_pi::PiConvo::new() }; + derive_pi_session_with(&manager, project, session) +} + +/// [`derive_pi_session`] against a caller-supplied manager. +pub(crate) fn derive_pi_session_with( + manager: &toolpath_pi::PiConvo, + project: &str, + session: &str, +) -> Result { let config = toolpath_pi::DeriveConfig::default(); let session = manager .read_session(project, session) diff --git a/crates/path-cli/src/cmd_share.rs b/crates/path-cli/src/cmd_share.rs index fba314ec..927da497 100644 --- a/crates/path-cli/src/cmd_share.rs +++ b/crates/path-cli/src/cmd_share.rs @@ -77,6 +77,16 @@ pub(crate) enum Harness { } impl Harness { + /// Every harness, in presentation order. + pub(crate) const ALL: [Harness; 6] = [ + Harness::Claude, + Harness::Gemini, + Harness::Codex, + Harness::Opencode, + Harness::Cursor, + Harness::Pi, + ]; + pub(crate) fn name(&self) -> &'static str { match self { Harness::Claude => "claude", @@ -836,10 +846,7 @@ fn harness_status_pi(bundle: &HarnessBundle, home: Option<&std::path::Path>) -> } } -fn harness_status_cursor( - bundle: &HarnessBundle, - home: Option<&std::path::Path>, -) -> HarnessStatus { +fn harness_status_cursor(bundle: &HarnessBundle, home: Option<&std::path::Path>) -> HarnessStatus { let Some(mgr) = &bundle.cursor else { return HarnessStatus::unresolved(); }; diff --git a/crates/path-cli/src/lib.rs b/crates/path-cli/src/lib.rs index 784cd337..011718e0 100644 --- a/crates/path-cli/src/lib.rs +++ b/crates/path-cli/src/lib.rs @@ -33,6 +33,8 @@ mod query; mod schema; #[cfg(all(not(target_os = "emscripten"), feature = "embedded-picker"))] mod skim_picker; +#[cfg(not(target_os = "emscripten"))] +mod sync; mod term; use anyhow::Result; diff --git a/crates/path-cli/src/sync.rs b/crates/path-cli/src/sync.rs new file mode 100644 index 00000000..3a116672 --- /dev/null +++ b/crates/path-cli/src/sync.rs @@ -0,0 +1,576 @@ +//! `path p cache sync` — incremental ingestion of agent sessions into +//! the document cache. +//! +//! Enumerates sessions across the requested agent harnesses (the same +//! aggregation `path share` uses), compares each against the sync +//! manifest at `$CONFIG_DIR/sync.json`, and derives + caches only what +//! is new or changed. The manifest maps artifact type → session id → +//! the fingerprint recorded at last sync (`last_activity` + +//! `message_count`), so an unchanged session costs a metadata read +//! instead of a re-derivation, and running sync twice in a row is a +//! no-op. Sessions deleted upstream keep both their cache document and +//! their manifest record — the cache is an archive, not a mirror. + +#![cfg(not(target_os = "emscripten"))] + +use anyhow::{Context, Result, anyhow}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::path::PathBuf; + +use crate::cmd_cache::write_cached; +use crate::cmd_import::DerivedDoc; +use crate::cmd_share::{Harness, HarnessArg, HarnessBundle, SessionRow, gather_sessions}; +use crate::config::config_dir; + +const MANIFEST_FILE: &str = "sync.json"; + +/// What the manifest remembers about one synced session. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub(crate) struct SyncRecord { + /// Project path, for the project-keyed providers (claude/gemini/pi). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) project: Option, + /// Cache entry the derived document was written to. + pub(crate) cache_id: String, + /// Fingerprint: the session's last activity as reported at sync time. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) last_activity: Option>, + /// Fingerprint: message count as reported at sync time. + pub(crate) message_count: usize, + pub(crate) synced_at: DateTime, +} + +/// The sync manifest: artifact type (`"claude"`, `"codex"`, …) → +/// session id → record. Kept as `BTreeMap`s so the JSON on disk is +/// stably ordered. +pub(crate) type Manifest = BTreeMap>; + +/// Per-type tally of what one sync run did. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SyncOutcome { + pub(crate) new: usize, + pub(crate) updated: usize, + pub(crate) unchanged: usize, + pub(crate) failed: usize, +} + +impl SyncOutcome { + fn total(&self) -> usize { + self.new + self.updated + self.unchanged + self.failed + } +} + +pub(crate) fn run(types: Vec) -> Result<()> { + let explicit = !types.is_empty(); + let types = resolve_types(&types); + let bundle = HarnessBundle::from_environment(); + let outcomes = sync_bundle(&bundle, &types)?; + eprint!("{}", render_summary(&outcomes, explicit)); + Ok(()) +} + +/// Explicit args → dedup'd harness list; no args → every harness. +fn resolve_types(args: &[HarnessArg]) -> Vec { + if args.is_empty() { + return Harness::ALL.to_vec(); + } + let mut out: Vec = Vec::with_capacity(args.len()); + for &a in args { + let h = Harness::from_arg(a); + if !out.contains(&h) { + out.push(h); + } + } + out +} + +/// Sync the given harness types from `bundle` into the cache. The +/// manifest is checkpointed after each type so an interrupted first +/// run doesn't forget the types it already finished. +pub(crate) fn sync_bundle( + bundle: &HarnessBundle, + types: &[Harness], +) -> Result> { + let mut manifest = load_manifest()?; + // Only ranking depends on cwd and sync ignores row order, so any + // directory works here. + let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + let mut out = Vec::with_capacity(types.len()); + for &harness in types { + let rows = gather_sessions(bundle, &cwd, Some(harness), None); + let mut records = manifest.get(harness.name()).cloned().unwrap_or_default(); + let outcome = sync_rows(bundle, &rows, &mut records)?; + if !records.is_empty() { + manifest.insert(harness.name().to_string(), records); + save_manifest(&manifest)?; + } + out.push((harness, outcome)); + } + Ok(out) +} + +/// Sync one harness's rows against its manifest records. Derivation +/// failures are warned and tallied, not fatal; cache-write failures +/// (disk, permissions) abort. +fn sync_rows( + bundle: &HarnessBundle, + rows: &[SessionRow], + records: &mut BTreeMap, +) -> Result { + let mut outcome = SyncOutcome::default(); + for row in rows { + let existing = records.get(&row.session_id); + let is_new = existing.is_none(); + if let Some(rec) = existing + && rec.last_activity == row.last_activity + && rec.message_count == row.message_count + { + outcome.unchanged += 1; + continue; + } + match derive_row(bundle, row) { + Ok(derived) => { + // force: sync owns refresh semantics — a re-sync or a + // prior manual `p import` of the same session must not + // error on the existing cache entry. + write_cached(&derived.cache_id, &derived.doc, true)?; + records.insert( + row.session_id.clone(), + SyncRecord { + project: row.project.clone(), + cache_id: derived.cache_id, + last_activity: row.last_activity, + message_count: row.message_count, + synced_at: Utc::now(), + }, + ); + if is_new { + outcome.new += 1; + } else { + outcome.updated += 1; + } + } + Err(e) => { + eprintln!( + "warning: sync {}: session {}: {e}", + row.harness.name(), + row.session_id + ); + outcome.failed += 1; + } + } + } + Ok(outcome) +} + +/// Derive one session through the same manager the row was enumerated +/// from, so listing and derivation always agree on provider roots. +fn derive_row(bundle: &HarnessBundle, row: &SessionRow) -> Result { + use crate::cmd_import as imp; + let project = || { + row.project + .as_deref() + .ok_or_else(|| anyhow!("session {} has no project path", row.session_id)) + }; + match row.harness { + Harness::Claude => { + imp::derive_claude_session_with(mgr(&bundle.claude)?, project()?, &row.session_id) + } + Harness::Gemini => imp::derive_gemini_session_with( + mgr(&bundle.gemini)?, + project()?, + &row.session_id, + false, + ), + Harness::Pi => imp::derive_pi_session_with(mgr(&bundle.pi)?, project()?, &row.session_id), + Harness::Codex => imp::derive_codex_session_with(mgr(&bundle.codex)?, &row.session_id), + Harness::Opencode => { + imp::derive_opencode_session_with(mgr(&bundle.opencode)?, &row.session_id, false) + } + Harness::Cursor => imp::derive_cursor_session_with(mgr(&bundle.cursor)?, &row.session_id), + } +} + +fn mgr(slot: &Option) -> Result<&T> { + slot.as_ref() + .ok_or_else(|| anyhow!("provider not available")) +} + +/// One stderr line per harness. Types the user didn't name are shown +/// only when they had sessions, so a default run doesn't list every +/// uninstalled harness. +fn render_summary(outcomes: &[(Harness, SyncOutcome)], explicit: bool) -> String { + let mut s = String::new(); + for (harness, o) in outcomes { + if o.total() == 0 && !explicit { + continue; + } + s.push_str(&format!( + "{} {} new, {} updated, {} unchanged", + harness.symbol(), + o.new, + o.updated, + o.unchanged + )); + if o.failed > 0 { + s.push_str(&format!(", {} failed", o.failed)); + } + s.push('\n'); + } + if s.is_empty() { + s.push_str("nothing to sync\n"); + } + s +} + +// ── manifest IO ──────────────────────────────────────────────────────── + +fn manifest_path() -> Result { + Ok(config_dir()?.join(MANIFEST_FILE)) +} + +pub(crate) fn load_manifest() -> Result { + let path = manifest_path()?; + let json = match std::fs::read_to_string(&path) { + Ok(s) => s, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Manifest::default()), + Err(e) => return Err(anyhow!("read {}: {e}", path.display())), + }; + serde_json::from_str(&json).with_context(|| { + format!( + "parse {}; delete it to re-sync from scratch", + path.display() + ) + }) +} + +/// Write the manifest atomically (temp file + rename) with the same +/// permissions as the rest of `$CONFIG_DIR`. +fn save_manifest(manifest: &Manifest) -> Result<()> { + let path = manifest_path()?; + let dir = path.parent().expect("manifest path has a parent"); + std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700)); + } + let json = serde_json::to_string_pretty(manifest)?; + let tmp = dir.join(format!("{MANIFEST_FILE}.{}.tmp", std::process::id())); + std::fs::write(&tmp, json).with_context(|| format!("write {}", tmp.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600)) + .with_context(|| format!("chmod 0600 {}", tmp.display()))?; + } + std::fs::rename(&tmp, &path) + .with_context(|| format!("rename {} → {}", tmp.display(), path.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{CONFIG_DIR_ENV, TEST_ENV_LOCK}; + use std::path::Path; + + /// Run `f` with `$TOOLPATH_CONFIG_DIR` pinned to `/.toolpath`; + /// `f` receives the tempdir root for building provider fixtures. + fn with_cfg R, R>(f: F) -> R { + let _g = TEST_ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let temp = tempfile::tempdir().unwrap(); + let prev = std::env::var_os(CONFIG_DIR_ENV); + unsafe { + std::env::set_var(CONFIG_DIR_ENV, temp.path().join(".toolpath")); + } + let result = f(temp.path()); + unsafe { + match prev { + Some(v) => std::env::set_var(CONFIG_DIR_ENV, v), + None => std::env::remove_var(CONFIG_DIR_ENV), + } + } + result + } + + fn write_claude_session(home: &Path, project_slug: &str, session: &str, prompt: &str) { + let project_dir = home.join(".claude/projects").join(project_slug); + std::fs::create_dir_all(&project_dir).unwrap(); + let user = format!( + r#"{{"type":"user","uuid":"u-{session}","timestamp":"2024-01-02T00:00:00Z","cwd":"/test/project","message":{{"role":"user","content":"{prompt}"}}}}"# + ); + let asst = format!( + r#"{{"type":"assistant","uuid":"a-{session}","timestamp":"2024-01-02T00:00:01Z","message":{{"role":"assistant","content":"hi"}}}}"# + ); + std::fs::write( + project_dir.join(format!("{session}.jsonl")), + format!("{user}\n{asst}\n"), + ) + .unwrap(); + } + + fn claude_bundle(home: &Path) -> HarnessBundle { + let resolver = toolpath_claude::PathResolver::new().with_claude_dir(home.join(".claude")); + HarnessBundle { + claude: Some(toolpath_claude::ClaudeConvo::with_resolver(resolver)), + ..Default::default() + } + } + + fn cached_step_count(cache_id: &str) -> usize { + let path = crate::cmd_cache::cache_path(cache_id).unwrap(); + let json = std::fs::read_to_string(path).unwrap(); + let doc = toolpath::v1::Graph::from_json(&json).unwrap(); + doc.single_path().map(|p| p.steps.len()).unwrap_or(0) + } + + #[test] + fn manifest_roundtrips_and_missing_is_empty() { + with_cfg(|_| { + assert!(load_manifest().unwrap().is_empty()); + + let mut manifest = Manifest::default(); + manifest.entry("claude".to_string()).or_default().insert( + "sess-1".to_string(), + SyncRecord { + project: Some("/test/project".to_string()), + cache_id: "claude-p1".to_string(), + last_activity: Some("2024-01-02T00:00:01Z".parse().unwrap()), + message_count: 2, + synced_at: "2026-07-09T00:00:00Z".parse().unwrap(), + }, + ); + save_manifest(&manifest).unwrap(); + assert_eq!(load_manifest().unwrap(), manifest); + }); + } + + #[cfg(unix)] + #[test] + fn manifest_file_is_0600() { + use std::os::unix::fs::PermissionsExt; + with_cfg(|_| { + save_manifest(&Manifest::default()).unwrap(); + let mode = std::fs::metadata(manifest_path().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + }); + } + + #[test] + fn corrupt_manifest_errors_with_hint() { + with_cfg(|_| { + save_manifest(&Manifest::default()).unwrap(); + std::fs::write(manifest_path().unwrap(), "not json").unwrap(); + let err = load_manifest().unwrap_err(); + assert!(err.to_string().contains("re-sync from scratch")); + }); + } + + #[test] + fn first_sync_ingests_then_second_is_unchanged() { + with_cfg(|home| { + write_claude_session(home, "-test-project", "sess-aaa", "Add a feature"); + write_claude_session(home, "-test-project", "sess-bbb", "Fix a bug"); + let bundle = claude_bundle(home); + + let outcomes = sync_bundle(&bundle, &[Harness::Claude]).unwrap(); + assert_eq!(outcomes.len(), 1); + let (_, first) = outcomes[0]; + assert_eq!( + (first.new, first.updated, first.unchanged, first.failed), + (2, 0, 0, 0) + ); + + let manifest = load_manifest().unwrap(); + let records = manifest.get("claude").unwrap(); + assert_eq!(records.len(), 2); + let rec = records.get("sess-aaa").unwrap(); + assert_eq!(rec.project.as_deref(), Some("/test/project")); + assert_eq!(rec.message_count, 2); + assert!( + crate::cmd_cache::cache_path(&rec.cache_id) + .unwrap() + .exists(), + "cache doc must exist for {}", + rec.cache_id + ); + + let (_, second) = sync_bundle(&bundle, &[Harness::Claude]).unwrap()[0]; + assert_eq!( + (second.new, second.updated, second.unchanged, second.failed), + (0, 0, 2, 0) + ); + }); + } + + #[test] + fn changed_session_is_rederived() { + with_cfg(|home| { + write_claude_session(home, "-test-project", "sess-aaa", "Add a feature"); + let bundle = claude_bundle(home); + sync_bundle(&bundle, &[Harness::Claude]).unwrap(); + + let cache_id = load_manifest().unwrap()["claude"]["sess-aaa"] + .cache_id + .clone(); + let steps_before = cached_step_count(&cache_id); + + // Session continues: a later user turn lands in the file. + let file = home.join(".claude/projects/-test-project/sess-aaa.jsonl"); + let mut body = std::fs::read_to_string(&file).unwrap(); + body.push_str( + r#"{"type":"user","uuid":"u-2","timestamp":"2024-01-02T00:05:00Z","cwd":"/test/project","message":{"role":"user","content":"And another thing"}}"#, + ); + body.push('\n'); + std::fs::write(&file, body).unwrap(); + + let (_, outcome) = sync_bundle(&bundle, &[Harness::Claude]).unwrap()[0]; + assert_eq!( + ( + outcome.new, + outcome.updated, + outcome.unchanged, + outcome.failed + ), + (0, 1, 0, 0) + ); + assert!( + cached_step_count(&cache_id) > steps_before, + "re-derived doc must contain the appended turn" + ); + }); + } + + #[test] + fn sync_touches_only_requested_types() { + with_cfg(|home| { + write_claude_session(home, "-test-project", "sess-aaa", "Add a feature"); + let bundle = claude_bundle(home); + + let outcomes = sync_bundle(&bundle, &[Harness::Codex]).unwrap(); + assert_eq!(outcomes[0].1, SyncOutcome::default()); + assert!( + load_manifest().unwrap().is_empty(), + "codex-only sync must not ingest claude sessions" + ); + }); + } + + #[test] + fn sync_overwrites_cache_entry_it_does_not_remember() { + with_cfg(|home| { + write_claude_session(home, "-test-project", "sess-aaa", "Add a feature"); + let bundle = claude_bundle(home); + sync_bundle(&bundle, &[Harness::Claude]).unwrap(); + + // Losing the manifest (or a prior manual `p import`) leaves a + // cache entry sync doesn't know about; re-syncing must + // overwrite it, not die on the exists-check. + std::fs::remove_file(manifest_path().unwrap()).unwrap(); + let (_, outcome) = sync_bundle(&bundle, &[Harness::Claude]).unwrap()[0]; + assert_eq!((outcome.new, outcome.failed), (1, 0)); + }); + } + + #[test] + fn failed_derivation_is_tallied_and_skipped() { + with_cfg(|home| { + write_claude_session(home, "-test-project", "sess-aaa", "Add a feature"); + let bundle = claude_bundle(home); + let cwd = std::env::current_dir().unwrap(); + let mut rows = gather_sessions(&bundle, &cwd, Some(Harness::Claude), None); + rows.push(SessionRow { + harness: Harness::Claude, + project: Some("/test/project".to_string()), + cwd: None, + session_id: "does-not-exist".to_string(), + title: String::new(), + last_activity: None, + message_count: 0, + matches_cwd: false, + }); + + let mut records = BTreeMap::new(); + let outcome = sync_rows(&bundle, &rows, &mut records).unwrap(); + assert_eq!((outcome.new, outcome.failed), (1, 1)); + assert!(records.contains_key("sess-aaa")); + assert!( + !records.contains_key("does-not-exist"), + "failed sessions must not be recorded as synced" + ); + }); + } + + #[test] + fn derive_row_errors_when_provider_missing() { + let bundle = HarnessBundle::default(); + let row = SessionRow { + harness: Harness::Claude, + project: Some("/test/project".to_string()), + cwd: None, + session_id: "sess".to_string(), + title: String::new(), + last_activity: None, + message_count: 0, + matches_cwd: false, + }; + let Err(err) = derive_row(&bundle, &row) else { + panic!("derive_row must fail without a claude manager"); + }; + assert!(err.to_string().contains("provider not available")); + } + + #[test] + fn resolve_types_defaults_to_all_and_dedups() { + assert_eq!(resolve_types(&[]), Harness::ALL.to_vec()); + assert_eq!( + resolve_types(&[HarnessArg::Codex, HarnessArg::Claude, HarnessArg::Codex]), + vec![Harness::Codex, Harness::Claude] + ); + } + + #[test] + fn render_summary_hides_empty_types_unless_explicit() { + let outcomes = vec![ + ( + Harness::Claude, + SyncOutcome { + new: 2, + updated: 1, + unchanged: 3, + failed: 0, + }, + ), + (Harness::Cursor, SyncOutcome::default()), + ]; + let default_run = render_summary(&outcomes, false); + assert!(default_run.contains("claude")); + assert!(!default_run.contains("cursor")); + + let explicit_run = render_summary(&outcomes, true); + assert!(explicit_run.contains("cursor")); + } + + #[test] + fn render_summary_shows_failures_and_empty_case() { + let outcomes = vec![( + Harness::Codex, + SyncOutcome { + new: 0, + updated: 0, + unchanged: 1, + failed: 2, + }, + )]; + let s = render_summary(&outcomes, false); + assert!(s.contains("2 failed")); + + assert_eq!(render_summary(&[], false), "nothing to sync\n"); + } +} diff --git a/crates/path-cli/tests/integration.rs b/crates/path-cli/tests/integration.rs index 9283acb9..97b5df85 100644 --- a/crates/path-cli/tests/integration.rs +++ b/crates/path-cli/tests/integration.rs @@ -542,6 +542,108 @@ fn cache_ls_after_import_lists_entry() { .stdout(predicate::str::contains("git-")); } +/// A `$HOME` with one Claude session. Returns (home-tempdir, session file). +fn claude_home_fixture() -> (tempfile::TempDir, PathBuf) { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + // toolpath-claude maps '/', '_', and '.' to '-' when sanitizing project + // paths into directory slugs — mirror that here so the fixture lands + // where the resolver looks for it. + let project_slug = project + .to_string_lossy() + .replace([std::path::MAIN_SEPARATOR, '_', '.'], "-"); + let project_dir = temp.path().join(".claude/projects").join(&project_slug); + std::fs::create_dir_all(&project_dir).unwrap(); + let session_file = project_dir.join("session-abc.jsonl"); + std::fs::write( + &session_file, + format!( + r#"{{"type":"user","uuid":"u-1","timestamp":"2024-01-01T00:00:00Z","cwd":"{cwd}","message":{{"role":"user","content":"hi"}}}} +{{"type":"assistant","uuid":"a-1","timestamp":"2024-01-01T00:00:01Z","message":{{"role":"assistant","content":"hello"}}}} +"#, + cwd = project.display() + ), + ) + .unwrap(); + (temp, session_file) +} + +#[test] +fn cache_sync_ingests_reskips_and_updates() { + let (home, session_file) = claude_home_fixture(); + let cfg = tempfile::tempdir().unwrap(); + let sync = || { + let mut c = cmd(); + c.env("HOME", home.path()) + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "cache", "sync", "claude"]); + c + }; + + // First run derives the session into the cache and records it. + sync() + .assert() + .success() + .stderr(predicate::str::contains("1 new, 0 updated, 0 unchanged")); + let manifest: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(cfg.path().join("sync.json")).unwrap()) + .unwrap(); + let record = &manifest["claude"]["session-abc"]; + let cache_id = record["cache_id"].as_str().unwrap(); + assert!( + cfg.path() + .join(format!("documents/{cache_id}.json")) + .exists() + ); + + // Nothing changed: the second run derives nothing. + sync() + .assert() + .success() + .stderr(predicate::str::contains("0 new, 0 updated, 1 unchanged")); + + // The session grows a turn; the third run re-derives it. + let mut body = std::fs::read_to_string(&session_file).unwrap(); + body.push_str( + r#"{"type":"user","uuid":"u-2","timestamp":"2024-01-01T00:05:00Z","cwd":"/x","message":{"role":"user","content":"more"}}"#, + ); + body.push('\n'); + std::fs::write(&session_file, body).unwrap(); + sync() + .assert() + .success() + .stderr(predicate::str::contains("0 new, 1 updated, 0 unchanged")); +} + +#[test] +fn cache_sync_default_run_with_no_sessions_reports_nothing() { + let home = tempfile::tempdir().unwrap(); + let cfg = tempfile::tempdir().unwrap(); + cmd() + .env("HOME", home.path()) + // opencode resolves through $XDG_DATA_HOME before $HOME — drop it + // so the sandboxed run can't see the developer's real database. + .env_remove("XDG_DATA_HOME") + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "cache", "sync"]) + .assert() + .success() + .stderr(predicate::str::contains("nothing to sync")); + assert!(!cfg.path().join("sync.json").exists()); +} + +#[test] +fn cache_sync_rejects_unknown_type() { + let cfg = tempfile::tempdir().unwrap(); + cmd() + .env("TOOLPATH_CONFIG_DIR", cfg.path()) + .args(["p", "cache", "sync", "frobnicate"]) + .assert() + .failure() + .stderr(predicate::str::contains("invalid value")); +} + #[test] fn export_pathbase_repo_flag_requires_login() { // `export pathbase` without --repo falls through to the anonymous diff --git a/crates/toolpath-cli/Cargo.toml b/crates/toolpath-cli/Cargo.toml index dd3ddc7f..54cf9fe6 100644 --- a/crates/toolpath-cli/Cargo.toml +++ b/crates/toolpath-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "toolpath-cli" -version = "0.15.0" +version = "0.16.0" edition = "2024" license = "Apache-2.0" repository = "https://github.com/empathic/toolpath" @@ -14,7 +14,7 @@ name = "path" path = "src/main.rs" [dependencies] -path-cli = { path = "../path-cli", version = "0.15.0" } +path-cli = { path = "../path-cli", version = "0.16.0" } anyhow = "1.0" [workspace] diff --git a/site/_data/crates.json b/site/_data/crates.json index 3996b79d..7e3ef5c3 100644 --- a/site/_data/crates.json +++ b/site/_data/crates.json @@ -105,7 +105,7 @@ }, { "name": "path-cli", - "version": "0.15.0", + "version": "0.16.0", "description": "Unified CLI (binary: path)", "docs": "https://docs.rs/path-cli", "crate": "https://crates.io/crates/path-cli", @@ -113,7 +113,7 @@ }, { "name": "toolpath-cli", - "version": "0.15.0", + "version": "0.16.0", "description": "Deprecated alias for path-cli", "docs": "https://docs.rs/toolpath-cli", "crate": "https://crates.io/crates/toolpath-cli",