From c6ba4cc3c1e607f8bd72ce30ea18d3f3267f8bf3 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Wed, 15 Apr 2026 17:54:09 +0200 Subject: [PATCH] Add compaction source pruning --- AGENTS.md | 31 ++++-- README.md | 11 ++- docs/trail-snippet.md | 27 +++++- package.json | 4 +- prpm.json | 2 +- src/cli/commands/compact.ts | 150 ++++++++++++++++++++++++++--- src/sdk/client.ts | 31 ++++-- tests/compact/llm-compact.test.ts | 65 +++++++++++++ tests/sdk/workflow-compact.test.ts | 17 ++++ 9 files changed, 299 insertions(+), 39 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6bba08b..251b7ac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ - + # Trail Record your work as a trajectory for future agents and humans to follow. @@ -82,6 +82,20 @@ When done, complete with a retrospective: trail complete --summary "Added JWT auth with refresh tokens" --confidence 0.85 ``` +After completing work, compact the finished trajectory or merged PR into a +durable summary. When the compacted summary is sufficient, discard the raw +source trajectories so `.trajectories/index.json` and list output stay focused: + +```bash +trail compact --discard-sources +# or after a PR merge: +trail compact --pr 42 --discard-sources +``` + +`--discard-sources` removes the source trajectory JSON/Markdown/trace files and +updates the index. Use it after confirming the compacted artifact is the record +you want to keep. + **Confidence levels:** - 0.9+ : High confidence, well-tested - 0.7-0.9 : Good confidence, standard implementation @@ -122,23 +136,26 @@ trail export --format markdown ## Compacting Trajectories -After a PR merge, compact related trajectories into a single summary: +After a PR merge, compact related trajectories into a single summary and prune +raw source trajectories when the summary should replace them: ```bash -trail compact --pr 42 +trail compact --pr 42 --discard-sources ``` Compact by branch: ```bash -trail compact --branch feature/auth +trail compact --branch feature/auth --discard-sources ``` Compact by commit range: ```bash -trail compact --commits abc123..def456 +trail compact --commits abc123..def456 --discard-sources ``` -Compaction consolidates decisions and creates a grouped summary, reducing noise while preserving key decisions. +Compaction consolidates decisions and creates a grouped summary. Adding +`--discard-sources` makes the compacted artifact the durable record by removing +the raw trajectories and their index entries. ## Why Trail? @@ -149,7 +166,7 @@ Your trajectory helps others understand: - **What challenges** you faced Future agents can query past trajectories to learn from your decisions. - + # Agent Relay diff --git a/README.md b/README.md index fc9c17a..4a2ab46 100644 --- a/README.md +++ b/README.md @@ -132,11 +132,14 @@ trail compact --commits abc1234,def5678 # Trajectories matching specific commit trail compact --pr 123 # Trajectories mentioning PR #123 trail compact --since 7d # Last 7 days trail compact --all # Everything (including previously compacted) +trail compact --pr 123 --discard-sources # Delete source trajectories and update index after compaction ``` ### Automatic Compaction (GitHub Action) -Add these steps to any workflow that runs on PR merge (e.g., your release or publish flow). Requires `ref: ${{ github.event.pull_request.base.ref }}` and `fetch-depth: 0` on checkout, plus `contents: write` permission: +Add these steps to any workflow that runs on PR merge (e.g., your release or publish flow). Requires `ref: ${{ github.event.pull_request.base.ref }}` and `fetch-depth: 0` on checkout, plus `contents: write` permission. + +Use `--discard-sources` when the compacted summary should replace the raw source trajectories. This removes the source JSON/Markdown/trace files and updates `.trajectories/index.json`, reducing future list/search noise. ```yaml - name: Compact trajectories @@ -144,13 +147,13 @@ Add these steps to any workflow that runs on PR merge (e.g., your release or pub PR_COMMITS=$(git log ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} --format=%H | paste -sd, -) OUTPUT=".trajectories/compacted/pr-${{ github.event.pull_request.number }}.json" if [ -n "$PR_COMMITS" ]; then - npx agent-trajectories compact --commits "$PR_COMMITS" --output "$OUTPUT" + npx agent-trajectories compact --commits "$PR_COMMITS" --output "$OUTPUT" --discard-sources else - npx agent-trajectories compact --pr ${{ github.event.pull_request.number }} --output "$OUTPUT" + npx agent-trajectories compact --pr ${{ github.event.pull_request.number }} --output "$OUTPUT" --discard-sources fi - name: Commit compacted trajectories run: | - git add .trajectories/compacted/ || true + git add .trajectories/ || true git diff --cached --quiet || \ (git commit -m "chore: compact trajectories for PR #${{ github.event.pull_request.number }}" && git push) ``` diff --git a/docs/trail-snippet.md b/docs/trail-snippet.md index 1f34d54..fa1b3d6 100644 --- a/docs/trail-snippet.md +++ b/docs/trail-snippet.md @@ -81,6 +81,20 @@ When done, complete with a retrospective: trail complete --summary "Added JWT auth with refresh tokens" --confidence 0.85 ``` +After completing work, compact the finished trajectory or merged PR into a +durable summary. When the compacted summary is sufficient, discard the raw +source trajectories so `.trajectories/index.json` and list output stay focused: + +```bash +trail compact --discard-sources +# or after a PR merge: +trail compact --pr 42 --discard-sources +``` + +`--discard-sources` removes the source trajectory JSON/Markdown/trace files and +updates the index. Use it after confirming the compacted artifact is the record +you want to keep. + **Confidence levels:** - 0.9+ : High confidence, well-tested - 0.7-0.9 : Good confidence, standard implementation @@ -121,23 +135,26 @@ trail export --format markdown ## Compacting Trajectories -After a PR merge, compact related trajectories into a single summary: +After a PR merge, compact related trajectories into a single summary and prune +raw source trajectories when the summary should replace them: ```bash -trail compact --pr 42 +trail compact --pr 42 --discard-sources ``` Compact by branch (finds trajectories with commits not in the specified base branch): ```bash -trail compact --branch main +trail compact --branch main --discard-sources ``` Compact by specific commits: ```bash -trail compact --commits abc123,def456 +trail compact --commits abc123,def456 --discard-sources ``` -Compaction consolidates decisions and creates a grouped summary, reducing noise while preserving key decisions. +Compaction consolidates decisions and creates a grouped summary. Adding +`--discard-sources` makes the compacted artifact the durable record by removing +the raw trajectories and their index entries. ## Why Trail? diff --git a/package.json b/package.json index 67a9f66..330aed1 100644 --- a/package.json +++ b/package.json @@ -48,9 +48,7 @@ "type": "git", "url": "https://github.com/AgentWorkforce/trajectories" }, - "files": [ - "dist" - ], + "files": ["dist"], "engines": { "node": ">=20.0.0" }, diff --git a/prpm.json b/prpm.json index ef227d9..d016616 100644 --- a/prpm.json +++ b/prpm.json @@ -9,7 +9,7 @@ "packages": [ { "name": "trail-snippet", - "version": "1.1.1", + "version": "1.1.2", "description": "AGENTS.md / CLAUDE.md snippet for agents on how to use trail to record their work", "format": "generic", "subtype": "snippet", diff --git a/src/cli/commands/compact.ts b/src/cli/commands/compact.ts index 3ec5f82..2717a1d 100644 --- a/src/cli/commands/compact.ts +++ b/src/cli/commands/compact.ts @@ -9,7 +9,13 @@ */ import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + readFileSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { dirname, join } from "node:path"; import type { Command } from "commander"; import { getCompactionConfig } from "../../compact/config.js"; @@ -75,6 +81,12 @@ interface IndexEntry { compactedInto?: string; } +interface TrajectoryIndex { + version: number; + lastUpdated: string; + trajectories: Record; +} + interface CompactCommandOptions { since?: string; until?: string; @@ -88,10 +100,18 @@ interface CompactCommandOptions { mechanical?: boolean; focus?: string; markdown?: boolean; + discardSources?: boolean; dryRun?: boolean; output?: string; } +interface DiscardSourcesSummary { + removedIndexEntries: number; + deletedJsonFiles: number; + deletedMarkdownFiles: number; + deletedTraceFiles: number; +} + interface LLMCompactionPlan { messages: Message[]; estimatedInputTokens: number; @@ -137,6 +157,10 @@ export function registerCompactCommand(program: Command): void { ) .option("--markdown", "Also write a Markdown companion file") .option("--no-markdown", "Skip writing a Markdown companion file") + .option( + "--discard-sources", + "After saving the compaction, delete source trajectory JSON/MD/trace files and remove their index entries", + ) .option("--dry-run", "Preview what would be compacted without saving") .option("--output ", "Output path for compacted trajectory") .action(async (options: CompactCommandOptions) => { @@ -193,7 +217,15 @@ export function registerCompactCommand(program: Command): void { outputPath, markdownEnabled, ); - await markTrajectoriesAsCompacted(trajectories, mechanicalCompacted.id); + if (options.discardSources) { + const discardSummary = discardSourceTrajectories(trajectories); + printDiscardSummary(discardSummary); + } else { + await markTrajectoriesAsCompacted( + trajectories, + mechanicalCompacted.id, + ); + } console.log(`\nCompacted trajectory saved to: ${outputPath}`); if (markdownEnabled) { @@ -252,7 +284,12 @@ export function registerCompactCommand(program: Command): void { const outputPath = options.output || getDefaultOutputPath(compacted, options.workflow); saveCompactionArtifacts(compacted, outputPath, markdownEnabled); - await markTrajectoriesAsCompacted(trajectories, compacted.id); + if (options.discardSources) { + const discardSummary = discardSourceTrajectories(trajectories); + printDiscardSummary(discardSummary); + } else { + await markTrajectoriesAsCompacted(trajectories, compacted.id); + } console.log(`\nCompacted trajectory saved to: ${outputPath}`); if (markdownEnabled) { @@ -441,9 +478,7 @@ function getCompactedTrajectoryIds(): Set { try { const indexContent = readFileSync(indexPath, "utf-8"); - const index = JSON.parse(indexContent) as { - trajectories: Record; - }; + const index = JSON.parse(indexContent) as TrajectoryIndex; for (const [id, entry] of Object.entries(index.trajectories || {})) { if (entry.compactedInto) { @@ -473,11 +508,7 @@ async function markTrajectoriesAsCompacted( try { const indexContent = readFileSync(indexPath, "utf-8"); - const index = JSON.parse(indexContent) as { - version: number; - lastUpdated: string; - trajectories: Record; - }; + const index = JSON.parse(indexContent) as TrajectoryIndex; let updated = false; for (const traj of trajectories) { @@ -497,6 +528,103 @@ async function markTrajectoriesAsCompacted( } } +/** + * Remove raw source trajectories after a durable compacted artifact has + * been written. This keeps compaction as the long-lived record and makes + * the index reflect only material that should remain visible. + */ +function discardSourceTrajectories( + trajectories: Trajectory[], +): DiscardSourcesSummary { + const sourceIds = new Set(trajectories.map((trajectory) => trajectory.id)); + const summary: DiscardSourcesSummary = { + removedIndexEntries: 0, + deletedJsonFiles: 0, + deletedMarkdownFiles: 0, + deletedTraceFiles: 0, + }; + + for (const searchPath of getSearchPaths()) { + const indexPath = join(searchPath, "index.json"); + if (!existsSync(indexPath)) continue; + + let index: TrajectoryIndex; + try { + const indexContent = readFileSync(indexPath, "utf-8"); + const parsedIndex = JSON.parse(indexContent) as unknown; + if (!isTrajectoryIndex(parsedIndex)) { + continue; + } + index = parsedIndex; + } catch { + // Keep behavior consistent with markTrajectoriesAsCompacted: malformed + // indexes are ignored instead of blocking an already-saved compaction. + continue; + } + + let updated = false; + for (const id of sourceIds) { + const entry = index.trajectories[id]; + if (!entry) continue; + + if (deleteFileIfExists(entry.path)) { + summary.deletedJsonFiles += 1; + } + if (deleteFileIfExists(getMarkdownOutputPath(entry.path))) { + summary.deletedMarkdownFiles += 1; + } + if (deleteFileIfExists(getTraceOutputPath(entry.path))) { + summary.deletedTraceFiles += 1; + } + + delete index.trajectories[id]; + summary.removedIndexEntries += 1; + updated = true; + } + + if (updated) { + index.lastUpdated = new Date().toISOString(); + writeFileSync(indexPath, JSON.stringify(index, null, 2)); + } + } + + return summary; +} + +function deleteFileIfExists(path: string): boolean { + if (!existsSync(path)) { + return false; + } + + unlinkSync(path); + return true; +} + +function isTrajectoryIndex(value: unknown): value is TrajectoryIndex { + if (value === null || typeof value !== "object") { + return false; + } + + const candidate = value as Partial; + return ( + candidate.trajectories !== null && + typeof candidate.trajectories === "object" && + !Array.isArray(candidate.trajectories) + ); +} + +function getTraceOutputPath(outputPath: string): string { + return outputPath.endsWith(".json") + ? outputPath.slice(0, -".json".length).concat(".trace.json") + : `${outputPath}.trace.json`; +} + +function printDiscardSummary(summary: DiscardSourcesSummary): void { + console.log( + `Discarded source trajectories: ${summary.removedIndexEntries} index entries, ${summary.deletedJsonFiles} JSON files, ${summary.deletedMarkdownFiles} Markdown files, ${summary.deletedTraceFiles} trace files`, + ); +} + function parseRelativeDate(input: string): Date { // Handle relative dates like "7d", "2w", "1m" const match = input.match(/^(\d+)([dwmh])$/); diff --git a/src/sdk/client.ts b/src/sdk/client.ts index 14b7be4..50db2e8 100644 --- a/src/sdk/client.ts +++ b/src/sdk/client.ts @@ -53,19 +53,22 @@ function normalizeOptionalString(value?: string): string | undefined { } function normalizeAutoCompactOptions( - autoCompact?: boolean | { mechanical?: boolean; markdown?: boolean }, -): false | { mechanical: boolean; markdown: boolean } { + autoCompact?: + | boolean + | { mechanical?: boolean; markdown?: boolean; discardSources?: boolean }, +): false | { mechanical: boolean; markdown: boolean; discardSources: boolean } { if (!autoCompact) { return false; } if (autoCompact === true) { - return { mechanical: false, markdown: true }; + return { mechanical: false, markdown: true, discardSources: false }; } return { mechanical: autoCompact.mechanical ?? false, markdown: autoCompact.markdown ?? true, + discardSources: autoCompact.discardSources ?? false, }; } @@ -137,7 +140,12 @@ function parseCompactWorkflowOutput(stdout: string): { export async function compactWorkflow( workflowId: string, - options?: { markdown?: boolean; mechanical?: boolean; cwd?: string }, + options?: { + markdown?: boolean; + mechanical?: boolean; + discardSources?: boolean; + cwd?: string; + }, ): Promise<{ compactedPath: string; markdownPath?: string }> { const normalizedWorkflowId = normalizeOptionalString(workflowId); if (!normalizedWorkflowId) { @@ -159,6 +167,9 @@ export async function compactWorkflow( if (options?.mechanical) { args.push("--mechanical"); } + if (options?.discardSources) { + args.push("--discard-sources"); + } return new Promise((resolve, reject) => { const child = spawn(cli.command, args, { @@ -221,9 +232,11 @@ export interface TrajectoryClientOptions { /** Whether to auto-save after each operation. Defaults to true */ autoSave?: boolean; /** - * When set, session.complete() and session.done() automatically run compactWorkflow() against the trajectory's workflowId. Default false. Pass an object to control the flags passed to the CLI — e.g. { mechanical: true } skips the LLM for deterministic compaction, { markdown: false } skips the .md companion. + * When set, session.complete() and session.done() automatically run compactWorkflow() against the trajectory's workflowId. Default false. Pass an object to control the flags passed to the CLI — e.g. { mechanical: true } skips the LLM for deterministic compaction, { markdown: false } skips the .md companion, { discardSources: true } prunes raw source trajectories after compaction. */ - autoCompact?: boolean | { mechanical?: boolean; markdown?: boolean }; + autoCompact?: + | boolean + | { mechanical?: boolean; markdown?: boolean; discardSources?: boolean }; } /** @@ -523,7 +536,7 @@ export class TrajectoryClient { private readonly autoCompactCwd?: string; private readonly autoCompact: | false - | { mechanical: boolean; markdown: boolean }; + | { mechanical: boolean; markdown: boolean; discardSources: boolean }; constructor(options: TrajectoryClientOptions = {}) { this.storage = options.storage ?? new FileStorage(options.dataDir); @@ -534,7 +547,9 @@ export class TrajectoryClient { this.autoCompactCwd = options.storage ? undefined : options.dataDir; } - getAutoCompactOptions(): false | { mechanical: boolean; markdown: boolean } { + getAutoCompactOptions(): + | false + | { mechanical: boolean; markdown: boolean; discardSources: boolean } { return this.autoCompact; } diff --git a/tests/compact/llm-compact.test.ts b/tests/compact/llm-compact.test.ts index f19bb11..1daff74 100644 --- a/tests/compact/llm-compact.test.ts +++ b/tests/compact/llm-compact.test.ts @@ -1,3 +1,4 @@ +import { existsSync } from "node:fs"; import { mkdtemp, readFile, readdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -296,6 +297,70 @@ describe("LLM compaction", () => { expect(compacted.filesAffected).toBeDefined(); }, ); + + it( + "can discard source trajectory files and index entries after compaction", + { timeout: 15_000 }, + async () => { + const started = await runCommand(["start", "Prune compacted sources"]); + expect(started.success).toBe(true); + + const decided = await runCommand([ + "decision", + "Discard raw trajectories after compaction", + "--reasoning", + "The compacted artifact becomes the durable record and list output stays focused.", + ]); + expect(decided.success).toBe(true); + + const completed = await runCommand([ + "complete", + "--summary", + "Finished source pruning flow", + "--confidence", + "0.9", + ]); + expect(completed.success).toBe(true); + + const indexPath = join(tempDir, ".trajectories", "index.json"); + const beforeIndex = JSON.parse(await readFile(indexPath, "utf-8")) as { + trajectories: Record; + }; + const sourceId = Object.keys(beforeIndex.trajectories)[0]; + expect(sourceId).toBeDefined(); + + const sourcePath = beforeIndex.trajectories[sourceId ?? ""]?.path; + expect(sourcePath).toBeDefined(); + expect(existsSync(sourcePath ?? "")).toBe(true); + expect(existsSync((sourcePath ?? "").replace(/\.json$/, ".md"))).toBe( + true, + ); + + const result = await runCommand([ + "compact", + "--mechanical", + "--discard-sources", + ]); + + expect(result.success).toBe(true); + expect(result.output).toContain("Compacted trajectory saved to:"); + expect(result.output).toContain("Discarded source trajectories:"); + + const compactedDir = join(tempDir, ".trajectories", "compacted"); + const compactedFiles = await readdir(compactedDir); + expect(compactedFiles.some((file) => file.endsWith(".json"))).toBe(true); + expect(compactedFiles.some((file) => file.endsWith(".md"))).toBe(true); + + const afterIndex = JSON.parse(await readFile(indexPath, "utf-8")) as { + trajectories: Record; + }; + expect(afterIndex.trajectories[sourceId ?? ""]).toBeUndefined(); + expect(existsSync(sourcePath ?? "")).toBe(false); + expect(existsSync((sourcePath ?? "").replace(/\.json$/, ".md"))).toBe( + false, + ); + }, + ); }); describe("CLI provider resolution", () => { diff --git a/tests/sdk/workflow-compact.test.ts b/tests/sdk/workflow-compact.test.ts index c9d8e78..a1359f5 100644 --- a/tests/sdk/workflow-compact.test.ts +++ b/tests/sdk/workflow-compact.test.ts @@ -274,13 +274,23 @@ describe("workflow compaction", () => { const session = await client.start("SDK helper task"); await session.decide("Which approach?", "Option A", "Cleaner abstraction"); await session.done("sdk helper done", 0.85); + const sessionId = session.id; clearEnv("TRAJECTORIES_WORKFLOW_ID"); await client.close(); + const indexPath = join(tempDir, ".trajectories", "index.json"); + const beforeIndex = JSON.parse(await readFile(indexPath, "utf-8")) as { + trajectories: Record; + }; + const sourcePath = beforeIndex.trajectories[sessionId]?.path; + expect(sourcePath).toBeDefined(); + expect(existsSync(sourcePath ?? "")).toBe(true); + const result = await compactWorkflow("wf-a", { mechanical: true, markdown: true, + discardSources: true, cwd: tempDir, }); @@ -296,6 +306,13 @@ describe("workflow compaction", () => { ) as { workflowId?: string; sourceTrajectories: string[] }; expect(compacted.workflowId).toBe("wf-a"); expect(compacted.sourceTrajectories).toHaveLength(1); + expect(compacted.sourceTrajectories).toContain(sessionId); + + const afterIndex = JSON.parse(await readFile(indexPath, "utf-8")) as { + trajectories: Record; + }; + expect(afterIndex.trajectories[sessionId]).toBeUndefined(); + expect(existsSync(sourcePath ?? "")).toBe(false); }, 60_000); it("does not drop trajectories that contain unknown event types", async () => {