From 20f5662b1fabadb759f831d7b4ce15eb87133878 Mon Sep 17 00:00:00 2001 From: stainlu Date: Tue, 12 May 2026 21:56:20 +0800 Subject: [PATCH] perf: budget review prompt file patches --- .github/workflows/ci.yml | 1 + CHANGELOG.md | 3 + src/clawsweeper.ts | 121 ++++++++++++++++++++++++++++++++++----- test/clawsweeper.test.ts | 47 +++++++++++++++ 4 files changed, 157 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad426234c0..b9fe6e68f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,7 @@ jobs: sparse-checkout: | .github config + dashboard docs instructions prompts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ae1e764e3..bb7e22ec31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,9 @@ checkpoint, and status-only commits are intentionally omitted. instead of reserving an entire 35-70 shard lane for every planning or publishing background run, so saturated backlog runs keep using available Codex capacity. +- Bounded the total pull request patch text kept in review prompts while + preserving hydrated file metadata, so large OpenClaw PR reviews spend fewer + tokens on repeated diff hunks. - Reserved pending/planning background sweep matrices at their quiet lane size and capped broad manual `shard_count` inputs by live scheduler allowance, so overlapping manual or scheduled review runs stay inside the Codex worker diff --git a/src/clawsweeper.ts b/src/clawsweeper.ts index 1b5c812500..e0b6ee13c3 100644 --- a/src/clawsweeper.ts +++ b/src/clawsweeper.ts @@ -322,6 +322,7 @@ interface ItemContext { pullFiles?: number; pullFilesHydrated?: number; pullFilesTruncated?: boolean; + pullFilePatchesTruncated?: number; pullCommits?: number; pullCommitsHydrated?: number; pullCommitsTruncated?: boolean; @@ -1619,20 +1620,35 @@ export function compactMappedSlice( return compactMappedWindow(items, items.length, limit, mapper); } -export function compactMappedWindow( +interface OmittedPromptEntries { + omitted: number; + note: string; +} + +function omittedPromptEntries(omitted: number): OmittedPromptEntries { + return { omitted, note: "middle entries omitted from prompt context" }; +} + +function isOmittedPromptEntries(entry: T | OmittedPromptEntries): entry is OmittedPromptEntries { + return ( + typeof entry === "object" && + entry !== null && + "omitted" in entry && + "note" in entry && + typeof (entry as { omitted?: unknown }).omitted === "number" + ); +} + +function compactWindowEntries( items: readonly T[], total: number, limit: number, - mapper: (item: T) => unknown, -): unknown[] { +): Array { const boundedLimit = Math.max(0, Math.floor(limit)); const boundedTotal = Math.max(0, Math.floor(total)); - if (boundedTotal <= boundedLimit && items.length <= boundedLimit) return items.map(mapper); - if (boundedLimit === 0) { - return boundedTotal > 0 - ? [{ omitted: boundedTotal, note: "middle entries omitted from prompt context" }] - : []; - } + if (boundedTotal <= boundedLimit && items.length <= boundedLimit) return [...items]; + if (boundedLimit === 0) return boundedTotal > 0 ? [omittedPromptEntries(boundedTotal)] : []; + const keepStart = Math.floor(boundedLimit / 2); const keepEnd = Math.max(0, boundedLimit - keepStart); const retained = @@ -1644,12 +1660,23 @@ export function compactMappedWindow( keepEnd > 0 ? retained.slice(Math.max(keepStart, retained.length - keepEnd)) : []; const omitted = Math.max(0, boundedTotal - retainedStart.length - retainedEnd.length); return [ - ...retainedStart.map(mapper), - ...(omitted > 0 ? [{ omitted, note: "middle entries omitted from prompt context" }] : []), - ...retainedEnd.map(mapper), + ...retainedStart, + ...(omitted > 0 ? [omittedPromptEntries(omitted)] : []), + ...retainedEnd, ]; } +export function compactMappedWindow( + items: readonly T[], + total: number, + limit: number, + mapper: (item: T) => unknown, +): unknown[] { + return compactWindowEntries(items, total, limit).map((entry) => + isOmittedPromptEntries(entry) ? entry : mapper(entry), + ); +} + function compactIssue(value: unknown): unknown { const issue = asRecord(value); return { @@ -2141,7 +2168,66 @@ export function sameAuthorCounterpartApplyReason( return null; } -function compactPullFile(value: unknown): unknown { +const PULL_FILE_PATCH_PROMPT_BUDGET_CHARS = 48_000; +const PULL_FILE_PATCH_PROMPT_MAX_CHARS = 2_000; + +interface CompactPullFilesForPromptOptions { + patchBudgetChars?: number; + maxPatchChars?: number; +} + +interface CompactPullFilesForPromptResult { + files: unknown[]; + patchesTruncated: number; +} + +export function compactPullFilesForPrompt( + files: readonly unknown[], + total: number, + limit: number, + options: CompactPullFilesForPromptOptions = {}, +): CompactPullFilesForPromptResult { + const windowEntries = compactWindowEntries(files, total, limit); + const retainedFiles = windowEntries.filter( + (entry): entry is unknown => !isOmittedPromptEntries(entry), + ); + let remainingPatchFiles = retainedFiles.filter((file) => { + const patch = asRecord(file).patch; + return typeof patch === "string" && patch.length > 0; + }).length; + let remainingPatchBudget = Math.max( + 0, + Math.floor(options.patchBudgetChars ?? PULL_FILE_PATCH_PROMPT_BUDGET_CHARS), + ); + const maxPatchChars = Math.max( + 0, + Math.floor(options.maxPatchChars ?? PULL_FILE_PATCH_PROMPT_MAX_CHARS), + ); + let patchesTruncated = 0; + const nextPatchLimit = (file: unknown): number => { + const patch = asRecord(file).patch; + if (typeof patch !== "string" || patch.length === 0) return maxPatchChars; + const fairLimit = + remainingPatchFiles > 0 ? Math.ceil(remainingPatchBudget / remainingPatchFiles) : 0; + const patchLimit = Math.max(0, Math.min(maxPatchChars, fairLimit)); + remainingPatchBudget = Math.max(0, remainingPatchBudget - Math.min(patch.length, patchLimit)); + remainingPatchFiles = Math.max(0, remainingPatchFiles - 1); + if (patch.length > patchLimit) patchesTruncated += 1; + return patchLimit; + }; + + return { + files: windowEntries.map((entry) => + isOmittedPromptEntries(entry) ? entry : compactPullFile(entry, nextPatchLimit(entry)), + ), + patchesTruncated, + }; +} + +function compactPullFile( + value: unknown, + patchMaxChars = PULL_FILE_PATCH_PROMPT_MAX_CHARS, +): unknown { const file = asRecord(value); return { filename: file.filename, @@ -2149,7 +2235,7 @@ function compactPullFile(value: unknown): unknown { additions: file.additions, deletions: file.deletions, changes: file.changes, - patch: truncateText(file.patch, 2000), + patch: truncateText(file.patch, patchMaxChars), }; } @@ -3439,6 +3525,7 @@ function collectItemContext(item: Item): ItemContext { 80, ); const pullFiles = pullFilesWindow.items; + const compactPullFiles = compactPullFilesForPrompt(pullFiles, pullFilesWindow.total, 80); const pullCommitsWindow = ghPagedContextWindow( `repos/${targetRepo()}/pulls/${item.number}/commits`, pullRecord.commits, @@ -3452,7 +3539,7 @@ function collectItemContext(item: Item): ItemContext { ); pullReviewComments = pullReviewCommentsWindow.items; context.pullRequest = compactPullRequest(pullRequest); - context.pullFiles = compactMappedWindow(pullFiles, pullFilesWindow.total, 80, compactPullFile); + context.pullFiles = compactPullFiles.files; context.pullCommits = compactMappedWindow( pullCommits, pullCommitsWindow.total, @@ -3474,6 +3561,7 @@ function collectItemContext(item: Item): ItemContext { pullFiles: pullFilesWindow.total, pullFilesHydrated: pullFilesWindow.hydrated, pullFilesTruncated: pullFilesWindow.truncated, + pullFilePatchesTruncated: compactPullFiles.patchesTruncated, pullCommits: pullCommitsWindow.total, pullCommitsHydrated: pullCommitsWindow.hydrated, pullCommitsTruncated: pullCommitsWindow.truncated, @@ -3505,6 +3593,8 @@ function collectItemContext(item: Item): ItemContext { counts.pullFilesHydrated = context.counts.pullFilesHydrated; if (context.counts?.pullFilesTruncated !== undefined) counts.pullFilesTruncated = context.counts.pullFilesTruncated; + if (context.counts?.pullFilePatchesTruncated !== undefined) + counts.pullFilePatchesTruncated = context.counts.pullFilePatchesTruncated; if (context.counts?.pullCommits !== undefined) counts.pullCommits = context.counts.pullCommits; if (context.counts?.pullCommitsHydrated !== undefined) counts.pullCommitsHydrated = context.counts.pullCommitsHydrated; @@ -6529,6 +6619,7 @@ ${options.action.closeComment ? options.action.closeComment : "_No close comment options.context.counts?.pullFilesHydrated, options.context.counts?.pullFilesTruncated, )} +- PR file patches truncated: ${options.context.counts?.pullFilePatchesTruncated ?? 0} - PR commits: ${contextCountText( options.context.counts?.pullCommits, options.context.pullCommits?.length ?? 0, diff --git a/test/clawsweeper.test.ts b/test/clawsweeper.test.ts index 0d665804bc..135a7241ea 100644 --- a/test/clawsweeper.test.ts +++ b/test/clawsweeper.test.ts @@ -18,6 +18,7 @@ import { closingPullRequestReferenceTarget, compactMappedSlice, compactMappedWindow, + compactPullFilesForPrompt, codexEnv, dashboardClosedAt, fixedPullRequestFromCommitPullsForTest, @@ -294,6 +295,52 @@ test("compactMappedWindow keeps bounded hydrated context when total is larger th assert.deepEqual(mapped, [1, 2, 99, 100]); }); +test("compactPullFilesForPrompt budgets patch text across retained files", () => { + const files = Array.from({ length: 4 }, (_, index) => ({ + filename: `src/file-${index}.ts`, + status: "modified", + additions: 100, + deletions: 0, + changes: 100, + patch: `${index}`.repeat(1000), + })); + + const result = compactPullFilesForPrompt(files, 4, 4, { + patchBudgetChars: 1000, + maxPatchChars: 1000, + }); + + assert.equal(result.patchesTruncated, 4); + assert.equal(result.files.length, 4); + for (const file of result.files) { + const patch = String((file as { patch: unknown }).patch); + assert.equal(patch.split("\n\n[truncated")[0].length, 250); + assert.match(patch, /\[truncated 750 chars\]/); + } +}); + +test("compactPullFilesForPrompt preserves file window omission markers", () => { + const files = Array.from({ length: 4 }, (_, index) => ({ + filename: `src/file-${index}.ts`, + status: "modified", + additions: 1, + deletions: 0, + changes: 1, + patch: `diff-${index}`, + })); + + const result = compactPullFilesForPrompt(files, 10, 4, { + patchBudgetChars: 1000, + maxPatchChars: 1000, + }); + + assert.deepEqual(result.files[2], { + omitted: 6, + note: "middle entries omitted from prompt context", + }); + assert.equal(result.patchesTruncated, 0); +}); + test("githubContextWindowPlan includes prior page when the tail crosses a page boundary", () => { assert.deepEqual(githubContextWindowPlan(101, 80), { keepStart: 40,