From f643f862f91c8eb4d9575ca31c7690c09ffb1390 Mon Sep 17 00:00:00 2001 From: Gavin Williams Date: Thu, 30 Apr 2026 10:15:35 +0100 Subject: [PATCH 1/3] feat(web): multi-phase review agent with per-file parallel LLM calls Replace the per-chunk (N LLM calls) architecture with a two-phase approach: 1. A single MR summary pass over all changed files to detect cross-file semantic changes (renames, signature changes, removed exports, etc.) that individual file reviewers should be aware of. 2. Per-file LLM reviews that batch all hunks for a file into a single call, parallelised across files via a concurrency-capped pool (MAX_CONCURRENT_FILE_REVIEWS = 5). This reduces LLM calls from one-per-hunk to one-per-file (plus one summary call), while giving each file review the full picture via the MR summary context. Additional changes: - Export `validateLogPath` from `invokeDiffReviewLlm` for reuse in the summary node - Add "How it works" section to the review agent docs Co-Authored-By: Claude Sonnet 4.6 --- docs/docs/features/agents/review-agent.mdx | 16 ++- .../nodes/generateDiffReviewPrompt.ts | 28 +++-- .../review-agent/nodes/generateMrSummary.ts | 97 +++++++++++++++ .../review-agent/nodes/generatePrReview.ts | 117 +++++++++++++----- .../review-agent/nodes/gitlabMrParser.ts | 2 +- .../review-agent/nodes/invokeDiffReviewLlm.ts | 2 +- 6 files changed, 213 insertions(+), 49 deletions(-) create mode 100644 packages/web/src/features/agents/review-agent/nodes/generateMrSummary.ts diff --git a/docs/docs/features/agents/review-agent.mdx b/docs/docs/features/agents/review-agent.mdx index b4dc9c2f6..18b94a3ea 100644 --- a/docs/docs/features/agents/review-agent.mdx +++ b/docs/docs/features/agents/review-agent.mdx @@ -3,12 +3,24 @@ title: AI Code Review Agent sidebarTitle: AI code review agent --- -This agent provides codebase-aware reviews for your GitHub PRs and GitLab MRs. For each diff, the agent fetches relevant context from your indexed codebase and feeds it into a configured language model for a detailed review. +This agent provides codebase-aware reviews for your GitHub PRs and GitLab MRs. When a review is triggered, the agent runs a two-phase LLM pipeline and posts inline comments on the changed files. The AI Code Review Agent is [fair source](https://github.com/sourcebot-dev/sourcebot/tree/main/packages/web/src/features/agents/review-agent) and packaged in [Sourcebot](https://github.com/sourcebot-dev/sourcebot). To get started, [deploy Sourcebot](/docs/deployment/docker-compose) and follow the configuration instructions below. ![AI Code Review Agent Example](/images/review_agent_example.png) +# How it works + +When a review is triggered, the agent runs the following steps: + +1. **MR summary pass.** A single LLM call analyses the full set of changed files to identify cross-file semantic changes, such as renamed functions, changed signatures, removed exports, or behaviour changes with cross-file implications. This summary is passed as additional context into each per-file review. + +2. **Per-file reviews.** One LLM call is made per changed file. Each call receives the file's complete diff (all hunks combined), the full file content, the PR title and description, the MR summary from step 1, and any configured context files. Reviews run in parallel, up to five files at a time. + +3. **Inline comments.** The agent posts the results as inline review comments on the PR or MR. + +If the MR summary pass finds no cross-file concerns, it returns nothing and the per-file reviews proceed without it. + # Language model The review agent uses whichever language model you have configured in your `config.json`. All providers supported by Sourcebot (OpenAI, Anthropic, AWS Bedrock, Azure OpenAI, and others) work out of the box. @@ -135,7 +147,7 @@ If you have multiple models configured, set `REVIEW_AGENT_MODEL` to the `display By default, the agent does not review PRs and MRs automatically. To enable automatic reviews on every new or updated PR/MR, set `REVIEW_AGENT_AUTO_REVIEW_ENABLED` to `true`. -You can also trigger a review manually by commenting `/review` on any PR or MR. To use a different command, set `REVIEW_AGENT_REVIEW_COMMAND` to your preferred value (without the leading slash). +You can also trigger a review manually by commenting `review` on any PR or MR. To use a different command, set `REVIEW_AGENT_REVIEW_COMMAND` to your preferred value. # Environment variable reference diff --git a/packages/web/src/features/agents/review-agent/nodes/generateDiffReviewPrompt.ts b/packages/web/src/features/agents/review-agent/nodes/generateDiffReviewPrompt.ts index f5eda1ca0..cfd71f75b 100644 --- a/packages/web/src/features/agents/review-agent/nodes/generateDiffReviewPrompt.ts +++ b/packages/web/src/features/agents/review-agent/nodes/generateDiffReviewPrompt.ts @@ -4,29 +4,35 @@ import { createLogger } from "@sourcebot/shared"; const logger = createLogger('generate-diff-review-prompt'); -export const generateDiffReviewPrompt = async (diff: sourcebot_diff, context: sourcebot_context[], rules: string[]) => { +export const generateDiffReviewPrompt = async (diffs: sourcebot_diff[], context: sourcebot_context[], rules: string[]) => { logger.debug("Executing generate_diff_review_prompt"); - - const prompt = ` - You are an expert software engineer that excels at reviewing code changes. Given the input, additional context, and rules defined below, review the code changes and provide a detailed review. The review you provide - must conform to all of the rules defined below. The output format of your review must conform to the output format defined below. - - # Input - The input is the old and new code snippets, which represent a single hunk from a git diff. The old code snippet is the code before the changes were made, and the new code snippet is the code after the changes were made. Each code snippet - is a sequence of lines each with a line number. + const hunksText = diffs.map((diff, i) => ` + ## Hunk ${i + 1} - ## Old Code Snippet + ### Old Code \`\`\` ${diff.oldSnippet} \`\`\` - ## New Code Snippet + ### New Code \`\`\` ${diff.newSnippet} \`\`\` + `).join('\n'); + + const prompt = ` + You are an expert software engineer that excels at reviewing code changes. Given the input, additional context, and rules defined below, review the code changes and provide a detailed review. The review you provide + must conform to all of the rules defined below. The output format of your review must conform to the output format defined below. + + # Input + + The input is the old and new code snippets for one or more hunks from a git diff for a single file. The old code snippet is the code before the changes were made, and the new code snippet is the code after the changes were made. Each code snippet + is a sequence of lines each with a line number. + + ${hunksText} # Additional Context diff --git a/packages/web/src/features/agents/review-agent/nodes/generateMrSummary.ts b/packages/web/src/features/agents/review-agent/nodes/generateMrSummary.ts new file mode 100644 index 000000000..7a2eaee83 --- /dev/null +++ b/packages/web/src/features/agents/review-agent/nodes/generateMrSummary.ts @@ -0,0 +1,97 @@ +import { sourcebot_context, sourcebot_pr_payload } from "@/features/agents/review-agent/types"; +import { getAISDKLanguageModelAndOptions, getConfiguredLanguageModels } from "@/features/chat/utils.server"; +import { validateLogPath } from "@/features/agents/review-agent/nodes/invokeDiffReviewLlm"; +import { env } from "@sourcebot/shared"; +import { generateText } from "ai"; +import { createLogger } from "@sourcebot/shared"; +import fs from "fs"; + +const logger = createLogger('generate-mr-summary'); + +/** + * Makes a single LLM call over the entire MR diff to identify cross-file + * semantic changes (renames, signature changes, removed exports, etc.) that + * individual per-file reviewers should be aware of. Returns null when there + * are no notable cross-file concerns or if the call fails — the per-file + * review pipeline always continues regardless. + */ +export const generateMrSummary = async ( + pr_payload: sourcebot_pr_payload, + reviewAgentLogPath: string | undefined, +): Promise => { + logger.debug("Executing generate_mr_summary"); + + const models = await getConfiguredLanguageModels(); + if (models.length === 0) { + logger.warn("No language models configured, skipping MR summary"); + return null; + } + + let selectedModel = models[0]; + if (env.REVIEW_AGENT_MODEL) { + const match = models.find((m) => m.displayName === modelName); + if (match) { + selectedModel = match; + } else { + logger.warn(`REVIEW_AGENT_MODEL="${env.REVIEW_AGENT_MODEL}" did not match any configured model displayName. Falling back to the first configured model.`); + } + } + + const { model, providerOptions, temperature } = await getAISDKLanguageModelAndOptions(selectedModel); + + const diffSummary = pr_payload.file_diffs.map((fileDiff) => { + const header = fileDiff.from !== fileDiff.to + ? `File: ${fileDiff.to} (renamed from ${fileDiff.from})` + : `File: ${fileDiff.to}`; + const hunks = fileDiff.diffs.map((d, i) => + `Hunk ${i + 1}:\n--- Old\n${d.oldSnippet}\n+++ New\n${d.newSnippet}` + ).join('\n\n'); + return `${header}\n${hunks}`; + }).join('\n\n---\n\n'); + + const prompt = `You are reviewing a pull request titled "${pr_payload.title}". + +Below are all the changed files and their diffs. Identify and summarise semantic changes that reviewers of individual files should be aware of — such as renamed functions or types, changed signatures or interfaces, removed exports, or behaviour changes with cross-file implications. + +If there are no noteworthy cross-file semantic concerns, respond with an empty string. + +# Changed Files + +${diffSummary}`; + + if (reviewAgentLogPath) { + validateLogPath(reviewAgentLogPath); + fs.appendFileSync(reviewAgentLogPath, `\n\nMR Summary Prompt:\n${prompt}`); + } + + try { + const result = await generateText({ + model, + system: "You are a code review assistant. Provide a concise plain-text summary of cross-file semantic changes in a pull request. Respond with an empty string if there are none.", + prompt, + providerOptions, + temperature, + }); + + const summary = result.text.trim(); + + if (reviewAgentLogPath) { + validateLogPath(reviewAgentLogPath); + fs.appendFileSync(reviewAgentLogPath, `\n\nMR Summary Response:\n${summary}`); + } + if (!summary) { + logger.debug("No cross-file semantic changes detected, skipping summary context"); + return null; + } + + logger.debug("Completed generate_mr_summary"); + return { + type: "pr_summary", + description: "A summary of cross-file semantic changes in this pull request", + context: summary, + }; + } catch (error) { + logger.error("Error generating MR summary, proceeding without it:", error); + return null; + } +}; diff --git a/packages/web/src/features/agents/review-agent/nodes/generatePrReview.ts b/packages/web/src/features/agents/review-agent/nodes/generatePrReview.ts index 4732049a3..b35fa1d35 100644 --- a/packages/web/src/features/agents/review-agent/nodes/generatePrReview.ts +++ b/packages/web/src/features/agents/review-agent/nodes/generatePrReview.ts @@ -1,52 +1,101 @@ -import { sourcebot_pr_payload, sourcebot_diff_review, sourcebot_file_diff_review, sourcebot_context } from "@/features/agents/review-agent/types"; +import { sourcebot_pr_payload, sourcebot_file_diff_review, sourcebot_context } from "@/features/agents/review-agent/types"; import { generateDiffReviewPrompt } from "@/features/agents/review-agent/nodes/generateDiffReviewPrompt"; import { invokeDiffReviewLlm } from "@/features/agents/review-agent/nodes/invokeDiffReviewLlm"; import { fetchFileContent } from "@/features/agents/review-agent/nodes/fetchFileContent"; +import { generateMrSummary } from "@/features/agents/review-agent/nodes/generateMrSummary"; import { createLogger } from "@sourcebot/shared"; const logger = createLogger('generate-pr-review'); -export const generatePrReviews = async (reviewAgentLogFileName: string | undefined, pr_payload: sourcebot_pr_payload, rules: string[]): Promise => { - logger.debug("Executing generate_pr_reviews"); +const MAX_CONCURRENT_FILE_REVIEWS = 5; - const file_diff_reviews: sourcebot_file_diff_review[] = []; - for (const file_diff of pr_payload.file_diffs) { - const reviews: sourcebot_diff_review[] = []; +/** + * Runs tasks with a bounded concurrency limit, returning results in the same + * order as the input array and using the same PromiseSettledResult shape as + * Promise.allSettled. + */ +async function withConcurrencyLimit( + tasks: Array<() => Promise>, + limit: number, +): Promise[]> { + const results: PromiseSettledResult[] = new Array(tasks.length); + let nextIndex = 0; - for (const diff of file_diff.diffs) { + async function worker() { + while (nextIndex < tasks.length) { + const index = nextIndex++; try { - const fileContentContext = await fetchFileContent(pr_payload, file_diff.to); - const context: sourcebot_context[] = [ - { - type: "pr_title", - description: "The title of the pull request", - context: pr_payload.title, - }, - { - type: "pr_description", - description: "The description of the pull request", - context: pr_payload.description, - }, - fileContentContext, - ]; - - const prompt = await generateDiffReviewPrompt(diff, context, rules); - - const diffReview = await invokeDiffReviewLlm(reviewAgentLogFileName, prompt); - reviews.push(...diffReview.reviews); - } catch (error) { - logger.error(`Error generating review for ${file_diff.to}: ${error}`); + results[index] = { status: 'fulfilled', value: await tasks[index]() }; + } catch (reason) { + results[index] = { status: 'rejected', reason }; } } - - if (reviews.length > 0) { - file_diff_reviews.push({ + } + + await Promise.all(Array.from({ length: Math.min(limit, tasks.length) }, worker)); + return results; +} + +export const generatePrReviews = async (reviewAgentLogFileName: string | undefined, pr_payload: sourcebot_pr_payload, rules: string[]): Promise => { + logger.debug("Executing generate_pr_reviews"); + + // Run MR summary upfront to detect cross-file semantic changes. + const mrSummaryResult = await Promise.allSettled([ + generateMrSummary(pr_payload, reviewAgentLogFileName), + ]); + + const mrSummaryContext: sourcebot_context[] = []; + if (mrSummaryResult[0].status === 'fulfilled' && mrSummaryResult[0].value !== null) { + mrSummaryContext.push(mrSummaryResult[0].value); + } else if (mrSummaryResult[0].status === 'rejected') { + logger.warn(`MR summary generation failed: ${mrSummaryResult[0].reason}`); + } + + // Per-file review — one LLM call per file, parallelised with a concurrency cap. + logger.debug(`Reviewing ${pr_payload.file_diffs.length} file(s)`); + const fileResults = await withConcurrencyLimit( + pr_payload.file_diffs.map((file_diff) => async () => { + const fileContentContext = await fetchFileContent(pr_payload, file_diff.to); + const context: sourcebot_context[] = [ + { + type: "pr_title", + description: "The title of the pull request", + context: pr_payload.title, + }, + { + type: "pr_description", + description: "The description of the pull request", + context: pr_payload.description, + }, + fileContentContext, + ...mrSummaryContext, + ]; + + const prompt = await generateDiffReviewPrompt(file_diff.diffs, context, rules); + const diffReview = await invokeDiffReviewLlm(reviewAgentLogFileName, prompt); + + if (diffReview.reviews.length === 0) { + return null; + } + + return { filename: file_diff.to, - reviews: reviews, - }); + oldFilename: file_diff.from, + reviews: diffReview.reviews, + } satisfies sourcebot_file_diff_review; + }), + MAX_CONCURRENT_FILE_REVIEWS, + ); + + const file_diff_reviews: sourcebot_file_diff_review[] = []; + for (const result of fileResults) { + if (result.status === 'rejected') { + logger.error(`Error generating review: ${result.reason}`); + } else if (result.value !== null) { + file_diff_reviews.push(result.value); } } logger.debug("Completed generate_pr_reviews"); return file_diff_reviews; -} \ No newline at end of file +} diff --git a/packages/web/src/features/agents/review-agent/nodes/gitlabMrParser.ts b/packages/web/src/features/agents/review-agent/nodes/gitlabMrParser.ts index 61b068a1c..709fbfa27 100644 --- a/packages/web/src/features/agents/review-agent/nodes/gitlabMrParser.ts +++ b/packages/web/src/features/agents/review-agent/nodes/gitlabMrParser.ts @@ -88,7 +88,7 @@ export const gitlabMrParser = async ( (file): file is sourcebot_file_diff => file !== null, ); - logger.debug("Completed gitlab_mr_parser"); + logger.debug(`Completed gitlab_mr_parser: ${filteredSourcebotFileDiffs.length} file(s) parsed`); return { title: mr.title, description: mr.description ?? "", diff --git a/packages/web/src/features/agents/review-agent/nodes/invokeDiffReviewLlm.ts b/packages/web/src/features/agents/review-agent/nodes/invokeDiffReviewLlm.ts index e43f137fe..baba99eb6 100644 --- a/packages/web/src/features/agents/review-agent/nodes/invokeDiffReviewLlm.ts +++ b/packages/web/src/features/agents/review-agent/nodes/invokeDiffReviewLlm.ts @@ -12,7 +12,7 @@ export const getReviewAgentLogDir = (): string => { return path.join(env.DATA_CACHE_DIR, 'review-agent'); }; -const validateLogPath = (logPath: string): void => { +export const validateLogPath = (logPath: string): void => { const resolved = path.resolve(logPath); const logDir = getReviewAgentLogDir(); if (!resolved.startsWith(logDir + path.sep)) { From 98b128a836fad770412e4c2e85471ad79db41853 Mon Sep 17 00:00:00 2001 From: Gavin Williams Date: Thu, 30 Apr 2026 10:45:48 +0100 Subject: [PATCH 2/3] Add changelog entry --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3dfd5050..4d5650726 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- [Experimental] Improved AI code review agent efficiency: replaced per-hunk LLM calls with a two-phase pipeline — a single MR summary pass followed by one parallelised LLM call per file (capped at 5 concurrent), reducing total LLM calls on large PRs. [#1164](https://github.com/sourcebot-dev/sourcebot/pull/1164) + ## [4.17.0] - 2026-04-30 ### Added From 08481cd0b8e75007e0bbda00c9cfbbd829b357fe Mon Sep 17 00:00:00 2001 From: Gavin Williams Date: Thu, 30 Apr 2026 10:47:29 +0100 Subject: [PATCH 3/3] Address review comments --- docs/docs/features/agents/review-agent.mdx | 8 ++++---- .../agents/review-agent/nodes/generateMrSummary.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/docs/features/agents/review-agent.mdx b/docs/docs/features/agents/review-agent.mdx index 18b94a3ea..c5ba12761 100644 --- a/docs/docs/features/agents/review-agent.mdx +++ b/docs/docs/features/agents/review-agent.mdx @@ -3,7 +3,7 @@ title: AI Code Review Agent sidebarTitle: AI code review agent --- -This agent provides codebase-aware reviews for your GitHub PRs and GitLab MRs. When a review is triggered, the agent runs a two-phase LLM pipeline and posts inline comments on the changed files. +The AI Code Review Agent gives you codebase-aware reviews for your GitHub PRs and GitLab MRs. When you trigger a review, it runs a two-phase LLM pipeline and posts inline comments on the changed files. The AI Code Review Agent is [fair source](https://github.com/sourcebot-dev/sourcebot/tree/main/packages/web/src/features/agents/review-agent) and packaged in [Sourcebot](https://github.com/sourcebot-dev/sourcebot). To get started, [deploy Sourcebot](/docs/deployment/docker-compose) and follow the configuration instructions below. @@ -11,15 +11,15 @@ The AI Code Review Agent is [fair source](https://github.com/sourcebot-dev/sourc # How it works -When a review is triggered, the agent runs the following steps: +When you trigger a review, the agent runs the following steps: 1. **MR summary pass.** A single LLM call analyses the full set of changed files to identify cross-file semantic changes, such as renamed functions, changed signatures, removed exports, or behaviour changes with cross-file implications. This summary is passed as additional context into each per-file review. 2. **Per-file reviews.** One LLM call is made per changed file. Each call receives the file's complete diff (all hunks combined), the full file content, the PR title and description, the MR summary from step 1, and any configured context files. Reviews run in parallel, up to five files at a time. -3. **Inline comments.** The agent posts the results as inline review comments on the PR or MR. +3. **Inline comments.** Results are posted as inline review comments on your PR or MR. -If the MR summary pass finds no cross-file concerns, it returns nothing and the per-file reviews proceed without it. +If the MR summary pass finds no cross-file concerns, it is skipped and the per-file reviews proceed without it. # Language model diff --git a/packages/web/src/features/agents/review-agent/nodes/generateMrSummary.ts b/packages/web/src/features/agents/review-agent/nodes/generateMrSummary.ts index 7a2eaee83..7b606ac7c 100644 --- a/packages/web/src/features/agents/review-agent/nodes/generateMrSummary.ts +++ b/packages/web/src/features/agents/review-agent/nodes/generateMrSummary.ts @@ -29,7 +29,7 @@ export const generateMrSummary = async ( let selectedModel = models[0]; if (env.REVIEW_AGENT_MODEL) { - const match = models.find((m) => m.displayName === modelName); + const match = models.find((m) => m.displayName === env.REVIEW_AGENT_MODEL); if (match) { selectedModel = match; } else {