From 92753c7c0b7d33972f5902cf17fb38f0e20df135 Mon Sep 17 00:00:00 2001 From: Contentrain Date: Tue, 14 Jul 2026 15:53:58 +0300 Subject: [PATCH] fix(chat): stop silent output-token truncation from stranding agent writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two staging conversations froze on a "…saving now:" preamble whose save never ran. Root cause: the tool loop called the model with a hardcoded `max_tokens: 4096`. A large write tool call (a full ui-strings dictionary, or several articles across locales in one turn) exceeded 4096 output tokens, so generation was cut off mid tool_use — no `tool_use_end` fired, the incomplete call never landed in `currentToolCalls`, and the loop's generic "not tool_use → end turn" check closed the turn silently. The user saw a frozen preamble with no error and nothing saved. Three complementary fixes: - Raise the output-token ceiling via the model catalog. `maxOutputTokens` is now a `ChatModelEntry` field (16K for every chat model, safely under each model's documented limit) plus `maxOutputTokensFor()` with an 8K fallback for Conversation-API / legacy model IDs. It is a cap, not a target — billed on actual output — so this is cost-neutral for normal turns and only prevents realistic bulk writes from truncating. Threaded through `ConversationConfig` and both callers (Studio chat + ee Conversation API). - Handle `stop_reason: max_tokens` explicitly in the loop instead of falling through to the silent end-of-turn: keep the partial text for the transcript, emit a typed `output_truncated` error, and close the turn. The client localizes it via `chat.output_truncated` (falls back to the generic send error for other error events). - Add the `rules.batch_writes` system-prompt rule steering the agent to split large writes across smaller save_content calls (a few entries per call, one locale per call) so it stops emitting oversized single tool calls in the first place. Adds a regression test asserting a `max_tokens` stop yields an `output_truncated` error and still closes the turn with the partial text. --- .../content/system/agent-prompts/en.json | 1 + .../content/system/ui-strings/en.json | 1 + app/composables/useChat.ts | 10 ++++-- ee/enterprise/conversation-api.ts | 3 +- .../projects/[projectId]/chat.post.ts | 4 +-- server/utils/agent-system-prompt.ts | 3 ++ server/utils/conversation-engine.ts | 34 +++++++++++++++++-- shared/utils/ai-models.ts | 33 ++++++++++++++++++ .../conversation-engine-regression.test.ts | 33 ++++++++++++++++++ 9 files changed, 115 insertions(+), 7 deletions(-) diff --git a/.contentrain/content/system/agent-prompts/en.json b/.contentrain/content/system/agent-prompts/en.json index 360f61e1..ba28ea99 100644 --- a/.contentrain/content/system/agent-prompts/en.json +++ b/.contentrain/content/system/agent-prompts/en.json @@ -34,6 +34,7 @@ "role.definition": "You are a Contentrain content management executor. You perform structured content operations on this Git-backed repository using the tools provided.\n\nCONSTRAINTS:\n- Execute content tasks using tools. Never output raw JSON for users to copy.\n- Do NOT explain what Contentrain is or how it works.\n- Do NOT have general knowledge conversations.\n- Respond in the user's language. Be concise - 1-2 sentences for confirmations.\n- If a request is outside content management, respond with ONE sentence redirecting to content tasks.", "rules.auto_merge_admin": "After save_content/save_model, changes are auto-merged (you have admin/owner privileges). Report the result directly.", "rules.auto_merge_owner": "After save_content/save_model/init_project, changes are auto-merged. Report the result directly.", + "rules.batch_writes": "Keep each response's writes small enough to finish within the output-token limit — one save_content carrying a very large payload (many entries at once, or a full dictionary) can be cut off mid-call and silently fail. When many collection entries need writing, split them across several save_content calls of a few entries each; save_content merges, so entries and fields you don't resend are kept. Splitting must never shift ids: reuse the existing id on updates, and keep the SAME entry id across every locale of an i18n entry.", "rules.collection_id": "For NEW collection entries, generate entry IDs as 12-character lowercase hex strings (e.g., a1b2c3d4e5f6).", "rules.context_infer": "Use the inferred model/locale/entry from context unless user explicitly overrides.", "rules.context_no_ask": "Never ask questions you can infer from context.", diff --git a/.contentrain/content/system/ui-strings/en.json b/.contentrain/content/system/ui-strings/en.json index 9153f34a..20a0ce7a 100644 --- a/.contentrain/content/system/ui-strings/en.json +++ b/.contentrain/content/system/ui-strings/en.json @@ -193,6 +193,7 @@ "chat.load_error": "Failed to load conversation", "chat.new_conversation": "New conversation", "chat.no_conversations": "No previous conversations", + "chat.output_truncated": "The response hit the output limit before finishing, so the operation didn’t complete. Try splitting it into smaller steps — for example, one language or a few entries at a time.", "chat.placeholder": "Ask about your content...", "chat.send": "Send", "chat.send_error": "Failed to send message. Please try again.", diff --git a/app/composables/useChat.ts b/app/composables/useChat.ts index b8851e94..0e933860 100644 --- a/app/composables/useChat.ts +++ b/app/composables/useChat.ts @@ -434,8 +434,14 @@ export function useChat(options?: { case 'error': { const { t } = useContent() - // SSE error events may contain raw backend messages — use fallback - error.value = t('chat.send_error') + // Output-token truncation is a distinct, actionable failure: the + // turn was cut off before a pending save/create ran. Give it its + // own localized message instead of the generic send failure. Any + // partial text already streamed stays in the bubble. + error.value = event.code === 'output_truncated' + ? t('chat.output_truncated') + // Other SSE error events may carry raw backend messages — use fallback + : t('chat.send_error') break } } diff --git a/ee/enterprise/conversation-api.ts b/ee/enterprise/conversation-api.ts index aadbe6f8..24fb5f97 100644 --- a/ee/enterprise/conversation-api.ts +++ b/ee/enterprise/conversation-api.ts @@ -15,6 +15,7 @@ import { errorMessage } from '../../server/utils/content-strings' import { normalizeContentRoot } from '../../server/utils/content-paths' import { runConversationLoop } from '../../server/utils/conversation-engine' import { buildPromptMessages, selectHistoryBudget } from '../../server/utils/conversation-history' +import { maxOutputTokensFor } from '../../shared/utils/ai-models' import { validateConversationKey } from '../../server/utils/conversation-keys' import { saveApiChatResult } from '../../server/utils/db' import { getPlanLimit, getWorkspacePlan, hasFeature } from '../../server/utils/license' @@ -326,7 +327,7 @@ async function runConversationMessage( let iterations: Array<{ iteration: number, assistantBlocks: AIContentBlock[], toolResultBlocks: AIContentBlock[] }> = [] for await (const evt of runConversationLoop( - { model, apiKey, systemPrompt, messages, tools: aiTools }, + { model, apiKey, systemPrompt, messages, tools: aiTools, maxOutputTokens: maxOutputTokensFor(model) }, { engine: contentEngine, git, diff --git a/server/api/workspaces/[workspaceId]/projects/[projectId]/chat.post.ts b/server/api/workspaces/[workspaceId]/projects/[projectId]/chat.post.ts index 2366e4f4..82b6bd81 100644 --- a/server/api/workspaces/[workspaceId]/projects/[projectId]/chat.post.ts +++ b/server/api/workspaces/[workspaceId]/projects/[projectId]/chat.post.ts @@ -6,7 +6,7 @@ import { deriveProjectPhase } from '~~/server/utils/agent-state-machine' import { classifyIntent } from '~~/server/utils/agent-context' import { runConversationLoop } from '~~/server/utils/conversation-engine' import { buildPromptMessages, selectHistoryBudget } from '~~/server/utils/conversation-history' -import { chatModelIdsFor, DEFAULT_CHAT_MODEL } from '../../../../../../shared/utils/ai-models' +import { chatModelIdsFor, DEFAULT_CHAT_MODEL, maxOutputTokensFor } from '../../../../../../shared/utils/ai-models' import { validateAttachmentBlocks } from '../../../../../utils/attachment-ingest' import { resolveEnterpriseChatApiKey } from '../../../../../utils/enterprise' import { getEdition } from '../../../../../utils/license' @@ -258,7 +258,7 @@ export default defineEventHandler(async (event) => { try { for await (const evt of runConversationLoop( - { model, apiKey, systemPrompt, messages, tools: aiTools, abortSignal: abortController.signal }, + { model, apiKey, systemPrompt, messages, tools: aiTools, maxOutputTokens: maxOutputTokensFor(model), abortSignal: abortController.signal }, { engine: contentEngine, git, userEmail: session.user.email ?? '', userId: session.user.id, contentRoot, workflow, permissions, plan, projectId, workspaceId, uiContext, phase }, )) { // Stop processing if client disconnected diff --git a/server/utils/agent-system-prompt.ts b/server/utils/agent-system-prompt.ts index 626c6f9a..7d4d2881 100644 --- a/server/utils/agent-system-prompt.ts +++ b/server/utils/agent-system-prompt.ts @@ -527,6 +527,9 @@ function buildBaseRulesSection(config: ContentrainConfig | null, permissions: Ag agentPrompt('rules.update_existing_id'), agentPrompt('rules.update_merge'), + // Write sizing — keep tool calls under the output-token ceiling + agentPrompt('rules.batch_writes'), + // Relations agentPrompt('rules.relation_value'), agentPrompt('rules.polymorphic_relation'), diff --git a/server/utils/conversation-engine.ts b/server/utils/conversation-engine.ts index ba19e033..52baee0d 100644 --- a/server/utils/conversation-engine.ts +++ b/server/utils/conversation-engine.ts @@ -3,6 +3,7 @@ import type { AIMessage, AIContentBlock, AISystemBlock, AITool, AIUsage } from ' import type { ChatUIContext, AffectedResources, ProjectPhase } from '~~/server/utils/agent-types' import type { AgentPermissions } from '~~/server/utils/agent-permissions' import type { ExpandModelView } from '~~/server/utils/relation-expand' +import { DEFAULT_MAX_OUTPUT_TOKENS } from '../../shared/utils/ai-models' /** * Conversation Engine — reusable AI conversation loop with tool execution. @@ -68,6 +69,15 @@ export interface ConversationConfig { tools: AITool[] maxToolIterations?: number maxToolResultLength?: number + /** + * Output-token ceiling (`max_tokens`) per model turn. A CAP, not a + * target — billed on actual output, so a generous value is + * cost-neutral and only prevents large tool calls (a full dictionary, + * many entries) from being truncated mid-generation. Callers pass the + * model-aware value from `maxOutputTokensFor`; defaults to + * `DEFAULT_MAX_OUTPUT_TOKENS` when omitted. + */ + maxOutputTokens?: number abortSignal?: AbortSignal } @@ -117,6 +127,7 @@ export async function* runConversationLoop( ): AsyncGenerator { const maxIterations = config.maxToolIterations ?? DEFAULT_MAX_TOOL_ITERATIONS const maxResultLength = config.maxToolResultLength ?? DEFAULT_MAX_TOOL_RESULT_LENGTH + const maxOutputTokens = config.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS const aiProvider = useAIProvider() let totalInputTokens = 0 @@ -218,7 +229,7 @@ export async function* runConversationLoop( iteration++ - const turn = yield* runModelStream(4096) + const turn = yield* runModelStream(maxOutputTokens) totalInputTokens += turn.usage.inputTokens totalOutputTokens += turn.usage.outputTokens totalCacheCreationInputTokens += turn.usage.cacheCreationInputTokens @@ -226,6 +237,25 @@ export async function* runConversationLoop( lastStopReason = turn.stopReason lastAssistantContent = turn.assistantBlocks + // === OUTPUT-TOKEN TRUNCATION === + // The model hit the `max_tokens` ceiling mid-response. If it was + // emitting a tool call, that call was cut off before completion — + // no `tool_use_end` fires, so it never lands in `currentToolCalls` + // and never executes. Treating this as a normal end-of-turn (the + // generic check below) strands the user on a "…saving now:" + // preamble whose action silently never ran. Instead: keep the + // partial (text-only) content for the transcript, surface a + // typed error the client localizes, and end the turn honestly. + if (turn.stopReason === 'max_tokens') { + trace.push({ iteration, assistantBlocks: turn.assistantBlocks, toolResultBlocks: [] }) + yield { + type: 'error', + code: 'output_truncated', + message: 'The response was cut off at the output-token limit before it finished. Any pending save/create did not run. Split large operations into smaller steps (e.g. one locale or a few entries per call) and retry.', + } + break + } + if (turn.stopReason !== 'tool_use' || turn.currentToolCalls.length === 0) { // Final iteration — no tool execution this turn. Persist the // assistant blocks alone (no tool_result row will exist for @@ -291,7 +321,7 @@ export async function* runConversationLoop( // tools-disabled streaming call so the model summarizes what it did — // streamed live, and recorded as the final visible assistant turn. if (!config.abortSignal?.aborted && iteration >= maxIterations && lastStopReason === 'tool_use') { - const wrap = yield* runModelStream(4096, []) + const wrap = yield* runModelStream(maxOutputTokens, []) totalInputTokens += wrap.usage.inputTokens totalOutputTokens += wrap.usage.outputTokens totalCacheCreationInputTokens += wrap.usage.cacheCreationInputTokens diff --git a/shared/utils/ai-models.ts b/shared/utils/ai-models.ts index a66eb58e..0852e9a7 100644 --- a/shared/utils/ai-models.ts +++ b/shared/utils/ai-models.ts @@ -34,6 +34,19 @@ export interface ChatModelEntry { * and source in `server/utils/conversation-history.ts`. */ historyBudget: number + /** + * Output-token ceiling (`max_tokens`) sent to the provider for this + * model. This is a CAP, not a target — the model bills only for what + * it actually generates, so a generous ceiling is cost-neutral for + * normal turns and only matters when a single response (notably a + * large write tool call — a full dictionary, many entries) would + * otherwise be cut off mid-generation. Too small a value silently + * truncates the tool call and the operation never runs; too large a + * value risks a provider 400 for exceeding the model's own limit. + * Values here stay comfortably within every listed model's documented + * output limit (Sonnet/Haiku 64K, Opus 32K). + */ + maxOutputTokens: number /** Command palette icon class. */ paletteIcon: string /** Command palette search keywords (base set; palette adds generics). */ @@ -47,6 +60,7 @@ export const CHAT_MODELS: readonly ChatModelEntry[] = [ description: 'Fast & economic', tier: 'starter', historyBudget: 12_000, + maxOutputTokens: 16_000, paletteIcon: 'icon-[annon--lightning]', paletteKeywords: ['haiku', 'fast', 'economic'], }, @@ -56,6 +70,7 @@ export const CHAT_MODELS: readonly ChatModelEntry[] = [ description: 'Balanced', tier: 'pro', historyBudget: 48_000, + maxOutputTokens: 16_000, paletteIcon: 'icon-[annon--star]', paletteKeywords: ['sonnet', 'balanced'], }, @@ -69,6 +84,7 @@ export const CHAT_MODELS: readonly ChatModelEntry[] = [ description: 'Balanced, newest generation', tier: 'pro', historyBudget: 48_000, + maxOutputTokens: 16_000, paletteIcon: 'icon-[annon--star]', paletteKeywords: ['sonnet', 'balanced', 'newest', 'sonnet 5'], }, @@ -78,6 +94,7 @@ export const CHAT_MODELS: readonly ChatModelEntry[] = [ description: 'Most capable', tier: 'pro', historyBudget: 48_000, + maxOutputTokens: 16_000, paletteIcon: 'icon-[annon--trophy]', paletteKeywords: ['opus', 'capable', 'best'], }, @@ -96,3 +113,19 @@ export const DEFAULT_CHAT_MODEL = 'claude-sonnet-4-6' export function chatModelIdsFor(hasStudioKey: boolean): string[] { return CHAT_MODELS.filter(m => hasStudioKey || m.tier === 'starter').map(m => m.id) } + +/** + * Fallback output-token ceiling for model IDs not in the chat catalog + * (Conversation-API / legacy models). Safe for every current Claude + * model and still double the historical 4K default the tool loop used. + */ +export const DEFAULT_MAX_OUTPUT_TOKENS = 8192 + +/** + * Output-token ceiling (`max_tokens`) for a model. Catalog-driven for + * chat-picker models; `DEFAULT_MAX_OUTPUT_TOKENS` for anything else + * (legacy / Conversation-API model IDs). See `ChatModelEntry.maxOutputTokens`. + */ +export function maxOutputTokensFor(modelId: string): number { + return CHAT_MODELS.find(m => m.id === modelId)?.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS +} diff --git a/tests/unit/conversation-engine-regression.test.ts b/tests/unit/conversation-engine-regression.test.ts index c53a7862..6a047f8b 100644 --- a/tests/unit/conversation-engine-regression.test.ts +++ b/tests/unit/conversation-engine-regression.test.ts @@ -119,6 +119,39 @@ describe('conversation engine regression', () => { expect(messages).toHaveLength(1) }) + it('surfaces a typed error when the model is cut off at the output-token limit', async () => { + // Regression: a `max_tokens` stop mid-response (the model was + // emitting a tool call that never completed) used to fall through + // the generic "final iteration" check and end the turn silently, + // stranding the user on a "…saving now:" preamble whose save never + // ran. It must instead emit an `output_truncated` error and still + // close the turn, preserving whatever text streamed. + const { events } = await collectConversationEvents({ + aiProvider: { + streamCompletion: async function* () { + yield { type: 'text', content: 'Saving now:' } + yield { + type: 'message_end', + stopReason: 'max_tokens', + usage: { inputTokens: 100, outputTokens: 4096 }, + } + }, + createCompletion: vi.fn(), + }, + }) + + const errorEvent = events.find(e => e.type === 'error') + expect(errorEvent).toBeTruthy() + expect(errorEvent).toMatchObject({ type: 'error', code: 'output_truncated' }) + + // The turn still closes cleanly and keeps the partial text. + const done = events[events.length - 1]! + expect(done).toMatchObject({ + type: 'done', + lastContent: [{ type: 'text', text: 'Saving now:' }], + }) + }) + it('streams the post-tool continuation instead of buffering it', async () => { // Every iteration now streams — the first turn AND the post-tool // continuation. The follow-up answer must arrive as streamed text