diff --git a/app/api/test-pipeline/route.ts b/app/api/test-pipeline/route.ts index e2d91e9..14c2b97 100644 --- a/app/api/test-pipeline/route.ts +++ b/app/api/test-pipeline/route.ts @@ -28,6 +28,13 @@ import { formatContextFromSources } from "@/lib/ai/rag" import { createMcpClient, checkMcpConnection } from "@/lib/mcp" import { agentMonitor } from "@/lib/agent-monitor" +type PreparedPipelineStep = { + planStepNumber: number + action: string + params: Record + description: string +} + // Define the tools available (same as BlenderAgent) const MCP_TOOLS = [ "get_scene_info", @@ -79,6 +86,24 @@ const MCP_TOOLS = [ "render_viewport_to_path", ] +const MCP_TOOL_SET = new Set(MCP_TOOLS) + +function assertAllowedMcpAction(action: string, step: number) { + if (MCP_TOOL_SET.has(action)) return + throw new Error(`Planner returned unsupported MCP action "${action}" at step ${step}`) +} + +function safeLogPreview(value: unknown, maxLength = 200) { + if (value === undefined) return "undefined" + + try { + const json = JSON.stringify(value) + return (json ?? String(value)).slice(0, maxLength) + } catch { + return "[unserializable]" + } +} + export async function GET(req: NextRequest) { if (process.env.NODE_ENV === "production") { return NextResponse.json({ error: "Test endpoint disabled in production" }, { status: 403 }) @@ -241,41 +266,62 @@ async function runPipeline(prompt: string, skipExecution = false, providerOverri reasoning: planResult.reasoning?.slice(0, 500), } - // ── Step 3: Code Generation for each execute_code step ── - const codeSteps = planResult.plan.steps.filter(s => s.action === "execute_code") - const generatedCodes: { description: string; code: string; lines: number }[] = [] - - for (const [idx, codeStep] of codeSteps.entries()) { - const codeDescription = typeof codeStep.parameters?.description === "string" - ? codeStep.parameters.description - : typeof codeStep.parameters?.code === "string" - ? codeStep.parameters.code // already has inline code - : codeStep.rationale ?? codeStep.expected_outcome ?? prompt - - // If the planner already provided code, use it directly - if (typeof codeStep.parameters?.code === "string") { - const code = codeStep.parameters.code as string - generatedCodes.push({ description: codeDescription, code, lines: code.split("\n").length }) - console.log(`[TEST] Step ${idx + 1}/${codeSteps.length}: Using inline code (${code.split("\n").length} lines)`) + // ── Step 3: Prepare direct MCP steps and generate execute_code payloads ── + const preparedSteps: PreparedPipelineStep[] = [] + const generatedCodes: { step: number; description: string; code: string; lines: number }[] = [] + + for (const [index, step] of planResult.plan.steps.entries()) { + const planStepNumber = index + 1 + assertAllowedMcpAction(step.action, planStepNumber) + + const codeDescription = typeof step.parameters?.description === "string" + ? step.parameters.description + : typeof step.parameters?.code === "string" + ? step.parameters.code + : step.rationale ?? step.expected_outcome ?? prompt + + if (step.action === "execute_code") { + let code: string + + // If the planner already provided code, use it directly. + if (typeof step.parameters?.code === "string") { + code = step.parameters.code as string + console.log(`[TEST] Step ${planStepNumber}: Using inline code (${code.split("\n").length} lines)`) + } else { + const codeStart = Date.now() + agentMonitor.log(sessionId, "codegen:start", { step: planStepNumber, descriptionChars: codeDescription.length }) + code = await generateCode({ + request: codeDescription, + context: ragContext, + applyMaterials: true, + namingPrefix: "ModelForge_", + }) + const codeMs = Date.now() - codeStart + timings[`codegen_step${planStepNumber}_ms`] = codeMs + agentMonitor.log(sessionId, "codegen:complete", { step: planStepNumber, lines: code.split("\n").length, chars: code.length }, codeMs) + } + + const lines = code.split("\n").length + generatedCodes.push({ step: planStepNumber, description: codeDescription, code, lines }) + preparedSteps.push({ + planStepNumber, + action: "execute_code", + params: { code }, + description: codeDescription, + }) continue } - const codeStart = Date.now() - agentMonitor.log(sessionId, "codegen:start", { step: idx + 1, descriptionChars: codeDescription.length }) - const code = await generateCode({ - request: codeDescription, - context: ragContext, - applyMaterials: true, - namingPrefix: "ModelForge_", + preparedSteps.push({ + planStepNumber, + action: step.action, + params: step.parameters ?? {}, + description: step.rationale ?? step.expected_outcome ?? step.action, }) - const codeMs = Date.now() - codeStart - timings[`codegen_step${idx + 1}_ms`] = codeMs - generatedCodes.push({ description: codeDescription, code, lines: code.split("\n").length }) - agentMonitor.log(sessionId, "codegen:complete", { step: idx + 1, lines: code.split("\n").length, chars: code.length }, codeMs) } - results.code = generatedCodes.map((gc, i) => ({ - step: i + 1, + results.code = generatedCodes.map((gc) => ({ + step: gc.step, description: gc.description.slice(0, 200), lines: gc.lines, length: gc.code.length, @@ -293,37 +339,51 @@ async function runPipeline(prompt: string, skipExecution = false, providerOverri } // ── Step 4: Execute in Blender via MCP ────────────────── - const executionResults: { step: number; status: string; output?: unknown; error?: string; time_ms: number }[] = [] + const executionResults: { step: number; action: string; status: string; output?: unknown; error?: string; time_ms: number }[] = [] - if (!skipExecution && mcpConnected && generatedCodes.length > 0) { - console.log(`[TEST] 🚀 Executing ${generatedCodes.length} code block(s) in Blender...`) + if (!skipExecution && mcpConnected && preparedSteps.length > 0) { + console.log(`[TEST] 🚀 Executing ${preparedSteps.length} planned step(s) in Blender...`) const client = createMcpClient() - for (const [idx, gc] of generatedCodes.entries()) { + for (const executable of preparedSteps) { const execStart = Date.now() try { - agentMonitor.log(sessionId, "mcp:execute", { step: idx + 1, command: "execute_code", codeChars: gc.code.length }) + const monitorPayload = executable.action === "execute_code" + ? { + step: executable.planStepNumber, + command: executable.action, + codeChars: typeof executable.params.code === "string" ? executable.params.code.length : 0, + } + : { + step: executable.planStepNumber, + command: executable.action, + params: executable.params, + } + + agentMonitor.log(sessionId, "mcp:execute", monitorPayload) const execResult = await client.execute({ - type: "execute_code", - params: { code: gc.code }, + type: executable.action, + params: executable.params, }) const execMs = Date.now() - execStart - timings[`exec_step${idx + 1}_ms`] = execMs + timings[`exec_step${executable.planStepNumber}_ms`] = execMs const stepResult = { - step: idx + 1, + step: executable.planStepNumber, + action: executable.action, status: execResult.status ?? "unknown", output: execResult.result ?? execResult.message, time_ms: execMs, } executionResults.push(stepResult) - agentMonitor.log(sessionId, "mcp:result", { step: idx + 1, status: stepResult.status, result: JSON.stringify(stepResult.output).slice(0, 200) }, execMs) + const outputPreview = safeLogPreview(stepResult.output) + agentMonitor.log(sessionId, "mcp:result", { step: executable.planStepNumber, command: executable.action, status: stepResult.status, result: outputPreview }, execMs) } catch (execErr) { const execMs = Date.now() - execStart - timings[`exec_step${idx + 1}_ms`] = execMs + timings[`exec_step${executable.planStepNumber}_ms`] = execMs const errMsg = execErr instanceof Error ? execErr.message : String(execErr) - executionResults.push({ step: idx + 1, status: "error", error: errMsg, time_ms: execMs }) - agentMonitor.log(sessionId, "mcp:error", { step: idx + 1, error: errMsg }, execMs) + executionResults.push({ step: executable.planStepNumber, action: executable.action, status: "error", error: errMsg, time_ms: execMs }) + agentMonitor.log(sessionId, "mcp:error", { step: executable.planStepNumber, command: executable.action, error: errMsg }, execMs) } } @@ -342,6 +402,8 @@ async function runPipeline(prompt: string, skipExecution = false, providerOverri await client.close() } else if (!skipExecution && !mcpConnected) { console.log(`[TEST] ⏭ Skipped execution — Blender not connected`) + } else if (!skipExecution && preparedSteps.length === 0) { + console.log(`[TEST] ⏭ Skipped execution — no executable steps`) } else { console.log(`[TEST] ⏭ Skipped execution (skipExecution=true)`) } diff --git a/scripts/test/test-test-pipeline-direct-step-execution.ts b/scripts/test/test-test-pipeline-direct-step-execution.ts new file mode 100644 index 0000000..a134ccb --- /dev/null +++ b/scripts/test/test-test-pipeline-direct-step-execution.ts @@ -0,0 +1,25 @@ +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" + +const source = readFileSync("app/api/test-pipeline/route.ts", "utf8") + +assert.match(source, /PreparedPipelineStep/) +assert.match(source, /preparedSteps/) +assert.match(source, /MCP_TOOL_SET/) +assert.match(source, /assertAllowedMcpAction/) +assert.match(source, /MCP_TOOL_SET\.has\(action\)/) +assert.match(source, /safeLogPreview/) +assert.match(source, /catch \{/) +assert.match(source, /\[unserializable\]/) +assert.match(source, /planResult\.plan\.steps\.entries\(\)/) +assert.match(source, /if \(step\.action === "execute_code"\)/) +assert.match(source, /generateCode\(/) +assert.match(source, /client\.execute\(\{\s*type: executable\.action,\s*params: executable\.params,\s*\}/s) +assert.match(source, /assertAllowedMcpAction\(step\.action, planStepNumber\)/) +assert.match(source, /result: outputPreview/) + +assert.doesNotMatch(source, /if \(!skipExecution && mcpConnected && generatedCodes\.length > 0\)/) +assert.doesNotMatch(source, /for \(const \[idx, gc\] of generatedCodes\.entries\(\)\)/) +assert.doesNotMatch(source, /result: JSON\.stringify\(stepResult\.output\)\.slice\(0, 200\)/) + +console.log("Test pipeline direct step execution tests passed")