Execute direct MCP steps in test pipeline

app/api/test-pipeline/route.ts

@@ -28,6 +28,13 @@ import { formatContextFromSources } from "@/lib/ai/rag"
 import { createMcpClient, checkMcpConnection } from "@/lib/mcp"
 import { agentMonitor } from "@/lib/agent-monitor"
 
+type PreparedPipelineStep = {
+  planStepNumber: number
+  action: string
+  params: Record<string, unknown>
+  description: string
+}
+
 // Define the tools available (same as BlenderAgent)
 const MCP_TOOLS = [
   "get_scene_info",
@@ -79,6 +86,24 @@ const MCP_TOOLS = [
   "render_viewport_to_path",
 ]
 
+const MCP_TOOL_SET = new Set<string>(MCP_TOOLS)
+
+function assertAllowedMcpAction(action: string, step: number) {
+  if (MCP_TOOL_SET.has(action)) return
+  throw new Error(`Planner returned unsupported MCP action "${action}" at step ${step}`)
+}
+
+function safeLogPreview(value: unknown, maxLength = 200) {
+  if (value === undefined) return "undefined"
+
+  try {
+    const json = JSON.stringify(value)
+    return (json ?? String(value)).slice(0, maxLength)
+  } catch {
+    return "[unserializable]"
+  }
+}
+
 export async function GET(req: NextRequest) {
   if (process.env.NODE_ENV === "production") {
     return NextResponse.json({ error: "Test endpoint disabled in production" }, { status: 403 })
@@ -241,41 +266,62 @@ async function runPipeline(prompt: string, skipExecution = false, providerOverri
       reasoning: planResult.reasoning?.slice(0, 500),
     }
 
-    // ── Step 3: Code Generation for each execute_code step ──
-    const codeSteps = planResult.plan.steps.filter(s => s.action === "execute_code")
-    const generatedCodes: { description: string; code: string; lines: number }[] = []
-
-    for (const [idx, codeStep] of codeSteps.entries()) {
-      const codeDescription = typeof codeStep.parameters?.description === "string"
-        ? codeStep.parameters.description
-        : typeof codeStep.parameters?.code === "string"
-          ? codeStep.parameters.code // already has inline code
-          : codeStep.rationale ?? codeStep.expected_outcome ?? prompt
-
-      // If the planner already provided code, use it directly
-      if (typeof codeStep.parameters?.code === "string") {
-        const code = codeStep.parameters.code as string
-        generatedCodes.push({ description: codeDescription, code, lines: code.split("\n").length })
-        console.log(`[TEST] Step ${idx + 1}/${codeSteps.length}: Using inline code (${code.split("\n").length} lines)`)
+    // ── Step 3: Prepare direct MCP steps and generate execute_code payloads ──
+    const preparedSteps: PreparedPipelineStep[] = []
+    const generatedCodes: { step: number; description: string; code: string; lines: number }[] = []
+
+    for (const [index, step] of planResult.plan.steps.entries()) {
+      const planStepNumber = index + 1
+      assertAllowedMcpAction(step.action, planStepNumber)
+
+      const codeDescription = typeof step.parameters?.description === "string"
+        ? step.parameters.description
+        : typeof step.parameters?.code === "string"
+          ? step.parameters.code
+          : step.rationale ?? step.expected_outcome ?? prompt
+
+      if (step.action === "execute_code") {
+        let code: string
+
+        // If the planner already provided code, use it directly.
+        if (typeof step.parameters?.code === "string") {
+          code = step.parameters.code as string
+          console.log(`[TEST] Step ${planStepNumber}: Using inline code (${code.split("\n").length} lines)`)
+        } else {
+          const codeStart = Date.now()
+          agentMonitor.log(sessionId, "codegen:start", { step: planStepNumber, descriptionChars: codeDescription.length })
+          code = await generateCode({
+            request: codeDescription,
+            context: ragContext,
+            applyMaterials: true,
+            namingPrefix: "ModelForge_",
+          })
+          const codeMs = Date.now() - codeStart
+          timings[`codegen_step${planStepNumber}_ms`] = codeMs
+          agentMonitor.log(sessionId, "codegen:complete", { step: planStepNumber, lines: code.split("\n").length, chars: code.length }, codeMs)
+        }
+
+        const lines = code.split("\n").length
+        generatedCodes.push({ step: planStepNumber, description: codeDescription, code, lines })
+        preparedSteps.push({
+          planStepNumber,
+          action: "execute_code",
+          params: { code },
+          description: codeDescription,
+        })
         continue
       }
 
-      const codeStart = Date.now()
-      agentMonitor.log(sessionId, "codegen:start", { step: idx + 1, descriptionChars: codeDescription.length })
-      const code = await generateCode({
-        request: codeDescription,
-        context: ragContext,
-        applyMaterials: true,
-        namingPrefix: "ModelForge_",
+      preparedSteps.push({
+        planStepNumber,
+        action: step.action,
+        params: step.parameters ?? {},
+        description: step.rationale ?? step.expected_outcome ?? step.action,
       })
-      const codeMs = Date.now() - codeStart
-      timings[`codegen_step${idx + 1}_ms`] = codeMs
-      generatedCodes.push({ description: codeDescription, code, lines: code.split("\n").length })
-      agentMonitor.log(sessionId, "codegen:complete", { step: idx + 1, lines: code.split("\n").length, chars: code.length }, codeMs)
     }
 
-    results.code = generatedCodes.map((gc, i) => ({
-      step: i + 1,
+    results.code = generatedCodes.map((gc) => ({
+      step: gc.step,
       description: gc.description.slice(0, 200),
       lines: gc.lines,
       length: gc.code.length,
@@ -293,37 +339,51 @@ async function runPipeline(prompt: string, skipExecution = false, providerOverri
     }
 
     // ── Step 4: Execute in Blender via MCP ──────────────────
-    const executionResults: { step: number; status: string; output?: unknown; error?: string; time_ms: number }[] = []
+    const executionResults: { step: number; action: string; status: string; output?: unknown; error?: string; time_ms: number }[] = []
 
-    if (!skipExecution && mcpConnected && generatedCodes.length > 0) {
-      console.log(`[TEST] 🚀 Executing ${generatedCodes.length} code block(s) in Blender...`)
+    if (!skipExecution && mcpConnected && preparedSteps.length > 0) {
+      console.log(`[TEST] 🚀 Executing ${preparedSteps.length} planned step(s) in Blender...`)
       const client = createMcpClient()
 
-      for (const [idx, gc] of generatedCodes.entries()) {
+      for (const executable of preparedSteps) {
         const execStart = Date.now()
         try {
-          agentMonitor.log(sessionId, "mcp:execute", { step: idx + 1, command: "execute_code", codeChars: gc.code.length })
+          const monitorPayload = executable.action === "execute_code"
+            ? {
+              step: executable.planStepNumber,
+              command: executable.action,
+              codeChars: typeof executable.params.code === "string" ? executable.params.code.length : 0,
+            }
+            : {
+              step: executable.planStepNumber,
+              command: executable.action,
+              params: executable.params,
+            }
+
+          agentMonitor.log(sessionId, "mcp:execute", monitorPayload)
           const execResult = await client.execute({
-            type: "execute_code",
-            params: { code: gc.code },
+            type: executable.action,
+            params: executable.params,
           })
           const execMs = Date.now() - execStart
-          timings[`exec_step${idx + 1}_ms`] = execMs
+          timings[`exec_step${executable.planStepNumber}_ms`] = execMs
 
           const stepResult = {
-            step: idx + 1,
+            step: executable.planStepNumber,
+            action: executable.action,
             status: execResult.status ?? "unknown",
             output: execResult.result ?? execResult.message,
             time_ms: execMs,
           }
           executionResults.push(stepResult)
-          agentMonitor.log(sessionId, "mcp:result", { step: idx + 1, status: stepResult.status, result: JSON.stringify(stepResult.output).slice(0, 200) }, execMs)
+          const outputPreview = safeLogPreview(stepResult.output)
+          agentMonitor.log(sessionId, "mcp:result", { step: executable.planStepNumber, command: executable.action, status: stepResult.status, result: outputPreview }, execMs)
         } catch (execErr) {
           const execMs = Date.now() - execStart
-          timings[`exec_step${idx + 1}_ms`] = execMs
+          timings[`exec_step${executable.planStepNumber}_ms`] = execMs
           const errMsg = execErr instanceof Error ? execErr.message : String(execErr)
-          executionResults.push({ step: idx + 1, status: "error", error: errMsg, time_ms: execMs })
-          agentMonitor.log(sessionId, "mcp:error", { step: idx + 1, error: errMsg }, execMs)
+          executionResults.push({ step: executable.planStepNumber, action: executable.action, status: "error", error: errMsg, time_ms: execMs })
+          agentMonitor.log(sessionId, "mcp:error", { step: executable.planStepNumber, command: executable.action, error: errMsg }, execMs)
         }
       }
 
@@ -342,6 +402,8 @@ async function runPipeline(prompt: string, skipExecution = false, providerOverri
       await client.close()
     } else if (!skipExecution && !mcpConnected) {
       console.log(`[TEST] ⏭ Skipped execution — Blender not connected`)
+    } else if (!skipExecution && preparedSteps.length === 0) {
+      console.log(`[TEST] ⏭ Skipped execution — no executable steps`)
     } else {
       console.log(`[TEST] ⏭ Skipped execution (skipExecution=true)`)
     }

scripts/test/test-test-pipeline-direct-step-execution.ts

@@ -0,0 +1,25 @@
+import assert from "node:assert/strict"
+import { readFileSync } from "node:fs"
+
+const source = readFileSync("app/api/test-pipeline/route.ts", "utf8")
+
+assert.match(source, /PreparedPipelineStep/)
+assert.match(source, /preparedSteps/)
+assert.match(source, /MCP_TOOL_SET/)
+assert.match(source, /assertAllowedMcpAction/)
+assert.match(source, /MCP_TOOL_SET\.has\(action\)/)
+assert.match(source, /safeLogPreview/)
+assert.match(source, /catch \{/)
+assert.match(source, /\[unserializable\]/)
+assert.match(source, /planResult\.plan\.steps\.entries\(\)/)
+assert.match(source, /if \(step\.action === "execute_code"\)/)
+assert.match(source, /generateCode\(/)
+assert.match(source, /client\.execute\(\{\s*type: executable\.action,\s*params: executable\.params,\s*\}/s)
+assert.match(source, /assertAllowedMcpAction\(step\.action, planStepNumber\)/)
+assert.match(source, /result: outputPreview/)
+
+assert.doesNotMatch(source, /if \(!skipExecution && mcpConnected && generatedCodes\.length > 0\)/)
+assert.doesNotMatch(source, /for \(const \[idx, gc\] of generatedCodes\.entries\(\)\)/)
+assert.doesNotMatch(source, /result: JSON\.stringify\(stepResult\.output\)\.slice\(0, 200\)/)
+
+console.log("Test pipeline direct step execution tests passed")