Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions app/api/test-pipeline/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,34 @@ const MCP_TOOLS = [
"focus_viewport_on_objects",
"select_scene_objects",
"set_active_collection",
"list_materials",
"delete_object",
"set_object_transform",
"rename_object",
"duplicate_object",
"join_objects",
"create_mesh_from_data",
"validate_mesh_geometry",
"inspect_retopology_readiness",
"normalize_vertex_group_weights",
"add_modifier",
"configure_modifier",
"configure_constraint",
"add_object_constraint",
"remove_object_constraint",
"apply_modifier",
"apply_transforms",
"shade_smooth",
"parent_set",
"parent_clear",
"set_origin",
"organize_collection_hierarchy",
"move_to_collection",
"set_visibility",
"export_object",
"list_installed_addons",
"create_material",
"assign_material",
"get_local_asset_library_status",
"search_local_assets",
"import_local_asset",
Expand All @@ -84,6 +104,12 @@ const MCP_TOOLS = [
"validate_studio_scene",
"render_thumbnail_to_path",
"render_viewport_to_path",
"add_light",
"set_light_properties",
"add_camera",
"set_camera_properties",
"set_render_settings",
"render_image",
]

const MCP_TOOL_SET = new Set<string>(MCP_TOOLS)
Expand Down
25 changes: 24 additions & 1 deletion lib/ai/agents.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1844,6 +1844,16 @@ const importLocalAsset = tool(
}
)

const getPolyhavenStatus = tool(
async () => executeMcpCommand("get_polyhaven_status"),
{
name: "get_polyhaven_status",
description:
"Check whether PolyHaven integration is configured and available before searching or downloading HDRIs, textures, or models.",
schema: z.object({}),
}
)

const getPolyhavenCategories = tool(
async ({ asset_type }: { asset_type: string }) =>
executeMcpCommand("get_polyhaven_categories", { asset_type }),
Expand Down Expand Up @@ -1903,6 +1913,16 @@ const setTexture = tool(

// ---------- Sketchfab Tools ---------

const getSketchfabStatus = tool(
async () => executeMcpCommand("get_sketchfab_status"),
{
name: "get_sketchfab_status",
description:
"Check whether Sketchfab integration is configured and available before searching or downloading Sketchfab models.",
schema: z.object({}),
}
)

const searchSketchfabModels = tool(
async ({ query, downloadable }: { query: string; downloadable?: boolean }) =>
executeMcpCommand("search_sketchfab_models", { query, downloadable }),
Expand Down Expand Up @@ -1984,8 +2004,9 @@ const importGeneratedAsset = tool(
// Tool Sets (filtered by config)
// ============================================================================

const SKETCHFAB_TOOL_NAMES = new Set(["search_sketchfab_models", "download_sketchfab_model"])
const SKETCHFAB_TOOL_NAMES = new Set(["get_sketchfab_status", "search_sketchfab_models", "download_sketchfab_model"])
const POLYHAVEN_TOOL_NAMES = new Set([
"get_polyhaven_status",
"get_polyhaven_categories",
"search_polyhaven_assets",
"download_polyhaven_asset",
Expand Down Expand Up @@ -2064,10 +2085,12 @@ const ALL_TOOLS = [
getLocalAssetLibraryStatus,
searchLocalAssets,
importLocalAsset,
getPolyhavenStatus,
getPolyhavenCategories,
searchPolyhavenAssets,
downloadPolyhavenAsset,
setTexture,
getSketchfabStatus,
searchSketchfabModels,
downloadSketchfabModel,
getHyper3dStatus,
Expand Down
10 changes: 5 additions & 5 deletions lib/orchestration/tool-filter.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import { TOOL_REGISTRY } from "./tool-registry"

const CATEGORY_GROUPS: Record<string, string[]> = {
inspection: ["get_scene_info", "get_object_info", "get_all_object_info", "inspect_blend_file_health", "inspect_modifier_constraint_stack", "inspect_rigging_data", "inspect_weight_paint_readiness", "inspect_animation_data", "inspect_collection_hierarchy", "inspect_viewport_areas", "set_viewport_shading", "focus_viewport_on_objects", "select_scene_objects", "set_active_collection", "get_viewport_screenshot", "render_viewport_to_path"],
geometry: ["create_mesh_from_data", "validate_mesh_geometry", "inspect_retopology_readiness", "inspect_modifier_constraint_stack", "inspect_rigging_data", "inspect_weight_paint_readiness", "normalize_vertex_group_weights", "configure_modifier", "configure_constraint", "add_object_constraint", "remove_object_constraint", "execute_code"],
inspection: ["get_scene_info", "get_object_info", "get_all_object_info", "inspect_blend_file_health", "inspect_modifier_constraint_stack", "inspect_rigging_data", "inspect_weight_paint_readiness", "inspect_animation_data", "inspect_collection_hierarchy", "inspect_viewport_areas", "set_viewport_shading", "focus_viewport_on_objects", "select_scene_objects", "set_active_collection", "get_viewport_screenshot", "render_viewport_to_path", "list_materials", "list_installed_addons"],
geometry: ["create_mesh_from_data", "validate_mesh_geometry", "inspect_retopology_readiness", "inspect_modifier_constraint_stack", "inspect_rigging_data", "inspect_weight_paint_readiness", "normalize_vertex_group_weights", "delete_object", "set_object_transform", "rename_object", "duplicate_object", "join_objects", "add_modifier", "configure_modifier", "configure_constraint", "add_object_constraint", "remove_object_constraint", "apply_modifier", "apply_transforms", "shade_smooth", "set_origin", "execute_code"],
animation: ["inspect_animation_data", "execute_code"],
organization: ["inspect_collection_hierarchy", "organize_collection_hierarchy", "select_scene_objects", "set_active_collection", "parent_set", "parent_clear", "move_to_collection", "set_visibility"],
materials: ["create_material_preset", "inspect_material_node_graph", "set_texture"],
materials: ["list_materials", "create_material", "assign_material", "create_material_preset", "inspect_material_node_graph", "set_texture"],
pipeline: ["prepare_uv_layout", "validate_export_readiness", "export_object"],
lighting: ["setup_studio_scene", "validate_studio_scene", "render_thumbnail_to_path", "add_light", "set_light_properties", "render_image"],
camera: ["inspect_viewport_areas", "focus_viewport_on_objects", "get_viewport_screenshot", "render_viewport_to_path", "setup_studio_scene", "validate_studio_scene", "render_thumbnail_to_path", "add_camera", "set_camera_properties", "render_image"],
lighting: ["setup_studio_scene", "validate_studio_scene", "render_thumbnail_to_path", "add_light", "set_light_properties", "set_render_settings", "render_image"],
camera: ["inspect_viewport_areas", "focus_viewport_on_objects", "get_viewport_screenshot", "render_viewport_to_path", "setup_studio_scene", "validate_studio_scene", "render_thumbnail_to_path", "add_camera", "set_camera_properties", "set_render_settings", "render_image"],
viewport: ["inspect_viewport_areas", "set_viewport_shading", "focus_viewport_on_objects", "select_scene_objects", "set_active_collection", "get_viewport_screenshot", "render_viewport_to_path"],
assets: [
"get_local_asset_library_status",
Expand Down
181 changes: 181 additions & 0 deletions lib/orchestration/tool-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,139 @@ export const TOOL_REGISTRY: ToolMetadata[] = [
category: "inspection",
parameters: "collection_name: string, create_new?: boolean",
},
{
name: "list_materials",
description:
"List existing Blender materials before reusing, assigning, or replacing material slots.",
category: "materials",
parameters: "(no parameters)",
},
{
name: "delete_object",
description:
"Delete a named object from the scene without generated Python. Use after inspecting scene state and confirming the exact object name.",
category: "geometry",
parameters: "name: string",
},
{
name: "set_object_transform",
description:
"Set object location, rotation, or scale directly with structured parameters instead of Python snippets.",
category: "geometry",
parameters: "name: string, location?: number[3], rotation?: number[3] degrees, scale?: number[3]",
},
{
name: "rename_object",
description:
"Rename an object with a deterministic direct tool so later tool calls can reference stable names.",
category: "geometry",
parameters: "name: string, new_name: string",
},
{
name: "duplicate_object",
description:
"Duplicate an existing object and optionally place it, preserving the source object as-is.",
category: "geometry",
parameters: "name: string, new_name?: string, linked?: boolean",
},
{
name: "join_objects",
description:
"Join multiple mesh objects into one named object when the scene needs a single combined mesh.",
category: "geometry",
parameters: "names: string[], new_name?: string",
},
{
name: "add_modifier",
description:
"Add a Blender modifier to an object using structured parameters before configuring or applying it.",
category: "geometry",
parameters: "name: string, modifier_type: string, modifier_name?: string",
},
{
name: "apply_modifier",
description:
"Apply a named modifier after stack inspection and validation.",
category: "geometry",
parameters: "name: string, modifier: string",
},
{
name: "apply_transforms",
description:
"Apply object location, rotation, or scale transforms with explicit booleans.",
category: "geometry",
parameters: "name: string, location?: boolean, rotation?: boolean, scale?: boolean",
},
{
name: "shade_smooth",
description:
"Set smooth or flat shading for mesh objects without Python.",
category: "geometry",
parameters: "name: string, smooth?: boolean",
},
{
name: "parent_set",
description:
"Parent one or more child objects to a parent while preserving the intended scene hierarchy.",
category: "geometry",
parameters: "child_name: string, parent_name: string, parent_type?: OBJECT|ARMATURE|BONE",
},
{
name: "parent_clear",
description:
"Clear object parenting with optional transform preservation.",
category: "geometry",
parameters: "name: string, keep_transform?: boolean",
},
{
name: "set_origin",
description:
"Set object origin using a bounded direct tool instead of context-sensitive Python operators.",
category: "geometry",
parameters: "name: string, origin_type?: ORIGIN_GEOMETRY|ORIGIN_CURSOR|GEOMETRY_ORIGIN|ORIGIN_CENTER_OF_VOLUME, center?: MEDIAN|BOUNDS",
},
{
name: "move_to_collection",
description:
"Move objects into an existing or newly created collection for deterministic scene organization.",
category: "geometry",
parameters: "name: string, collection_name: string, create_new?: boolean",
},
Comment on lines +109 to +178
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

Fix TOOL_REGISTRY parameter contract drift for direct tools.

Several metadata signatures don’t match the actual tool wrappers, which can cause planner-generated invalid params at runtime (e.g., duplicate_object, join_objects, parent_set, parent_clear, move_to_collection, set_origin enum wording).

Proposed metadata fixes
-    parameters: "name: string, new_name?: string, location?: number[3]",
+    parameters: "name: string, new_name?: string, linked?: boolean",

-    parameters: "names: string[], new_name?: string",
+    parameters: "names: string[]",

-    parameters: "children: string[], parent: string, keep_transform?: boolean",
+    parameters: "child_name: string, parent_name: string, parent_type?: string",

-    parameters: "names: string[], keep_transform?: boolean",
+    parameters: "name: string, keep_transform?: boolean",

-    parameters: "name: string, origin_type?: GEOMETRY_ORIGIN|ORIGIN_GEOMETRY|ORIGIN_CURSOR|ORIGIN_CENTER_OF_MASS, center?: MEDIAN|BOUNDS",
+    parameters: "name: string, origin_type?: GEOMETRY_ORIGIN|ORIGIN_GEOMETRY|ORIGIN_CURSOR|ORIGIN_CENTER_OF_VOLUME, center?: MEDIAN|BOUNDS",

-    parameters: "names: string[], collection_name: string, create_new?: boolean",
+    parameters: "name: string, collection_name: string, create_new?: boolean",
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
{
name: "duplicate_object",
description:
"Duplicate an existing object and optionally place it, preserving the source object as-is.",
category: "geometry",
parameters: "name: string, new_name?: string, location?: number[3]",
},
{
name: "join_objects",
description:
"Join multiple mesh objects into one named object when the scene needs a single combined mesh.",
category: "geometry",
parameters: "names: string[], new_name?: string",
},
{
name: "add_modifier",
description:
"Add a Blender modifier to an object using structured parameters before configuring or applying it.",
category: "geometry",
parameters: "name: string, modifier_type: string, modifier_name?: string",
},
{
name: "apply_modifier",
description:
"Apply a named modifier after stack inspection and validation.",
category: "geometry",
parameters: "name: string, modifier: string",
},
{
name: "apply_transforms",
description:
"Apply object location, rotation, or scale transforms with explicit booleans.",
category: "geometry",
parameters: "name: string, location?: boolean, rotation?: boolean, scale?: boolean",
},
{
name: "shade_smooth",
description:
"Set smooth or flat shading for mesh objects without Python.",
category: "geometry",
parameters: "name: string, smooth?: boolean",
},
{
name: "parent_set",
description:
"Parent one or more child objects to a parent while preserving the intended scene hierarchy.",
category: "geometry",
parameters: "children: string[], parent: string, keep_transform?: boolean",
},
{
name: "parent_clear",
description:
"Clear object parenting with optional transform preservation.",
category: "geometry",
parameters: "names: string[], keep_transform?: boolean",
},
{
name: "set_origin",
description:
"Set object origin using a bounded direct tool instead of context-sensitive Python operators.",
category: "geometry",
parameters: "name: string, origin_type?: GEOMETRY_ORIGIN|ORIGIN_GEOMETRY|ORIGIN_CURSOR|ORIGIN_CENTER_OF_MASS, center?: MEDIAN|BOUNDS",
},
{
name: "move_to_collection",
description:
"Move objects into an existing or newly created collection for deterministic scene organization.",
category: "geometry",
parameters: "names: string[], collection_name: string, create_new?: boolean",
},
{
name: "duplicate_object",
description:
"Duplicate an existing object and optionally place it, preserving the source object as-is.",
category: "geometry",
parameters: "name: string, new_name?: string, linked?: boolean",
},
{
name: "join_objects",
description:
"Join multiple mesh objects into one named object when the scene needs a single combined mesh.",
category: "geometry",
parameters: "names: string[]",
},
{
name: "add_modifier",
description:
"Add a Blender modifier to an object using structured parameters before configuring or applying it.",
category: "geometry",
parameters: "name: string, modifier_type: string, modifier_name?: string",
},
{
name: "apply_modifier",
description:
"Apply a named modifier after stack inspection and validation.",
category: "geometry",
parameters: "name: string, modifier: string",
},
{
name: "apply_transforms",
description:
"Apply object location, rotation, or scale transforms with explicit booleans.",
category: "geometry",
parameters: "name: string, location?: boolean, rotation?: boolean, scale?: boolean",
},
{
name: "shade_smooth",
description:
"Set smooth or flat shading for mesh objects without Python.",
category: "geometry",
parameters: "name: string, smooth?: boolean",
},
{
name: "parent_set",
description:
"Parent one or more child objects to a parent while preserving the intended scene hierarchy.",
category: "geometry",
parameters: "child_name: string, parent_name: string, parent_type?: string",
},
{
name: "parent_clear",
description:
"Clear object parenting with optional transform preservation.",
category: "geometry",
parameters: "name: string, keep_transform?: boolean",
},
{
name: "set_origin",
description:
"Set object origin using a bounded direct tool instead of context-sensitive Python operators.",
category: "geometry",
parameters: "name: string, origin_type?: GEOMETRY_ORIGIN|ORIGIN_GEOMETRY|ORIGIN_CURSOR|ORIGIN_CENTER_OF_VOLUME, center?: MEDIAN|BOUNDS",
},
{
name: "move_to_collection",
description:
"Move objects into an existing or newly created collection for deterministic scene organization.",
category: "geometry",
parameters: "name: string, collection_name: string, create_new?: boolean",
},
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@lib/orchestration/tool-registry.ts` around lines 109 - 178, The TOOL_REGISTRY
entries for direct tools have metadata that doesn't match their actual wrappers:
update the parameter contract strings for duplicate_object, join_objects,
parent_set, parent_clear, move_to_collection, and set_origin so they reflect the
real parameter names/types and enum values used by the implementation;
specifically ensure duplicate_object uses "name: string, new_name?: string,
location?: number[]", join_objects uses "names: string[], new_name?: string",
parent_set uses "children: string[], parent: string, keep_transform?: boolean",
parent_clear uses "names: string[], keep_transform?: boolean",
move_to_collection uses "names: string[], collection_name: string, create_new?:
boolean", and set_origin uses the exact enum labels the code expects (e.g.,
ORIGIN_GEOMETRY|ORIGIN_CURSOR|ORIGIN_CENTER_OF_MASS or the implementation's enum
identifiers) so planner-generated params match the tool wrappers.

{
name: "set_visibility",
description:
"Set viewport and/or render visibility for a named object.",
category: "geometry",
parameters: "name: string, hide_viewport?: boolean, hide_render?: boolean",
},
{
name: "export_object",
description:
"Export named objects to GLB, GLTF, FBX, OBJ, or STL after export readiness validation.",
category: "advanced",
parameters: "names: string[], filepath: string, file_format?: GLB|GLTF|FBX|OBJ|STL",
},
{
name: "list_installed_addons",
description:
"List enabled Blender addons so the agent can adapt to available capabilities and avoid assuming unavailable integrations.",
category: "inspection",
parameters: "(no parameters)",
},
{
name: "create_material",
description:
"Create a basic Blender material with structured color and shader parameters when a full preset is unnecessary.",
category: "materials",
parameters: "name: string, color?: number[4], roughness?: number, metallic?: number",
},
{
name: "assign_material",
description:
"Assign an existing material to an object by name without generated Python.",
category: "materials",
parameters: "object_name: string, material_name: string, slot_index?: number",
},
{
name: "execute_code",
description:
Expand Down Expand Up @@ -196,6 +329,54 @@ export const TOOL_REGISTRY: ToolMetadata[] = [
parameters:
"output_path?: string, target_names?: string[], preset?: studio|product|indoor|exterior|night, camera_name?: string, resolution?: number, samples?: number, file_format?: string, frame_camera?: boolean, distance_multiplier?: number, focal_length?: number",
},
{
name: "render_image",
description:
"Render the current scene to an image file using current render settings, optionally overriding output path or file format.",
category: "lighting",
parameters:
"output_path?: string, file_format?: PNG|JPEG|OPEN_EXR|TIFF",
},
{
name: "add_light",
description:
"Add a new POINT, SUN, SPOT, or AREA light with optional location, energy, and RGB color. Prefer this over execute_code for simple scene lighting.",
category: "lighting",
parameters:
"light_type?: POINT|SUN|SPOT|AREA, name?: string, location?: number[3], energy?: number, color?: number[3]",
},
{
name: "set_light_properties",
description:
"Modify an existing light's energy, color, softness, spot cone, spot blend, or area size without generated Python.",
category: "lighting",
parameters:
"name: string, energy?: number, color?: number[3], shadow_soft_size?: number, spot_size?: number, spot_blend?: number, size?: number",
},
{
name: "add_camera",
description:
"Add a new camera with optional location, rotation in degrees, lens, and sensor width. Use for explicit camera creation before rendering.",
category: "lighting",
parameters:
"name?: string, location?: number[3], rotation?: number[3] degrees, lens?: number, sensor_width?: number",
},
{
name: "set_camera_properties",
description:
"Modify an existing camera's lens, sensor width, clipping, depth of field, and active-scene-camera state. Make the camera active with set_active=true before render_image.",
category: "lighting",
parameters:
"name: string, lens?: number, sensor_width?: number, clip_start?: number, clip_end?: number, dof_use?: boolean, dof_focus_distance?: number, dof_aperture_fstop?: number, set_active?: boolean",
},
{
name: "set_render_settings",
description:
"Configure render engine, resolution, samples, denoising, transparency, output path, and file format before preview or final renders.",
category: "lighting",
parameters:
"engine?: BLENDER_EEVEE_NEXT|BLENDER_EEVEE|CYCLES|BLENDER_WORKBENCH, resolution_x?: number, resolution_y?: number, resolution_percentage?: number, samples?: number, use_denoising?: boolean, film_transparent?: boolean, output_path?: string, file_format?: PNG|JPEG|OPEN_EXR|TIFF",
},
{
name: "create_mesh_from_data",
description:
Expand Down
37 changes: 37 additions & 0 deletions scripts/test/test-camera-light-render-tool-registry.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import assert from "node:assert/strict"
import { readFileSync } from "node:fs"

import { TOOL_REGISTRY } from "../../lib/orchestration/tool-registry"
import { filterRelevantTools, formatToolListForPrompt } from "../../lib/orchestration/tool-filter"

const expectedTools = [
"add_light",
"set_light_properties",
"add_camera",
"set_camera_properties",
"set_render_settings",
]

for (const toolName of expectedTools) {
const metadata = TOOL_REGISTRY.find((tool) => tool.name === toolName)
assert.ok(metadata, `${toolName} should be described in TOOL_REGISTRY`)
assert.ok(metadata.description.length > 40, `${toolName} should have useful planner guidance`)
assert.ok(metadata.parameters?.length, `${toolName} should document its parameters`)
}

const lightingTools = filterRelevantTools("Add a warm spotlight, a camera, set render settings, and render a preview")
for (const toolName of expectedTools) {
assert.ok(lightingTools.includes(toolName), `${toolName} should be selected for camera/light/render requests`)
}

const promptToolList = formatToolListForPrompt(expectedTools)
assert.match(promptToolList, /Add a new POINT, SUN, SPOT, or AREA light/)
assert.match(promptToolList, /Make the camera active/)
assert.match(promptToolList, /Configure render engine/)

const testPipelineSource = readFileSync("app/api/test-pipeline/route.ts", "utf8")
for (const toolName of expectedTools) {
assert.match(testPipelineSource, new RegExp(`"${toolName}"`), `${toolName} should be allowed in the dev test pipeline`)
}

console.log("Camera, light, and render tool registry tests passed")
Loading
Loading