Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions mcp-servers/litert-mcp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ Runs inference using the configured LiteRT-LM model.
}
```

**Note**: The current CLI wrapper only supports text-only inference. For multimodal capabilities (image/audio), use the LiteRT-LM C++ or Python API directly.

## Setup for Development

This server uses a manual JSON-RPC implementation to avoid external dependencies in the base environment. Just run:
Expand Down
21 changes: 19 additions & 2 deletions mcp-servers/litert-mcp/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,10 +147,26 @@ async def _handle_tools_call(self, request_id, params):

async def _run_inference(self, args: Dict[str, Any]) -> Dict[str, Any]:
prompt = args.get("prompt")

# Validate prompt parameter
if not prompt or not isinstance(prompt, str) or not prompt.strip():
return {
"status": "error",
"message": "Invalid or empty prompt. Please provide a non-empty text prompt."
}

model_path = args.get("model_path") or self.default_model_path
image_path = args.get("image_path")
audio_path = args.get("audio_path")
backend = args.get("backend", "cpu")

# Validate backend parameter
valid_backends = ["cpu", "gpu", "npu"]
if backend not in valid_backends:
return {
"status": "error",
"message": f"Invalid backend '{backend}'. Must be one of {valid_backends}."
}

# Validate Prompt
if not prompt:
Expand Down Expand Up @@ -251,7 +267,7 @@ async def main():
)
writer = asyncio.StreamWriter(w_transport, w_protocol, None, asyncio.get_event_loop())
except Exception as e:
LOGGER.warning(f"Could not connect write pipe to stdout: {e}. Falling back to print.")
LOGGER.warning(f"Could not connect write pipe to stdout: {e}. Falling back to sys.stdout.write().")
writer = None
else:
# Windows fallback:
Expand Down Expand Up @@ -281,7 +297,8 @@ async def main():
writer = None
print(response_str, flush=True)
else:
print(response_str, flush=True)
sys.stdout.write(response_str)
sys.stdout.flush()

except json.JSONDecodeError:
LOGGER.error(f"Invalid JSON received: {line}")
Expand Down
Loading