From 26c3041b2fd6972e119dce5139f1e86560a83dea Mon Sep 17 00:00:00 2001 From: Tan Jia Huei Date: Tue, 24 Mar 2026 04:08:54 +0000 Subject: [PATCH 1/3] Add Responses MVP --- .env.example | 29 + .gitattributes | 66 + .github/workflows/ci.yml | 77 + .github/workflows/lint.yml | 106 + .github/workflows/pages.yml | 66 + .github/workflows/release-wheels.yml | 130 + .github/workflows/wheels.yml | 76 + .gitignore | 66 + .markdownlint.yaml | 31 + .prettierignore | 49 + .prettierrc | 5 + LICENSE | 28 +- README.md | 218 +- THIRD_PARTY_NOTICES.md | 14 + docs/deployment/configuration.md | 260 + docs/deployment/observability.md | 204 + docs/examples/code-interpreter-examples.md | 95 + docs/examples/hosted-mcp-examples.md | 107 + docs/examples/stateful-conversations.md | 70 + docs/examples/tool-loop-examples.md | 90 + docs/examples/web-search-examples.md | 85 + docs/features/built-in-tools.md | 106 + docs/features/hosted-mcp.md | 131 + docs/features/statefulness.md | 64 + docs/features/web-search.md | 117 + docs/getting-started/architecture.md | 83 + docs/getting-started/installation.md | 107 + docs/getting-started/quickstart.md | 160 + docs/index.md | 50 + docs/reference/api-reference.md | 277 + docs/reference/configuration.md | 139 + docs/reference/events.md | 197 + docs/snippets/mcp_builtin_request_payload.txt | 20 + .../mcp_discover_servers_tools_curl.txt | 2 + docs/snippets/mcp_runtime_config_example.txt | 20 + docs/snippets/openai_client_local_gateway.py | 8 + docs/snippets/serve_external_upstream_cmd.txt | 1 + docs/usage/command-reference.md | 157 + docs/usage/running-the-gateway.md | 127 + docs/usage/streaming-events.md | 212 + mkdocs.yml | 89 + responses/LICENSE | 177 + responses/MANIFEST.in | 10 + responses/README.md | 7 + responses/pyproject.toml | 190 + responses/python/vllm_responses/__init__.py | 0 .../python/vllm_responses/configs/__init__.py | 17 + .../python/vllm_responses/configs/builders.py | 355 ++ .../python/vllm_responses/configs/config.json | 0 .../python/vllm_responses/configs/defaults.py | 50 + .../python/vllm_responses/configs/mock_llm.py | 64 + .../python/vllm_responses/configs/runtime.py | 68 + .../python/vllm_responses/configs/sources.py | 57 + .../python/vllm_responses/configs/startup.py | 431 ++ responses/python/vllm_responses/db.py | 207 + .../vllm_responses/entrypoints/__init__.py | 0 .../entrypoints/_helper_runtime.py | 342 ++ .../entrypoints/_serve/__init__.py | 0 .../entrypoints/_serve/_runtime.py | 305 + .../entrypoints/_serve/_spec.py | 118 + .../entrypoints/_serve_utils.py | 263 + .../vllm_responses/entrypoints/_state.py | 88 + .../python/vllm_responses/entrypoints/api.py | 148 + .../entrypoints/gateway/__init__.py | 0 .../entrypoints/gateway/_app.py | 299 + .../entrypoints/gunicorn_conf.py | 32 + .../python/vllm_responses/entrypoints/llm.py | 184 + .../vllm_responses/entrypoints/mcp_runtime.py | 183 + .../vllm_responses/entrypoints/serve.py | 84 + .../entrypoints/vllm/__init__.py | 0 .../entrypoints/vllm/_adapter.py | 77 + .../entrypoints/vllm/_runtime.py | 419 ++ .../vllm_responses/entrypoints/vllm/_spec.py | 99 + .../vllm_responses/entrypoints/vllm_cli.py | 38 + responses/python/vllm_responses/lm.py | 343 ++ .../python/vllm_responses/lm_failures.py | 232 + .../python/vllm_responses/mcp/__init__.py | 12 + responses/python/vllm_responses/mcp/config.py | 263 + .../vllm_responses/mcp/fastmcp_runtime.py | 71 + .../vllm_responses/mcp/gateway_toolset.py | 192 + .../vllm_responses/mcp/hosted_registry.py | 291 + responses/python/vllm_responses/mcp/policy.py | 80 + .../python/vllm_responses/mcp/resolver.py | 230 + .../vllm_responses/mcp/runtime_client.py | 220 + .../vllm_responses/mcp/runtime_toolset.py | 70 + responses/python/vllm_responses/mcp/types.py | 57 + responses/python/vllm_responses/mcp/utils.py | 155 + .../vllm_responses/observability/__init__.py | 5 + .../vllm_responses/observability/metrics.py | 369 ++ .../vllm_responses/observability/tracing.py | 104 + .../vllm_responses/responses_core/__init__.py | 7 + .../vllm_responses/responses_core/composer.py | 726 +++ .../vllm_responses/responses_core/models.py | 190 + .../responses_core/normalizer.py | 589 ++ .../vllm_responses/responses_core/sse.py | 37 + .../vllm_responses/responses_core/store.py | 514 ++ .../python/vllm_responses/routers/__init__.py | 0 .../python/vllm_responses/routers/mcp.py | 96 + .../python/vllm_responses/routers/serving.py | 111 + .../vllm_responses/routers/upstream_proxy.py | 221 + .../python/vllm_responses/tools/__init__.py | 39 + .../vllm_responses/tools/base/__init__.py | 0 .../vllm_responses/tools/base/config.py | 7 + .../vllm_responses/tools/base/runtime.py | 34 + .../python/vllm_responses/tools/base/types.py | 82 + .../python/vllm_responses/tools/bootstrap.py | 9 + .../tools/code_interpreter/CLAUDE.md | 25 + .../tools/code_interpreter/README.md | 363 ++ .../tools/code_interpreter/__init__.py | 267 + .../tools/code_interpreter/package.json | 19 + .../tools/code_interpreter/src/index.test.ts | 95 + .../tools/code_interpreter/src/index.ts | 82 + .../code_interpreter/src/pyodide-manager.ts | 346 ++ .../tools/code_interpreter/src/repl.ts | 60 + .../tools/code_interpreter/src/server.test.ts | 240 + .../tools/code_interpreter/src/server.ts | 169 + .../tools/code_interpreter/src/types.ts | 57 + .../tools/code_interpreter/src/worker-pool.ts | 198 + .../tools/code_interpreter/src/worker.ts | 93 + .../src/xmlhttprequest-ssl.d.ts | 1 + .../tools/code_interpreter/tsconfig.json | 29 + .../tools/code_interpreter/worker.ts | 6 + responses/python/vllm_responses/tools/ids.py | 4 + .../tools/profile_resolution.py | 100 + .../python/vllm_responses/tools/runtime.py | 45 + .../tools/web_search/__init__.py | 0 .../tools/web_search/adapters/__init__.py | 50 + .../tools/web_search/adapters/base.py | 52 + .../web_search/adapters/duckduckgo_common.py | 102 + .../tools/web_search/adapters/exa_mcp.py | 181 + .../tools/web_search/adapters/fetch_mcp.py | 67 + .../vllm_responses/tools/web_search/config.py | 51 + .../tools/web_search/executor.py | 266 + .../tools/web_search/mcp_provision.py | 33 + .../tools/web_search/page_cache.py | 43 + .../tools/web_search/profiles.py | 159 + .../tools/web_search/runtime.py | 72 + .../vllm_responses/tools/web_search/tool.py | 81 + .../vllm_responses/tools/web_search/types.py | 134 + .../python/vllm_responses/types/__init__.py | 0 responses/python/vllm_responses/types/api.py | 150 + .../python/vllm_responses/types/openai.py | 2912 +++++++++ .../python/vllm_responses/utils/__init__.py | 56 + .../python/vllm_responses/utils/cache.py | 136 + .../vllm_responses/utils/cassette_replay.py | 219 + .../python/vllm_responses/utils/crypt.py | 273 + .../python/vllm_responses/utils/dates.py | 67 + .../python/vllm_responses/utils/exceptions.py | 260 + .../python/vllm_responses/utils/handlers.py | 486 ++ responses/python/vllm_responses/utils/io.py | 144 + .../python/vllm_responses/utils/logging.py | 131 + .../utils/loguru_otlp_handler.py | 284 + .../python/vllm_responses/utils/types.py | 266 + responses/python/vllm_responses/utils/urls.py | 10 + responses/python/vllm_responses/version.py | 1 + responses/setup.py | 45 + .../mcp-hosted-step1-stream.yaml | 107 + .../mcp-hosted-step2-stream.yaml | 386 ++ .../model-not-found-stream.yaml | 145 + .../text-max_tokens-length-stream.yaml | 51 + .../chat_completion/text-single-stream.yaml | 57 + .../chat_completion/text-single.yaml | 53 + .../vllm-code_interpreter-step1-stream.yaml | 353 ++ .../vllm-code_interpreter-step2-stream.yaml | 424 ++ ...tool-code_interpreter-function-stream.yaml | 3973 +++++++++++++ ...uiltin-tool-code_interpreter-function.yaml | 141 + ...code_interpreter-outputs-multi-stream.yaml | 297 + .../builtin-tool-code_interpreter-stream.yaml | 528 ++ .../builtin-tool-code_interpreter.yaml | 124 + .../builtin-tool-web_search-find_in_page.yaml | 120 + ...tin-tool-web_search-open_page-sources.yaml | 112 + .../builtin-tool-web_search-open_page.yaml | 112 + .../builtin-tool-web_search-sources.yaml | 214 + .../builtin-tool-web_search-stream.yaml | 571 ++ .../responses/builtin-tool-web_search.yaml | 134 + .../context_length_exceeded-stream.yaml | 64 + .../errors/context_length_exceeded.yaml | 31 + .../errors/missing_required_parameter.yaml | 37 + ...tool-calls-code-interpreter-stream-v1.yaml | 1295 ++++ ...calls-code-interpreter-stream-v2-zero.yaml | 41 + ...-calls-code-interpreter-stream-v3-two.yaml | 755 +++ .../max-tool-calls-web-search-v1.yaml | 645 ++ .../max-tool-calls-web-search-v2.yaml | 753 +++ ...t-max_output_tokens-incomplete-stream.yaml | 75 + .../text-max_output_tokens-incomplete.yaml | 80 + .../responses/text-single-stream.yaml | 133 + .../cassettes/responses/text-single.yaml | 82 + .../responses/tool-function-stream.yaml | 1030 ++++ .../tool-function-tool_loop-step1-stream.yaml | 179 + .../tool-function-tool_loop-step1.yaml | 113 + ...ion-tool_loop-step2-mixed-input-step1.yaml | 113 + ...-function-tool_loop-step2-mixed-input.yaml | 92 + ...ion-tool_loop-step2-omit-tools-stream.yaml | 355 ++ ...l-function-tool_loop-step2-omit-tools.yaml | 92 + ...p-step2-tool_choice-without-tools-400.yaml | 41 + .../cassettes/responses/tool-function.yaml | 183 + responses/tests/conftest.py | 159 + responses/tests/sse_test_utils.py | 86 + responses/tests/test_app_extensions.py | 167 + responses/tests/test_code_interpreter.py | 259 + .../tests/test_gateway_cassette_replay.py | 250 + responses/tests/test_gateway_statefulness.py | 259 + responses/tests/test_gateway_tool_metrics.py | 126 + responses/tests/test_integrated_app.py | 382 ++ responses/tests/test_integrated_runtime.py | 508 ++ responses/tests/test_integrated_spec.py | 156 + responses/tests/test_mcp.py | 3839 ++++++++++++ responses/tests/test_mcp_runtime_client.py | 210 + .../tests/test_mcp_runtime_entrypoint.py | 162 + responses/tests/test_observability_metrics.py | 92 + .../tests/test_openresponses_conformance.py | 152 + responses/tests/test_packaging_contract.py | 13 + .../tests/test_pydantic_ai_normalizer.py | 131 + responses/tests/test_response_composer.py | 387 ++ responses/tests/test_response_store.py | 358 ++ responses/tests/test_runtime_config.py | 230 + responses/tests/test_serve_cli_parsing.py | 53 + responses/tests/test_serve_spec.py | 361 ++ responses/tests/test_serve_supervisor.py | 194 + responses/tests/test_serve_utils.py | 169 + .../tests/test_store_schema_init_multicore.py | 56 + responses/tests/test_upstream_proxy.py | 346 ++ responses/tests/test_vllm_cli.py | 100 + responses/tests/test_web_search.py | 688 +++ responses/uv.lock | 5201 +++++++++++++++++ scripts/ci/bootstrap_pyodide_cache.py | 88 + scripts/ci/container_smoke_test.py | 77 + .../prebuild_code_interpreter_linux_x86_64.sh | 43 + scripts/record_openai_mcp_chat_cassettes.py | 258 + scripts/recorder_proxy.py | 299 + 230 files changed, 52459 insertions(+), 28 deletions(-) create mode 100644 .env.example create mode 100644 .gitattributes create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pages.yml create mode 100644 .github/workflows/release-wheels.yml create mode 100644 .github/workflows/wheels.yml create mode 100644 .gitignore create mode 100644 .markdownlint.yaml create mode 100644 .prettierignore create mode 100644 .prettierrc create mode 100644 THIRD_PARTY_NOTICES.md create mode 100644 docs/deployment/configuration.md create mode 100644 docs/deployment/observability.md create mode 100644 docs/examples/code-interpreter-examples.md create mode 100644 docs/examples/hosted-mcp-examples.md create mode 100644 docs/examples/stateful-conversations.md create mode 100644 docs/examples/tool-loop-examples.md create mode 100644 docs/examples/web-search-examples.md create mode 100644 docs/features/built-in-tools.md create mode 100644 docs/features/hosted-mcp.md create mode 100644 docs/features/statefulness.md create mode 100644 docs/features/web-search.md create mode 100644 docs/getting-started/architecture.md create mode 100644 docs/getting-started/installation.md create mode 100644 docs/getting-started/quickstart.md create mode 100644 docs/index.md create mode 100644 docs/reference/api-reference.md create mode 100644 docs/reference/configuration.md create mode 100644 docs/reference/events.md create mode 100644 docs/snippets/mcp_builtin_request_payload.txt create mode 100644 docs/snippets/mcp_discover_servers_tools_curl.txt create mode 100644 docs/snippets/mcp_runtime_config_example.txt create mode 100644 docs/snippets/openai_client_local_gateway.py create mode 100644 docs/snippets/serve_external_upstream_cmd.txt create mode 100644 docs/usage/command-reference.md create mode 100644 docs/usage/running-the-gateway.md create mode 100644 docs/usage/streaming-events.md create mode 100644 mkdocs.yml create mode 100644 responses/LICENSE create mode 100644 responses/MANIFEST.in create mode 100644 responses/README.md create mode 100644 responses/pyproject.toml create mode 100644 responses/python/vllm_responses/__init__.py create mode 100644 responses/python/vllm_responses/configs/__init__.py create mode 100644 responses/python/vllm_responses/configs/builders.py create mode 100644 responses/python/vllm_responses/configs/config.json create mode 100644 responses/python/vllm_responses/configs/defaults.py create mode 100644 responses/python/vllm_responses/configs/mock_llm.py create mode 100644 responses/python/vllm_responses/configs/runtime.py create mode 100644 responses/python/vllm_responses/configs/sources.py create mode 100644 responses/python/vllm_responses/configs/startup.py create mode 100644 responses/python/vllm_responses/db.py create mode 100644 responses/python/vllm_responses/entrypoints/__init__.py create mode 100644 responses/python/vllm_responses/entrypoints/_helper_runtime.py create mode 100644 responses/python/vllm_responses/entrypoints/_serve/__init__.py create mode 100644 responses/python/vllm_responses/entrypoints/_serve/_runtime.py create mode 100644 responses/python/vllm_responses/entrypoints/_serve/_spec.py create mode 100644 responses/python/vllm_responses/entrypoints/_serve_utils.py create mode 100644 responses/python/vllm_responses/entrypoints/_state.py create mode 100644 responses/python/vllm_responses/entrypoints/api.py create mode 100644 responses/python/vllm_responses/entrypoints/gateway/__init__.py create mode 100644 responses/python/vllm_responses/entrypoints/gateway/_app.py create mode 100644 responses/python/vllm_responses/entrypoints/gunicorn_conf.py create mode 100644 responses/python/vllm_responses/entrypoints/llm.py create mode 100644 responses/python/vllm_responses/entrypoints/mcp_runtime.py create mode 100644 responses/python/vllm_responses/entrypoints/serve.py create mode 100644 responses/python/vllm_responses/entrypoints/vllm/__init__.py create mode 100644 responses/python/vllm_responses/entrypoints/vllm/_adapter.py create mode 100644 responses/python/vllm_responses/entrypoints/vllm/_runtime.py create mode 100644 responses/python/vllm_responses/entrypoints/vllm/_spec.py create mode 100644 responses/python/vllm_responses/entrypoints/vllm_cli.py create mode 100644 responses/python/vllm_responses/lm.py create mode 100644 responses/python/vllm_responses/lm_failures.py create mode 100644 responses/python/vllm_responses/mcp/__init__.py create mode 100644 responses/python/vllm_responses/mcp/config.py create mode 100644 responses/python/vllm_responses/mcp/fastmcp_runtime.py create mode 100644 responses/python/vllm_responses/mcp/gateway_toolset.py create mode 100644 responses/python/vllm_responses/mcp/hosted_registry.py create mode 100644 responses/python/vllm_responses/mcp/policy.py create mode 100644 responses/python/vllm_responses/mcp/resolver.py create mode 100644 responses/python/vllm_responses/mcp/runtime_client.py create mode 100644 responses/python/vllm_responses/mcp/runtime_toolset.py create mode 100644 responses/python/vllm_responses/mcp/types.py create mode 100644 responses/python/vllm_responses/mcp/utils.py create mode 100644 responses/python/vllm_responses/observability/__init__.py create mode 100644 responses/python/vllm_responses/observability/metrics.py create mode 100644 responses/python/vllm_responses/observability/tracing.py create mode 100644 responses/python/vllm_responses/responses_core/__init__.py create mode 100644 responses/python/vllm_responses/responses_core/composer.py create mode 100644 responses/python/vllm_responses/responses_core/models.py create mode 100644 responses/python/vllm_responses/responses_core/normalizer.py create mode 100644 responses/python/vllm_responses/responses_core/sse.py create mode 100644 responses/python/vllm_responses/responses_core/store.py create mode 100644 responses/python/vllm_responses/routers/__init__.py create mode 100644 responses/python/vllm_responses/routers/mcp.py create mode 100644 responses/python/vllm_responses/routers/serving.py create mode 100644 responses/python/vllm_responses/routers/upstream_proxy.py create mode 100644 responses/python/vllm_responses/tools/__init__.py create mode 100644 responses/python/vllm_responses/tools/base/__init__.py create mode 100644 responses/python/vllm_responses/tools/base/config.py create mode 100644 responses/python/vllm_responses/tools/base/runtime.py create mode 100644 responses/python/vllm_responses/tools/base/types.py create mode 100644 responses/python/vllm_responses/tools/bootstrap.py create mode 100644 responses/python/vllm_responses/tools/code_interpreter/CLAUDE.md create mode 100644 responses/python/vllm_responses/tools/code_interpreter/README.md create mode 100644 responses/python/vllm_responses/tools/code_interpreter/__init__.py create mode 100644 responses/python/vllm_responses/tools/code_interpreter/package.json create mode 100644 responses/python/vllm_responses/tools/code_interpreter/src/index.test.ts create mode 100644 responses/python/vllm_responses/tools/code_interpreter/src/index.ts create mode 100644 responses/python/vllm_responses/tools/code_interpreter/src/pyodide-manager.ts create mode 100644 responses/python/vllm_responses/tools/code_interpreter/src/repl.ts create mode 100644 responses/python/vllm_responses/tools/code_interpreter/src/server.test.ts create mode 100644 responses/python/vllm_responses/tools/code_interpreter/src/server.ts create mode 100644 responses/python/vllm_responses/tools/code_interpreter/src/types.ts create mode 100644 responses/python/vllm_responses/tools/code_interpreter/src/worker-pool.ts create mode 100644 responses/python/vllm_responses/tools/code_interpreter/src/worker.ts create mode 100644 responses/python/vllm_responses/tools/code_interpreter/src/xmlhttprequest-ssl.d.ts create mode 100644 responses/python/vllm_responses/tools/code_interpreter/tsconfig.json create mode 100644 responses/python/vllm_responses/tools/code_interpreter/worker.ts create mode 100644 responses/python/vllm_responses/tools/ids.py create mode 100644 responses/python/vllm_responses/tools/profile_resolution.py create mode 100644 responses/python/vllm_responses/tools/runtime.py create mode 100644 responses/python/vllm_responses/tools/web_search/__init__.py create mode 100644 responses/python/vllm_responses/tools/web_search/adapters/__init__.py create mode 100644 responses/python/vllm_responses/tools/web_search/adapters/base.py create mode 100644 responses/python/vllm_responses/tools/web_search/adapters/duckduckgo_common.py create mode 100644 responses/python/vllm_responses/tools/web_search/adapters/exa_mcp.py create mode 100644 responses/python/vllm_responses/tools/web_search/adapters/fetch_mcp.py create mode 100644 responses/python/vllm_responses/tools/web_search/config.py create mode 100644 responses/python/vllm_responses/tools/web_search/executor.py create mode 100644 responses/python/vllm_responses/tools/web_search/mcp_provision.py create mode 100644 responses/python/vllm_responses/tools/web_search/page_cache.py create mode 100644 responses/python/vllm_responses/tools/web_search/profiles.py create mode 100644 responses/python/vllm_responses/tools/web_search/runtime.py create mode 100644 responses/python/vllm_responses/tools/web_search/tool.py create mode 100644 responses/python/vllm_responses/tools/web_search/types.py create mode 100644 responses/python/vllm_responses/types/__init__.py create mode 100644 responses/python/vllm_responses/types/api.py create mode 100644 responses/python/vllm_responses/types/openai.py create mode 100644 responses/python/vllm_responses/utils/__init__.py create mode 100644 responses/python/vllm_responses/utils/cache.py create mode 100644 responses/python/vllm_responses/utils/cassette_replay.py create mode 100644 responses/python/vllm_responses/utils/crypt.py create mode 100644 responses/python/vllm_responses/utils/dates.py create mode 100644 responses/python/vllm_responses/utils/exceptions.py create mode 100644 responses/python/vllm_responses/utils/handlers.py create mode 100644 responses/python/vllm_responses/utils/io.py create mode 100644 responses/python/vllm_responses/utils/logging.py create mode 100644 responses/python/vllm_responses/utils/loguru_otlp_handler.py create mode 100644 responses/python/vllm_responses/utils/types.py create mode 100644 responses/python/vllm_responses/utils/urls.py create mode 100644 responses/python/vllm_responses/version.py create mode 100644 responses/setup.py create mode 100644 responses/tests/cassettes/chat_completion/mcp-hosted-step1-stream.yaml create mode 100644 responses/tests/cassettes/chat_completion/mcp-hosted-step2-stream.yaml create mode 100644 responses/tests/cassettes/chat_completion/model-not-found-stream.yaml create mode 100644 responses/tests/cassettes/chat_completion/text-max_tokens-length-stream.yaml create mode 100644 responses/tests/cassettes/chat_completion/text-single-stream.yaml create mode 100644 responses/tests/cassettes/chat_completion/text-single.yaml create mode 100644 responses/tests/cassettes/chat_completion/vllm-code_interpreter-step1-stream.yaml create mode 100644 responses/tests/cassettes/chat_completion/vllm-code_interpreter-step2-stream.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-code_interpreter-function-stream.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-code_interpreter-function.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-code_interpreter-outputs-multi-stream.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-code_interpreter-stream.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-code_interpreter.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-web_search-find_in_page.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-web_search-open_page-sources.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-web_search-open_page.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-web_search-sources.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-web_search-stream.yaml create mode 100644 responses/tests/cassettes/responses/builtin-tool-web_search.yaml create mode 100644 responses/tests/cassettes/responses/errors/context_length_exceeded-stream.yaml create mode 100644 responses/tests/cassettes/responses/errors/context_length_exceeded.yaml create mode 100644 responses/tests/cassettes/responses/errors/missing_required_parameter.yaml create mode 100644 responses/tests/cassettes/responses/max-tool-calls-code-interpreter-stream-v1.yaml create mode 100644 responses/tests/cassettes/responses/max-tool-calls-code-interpreter-stream-v2-zero.yaml create mode 100644 responses/tests/cassettes/responses/max-tool-calls-code-interpreter-stream-v3-two.yaml create mode 100644 responses/tests/cassettes/responses/max-tool-calls-web-search-v1.yaml create mode 100644 responses/tests/cassettes/responses/max-tool-calls-web-search-v2.yaml create mode 100644 responses/tests/cassettes/responses/text-max_output_tokens-incomplete-stream.yaml create mode 100644 responses/tests/cassettes/responses/text-max_output_tokens-incomplete.yaml create mode 100644 responses/tests/cassettes/responses/text-single-stream.yaml create mode 100644 responses/tests/cassettes/responses/text-single.yaml create mode 100644 responses/tests/cassettes/responses/tool-function-stream.yaml create mode 100644 responses/tests/cassettes/responses/tool-function-tool_loop-step1-stream.yaml create mode 100644 responses/tests/cassettes/responses/tool-function-tool_loop-step1.yaml create mode 100644 responses/tests/cassettes/responses/tool-function-tool_loop-step2-mixed-input-step1.yaml create mode 100644 responses/tests/cassettes/responses/tool-function-tool_loop-step2-mixed-input.yaml create mode 100644 responses/tests/cassettes/responses/tool-function-tool_loop-step2-omit-tools-stream.yaml create mode 100644 responses/tests/cassettes/responses/tool-function-tool_loop-step2-omit-tools.yaml create mode 100644 responses/tests/cassettes/responses/tool-function-tool_loop-step2-tool_choice-without-tools-400.yaml create mode 100644 responses/tests/cassettes/responses/tool-function.yaml create mode 100644 responses/tests/conftest.py create mode 100644 responses/tests/sse_test_utils.py create mode 100644 responses/tests/test_app_extensions.py create mode 100644 responses/tests/test_code_interpreter.py create mode 100644 responses/tests/test_gateway_cassette_replay.py create mode 100644 responses/tests/test_gateway_statefulness.py create mode 100644 responses/tests/test_gateway_tool_metrics.py create mode 100644 responses/tests/test_integrated_app.py create mode 100644 responses/tests/test_integrated_runtime.py create mode 100644 responses/tests/test_integrated_spec.py create mode 100644 responses/tests/test_mcp.py create mode 100644 responses/tests/test_mcp_runtime_client.py create mode 100644 responses/tests/test_mcp_runtime_entrypoint.py create mode 100644 responses/tests/test_observability_metrics.py create mode 100644 responses/tests/test_openresponses_conformance.py create mode 100644 responses/tests/test_packaging_contract.py create mode 100644 responses/tests/test_pydantic_ai_normalizer.py create mode 100644 responses/tests/test_response_composer.py create mode 100644 responses/tests/test_response_store.py create mode 100644 responses/tests/test_runtime_config.py create mode 100644 responses/tests/test_serve_cli_parsing.py create mode 100644 responses/tests/test_serve_spec.py create mode 100644 responses/tests/test_serve_supervisor.py create mode 100644 responses/tests/test_serve_utils.py create mode 100644 responses/tests/test_store_schema_init_multicore.py create mode 100644 responses/tests/test_upstream_proxy.py create mode 100644 responses/tests/test_vllm_cli.py create mode 100644 responses/tests/test_web_search.py create mode 100644 responses/uv.lock create mode 100644 scripts/ci/bootstrap_pyodide_cache.py create mode 100644 scripts/ci/container_smoke_test.py create mode 100644 scripts/ci/prebuild_code_interpreter_linux_x86_64.sh create mode 100644 scripts/record_openai_mcp_chat_cassettes.py create mode 100644 scripts/recorder_proxy.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2eb44c7 --- /dev/null +++ b/.env.example @@ -0,0 +1,29 @@ +# External API keys +VR_OPENAI_API_KEY= + +# Service connection (dev) +# VR_DB_PATH=postgresql+psycopg://owlpguser:owlpgpassword@localhost:5432/jamaibase_owl +# VR_OPENTELEMETRY_HOST=localhost +# VR_OPENTELEMETRY_PORT=4317 +# VR_REDIS_HOST=localhost +# VR_REDIS_PORT=6379 +# Optional: ResponseStore hot cache (Redis). Disabled by default. +# VR_RESPONSE_STORE_CACHE=0 # 0|1 +# VR_RESPONSE_STORE_CACHE_TTL_SECONDS=3600 +# VR_VICTORIA_METRICS_HOST=localhost +# VR_VICTORIA_LOGS_HOST=localhost +# VR_LLM_API_BASE=http://localhost:6970/v1 +# +# Tools +# VR_CODE_INTERPRETER_MODE=spawn # spawn | external | disabled +# VR_CODE_INTERPRETER_PORT=5970 +# VR_CODE_INTERPRETER_WORKERS=0 +# VR_PYODIDE_CACHE_DIR=~/.cache/vllm-responses/pyodide +# VR_CODE_INTERPRETER_DEV_BUN_FALLBACK=0 # 0|1 (dev-only; allows bun src fallback when no binary exists) + +# Configuration +VR_PORT=6969 +VR_WORKERS=3 +PB_MAX_CLIENT_CONN=500 +PB_MAX_CLIENT_CONN=80 +PG_MAX_CONNECTIONS=100 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2c04005 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,66 @@ +### https://git-scm.com/docs/gitattributes + +# When `text` is set to "auto", the path is marked for automatic end-of-line conversion. +# If Git decides that the content is text, its line endings are converted to LF on checkin. +# When the file has been committed with CRLF, no conversion is done. +# https://git-scm.com/docs/gitattributes#Documentation/gitattributes.txt-Settostringvalueauto +* text=auto eol=lf + +# Declare files that will always have CRLF line endings on checkout. +*.bat text eol=crlf + +# Declare files that will always have LF line endings on checkout. +*.sh text eol=lf + + +# These files are text and should be normalized (Convert crlf => lf) +# Setting the `text` attribute on a path enables end-of-line normalization and marks the path as a text file. +# End-of-line conversion takes place without guessing the content type. +# https://git-scm.com/docs/gitattributes#_text +*.css text +*.html text +*.js* text +*.md text +*.py text +*.sh text + +# These files are binary and should be left untouched +# `binary` is a built-in macro for `-text` `-diff` (ie unset `text` and `diff`) +# https://git-scm.com/docs/gitattributes#_using_macro_attributes +*.7z binary +*.bin binary +*.cu.o binary +*.db binary +*.doc binary +*.docx binary +*.gif binary +*.gz binary +*.heic* binary +*.heif* binary +*.jar binary +*.jpeg binary +*.jpg binary +*.mov binary +*.mp* binary +*.npy binary +*.npz binary +*.parquet binary +*.pcd binary +*.pdf binary +*.pkl binary +*.png binary +*.ppt binary +*.pptx binary +*.pth binary +*.so binary +*.webp binary +*.xls binary +*.xlsx binary +*.zip binary + +# Track with LFS +# *.pth filter=lfs diff=lfs merge=lfs -text + +# These files should not be processed by Linguist for language detection on GitHub.com +*.p linguist-detectable=false +*.gz linguist-detectable=false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ed23c3c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,77 @@ +name: CI Workflow + +on: + pull_request: + push: + branches: + - "main" + workflow_dispatch: + +jobs: + docs_build: + name: Docs build (MkDocs --strict) + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.8.14" + enable-cache: true + + - name: Build docs + working-directory: responses + run: | + uv sync --frozen --extra docs + uv run --no-sync mkdocs build --strict -f ../mkdocs.yml + + pytest: + name: Pytest + runs-on: ubuntu-latest + env: + VR_PYODIDE_CACHE_DIR: ~/.cache/vllm-responses/pyodide + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.8.14" + enable-cache: true + + - name: Cache Pyodide runtime + uses: actions/cache@v4 + with: + path: ~/.cache/vllm-responses/pyodide + key: pyodide-0.29.1-${{ runner.os }} + + - name: Sync (locked) + working-directory: responses + run: uv sync --frozen --extra test + + - name: Bootstrap Pyodide cache (required for code interpreter tests) + run: | + python scripts/ci/bootstrap_pyodide_cache.py + + - name: Build code interpreter binary (required for code interpreter tests) + run: | + bash scripts/ci/prebuild_code_interpreter_linux_x86_64.sh responses + + - name: Run tests + working-directory: responses + run: uv run --no-sync pytest -q diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..2e725d8 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,106 @@ +name: Linting Workflow + +on: + pull_request: + push: + branches: + - "*" + +jobs: + python_lint: + name: Python linting and formatting checks + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.8.14" + enable-cache: true + + - name: Sync (locked) + working-directory: responses + run: uv sync --frozen --extra lint + + - name: Check Python files using Ruff + working-directory: responses + run: | + uv run --no-sync ruff check --output-format github --config pyproject.toml .. + uv run --no-sync ruff format --diff --config pyproject.toml .. + + prettier_lint: + name: Prettier Checks + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 24 + + - name: Check files using Prettier + # .prettierignore excludes Markdown files (handled by mdformat) + run: npx prettier@3.8 --check . + + markdown_format: + name: Markdown Format Check (mdformat) + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.8.14" + enable-cache: true + + - name: Sync (locked) + working-directory: responses + run: uv sync --frozen --extra lint + + - name: Check Markdown formatting + working-directory: responses + run: | + # --no-validate allows files with complex MkDocs syntax to be checked + # Some files with advanced admonitions may fail strict HTML validation + uv run --no-sync mdformat --check --no-validate ../docs/ || (echo "Run 'cd responses && uv sync --active --extra lint' then 'uv run mdformat ../docs/' locally to fix" && exit 1) + + markdown_lint: + name: Markdown Lint (markdownlint-cli) + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: 24 + + - name: Lint Markdown files + run: npx markdownlint-cli@0.47.0 docs/ diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..3f2ad49 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,66 @@ +name: Deploy Docs (GitHub Pages) + +on: + push: + branches: + - "main" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + name: Build MkDocs site + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.8.14" + enable-cache: true + + - name: Build docs (locked) + working-directory: responses + run: | + uv sync --frozen --extra docs + uv run --no-sync mkdocs build --strict -f ../mkdocs.yml -d "${GITHUB_WORKSPACE}/site" + + - name: Verify site artifact exists + run: | + test -d site + test -f site/index.html + ls -la site | head -n 50 + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + name: Deploy to GitHub Pages + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Deploy + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/release-wheels.yml b/.github/workflows/release-wheels.yml new file mode 100644 index 0000000..f3951ba --- /dev/null +++ b/.github/workflows/release-wheels.yml @@ -0,0 +1,130 @@ +name: Release Wheels (GitHub Releases) + +on: + release: + types: + - published + workflow_dispatch: + inputs: + tag: + description: "Existing git tag to release (for example v0.2.0)" + required: true + type: string + +permissions: + contents: write + +jobs: + release_linux_x86_64: + name: Build + publish Linux wheels/sdist + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Resolve release tag + id: meta + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + TAG="${{ inputs.tag }}" + git fetch --tags --force + git rev-parse "refs/tags/${TAG}" >/dev/null + git checkout "refs/tags/${TAG}" + elif [ "${{ github.event_name }}" = "release" ]; then + TAG="${{ github.event.release.tag_name }}" + git fetch --tags --force + git rev-parse "refs/tags/${TAG}" >/dev/null + git checkout "refs/tags/${TAG}" + else + TAG="${GITHUB_REF_NAME}" + fi + echo "tag=${TAG}" >> "${GITHUB_OUTPUT}" + + - name: Ensure release tag commit is on main + run: | + set -euo pipefail + TAG="${{ steps.meta.outputs.tag }}" + git fetch origin main --depth=1 + TAG_SHA="$(git rev-list -n1 "refs/tags/${TAG}")" + if ! git merge-base --is-ancestor "${TAG_SHA}" origin/main; then + echo "Tag ${TAG} (${TAG_SHA}) is not on origin/main; refusing release build." + exit 1 + fi + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install build tooling + run: | + python -m pip install --upgrade pip + python -m pip install cibuildwheel==2.21.3 build==1.3.0 + + - name: Build code interpreter binary (host) + run: | + bash scripts/ci/prebuild_code_interpreter_linux_x86_64.sh responses + + - name: Build wheels (cibuildwheel) + working-directory: responses + env: + CIBW_BUILD: "cp312-manylinux_x86_64" + CIBW_ARCHS_LINUX: "x86_64" + CIBW_REPAIR_WHEEL_COMMAND: "" + run: | + python -m cibuildwheel --output-dir wheelhouse + + - name: Build sdist + working-directory: responses + run: | + python -m build --sdist --outdir dist + + - name: Artifact inspection (forbidden paths) + working-directory: responses + run: | + set -euo pipefail + echo "[inspect] wheels:" + python -m zipfile -l wheelhouse/*.whl | grep -E "vllm_responses/tools/code_interpreter/(node_modules/|pyodide-.*\\.tar\\.bz2)" && exit 1 || true + echo "[inspect] sdist:" + tar -tzf dist/*.tar.gz | grep -E "vllm_responses/tools/code_interpreter/(node_modules/|pyodide-.*\\.tar\\.bz2)" && exit 1 || true + + - name: Minimal container smoke test (tar missing is actionable) + working-directory: responses + run: | + set -euo pipefail + docker run --rm \ + -v "$(pwd)/wheelhouse:/wheels:ro" \ + -v "$(pwd)/../scripts/ci:/scripts:ro" \ + python:3.12-slim \ + sh -lc 'python -m venv /tmp/venv && . /tmp/venv/bin/activate && pip install /wheels/*.whl && python /scripts/container_smoke_test.py' + + - name: Generate checksums + working-directory: responses + run: | + set -euo pipefail + sha256sum wheelhouse/*.whl dist/*.tar.gz > dist/SHA256SUMS.txt + cat dist/SHA256SUMS.txt + + - name: Upload build artifacts (workflow) + uses: actions/upload-artifact@v4 + with: + name: release-assets-${{ steps.meta.outputs.tag }} + path: | + responses/wheelhouse/*.whl + responses/dist/*.tar.gz + responses/dist/SHA256SUMS.txt + + - name: Publish GitHub Release assets + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.meta.outputs.tag }} + generate_release_notes: true + fail_on_unmatched_files: true + files: | + responses/wheelhouse/*.whl + responses/dist/*.tar.gz + responses/dist/SHA256SUMS.txt diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..b95fb17 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,76 @@ +name: Build Wheels (Linux x86_64) + +on: + push: + branches: + - "main" + workflow_dispatch: + +jobs: + wheels_linux_x86_64: + name: manylinux wheels + smoke checks + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install build tooling + run: | + python -m pip install --upgrade pip + python -m pip install cibuildwheel==2.21.3 build==1.3.0 + + - name: Build code interpreter binary (host) + run: | + bash scripts/ci/prebuild_code_interpreter_linux_x86_64.sh responses + + - name: Build wheels (cibuildwheel) + working-directory: responses + env: + CIBW_BUILD: "cp312-manylinux_x86_64" + CIBW_ARCHS_LINUX: "x86_64" + CIBW_REPAIR_WHEEL_COMMAND: "" + + run: | + python -m cibuildwheel --output-dir wheelhouse + + - name: Build sdist + working-directory: responses + run: | + python -m build --sdist --outdir dist + + - name: Artifact inspection (forbidden paths) + working-directory: responses + run: | + set -euo pipefail + echo "[inspect] wheels:" + python -m zipfile -l wheelhouse/*.whl | grep -E "vllm_responses/tools/code_interpreter/(node_modules/|pyodide-.*\\.tar\\.bz2)" && exit 1 || true + echo "[inspect] sdist:" + tar -tzf dist/*.tar.gz | grep -E "vllm_responses/tools/code_interpreter/(node_modules/|pyodide-.*\\.tar\\.bz2)" && exit 1 || true + + - name: Minimal container smoke test (tar missing is actionable) + working-directory: responses + run: | + set -euo pipefail + docker run --rm \ + -v "$(pwd)/wheelhouse:/wheels:ro" \ + -v "$(pwd)/../scripts/ci:/scripts:ro" \ + python:3.12-slim \ + sh -lc 'python -m venv /tmp/venv && . /tmp/venv/bin/activate && pip install /wheels/*.whl && python /scripts/container_smoke_test.py' + + - name: Upload wheel artifacts + uses: actions/upload-artifact@v4 + with: + name: wheelhouse-linux-x86_64 + path: responses/wheelhouse/*.whl + + - name: Upload sdist artifact + uses: actions/upload-artifact@v4 + with: + name: sdist + path: responses/dist/*.tar.gz diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d5474a8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,66 @@ +# OS +thumbs.db +.DS_Store + +# Internal references, dependencies, temporary folders & files +.env +**/__ref__/ +*.log +*.lock +!uv.lock +!responses/uv.lock +*.db* +*.parquet +.cache/ + +# Research artifacts / local harnesses (keep out of git; curated fixtures live under `responses/tests/cassettes/`) +/cassettes/ +/study/ +/plans/ +/skills/ + +# Docker +/docker_data/ + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.egg-info +*.whl +.pytest_cache +.ipynb_checkpoints +venv/ +responses/dist +responses/wheelhouse/ +responses/python/vllm_responses/tools/code_interpreter/bin + +# pip +**/build/ + +# pytest-cov +**/coverage.xml +**/.coverage* +/junit +/htmlcov +/coverage.xml + +# ruff +.ruff_cache/ + +# jest-cov +**/coverage/* + +# JavaScript +**/node_modules/ +**/logs +responses/python/vllm_responses/tools/code_interpreter/pyodide-*.tar.bz2 + +# db related +**/ch_data/ +**/vm_data/ +**/ch_logs/ +/db/ +**/vl_data/ +**/rqlite_db/ +responses/python/vllm_responses/tools/code_interpreter/**/code-interpreter-server diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..fdf9b06 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,31 @@ +# Markdownlint configuration for MkDocs/Material documentation +# https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md + +# List item indentation (4 spaces matches mdformat-mkdocs) +MD007: + indent: 4 + +# Disable line length limit (not enforced for docs) +MD013: false + +# Allow duplicate headers at different levels (for MkDocs structure) +MD024: + siblings_only: true + +# Allow code blocks without blank lines in lists (important for admonitions) +MD031: + list_items: false + +# Allow inline HTML (MkDocs grids, admonitions, etc.) +MD033: false + +# Allow both fenced and indented code blocks +MD046: false + +# Allow reference-style links +MD052: false + +# Allow relative links (MkDocs handles these) +MD059: false + +MD036: true diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..20e61ff --- /dev/null +++ b/.prettierignore @@ -0,0 +1,49 @@ +# OS +thumbs.db +.DS_Store + +# Cassettes +responses/python/tests/cassettes/**/*.yaml + +# Internal references, dependencies, temporary folders & files +.env +archive/ +# **/__ref__/ +*.log +*.lock +*.db +*.parquet + +# Python +__pycache__/ +*.py* +*$py.class +*.egg-info +.pytest_cache +.ipynb_checkpoints +.venv/ +**/.venv/ +venv/ +clients/python/tests/**/* + +# pip +**/build/ + +# pytest-cov +**/.coverage* +/junit +/htmlcov +/coverage.xml + +# jest-cov +**/coverage/* + +# JavaScript +**/node_modules/ + +# Frontend +services/app +clients/typescript + +# Markdown files - handled by mdformat +**/*.md diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..95e419e --- /dev/null +++ b/.prettierrc @@ -0,0 +1,5 @@ +{ + "printWidth": 150, + "proseWrap": "never", + "trailingComma": "all" +} diff --git a/LICENSE b/LICENSE index 261eeb9..eda7b0a 100644 --- a/LICENSE +++ b/LICENSE @@ -167,35 +167,11 @@ and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, + on Your own behalf and on Your sole responsibility, not on behalf of + any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.md b/README.md index 19a795d..ea68704 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,216 @@ -# vllm-agentic-stack -Stateful API logic for agentic applications using vLLM +# vLLM Responses + +FastAPI gateway that exposes an OpenAI-style **Responses API** (`/v1/responses`) in front of a vLLM **OpenAI-compatible** server (`/v1/chat/completions`), with: + +- SSE streaming event shape + ordering +- `previous_response_id` statefulness (ResponseStore) +- gateway-executed built-in tool: `code_interpreter` +- gateway-hosted MCP tools (`tools[].type="mcp"` with configured `server_label`) + +Current MCP boundary: + +- `tools[].type="mcp"` is gateway-hosted MCP resolved via `VR_MCP_CONFIG_PATH`. +- Request-declared MCP targets (`server_url`, `connector_id`) are not supported yet. + +**[📚 Full User Documentation](https://embeddedllm.github.io/vllm-responses/)** (Guides, API Reference, Examples) + +Design docs (maintainer-facing): `design_docs/index.md`. + +## Install + +The `vllm-responses` CLI is provided by the Python package in `responses/`. + +**Prerequisites:** Python 3.12+ and `uv`. + +### Install from a prebuilt wheel (Linux x86_64) (Recommended) + +Download a prebuilt wheel (`vllm_responses-*.whl`) from GitHub Releases (preferred) or a CI run artifact, then install it: + +```bash +uv venv --python=3.12 +source .venv/bin/activate +uv pip install vllm +uv pip install path/to/vllm_responses-*.whl +``` + +On Linux x86_64 wheels, the Code Interpreter server binary is bundled, so **Bun is not required**. +Currently, wheels are only built for Linux x86_64. + +Installing `vllm-responses` provides: + +- `vllm-responses` for the standalone supervisor mode +- `vllm` as a CLI shim that supports `vllm serve --responses` and delegates all non-Responses paths to the upstream + `vllm` Python package + +### Install from source (repo checkout) (Development) + +```bash +git clone https://github.com/EmbeddedLLM/vllm-responses +cd vllm-responses + +uv venv --python=3.12 +source .venv/bin/activate +uv pip install vllm +uv pip install -e ./responses + +# Development: enable Code Interpreter via Bun fallback +# - Required for source checkouts when running with `code_interpreter` enabled (default) +cd responses/python/vllm_responses/tools/code_interpreter +bun install +export VR_CODE_INTERPRETER_DEV_BUN_FALLBACK=1 +cd - + +vllm-responses --help +``` + +Verify installation: + +```bash +vllm-responses --help +vllm --help +``` + +### Optional dependency sets (extras) + +Install any combination via: + +```bash +uv pip install -e './responses[,]' +``` + +Available extras: + +- `docs`: MkDocs toolchain (contributors). +- `lint`: Ruff + Markdown formatting. +- `test`: Pytest + coverage + load testing tools. +- `tracing`: OpenTelemetry tracing support (only needed if you enable `VR_TRACING_ENABLED=true`). +- `build`: Package build/publish tools. +- `all`: Everything above. + +## Build a wheel from source + +If you want to produce a local wheel from this checkout, build from the +`responses/` package directory. + +### Rebuild the bundled Code Interpreter binary (Linux x86_64 only) + +This step is only needed if you want the wheel to include a freshly compiled +Code Interpreter binary. + +```bash +bash scripts/ci/prebuild_code_interpreter_linux_x86_64.sh responses +``` + +The script writes the bundled executable under: + +- `responses/python/vllm_responses/tools/code_interpreter/bin/linux/x86_64/code-interpreter-server` + +### Build wheel and sdist + +```bash +uv pip install -e './responses[build]' +cd responses +python -m build --wheel --sdist +``` + +Build artifacts are written to: + +- `responses/dist/` + +On Linux x86_64, wheels built after the prebuild step bundle the native Code +Interpreter binary. On other platforms, use the source-install Bun fallback or +disable Code Interpreter. + +## Run + +### remote-upstream gateway mode (`vllm-responses serve`) + +Prereqs: + +- If `code_interpreter` is enabled (default), the first start may download the Pyodide runtime (~400MB) into a cache + directory (see `VR_PYODIDE_CACHE_DIR`). This requires `tar` to be installed. +- For non-Linux platforms (or source installs without the bundled binary), you can disable the tool via + `--code-interpreter disabled`. For development you can also enable the Bun-based fallback via + `VR_CODE_INTERPRETER_DEV_BUN_FALLBACK=1`. + +External upstream (you start vLLM yourself; `/v1` is optional): + +```bash +vllm-responses serve --upstream http://127.0.0.1:8457 +``` + +The Responses endpoint is: + +- `POST http://127.0.0.1:5969/v1/responses` + +Remote access note: + +- If you bind the gateway with `--gateway-host 0.0.0.0`, use the machine’s IP/hostname to connect (not `0.0.0.0`). + +### integrated runtime (`vllm serve --responses`) + +Prereq: + +- install upstream `vllm` first, then install `vllm-responses` into the same environment + +Example: + +```bash +CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3.5-0.8B \ + --responses \ + --reasoning-parser qwen3 \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_coder \ + --host 0.0.0.0 \ + --port 8457 +``` + +CLI help: + +- `vllm serve --help` shows upstream vLLM help +- `vllm serve --responses --help` shows the Responses-owned integrated flags + +### Optional: ResponseStore hot cache (Redis) + +`previous_response_id` hydration reads the previous response state from the DB. For multi-worker deployments, you can optionally enable a Redis-backed hot cache to reduce DB reads/latency. + +Env vars (default off): + +- `VR_RESPONSE_STORE_CACHE=1` +- `VR_RESPONSE_STORE_CACHE_TTL_SECONDS=3600` + +Redis connection: + +- `VR_REDIS_HOST`, `VR_REDIS_PORT` + +## Quick smoke test (OpenAI Python SDK) + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://127.0.0.1:5969/v1", api_key="dummy") + +with client.responses.stream( + model="MiniMaxAI/MiniMax-M2.1", + input=[{"role": "user", "content": "You MUST call the code_interpreter tool. Execute: 2+2. Reply with ONLY the number."}], + tools=[{"type": "code_interpreter"}], + tool_choice="auto", + include=["code_interpreter_call.outputs"], +) as stream: + for evt in stream: + if getattr(evt, "type", "").endswith(".delta"): + continue + print(getattr(evt, "type", evt)) + r1 = stream.get_final_response().id + +with client.responses.stream( + model="MiniMaxAI/MiniMax-M2.1", + previous_response_id=r1, + input=[{"role": "user", "content": "What number did you just compute? Reply with ONLY the number."}], + tool_choice="none", +) as stream: + for evt in stream: + if getattr(evt, "type", "").endswith(".delta"): + continue + print(getattr(evt, "type", evt)) +``` diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..22c92b5 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,14 @@ +# Third-party notices + +This repository and its published artifacts may include or depend on third-party software. + +## Pyodide + +The built-in `code_interpreter` runtime uses Pyodide. + +- Project: Pyodide +- License: Mozilla Public License 2.0 (MPL-2.0) +- Source: https://github.com/pyodide/pyodide + +Pyodide is downloaded at runtime into a local cache directory (it is not bundled in v1 wheels). + diff --git a/docs/deployment/configuration.md b/docs/deployment/configuration.md new file mode 100644 index 0000000..f96dffc --- /dev/null +++ b/docs/deployment/configuration.md @@ -0,0 +1,260 @@ +# Configuration Guide + +Configure the gateway's database, caching, workers, and service architecture for your deployment needs. + +## Overview + +This guide covers configuration options for: + +- **Storage backend** (SQLite vs PostgreSQL) +- **Worker processes** (single vs multiple) +- **Response caching optimization** (optional Redis integration) +- **Service architecture** (single-command runtime vs disaggregated) + +For complete environment variable reference, see [Configuration Reference](../reference/configuration.md). + +## Storage Backend + +The gateway stores conversation state for `previous_response_id` functionality. Choose the storage backend that fits your deployment model. + +Stored continuation anchors include terminal responses with `status="completed"` and `status="incomplete"` (when `store=true`). + +### SQLite (Default) + +Zero-configuration storage using a local SQLite database file. + +```bash +# Default - no configuration needed +--8<-- "snippets/serve_external_upstream_cmd.txt" +``` + +**Characteristics:** + +- Zero setup required +- Single file database (`vllm_responses.db`) +- Works with multiple workers on the same machine (uses WAL mode) +- Does NOT work across multiple machines + +### PostgreSQL + +Required for multi-machine deployments and high-availability scenarios. + +```bash +export VR_DB_PATH="postgresql+asyncpg://user:password@db-host:5432/vllm_responses" +--8<-- "snippets/serve_external_upstream_cmd.txt" +``` + +**Migration notes:** When moving from SQLite to PostgreSQL: + +1. Set `VR_DB_PATH` to your PostgreSQL connection string +1. Restart the gateway - tables will be created automatically +1. Existing SQLite data will NOT be migrated + +______________________________________________________________________ + +## Worker Configuration + +Control gateway throughput by adjusting the number of worker processes. + +### Single Worker (Default) + +The default configuration runs one worker process. + +```bash +--8<-- "snippets/serve_external_upstream_cmd.txt" +``` + +**When this is sufficient:** + +- Local development +- Low to moderate traffic (\<100 concurrent requests) +- Testing and experimentation + +### Multiple Workers + +Increase concurrency by running multiple worker processes. + +```bash +vllm-responses serve --gateway-workers 4 --upstream http://127.0.0.1:8000/v1 +``` + +**What this does:** + +- Handles more concurrent requests +- Utilizes multiple CPU cores +- Each worker shares the same database + +**Compatibility notes:** + +- **SQLite:** Works fine with multiple workers on the same machine (uses WAL mode for concurrent access) +- **PostgreSQL:** Required for multiple workers across multiple machines (Kubernetes, multi-VM setups) + +### Upstream Readiness Controls + +Tune how long the supervisor waits for an external upstream to become ready. + +```bash +vllm-responses serve \ + --upstream http://127.0.0.1:8000/v1 \ + --upstream-ready-timeout 900 \ + --upstream-ready-interval 2 +``` + +Use these when the upstream has a slow cold start or when you want faster failure detection during rollout. + +______________________________________________________________________ + +## Response Caching Optimization (Optional) + +Add Redis caching to reduce database load for `previous_response_id` lookups. + +### Configuration + +```bash +export VR_RESPONSE_STORE_CACHE=1 +export VR_REDIS_HOST=localhost +export VR_REDIS_PORT=6379 +export VR_RESPONSE_STORE_CACHE_TTL_SECONDS=3600 # 1 hour + +--8<-- "snippets/serve_external_upstream_cmd.txt" +``` + +### How It Works + +Recent responses are cached in Redis. When a request includes `previous_response_id`, the gateway checks Redis first before querying the database. This significantly reduces database load and latency for active conversations. + +**Performance impact:** + +- Cache hits: fast retrieval +- Reduces database connection pool pressure +- Especially beneficial with PostgreSQL over network + +______________________________________________________________________ + +## MCP Configuration (Optional) + +Enable Built-in MCP by providing a runtime config file on the active entrypoint. + +### Minimal Setup + +```bash +vllm-responses serve \ + --upstream http://127.0.0.1:8000/v1 \ + --mcp-config /etc/vllm-responses/mcp.json +``` + +For `mcp.json` examples (URL + stdio styles), see +[MCP Examples -> Built-in MCP Runtime Config](../examples/hosted-mcp-examples.md#built-in-mcp-runtime-config-mcpjson). + +### Operational Notes + +- If `--mcp-config` is omitted, Built-in MCP is disabled. +- With `vllm-responses serve`, Built-in MCP runs in a singleton internal runtime process shared by all gateway workers. +- The supervisor injects `VR_MCP_BUILTIN_RUNTIME_URL` for gateway workers automatically. +- Built-in MCP startup and call timeouts are configured globally: + - `VR_MCP_HOSTED_STARTUP_TIMEOUT_SEC` + - `VR_MCP_HOSTED_TOOL_TIMEOUT_SEC` +- Runtime discovery endpoints: + - `GET /v1/mcp/servers` + - `GET /v1/mcp/servers/{server_label}/tools` + +### Remote MCP Gate + +Remote MCP declarations (`tools[].type="mcp"` with `server_url`) are enabled by default. + +```bash +export VR_MCP_REQUEST_REMOTE_ENABLED=false +``` + +When disabled, any Remote MCP declaration is rejected as a request-level policy error. Built-in MCP mode is unaffected. + +### Remote MCP URL Policy Checks + +Gateway URL policy checks for Remote MCP are enabled by default. + +```bash +export VR_MCP_REQUEST_REMOTE_URL_CHECKS=true +``` + +Set to `false` to bypass gateway-side URL validation checks. + +```bash +export VR_MCP_REQUEST_REMOTE_URL_CHECKS=false +``` + +Warning: disabling URL checks increases SSRF and unsafe-endpoint risk and should only be used in tightly controlled environments. + +## Service Architecture Patterns + +The gateway can run in different architectural configurations depending on your scaling and operational needs. + +### Integrated Single-Command Runtime + +Use `vllm serve --responses` when you want the colocated local stack on one public API server. + +```bash +vllm serve meta-llama/Llama-3.2-3B-Instruct --responses +``` + +**Components:** + +- vLLM API server +- Gateway routes mounted into the same FastAPI app +- Code interpreter helper runtime (optional) +- `web_search` built-in tool support (optional, when `--responses-web-search-profile` is set) +- Built-in MCP integration (optional, when `--responses-mcp-config` is set) + - runs as a loopback helper runtime when enabled + - shipped `web_search` profiles can also cause this helper runtime to be started automatically + +Integrated mode example with `web_search`: + +```bash +vllm serve meta-llama/Llama-3.2-3B-Instruct \ + --responses \ + --responses-web-search-profile exa_mcp +``` + +If the shipped `exa_mcp` profile should use an operator Exa key instead of the +anonymous default, set `EXA_API_KEY` in the gateway environment before startup. + +Integrated mode example with explicit Built-in MCP config: + +```bash +vllm serve meta-llama/Llama-3.2-3B-Instruct \ + --responses \ + --responses-mcp-config /etc/vllm-responses/mcp.json +``` + +### Remote-Upstream Gateway Mode + +Use `vllm-responses serve` when inference and gateway should remain separate. + +```bash +vllm-responses serve --upstream http://127.0.0.1:8000/v1 +``` + +**When to use:** + +- Separate scaling of inference and gateway +- Using existing vLLM infrastructure +- Avoiding model reload when restarting gateway + +______________________________________________________________________ + +## Configuration Quick Reference + +| Configuration | Command/Environment | +| ------------------------- | ------------------------------------------------------------------------ | +| **Database (PostgreSQL)** | `export VR_DB_PATH="postgresql+asyncpg://..."` | +| **Multiple workers** | `--gateway-workers 4` | +| **Redis cache** | `export VR_RESPONSE_STORE_CACHE=1` | +| **Built-in MCP config** | `--mcp-config /path/mcp.json` or `--responses-mcp-config /path/mcp.json` | +| **Remote MCP** | `export VR_MCP_REQUEST_REMOTE_ENABLED=false` | +| **Remote URL checks** | `export VR_MCP_REQUEST_REMOTE_URL_CHECKS=false` | +| **External vLLM** | `--upstream http://vllm:8000/v1` | + +______________________________________________________________________ + +## Next Steps + +- **For complete environment variables:** See [Configuration Reference](../reference/configuration.md) diff --git a/docs/deployment/observability.md b/docs/deployment/observability.md new file mode 100644 index 0000000..6343ae8 --- /dev/null +++ b/docs/deployment/observability.md @@ -0,0 +1,204 @@ +# Observability + +Monitor the gateway using Prometheus metrics and optional OpenTelemetry tracing. + +## Overview + +The gateway exposes operational data through: + +- **Prometheus metrics** at `GET /metrics` (enabled by default) +- **Health checks** at `GET /health` for load balancer probes +- **OpenTelemetry tracing** via OTLP export (optional, disabled by default) + +## Prometheus Metrics + +### Scraping Configuration + +The metrics endpoint is unauthenticated and available at: + +- **Endpoint**: `GET /metrics` (configurable via `VR_METRICS_PATH`) +- **Port**: Same as the active public API listener + - `vllm-responses serve`: default `5969` + - `vllm serve --responses`: default `8000` unless overridden by vLLM flags +- **Format**: Prometheus text exposition format + +If you change `VR_METRICS_PATH`, update your Prometheus `metrics_path` accordingly. + +Example Prometheus `scrape_configs`: + +```yaml +scrape_configs: + # Gateway metrics + - job_name: "vllm-responses" + static_configs: + - targets: ["gateway-host:5969"] + metrics_path: "/metrics" + scrape_interval: 15s + + # Also scrape vLLM metrics (if running locally) + - job_name: "vllm" + static_configs: + - targets: ["localhost:8457"] + metrics_path: "/metrics" + scrape_interval: 15s +``` + +!!! note "Also scrape vLLM" + + Remember to scrape your vLLM instance separately (it is a different service, with its own metrics). See [vLLM metrics documentation](https://docs.vllm.ai/en/stable/design/metrics/) for details. + +### Multi-Worker Support + +When running with multiple workers (`--gateway-workers N`), metrics are automatically aggregated across all workers when using `vllm-responses serve`. + +**Using `vllm-responses serve` (recommended):** + +Multi-process metrics are handled automatically. The supervisor creates a temporary directory for Prometheus multi-process mode and configures the workers correctly. + +We intentionally do not document manual multi-worker ASGI server setups here (Gunicorn/Uvicorn/etc.), because correct multi-process Prometheus aggregation requires additional lifecycle wiring (per-run multiprocess directories and worker-exit cleanup hooks). + +### Available Metrics + +All metrics use the `vllm_responses_` prefix. + +#### HTTP Metrics + +| Metric | Type | Labels | Description | +| ---------------------------------------------- | --------- | --------------------------- | -------------------------------------------------------------------- | +| `vllm_responses_http_requests_total` | Counter | `method`, `route`, `status` | Total HTTP requests completed | +| `vllm_responses_http_request_duration_seconds` | Histogram | `method`, `route` | HTTP handler duration (excludes SSE stream lifetime) | +| `vllm_responses_http_in_flight_requests` | Gauge | - | Requests currently being handled (does not include open SSE streams) | + +#### SSE Streaming Metrics + +| Metric | Type | Labels | Description | +| -------------------------------------------- | --------- | ------- | ------------------------------------------------------------------------------------------ | +| `vllm_responses_sse_connections_in_flight` | Gauge | - | SSE connections currently open | +| `vllm_responses_sse_stream_duration_seconds` | Histogram | `route` | Full SSE stream lifetime from request start (including time-to-first-chunk) to termination | + +SSE metrics capture the end-to-end streaming duration for the `/v1/responses` endpoint, which is the primary API path. + +#### Tool Metrics + +| Metric | Type | Labels | Description | +| ------------------------------------------------ | --------- | ------------------------ | ------------------------------------ | +| `vllm_responses_tool_calls_requested_total` | Counter | `tool_type` | Tool calls requested by the model | +| `vllm_responses_tool_calls_executed_total` | Counter | `tool_type` | Tool calls executed by the gateway | +| `vllm_responses_tool_execution_duration_seconds` | Histogram | `tool_type` | Tool execution wall-clock duration | +| `vllm_responses_tool_errors_total` | Counter | `tool_type` | Tool execution errors | +| `vllm_responses_mcp_server_startup_total` | Counter | `server_label`, `status` | Built-in MCP server startup outcomes | + +**Tool types:** + +- `function` - Client-executed function tools (request count only) +- `code_interpreter` - Gateway-executed code interpreter (request + execution/error metrics) +- `web_search` - Gateway-executed web search tool calls (request + execution/error metrics) +- `mcp` - MCP tool calls requested by the model (covers Built-in MCP and Remote MCP modes) + +Notes: + +- MCP metrics are not split by mode (Built-in MCP vs Remote MCP) in current metrics. +- `vllm_responses_mcp_server_startup_total` is hosted-only (there is no startup metric for Client-Specified Remote declarations). + +#### Metric Labels + +Labels are bounded to prevent cardinality explosion: + +- `method`: HTTP method (GET, POST, etc.) +- `route`: Route template (e.g., `/v1/responses`), not raw paths +- `status`: HTTP status code (200, 400, 500, etc.) +- `tool_type`: Tool category (`function`, `code_interpreter`, `web_search`, `mcp`) +- `server_label`: Built-in MCP server label configured in runtime config +- `status` (MCP startup metric): `ok` or `error` + +## OpenTelemetry Tracing + +Tracing is disabled by default. Enable it for distributed tracing of requests through the gateway. + +### Installation (optional dependencies) + +OpenTelemetry tracing requires the `tracing` optional dependencies. + +### Configuration + +Set the following environment variables: + +```bash +# Enable tracing +export VR_TRACING_ENABLED=true + +# Sampling ratio (0.0 to 1.0) +export VR_TRACING_SAMPLE_RATIO=0.01 + +# OTLP endpoint (gRPC) +export VR_OPENTELEMETRY_HOST=otel-collector +export VR_OPENTELEMETRY_PORT=4317 + +# Service name in traces +export VR_OTEL_SERVICE_NAME=vllm_responses +``` + +### What Gets Traced + +When enabled, the gateway instruments: + +- **Inbound HTTP requests** (FastAPI/ASGI) +- **Outbound HTTP to vLLM** (HTTPX client) + +### Prerequisites + +You need an OTLP-compatible collector or backend to receive traces: + +- [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) +- [Jaeger](https://www.jaegertracing.io/) +- [Tempo](https://grafana.com/docs/tempo/) +- Any other OTLP gRPC-compatible backend + +Example minimal OpenTelemetry Collector configuration: + +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +exporters: + jaeger: + endpoint: jaeger:14250 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [jaeger] +``` + +## Health Checks + +The gateway exposes a health endpoint for load balancer health probes: + +- **Endpoint**: `GET /health` +- **Response**: `200 OK` with empty JSON object `{}` +- **Authentication**: None + +Example: + +```bash +curl http://localhost:5969/health +``` + +In integrated mode, use the `vllm serve` host/port instead (for example `http://localhost:8000/health` with defaults). + +Use this endpoint for: + +- Kubernetes liveness and readiness probes +- AWS ALB/NLB health checks +- Docker health checks +- Load balancer health monitoring + +## Configuration Reference + +For a complete list of observability environment variables and their defaults, see the [Configuration Reference](../reference/configuration.md). diff --git a/docs/examples/code-interpreter-examples.md b/docs/examples/code-interpreter-examples.md new file mode 100644 index 0000000..4cc908a --- /dev/null +++ b/docs/examples/code-interpreter-examples.md @@ -0,0 +1,95 @@ +# Code Interpreter Examples + +Use the built-in Code Interpreter to perform calculations, data analysis, and more. + +## Prerequisite + +Ensure your client request includes the tool definition and `include` parameter: + +```python +# Common setup for all examples +client = OpenAI(...) +kwargs = { + "model": "meta-llama/Llama-3.2-3B-Instruct", + "tools": [{"type": "code_interpreter"}], + "include": ["code_interpreter_call.outputs"] +} +``` + +### Output semantics (what you’ll see in `code_interpreter_call.outputs`) + +When `include=["code_interpreter_call.outputs"]` is present, the gateway populates `outputs` with up to two log entries: + +1. Aggregated stdout/stderr (e.g. everything from `print(...)`). +1. The final expression display value (if the last statement is a bare expression). + +If your code ends with `print(...)` and no final expression, you’ll typically only see the first logs entry. + +## 1. Mathematical Calculation + +The model can solve math problems by writing Python code, which avoids the common arithmetic errors LLMs make. + +**Prompt:** "Calculate the factorial of 50." + +**Model Execution:** + +```python +import math +print(math.factorial(50)) +``` + +**Output:** + +```text +30414093201713378043612608166064768844377641568960512000000000000 +``` + +## 2. Data Analysis with Pandas + +The environment includes `pandas` and `numpy`. + +**Prompt:** "Create a DataFrame with 5 random numbers and calculate their mean." + +**Model Execution:** + +```python +import pandas as pd +import numpy as np + +df = pd.DataFrame({'values': np.random.rand(5)}) +print(f"Mean: {df['values'].mean()}") +print(df) +``` + +## 3. String Processing + +Use Python's powerful string manipulation capabilities. + +**Prompt:** "Reverse the string 'Hello World' and count the vowels." + +**Model Execution:** + +```python +s = "Hello World" +reversed_s = s[::-1] +vowel_count = sum(1 for char in s.lower() if char in 'aeiou') +print(f"Reversed: {reversed_s}") +print(f"Vowels: {vowel_count}") +``` + +## 4. HTTP Requests + +The sandbox allows HTTP requests (via a proxy) using `httpx`. + +**Prompt:** "Fetch the current time from an API." + +**Model Execution:** + +```python +import httpx +r = httpx.get("https://worldtimeapi.org/api/ip") +data = r.json() +print(f"Current time: {data['datetime']}") +``` + +(Note: Network access depends on your deployment configuration.) diff --git a/docs/examples/hosted-mcp-examples.md b/docs/examples/hosted-mcp-examples.md new file mode 100644 index 0000000..6e1c007 --- /dev/null +++ b/docs/examples/hosted-mcp-examples.md @@ -0,0 +1,107 @@ +# MCP Examples (Built-in MCP + Remote MCP) + +Use MCP tools through the Responses API in either Built-in MCP mode or Remote MCP mode. + +## Built-in MCP Runtime Config (`mcp.json`) + +Pass `--mcp-config` (or `--responses-mcp-config` in integrated mode) to point to an MCP runtime config file: + +```bash +vllm-responses serve \ + --upstream http://127.0.0.1:8000/v1 \ + --mcp-config /etc/vllm-responses/mcp.json +``` + +Expected shape: top-level `mcpServers`, with each key as your `server_label` and each value as one MCP server entry. +This is intentionally close to common MCP client config formats, so you can usually copy existing entries directly. + +Canonical example (`url` + `stdio` styles): + +--8<-- "snippets/mcp_runtime_config_example.txt" + +## Built-in MCP: Discover Available Servers and Tools + +Before sending tool requests, inspect runtime availability: + +Start with a supported entrypoint that enables Built-in MCP: + +- `vllm-responses serve --mcp-config ...` +- `vllm serve --responses --responses-mcp-config ...` + +```bash +--8<-- "snippets/mcp_discover_servers_tools_curl.txt" +``` + +The snippet defaults to `http://127.0.0.1:5969`. In integrated mode, set +`VLLM_RESPONSES_HTTP_BASE=http://127.0.0.1:8000` first, or replace the base URL with your +configured `vllm serve` address. + +## Built-in MCP: Force an MCP Tool Call + +Canonical request payload: + +--8<-- "snippets/mcp_builtin_request_payload.txt" + +Python SDK equivalent: + +```python +response = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + stream=True, + input=[{"role": "user", "content": "Find migration notes in docs."}], + tools=[ + { + "type": "mcp", + "server_label": "github_docs", + "allowed_tools": ["search_docs"], + "require_approval": "never", + } + ], + tool_choice={ + "type": "mcp", + "server_label": "github_docs", + "name": "search_docs", + }, +) +``` + +Expected stream events include: + +- `response.mcp_call.in_progress` +- `response.mcp_call_arguments.delta` +- `response.mcp_call_arguments.done` +- `response.mcp_call.completed` or `response.mcp_call.failed` + +## Continue with `previous_response_id` + +If `store=true` (default), the response ID from a terminal response can be reused: + +```python +follow_up = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + previous_response_id=response.id, + input=[{"role": "user", "content": "Summarize that in one sentence."}], +) +``` + +## Remote MCP Mode: Quick Example + +Use Remote MCP mode when you want to declare an MCP endpoint directly in the request instead of using Built-in MCP registry config. + +```python +remote = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[{"role": "user", "content": "Search remote docs for migration notes."}], + tools=[ + { + "type": "mcp", + "server_label": "docs_remote", + "server_url": "https://mcp.example.com/sse", + "authorization": "YOUR_TOKEN", + "allowed_tools": ["search_docs"], + "require_approval": "never", + } + ], + tool_choice={"type": "mcp", "server_label": "docs_remote", "name": "search_docs"}, +) +``` diff --git a/docs/examples/stateful-conversations.md b/docs/examples/stateful-conversations.md new file mode 100644 index 0000000..90dfb19 --- /dev/null +++ b/docs/examples/stateful-conversations.md @@ -0,0 +1,70 @@ +# Stateful Conversation Examples + +Use `previous_response_id` to maintain conversation history efficiently. + +## Basic Chat + +This example simulates a conversation between a user and an assistant. + +### Turn 1: User Greeting + +```python +# Initial request +response_1 = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[ + {"role": "system", "content": "You are a helpful pirate."}, + {"role": "user", "content": "Hello!"} + ] +) + +print(response_1.output[0].content) +# "Ahoy there, matey!" +``` + +### Turn 2: User Follow-up + +For the next turn, we **do not** send the previous messages. We only send the new input and the ID of the last response. + +```python +response_2 = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + previous_response_id=response_1.id, + input=[{"role": "user", "content": "Where is the treasure?"}] +) + +print(response_2.output[0].content) +# "X marks the spot on the map!" +``` + +### Turn 3: Assistant Continues + +We continue the chain using `response_2.id`. + +```python +response_3 = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + previous_response_id=response_2.id, + input=[{"role": "user", "content": "Can you show me the map?"}] +) + +print(response_3.output[0].content) +# "Aye, here it be..." +``` + +## Changing Instructions + +You can change the system prompt mid-conversation. The `instructions` parameter (system prompt) is **not** carried over by `previous_response_id`. You must provide it if you want it to persist or change. + +```python +# Change persona to a ninja +response_4 = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + previous_response_id=response_3.id, + input=[{"role": "user", "content": "Wait, be quiet now."}], + instructions="You are a stealthy ninja." +) + +print(response_4.output[0].content) +# "(Whispers) Understood. I vanish into the shadows." +``` diff --git a/docs/examples/tool-loop-examples.md b/docs/examples/tool-loop-examples.md new file mode 100644 index 0000000..f3bfbd8 --- /dev/null +++ b/docs/examples/tool-loop-examples.md @@ -0,0 +1,90 @@ +# Custom Tool Loop Examples + +How to implement the client-side tool loop for custom functions. + +## Weather Example + +This example demonstrates a complete cycle: Model requests tool -> Client executes -> Model uses result. + +### 1. Define the Tool + +First, tell the model about the tool. + +```python +tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } +] +``` + +### 2. Initial Request + +```python +response = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[{"role": "user", "content": "What's the weather in Boston?"}], + tools=tools, + tool_choice="auto", +) + +# Store the response ID for the next step +response_id = response.id +``` + +### 3. Handle Tool Call + +Check if the model wants to call a function. + +```python +output_item = response.output[0] + +if output_item.type == "function_call": + tool_name = output_item.name + tool_args = output_item.arguments + call_id = output_item.call_id + + print(f"Model called {tool_name} with {tool_args}") + + # In a real app, you would parse args and call your API + # function_response = my_weather_api(location="Boston") + function_response = '{"temperature": "22", "unit": "celsius", "description": "Sunny"}' +``` + +### 4. Submit Result + +Send the tool output back to the model using `previous_response_id`. + +```python +final_response = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + previous_response_id=response_id, + input=[ + { + "type": "function_call_output", + "call_id": call_id, + "output": function_response + } + ] +) + +print(final_response.output[0].content) +# "The weather in Boston is 22°C and sunny." +``` + +For MCP examples, see [MCP Examples](hosted-mcp-examples.md). diff --git a/docs/examples/web-search-examples.md b/docs/examples/web-search-examples.md new file mode 100644 index 0000000..f763d7b --- /dev/null +++ b/docs/examples/web-search-examples.md @@ -0,0 +1,85 @@ +# Web Search Examples + +Use the gateway-owned `web_search` built-in to let the model search the web, +open pages, and reuse page text inside the same request. + +## Prerequisite + +Start a supported entrypoint with a shipped profile: + +### `vllm-responses serve` + +```bash +export EXA_API_KEY="your-exa-api-key" # optional for exa_mcp +vllm-responses serve \ + --upstream http://127.0.0.1:8000/v1 \ + --web-search-profile exa_mcp +``` + +### `vllm serve --responses` + +```bash +export EXA_API_KEY="your-exa-api-key" # optional for exa_mcp +vllm serve meta-llama/Llama-3.2-3B-Instruct \ + --responses \ + --responses-web-search-profile exa_mcp +``` + +If `EXA_API_KEY` is omitted, the shipped `exa_mcp` helper uses the anonymous +default Exa MCP URL. + +## 1. Minimal Request + +```python +response = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[{"role": "user", "content": "Find the official vLLM documentation site."}], + tools=[{"type": "web_search"}], +) +``` + +## 2. Return Search Sources + +Add `web_search_call.action.sources` to `include` when you want the final +search action sources expanded in the response item. + +```python +response = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[{"role": "user", "content": "Find migration notes for vLLM and cite the sources."}], + tools=[{"type": "web_search"}], + include=["web_search_call.action.sources"], +) +``` + +## 3. Streaming Web Search Events + +```python +with client.responses.stream( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[{"role": "user", "content": "Search for the latest vLLM release notes."}], + tools=[{"type": "web_search"}], + include=["web_search_call.action.sources"], +) as stream: + for event in stream: + print(event) +``` + +Expected event families include: + +- `response.web_search_call.in_progress` +- `response.web_search_call.searching` +- `response.web_search_call.completed` + +## 4. Search Then Reuse Page Text + +`find_in_page` is backed by request-local cached page text. In practice, that +means the model must first open a page before it can search inside it during +the same request. + +Prompt pattern: + +```text +Search for the vLLM migration notes, open the most relevant page, then find the +section mentioning breaking changes. +``` diff --git a/docs/features/built-in-tools.md b/docs/features/built-in-tools.md new file mode 100644 index 0000000..5d75ad2 --- /dev/null +++ b/docs/features/built-in-tools.md @@ -0,0 +1,106 @@ +# Built-in Tools + +The gateway can execute certain built-in tools on your behalf and stream the +results back through the Responses API. + +Current built-in tools: + +- **Code Interpreter** +- **Web Search** + +For the dedicated `web_search` guide, see [Web Search](web-search.md). + +## Code Interpreter + +The Code Interpreter allows the model to write and execute Python code in a sandboxed environment. This is useful for: + +- Mathematical calculations +- Data analysis and visualization +- String manipulation +- Solving logic puzzles + +### Enabling the Tool + +To use the code interpreter, you must: + +1. Include it in the `tools` list with type `code_interpreter`. +1. (Optional but recommended) Add `code_interpreter_call.outputs` to the `include` list if you want to receive: + - the captured stdout/stderr stream (e.g. `print(...)` output), and + - the final expression display value (if any), in the response object. + +```python +response = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[{"role": "user", "content": "Calculate the 10th Fibonacci number."}], + tools=[{"type": "code_interpreter"}], + include=["code_interpreter_call.outputs"] +) +``` + +### Response Structure + +When the model uses the code interpreter, the response will contain a `code_interpreter_call` output item. + +```json +{ + "type": "code_interpreter_call", + "id": "ci_123", + "container_id": "pyodide-worker-1", + "code": "def fib(n):...", + "status": "completed", + "outputs": [ + { + "type": "logs", + "logs": "P1\nP2\n" + }, + { + "type": "logs", + "logs": "6" + } + ] +} +``` + +**Output types:** + +- **Logs**: `{ "type": "logs", "logs": "stdout/stderr text" }` +- **Images**: `{ "type": "image", "url": "data:image/png;base64,..." }` + +### Streaming Execution + +One of the biggest benefits of the built-in runtime is **streaming**. You receive events _while the code is running_. + +1. `response.code_interpreter_call.in_progress`: The tool call has started. +1. `response.code_interpreter_call_code.delta`: The model is writing the code. +1. `response.code_interpreter_call.interpreting`: The code is finished and is now executing. +1. `response.code_interpreter_call.completed`: Execution finished. +1. `response.output_item.done`: The item is finalized, including outputs. + +### Security and Sandboxing + +The code interpreter runs in a sandboxed environment: + +- **Runtime**: [Pyodide](https://pyodide.org/) (Python compiled to WebAssembly) running inside a local Code Interpreter + service. On Linux x86_64 wheels, the server is bundled as a native binary; for development/source installs it can run + via [Bun](https://bun.sh/). +- **Isolation**: Runs in a WebAssembly sandbox with no direct host file system access. +- **Network Access**: HTTP requests are available via `httpx` (useful for API calls, data fetching). +- **Resource Limits**: Execution time is capped (configurable via startup flags). + +!!! note "First start download" + + If the tool is enabled, the first start may download the Pyodide runtime (~400MB) into a cache directory and extract + it. You can control the cache location via `VR_PYODIDE_CACHE_DIR`. + +!!! note "Concurrency" + + If you need more code-interpreter throughput, you can configure a worker pool for the Code Interpreter service via + `--code-interpreter-workers` under `vllm-responses serve`, or + `--responses-code-interpreter-workers` in integrated mode. This uses [Bun Workers + **experimental**](https://bun.com/docs/runtime/workers). Use `2+` for actual parallelism; `1` enables worker mode + but does not increase throughput. Each worker loads its own Pyodide runtime, so higher worker counts increase RAM + usage and startup time. + +!!! warning "Production Deployment" + + While the sandbox provides isolation, running arbitrary code from an LLM always carries risks. Ensure your deployment environment is properly secured and monitored. diff --git a/docs/features/hosted-mcp.md b/docs/features/hosted-mcp.md new file mode 100644 index 0000000..f491093 --- /dev/null +++ b/docs/features/hosted-mcp.md @@ -0,0 +1,131 @@ +# MCP Integration (Built-in MCP + Remote MCP) + +The gateway supports two MCP declaration modes in `tools`: + +- Built-in MCP mode: reference a configured server by `server_label`. +- Remote MCP mode: provide request `server_url` (and request-scoped auth/headers). + +This page focuses on Built-in MCP setup and call flow, then summarizes Remote MCP mode differences. + +## Choose a Mode + +| Mode | Best When | Request Shape | +| ------------ | --------------------------------------------------------- | --------------------------------------------- | +| Built-in MCP | You want centrally managed server inventory and policy | `type: "mcp"` + `server_label` | +| Remote MCP | You want to point directly to an MCP endpoint per request | `type: "mcp"` + `server_label` + `server_url` | + +## What It Solves + +- Keep MCP execution inside the gateway request lifecycle. +- Use Responses-style streaming events for MCP call progress/results. +- Reuse response IDs with `previous_response_id` just like other tool flows. + +## Prerequisites + +1. Configure MCP runtime servers via `--mcp-config` (or `--responses-mcp-config` in integrated mode). +1. Ensure the target `server_label` is available (`GET /v1/mcp/servers`). +1. Start the gateway with either: + - `vllm-responses serve --mcp-config ...`, or + - `vllm serve --responses --responses-mcp-config ...` + +## Built-in MCP Setup + +Pass the Built-in MCP config path on the entrypoint CLI: + +```bash +vllm-responses serve \ + --upstream http://127.0.0.1:8000/v1 \ + --mcp-config /etc/vllm-responses/mcp.json +``` + +`mcp.json` follows the common MCP client-style shape: a top-level `mcpServers` object keyed by your server labels. +In most cases, you can copy an MCP server entry from another MCP client config and reuse it here with minimal changes. +For canonical examples (URL + stdio styles), see [MCP Examples -> Built-in MCP Runtime Config](../examples/hosted-mcp-examples.md#built-in-mcp-runtime-config-mcpjson). + +Built-in URL-style entries accept both `http://` and `https://` URLs. This differs from Remote MCP request URLs, which are policy-checked as `https://` by default. + +Verify server availability before requests: + +```bash +--8<-- "snippets/mcp_discover_servers_tools_curl.txt" +``` + +For integrated mode, set `VLLM_RESPONSES_HTTP_BASE=http://127.0.0.1:8000` first (or use your +custom `vllm serve` host/port). + +Runtime architecture note: + +- `vllm-responses serve` starts one internal Built-in MCP runtime process on loopback. +- `vllm serve --responses --responses-mcp-config ...` also starts one loopback Built-in MCP helper for the combined app. +- All gateway workers share that runtime, so Built-in MCP startup/discovery/session state is not duplicated per worker. + +## Built-in MCP Usage + +Use one complete request payload including both MCP declaration and tool choice: + +--8<-- "snippets/mcp_builtin_request_payload.txt" + +### cURL + +```bash +curl -X POST http://127.0.0.1:5969/v1/responses \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer dummy" \ + -d '{ + "model": "meta-llama/Llama-3.2-3B-Instruct", + "stream": true, + "input": [{"role":"user","content":"Find migration notes in docs."}], + "tools": [{"type":"mcp","server_label":"github_docs","allowed_tools":["search_docs"],"require_approval":"never"}], + "tool_choice": {"type":"mcp","server_label":"github_docs","name":"search_docs"} + }' +``` + +If you are using integrated mode, replace `http://127.0.0.1:5969` with your `vllm serve` +base URL (default `http://127.0.0.1:8000`). + +### OpenAI Python SDK + +```python +--8<-- "snippets/openai_client_local_gateway.py" + +# For integrated mode, set: +# export VLLM_RESPONSES_BASE_URL=http://127.0.0.1:8000/v1 + +with client.responses.stream( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[{"role": "user", "content": "Find migration notes in docs."}], + tools=[ + { + "type": "mcp", + "server_label": "github_docs", + "allowed_tools": ["search_docs"], + "require_approval": "never", + } + ], + tool_choice={"type": "mcp", "server_label": "github_docs", "name": "search_docs"}, +) as stream: + for event in stream: + print(event.type) +``` + +## MCP Event Lifecycle + +Both MCP modes stream these event types: + +- `response.mcp_call.in_progress` +- `response.mcp_call_arguments.delta` +- `response.mcp_call_arguments.done` +- `response.mcp_call.completed` or `response.mcp_call.failed` + +See [Events Reference](../reference/events.md) for payload details. + +## Remote MCP Mode Notes + +- Built-in MCP requests reference configured servers by `server_label` only. +- Remote MCP via request `server_url` does not require server registration in the Built-in MCP config file. +- Remote MCP transport selection is delegated to FastMCP from request `server_url` and headers. +- `require_approval` currently supports `never` only. +- Remote MCP host policy rejects `localhost`, `*.localhost`, and IP-literal hosts, and only `https` is accepted. +- For Remote MCP field compatibility (`server_url`, `connector_id`, `headers`), see [API Reference](../reference/api-reference.md#mcp-compatibility-matrix-current). + +For end-to-end examples, see [MCP Examples](../examples/hosted-mcp-examples.md). diff --git a/docs/features/statefulness.md b/docs/features/statefulness.md new file mode 100644 index 0000000..e81e843 --- /dev/null +++ b/docs/features/statefulness.md @@ -0,0 +1,64 @@ +# Stateful Conversations + +One of the most powerful features of the Responses API is the ability to maintain conversation state on the server. + +## Overview + +In the standard Chat Completions API, the client is responsible for managing the entire conversation history. Every new request must include the full list of previous messages (`messages=[...]`). This is bandwidth-intensive and requires complex client-side state management. + +The Responses API introduces **Statefulness** via the `previous_response_id` parameter. + +## How It Works + +1. **Initial Request**: You send a request with your initial input (e.g., a user message). +1. **Storage**: The gateway generates a response and stores the _entire conversation context_ (including your input and its output) in its database for final responses (`completed` and `incomplete`, when `store=true`). +1. **Continuation**: When you want to reply, you send _only_ your new input and the `previous_response_id` from the last response. +1. **Rehydration**: The gateway looks up the previous response, reconstructs the full history, and sends it to the model. + +### Example Flow + +#### Step 1: Start the conversation + +```python +# No previous ID provided +response_1 = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[{"role": "user", "content": "My name is Alice."}] +) + +print(response_1.output[0].content) +# "Hello Alice!" + +print(response_1.id) +# "resp_01J..." +``` + +#### Step 2: Continue the conversation + +```python +# Pass the ID from Step 1 +response_2 = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + previous_response_id=response_1.id, + input=[{"role": "user", "content": "What is my name?"}] +) + +print(response_2.output[0].content) +# "Your name is Alice." +``` + +## Storage Backends + +Statefulness is powered by the **ResponseStore**. + +- **Development**: By default, `vLLM Responses` uses a local **SQLite** database (`vllm_responses.db`). This works great for local setups and single-machine deployments. +- **Production**: For multi-machine deployments or high-traffic production, you should configure a **PostgreSQL** database. + +See [Configuration Reference](../reference/configuration.md) and [Configuration Guide](../deployment/configuration.md) for details. + +## Security & Lifecycle + +- **Capability-Based Access**: The `response_id` acts as a capability token. Anyone who possesses the ID can continue the conversation. Treat these IDs as secrets (like session tokens). +- **Persistence**: By default, responses are stored indefinitely (or until an expiration policy is configured/implemented). +- **`store` Parameter**: You can control whether a response is stored using the `store` parameter (default: `true`). If `store=false`, the response is not persisted and cannot be retrieved later or used as a `previous_response_id`. +- **Terminal Statuses**: Stored terminal responses include both `completed` and `incomplete`. Non-terminal and failed states are not continuation anchors. diff --git a/docs/features/web-search.md b/docs/features/web-search.md new file mode 100644 index 0000000..0e61351 --- /dev/null +++ b/docs/features/web-search.md @@ -0,0 +1,117 @@ +# Web Search + +Use the gateway-owned `web_search` built-in when you want the model to search +the web, open a page, or search inside a page it already opened during the same +request. + +`web_search` is one public built-in tool: + +```json +{"type": "web_search"} +``` + +The gateway owns how that public tool is realized internally. Operators enable +one shipped profile at startup, and clients continue to use the same public +tool shape regardless of which profile is active. + +## Shipped Profiles + +Current shipped profiles: + +- `exa_mcp` +- `duckduckgo_plus_fetch` + +### `exa_mcp` + +- Uses Exa-backed MCP tools for search and page opening. +- Requires the Built-in MCP runtime, but the shipped helper entry is provisioned + automatically when the profile is enabled. +- If `EXA_API_KEY` is set in the gateway environment, the shipped Exa MCP entry + appends it automatically to the Exa MCP URL. + +### `duckduckgo_plus_fetch` + +- Uses DuckDuckGo for search and the Fetch MCP server for page opening. +- Also requires the Built-in MCP runtime, with the shipped Fetch helper entry + provisioned automatically when the profile is enabled. + +## Principles + +- `web_search` stays one public built-in tool even though the gateway may + realize it internally with multiple actions. +- Profile selection is an operator decision made at startup, not a per-request + client parameter. +- The gateway keeps backend/provider details out of the public Responses API + request shape. +- `find_in_page` works over request-local cached page text from a prior + `open_page` result in the same request. + +## Enable the Tool + +### `vllm-responses serve` + +```bash +vllm-responses serve \ + --upstream http://127.0.0.1:8000/v1 \ + --web-search-profile exa_mcp +``` + +### `vllm serve --responses` + +```bash +vllm serve meta-llama/Llama-3.2-3B-Instruct \ + --responses \ + --responses-web-search-profile exa_mcp +``` + +Notes: + +- If the profile flag is omitted, `web_search` is disabled. +- For supported entrypoints, web-search enablement is CLI-owned. +- Shipped profiles that need Built-in MCP helper servers provision their + default helper entries automatically. You do not need `--mcp-config` just to + enable a shipped `web_search` profile. + +## Use the Tool + +Minimal Python SDK example: + +```python +response = client.responses.create( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[{"role": "user", "content": "Find the latest migration notes for vLLM."}], + tools=[{"type": "web_search"}], +) +``` + +When the model uses the tool, the response contains a `web_search_call` output +item. + +If you want source expansion on search results, add: + +```python +include=["web_search_call.action.sources"] +``` + +Streaming responses emit the `web_search_call` lifecycle family, including: + +- `response.web_search_call.in_progress` +- `response.web_search_call.searching` +- `response.web_search_call.completed` + +## Current Behavior and Limitations + +- Profile selection happens at startup. Requests do not choose between shipped + profiles. +- Current shipped profiles are limited to `exa_mcp` and + `duckduckgo_plus_fetch`. +- Completed `web_search_call` output items follow normal Responses storage + behavior when `store=true`. +- `find_in_page` depends on page text cached from an earlier `open_page` + action in the same request, and that page cache is not persisted across + requests or `previous_response_id` continuations. +- `include=["web_search_call.action.sources"]` controls source expansion for + search results. +- Built-in MCP auto-provision covers the shipped helper defaults. Use + `--mcp-config` or `--responses-mcp-config` when you also need extra MCP + inventory or explicit overrides. diff --git a/docs/getting-started/architecture.md b/docs/getting-started/architecture.md new file mode 100644 index 0000000..619dea1 --- /dev/null +++ b/docs/getting-started/architecture.md @@ -0,0 +1,83 @@ +# Architecture + +Understand how `vLLM Responses` bridges the gap between the OpenAI Responses API and vLLM. + +## Overview + +At its core, `vLLM Responses` is a translation layer (or "gateway"). It sits between your client application and your vLLM inference server, adding statefulness, built-in tool execution (including Code Interpreter and `web_search`), MCP integration (Built-in MCP and Remote MCP), and spec-compliant streaming. + +```mermaid +sequenceDiagram + participant Client + participant Gateway as vLLM Responses
Gateway + participant DB as ResponseStore
(Database) + participant vLLM as vLLM Server + participant CI as Code Interpreter
Runtime + participant MCP as MCP Runtime/Server
(Built-in/Remote) + + Client->>Gateway: POST /v1/responses
(with previous_response_id) + Gateway->>DB: Load conversation history + DB-->>Gateway: Previous context + Gateway->>vLLM: POST /v1/chat/completions
(with full history) + vLLM-->>Gateway: Streaming chunks + + opt If model requests code_interpreter + Gateway->>CI: Execute code + CI-->>Gateway: Execution results + Gateway->>vLLM: Continue with results + vLLM-->>Gateway: Final response + end + + opt If model requests MCP tool + Gateway->>MCP: Execute tool call + MCP-->>Gateway: Tool output/error + Gateway->>vLLM: Continue with MCP result + vLLM-->>Gateway: Final response + end + + Gateway-->>Client: SSE: Responses events + Gateway->>DB: Store response state +``` + +### Request Flow + +1. **Client** sends a request to `/v1/responses`, optionally including a `previous_response_id` to continue a conversation. +1. **Gateway** rehydrates conversation history from the **ResponseStore** (database) if a `previous_response_id` is provided. +1. **Gateway** transforms the Responses request into a Chat Completions request and sends it to **vLLM** with the full conversation history. +1. **vLLM** generates tokens and streams them back to the gateway as Chat Completions chunks. +1. **Gateway** normalizes vLLM output into a stable internal event stream and composes spec-compliant Responses SSE events. +1. If the model requests a **gateway-executed tool** (for example Code Interpreter, `web_search`, or an MCP tool): + - The gateway executes the tool call (locally for built-ins, or via Built-in MCP / Remote MCP transport). + - Results are fed back to vLLM to continue generation. + - All of this happens within a single API request. +1. **Gateway** streams Responses events back to the client in real-time. +1. **Gateway** persists terminal storable response state (`completed` and `incomplete`, when `store=true`) to the database for future `previous_response_id` lookups. + +______________________________________________________________________ + +## Key Concepts + +### The Responses API + +The [OpenAI Responses API](https://www.openresponses.org/specification) is a newer, richer protocol than Chat Completions. It treats a "Response" as a first-class object that can contain multiple output items (messages, tool calls, thinking blocks). This gateway implements the OpenResponses specification, ensuring compatibility with OpenAI's Responses API and other compliant providers. + +### Statefulness + +Unlike standard Chat Completions where you must send the entire conversation history with every request, the Responses API allows for **stateful conversations**. + +- You send an initial request. +- The gateway returns a `response.id`. +- For the next turn, you simply pass `previous_response_id="..."` along with your new input. +- The gateway rehydrates the full context from its **ResponseStore**. + +This significantly reduces bandwidth and complexity for client applications. + +### Built-in Tools + +The gateway includes runtimes for **built-in tools**. Current examples are the **Code Interpreter** and **`web_search`**. When the model decides to use one of these tools, the gateway executes it and returns the results within the same API request lifecycle. + +For `web_search`, the gateway resolves a startup-selected profile, executes search and page-opening actions behind the public `{"type": "web_search"}` tool shape, and keeps `find_in_page` page text in a request-local cache. + +### MCP Integration + +The gateway can execute MCP tools declared in the request. Built-in MCP uses configured `server_label` inventory via a singleton internal runtime process shared by gateway workers; Remote MCP uses request `server_url`. Both stay in the same Responses lifecycle with consistent `mcp_call` events and output items. diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 0000000..2ab44a6 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,107 @@ +# Installation + +Get started with `vLLM Responses` by installing the package and its dependencies. + +## Prerequisites + +- **Python 3.12+**: Ensure you have a compatible Python version installed. +- **uv** (Recommended): We recommend using [uv](https://github.com/astral-sh/uv) for fast, reliable dependency management. +- **tar**: If you use the built-in **Code Interpreter** (enabled by default), the first start may download the Pyodide + runtime (~400MB) and extract it; `tar` must be available. +- **Bun** (Development): Required for source checkouts if you want the built-in Code Interpreter to work (enabled by + default). Wheels for Linux x86_64 bundle a native Code Interpreter binary and do not require Bun. +- **vLLM**: Required if you want the integrated colocated runtime via `vllm serve --responses`. + +## Install the CLI + +We recommend setting up a virtual environment using `uv`. + +### Install from a prebuilt wheel (Linux x86_64) (Recommended) + +Download a prebuilt wheel (`vllm_responses-*.whl`) from GitHub Releases (preferred) or a CI run artifact, then install it: + +```bash +uv venv --python=3.12 +source .venv/bin/activate +uv pip install path/to/vllm_responses-*.whl +vllm-responses --help +``` + +On Linux x86_64 wheels, the Code Interpreter server binary is bundled, so **Bun is not required**. + +!!! note "Non-Linux platforms" + + The gateway is a Python service and can run on other platforms, but the bundled Code Interpreter binary is currently + only shipped in Linux x86_64 wheels. On other platforms, either disable the tool via `--code-interpreter disabled`, + or run from a source checkout and use the (development-only) Bun fallback. + +### Install from source (repo checkout) + +If you are working from a source checkout and want the gateway to work with the default configuration (Code Interpreter +enabled), use the Bun fallback: + +```bash +git clone https://github.com/EmbeddedLLM/vllm-responses +cd vllm-responses + +uv venv --python=3.12 +source .venv/bin/activate +uv pip install -e ./responses + +cd responses/python/vllm_responses/tools/code_interpreter +bun install +export VR_CODE_INTERPRETER_DEV_BUN_FALLBACK=1 +cd - + +vllm-responses --help +``` + +### First start: Pyodide download (Code Interpreter) + +If `code_interpreter` is enabled (default), the first start may download the Pyodide runtime (~400MB) into a cache +directory and extract it. Subsequent starts reuse the cache. + +- Default cache: `${XDG_CACHE_HOME:-$HOME/.cache}/vllm-responses/pyodide` +- Override: set `VR_PYODIDE_CACHE_DIR` to a persistent directory with enough free disk space. + +## Build a wheel from a source checkout + +If you want to produce a local wheel from this repo, build from the +`responses/` package directory. + +### Rebuild the bundled Code Interpreter binary (Linux x86_64 only) + +This step is only needed when you want the built wheel to include a freshly +compiled native Code Interpreter binary. + +```bash +bash scripts/ci/prebuild_code_interpreter_linux_x86_64.sh responses +``` + +### Build wheel and sdist + +```bash +uv pip install -e './responses[build]' +cd responses +python -m build --wheel --sdist +``` + +Artifacts are written to: + +- `responses/dist/` + +On Linux x86_64, wheels built after the prebuild step bundle the native Code +Interpreter binary. On other platforms, use the source-install Bun fallback or +disable Code Interpreter. + +## Optional dependency sets + +Some features require additional optional dependencies. + +### OpenTelemetry tracing (optional) + +If you want to enable OpenTelemetry tracing (`VR_TRACING_ENABLED=true`), install with the `tracing` extra. + +### Documentation toolchain (contributors) + +If you want to build/serve the MkDocs site locally, install with the `docs` extra. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md new file mode 100644 index 0000000..d2c89ae --- /dev/null +++ b/docs/getting-started/quickstart.md @@ -0,0 +1,160 @@ +# Quickstart + +Get your Responses API gateway running in under 5 minutes. + +## Prerequisites + +- Completed [Installation](installation.md) + +______________________________________________________________________ + +## 1. Start the Gateway + +=== "Integrated Colocated Mode" + + Start vLLM and the Responses gateway together on one public API server: + + ```bash + vllm serve meta-llama/Llama-3.2-3B-Instruct --responses + ``` + +=== "Remote-Upstream Gateway Mode" + + If you already have vLLM running on port 8457: + + ```bash + --8<-- "snippets/serve_external_upstream_cmd.txt" + ``` + +Base URL by mode: + +- `vllm-responses serve`: `http://127.0.0.1:5969` by default +- `vllm serve --responses`: same host/port as `vllm serve` (default `http://127.0.0.1:8000`) + +______________________________________________________________________ + +## 2. Send a Request + +Now, send a request to the **Responses API** endpoint (`/v1/responses`). + +The cURL examples below use the default `vllm-responses serve` URL (`http://127.0.0.1:5969`). +If you started integrated mode with `vllm serve --responses`, replace that base URL with your +vLLM bind address (default `http://127.0.0.1:8000`). + +=== "cURL (streaming with Code Interpreter)" + + ```bash + curl -X POST http://127.0.0.1:5969/v1/responses \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer dummy" \ + -d '{ + "model": "meta-llama/Llama-3.2-3B-Instruct", + "input": [{"role": "user", "content": "Calculate the factorial of 5"}], + "stream": true, + "tools": [{"type": "code_interpreter"}], + "include": ["code_interpreter_call.outputs"] + }' + ``` + +=== "cURL (non-streaming)" + + ```bash + curl -X POST http://127.0.0.1:5969/v1/responses \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer dummy" \ + -d '{ + "model": "meta-llama/Llama-3.2-3B-Instruct", + "input": [{"role": "user", "content": "Calculate the factorial of 5"}], + "tools": [{"type": "code_interpreter"}], + "include": ["code_interpreter_call.outputs"] + }' + ``` + +=== "Python (OpenAI SDK)" + + ```python + --8<-- "snippets/openai_client_local_gateway.py" + + # For integrated mode, export: + # VLLM_RESPONSES_BASE_URL=http://127.0.0.1:8000/v1 + # + # For `vllm-responses serve`, the default snippet base URL already matches: + # http://127.0.0.1:5969/v1 + + with client.responses.stream( + model="meta-llama/Llama-3.2-3B-Instruct", + input=[{"role": "user", "content": "Calculate the factorial of 5"}], + tools=[{"type": "code_interpreter"}], + include=["code_interpreter_call.outputs"], + ) as stream: + for event in stream: + print(event) + ``` + +______________________________________________________________________ + +## 3. Observe the Response + +If you used `stream=true`, you will see **Server-Sent Events (SSE)**. Unlike standard Chat Completions, the Responses API provides rich lifecycle events: + +```text +event: response.created +data: {"response":{...}} + +event: response.output_item.added +data: {"output_item":{"type":"message", ...}} + +event: response.content_part.added +data: {"part":{"type":"text", "text":""}, ...} + +event: response.output_text.delta +data: {"delta":"I am a large language model...", ...} + +... + +event: response.completed +data: {"response":{...}} +``` + +## 4. Optional: MCP Smoke Test (Built-in MCP) + +If you enabled Built-in MCP on your active entrypoint, you can run a minimal forced tool call: + +- `vllm-responses serve ... --mcp-config /path/to/mcp.json` +- `vllm serve ... --responses --responses-mcp-config /path/to/mcp.json` + +Need the Built-in MCP `mcp.json` format first? See: + +- [MCP Examples -> Built-in MCP Runtime Config](../examples/hosted-mcp-examples.md#built-in-mcp-runtime-config-mcpjson) + +```bash +curl -X POST http://127.0.0.1:5969/v1/responses \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer dummy" \ + -d '{ + "model": "meta-llama/Llama-3.2-3B-Instruct", + "stream": true, + "input": [{"role":"user","content":"Use the MCP docs tool to search for migration notes."}], + "tools": [{"type":"mcp","server_label":"github_docs"}], + "tool_choice": {"type":"mcp","server_label":"github_docs","name":"search_docs"} + }' +``` + +If you are running integrated mode, replace `http://127.0.0.1:5969` with your `vllm serve` +base URL (default `http://127.0.0.1:8000`). + +In the stream, you should see MCP lifecycle events such as: + +- `response.mcp_call.in_progress` +- `response.mcp_call_arguments.done` +- `response.mcp_call.completed` (or `response.mcp_call.failed`) + +## Next Steps + +Now that you have the basic loop working, try the advanced features: + +- **[Code Interpreter](../features/built-in-tools.md)**: Ask the model to write and execute code. +- **[Web Search](../features/web-search.md)**: Let the model search the web with a shipped gateway profile. +- **[Stateful Conversations](../features/statefulness.md)**: Use `previous_response_id` to continue a chat. +- **[MCP Integration](../features/hosted-mcp.md)**: Use Built-in MCP or Remote MCP declarations. +- **[Architecture](architecture.md)**: Learn how the gateway processes your request. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..643cadf --- /dev/null +++ b/docs/index.md @@ -0,0 +1,50 @@ +# vLLM Responses + +
+ + OpenResponses Compliant + +
+ +**FastAPI gateway for the OpenAI-style Responses API.** + +`vLLM Responses` sits in front of a vLLM server and transforms its standard Chat Completions output into the rich, stateful **Responses API** format. It gives you advanced capabilities like server-side tool execution and conversation state management without modifying your inference backend. + +______________________________________________________________________ + +## Why use this gateway? + +- **Stateful Conversations**: Maintain conversation history automatically using `previous_response_id`, backed by a persistent store (SQLite/Postgres). +- **Built-in Code Interpreter**: Let the model write and execute code in a sandboxed environment on the gateway. +- **Built-in Web Search**: Let the model search the web, open pages, and search within page text through the gateway-owned `web_search` tool. +- **MCP Integration (Built-in MCP + Remote MCP)**: Use configured Built-in MCP servers or Remote MCP declarations with Responses-style streaming events. +- **Correct SSE Streaming**: Receive spec-compliant Server-Sent Events with precise ordering and shape guarantees. +- **Drop-in Compatibility**: Works with any standard vLLM OpenAI-compatible endpoint. + +## Getting Started + +
+ +- :rocket: **[Quickstart](getting-started/quickstart.md)**
Get up and running in 5 minutes. + +- :material-download: **[Installation](getting-started/installation.md)**
Install the CLI and dependencies. + +- :material-server: **[Running the Gateway](usage/running-the-gateway.md)**
Deploy alongside vLLM or as a standalone service. + +- :material-school: **[Architecture](getting-started/architecture.md)**
Understand how the gateway works. + +
+ +## Documentation Map + +- **[Usage](usage/running-the-gateway.md)**: Choose a runtime mode and get the gateway running before diving into flags. +- **[Features](features/statefulness.md)**: Deep dive into statefulness, built-in tools, and MCP integration. +- **[Reference](reference/api-reference.md)**: API endpoint details, configuration variables, and event schemas. +- **[Deployment](deployment/configuration.md)**: Production configuration. +- **[Examples](examples/code-interpreter-examples.md)**: Code snippets for code interpreter, MCP usage, and tool loops. + +______________________________________________________________________ + +!!! tip "New to the Responses API?" + + Start with the **[Quickstart](getting-started/quickstart.md)** to see the API in action, then check out **[Architecture](getting-started/architecture.md)** to understand the concepts. diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md new file mode 100644 index 0000000..d4060b8 --- /dev/null +++ b/docs/reference/api-reference.md @@ -0,0 +1,277 @@ +# API Reference + +The gateway exposes a primary OpenAI Responses endpoint plus compatibility passthrough endpoints. + +!!! tip "Spec Conformance" + + The gateway implements the [OpenResponses](https://www.openresponses.org) specification. This ensures schema validity and correct event ordering. + +______________________________________________________________________ + +## OpenAI Responses API Compatibility + +The gateway implements the OpenAI Responses API specification as defined in the [OpenResponses specification](https://www.openresponses.org/specification). This includes: + +- **Response structure**: Full `ResponseResource` shape with all required fields +- **Streaming events**: Complete SSE event lifecycle with correct ordering +- **Output items**: Support for messages, function calls, reasoning, and built-in tools +- **Statefulness**: `previous_response_id` for multi-turn conversations + +For details on the underlying architecture, see [Architecture](../getting-started/architecture.md). + +### Gateway Extensions + +In addition to OpenResponses contract compatibility, this gateway provides MCP integration: + +- Request-time MCP declarations in Built-in MCP mode (`server_label`) and Remote MCP mode (`server_url`) +- Built-in MCP discovery endpoints (`/v1/mcp/servers`, `/v1/mcp/servers/{server_label}/tools`) +- MCP lifecycle stream events (`response.mcp_call.*`) for both Built-in MCP and Remote MCP calls + +#### MCP Compatibility Matrix (Current) + +| Capability | OpenAI Responses API | This Gateway (Current) | +| ----------------------------------------------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------- | +| Request-declared remote MCP (`tools[].mcp.server_url`) | Supported | Supported as Remote MCP (URL checks enabled by default; no connector mode) | +| Request-declared connector MCP (`tools[].mcp.connector_id`) | Supported | Not supported | +| `server_label` resolution | Request-local label associated with request-declared MCP target | Built-in MCP mode: gateway-configured label. Remote MCP mode: request-local label. | +| Where MCP transport is defined | Request payload | Built-in MCP: gateway config; Remote MCP: request `server_url` | +| Local `stdio` MCP servers | Not available via request-declared MCP in OpenAI cloud runtime | Supported in Built-in MCP config (`mcpServers.