diff --git a/docs/weblog/end-to-end_weblog.md b/docs/weblog/end-to-end_weblog.md index 0995e5e6492..a0c4ced7090 100644 --- a/docs/weblog/end-to-end_weblog.md +++ b/docs/weblog/end-to-end_weblog.md @@ -1263,6 +1263,33 @@ It takes a raw (unparsed) request body, and a signature located in header `Strip The endpoint must return as JSON in the response body, the sub-object `event.data.object` returned by the `constructEvent()` Stripe SDK method. If an error happens, the endpoint must respond with a 403 error code. +### GET /llm + +This endpoint is implemented by Python, Node.js, and PHP (using openai-php/client). + +This endpoint collects interactions with LLMs. The request will have the following query parameters +- `model`: Identifies the LLM model invoked. Examples are: `gpt-4.1`, `gpt-4o-mini`, `text-davinci-003`. +- `operation`: Instead of having one each point for each function wrapped, this parameter will be used to decide what method to trigger. The following table maps all operation values to wrapped method: +| Value | Python mapped method | Node.js mapped method | PHP mapped method | +| --- | --- | --- | --- | +| `openai-latest-responses.create` | `OpenAI().responses.create(...)` | `client.responses.create` | `$client->responses()->create(...)` | +| `openai-latest-chat.completions.create` | `OpenAI().chat.completions.create(...)` | `client.chat.completions.create` | `$client->chat()->create(...)` | +| `openai-latest-completions.create` | `OpenAI().completions.create(...)` | `client.completions.create` | `$client->completions()->create(...)` | +| `openai-legacy-chat.completions.create` | `openai.ChatCompletion.create` | `openai.createChatCompletion` | `$client->chat()->create(...)` | +| `openai-legacy-completions.create` | `openai.Completion.create` | `openai.createCompletion` | `$client->completions()->create(...)` | +| `openai-async-responses.create` | `AsyncOpenAI().responses.create(...)` | not implemented | `$client->responses()->create(...)` | +| `openai-async-chat.completions.create` | `AsyncOpenAI().chat.completions.create(...)` | not implemented | `$client->chat()->create(...)` | +| `openai-async-completions.create` | `AsyncOpenAI().completions.create(...)` | not implemented | `$client->completions()->create(...)` | + +For example a call to `/llm?model=gpt-4.1&operation=openai-latest-responses.create` (URL-encoded) will require that Python does the following call +``` +OpenAI().responses.create(model="gpt-4.1", ...) +``` + +This approach makes the endpoint ready to be expanded in the future. + +In scenarios that use this endpoint, `OPENAI_BASE_URL` is set to the system-tests internal server, which mocks the OpenAI API responses. + ## Weblog specification There are several rules shared between all the existing end-to-end weblogs. diff --git a/manifests/cpp.yml b/manifests/cpp.yml index d5fc06351a8..1a005c2350b 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -9,6 +9,15 @@ # use `>1.0.0` to indicate that requirement. # NOTE: only parametric tests are run for cpp, there is no need to mark other tests as "missing_feature" here. manifest: + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: ">1.0.0" tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_b3single_128_bit_generation_disabled: missing_feature (propagation style not supported) tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_b3single_128_bit_generation_enabled: missing_feature (propagation style not supported) diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index e017bae5c1b..3799bb2366b 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -18,6 +18,15 @@ manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_SDK_Disabled: missing_feature tests/apm_tracing_e2e/: missing_feature (missing /e2e_otel_span endpoint on weblog) tests/appsec/: irrelevant (ASM is not implemented in C++) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/debugger/: irrelevant tests/ffe/test_dynamic_evaluation.py: missing_feature tests/ffe/test_exposures.py: missing_feature diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 8582f95b8c9..2aa621c0fa2 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -30,6 +30,15 @@ manifest: tests/appsec/api_security/test_custom_data_classification.py::Test_API_Security_Custom_Data_Classification_Scanner: v1.10.0 tests/appsec/api_security/test_endpoint_discovery.py: irrelevant (not applicable to proxies) tests/appsec/api_security/test_endpoint_fallback.py: irrelevant (not applicable to proxies) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: v1.8.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_Cookies: v1.8.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_FormUrlEncoded_Body: v1.8.0 diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 9ae81f5dc71..6c3db2a4584 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -35,6 +35,15 @@ manifest: tests/appsec/api_security/test_custom_data_classification.py::Test_API_Security_Custom_Data_Classification_Scanner: missing_feature tests/appsec/api_security/test_endpoint_discovery.py::Test_Endpoint_Discovery: v3.24.0 tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: v2.46.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_Cookies: v2.46.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_FormUrlEncoded_Body: v2.46.0 diff --git a/manifests/envoy.yml b/manifests/envoy.yml index 59cc8bc2503..e82ffb81ef6 100644 --- a/manifests/envoy.yml +++ b/manifests/envoy.yml @@ -1,6 +1,15 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/DataDog/system-tests/refs/heads/main/utils/manifest/schema.json --- manifest: + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/test_alpha.py: v1.72.0 tests/appsec/test_blocking_addresses.py: v1.72.0 tests/appsec/test_blocking_addresses.py::Test_BlockingGraphqlResolvers: irrelevant diff --git a/manifests/golang.yml b/manifests/golang.yml index c88bfcaa2bd..73aec7fe411 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -38,6 +38,15 @@ manifest: tests/appsec/api_security/test_custom_data_classification.py::Test_API_Security_Custom_Data_Classification_Scanner: v2.4.0 tests/appsec/api_security/test_endpoint_discovery.py::Test_Endpoint_Discovery: missing_feature tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: v2.0.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_Cookies: v1.60.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_FormUrlEncoded_Body: v1.60.0 diff --git a/manifests/haproxy.yml b/manifests/haproxy.yml index 7ff78699a79..39e4aaf0a3b 100644 --- a/manifests/haproxy.yml +++ b/manifests/haproxy.yml @@ -1,6 +1,15 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/DataDog/system-tests/refs/heads/main/utils/manifest/schema.json --- manifest: + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/test_alpha.py: v2.4.0 tests/appsec/test_blocking_addresses.py: v2.4.0 tests/appsec/test_blocking_addresses.py::Test_BlockingGraphqlResolvers: irrelevant diff --git a/manifests/java.yml b/manifests/java.yml index 40c1e0483b0..018942c254c 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -149,6 +149,15 @@ manifest: uds-spring-boot: irrelevant (Not applicable to weblog variant) spring-boot-jetty: irrelevant (Not applicable to weblog variant) tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: - weblog_declaration: "*": v1.31.0 diff --git a/manifests/java_otel.yml b/manifests/java_otel.yml index ee4aa5e087e..7bb9eb236c0 100644 --- a/manifests/java_otel.yml +++ b/manifests/java_otel.yml @@ -1,6 +1,15 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/DataDog/system-tests/refs/heads/main/utils/manifest/schema.json --- manifest: + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/integrations/test_open_telemetry.py::Test_MsSql::test_obfuscate_query: bug (OTEL-2778) tests/integrations/test_open_telemetry.py::Test_MySql::test_obfuscate_query: bug (OTEL-2778) tests/integrations/test_open_telemetry.py::Test_MySql::test_properties: bug (OTEL-2778) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 2daba259a4e..62b008a8b4c 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -135,6 +135,15 @@ manifest: - component_version: "5.82.0" declaration: flaky (APPSEC-60648) tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: missing_feature tests/appsec/api_security/test_schemas.py::Test_Scanners: - weblog_declaration: "*": *ref_4_21_0 diff --git a/manifests/php.yml b/manifests/php.yml index 81de35a47c3..4c5c46500c4 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -32,6 +32,15 @@ manifest: tests/appsec/api_security/test_endpoint_discovery.py::Test_Endpoint_Discovery::test_optional_response_code: irrelevant (Not supported) tests/appsec/api_security/test_endpoint_discovery.py::Test_Endpoint_Discovery::test_optional_type: irrelevant (Not supported) tests/appsec/api_security/test_endpoint_fallback.py: v1.16.0 + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: missing_feature (waiting until PR gets merged) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: missing_feature (waiting until PR gets merged) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: missing_feature (waiting until PR gets merged) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: - weblog_declaration: "*": v1.11.0-dev diff --git a/manifests/python.yml b/manifests/python.yml index fc995df1a73..aebd82ca4c0 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -66,6 +66,15 @@ manifest: fastapi: v3.13.0.dev tests/appsec/api_security/test_endpoint_discovery.py::Test_Endpoint_Discovery::test_optional_type: irrelevant tests/appsec/api_security/test_endpoint_fallback.py: irrelevant (python weblogs always have http.route) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: missing_feature tests/appsec/api_security/test_schemas.py::Test_Scanners: - weblog_declaration: "*": v2.4.0 diff --git a/manifests/ruby.yml b/manifests/ruby.yml index db697e33499..b2d81de4766 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -75,6 +75,15 @@ manifest: - declaration: irrelevant weblog: [rack, rails42] tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: v2.19.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_Cookies: v1.15.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_FormUrlEncoded_Body: missing_feature diff --git a/manifests/rust.yml b/manifests/rust.yml index 9908665b134..4938423b991 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -4,6 +4,15 @@ refs: - &ref_0_0_1 "0.0.1" manifest: tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/auto_inject/test_auto_inject_install.py::TestContainerAutoInjectInstallScriptAppsec: missing_feature tests/auto_inject/test_auto_inject_install.py::TestHostAutoInjectInstallScriptAppsec: missing_feature tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: missing_feature diff --git a/tests/appsec/api_security/test_endpoints.py b/tests/appsec/api_security/test_endpoints.py new file mode 100644 index 00000000000..df0e01a28d9 --- /dev/null +++ b/tests/appsec/api_security/test_endpoints.py @@ -0,0 +1,80 @@ +from utils import interfaces, rfc, scenarios, weblog, features +from utils._weblog import HttpResponse + + +def assert_llm_span(request: HttpResponse, model: str) -> None: + """Common assertions for LLM endpoint spans.""" + span = interfaces.library.get_root_span(request) + assert span["meta"]["appsec.events.llm.call.provider"] == "openai" + assert span["meta"]["appsec.events.llm.call.model"] == model + assert span["metrics"]["_sampling_priority_v1"] == 2 + + +@rfc( + "https://docs.google.com/document/d/1TIFxbtbkldjOA6S5JFlCTMfqniXfJXZmDKptI5w2pnk/edit?tab=t.0#heading=h.xtljwwxyhqk7" +) +@scenarios.appsec_rasp +@features.api_llm_endpoint +class Test_LLM_Endpoint: + """Tests for the /llm endpoint capturing LLM interaction metadata.""" + + MODEL = "gpt-4.1" + + def setup_openai_latest_responses_create(self): + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-latest-responses.create") + + def test_openai_latest_responses_create(self): + assert_llm_span(self.request, self.MODEL) + + def setup_openai_latest_chat_completions_create(self): + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-latest-chat.completions.create") + + def test_openai_latest_chat_completions_create(self): + assert_llm_span(self.request, self.MODEL) + + def setup_openai_latest_completions_create(self): + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-latest-completions.create") + + def test_openai_latest_completions_create(self): + assert_llm_span(self.request, self.MODEL) + + def setup_openai_legacy_chat_completions_create(self): + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-legacy-chat.completions.create") + + def test_openai_legacy_chat_completions_create(self): + assert_llm_span(self.request, self.MODEL) + + def setup_openai_legacy_completions_create(self): + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-legacy-completions.create") + + def test_openai_legacy_completions_create(self): + assert_llm_span(self.request, self.MODEL) + + def setup_openai_async_responses_create(self): + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-async-responses.create") + + def test_openai_async_responses_create(self): + assert_llm_span(self.request, self.MODEL) + + def setup_openai_async_chat_completions_create(self): + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-async-chat.completions.create") + + def test_openai_async_chat_completions_create(self): + assert_llm_span(self.request, self.MODEL) + + def setup_openai_async_completions_create(self): + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-async-completions.create") + + def test_openai_async_completions_create(self): + assert_llm_span(self.request, self.MODEL) + + def setup_root_no_llm(self): + # Baseline request to root endpoint should not produce LLM tags + self.root_request = weblog.get("/") + + def test_root_has_no_llm_tags(self): + """Assert that LLM-specific meta tags are not present on the span.""" + span = interfaces.library.get_root_span(self.root_request) + meta = span.get("meta", {}) + assert "appsec.events.llm.call.provider" not in meta + assert "appsec.events.llm.call.model" not in meta diff --git a/tests/appsec/rasp/rasp_ruleset.json b/tests/appsec/rasp/rasp_ruleset.json index 53459732cc5..088db6a7f77 100644 --- a/tests/appsec/rasp/rasp_ruleset.json +++ b/tests/appsec/rasp/rasp_ruleset.json @@ -823,6 +823,50 @@ } } } + }, + { + "id": "llm-001-000", + "name": "LLM call", + "tags": { + "type": "llm.event", + "category": "business_logic", + "module": "business_logic" + }, + "min_version": "1.25.0", + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.business_logic.llm.event", + "key_path": [ + "provider" + ] + } + ] + }, + "operator": "exists" + } + ], + "transformers": [], + "output": { + "event": false, + "keep": true, + "attributes": { + "appsec.events.llm.call.provider": { + "address": "server.business_logic.llm.event", + "key_path": [ + "provider" + ] + }, + "appsec.events.llm.call.model": { + "address": "server.business_logic.llm.event", + "key_path": [ + "model" + ] + } + } + } } ] } \ No newline at end of file diff --git a/utils/_context/_scenarios/appsec_rasp.py b/utils/_context/_scenarios/appsec_rasp.py index 7ffd6f48fa6..4d797e034a6 100644 --- a/utils/_context/_scenarios/appsec_rasp.py +++ b/utils/_context/_scenarios/appsec_rasp.py @@ -24,6 +24,7 @@ def __init__( # added to test Test_ExtendedRequestBodyCollection "DD_APPSEC_RASP_COLLECT_REQUEST_BODY": "true", "DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE": "1.0", + "OPENAI_BASE_URL": "http://internal_server:8089", } merged_env = default_env | weblog_env diff --git a/utils/_features.py b/utils/_features.py index 9b895dedb05..cd65c496df4 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2574,6 +2574,14 @@ def appsec_extended_data_collection(test_object): """ return _mark_test_object(test_object, feature_id=492, owner=_Owner.asm) + @staticmethod + def api_llm_endpoint(test_object): + """API Security - Business logic events for LLM-based SDKs + + https://feature-parity.us1.prod.dog/#/?feature=543 + """ + return _mark_test_object(test_object, feature_id=NOT_REPORTED_ID, owner=_Owner.asm) + @staticmethod def agent_data_integrity(test_object): """Data integrity diff --git a/utils/build/docker/internal_server/app.py b/utils/build/docker/internal_server/app.py index 4e50665ce44..ecec8be5eac 100644 --- a/utils/build/docker/internal_server/app.py +++ b/utils/build/docker/internal_server/app.py @@ -1,5 +1,6 @@ import os import signal +import time import urllib.parse import fastapi @@ -146,6 +147,115 @@ async def payment_intents(request: fastapi.Request): return fastapi.responses.JSONResponse({"error": {"type": "api_error", "message": str(e)}}, status_code=500) +# Mock OpenAI API endpoints for LLM tests (OPENAI_BASE_URL=http://internal_server:8089). +def _openai_fake_usage() -> dict: + return {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3} + + +@app.post("/chat/completions", response_class=fastapi.responses.JSONResponse) +async def openai_chat_completions(request: fastapi.Request): + """Mock for OpenAI Chat Completions API (POST /chat/completions).""" + try: + body = await request.json() if request.headers.get("content-length") else {} + except Exception: + body = {} + model = body.get("model", "gpt-4.1") + return fastapi.responses.JSONResponse( + { + "id": "chatcmpl-fake-internal", + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Fake response from internal_server mock."}, + "finish_reason": "stop", + } + ], + "usage": _openai_fake_usage(), + }, + status_code=200, + ) + + +@app.post("/completions", response_class=fastapi.responses.JSONResponse) +async def openai_completions(request: fastapi.Request): + """Mock for OpenAI Completions API (POST /completions, legacy).""" + try: + body = await request.json() if request.headers.get("content-length") else {} + except Exception: + body = {} + model = body.get("model", "text-davinci-003") + return fastapi.responses.JSONResponse( + { + "id": "cmpl-fake-internal", + "object": "text_completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "text": "Fake completion from internal_server mock.", + "index": 0, + "finish_reason": "stop", + "logprobs": None, + } + ], + "usage": _openai_fake_usage(), + }, + status_code=200, + ) + + +@app.post("/responses", response_class=fastapi.responses.JSONResponse) +async def openai_responses(request: fastapi.Request): + """Mock for OpenAI Responses API (POST /responses). + Shape matches openai-php/client CreateResponse (created_at, status, output with output_text content). + """ + try: + body = await request.json() if request.headers.get("content-length") else {} + except Exception: + body = {} + model = body.get("model", "gpt-4.1") + return fastapi.responses.JSONResponse( + { + "id": "resp-fake-internal", + "object": "response", + "created_at": int(time.time()), + "status": "completed", + "model": model, + "output": [ + { + "type": "message", + "id": "msg-fake-internal", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Fake response from internal_server mock.", + "annotations": [], + } + ], + } + ], + "output_text": "Fake response from internal_server mock.", + "parallel_tool_calls": False, + "tool_choice": "none", + "tools": [], + "store": True, + "usage": { + "input_tokens": 1, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 3, + }, + }, + status_code=200, + ) + + @app.get("/shutdown") async def shutdown(): os.kill(os.getpid(), signal.SIGTERM) diff --git a/utils/build/docker/php/apache-mod/php.conf b/utils/build/docker/php/apache-mod/php.conf index 1c56b062c20..aaf88b2f4e9 100644 --- a/utils/build/docker/php/apache-mod/php.conf +++ b/utils/build/docker/php/apache-mod/php.conf @@ -27,6 +27,7 @@ RewriteRule "^/load_dependency$" "/load_dependency/" RewriteRule "^/signup$" "/signup/" RewriteRule "^/shell_execution$" "/shell_execution/" + RewriteRule "^/llm$" "/llm/" RewriteCond /var/www/html/%{REQUEST_URI} !-f RewriteRule "^/rasp/(.*)" "/rasp/$1.php" [L] RewriteRule "^/api_security.sampling/.*" "/api_security_sampling.php$0" [L] diff --git a/utils/build/docker/php/common/composer.json b/utils/build/docker/php/common/composer.json index 4abe446e7d4..b82c054e130 100644 --- a/utils/build/docker/php/common/composer.json +++ b/utils/build/docker/php/common/composer.json @@ -3,7 +3,9 @@ "type": "project", "require": { "weblog/acme": "*", - "monolog/monolog": "*" + "monolog/monolog": "*", + "openai-php/client": "*", + "guzzlehttp/guzzle": "*" }, "repositories": [ { diff --git a/utils/build/docker/php/common/llm.php b/utils/build/docker/php/common/llm.php new file mode 100644 index 00000000000..b534479973c --- /dev/null +++ b/utils/build/docker/php/common/llm.php @@ -0,0 +1,70 @@ + 'Missing or empty query parameters: model, operation']); + exit; +} + +try { + $baseUri = (string) (getenv('OPENAI_BASE_URL') ?: 'http://internal_server:8089/v1'); + $client = \OpenAI::factory() + ->withApiKey('sk-fake') + ->withBaseUri($baseUri) + ->make(); +} catch (Throwable $e) { + http_response_code(500); + echo json_encode(['error' => 'OpenAI client init failed: ' . $e->getMessage()]); + exit; +} + +$params = ['model' => $model]; + +try { + switch ($operation) { + case 'openai-latest-responses.create': + $params['input'] = 'Hello'; + $response = $client->responses()->create($params); + break; + case 'openai-latest-chat.completions.create': + $params['messages'] = [['role' => 'user', 'content' => 'Hello']]; + $response = $client->chat()->create($params); + break; + case 'openai-latest-completions.create': + $params['prompt'] = 'Hello'; + $response = $client->completions()->create($params); + break; + default: + http_response_code(400); + echo json_encode(['error' => 'Unknown operation: ' . $operation]); + exit; + } + + // Return a simple success payload; the client returns response objects + echo json_encode([ + 'model' => $model, + 'operation' => $operation, + 'status' => 'ok', + ]); +} catch (Throwable $e) { + http_response_code(500); + echo json_encode([ + 'error' => $e->getMessage(), + 'operation' => $operation, + ]); +} diff --git a/utils/build/docker/php/php-fpm/php-fpm.conf b/utils/build/docker/php/php-fpm/php-fpm.conf index 47e2bffb185..e54aa228a5c 100644 --- a/utils/build/docker/php/php-fpm/php-fpm.conf +++ b/utils/build/docker/php/php-fpm/php-fpm.conf @@ -27,6 +27,7 @@ RewriteRule "^/requestdownstream$" "/requestdownstream/" RewriteRule "^/resource_renaming$" "/resource_renaming/" RewriteRule "^/returnheaders$" "/returnheaders/" + RewriteRule "^/llm$" "/llm/" RewriteRule "^/session/new$" "/session_new.php" RewriteRule "^/user_login_failure_event_v2$" "/user_login_failure_event_v2/" RewriteRule "^/user_login_success_event_v2$" "/user_login_success_event_v2/"