From 6772469f10a6d995f591cba087dd7b3d82965ff1 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Wed, 18 Feb 2026 16:47:42 +0100 Subject: [PATCH 01/10] Add tests --- docs/weblog/end-to-end_weblog.md | 26 ++++++ manifests/cpp.yml | 9 ++ manifests/cpp_httpd.yml | 9 ++ manifests/cpp_nginx.yml | 9 ++ manifests/dotnet.yml | 9 ++ manifests/envoy.yml | 9 ++ manifests/golang.yml | 9 ++ manifests/haproxy.yml | 9 ++ manifests/java.yml | 9 ++ manifests/java_otel.yml | 9 ++ manifests/nodejs.yml | 9 ++ manifests/php.yml | 9 ++ manifests/python.yml | 9 ++ manifests/ruby.yml | 9 ++ manifests/rust.yml | 9 ++ tests/appsec/api_security/test_endpoints.py | 94 +++++++++++++++++++++ utils/_features.py | 8 ++ 17 files changed, 254 insertions(+) create mode 100644 tests/appsec/api_security/test_endpoints.py diff --git a/docs/weblog/end-to-end_weblog.md b/docs/weblog/end-to-end_weblog.md index 0995e5e6492..d2b3386c3cb 100644 --- a/docs/weblog/end-to-end_weblog.md +++ b/docs/weblog/end-to-end_weblog.md @@ -1263,6 +1263,32 @@ It takes a raw (unparsed) request body, and a signature located in header `Strip The endpoint must return as JSON in the response body, the sub-object `event.data.object` returned by the `constructEvent()` Stripe SDK method. If an error happens, the endpoint must respond with a 403 error code. +### GET /llm + +For now this endpoint will only be implemented by Python and Node.js + +This endpoint collects interactions with LLMs. The request will have the following query parameters +- `model`: Identifies the LLM model invoked. Examples are: `gpt-4.1`, `gpt-4o-mini`, `text-davinci-003`. +- `operation`: Instead of having one each point for each function wrapped, this parameter will be used to decide what method to trigger. The following table maps all operation values to wrapped method: +| Value | Python mapped method | Node.js mapped method | +| --- | --- | --- | +| `openai-latest-responses.create` | `OpenAI().responses.create(...)` | `client.responses.create` | +| `openai-latest-chat.completions.create` | `OpenAI().chat.completions.create(...)` | `client.chat.completions.create` | +| `openai-latest-completions.create` | `OpenAI().completions.create(...)` | `client.completions.create` | +| `openai-legacy-chat.completions.create` | `openai.ChatCompletion.create` | `openai.createChatCompletion` | +| `openai-legacy-completions.create` | `openai.Completion.create` | `openai.createCompletion` | +| `openai-async-responses.create` | `AsyncOpenAI().responses.create(...)` | not implemented | +| `openai-async-chat.completions.create` | `AsyncOpenAI().chat.completions.create(...)` | not implemented | +| `openai-async-completions.create` | `AsyncOpenAI().completions.create(...)` | not implemented | + +For exmaple a call to `/llm?model=gpt-4.1&operation=openai-latest-responses.create`(url encoded) will do the will require that python does thes following call +``` +OpenAI().responses.create(model="gpt-4.1", ...) +``` + +This approach makes the endpoint ready to be expanded in the future. + + ## Weblog specification There are several rules shared between all the existing end-to-end weblogs. diff --git a/manifests/cpp.yml b/manifests/cpp.yml index d5fc06351a8..1a005c2350b 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -9,6 +9,15 @@ # use `>1.0.0` to indicate that requirement. # NOTE: only parametric tests are run for cpp, there is no need to mark other tests as "missing_feature" here. manifest: + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: ">1.0.0" tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_b3single_128_bit_generation_disabled: missing_feature (propagation style not supported) tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_b3single_128_bit_generation_enabled: missing_feature (propagation style not supported) diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index e017bae5c1b..3799bb2366b 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -18,6 +18,15 @@ manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_SDK_Disabled: missing_feature tests/apm_tracing_e2e/: missing_feature (missing /e2e_otel_span endpoint on weblog) tests/appsec/: irrelevant (ASM is not implemented in C++) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/debugger/: irrelevant tests/ffe/test_dynamic_evaluation.py: missing_feature tests/ffe/test_exposures.py: missing_feature diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 8582f95b8c9..2aa621c0fa2 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -30,6 +30,15 @@ manifest: tests/appsec/api_security/test_custom_data_classification.py::Test_API_Security_Custom_Data_Classification_Scanner: v1.10.0 tests/appsec/api_security/test_endpoint_discovery.py: irrelevant (not applicable to proxies) tests/appsec/api_security/test_endpoint_fallback.py: irrelevant (not applicable to proxies) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: v1.8.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_Cookies: v1.8.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_FormUrlEncoded_Body: v1.8.0 diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 9ae81f5dc71..6c3db2a4584 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -35,6 +35,15 @@ manifest: tests/appsec/api_security/test_custom_data_classification.py::Test_API_Security_Custom_Data_Classification_Scanner: missing_feature tests/appsec/api_security/test_endpoint_discovery.py::Test_Endpoint_Discovery: v3.24.0 tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: v2.46.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_Cookies: v2.46.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_FormUrlEncoded_Body: v2.46.0 diff --git a/manifests/envoy.yml b/manifests/envoy.yml index 59cc8bc2503..e82ffb81ef6 100644 --- a/manifests/envoy.yml +++ b/manifests/envoy.yml @@ -1,6 +1,15 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/DataDog/system-tests/refs/heads/main/utils/manifest/schema.json --- manifest: + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/test_alpha.py: v1.72.0 tests/appsec/test_blocking_addresses.py: v1.72.0 tests/appsec/test_blocking_addresses.py::Test_BlockingGraphqlResolvers: irrelevant diff --git a/manifests/golang.yml b/manifests/golang.yml index c88bfcaa2bd..73aec7fe411 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -38,6 +38,15 @@ manifest: tests/appsec/api_security/test_custom_data_classification.py::Test_API_Security_Custom_Data_Classification_Scanner: v2.4.0 tests/appsec/api_security/test_endpoint_discovery.py::Test_Endpoint_Discovery: missing_feature tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: v2.0.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_Cookies: v1.60.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_FormUrlEncoded_Body: v1.60.0 diff --git a/manifests/haproxy.yml b/manifests/haproxy.yml index 7ff78699a79..39e4aaf0a3b 100644 --- a/manifests/haproxy.yml +++ b/manifests/haproxy.yml @@ -1,6 +1,15 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/DataDog/system-tests/refs/heads/main/utils/manifest/schema.json --- manifest: + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/test_alpha.py: v2.4.0 tests/appsec/test_blocking_addresses.py: v2.4.0 tests/appsec/test_blocking_addresses.py::Test_BlockingGraphqlResolvers: irrelevant diff --git a/manifests/java.yml b/manifests/java.yml index 40c1e0483b0..018942c254c 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -149,6 +149,15 @@ manifest: uds-spring-boot: irrelevant (Not applicable to weblog variant) spring-boot-jetty: irrelevant (Not applicable to weblog variant) tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: - weblog_declaration: "*": v1.31.0 diff --git a/manifests/java_otel.yml b/manifests/java_otel.yml index ee4aa5e087e..7bb9eb236c0 100644 --- a/manifests/java_otel.yml +++ b/manifests/java_otel.yml @@ -1,6 +1,15 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/DataDog/system-tests/refs/heads/main/utils/manifest/schema.json --- manifest: + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/integrations/test_open_telemetry.py::Test_MsSql::test_obfuscate_query: bug (OTEL-2778) tests/integrations/test_open_telemetry.py::Test_MySql::test_obfuscate_query: bug (OTEL-2778) tests/integrations/test_open_telemetry.py::Test_MySql::test_properties: bug (OTEL-2778) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 2daba259a4e..76200e9a22c 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -135,6 +135,15 @@ manifest: - component_version: "5.82.0" declaration: flaky (APPSEC-60648) tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: missing_feature tests/appsec/api_security/test_schemas.py::Test_Scanners: - weblog_declaration: "*": *ref_4_21_0 diff --git a/manifests/php.yml b/manifests/php.yml index 81de35a47c3..0f6d91dbeee 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -32,6 +32,15 @@ manifest: tests/appsec/api_security/test_endpoint_discovery.py::Test_Endpoint_Discovery::test_optional_response_code: irrelevant (Not supported) tests/appsec/api_security/test_endpoint_discovery.py::Test_Endpoint_Discovery::test_optional_type: irrelevant (Not supported) tests/appsec/api_security/test_endpoint_fallback.py: v1.16.0 + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: - weblog_declaration: "*": v1.11.0-dev diff --git a/manifests/python.yml b/manifests/python.yml index fc995df1a73..aebd82ca4c0 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -66,6 +66,15 @@ manifest: fastapi: v3.13.0.dev tests/appsec/api_security/test_endpoint_discovery.py::Test_Endpoint_Discovery::test_optional_type: irrelevant tests/appsec/api_security/test_endpoint_fallback.py: irrelevant (python weblogs always have http.route) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: missing_feature tests/appsec/api_security/test_schemas.py::Test_Scanners: - weblog_declaration: "*": v2.4.0 diff --git a/manifests/ruby.yml b/manifests/ruby.yml index db697e33499..b2d81de4766 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -75,6 +75,15 @@ manifest: - declaration: irrelevant weblog: [rack, rails42] tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/appsec/api_security/test_schemas.py::Test_Scanners: v2.19.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_Cookies: v1.15.0 tests/appsec/api_security/test_schemas.py::Test_Schema_Request_FormUrlEncoded_Body: missing_feature diff --git a/manifests/rust.yml b/manifests/rust.yml index 9908665b134..4938423b991 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -4,6 +4,15 @@ refs: - &ref_0_0_1 "0.0.1" manifest: tests/appsec/api_security/test_endpoint_fallback.py: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) tests/auto_inject/test_auto_inject_install.py::TestContainerAutoInjectInstallScriptAppsec: missing_feature tests/auto_inject/test_auto_inject_install.py::TestHostAutoInjectInstallScriptAppsec: missing_feature tests/auto_inject/test_auto_inject_install.py::TestSimpleInstallerAutoInjectManualAppsec: missing_feature diff --git a/tests/appsec/api_security/test_endpoints.py b/tests/appsec/api_security/test_endpoints.py new file mode 100644 index 00000000000..61664d79913 --- /dev/null +++ b/tests/appsec/api_security/test_endpoints.py @@ -0,0 +1,94 @@ +from utils import interfaces, rfc, scenarios, weblog, features, irrelevant, context +from utils._weblog import HttpResponse + + +def assert_llm_span(request: HttpResponse, model: str) -> None: + """Common assertions for LLM endpoint spans.""" + span = interfaces.library.get_root_span(request) + assert span["meta"]["appsec.events.llm.call.provider"] == "openai" + assert span["meta"]["appsec.events.llm.call.model"] == model + assert span["metrics"]["_sampling_priority_v1"] == 1 + + +@rfc( + "https://docs.google.com/document/d/1TIFxbtbkldjOA6S5JFlCTMfqniXfJXZmDKptI5w2pnk/edit?tab=t.0#heading=h.xtljwwxyhqk7" +) +@scenarios.appsec_api_security +@features.api_llm_endpoint +class Test_LLM_Endpoint: + """Tests for the /llm endpoint capturing LLM interaction metadata.""" + + def setup_openai_latest_responses_create(self): + self.model = "gpt-4.1" + self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-latest-responses.create") + + @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") + def test_openai_latest_responses_create(self): + assert_llm_span(self.request, self.model) + + def setup_openai_latest_chat_completions_create(self): + self.model = "gpt-4.1" + self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-latest-chat.completions.create") + + @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") + def test_openai_latest_chat_completions_create(self): + assert_llm_span(self.request, self.model) + + def setup_openai_latest_completions_create(self): + self.model = "gpt-4.1" + self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-latest-completions.create") + + @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") + def test_openai_latest_completions_create(self): + assert_llm_span(self.request, self.model) + + def setup_openai_legacy_chat_completions_create(self): + self.model = "gpt-4.1" + self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-legacy-chat.completions.create") + + @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") + def test_openai_legacy_chat_completions_create(self): + assert_llm_span(self.request, self.model) + + def setup_openai_legacy_completions_create(self): + self.model = "gpt-4.1" + self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-legacy-completions.create") + + @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") + def test_openai_legacy_completions_create(self): + assert_llm_span(self.request, self.model) + + def setup_openai_async_responses_create(self): + self.model = "gpt-4.1" + self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-async-responses.create") + + @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") + def test_openai_async_responses_create(self): + assert_llm_span(self.request, self.model) + + def setup_openai_async_chat_completions_create(self): + self.model = "gpt-4.1" + self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-async-chat.completions.create") + + @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") + def test_openai_async_chat_completions_create(self): + assert_llm_span(self.request, self.model) + + def setup_openai_async_completions_create(self): + self.model = "gpt-4.1" + self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-async-completions.create") + + @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") + def test_openai_async_completions_create(self): + assert_llm_span(self.request, self.model) + + def setup_root_no_llm(self): + # Baseline request to root endpoint should not produce LLM tags + self.root_request = weblog.get("/") + + def test_root_has_no_llm_tags(self): + """Assert that LLM-specific meta tags are not present on the span.""" + span = interfaces.library.get_root_span(self.root_request) + meta = span.get("meta", {}) + assert "appsec.events.llm.call.provider" not in meta + assert "appsec.events.llm.call.model" not in meta diff --git a/utils/_features.py b/utils/_features.py index 9b895dedb05..cd65c496df4 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2574,6 +2574,14 @@ def appsec_extended_data_collection(test_object): """ return _mark_test_object(test_object, feature_id=492, owner=_Owner.asm) + @staticmethod + def api_llm_endpoint(test_object): + """API Security - Business logic events for LLM-based SDKs + + https://feature-parity.us1.prod.dog/#/?feature=543 + """ + return _mark_test_object(test_object, feature_id=NOT_REPORTED_ID, owner=_Owner.asm) + @staticmethod def agent_data_integrity(test_object): """Data integrity From cf24a2f05ff2478e79222beecbbff9ba5b9c881e Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Thu, 19 Feb 2026 12:17:04 +0100 Subject: [PATCH 02/10] Remove irrelevant tags not used anymore --- tests/appsec/api_security/test_endpoints.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/appsec/api_security/test_endpoints.py b/tests/appsec/api_security/test_endpoints.py index 61664d79913..0ac6865bd34 100644 --- a/tests/appsec/api_security/test_endpoints.py +++ b/tests/appsec/api_security/test_endpoints.py @@ -1,4 +1,4 @@ -from utils import interfaces, rfc, scenarios, weblog, features, irrelevant, context +from utils import interfaces, rfc, scenarios, weblog, features, context from utils._weblog import HttpResponse @@ -22,7 +22,6 @@ def setup_openai_latest_responses_create(self): self.model = "gpt-4.1" self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-latest-responses.create") - @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") def test_openai_latest_responses_create(self): assert_llm_span(self.request, self.model) @@ -30,7 +29,6 @@ def setup_openai_latest_chat_completions_create(self): self.model = "gpt-4.1" self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-latest-chat.completions.create") - @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") def test_openai_latest_chat_completions_create(self): assert_llm_span(self.request, self.model) @@ -38,7 +36,6 @@ def setup_openai_latest_completions_create(self): self.model = "gpt-4.1" self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-latest-completions.create") - @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") def test_openai_latest_completions_create(self): assert_llm_span(self.request, self.model) @@ -46,7 +43,6 @@ def setup_openai_legacy_chat_completions_create(self): self.model = "gpt-4.1" self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-legacy-chat.completions.create") - @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") def test_openai_legacy_chat_completions_create(self): assert_llm_span(self.request, self.model) @@ -54,7 +50,6 @@ def setup_openai_legacy_completions_create(self): self.model = "gpt-4.1" self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-legacy-completions.create") - @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") def test_openai_legacy_completions_create(self): assert_llm_span(self.request, self.model) @@ -62,7 +57,6 @@ def setup_openai_async_responses_create(self): self.model = "gpt-4.1" self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-async-responses.create") - @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") def test_openai_async_responses_create(self): assert_llm_span(self.request, self.model) @@ -70,7 +64,6 @@ def setup_openai_async_chat_completions_create(self): self.model = "gpt-4.1" self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-async-chat.completions.create") - @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") def test_openai_async_chat_completions_create(self): assert_llm_span(self.request, self.model) @@ -78,7 +71,6 @@ def setup_openai_async_completions_create(self): self.model = "gpt-4.1" self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-async-completions.create") - @irrelevant(context.library not in ["python", "nodejs"], reason="LLM endpoint only implemented for Python/Node.js") def test_openai_async_completions_create(self): assert_llm_span(self.request, self.model) From db8495814aeea01f7ccc45d5940d5fba4734b1a3 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Thu, 19 Feb 2026 12:27:21 +0100 Subject: [PATCH 03/10] Mark irrelevant nodejs tests --- manifests/nodejs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 76200e9a22c..62b008a8b4c 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -135,9 +135,9 @@ manifest: - component_version: "5.82.0" declaration: flaky (APPSEC-60648) tests/appsec/api_security/test_endpoint_fallback.py: missing_feature - tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: missing_feature - tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: missing_feature - tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: missing_feature + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: missing_feature tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: missing_feature tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: missing_feature From fc2fb08dab1299c088483072e022c5e14cb27ccf Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Thu, 19 Feb 2026 12:27:37 +0100 Subject: [PATCH 04/10] Refactor tests --- tests/appsec/api_security/test_endpoints.py | 44 +++++++++------------ 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/tests/appsec/api_security/test_endpoints.py b/tests/appsec/api_security/test_endpoints.py index 0ac6865bd34..525327254ff 100644 --- a/tests/appsec/api_security/test_endpoints.py +++ b/tests/appsec/api_security/test_endpoints.py @@ -1,4 +1,4 @@ -from utils import interfaces, rfc, scenarios, weblog, features, context +from utils import interfaces, rfc, scenarios, weblog, features from utils._weblog import HttpResponse @@ -18,61 +18,55 @@ def assert_llm_span(request: HttpResponse, model: str) -> None: class Test_LLM_Endpoint: """Tests for the /llm endpoint capturing LLM interaction metadata.""" + MODEL = "gpt-4.1" + def setup_openai_latest_responses_create(self): - self.model = "gpt-4.1" - self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-latest-responses.create") + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-latest-responses.create") def test_openai_latest_responses_create(self): - assert_llm_span(self.request, self.model) + assert_llm_span(self.request, self.MODEL) def setup_openai_latest_chat_completions_create(self): - self.model = "gpt-4.1" - self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-latest-chat.completions.create") + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-latest-chat.completions.create") def test_openai_latest_chat_completions_create(self): - assert_llm_span(self.request, self.model) + assert_llm_span(self.request, self.MODEL) def setup_openai_latest_completions_create(self): - self.model = "gpt-4.1" - self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-latest-completions.create") + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-latest-completions.create") def test_openai_latest_completions_create(self): - assert_llm_span(self.request, self.model) + assert_llm_span(self.request, self.MODEL) def setup_openai_legacy_chat_completions_create(self): - self.model = "gpt-4.1" - self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-legacy-chat.completions.create") + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-legacy-chat.completions.create") def test_openai_legacy_chat_completions_create(self): - assert_llm_span(self.request, self.model) + assert_llm_span(self.request, self.MODEL) def setup_openai_legacy_completions_create(self): - self.model = "gpt-4.1" - self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-legacy-completions.create") + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-legacy-completions.create") def test_openai_legacy_completions_create(self): - assert_llm_span(self.request, self.model) + assert_llm_span(self.request, self.MODEL) def setup_openai_async_responses_create(self): - self.model = "gpt-4.1" - self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-async-responses.create") + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-async-responses.create") def test_openai_async_responses_create(self): - assert_llm_span(self.request, self.model) + assert_llm_span(self.request, self.MODEL) def setup_openai_async_chat_completions_create(self): - self.model = "gpt-4.1" - self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-async-chat.completions.create") + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-async-chat.completions.create") def test_openai_async_chat_completions_create(self): - assert_llm_span(self.request, self.model) + assert_llm_span(self.request, self.MODEL) def setup_openai_async_completions_create(self): - self.model = "gpt-4.1" - self.request = weblog.get("/llm?model=gpt-4.1&operation=openai-async-completions.create") + self.request = weblog.get(f"/llm?model={self.MODEL}&operation=openai-async-completions.create") def test_openai_async_completions_create(self): - assert_llm_span(self.request, self.model) + assert_llm_span(self.request, self.MODEL) def setup_root_no_llm(self): # Baseline request to root endpoint should not produce LLM tags From 857c6b5ace44e5909c5b7f1a209897044eb5803d Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Thu, 19 Feb 2026 12:52:55 +0100 Subject: [PATCH 05/10] Inject rules --- tests/appsec/api_security/test_endpoints.py | 4 +- tests/appsec/rasp/rasp_ruleset.json | 44 +++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/tests/appsec/api_security/test_endpoints.py b/tests/appsec/api_security/test_endpoints.py index 525327254ff..df0e01a28d9 100644 --- a/tests/appsec/api_security/test_endpoints.py +++ b/tests/appsec/api_security/test_endpoints.py @@ -7,13 +7,13 @@ def assert_llm_span(request: HttpResponse, model: str) -> None: span = interfaces.library.get_root_span(request) assert span["meta"]["appsec.events.llm.call.provider"] == "openai" assert span["meta"]["appsec.events.llm.call.model"] == model - assert span["metrics"]["_sampling_priority_v1"] == 1 + assert span["metrics"]["_sampling_priority_v1"] == 2 @rfc( "https://docs.google.com/document/d/1TIFxbtbkldjOA6S5JFlCTMfqniXfJXZmDKptI5w2pnk/edit?tab=t.0#heading=h.xtljwwxyhqk7" ) -@scenarios.appsec_api_security +@scenarios.appsec_rasp @features.api_llm_endpoint class Test_LLM_Endpoint: """Tests for the /llm endpoint capturing LLM interaction metadata.""" diff --git a/tests/appsec/rasp/rasp_ruleset.json b/tests/appsec/rasp/rasp_ruleset.json index 53459732cc5..088db6a7f77 100644 --- a/tests/appsec/rasp/rasp_ruleset.json +++ b/tests/appsec/rasp/rasp_ruleset.json @@ -823,6 +823,50 @@ } } } + }, + { + "id": "llm-001-000", + "name": "LLM call", + "tags": { + "type": "llm.event", + "category": "business_logic", + "module": "business_logic" + }, + "min_version": "1.25.0", + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.business_logic.llm.event", + "key_path": [ + "provider" + ] + } + ] + }, + "operator": "exists" + } + ], + "transformers": [], + "output": { + "event": false, + "keep": true, + "attributes": { + "appsec.events.llm.call.provider": { + "address": "server.business_logic.llm.event", + "key_path": [ + "provider" + ] + }, + "appsec.events.llm.call.model": { + "address": "server.business_logic.llm.event", + "key_path": [ + "model" + ] + } + } + } } ] } \ No newline at end of file From 7cb22da4959337acd1aeefbb824f1ab20e313805 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Thu, 19 Feb 2026 15:57:33 +0100 Subject: [PATCH 06/10] Add fake responses to openai calls --- utils/_context/_scenarios/appsec_rasp.py | 1 + utils/build/docker/internal_server/app.py | 92 +++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/utils/_context/_scenarios/appsec_rasp.py b/utils/_context/_scenarios/appsec_rasp.py index 7ffd6f48fa6..4d797e034a6 100644 --- a/utils/_context/_scenarios/appsec_rasp.py +++ b/utils/_context/_scenarios/appsec_rasp.py @@ -24,6 +24,7 @@ def __init__( # added to test Test_ExtendedRequestBodyCollection "DD_APPSEC_RASP_COLLECT_REQUEST_BODY": "true", "DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE": "1.0", + "OPENAI_BASE_URL": "http://internal_server:8089", } merged_env = default_env | weblog_env diff --git a/utils/build/docker/internal_server/app.py b/utils/build/docker/internal_server/app.py index 4e50665ce44..bb044f5159f 100644 --- a/utils/build/docker/internal_server/app.py +++ b/utils/build/docker/internal_server/app.py @@ -1,5 +1,6 @@ import os import signal +import time import urllib.parse import fastapi @@ -146,6 +147,97 @@ async def payment_intents(request: fastapi.Request): return fastapi.responses.JSONResponse({"error": {"type": "api_error", "message": str(e)}}, status_code=500) +# Mock OpenAI API endpoints for LLM tests (OPENAI_BASE_URL=http://internal_server:8089). +def _openai_fake_usage() -> dict: + return {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3} + + +@app.post("/v1/chat/completions", response_class=fastapi.responses.JSONResponse) +async def openai_chat_completions(request: fastapi.Request): + """Mock for OpenAI Chat Completions API (POST /chat/completions).""" + try: + body = await request.json() if request.headers.get("content-length") else {} + except Exception: + body = {} + model = body.get("model", "gpt-4.1") + return fastapi.responses.JSONResponse( + { + "id": "chatcmpl-fake-internal", + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Fake response from internal_server mock."}, + "finish_reason": "stop", + } + ], + "usage": _openai_fake_usage(), + }, + status_code=200, + ) + + +@app.post("/v1/completions", response_class=fastapi.responses.JSONResponse) +async def openai_completions(request: fastapi.Request): + """Mock for OpenAI Completions API (POST /completions, legacy).""" + try: + body = await request.json() if request.headers.get("content-length") else {} + except Exception: + body = {} + model = body.get("model", "text-davinci-003") + return fastapi.responses.JSONResponse( + { + "id": "cmpl-fake-internal", + "object": "text_completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "text": "Fake completion from internal_server mock.", + "index": 0, + "finish_reason": "stop", + "logprobs": None, + } + ], + "usage": _openai_fake_usage(), + }, + status_code=200, + ) + + +@app.post("/v1/responses", response_class=fastapi.responses.JSONResponse) +async def openai_responses(request: fastapi.Request): + """Mock for OpenAI Responses API (POST /responses).""" + try: + body = await request.json() if request.headers.get("content-length") else {} + except Exception: + body = {} + model = body.get("model", "gpt-4.1") + return fastapi.responses.JSONResponse( + { + "id": "resp-fake-internal", + "object": "response", + "created": int(time.time()), + "model": model, + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Fake response from internal_server mock."}], + } + ], + "usage": { + "input_tokens": 1, + "output_tokens": 2, + "total_tokens": 3, + }, + }, + status_code=200, + ) + + @app.get("/shutdown") async def shutdown(): os.kill(os.getpid(), signal.SIGTERM) From 349ab131b1812a657e3305dd30d0b9d9a47feaa3 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Fri, 20 Feb 2026 11:59:49 +0100 Subject: [PATCH 07/10] Fix mock responses --- docs/weblog/end-to-end_weblog.md | 26 +++---- utils/build/docker/internal_server/app.py | 30 +++++-- utils/build/docker/php/apache-mod/php.conf | 1 + utils/build/docker/php/common/composer.json | 3 +- utils/build/docker/php/common/llm.php | 86 +++++++++++++++++++++ utils/build/docker/php/php-fpm/php-fpm.conf | 1 + 6 files changed, 127 insertions(+), 20 deletions(-) create mode 100644 utils/build/docker/php/common/llm.php diff --git a/docs/weblog/end-to-end_weblog.md b/docs/weblog/end-to-end_weblog.md index d2b3386c3cb..fa51fa790e9 100644 --- a/docs/weblog/end-to-end_weblog.md +++ b/docs/weblog/end-to-end_weblog.md @@ -1265,23 +1265,23 @@ If an error happens, the endpoint must respond with a 403 error code. ### GET /llm -For now this endpoint will only be implemented by Python and Node.js +This endpoint is implemented by Python, Node.js, and PHP (using openai-php/client). This endpoint collects interactions with LLMs. The request will have the following query parameters - `model`: Identifies the LLM model invoked. Examples are: `gpt-4.1`, `gpt-4o-mini`, `text-davinci-003`. - `operation`: Instead of having one each point for each function wrapped, this parameter will be used to decide what method to trigger. The following table maps all operation values to wrapped method: -| Value | Python mapped method | Node.js mapped method | -| --- | --- | --- | -| `openai-latest-responses.create` | `OpenAI().responses.create(...)` | `client.responses.create` | -| `openai-latest-chat.completions.create` | `OpenAI().chat.completions.create(...)` | `client.chat.completions.create` | -| `openai-latest-completions.create` | `OpenAI().completions.create(...)` | `client.completions.create` | -| `openai-legacy-chat.completions.create` | `openai.ChatCompletion.create` | `openai.createChatCompletion` | -| `openai-legacy-completions.create` | `openai.Completion.create` | `openai.createCompletion` | -| `openai-async-responses.create` | `AsyncOpenAI().responses.create(...)` | not implemented | -| `openai-async-chat.completions.create` | `AsyncOpenAI().chat.completions.create(...)` | not implemented | -| `openai-async-completions.create` | `AsyncOpenAI().completions.create(...)` | not implemented | - -For exmaple a call to `/llm?model=gpt-4.1&operation=openai-latest-responses.create`(url encoded) will do the will require that python does thes following call +| Value | Python mapped method | Node.js mapped method | PHP mapped method | +| --- | --- | --- | --- | +| `openai-latest-responses.create` | `OpenAI().responses.create(...)` | `client.responses.create` | `$client->responses()->create(...)` | +| `openai-latest-chat.completions.create` | `OpenAI().chat.completions.create(...)` | `client.chat.completions.create` | `$client->chat()->create(...)` | +| `openai-latest-completions.create` | `OpenAI().completions.create(...)` | `client.completions.create` | `$client->completions()->create(...)` | +| `openai-legacy-chat.completions.create` | `openai.ChatCompletion.create` | `openai.createChatCompletion` | `$client->chat()->create(...)` | +| `openai-legacy-completions.create` | `openai.Completion.create` | `openai.createCompletion` | `$client->completions()->create(...)` | +| `openai-async-responses.create` | `AsyncOpenAI().responses.create(...)` | not implemented | `$client->responses()->create(...)` | +| `openai-async-chat.completions.create` | `AsyncOpenAI().chat.completions.create(...)` | not implemented | `$client->chat()->create(...)` | +| `openai-async-completions.create` | `AsyncOpenAI().completions.create(...)` | not implemented | `$client->completions()->create(...)` | + +For example a call to `/llm?model=gpt-4.1&operation=openai-latest-responses.create` (URL-encoded) will require that Python does the following call ``` OpenAI().responses.create(model="gpt-4.1", ...) ``` diff --git a/utils/build/docker/internal_server/app.py b/utils/build/docker/internal_server/app.py index bb044f5159f..ecec8be5eac 100644 --- a/utils/build/docker/internal_server/app.py +++ b/utils/build/docker/internal_server/app.py @@ -152,7 +152,7 @@ def _openai_fake_usage() -> dict: return {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3} -@app.post("/v1/chat/completions", response_class=fastapi.responses.JSONResponse) +@app.post("/chat/completions", response_class=fastapi.responses.JSONResponse) async def openai_chat_completions(request: fastapi.Request): """Mock for OpenAI Chat Completions API (POST /chat/completions).""" try: @@ -179,7 +179,7 @@ async def openai_chat_completions(request: fastapi.Request): ) -@app.post("/v1/completions", response_class=fastapi.responses.JSONResponse) +@app.post("/completions", response_class=fastapi.responses.JSONResponse) async def openai_completions(request: fastapi.Request): """Mock for OpenAI Completions API (POST /completions, legacy).""" try: @@ -207,9 +207,11 @@ async def openai_completions(request: fastapi.Request): ) -@app.post("/v1/responses", response_class=fastapi.responses.JSONResponse) +@app.post("/responses", response_class=fastapi.responses.JSONResponse) async def openai_responses(request: fastapi.Request): - """Mock for OpenAI Responses API (POST /responses).""" + """Mock for OpenAI Responses API (POST /responses). + Shape matches openai-php/client CreateResponse (created_at, status, output with output_text content). + """ try: body = await request.json() if request.headers.get("content-length") else {} except Exception: @@ -219,18 +221,34 @@ async def openai_responses(request: fastapi.Request): { "id": "resp-fake-internal", "object": "response", - "created": int(time.time()), + "created_at": int(time.time()), + "status": "completed", "model": model, "output": [ { "type": "message", + "id": "msg-fake-internal", "role": "assistant", - "content": [{"type": "text", "text": "Fake response from internal_server mock."}], + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Fake response from internal_server mock.", + "annotations": [], + } + ], } ], + "output_text": "Fake response from internal_server mock.", + "parallel_tool_calls": False, + "tool_choice": "none", + "tools": [], + "store": True, "usage": { "input_tokens": 1, + "input_tokens_details": {"cached_tokens": 0}, "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3, }, }, diff --git a/utils/build/docker/php/apache-mod/php.conf b/utils/build/docker/php/apache-mod/php.conf index 1c56b062c20..aaf88b2f4e9 100644 --- a/utils/build/docker/php/apache-mod/php.conf +++ b/utils/build/docker/php/apache-mod/php.conf @@ -27,6 +27,7 @@ RewriteRule "^/load_dependency$" "/load_dependency/" RewriteRule "^/signup$" "/signup/" RewriteRule "^/shell_execution$" "/shell_execution/" + RewriteRule "^/llm$" "/llm/" RewriteCond /var/www/html/%{REQUEST_URI} !-f RewriteRule "^/rasp/(.*)" "/rasp/$1.php" [L] RewriteRule "^/api_security.sampling/.*" "/api_security_sampling.php$0" [L] diff --git a/utils/build/docker/php/common/composer.json b/utils/build/docker/php/common/composer.json index 4abe446e7d4..01981163618 100644 --- a/utils/build/docker/php/common/composer.json +++ b/utils/build/docker/php/common/composer.json @@ -3,7 +3,8 @@ "type": "project", "require": { "weblog/acme": "*", - "monolog/monolog": "*" + "monolog/monolog": "*", + "openai-php/client": "*" }, "repositories": [ { diff --git a/utils/build/docker/php/common/llm.php b/utils/build/docker/php/common/llm.php new file mode 100644 index 00000000000..c203ab2a29a --- /dev/null +++ b/utils/build/docker/php/common/llm.php @@ -0,0 +1,86 @@ + 'Missing or empty query parameters: model, operation']); + exit; +} + +$baseUri = getenv('OPENAI_BASE_URL') ?: 'http://internal_server:8089/v1'; +$client = OpenAI::factory() + ->withApiKey('sk-fake') + ->withBaseUri($baseUri) + ->make(); + +$params = ['model' => $model]; + +try { + switch ($operation) { + case 'openai-latest-responses.create': + $params['input'] = 'Hello'; + $response = $client->responses()->create($params); + break; + case 'openai-latest-chat.completions.create': + $params['messages'] = [['role' => 'user', 'content' => 'Hello']]; + $response = $client->chat()->create($params); + break; + case 'openai-latest-completions.create': + $params['prompt'] = 'Hello'; + $response = $client->completions()->create($params); + break; + case 'openai-legacy-chat.completions.create': + $params['messages'] = [['role' => 'user', 'content' => 'Hello']]; + $response = $client->chat()->create($params); + break; + case 'openai-legacy-completions.create': + $params['prompt'] = 'Hello'; + $response = $client->completions()->create($params); + break; + case 'openai-async-responses.create': + $params['input'] = 'Hello'; + $response = $client->responses()->create($params); + break; + case 'openai-async-chat.completions.create': + $params['messages'] = [['role' => 'user', 'content' => 'Hello']]; + $response = $client->chat()->create($params); + break; + case 'openai-async-completions.create': + $params['prompt'] = 'Hello'; + $response = $client->completions()->create($params); + break; + default: + http_response_code(400); + echo json_encode(['error' => 'Unknown operation: ' . $operation]); + exit; + } + + // Return a simple success payload; the client returns response objects + echo json_encode([ + 'model' => $model, + 'operation' => $operation, + 'status' => 'ok', + ]); +} catch (Throwable $e) { + http_response_code(500); + echo json_encode([ + 'error' => $e->getMessage(), + 'operation' => $operation, + ]); +} diff --git a/utils/build/docker/php/php-fpm/php-fpm.conf b/utils/build/docker/php/php-fpm/php-fpm.conf index 47e2bffb185..e54aa228a5c 100644 --- a/utils/build/docker/php/php-fpm/php-fpm.conf +++ b/utils/build/docker/php/php-fpm/php-fpm.conf @@ -27,6 +27,7 @@ RewriteRule "^/requestdownstream$" "/requestdownstream/" RewriteRule "^/resource_renaming$" "/resource_renaming/" RewriteRule "^/returnheaders$" "/returnheaders/" + RewriteRule "^/llm$" "/llm/" RewriteRule "^/session/new$" "/session_new.php" RewriteRule "^/user_login_failure_event_v2$" "/user_login_failure_event_v2/" RewriteRule "^/user_login_success_event_v2$" "/user_login_success_event_v2/" From cc5dc1aad208deb31aef5110ecef0c5ec5652bbb Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Fri, 20 Feb 2026 12:11:49 +0100 Subject: [PATCH 08/10] Explains mock openai responses --- docs/weblog/end-to-end_weblog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/weblog/end-to-end_weblog.md b/docs/weblog/end-to-end_weblog.md index fa51fa790e9..a0c4ced7090 100644 --- a/docs/weblog/end-to-end_weblog.md +++ b/docs/weblog/end-to-end_weblog.md @@ -1288,6 +1288,7 @@ OpenAI().responses.create(model="gpt-4.1", ...) This approach makes the endpoint ready to be expanded in the future. +In scenarios that use this endpoint, `OPENAI_BASE_URL` is set to the system-tests internal server, which mocks the OpenAI API responses. ## Weblog specification From 888f9c33cb6c6dbce4f602f64fe4ef7500908fa4 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Mon, 23 Feb 2026 10:13:35 +0100 Subject: [PATCH 09/10] [PHP] Enable llm tests php (#6354) --- manifests/php.yml | 6 ++-- utils/build/docker/php/common/composer.json | 3 +- utils/build/docker/php/common/llm.php | 38 ++++++--------------- 3 files changed, 16 insertions(+), 31 deletions(-) diff --git a/manifests/php.yml b/manifests/php.yml index 0f6d91dbeee..4fb70a976f0 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -35,9 +35,9 @@ manifest: tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) - tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: irrelevant (language not implementing this feature) - tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: irrelevant (language not implementing this feature) - tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: irrelevant (language not implementing this feature) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: v1.17.0-dev + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: v1.17.0-dev + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: v1.17.0-dev tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature) diff --git a/utils/build/docker/php/common/composer.json b/utils/build/docker/php/common/composer.json index 01981163618..b82c054e130 100644 --- a/utils/build/docker/php/common/composer.json +++ b/utils/build/docker/php/common/composer.json @@ -4,7 +4,8 @@ "require": { "weblog/acme": "*", "monolog/monolog": "*", - "openai-php/client": "*" + "openai-php/client": "*", + "guzzlehttp/guzzle": "*" }, "repositories": [ { diff --git a/utils/build/docker/php/common/llm.php b/utils/build/docker/php/common/llm.php index c203ab2a29a..b534479973c 100644 --- a/utils/build/docker/php/common/llm.php +++ b/utils/build/docker/php/common/llm.php @@ -10,8 +10,6 @@ require_once __DIR__ . '/vendor/autoload.php'; -use OpenAI\OpenAI; - header('Content-Type: application/json'); $model = $_GET['model'] ?? null; @@ -23,11 +21,17 @@ exit; } -$baseUri = getenv('OPENAI_BASE_URL') ?: 'http://internal_server:8089/v1'; -$client = OpenAI::factory() - ->withApiKey('sk-fake') - ->withBaseUri($baseUri) - ->make(); +try { + $baseUri = (string) (getenv('OPENAI_BASE_URL') ?: 'http://internal_server:8089/v1'); + $client = \OpenAI::factory() + ->withApiKey('sk-fake') + ->withBaseUri($baseUri) + ->make(); +} catch (Throwable $e) { + http_response_code(500); + echo json_encode(['error' => 'OpenAI client init failed: ' . $e->getMessage()]); + exit; +} $params = ['model' => $model]; @@ -45,26 +49,6 @@ $params['prompt'] = 'Hello'; $response = $client->completions()->create($params); break; - case 'openai-legacy-chat.completions.create': - $params['messages'] = [['role' => 'user', 'content' => 'Hello']]; - $response = $client->chat()->create($params); - break; - case 'openai-legacy-completions.create': - $params['prompt'] = 'Hello'; - $response = $client->completions()->create($params); - break; - case 'openai-async-responses.create': - $params['input'] = 'Hello'; - $response = $client->responses()->create($params); - break; - case 'openai-async-chat.completions.create': - $params['messages'] = [['role' => 'user', 'content' => 'Hello']]; - $response = $client->chat()->create($params); - break; - case 'openai-async-completions.create': - $params['prompt'] = 'Hello'; - $response = $client->completions()->create($params); - break; default: http_response_code(400); echo json_encode(['error' => 'Unknown operation: ' . $operation]); From 4a7f6a5289647e875a457d26acdea0337af3d063 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Mon, 23 Feb 2026 10:49:02 +0100 Subject: [PATCH 10/10] Mark php tests as missing --- manifests/php.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/php.yml b/manifests/php.yml index 4fb70a976f0..4c5c46500c4 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -35,9 +35,9 @@ manifest: tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_chat_completions_create: irrelevant (language not implementing this feature) tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_completions_create: irrelevant (language not implementing this feature) tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_async_responses_create: irrelevant (language not implementing this feature) - tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: v1.17.0-dev - tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: v1.17.0-dev - tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: v1.17.0-dev + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_chat_completions_create: missing_feature (waiting until PR gets merged) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_completions_create: missing_feature (waiting until PR gets merged) + tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_latest_responses_create: missing_feature (waiting until PR gets merged) tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_chat_completions_create: irrelevant (language not implementing this feature) tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_openai_legacy_completions_create: irrelevant (language not implementing this feature) tests/appsec/api_security/test_endpoints.py::Test_LLM_Endpoint::test_root_has_no_llm_tags: irrelevant (language not implementing this feature)