From 0fe616dc6e365e454537b40f4392ec92706fcd27 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Thu, 19 Feb 2026 16:17:47 +0100 Subject: [PATCH 01/17] Push llm event address --- .../Integrations/OpenAI/OpenAIIntegration.php | 82 ++++++++++++------- 1 file changed, 51 insertions(+), 31 deletions(-) diff --git a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php index d3ad0024e67..f30e051ea3b 100644 --- a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php +++ b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php @@ -67,34 +67,34 @@ public static function init(): int $logger = \dd_trace_env_config('DD_OPENAI_LOGS_ENABLED') ? new DatadogLogger() : null; $targets = [ - ['OpenAI\Resources\Completions', 'create', 'createCompletion', 'POST', '/v1/completions'], - ['OpenAI\Resources\Chat', 'create', 'createChatCompletion', 'POST', '/v1/chat/completions'], - ['OpenAI\Resources\Embeddings', 'create', 'createEmbedding', 'POST', '/v1/embeddings'], - ['OpenAI\Resources\Models', 'list', 'listModels', 'GET', '/v1/models'], - ['OpenAI\Resources\Files', 'list', 'listFiles', 'GET', '/v1/files'], - ['OpenAI\Resources\FineTuning', 'listJobs', 'listFineTunes', 'GET', '/v1/fine-tunes'], - ['OpenAI\Resources\Models', 'retrieve', 'retrieveModel', 'GET', '/v1/models/*'], - ['OpenAI\Resources\Files', 'retrieve', 'retrieveFile', 'GET', '/v1/files/*'], - ['OpenAI\Resources\FineTuning', 'retrieveJob', 'retrieveFineTune', 'GET', '/v1/fine-tunes/*'], - ['OpenAI\Resources\Models', 'delete', 'deleteModel', 'DELETE', '/v1/models/*'], - ['OpenAI\Resources\Files', 'delete', 'deleteFile', 'DELETE', '/v1/files/*'], - ['OpenAI\Resources\Images', 'create', 'createImage', 'POST', '/v1/images/generations'], - ['OpenAI\Resources\Images', 'edit', 'createImageEdit', 'POST', '/v1/images/edits'], - ['OpenAI\Resources\Images', 'variation', 'createImageVariation', 'POST', '/v1/images/variations'], - ['OpenAI\Resources\Audio', 'transcribe', 'createTranscription', 'POST', '/v1/audio/transcriptions'], - ['OpenAI\Resources\Audio', 'translate', 'createTranslation', 'POST', '/v1/audio/translations'], - ['OpenAI\Resources\Moderations', 'create', 'createModeration', 'POST', '/v1/moderations'], - ['OpenAI\Resources\Files', 'upload', 'createFile', 'POST', '/v1/files'], - ['OpenAI\Resources\Files', 'download', 'downloadFile', 'GET', '/v1/files/*/content'], - ['OpenAI\Resources\FineTuning', 'createJob', 'createFineTune', 'POST', '/v1/fine-tunes'], - ['OpenAI\Resources\FineTunes', 'cancel', 'cancelFineTune', 'POST', '/v1/fine-tunes/*/cancel'], - ['OpenAI\Resources\FineTunes', 'listEvents', 'listFineTuneEvents', 'GET', '/v1/fine-tunes/*/events'], + ['OpenAI\Resources\Completions', 'create', 'createCompletion', 'POST', '/v1/completions', true], + ['OpenAI\Resources\Chat', 'create', 'createChatCompletion', 'POST', '/v1/chat/completions', true], + ['OpenAI\Resources\Embeddings', 'create', 'createEmbedding', 'POST', '/v1/embeddings', false], + ['OpenAI\Resources\Models', 'list', 'listModels', 'GET', '/v1/models', false], + ['OpenAI\Resources\Files', 'list', 'listFiles', 'GET', '/v1/files', false], + ['OpenAI\Resources\FineTuning', 'listJobs', 'listFineTunes', 'GET', '/v1/fine-tunes', false], + ['OpenAI\Resources\Models', 'retrieve', 'retrieveModel', 'GET', '/v1/models/*', false], + ['OpenAI\Resources\Files', 'retrieve', 'retrieveFile', 'GET', '/v1/files/*', false], + ['OpenAI\Resources\FineTuning', 'retrieveJob', 'retrieveFineTune', 'GET', '/v1/fine-tunes/*', false], + ['OpenAI\Resources\Models', 'delete', 'deleteModel', 'DELETE', '/v1/models/*', false], + ['OpenAI\Resources\Files', 'delete', 'deleteFile', 'DELETE', '/v1/files/*', false], + ['OpenAI\Resources\Images', 'create', 'createImage', 'POST', '/v1/images/generations', false], + ['OpenAI\Resources\Images', 'edit', 'createImageEdit', 'POST', '/v1/images/edits', false], + ['OpenAI\Resources\Images', 'variation', 'createImageVariation', 'POST', '/v1/images/variations', false], + ['OpenAI\Resources\Audio', 'transcribe', 'createTranscription', 'POST', '/v1/audio/transcriptions', false], + ['OpenAI\Resources\Audio', 'translate', 'createTranslation', 'POST', '/v1/audio/translations', false], + ['OpenAI\Resources\Moderations', 'create', 'createModeration', 'POST', '/v1/moderations', false], + ['OpenAI\Resources\Files', 'upload', 'createFile', 'POST', '/v1/files', false], + ['OpenAI\Resources\Files', 'download', 'downloadFile', 'GET', '/v1/files/*/content', false], + ['OpenAI\Resources\FineTuning', 'createJob', 'createFineTune', 'POST', '/v1/fine-tunes', false], + ['OpenAI\Resources\FineTunes', 'cancel', 'cancelFineTune', 'POST', '/v1/fine-tunes/*/cancel', false], + ['OpenAI\Resources\FineTunes', 'listEvents', 'listFineTuneEvents', 'GET', '/v1/fine-tunes/*/events', false], ]; $streamedTargets = [ - ['OpenAI\Resources\Completions', 'createStreamed', 'createCompletion', 'POST', '/v1/completions'], - ['OpenAI\Resources\Chat', 'createStreamed', 'createChatCompletion', 'POST', '/v1/chat/completions'], - ['OpenAI\Resources\FineTunes', 'listEventsStreamed', 'listFineTuneEvents', 'GET', '/v1/fine-tunes/*/events'], + ['OpenAI\Resources\Completions', 'createStreamed', 'createCompletion', 'POST', '/v1/completions', true], + ['OpenAI\Resources\Chat', 'createStreamed', 'createChatCompletion', 'POST', '/v1/chat/completions', true], + ['OpenAI\Resources\FineTunes', 'listEventsStreamed', 'listFineTuneEvents', 'GET', '/v1/fine-tunes/*/events', false], ]; \DDTrace\hook_method( @@ -152,13 +152,13 @@ static function ($This, $scope, $args) { ); }; - foreach ($targets as [$class, $method, $operationID, $httpMethod, $endpoint]) { + foreach ($targets as [$class, $method, $operationID, $httpMethod, $endpoint, $reportAppsec]) { \DDTrace\trace_method( $class, $method, [ 'prehook' => $handleRequestPrehook(false, $operationID), - 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint) { + 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportAppsec) { /** @var (\OpenAI\Contracts\ResponseContract&\OpenAI\Contracts\ResponseHasMetaInformationContract)|string $response */ // Files::download - i.e., downloadFile - returns a string instead of a Response instance self::handleResponse( @@ -168,19 +168,20 @@ static function ($This, $scope, $args) { response: \is_string($response) ? $response : ($response ? $response->toArray() : []), httpMethod: $httpMethod, endpoint: $endpoint, + reportAppsec: $reportAppsec, ); } ] ); } - foreach ($streamedTargets as [$class, $method, $operationID, $httpMethod, $endpoint]) { + foreach ($streamedTargets as [$class, $method, $operationID, $httpMethod, $endpoint, $reportAppsec]) { \DDTrace\trace_method( $class, $method, [ 'prehook' => $handleRequestPrehook(true, $operationID), - 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint) { + 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportAppsec) { /** @var \OpenAI\Responses\StreamResponse $response */ self::handleStreamedResponse( span: $span, @@ -189,6 +190,7 @@ static function ($This, $scope, $args) { response: $response, httpMethod: $httpMethod, endpoint: $endpoint, + reportAppsec: $reportAppsec, ); } ] @@ -419,6 +421,7 @@ public static function handleResponse( array|string $response, string $httpMethod, string $endpoint, + bool $reportAppsec = false, ) { $operationID = \explode('/', $span->resource)[0]; @@ -427,6 +430,14 @@ public static function handleResponse( $response = ['bytes' => \strlen($response)]; } + $model = $headers['openai-model'] ?? $response['model'] ?? null; + if ($reportAppsec && function_exists('datadog\appsec\push_addresses')) { + \datadog\appsec\push_addresses(["server.business_logic.llm.event" => [ + 'provider' => 'openai', + 'model' => $model, + ]]); + } + $tags = [ 'openai.request.endpoint' => $endpoint, 'openai.request.method' => $httpMethod, @@ -434,7 +445,7 @@ public static function handleResponse( 'openai.organization.id' => $response['organization_id'] ?? null, // Only available in fine-tunes endpoint 'openai.organization.name' => $headers['openai-organization'] ?? null, - 'openai.response.model' => $headers['openai-model'] ?? $response['model'] ?? null, // Specific model, often undefined + 'openai.response.model' => $model, // Specific model, often undefined 'openai.response.id' => $headers['x-request-id'] ?? $response['id'] ?? null, // Common creation value, numeric epoch 'openai.response.deleted' => $response['deleted'] ?? null, // Common boolean field in delete responses @@ -1011,17 +1022,26 @@ public static function handleStreamedResponse( StreamResponse $response, string $httpMethod, string $endpoint, + bool $reportAppsec = false, ) { $responseArray = self::readAndStoreStreamedResponse($span, $response); $operationID = \explode('/', $span->resource)[0]; + $model = $headers['openai-model'] ?? null; + if ($reportAppsec && function_exists('datadog\appsec\push_addresses')) { + \datadog\appsec\push_addresses(["server.business_logic.llm.event" => [ + 'provider' => 'openai', + 'model' => $model, + ]]); + } + $tags = [ 'openai.request.endpoint' => $endpoint, 'openai.request.method' => $httpMethod, 'openai.organization.name' => $headers['openai-organization'] ?? null, - 'openai.response.model' => $headers['openai-model'] ?? null, // Specific model, often undefined + 'openai.response.model' => $model, // Specific model, often undefined 'openai.response.id' => $headers['x-request-id'] ?? null, // Common creation value, numeric epoch ]; From ddb5115172212e5af9757c902a9568380e309063 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Thu, 19 Feb 2026 18:19:45 +0100 Subject: [PATCH 02/17] Add missing operations --- .../Integrations/OpenAI/OpenAIIntegration.php | 92 +++++++++++-------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php index f30e051ea3b..cd8cb2df31a 100644 --- a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php +++ b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php @@ -67,34 +67,36 @@ public static function init(): int $logger = \dd_trace_env_config('DD_OPENAI_LOGS_ENABLED') ? new DatadogLogger() : null; $targets = [ - ['OpenAI\Resources\Completions', 'create', 'createCompletion', 'POST', '/v1/completions', true], - ['OpenAI\Resources\Chat', 'create', 'createChatCompletion', 'POST', '/v1/chat/completions', true], - ['OpenAI\Resources\Embeddings', 'create', 'createEmbedding', 'POST', '/v1/embeddings', false], - ['OpenAI\Resources\Models', 'list', 'listModels', 'GET', '/v1/models', false], - ['OpenAI\Resources\Files', 'list', 'listFiles', 'GET', '/v1/files', false], - ['OpenAI\Resources\FineTuning', 'listJobs', 'listFineTunes', 'GET', '/v1/fine-tunes', false], - ['OpenAI\Resources\Models', 'retrieve', 'retrieveModel', 'GET', '/v1/models/*', false], - ['OpenAI\Resources\Files', 'retrieve', 'retrieveFile', 'GET', '/v1/files/*', false], - ['OpenAI\Resources\FineTuning', 'retrieveJob', 'retrieveFineTune', 'GET', '/v1/fine-tunes/*', false], - ['OpenAI\Resources\Models', 'delete', 'deleteModel', 'DELETE', '/v1/models/*', false], - ['OpenAI\Resources\Files', 'delete', 'deleteFile', 'DELETE', '/v1/files/*', false], - ['OpenAI\Resources\Images', 'create', 'createImage', 'POST', '/v1/images/generations', false], - ['OpenAI\Resources\Images', 'edit', 'createImageEdit', 'POST', '/v1/images/edits', false], - ['OpenAI\Resources\Images', 'variation', 'createImageVariation', 'POST', '/v1/images/variations', false], - ['OpenAI\Resources\Audio', 'transcribe', 'createTranscription', 'POST', '/v1/audio/transcriptions', false], - ['OpenAI\Resources\Audio', 'translate', 'createTranslation', 'POST', '/v1/audio/translations', false], - ['OpenAI\Resources\Moderations', 'create', 'createModeration', 'POST', '/v1/moderations', false], - ['OpenAI\Resources\Files', 'upload', 'createFile', 'POST', '/v1/files', false], - ['OpenAI\Resources\Files', 'download', 'downloadFile', 'GET', '/v1/files/*/content', false], - ['OpenAI\Resources\FineTuning', 'createJob', 'createFineTune', 'POST', '/v1/fine-tunes', false], - ['OpenAI\Resources\FineTunes', 'cancel', 'cancelFineTune', 'POST', '/v1/fine-tunes/*/cancel', false], - ['OpenAI\Resources\FineTunes', 'listEvents', 'listFineTuneEvents', 'GET', '/v1/fine-tunes/*/events', false], + ['OpenAI\Resources\Completions', 'create', 'createCompletion', 'POST', '/v1/completions', true, true], + ['OpenAI\Resources\Chat', 'create', 'createChatCompletion', 'POST', '/v1/chat/completions', true, true], + ['OpenAI\Resources\Embeddings', 'create', 'createEmbedding', 'POST', '/v1/embeddings', true, false], + ['OpenAI\Resources\Models', 'list', 'listModels', 'GET', '/v1/models', true, false], + ['OpenAI\Resources\Files', 'list', 'listFiles', 'GET', '/v1/files', true, false], + ['OpenAI\Resources\FineTuning', 'listJobs', 'listFineTunes', 'GET', '/v1/fine-tunes', true, false], + ['OpenAI\Resources\Models', 'retrieve', 'retrieveModel', 'GET', '/v1/models/*', true, false], + ['OpenAI\Resources\Files', 'retrieve', 'retrieveFile', 'GET', '/v1/files/*', true, false], + ['OpenAI\Resources\FineTuning', 'retrieveJob', 'retrieveFineTune', 'GET', '/v1/fine-tunes/*', true, false], + ['OpenAI\Resources\Models', 'delete', 'deleteModel', 'DELETE', '/v1/models/*', true, false], + ['OpenAI\Resources\Files', 'delete', 'deleteFile', 'DELETE', '/v1/files/*', true, false], + ['OpenAI\Resources\Images', 'create', 'createImage', 'POST', '/v1/images/generations', true, false], + ['OpenAI\Resources\Images', 'edit', 'createImageEdit', 'POST', '/v1/images/edits', true, false], + ['OpenAI\Resources\Images', 'variation', 'createImageVariation', 'POST', '/v1/images/variations', true, false], + ['OpenAI\Resources\Audio', 'transcribe', 'createTranscription', 'POST', '/v1/audio/transcriptions', true, false], + ['OpenAI\Resources\Audio', 'translate', 'createTranslation', 'POST', '/v1/audio/translations', true, false], + ['OpenAI\Resources\Moderations', 'create', 'createModeration', 'POST', '/v1/moderations', true, false], + ['OpenAI\Resources\Files', 'upload', 'createFile', 'POST', '/v1/files', true, false], + ['OpenAI\Resources\Files', 'download', 'downloadFile', 'GET', '/v1/files/*/content', true, false], + ['OpenAI\Resources\FineTuning', 'createJob', 'createFineTune', 'POST', '/v1/fine-tunes', true, false], + ['OpenAI\Resources\FineTunes', 'cancel', 'cancelFineTune', 'POST', '/v1/fine-tunes/*/cancel', true, false], + ['OpenAI\Resources\FineTunes', 'listEvents', 'listFineTuneEvents', 'GET', '/v1/fine-tunes/*/events', true, false], + ['OpenAI\Resources\Responses', 'create', 'createResponse', 'POST', '/v1/responses', false, true], ]; $streamedTargets = [ - ['OpenAI\Resources\Completions', 'createStreamed', 'createCompletion', 'POST', '/v1/completions', true], - ['OpenAI\Resources\Chat', 'createStreamed', 'createChatCompletion', 'POST', '/v1/chat/completions', true], - ['OpenAI\Resources\FineTunes', 'listEventsStreamed', 'listFineTuneEvents', 'GET', '/v1/fine-tunes/*/events', false], + ['OpenAI\Resources\Completions', 'createStreamed', 'createCompletion', 'POST', '/v1/completions', true, true], + ['OpenAI\Resources\Chat', 'createStreamed', 'createChatCompletion', 'POST', '/v1/chat/completions', true, true], + ['OpenAI\Resources\FineTunes', 'listEventsStreamed', 'listFineTuneEvents', 'GET', '/v1/fine-tunes/*/events', true, false], + ['OpenAI\Resources\Responses', 'create', 'createResponse', 'POST', '/v1/responses', false, true], ]; \DDTrace\hook_method( @@ -152,13 +154,13 @@ static function ($This, $scope, $args) { ); }; - foreach ($targets as [$class, $method, $operationID, $httpMethod, $endpoint, $reportAppsec]) { + foreach ($targets as [$class, $method, $operationID, $httpMethod, $endpoint, $reportApm, $reportAppsec]) { \DDTrace\trace_method( $class, $method, [ 'prehook' => $handleRequestPrehook(false, $operationID), - 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportAppsec) { + 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm, $reportAppsec) { /** @var (\OpenAI\Contracts\ResponseContract&\OpenAI\Contracts\ResponseHasMetaInformationContract)|string $response */ // Files::download - i.e., downloadFile - returns a string instead of a Response instance self::handleResponse( @@ -168,6 +170,7 @@ static function ($This, $scope, $args) { response: \is_string($response) ? $response : ($response ? $response->toArray() : []), httpMethod: $httpMethod, endpoint: $endpoint, + reportApm: $reportApm, reportAppsec: $reportAppsec, ); } @@ -181,7 +184,7 @@ static function ($This, $scope, $args) { $method, [ 'prehook' => $handleRequestPrehook(true, $operationID), - 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportAppsec) { + 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm, $reportAppsec) { /** @var \OpenAI\Responses\StreamResponse $response */ self::handleStreamedResponse( span: $span, @@ -190,6 +193,7 @@ static function ($This, $scope, $args) { response: $response, httpMethod: $httpMethod, endpoint: $endpoint, + reportApm: $reportApm, reportAppsec: $reportAppsec, ); } @@ -421,15 +425,10 @@ public static function handleResponse( array|string $response, string $httpMethod, string $endpoint, + bool $reportApm = true, bool $reportAppsec = false, ) { - $operationID = \explode('/', $span->resource)[0]; - - if ($operationID === 'downloadFile') { - $response = ['bytes' => \strlen($response)]; - } - $model = $headers['openai-model'] ?? $response['model'] ?? null; if ($reportAppsec && function_exists('datadog\appsec\push_addresses')) { \datadog\appsec\push_addresses(["server.business_logic.llm.event" => [ @@ -438,6 +437,18 @@ public static function handleResponse( ]]); } + if (!$reportApm) { + return; + } + + $operationID = \explode('/', $span->resource)[0]; + + if ($operationID === 'downloadFile') { + $response = ['bytes' => \strlen($response)]; + } + + + $tags = [ 'openai.request.endpoint' => $endpoint, 'openai.request.method' => $httpMethod, @@ -1022,13 +1033,10 @@ public static function handleStreamedResponse( StreamResponse $response, string $httpMethod, string $endpoint, + bool $reportApm = true, bool $reportAppsec = false, ) { - $responseArray = self::readAndStoreStreamedResponse($span, $response); - - $operationID = \explode('/', $span->resource)[0]; - $model = $headers['openai-model'] ?? null; if ($reportAppsec && function_exists('datadog\appsec\push_addresses')) { \datadog\appsec\push_addresses(["server.business_logic.llm.event" => [ @@ -1037,6 +1045,14 @@ public static function handleStreamedResponse( ]]); } + if (!$reportApm) { + return; + } + + $responseArray = self::readAndStoreStreamedResponse($span, $response); + + $operationID = \explode('/', $span->resource)[0]; + $tags = [ 'openai.request.endpoint' => $endpoint, 'openai.request.method' => $httpMethod, From c826cf7ed85a8712284c79d74cd68982016ecb31 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Fri, 20 Feb 2026 11:29:18 +0100 Subject: [PATCH 03/17] Add tests --- .../Integrations/OpenAI/OpenAIIntegration.php | 13 +++-- .../Integrations/OpenAI/Latest/OpenAITest.php | 47 +++++++++++++++++++ .../Integrations/OpenAI/Latest/composer.json | 3 ++ tests/OpenAI/Fixtures/Chat.php | 18 +++++++ .../Fixtures/Streams/ResponseCreate.txt | 10 ++++ 5 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 tests/OpenAI/Fixtures/Streams/ResponseCreate.txt diff --git a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php index cd8cb2df31a..d6539914474 100644 --- a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php +++ b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php @@ -135,14 +135,17 @@ static function ($This, $scope, $args) { } ); - $handleRequestPrehook = fn ($streamed, $operationID) => function (\DDTrace\SpanData $span, $args) use ($operationID, $streamed) { + $handleRequestPrehook = fn ($streamed, $operationID, $reportApm) => function (\DDTrace\SpanData $span, $args) use ($operationID, $streamed, $reportApm) { + if (!$reportApm) { + return; + } OpenAIIntegration::setServiceName($span); $clientData = ObjectKVStore::get($this, 'client_data'); if (\is_null($clientData)) { $transporter = ObjectKVStore::get($this, 'transporter'); $clientData = ObjectKVStore::get($transporter, 'client_data'); ObjectKVStore::put($this, 'client_data', $clientData); - } + } /** @var array{baseUri: string, headers: string, apiKey: ?string} $clientData */ OpenAIIntegration::handleRequest( span: $span, @@ -159,7 +162,7 @@ static function ($This, $scope, $args) { $class, $method, [ - 'prehook' => $handleRequestPrehook(false, $operationID), + 'prehook' => $handleRequestPrehook(false, $operationID, $reportApm), 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm, $reportAppsec) { /** @var (\OpenAI\Contracts\ResponseContract&\OpenAI\Contracts\ResponseHasMetaInformationContract)|string $response */ // Files::download - i.e., downloadFile - returns a string instead of a Response instance @@ -178,12 +181,12 @@ static function ($This, $scope, $args) { ); } - foreach ($streamedTargets as [$class, $method, $operationID, $httpMethod, $endpoint, $reportAppsec]) { + foreach ($streamedTargets as [$class, $method, $operationID, $httpMethod, $endpoint, $reportApm, $reportAppsec]) { \DDTrace\trace_method( $class, $method, [ - 'prehook' => $handleRequestPrehook(true, $operationID), + 'prehook' => $handleRequestPrehook(true, $operationID, $reportApm), 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm, $reportAppsec) { /** @var \OpenAI\Responses\StreamResponse $response */ self::handleStreamedResponse( diff --git a/tests/Integrations/OpenAI/Latest/OpenAITest.php b/tests/Integrations/OpenAI/Latest/OpenAITest.php index b536e49451b..f03e7ec15b9 100644 --- a/tests/Integrations/OpenAI/Latest/OpenAITest.php +++ b/tests/Integrations/OpenAI/Latest/OpenAITest.php @@ -5,6 +5,7 @@ use DDTrace\Tests\Common\IntegrationTestCase; use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Stream; +use datadog\appsec\AppsecStatus; class OpenAITest extends IntegrationTestCase { @@ -116,6 +117,11 @@ public function testCreateCompletion() ], 'user' => 'dd-trace' ]); + + $events = AppsecStatus::getInstance()->getEvents(['push_addresses'], ['server.business_logic.llm.event']); + $this->assertEquals(1, count($events)); + $this->assertEquals('openai', $events[0][0]["server.business_logic.llm.event"]['provider']); + $this->assertEquals('da-vince', $events[0][0]["server.business_logic.llm.event"]['model']); } public function testCreateChatCompletion() @@ -124,6 +130,24 @@ public function testCreateChatCompletion() 'model' => 'gpt-3.5-turbo', 'messages' => ['role' => 'user', 'content' => 'Hello!'], ]); + + $events = AppsecStatus::getInstance()->getEvents(['push_addresses'], ['server.business_logic.llm.event']); + $this->assertEquals(1, count($events)); + $this->assertEquals('openai', $events[0][0]["server.business_logic.llm.event"]['provider']); + $this->assertEquals('gpt-3.5-turbo', $events[0][0]["server.business_logic.llm.event"]['model']); + } + + public function testCreateResponse() + { + $this->call('responses', 'create', metaHeaders(), response(), [ + 'model' => 'gpt-3.5-turbo', + 'messages' => ['role' => 'user', 'content' => 'Hello!'], + ]); + + $events = AppsecStatus::getInstance()->getEvents(['push_addresses'], ['server.business_logic.llm.event']); + $this->assertEquals(1, count($events)); + $this->assertEquals('openai', $events[0][0]["server.business_logic.llm.event"]['provider']); + $this->assertEquals('gpt-3.5-turbo', $events[0][0]["server.business_logic.llm.event"]['model']); } public function testCreateChatCompletionWithMultipleRoles() @@ -404,6 +428,11 @@ public function testCreateCompletionStream() 'model' => 'gpt-3.5-turbo-instruct', 'prompt' => 'hi', ]); + + $events = AppsecStatus::getInstance()->getEvents(['push_addresses'], ['server.business_logic.llm.event']); + $this->assertEquals(1, count($events)); + $this->assertEquals('openai', $events[0][0]["server.business_logic.llm.event"]['provider']); + $this->assertEquals('gpt-3.5-turbo-instruct', $events[0][0]["server.business_logic.llm.event"]['model']); } public function testCreateChatCompletionStream() @@ -412,6 +441,24 @@ public function testCreateChatCompletionStream() 'model' => 'gpt-3.5-turbo', 'messages' => ['role' => 'user', 'content' => 'Hello!'], ]); + + $events = AppsecStatus::getInstance()->getEvents(['push_addresses'], ['server.business_logic.llm.event']); + $this->assertEquals(1, count($events)); + $this->assertEquals('openai', $events[0][0]["server.business_logic.llm.event"]['provider']); + $this->assertEquals('gpt-3.5-turbo', $events[0][0]["server.business_logic.llm.event"]['model']); + } + + public function testCreateResponseStream() + { + $this->callStreamed('responses', 'createStreamed', metaHeaders(), responseStream(), [ + 'model' => 'gpt-3.5-turbo', + 'messages' => ['role' => 'user', 'content' => 'Hello!'], + ]); + + $events = AppsecStatus::getInstance()->getEvents(['push_addresses'], ['server.business_logic.llm.event']); + $this->assertEquals(1, count($events)); + $this->assertEquals('openai', $events[0][0]["server.business_logic.llm.event"]['provider']); + $this->assertEquals('gpt-3.5-turbo', $events[0][0]["server.business_logic.llm.event"]['model']); } public function testListFineTuneEventsStream() diff --git a/tests/Integrations/OpenAI/Latest/composer.json b/tests/Integrations/OpenAI/Latest/composer.json index 0445bb70c63..304d72f18e0 100644 --- a/tests/Integrations/OpenAI/Latest/composer.json +++ b/tests/Integrations/OpenAI/Latest/composer.json @@ -3,5 +3,8 @@ "openai-php/client": "0.16.1", "guzzlehttp/guzzle": "^7.8.1", "guzzlehttp/psr7": "^2.6.2" + }, + "autoload-dev": { + "files": ["../../../../tests/Appsec/Mock.php"] } } \ No newline at end of file diff --git a/tests/OpenAI/Fixtures/Chat.php b/tests/OpenAI/Fixtures/Chat.php index 9bb471a22a3..e9be176f159 100644 --- a/tests/OpenAI/Fixtures/Chat.php +++ b/tests/OpenAI/Fixtures/Chat.php @@ -30,6 +30,16 @@ function chatCompletion(): array ]; } +function response(): array +{ + return [ + 'id' => 'chatcmpl-123', + 'object' => 'chat.completion', + 'created' => 1677652288, + 'model' => 'gpt-3.5-turbo-0125', + ]; +} + function chatCompletionDefaultExample(): array { return [ @@ -136,3 +146,11 @@ function chatCompletionStreamError() { return fopen(__DIR__.'/Streams/ChatCompletionCreateError.txt', 'r'); } + +/** + * @return resource + */ +function responseStream() +{ + return fopen(__DIR__.'/Streams/ResponseCreate.txt', 'r'); +} \ No newline at end of file diff --git a/tests/OpenAI/Fixtures/Streams/ResponseCreate.txt b/tests/OpenAI/Fixtures/Streams/ResponseCreate.txt new file mode 100644 index 00000000000..478ffae2e84 --- /dev/null +++ b/tests/OpenAI/Fixtures/Streams/ResponseCreate.txt @@ -0,0 +1,10 @@ +data: {"type":"response.created","response":{"id":"resp_67c9fdcecf488190bdd9a0409de3a1ec07b8b0ad4e5eb654","object":"response","created_at":1741290958,"status":"in_progress","error":null,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} +data: {"type":"response.in_progress","response":{"id":"resp_67c9fdcecf488190bdd9a0409de3a1ec07b8b0ad4e5eb654","object":"response","created_at":1741290958,"status":"in_progress","error":null,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} +data: {"type":"response.output_item.added","output_index":0,"item":{"id":"msg_67c9fdcf37fc8190ba82116e33fb28c507b8b0ad4e5eb654","type":"message","status":"in_progress","role":"assistant","content":[]}} +data: {"type":"response.content_part.added","item_id":"msg_67c9fdcf37fc8190ba82116e33fb28c507b8b0ad4e5eb654","output_index":0,"content_index":0,"part":{"type":"output_text","text":"","annotations":[]}} +data: {"type":"response.output_text.delta","item_id":"msg_67c9fdcf37fc8190ba82116e33fb28c507b8b0ad4e5eb654","output_index":0,"content_index":0,"delta":"Hi","sequence_number":0} +data: {"type":"response.output_text.done","item_id":"msg_67c9fdcf37fc8190ba82116e33fb28c507b8b0ad4e5eb654","output_index":0,"content_index":0,"text":"Hi there! How can I assist you today?"} +data: {"type":"response.content_part.done","item_id":"msg_67c9fdcf37fc8190ba82116e33fb28c507b8b0ad4e5eb654","output_index":0,"content_index":0,"part":{"type":"output_text","text":"Hi there! How can I assist you today?","annotations":[]}} +data: {"type":"response.output_item.done","output_index":0,"item":{"id":"msg_67c9fdcf37fc8190ba82116e33fb28c507b8b0ad4e5eb654","type":"message","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Hi there! How can I assist you today?","annotations":[]}]}} +data: {"type":"response.completed","response":{"id":"resp_67c9fdcecf488190bdd9a0409de3a1ec07b8b0ad4e5eb654","object":"response","created_at":1741290958,"status":"completed","error":null,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"model":"gpt-4o-2024-08-06","output":[{"id":"msg_67c9fdcf37fc8190ba82116e33fb28c507b8b0ad4e5eb654","type":"message","status":"completed","role":"assistant","content":[{"type":"output_text","text":"Hi there! How can I assist you today?","annotations":[]}]}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":37,"output_tokens":11,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":48},"user":null,"metadata":{}}} +data: [DONE] From 70dac6e3744f407009fa279cb046a53f391ed2f3 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Fri, 20 Feb 2026 12:38:40 +0100 Subject: [PATCH 04/17] Make appsec mock to work in memory --- tests/Appsec/Mock.php | 209 +++++++++++++----- .../Integrations/OpenAI/Latest/OpenAITest.php | 10 + 2 files changed, 159 insertions(+), 60 deletions(-) diff --git a/tests/Appsec/Mock.php b/tests/Appsec/Mock.php index 1ae8da21a6c..8072b7ce1de 100644 --- a/tests/Appsec/Mock.php +++ b/tests/Appsec/Mock.php @@ -2,64 +2,147 @@ namespace datadog\appsec; -if (!class_exists('datadog\appsec\AppsecStatus')) { - class AppsecStatus + +if (!class_exists('datadog\appsec\AppsecStatusBase')) { + /** + * Shared logic for filtering events by names and addresses. + * + * @internal + */ + abstract class AppsecStatusBase { - private static $instance = null; - private $connection; + /** + * @param array $rows + * @return array> + */ + protected static function filterEventsByNamesAndAddresses(array $rows, string $token, array $names, array $addresses): array + { + $result = []; + foreach ($rows as $row) { + if ($row['token'] !== $token) { + continue; + } + $new = json_decode($row['event'], true); + if ($new === null) { + continue; + } + if (empty($names) || in_array($new['eventName'], $names) && + (empty($addresses) || !empty(array_intersect($addresses, array_keys($new[0] ?? []))))) { + $result[] = $new; + } + } + return $result; + } - protected function __construct() + abstract public function init(): void; + + abstract public function setDefaults(): void; + + /** + * @param array $event + */ + abstract public function addEvent(array $event, string $eventName): void; + + /** + * @return array> + */ + abstract public function getEvents(array $names = [], array $addresses = []): array; + + /** + * @param array $event + */ + abstract public function simulateBlockOnEvent($event): void; + } +} +if (!class_exists('datadog\appsec\AppsecStatusInMemory')) { + final class AppsecStatusInMemory extends AppsecStatusBase + { + /** @var array */ + private $events = []; + /** @var array */ + private $blockedEvents = []; + + public function init(): void { + $this->events = []; + $this->blockedEvents = []; } - public static function getInstance() + public function setDefaults(): void { - if (!self::$instance) { - self::$instance = new static(); + $token = ini_get("datadog.trace.agent_test_session_token"); + $this->events = array_values(array_filter($this->events, function ($row) use ($token) { + return $row['token'] !== $token; + })); + $this->blockedEvents = array_values(array_filter($this->blockedEvents, function ($row) use ($token) { + return $row['token'] !== $token; + })); + } + + public function addEvent(array $event, $eventName): void + { + $event['eventName'] = $eventName; + $jsonEvent = json_encode($event); + $token = ini_get("datadog.trace.agent_test_session_token"); + + $eventIsBlocked = false; + foreach ($this->blockedEvents as $row) { + if ($row['event'] === $jsonEvent && $row['token'] === $token) { + $eventIsBlocked = true; + break; + } + } + $this->events[] = ['event' => $jsonEvent, 'token' => $token]; + if ($eventIsBlocked) { + \DDTrace\Testing\trigger_error("Datadog blocked the request and NON RELEVANT TEXT FROM HERE", E_ERROR); } + } - return self::$instance; + public function getEvents(array $names = [], array $addresses = []): array + { + $token = ini_get("datadog.trace.agent_test_session_token"); + return self::filterEventsByNamesAndAddresses($this->events, $token, $names, $addresses); } - protected function getDbPdo() + public function simulateBlockOnEvent($event): void { - if (!isset($this->connection)) { - $pdo = new \PDO('mysql:host=mysql-integration', 'test', 'test'); - $pdo->exec("CREATE DATABASE IF NOT EXISTS test"); - $this->connection = new \PDO('mysql:host=mysql-integration;dbname=test', 'test', 'test'); - $this->connection->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); - } - return $this->connection; + $jsonEvent = json_encode($event); + $token = ini_get("datadog.trace.agent_test_session_token"); + $this->blockedEvents[] = ['event' => $jsonEvent, 'token' => $token]; } + } +} - /** - * Not all test are interested on events but frameworks are instrumented so this check is to avoid errors - */ - private function initiated() +if (!class_exists('datadog\appsec\AppsecStatusMysql')) { + final class AppsecStatusMysql extends AppsecStatusBase + { + + private function initiated(): bool { $stmt = $this->getDbPdo()->prepare("SELECT * FROM information_schema.tables WHERE table_name = :table_name"); $stmt->execute(['table_name' => 'appsec_events']); return $stmt->rowCount() > 0; } - public function init() + public function init(): void { $this->getDbPdo()->exec("CREATE TABLE IF NOT EXISTS appsec_events (event varchar(1000), token varchar(100))"); $this->getDbPdo()->exec("CREATE TABLE IF NOT EXISTS appsec_blocked_events (event varchar(1000), token varchar(100))"); } - public function setDefaults() + public function setDefaults(): void { if (!$this->initiated()) { return; } + $token = ini_get("datadog.trace.agent_test_session_token"); $stmt = $this->getDbPdo()->prepare("DELETE FROM appsec_events WHERE token = :token"); - $stmt->execute(['token' => ini_get("datadog.trace.agent_test_session_token")]); + $stmt->execute(['token' => $token]); $stmt = $this->getDbPdo()->prepare("DELETE FROM appsec_blocked_events WHERE token = :token"); - $stmt->execute(['token' => ini_get("datadog.trace.agent_test_session_token")]); + $stmt->execute(['token' => $token]); } - public function addEvent(array $event, $eventName) + public function addEvent(array $event, $eventName): void { if (!$this->initiated()) { return; @@ -69,57 +152,63 @@ public function addEvent(array $event, $eventName) $token = ini_get("datadog.trace.agent_test_session_token"); $stmt = $this->getDbPdo()->prepare("SELECT * from appsec_blocked_events where event=:event and token=:token"); - $stmt->execute([ - 'event' => $jsonEvent, - 'token' => $token - ]); + $stmt->execute(['event' => $jsonEvent, 'token' => $token]); $eventIsBlocked = $stmt->rowCount() > 0; - + $stmt = $this->getDbPdo()->prepare("INSERT INTO appsec_events VALUES (:event, :token)"); - $stmt->execute([ - 'event' => $jsonEvent, - 'token' => $token - ]); + $stmt->execute(['event' => $jsonEvent, 'token' => $token]); if ($eventIsBlocked) { \DDTrace\Testing\trigger_error("Datadog blocked the request and NON RELEVANT TEXT FROM HERE", E_ERROR); } } - public function getEvents(array $names = [], array $addresses = []) + public function getEvents(array $names = [], array $addresses = []): array { - $result = []; - if (!$this->initiated()) { - return $result; + return []; } - + $token = ini_get("datadog.trace.agent_test_session_token"); $stmt = $this->getDbPdo()->prepare("SELECT * FROM appsec_events WHERE token = :token"); - $stmt->execute(['token' => ini_get("datadog.trace.agent_test_session_token")]); - $events = $stmt->fetchAll(); - - foreach ($events as $event) { - $new = json_decode($event['event'], true); - if ($new === null) { - continue; - } - if (empty($names) || in_array($new['eventName'], $names) && - (empty($addresses) || !empty(array_intersect($addresses, array_keys($new[0]))))) { - $result[] = $new; - } - } - - return $result; + $stmt->execute(['token' => $token]); + $events = $stmt->fetchAll(\PDO::FETCH_ASSOC); + return self::filterEventsByNamesAndAddresses($events, $token, $names, $addresses); } - public function simulateBlockOnEvent($event) { + public function simulateBlockOnEvent($event): void + { $jsonEvent = json_encode($event); $token = ini_get("datadog.trace.agent_test_session_token"); $stmt = $this->getDbPdo()->prepare("INSERT INTO appsec_blocked_events VALUES (:event, :token)"); - $stmt->execute([ - 'event' => $jsonEvent, - 'token' => $token - ]); + $stmt->execute(['event' => $jsonEvent, 'token' => $token]); + } + } +} + +if (!class_exists('datadog\appsec\AppsecStatus')) { + class AppsecStatus + { + /** @var AppsecStatusBase|null */ + private static $instance = null; + + /** + * The first call defines the mode: getInstance(true) = in-memory, getInstance() or getInstance(false) = MySQL. + * Mode is fixed until clearInstances() is called. + * + * @param bool $inMemory When true, use in-memory storage (only applied on first call) + * @return AppsecStatusBase + */ + public static function getInstance(bool $inMemory = false) + { + if (self::$instance === null) { + self::$instance = $inMemory ? new AppsecStatusInMemory() : new AppsecStatusMysql(); + } + return self::$instance; + } + + public static function clearInstances(): void + { + self::$instance = null; } } } diff --git a/tests/Integrations/OpenAI/Latest/OpenAITest.php b/tests/Integrations/OpenAI/Latest/OpenAITest.php index f03e7ec15b9..a15a7af5e93 100644 --- a/tests/Integrations/OpenAI/Latest/OpenAITest.php +++ b/tests/Integrations/OpenAI/Latest/OpenAITest.php @@ -14,6 +14,15 @@ class OpenAITest extends IntegrationTestCase public static function ddSetUpBeforeClass() { parent::ddSetUpBeforeClass(); + self::putEnv('APPSEC_MOCK_ENABLED=true'); + AppsecStatus::getInstance(true); + } + + public static function ddTearDownAfterClass() + { + parent::ddTearDownAfterClass(); + AppsecStatus::clearInstances(); + self::putEnv('APPSEC_MOCK_ENABLED'); } private function checkErrors() @@ -48,6 +57,7 @@ protected function ddSetUp() } else { $this->errorLogSize = 0; } + AppsecStatus::getInstance()->setDefaults(); } protected function envsToCleanUpAtTearDown() From 0b41651328ac7b6ac057a7a548305c62bc51572c Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Fri, 20 Feb 2026 15:47:28 +0100 Subject: [PATCH 05/17] Move push address to prehook --- .../Integrations/OpenAI/OpenAIIntegration.php | 42 ++++++------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php index d6539914474..095b06e98ae 100644 --- a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php +++ b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php @@ -135,7 +135,13 @@ static function ($This, $scope, $args) { } ); - $handleRequestPrehook = fn ($streamed, $operationID, $reportApm) => function (\DDTrace\SpanData $span, $args) use ($operationID, $streamed, $reportApm) { + $handleRequestPrehook = fn ($streamed, $operationID, $reportApm, $reportAppsec) => function (\DDTrace\SpanData $span, $args) use ($operationID, $streamed, $reportApm, $reportAppsec) { + if ($reportAppsec && function_exists('datadog\appsec\push_addresses')) { + \datadog\appsec\push_addresses(["server.business_logic.llm.event" => [ + 'provider' => 'openai', + 'model' => $args[0]['model'] ?? null, + ]]); + } if (!$reportApm) { return; } @@ -162,8 +168,8 @@ static function ($This, $scope, $args) { $class, $method, [ - 'prehook' => $handleRequestPrehook(false, $operationID, $reportApm), - 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm, $reportAppsec) { + 'prehook' => $handleRequestPrehook(false, $operationID, $reportApm, $reportAppsec), + 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm) { /** @var (\OpenAI\Contracts\ResponseContract&\OpenAI\Contracts\ResponseHasMetaInformationContract)|string $response */ // Files::download - i.e., downloadFile - returns a string instead of a Response instance self::handleResponse( @@ -174,7 +180,6 @@ static function ($This, $scope, $args) { httpMethod: $httpMethod, endpoint: $endpoint, reportApm: $reportApm, - reportAppsec: $reportAppsec, ); } ] @@ -186,8 +191,8 @@ static function ($This, $scope, $args) { $class, $method, [ - 'prehook' => $handleRequestPrehook(true, $operationID, $reportApm), - 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm, $reportAppsec) { + 'prehook' => $handleRequestPrehook(true, $operationID, $reportApm, $reportAppsec), + 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm) { /** @var \OpenAI\Responses\StreamResponse $response */ self::handleStreamedResponse( span: $span, @@ -197,7 +202,6 @@ static function ($This, $scope, $args) { httpMethod: $httpMethod, endpoint: $endpoint, reportApm: $reportApm, - reportAppsec: $reportAppsec, ); } ] @@ -429,17 +433,8 @@ public static function handleResponse( string $httpMethod, string $endpoint, bool $reportApm = true, - bool $reportAppsec = false, ) { - $model = $headers['openai-model'] ?? $response['model'] ?? null; - if ($reportAppsec && function_exists('datadog\appsec\push_addresses')) { - \datadog\appsec\push_addresses(["server.business_logic.llm.event" => [ - 'provider' => 'openai', - 'model' => $model, - ]]); - } - if (!$reportApm) { return; } @@ -450,8 +445,6 @@ public static function handleResponse( $response = ['bytes' => \strlen($response)]; } - - $tags = [ 'openai.request.endpoint' => $endpoint, 'openai.request.method' => $httpMethod, @@ -459,7 +452,7 @@ public static function handleResponse( 'openai.organization.id' => $response['organization_id'] ?? null, // Only available in fine-tunes endpoint 'openai.organization.name' => $headers['openai-organization'] ?? null, - 'openai.response.model' => $model, // Specific model, often undefined + 'openai.response.model' => $headers['openai-model'] ?? $response['model'] ?? null, // Specific model, often undefined 'openai.response.id' => $headers['x-request-id'] ?? $response['id'] ?? null, // Common creation value, numeric epoch 'openai.response.deleted' => $response['deleted'] ?? null, // Common boolean field in delete responses @@ -1037,17 +1030,8 @@ public static function handleStreamedResponse( string $httpMethod, string $endpoint, bool $reportApm = true, - bool $reportAppsec = false, ) { - $model = $headers['openai-model'] ?? null; - if ($reportAppsec && function_exists('datadog\appsec\push_addresses')) { - \datadog\appsec\push_addresses(["server.business_logic.llm.event" => [ - 'provider' => 'openai', - 'model' => $model, - ]]); - } - if (!$reportApm) { return; } @@ -1060,7 +1044,7 @@ public static function handleStreamedResponse( 'openai.request.endpoint' => $endpoint, 'openai.request.method' => $httpMethod, 'openai.organization.name' => $headers['openai-organization'] ?? null, - 'openai.response.model' => $model, // Specific model, often undefined + 'openai.response.model' => $headers['openai-model'] ?? null, // Specific model, often undefined 'openai.response.id' => $headers['x-request-id'] ?? null, // Common creation value, numeric epoch ]; From 123295cb9c1de3276cf03c531883ac5846fc2fb3 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Fri, 20 Feb 2026 17:39:08 +0100 Subject: [PATCH 06/17] Create fake response --- tests/OpenAI/Fixtures/Chat.php | 18 -------- tests/OpenAI/Fixtures/Response.php | 69 ++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 18 deletions(-) create mode 100644 tests/OpenAI/Fixtures/Response.php diff --git a/tests/OpenAI/Fixtures/Chat.php b/tests/OpenAI/Fixtures/Chat.php index e9be176f159..9bb471a22a3 100644 --- a/tests/OpenAI/Fixtures/Chat.php +++ b/tests/OpenAI/Fixtures/Chat.php @@ -30,16 +30,6 @@ function chatCompletion(): array ]; } -function response(): array -{ - return [ - 'id' => 'chatcmpl-123', - 'object' => 'chat.completion', - 'created' => 1677652288, - 'model' => 'gpt-3.5-turbo-0125', - ]; -} - function chatCompletionDefaultExample(): array { return [ @@ -146,11 +136,3 @@ function chatCompletionStreamError() { return fopen(__DIR__.'/Streams/ChatCompletionCreateError.txt', 'r'); } - -/** - * @return resource - */ -function responseStream() -{ - return fopen(__DIR__.'/Streams/ResponseCreate.txt', 'r'); -} \ No newline at end of file diff --git a/tests/OpenAI/Fixtures/Response.php b/tests/OpenAI/Fixtures/Response.php new file mode 100644 index 00000000000..3aa5e490023 --- /dev/null +++ b/tests/OpenAI/Fixtures/Response.php @@ -0,0 +1,69 @@ + 'resp-fake-internal', + 'object' => 'response', + 'created_at' => 1677652288, + 'status' => 'completed', + 'model' => 'gpt-3.5-turbo-0125', + // Fields commonly expected by the OpenAI PHP client for Responses API + 'error' => null, + 'incomplete_details' => null, + 'instructions' => 'You are a helpful assistant.', + 'max_output_tokens' => null, + 'output' => [ + [ + 'type' => 'message', + 'id' => 'msg-fake-internal', + 'role' => 'assistant', + 'status' => 'completed', + 'content' => [ + [ + 'type' => 'output_text', + 'text' => 'Fake response from internal_server mock.', + 'annotations' => [], + ], + ], + ], + ], + 'output_text' => 'Fake response from internal_server mock.', + 'previous_response_id' => null, + 'parallel_tool_calls' => false, + 'tool_choice' => 'none', + 'tools' => [], + 'store' => true, + 'reasoning' => [ + 'effort' => null, + 'summary' => null, + ], + 'temperature' => 1.0, + 'text' => [ + 'format' => [ + 'type' => 'text', + ], + ], + 'top_p' => 1.0, + 'truncation' => 'disabled', + 'user' => null, + 'metadata' => [], + 'usage' => [ + 'input_tokens' => 1, + 'input_tokens_details' => ['cached_tokens' => 0], + 'output_tokens' => 2, + 'output_tokens_details' => ['reasoning_tokens' => 0], + 'total_tokens' => 3, + ], + ]; +} + +/** + * @return resource + */ +function responseStream() +{ + return fopen(__DIR__.'/Streams/ResponseCreate.txt', 'r'); +} \ No newline at end of file From b5e2c9ad3eb613b9709ec0f60bde950613917743 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Sat, 21 Feb 2026 12:27:54 +0100 Subject: [PATCH 07/17] Update snapshots --- .../Integrations/OpenAI/OpenAIIntegration.php | 2 +- .../Integrations/OpenAI/Latest/OpenAITest.php | 7 +++- ...n_ai.open_ai_test.test_create_response.txt | 0 ...en_ai_test.test_create_response_stream.txt | 0 ..._ai.open_ai_test.test_create_response.json | 40 +++++++++++++++++++ ...n_ai_test.test_create_response_stream.json | 40 +++++++++++++++++++ 6 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 tests/snapshots/logs/tests.integrations.open_ai.open_ai_test.test_create_response.txt create mode 100644 tests/snapshots/logs/tests.integrations.open_ai.open_ai_test.test_create_response_stream.txt create mode 100644 tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response.json create mode 100644 tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response_stream.json diff --git a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php index 095b06e98ae..9f3b9988d13 100644 --- a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php +++ b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php @@ -96,7 +96,7 @@ public static function init(): int ['OpenAI\Resources\Completions', 'createStreamed', 'createCompletion', 'POST', '/v1/completions', true, true], ['OpenAI\Resources\Chat', 'createStreamed', 'createChatCompletion', 'POST', '/v1/chat/completions', true, true], ['OpenAI\Resources\FineTunes', 'listEventsStreamed', 'listFineTuneEvents', 'GET', '/v1/fine-tunes/*/events', true, false], - ['OpenAI\Resources\Responses', 'create', 'createResponse', 'POST', '/v1/responses', false, true], + ['OpenAI\Resources\Responses', 'createStreamed', 'createResponse', 'POST', '/v1/responses', false, true], ]; \DDTrace\hook_method( diff --git a/tests/Integrations/OpenAI/Latest/OpenAITest.php b/tests/Integrations/OpenAI/Latest/OpenAITest.php index a15a7af5e93..89293eb7075 100644 --- a/tests/Integrations/OpenAI/Latest/OpenAITest.php +++ b/tests/Integrations/OpenAI/Latest/OpenAITest.php @@ -6,9 +6,12 @@ use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Stream; use datadog\appsec\AppsecStatus; +use DDTrace\Tests\Common\SnapshotTestTrait; class OpenAITest extends IntegrationTestCase { + use SnapshotTestTrait +; private $errorLogSize = 0; public static function ddSetUpBeforeClass() @@ -51,6 +54,7 @@ protected function ddSetUp() 'DD_SERVICE=openai-test', 'DD_ENV=test', 'DD_VERSION=1.0', + 'APPSEC_MOCK_ENABLED=true', ]); if (file_exists(__DIR__ . "/openai.log")) { $this->errorLogSize = (int)filesize(__DIR__ . "/openai.log"); @@ -58,6 +62,8 @@ protected function ddSetUp() $this->errorLogSize = 0; } AppsecStatus::getInstance()->setDefaults(); + $token = $this->generateToken(); + update_test_agent_session_token($token); } protected function envsToCleanUpAtTearDown() @@ -127,7 +133,6 @@ public function testCreateCompletion() ], 'user' => 'dd-trace' ]); - $events = AppsecStatus::getInstance()->getEvents(['push_addresses'], ['server.business_logic.llm.event']); $this->assertEquals(1, count($events)); $this->assertEquals('openai', $events[0][0]["server.business_logic.llm.event"]['provider']); diff --git a/tests/snapshots/logs/tests.integrations.open_ai.open_ai_test.test_create_response.txt b/tests/snapshots/logs/tests.integrations.open_ai.open_ai_test.test_create_response.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/snapshots/logs/tests.integrations.open_ai.open_ai_test.test_create_response_stream.txt b/tests/snapshots/logs/tests.integrations.open_ai.open_ai_test.test_create_response_stream.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response.json b/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response.json new file mode 100644 index 00000000000..6a98529c440 --- /dev/null +++ b/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response.json @@ -0,0 +1,40 @@ +[[ + { + "name": "OpenAI\\Resources\\Responses.create", + "service": "openai-test", + "resource": "OpenAI\\Resources\\Responses.create", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "cli", + "meta": { + "_dd.p.dm": "0", + "_dd.p.tid": "6999965d00000000", + "env": "test", + "runtime-id": "a8372150-8faa-429f-93f0-5ea8049895e8", + "version": "1.0" + }, + "metrics": { + "_dd.agent_psr": 1, + "_sampling_priority_v1": 1 + } + }, + { + "name": "Psr\\Http\\Client\\ClientInterface.sendRequest", + "service": "openai-test", + "resource": "sendRequest", + "trace_id": 0, + "span_id": 2, + "parent_id": 1, + "type": "http", + "meta": { + "component": "psr18", + "env": "test", + "http.method": "POST", + "http.status_code": "200", + "http.url": "https://api.openai.com/v1/responses?foo=bar", + "network.destination.name": "api.openai.com", + "span.kind": "client", + "version": "1.0" + } + }]] diff --git a/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response_stream.json b/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response_stream.json new file mode 100644 index 00000000000..766335f8bd0 --- /dev/null +++ b/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response_stream.json @@ -0,0 +1,40 @@ +[[ + { + "name": "OpenAI\\Resources\\Responses.createStreamed", + "service": "openai-test", + "resource": "OpenAI\\Resources\\Responses.createStreamed", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "cli", + "meta": { + "_dd.p.dm": "0", + "_dd.p.tid": "6999966c00000000", + "env": "test", + "runtime-id": "a8372150-8faa-429f-93f0-5ea8049895e8", + "version": "1.0" + }, + "metrics": { + "_dd.agent_psr": 1, + "_sampling_priority_v1": 1 + } + }, + { + "name": "Psr\\Http\\Client\\ClientInterface.sendRequest", + "service": "openai-test", + "resource": "sendRequest", + "trace_id": 0, + "span_id": 2, + "parent_id": 1, + "type": "http", + "meta": { + "component": "psr18", + "env": "test", + "http.method": "POST", + "http.status_code": "200", + "http.url": "https://api.openai.com/v1/responses?foo=bar", + "network.destination.name": "api.openai.com", + "span.kind": "client", + "version": "1.0" + } + }]] From 6028af2a7ea8d574cdbd3acbaadff068d37e4056 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Mon, 23 Feb 2026 10:54:25 +0100 Subject: [PATCH 08/17] Fix Mysql implementation of mock --- tests/Appsec/Mock.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/Appsec/Mock.php b/tests/Appsec/Mock.php index 8072b7ce1de..0124fc32461 100644 --- a/tests/Appsec/Mock.php +++ b/tests/Appsec/Mock.php @@ -116,6 +116,18 @@ public function simulateBlockOnEvent($event): void if (!class_exists('datadog\appsec\AppsecStatusMysql')) { final class AppsecStatusMysql extends AppsecStatusBase { + private $connection = null; + + protected function getDbPdo() + { + if (!isset($this->connection)) { + $pdo = new \PDO('mysql:host=mysql-integration', 'test', 'test'); + $pdo->exec("CREATE DATABASE IF NOT EXISTS test"); + $this->connection = new \PDO('mysql:host=mysql-integration;dbname=test', 'test', 'test'); + $this->connection->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); + } + return $this->connection; + } private function initiated(): bool { From b64379c855bc136d2a50aa97c743e8a2fe18c6c8 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Mon, 23 Feb 2026 11:20:54 +0100 Subject: [PATCH 09/17] Fix addEvent signature --- tests/Appsec/Mock.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Appsec/Mock.php b/tests/Appsec/Mock.php index 0124fc32461..595710d7224 100644 --- a/tests/Appsec/Mock.php +++ b/tests/Appsec/Mock.php @@ -41,7 +41,7 @@ abstract public function setDefaults(): void; /** * @param array $event */ - abstract public function addEvent(array $event, string $eventName): void; + abstract public function addEvent(array $event, $eventName): void; /** * @return array> From 86bde1a074369aa5fd4adfd93e5cd75e9f462a36 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Tue, 24 Feb 2026 13:24:39 +0100 Subject: [PATCH 10/17] Explain the values of the array --- src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php index 9f3b9988d13..c5321682688 100644 --- a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php +++ b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php @@ -66,6 +66,7 @@ public static function init(): int { $logger = \dd_trace_env_config('DD_OPENAI_LOGS_ENABLED') ? new DatadogLogger() : null; + // [class, method, operationID, httpMethod, endpoint, reportApm, reportAppsec] $targets = [ ['OpenAI\Resources\Completions', 'create', 'createCompletion', 'POST', '/v1/completions', true, true], ['OpenAI\Resources\Chat', 'create', 'createChatCompletion', 'POST', '/v1/chat/completions', true, true], @@ -92,6 +93,7 @@ public static function init(): int ['OpenAI\Resources\Responses', 'create', 'createResponse', 'POST', '/v1/responses', false, true], ]; + // [class, method, operationID, httpMethod, endpoint, reportApm, reportAppsec] $streamedTargets = [ ['OpenAI\Resources\Completions', 'createStreamed', 'createCompletion', 'POST', '/v1/completions', true, true], ['OpenAI\Resources\Chat', 'createStreamed', 'createChatCompletion', 'POST', '/v1/chat/completions', true, true], From 905ba6a98ab7ef2c04a3d2f692b3b846c7af432e Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Wed, 25 Feb 2026 17:57:01 +0100 Subject: [PATCH 11/17] Add integration tests for llm events --- .../appsec/php/docker/AppSecContainer.groovy | 3 + .../php/mock_agent/MockOpenAIServer.groovy | 154 ++++++++++++++++++ .../php/integration/LlmEventsTests.groovy | 112 +++++++++++++ .../integration/src/test/waf/recommended.json | 44 +++++ .../src/test/www/base/composer.json | 8 + .../src/test/www/base/initialize.sh | 6 + .../src/test/www/base/public/llm.php | 71 ++++++++ 7 files changed, 398 insertions(+) create mode 100644 appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy create mode 100644 appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy create mode 100644 appsec/tests/integration/src/test/www/base/composer.json create mode 100755 appsec/tests/integration/src/test/www/base/initialize.sh create mode 100644 appsec/tests/integration/src/test/www/base/public/llm.php diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy index 7e985539c56..aed97fda509 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy @@ -1,6 +1,7 @@ package com.datadog.appsec.php.docker import com.datadog.appsec.php.mock_agent.MockDatadogAgent +import com.datadog.appsec.php.mock_agent.MockOpenAIServer import com.datadog.appsec.php.mock_agent.rem_cfg.RemoteConfigRequest import com.datadog.appsec.php.mock_agent.rem_cfg.RemoteConfigResponse import com.datadog.appsec.php.mock_agent.rem_cfg.Target @@ -56,11 +57,13 @@ class AppSecContainer> extends GenericContain .build() private MockDatadogAgent mockDatadogAgent = new MockDatadogAgent() + public MockOpenAIServer mockOpenAIServer = new MockOpenAIServer() AppSecContainer(Map options) { super(imageNameFuture(options)) processOptions(options) dependsOn mockDatadogAgent + dependsOn mockOpenAIServer withCreateContainerCmdModifier(cmd -> { cmd.hostConfig.withInit(true) }) diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy new file mode 100644 index 00000000000..05361c020d7 --- /dev/null +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy @@ -0,0 +1,154 @@ +package com.datadog.appsec.php.mock_agent + +import groovy.json.JsonSlurper +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.javalin.Javalin +import io.javalin.http.Context +import org.testcontainers.lifecycle.Startable + +@Slf4j +@CompileStatic +class MockOpenAIServer implements Startable { + private static final int PORT = 8089 + Javalin httpServer + + @Override + void start() { + this.httpServer = Javalin.create(config -> { + config.showJavalinBanner = false + }) + + // Support both /path and /v1/path for OpenAI client compatibility + this.httpServer.post('/chat/completions', this.&handleChatCompletions) + this.httpServer.post('/v1/chat/completions', this.&handleChatCompletions) + this.httpServer.post('/completions', this.&handleCompletions) + this.httpServer.post('/v1/completions', this.&handleCompletions) + this.httpServer.post('/responses', this.&handleResponses) + this.httpServer.post('/v1/responses', this.&handleResponses) + + this.httpServer.error(404, ctx -> { + log.info("Unmatched OpenAI mock request: ${ctx.method()} ${ctx.path()}") + ctx.status(404).json(['error': 'Not Found']) + }) + this.httpServer.error(405, ctx -> { + ctx.status(405).json(['error': 'Method Not Allowed']) + }) + + this.httpServer.start(PORT) + } + + int getPort() { + PORT + } + + @Override + void stop() { + if (httpServer != null) { + this.httpServer.stop() + this.httpServer = null + } + } + + private static Map parseBody(Context ctx) { + String raw = ctx.body() + if (raw == null || raw.isEmpty()) { + return [:] + } + try { + def decoded = new JsonSlurper().parseText(raw) + return decoded instanceof Map ? (Map) decoded : [:] + } catch (Exception e) { + return [:] + } + } + + private static Map fakeUsage() { + [ + 'prompt_tokens' : 1, + 'completion_tokens': 2, + 'total_tokens' : 3, + ] + } + + private void handleChatCompletions(Context ctx) { + Map body = parseBody(ctx) + String model = (body['model'] as String) ?: 'gpt-4.1' + ctx.json([ + 'id' : 'chatcmpl-fake-internal', + 'object' : 'chat.completion', + 'created': (long)(System.currentTimeMillis() / 1000), + 'model' : model, + 'choices': [ + [ + 'index' : 0, + 'message' : [ + 'role' : 'assistant', + 'content': 'Fake response from internal_server mock.', + ], + 'finish_reason': 'stop', + ], + ], + 'usage' : fakeUsage(), + ]) + } + + private void handleCompletions(Context ctx) { + Map body = parseBody(ctx) + String model = (body['model'] as String) ?: 'text-davinci-003' + ctx.json([ + 'id' : 'cmpl-fake-internal', + 'object' : 'text_completion', + 'created': (long)(System.currentTimeMillis() / 1000), + 'model' : model, + 'choices': [ + [ + 'text' : 'Fake completion from internal_server mock.', + 'index' : 0, + 'finish_reason': 'stop', + 'logprobs' : null, + ], + ], + 'usage' : fakeUsage(), + ]) + } + + private void handleResponses(Context ctx) { + Map body = parseBody(ctx) + String model = (body['model'] as String) ?: 'gpt-4.1' + ctx.json([ + 'id' : 'resp-fake-internal', + 'object' : 'response', + 'created_at' : (long)(System.currentTimeMillis() / 1000), + 'status' : 'completed', + 'model' : model, + 'output' : [ + [ + 'type' : 'message', + 'id' : 'msg-fake-internal', + 'role' : 'assistant', + 'status' : 'completed', + 'content': [ + [ + 'type' : 'output_text', + 'text' : 'Fake response from internal_server mock.', + 'annotations': [], + ], + ], + ], + ], + 'output_text' : 'Fake response from internal_server mock.', + 'parallel_tool_calls' : false, + 'tool_choice' : 'none', + 'tools' : [], + 'store' : true, + 'usage' : [ + 'input_tokens' : 1, + 'input_tokens_details' : ['cached_tokens': 0], + 'output_tokens' : 2, + 'output_tokens_details' : ['reasoning_tokens': 0], + 'total_tokens' : 3, + ], + ]) + } +} diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy new file mode 100644 index 00000000000..3cfaaf50ba0 --- /dev/null +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy @@ -0,0 +1,112 @@ +package com.datadog.appsec.php.integration + +import com.datadog.appsec.php.docker.AppSecContainer +import com.datadog.appsec.php.docker.FailOnUnmatchedTraces +import com.datadog.appsec.php.docker.InspectContainerHelper +import com.datadog.appsec.php.model.Span +import com.datadog.appsec.php.model.Trace +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestMethodOrder +import org.junit.jupiter.api.condition.EnabledIf +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers + +import java.io.InputStream + +import static org.testcontainers.containers.Container.ExecResult +import java.net.http.HttpRequest +import java.net.http.HttpResponse + +import static com.datadog.appsec.php.integration.TestParams.getPhpVersion +import static com.datadog.appsec.php.integration.TestParams.getVariant +import static com.datadog.appsec.php.integration.TestParams.phpVersionAtLeast +import com.datadog.appsec.php.TelemetryHelpers +import static java.net.http.HttpResponse.BodyHandlers.ofString + +@Testcontainers +@EnabledIf('isExpectedVersion') +class LlmEventsTests { + static final String MODEL = 'gpt-4.1' + static boolean expectedVersion = phpVersionAtLeast('8.2') && !variant.contains('zts') + + AppSecContainer getContainer() { + getClass().CONTAINER + } + + @Container + @FailOnUnmatchedTraces + public static final AppSecContainer CONTAINER = + new AppSecContainer( + workVolume: this.name, + baseTag: 'apache2-mod-php', + phpVersion: phpVersion, + phpVariant: variant, + www: 'base', + ) { + @Override + void configure() { + super.configure() + org.testcontainers.Testcontainers.exposeHostPorts(mockOpenAIServer.port) + withEnv('OPENAI_BASE_URL', "http://host.testcontainers.internal:${mockOpenAIServer.port}/v1") + } + } + + @BeforeAll + static void setMaxRequestWorkers() { + ExecResult res = CONTAINER.execInContainer( + 'sed', '-i', 's/MaxRequestWorkers.*/MaxRequestWorkers 2/', '/etc/apache2/mods-available/mpm_prefork.conf') + assert res.exitCode == 0 : "Failed setting mpm_prefork MaxRequestWorkers: $res.stderr" + res = CONTAINER.execInContainer( + 'sed', '-i', 's/MaxRequestWorkers.*/MaxRequestWorkers 2/', '/etc/apache2/mods-available/mpm_worker.conf') + assert res.exitCode == 0 : "Failed setting mpm_worker MaxRequestWorkers: $res.stderr" + res = CONTAINER.execInContainer('service', 'apache2', 'restart') + assert res.exitCode == 0 : "Failed restarting apache2: $res.stderr" + } + + static void main(String[] args) { + InspectContainerHelper.run(CONTAINER) + } + + /** Common assertions for LLM endpoint spans. */ + static void assertLlmSpan(Trace trace, String model) { + Span span = trace.first() + assert span.meta.'appsec.events.llm.call.provider' == 'openai' + assert span.meta.'appsec.events.llm.call.model' == model + assert span.metrics._sampling_priority_v1 == 2.0d + } + + @Test + void 'OpenAI latest responses create'() { + def trace = container.traceFromRequest("/llm.php?model=${MODEL}&operation=openai-latest-responses.create") { HttpResponse resp -> + assert resp.statusCode() == 200 + } + assertLlmSpan(trace, MODEL) + } + + @Test + void 'OpenAI latest chat completions create'() { + def trace = container.traceFromRequest("/llm.php?model=${MODEL}&operation=openai-latest-chat.completions.create") { HttpResponse resp -> + assert resp.statusCode() == 200 + } + assertLlmSpan(trace, MODEL) + } + + @Test + void 'OpenAI latest completions create'() { + def trace = container.traceFromRequest("/llm.php?model=${MODEL}&operation=openai-latest-completions.create") { HttpResponse resp -> + assert resp.statusCode() == 200 + } + assertLlmSpan(trace, MODEL) + } + + @Test + void 'Root has no LLM tags'() { + def trace = container.traceFromRequest('/hello.php') { HttpResponse resp -> + assert resp.statusCode() == 200 + } + Span span = trace.first() + assert !span.meta.containsKey('appsec.events.llm.call.provider') + assert !span.meta.containsKey('appsec.events.llm.call.model') + } +} diff --git a/appsec/tests/integration/src/test/waf/recommended.json b/appsec/tests/integration/src/test/waf/recommended.json index 19d24c1d196..2766a4a9e34 100644 --- a/appsec/tests/integration/src/test/waf/recommended.json +++ b/appsec/tests/integration/src/test/waf/recommended.json @@ -6933,6 +6933,50 @@ "on_match": [ "stack_trace" ] + }, + { + "id": "llm-001-000", + "name": "LLM call", + "tags": { + "type": "llm.event", + "category": "business_logic", + "module": "business_logic" + }, + "min_version": "1.25.0", + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.business_logic.llm.event", + "key_path": [ + "provider" + ] + } + ] + }, + "operator": "exists" + } + ], + "transformers": [], + "output": { + "event": false, + "keep": true, + "attributes": { + "appsec.events.llm.call.provider": { + "address": "server.business_logic.llm.event", + "key_path": [ + "provider" + ] + }, + "appsec.events.llm.call.model": { + "address": "server.business_logic.llm.event", + "key_path": [ + "model" + ] + } + } + } } ], "rules_compat": [ diff --git a/appsec/tests/integration/src/test/www/base/composer.json b/appsec/tests/integration/src/test/www/base/composer.json new file mode 100644 index 00000000000..7e8e5a2e134 --- /dev/null +++ b/appsec/tests/integration/src/test/www/base/composer.json @@ -0,0 +1,8 @@ +{ + "name": "datadog/appsec-integration-tests", + "type": "project", + "require": { + "openai-php/client": "*", + "guzzlehttp/guzzle": "*" + } +} \ No newline at end of file diff --git a/appsec/tests/integration/src/test/www/base/initialize.sh b/appsec/tests/integration/src/test/www/base/initialize.sh new file mode 100755 index 00000000000..6d9ebb400b9 --- /dev/null +++ b/appsec/tests/integration/src/test/www/base/initialize.sh @@ -0,0 +1,6 @@ +#!/bin/bash -e + +cd /var/www + +composer install --no-dev +chown -R www-data.www-data vendor diff --git a/appsec/tests/integration/src/test/www/base/public/llm.php b/appsec/tests/integration/src/test/www/base/public/llm.php new file mode 100644 index 00000000000..098404d2ea2 --- /dev/null +++ b/appsec/tests/integration/src/test/www/base/public/llm.php @@ -0,0 +1,71 @@ + 'Missing or empty query parameters: model, operation']); + exit; +} + +try { + $baseUri = getenv('OPENAI_BASE_URL') ?: 'http://localhost/mockOpenAi/'; + $client = \OpenAI::factory() + ->withApiKey('sk-fake') + ->withBaseUri($baseUri) + ->make(); +} catch (Throwable $e) { + http_response_code(500); + echo json_encode(['error' => 'OpenAI client init failed: ' . $e->getMessage()]); + exit; +} + +$params = ['model' => $model]; + +try { + switch ($operation) { + case 'openai-latest-responses.create': + $params['input'] = 'Hello'; + $response = $client->responses()->create($params); + break; + case 'openai-latest-chat.completions.create': + $params['messages'] = [['role' => 'user', 'content' => 'Hello']]; + $response = $client->chat()->create($params); + break; + case 'openai-latest-completions.create': + $params['prompt'] = 'Hello'; + $response = $client->completions()->create($params); + break; + default: + http_response_code(400); + echo json_encode(['error' => 'Unknown operation: ' . $operation]); + exit; + } + + // Return a simple success payload; the client returns response objects + echo json_encode([ + 'model' => $model, + 'operation' => $operation, + 'status' => 'ok', + ]); +} catch (Throwable $e) { + http_response_code(500); + echo json_encode([ + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + 'operation' => $operation, + ]); +} \ No newline at end of file From 0eb5a024646e357b1d467d08a0149f15c3b3944a Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Wed, 25 Feb 2026 21:28:51 +0100 Subject: [PATCH 12/17] Fix port --- .../datadog/appsec/php/mock_agent/MockOpenAIServer.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy index 05361c020d7..2a32ef9c77e 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy @@ -10,7 +10,6 @@ import org.testcontainers.lifecycle.Startable @Slf4j @CompileStatic class MockOpenAIServer implements Startable { - private static final int PORT = 8089 Javalin httpServer @Override @@ -35,11 +34,11 @@ class MockOpenAIServer implements Startable { ctx.status(405).json(['error': 'Method Not Allowed']) }) - this.httpServer.start(PORT) + this.httpServer.start(0) } int getPort() { - PORT + this.httpServer.port() } @Override From 3c92b6746ec61b6da4e4a212163951033fed3968 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Wed, 25 Feb 2026 22:26:12 +0100 Subject: [PATCH 13/17] Make it compatible with php 7.0 --- tests/Appsec/Mock.php | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/tests/Appsec/Mock.php b/tests/Appsec/Mock.php index 595710d7224..386846362e5 100644 --- a/tests/Appsec/Mock.php +++ b/tests/Appsec/Mock.php @@ -34,14 +34,21 @@ protected static function filterEventsByNamesAndAddresses(array $rows, string $t return $result; } - abstract public function init(): void; + /** + * @return void + */ + abstract public function init(); - abstract public function setDefaults(): void; + /** + * @return void + */ + abstract public function setDefaults(); /** * @param array $event + * @return void */ - abstract public function addEvent(array $event, $eventName): void; + abstract public function addEvent(array $event, $eventName); /** * @return array> @@ -50,8 +57,9 @@ abstract public function getEvents(array $names = [], array $addresses = []): ar /** * @param array $event + * @return void */ - abstract public function simulateBlockOnEvent($event): void; + abstract public function simulateBlockOnEvent($event); } } if (!class_exists('datadog\appsec\AppsecStatusInMemory')) { @@ -62,13 +70,13 @@ final class AppsecStatusInMemory extends AppsecStatusBase /** @var array */ private $blockedEvents = []; - public function init(): void + public function init() { $this->events = []; $this->blockedEvents = []; } - public function setDefaults(): void + public function setDefaults() { $token = ini_get("datadog.trace.agent_test_session_token"); $this->events = array_values(array_filter($this->events, function ($row) use ($token) { @@ -79,7 +87,7 @@ public function setDefaults(): void })); } - public function addEvent(array $event, $eventName): void + public function addEvent(array $event, $eventName) { $event['eventName'] = $eventName; $jsonEvent = json_encode($event); @@ -104,7 +112,7 @@ public function getEvents(array $names = [], array $addresses = []): array return self::filterEventsByNamesAndAddresses($this->events, $token, $names, $addresses); } - public function simulateBlockOnEvent($event): void + public function simulateBlockOnEvent($event) { $jsonEvent = json_encode($event); $token = ini_get("datadog.trace.agent_test_session_token"); @@ -136,13 +144,13 @@ private function initiated(): bool return $stmt->rowCount() > 0; } - public function init(): void + public function init() { $this->getDbPdo()->exec("CREATE TABLE IF NOT EXISTS appsec_events (event varchar(1000), token varchar(100))"); $this->getDbPdo()->exec("CREATE TABLE IF NOT EXISTS appsec_blocked_events (event varchar(1000), token varchar(100))"); } - public function setDefaults(): void + public function setDefaults() { if (!$this->initiated()) { return; @@ -154,7 +162,7 @@ public function setDefaults(): void $stmt->execute(['token' => $token]); } - public function addEvent(array $event, $eventName): void + public function addEvent(array $event, $eventName) { if (!$this->initiated()) { return; @@ -187,7 +195,7 @@ public function getEvents(array $names = [], array $addresses = []): array return self::filterEventsByNamesAndAddresses($events, $token, $names, $addresses); } - public function simulateBlockOnEvent($event): void + public function simulateBlockOnEvent($event) { $jsonEvent = json_encode($event); $token = ini_get("datadog.trace.agent_test_session_token"); @@ -218,7 +226,10 @@ public static function getInstance(bool $inMemory = false) return self::$instance; } - public static function clearInstances(): void + /** + * @return void + */ + public static function clearInstances() { self::$instance = null; } From 193aafa3c00af1f19bfabfdec59ecc8b13fcc718 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Fri, 27 Feb 2026 12:33:45 +0100 Subject: [PATCH 14/17] Amend comments from PR --- .../appsec/php/docker/AppSecContainer.groovy | 6 ++--- .../MockOpenAIServer.groovy | 2 +- .../php/integration/LlmEventsTests.groovy | 24 ++++++++----------- .../src/test/www/{base => llm}/composer.json | 0 .../src/test/www/{base => llm}/initialize.sh | 0 .../src/test/www/llm/public/hello.php | 8 +++++++ .../src/test/www/{base => llm}/public/llm.php | 0 7 files changed, 21 insertions(+), 19 deletions(-) rename appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/{mock_agent => mock_openai}/MockOpenAIServer.groovy (99%) rename appsec/tests/integration/src/test/www/{base => llm}/composer.json (100%) rename appsec/tests/integration/src/test/www/{base => llm}/initialize.sh (100%) create mode 100644 appsec/tests/integration/src/test/www/llm/public/hello.php rename appsec/tests/integration/src/test/www/{base => llm}/public/llm.php (100%) diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy index aed97fda509..894ee73e169 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy @@ -1,7 +1,7 @@ package com.datadog.appsec.php.docker import com.datadog.appsec.php.mock_agent.MockDatadogAgent -import com.datadog.appsec.php.mock_agent.MockOpenAIServer +import com.datadog.appsec.php.mock_openai.MockOpenAIServer import com.datadog.appsec.php.mock_agent.rem_cfg.RemoteConfigRequest import com.datadog.appsec.php.mock_agent.rem_cfg.RemoteConfigResponse import com.datadog.appsec.php.mock_agent.rem_cfg.Target @@ -56,14 +56,12 @@ class AppSecContainer> extends GenericContain .connectTimeout(Duration.ofSeconds(5)) .build() - private MockDatadogAgent mockDatadogAgent = new MockDatadogAgent() - public MockOpenAIServer mockOpenAIServer = new MockOpenAIServer() + private MockDatadogAgent mockDatadogAgent = new MockDatadogAgent() AppSecContainer(Map options) { super(imageNameFuture(options)) processOptions(options) dependsOn mockDatadogAgent - dependsOn mockOpenAIServer withCreateContainerCmdModifier(cmd -> { cmd.hostConfig.withInit(true) }) diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_openai/MockOpenAIServer.groovy similarity index 99% rename from appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy rename to appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_openai/MockOpenAIServer.groovy index 2a32ef9c77e..a88e4ba1f35 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/MockOpenAIServer.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_openai/MockOpenAIServer.groovy @@ -1,4 +1,4 @@ -package com.datadog.appsec.php.mock_agent +package com.datadog.appsec.php.mock_openai import groovy.json.JsonSlurper import groovy.transform.CompileStatic diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy index 3cfaaf50ba0..a41ff8a2048 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy @@ -2,6 +2,7 @@ package com.datadog.appsec.php.integration import com.datadog.appsec.php.docker.AppSecContainer import com.datadog.appsec.php.docker.FailOnUnmatchedTraces +import com.datadog.appsec.php.mock_openai.MockOpenAIServer import com.datadog.appsec.php.docker.InspectContainerHelper import com.datadog.appsec.php.model.Span import com.datadog.appsec.php.model.Trace @@ -34,6 +35,9 @@ class LlmEventsTests { getClass().CONTAINER } + @Container + public static final MockOpenAIServer mockOpenAIServer = new MockOpenAIServer() + @Container @FailOnUnmatchedTraces public static final AppSecContainer CONTAINER = @@ -42,27 +46,19 @@ class LlmEventsTests { baseTag: 'apache2-mod-php', phpVersion: phpVersion, phpVariant: variant, - www: 'base', + www: 'llm', ) { + { + dependsOn mockOpenAIServer + } + @Override void configure() { super.configure() org.testcontainers.Testcontainers.exposeHostPorts(mockOpenAIServer.port) withEnv('OPENAI_BASE_URL', "http://host.testcontainers.internal:${mockOpenAIServer.port}/v1") } - } - - @BeforeAll - static void setMaxRequestWorkers() { - ExecResult res = CONTAINER.execInContainer( - 'sed', '-i', 's/MaxRequestWorkers.*/MaxRequestWorkers 2/', '/etc/apache2/mods-available/mpm_prefork.conf') - assert res.exitCode == 0 : "Failed setting mpm_prefork MaxRequestWorkers: $res.stderr" - res = CONTAINER.execInContainer( - 'sed', '-i', 's/MaxRequestWorkers.*/MaxRequestWorkers 2/', '/etc/apache2/mods-available/mpm_worker.conf') - assert res.exitCode == 0 : "Failed setting mpm_worker MaxRequestWorkers: $res.stderr" - res = CONTAINER.execInContainer('service', 'apache2', 'restart') - assert res.exitCode == 0 : "Failed restarting apache2: $res.stderr" - } + } static void main(String[] args) { InspectContainerHelper.run(CONTAINER) diff --git a/appsec/tests/integration/src/test/www/base/composer.json b/appsec/tests/integration/src/test/www/llm/composer.json similarity index 100% rename from appsec/tests/integration/src/test/www/base/composer.json rename to appsec/tests/integration/src/test/www/llm/composer.json diff --git a/appsec/tests/integration/src/test/www/base/initialize.sh b/appsec/tests/integration/src/test/www/llm/initialize.sh similarity index 100% rename from appsec/tests/integration/src/test/www/base/initialize.sh rename to appsec/tests/integration/src/test/www/llm/initialize.sh diff --git a/appsec/tests/integration/src/test/www/llm/public/hello.php b/appsec/tests/integration/src/test/www/llm/public/hello.php new file mode 100644 index 00000000000..5d4a4097739 --- /dev/null +++ b/appsec/tests/integration/src/test/www/llm/public/hello.php @@ -0,0 +1,8 @@ + Date: Fri, 27 Feb 2026 17:16:58 +0100 Subject: [PATCH 15/17] Avoid creating not required span --- .../Integrations/OpenAI/OpenAIIntegration.php | 108 +++++++++++------- 1 file changed, 66 insertions(+), 42 deletions(-) diff --git a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php index c5321682688..4fd27ebc57a 100644 --- a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php +++ b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php @@ -137,13 +137,25 @@ static function ($This, $scope, $args) { } ); - $handleRequestPrehook = fn ($streamed, $operationID, $reportApm, $reportAppsec) => function (\DDTrace\SpanData $span, $args) use ($operationID, $streamed, $reportApm, $reportAppsec) { + $pushAppsecEvent = static function (bool $reportAppsec, array $args): void { if ($reportAppsec && function_exists('datadog\appsec\push_addresses')) { \datadog\appsec\push_addresses(["server.business_logic.llm.event" => [ 'provider' => 'openai', 'model' => $args[0]['model'] ?? null, ]]); } + }; + + $appsecOnlyPrehook = static function (bool $reportAppsec) use ($pushAppsecEvent) { + return static function ($This, $scope, $args) use ($reportAppsec, $pushAppsecEvent): void { + $pushAppsecEvent($reportAppsec, $args); + }; + }; + + $handleRequestPrehook = fn ($streamed, $operationID, $reportApm, $reportAppsec) => function (\DDTrace\SpanData $span, $args) use ($operationID, $streamed, $reportApm, $reportAppsec, $pushAppsecEvent) { + $pushAppsecEvent($reportAppsec, $args); + + // This should not happen but just in case if (!$reportApm) { return; } @@ -153,7 +165,7 @@ static function ($This, $scope, $args) { $transporter = ObjectKVStore::get($this, 'transporter'); $clientData = ObjectKVStore::get($transporter, 'client_data'); ObjectKVStore::put($this, 'client_data', $clientData); - } + } /** @var array{baseUri: string, headers: string, apiKey: ?string} $clientData */ OpenAIIntegration::handleRequest( span: $span, @@ -166,48 +178,59 @@ static function ($This, $scope, $args) { }; foreach ($targets as [$class, $method, $operationID, $httpMethod, $endpoint, $reportApm, $reportAppsec]) { - \DDTrace\trace_method( - $class, - $method, - [ - 'prehook' => $handleRequestPrehook(false, $operationID, $reportApm, $reportAppsec), - 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm) { - /** @var (\OpenAI\Contracts\ResponseContract&\OpenAI\Contracts\ResponseHasMetaInformationContract)|string $response */ - // Files::download - i.e., downloadFile - returns a string instead of a Response instance - self::handleResponse( - span: $span, - logger: $logger, - headers: $response ? (method_exists($response, 'meta') ? $response->meta()->toArray() : []) : [], - response: \is_string($response) ? $response : ($response ? $response->toArray() : []), - httpMethod: $httpMethod, - endpoint: $endpoint, - reportApm: $reportApm, - ); - } - ] - ); + if ($reportApm) { + \DDTrace\trace_method( + $class, + $method, + [ + 'prehook' => $handleRequestPrehook(false, $operationID, $reportApm, $reportAppsec), + 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm) { + /** @var (\OpenAI\Contracts\ResponseContract&\OpenAI\Contracts\ResponseHasMetaInformationContract)|string $response */ + // Files::download - i.e., downloadFile - returns a string instead of a Response instance + self::handleResponse( + span: $span, + logger: $logger, + headers: $response ? (method_exists($response, 'meta') ? $response->meta()->toArray() : []) : [], + response: \is_string($response) ? $response : ($response ? $response->toArray() : []), + httpMethod: $httpMethod, + endpoint: $endpoint, + reportApm: $reportApm, + ); + } + ] + ); + } elseif ($reportAppsec) { + // Only appsec reporting, use hook_method to avoid creating a span + \DDTrace\hook_method($class, $method, $appsecOnlyPrehook($reportAppsec)); + } } foreach ($streamedTargets as [$class, $method, $operationID, $httpMethod, $endpoint, $reportApm, $reportAppsec]) { - \DDTrace\trace_method( - $class, - $method, - [ - 'prehook' => $handleRequestPrehook(true, $operationID, $reportApm, $reportAppsec), - 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm) { - /** @var \OpenAI\Responses\StreamResponse $response */ - self::handleStreamedResponse( - span: $span, - logger: $logger, - headers: $response->meta()->toArray(), - response: $response, - httpMethod: $httpMethod, - endpoint: $endpoint, - reportApm: $reportApm, - ); - } - ] - ); + if ($reportApm) { + // Use trace_method which handles both APM and appsec in the prehook + \DDTrace\trace_method( + $class, + $method, + [ + 'prehook' => $handleRequestPrehook(true, $operationID, $reportApm, $reportAppsec), + 'posthook' => static function (\DDTrace\SpanData $span, $args, $response) use ($logger, $httpMethod, $endpoint, $reportApm) { + /** @var \OpenAI\Responses\StreamResponse $response */ + self::handleStreamedResponse( + span: $span, + logger: $logger, + headers: $response->meta()->toArray(), + response: $response, + httpMethod: $httpMethod, + endpoint: $endpoint, + reportApm: $reportApm, + ); + } + ] + ); + } elseif ($reportAppsec) { + // Only appsec reporting, use hook_method to avoid creating a span + \DDTrace\hook_method($class, $method, $appsecOnlyPrehook($reportAppsec)); + } } \DDTrace\install_hook( @@ -437,6 +460,7 @@ public static function handleResponse( bool $reportApm = true, ) { + // This should not happen but just in case if (!$reportApm) { return; } @@ -1041,7 +1065,7 @@ public static function handleStreamedResponse( $responseArray = self::readAndStoreStreamedResponse($span, $response); $operationID = \explode('/', $span->resource)[0]; - + $tags = [ 'openai.request.endpoint' => $endpoint, 'openai.request.method' => $httpMethod, From e6187281d0d60c9bb5ca09570e07255d4db628c3 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Fri, 27 Feb 2026 17:54:04 +0100 Subject: [PATCH 16/17] Fix tests --- ...n_ai.open_ai_test.test_create_response.txt | 0 ..._ai.open_ai_test.test_create_response.json | 37 ++++++------------- ...n_ai_test.test_create_response_stream.json | 37 ++++++------------- 3 files changed, 24 insertions(+), 50 deletions(-) create mode 100644 tests/snapshots/metrics/tests.integrations.open_ai.open_ai_test.test_create_response.txt diff --git a/tests/snapshots/metrics/tests.integrations.open_ai.open_ai_test.test_create_response.txt b/tests/snapshots/metrics/tests.integrations.open_ai.open_ai_test.test_create_response.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response.json b/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response.json index 6a98529c440..76291b82343 100644 --- a/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response.json +++ b/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response.json @@ -1,40 +1,27 @@ [[ { - "name": "OpenAI\\Resources\\Responses.create", + "name": "Psr\\Http\\Client\\ClientInterface.sendRequest", "service": "openai-test", - "resource": "OpenAI\\Resources\\Responses.create", + "resource": "sendRequest", "trace_id": 0, "span_id": 1, "parent_id": 0, - "type": "cli", + "type": "http", "meta": { "_dd.p.dm": "0", - "_dd.p.tid": "6999965d00000000", + "_dd.p.tid": "69a1cb2d00000000", + "component": "psr18", "env": "test", - "runtime-id": "a8372150-8faa-429f-93f0-5ea8049895e8", + "http.method": "POST", + "http.status_code": "200", + "http.url": "https://api.openai.com/v1/responses?foo=bar", + "network.destination.name": "api.openai.com", + "runtime-id": "52edddee-1942-4e53-8c67-ce2f4cafbb79", + "span.kind": "client", "version": "1.0" }, "metrics": { "_dd.agent_psr": 1, "_sampling_priority_v1": 1 } - }, - { - "name": "Psr\\Http\\Client\\ClientInterface.sendRequest", - "service": "openai-test", - "resource": "sendRequest", - "trace_id": 0, - "span_id": 2, - "parent_id": 1, - "type": "http", - "meta": { - "component": "psr18", - "env": "test", - "http.method": "POST", - "http.status_code": "200", - "http.url": "https://api.openai.com/v1/responses?foo=bar", - "network.destination.name": "api.openai.com", - "span.kind": "client", - "version": "1.0" - } - }]] + }]] diff --git a/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response_stream.json b/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response_stream.json index 766335f8bd0..9fe0f6e8c80 100644 --- a/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response_stream.json +++ b/tests/snapshots/tests.integrations.open_ai.open_ai_test.test_create_response_stream.json @@ -1,40 +1,27 @@ [[ { - "name": "OpenAI\\Resources\\Responses.createStreamed", + "name": "Psr\\Http\\Client\\ClientInterface.sendRequest", "service": "openai-test", - "resource": "OpenAI\\Resources\\Responses.createStreamed", + "resource": "sendRequest", "trace_id": 0, "span_id": 1, "parent_id": 0, - "type": "cli", + "type": "http", "meta": { "_dd.p.dm": "0", - "_dd.p.tid": "6999966c00000000", + "_dd.p.tid": "69a1cb3c00000000", + "component": "psr18", "env": "test", - "runtime-id": "a8372150-8faa-429f-93f0-5ea8049895e8", + "http.method": "POST", + "http.status_code": "200", + "http.url": "https://api.openai.com/v1/responses?foo=bar", + "network.destination.name": "api.openai.com", + "runtime-id": "52edddee-1942-4e53-8c67-ce2f4cafbb79", + "span.kind": "client", "version": "1.0" }, "metrics": { "_dd.agent_psr": 1, "_sampling_priority_v1": 1 } - }, - { - "name": "Psr\\Http\\Client\\ClientInterface.sendRequest", - "service": "openai-test", - "resource": "sendRequest", - "trace_id": 0, - "span_id": 2, - "parent_id": 1, - "type": "http", - "meta": { - "component": "psr18", - "env": "test", - "http.method": "POST", - "http.status_code": "200", - "http.url": "https://api.openai.com/v1/responses?foo=bar", - "network.destination.name": "api.openai.com", - "span.kind": "client", - "version": "1.0" - } - }]] + }]] From dc468ff693f276e3940064fd80a2de7acbc91673 Mon Sep 17 00:00:00 2001 From: Alejandro Estringana Ruiz Date: Mon, 2 Mar 2026 10:52:38 +0100 Subject: [PATCH 17/17] Amend pr comments --- .../php/integration/LlmEventsTests.groovy | 1 - .../Integrations/OpenAI/OpenAIIntegration.php | 75 ++++++++++--------- 2 files changed, 41 insertions(+), 35 deletions(-) diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy index a41ff8a2048..4833bcf5571 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/LlmEventsTests.groovy @@ -35,7 +35,6 @@ class LlmEventsTests { getClass().CONTAINER } - @Container public static final MockOpenAIServer mockOpenAIServer = new MockOpenAIServer() @Container diff --git a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php index 4fd27ebc57a..cbe6cabc0f2 100644 --- a/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php +++ b/src/DDTrace/Integrations/OpenAI/OpenAIIntegration.php @@ -59,6 +59,16 @@ public static function setServiceName(SpanData $span) } } + public static function pushLlmEventForAppsec($model = null) + { + if (function_exists('datadog\appsec\push_addresses')) { + \datadog\appsec\push_addresses(["server.business_logic.llm.event" => [ + 'provider' => 'openai', + 'model' => $model, + ]]); + } + } + /** * Add instrumentation to OpenAI API Requests */ @@ -137,44 +147,41 @@ static function ($This, $scope, $args) { } ); - $pushAppsecEvent = static function (bool $reportAppsec, array $args): void { - if ($reportAppsec && function_exists('datadog\appsec\push_addresses')) { - \datadog\appsec\push_addresses(["server.business_logic.llm.event" => [ - 'provider' => 'openai', - 'model' => $args[0]['model'] ?? null, - ]]); - } - }; - - $appsecOnlyPrehook = static function (bool $reportAppsec) use ($pushAppsecEvent) { - return static function ($This, $scope, $args) use ($reportAppsec, $pushAppsecEvent): void { - $pushAppsecEvent($reportAppsec, $args); + $appsecOnlyPrehook = static function (bool $reportAppsec) { + return static function ($This, $scope, $args) use ($reportAppsec): void { + if ($reportAppsec) { + OpenAIIntegration::pushLlmEventForAppsec($args[0]['model'] ?? null); + } }; }; - $handleRequestPrehook = fn ($streamed, $operationID, $reportApm, $reportAppsec) => function (\DDTrace\SpanData $span, $args) use ($operationID, $streamed, $reportApm, $reportAppsec, $pushAppsecEvent) { - $pushAppsecEvent($reportAppsec, $args); + $handleRequestPrehook = static function ($streamed, $operationID, $reportApm, $reportAppsec) { + return function (\DDTrace\SpanData $span, $args) use ($operationID, $streamed, $reportApm, $reportAppsec) { + if ($reportAppsec) { + OpenAIIntegration::pushLlmEventForAppsec($args[0]['model'] ?? null); + } - // This should not happen but just in case - if (!$reportApm) { - return; - } - OpenAIIntegration::setServiceName($span); - $clientData = ObjectKVStore::get($this, 'client_data'); - if (\is_null($clientData)) { - $transporter = ObjectKVStore::get($this, 'transporter'); - $clientData = ObjectKVStore::get($transporter, 'client_data'); - ObjectKVStore::put($this, 'client_data', $clientData); - } - /** @var array{baseUri: string, headers: string, apiKey: ?string} $clientData */ - OpenAIIntegration::handleRequest( - span: $span, - operationID: $operationID, - args: $args, - basePath: $clientData['baseUri'], - apiKey: $clientData['apiKey'], - streamed: $streamed - ); + // This should not happen but just in case + if (!$reportApm) { + return; + } + OpenAIIntegration::setServiceName($span); + $clientData = ObjectKVStore::get($this, 'client_data'); + if (\is_null($clientData)) { + $transporter = ObjectKVStore::get($this, 'transporter'); + $clientData = ObjectKVStore::get($transporter, 'client_data'); + ObjectKVStore::put($this, 'client_data', $clientData); + } + /** @var array{baseUri: string, headers: string, apiKey: ?string} $clientData */ + OpenAIIntegration::handleRequest( + span: $span, + operationID: $operationID, + args: $args, + basePath: $clientData['baseUri'], + apiKey: $clientData['apiKey'], + streamed: $streamed + ); + }; }; foreach ($targets as [$class, $method, $operationID, $httpMethod, $endpoint, $reportApm, $reportAppsec]) {