From 6053da9460c3d521e28ddb42acaa23aa848f5077 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 10 Apr 2026 12:09:18 +0200 Subject: [PATCH 1/4] feat(llma): add $ai_stop_reason property to LLM analytics Captures the LLM's reason for stopping generation (e.g. "stop", "end_turn", "tool_use", "SAFETY") as a top-level queryable event property. Previously, finish_reason was stripped from OTel events and never extracted from SDK wrappers. --- .../core-filter-definitions-by-group.json | 98 +++++++++++++++++++ .../ai/otel/middleware/traceloop.test.ts | 2 + .../ingestion/ai/otel/middleware/traceloop.ts | 12 ++- .../ai/otel/middleware/vercel-ai.test.ts | 1 + .../ingestion/ai/otel/middleware/vercel-ai.ts | 15 ++- posthog/hogql/ai.py | 5 + posthog/taxonomy/taxonomy.py | 5 + 7 files changed, 134 insertions(+), 4 deletions(-) diff --git a/frontend/src/taxonomy/core-filter-definitions-by-group.json b/frontend/src/taxonomy/core-filter-definitions-by-group.json index ff1d73f3200a..1501c3be09e2 100644 --- a/frontend/src/taxonomy/core-filter-definitions-by-group.json +++ b/frontend/src/taxonomy/core-filter-definitions-by-group.json @@ -154,6 +154,30 @@ ], "label": "AI cost model source (LLM)" }, + "$ai_error": { + "description": "The error message from a failed AI event.", + "examples": [ + "Rate limit exceeded", + "Invalid API key" + ], + "label": "AI Error (LLM)" + }, + "$ai_error_normalized": { + "description": "A normalized version of the AI error message for grouping similar errors.", + "examples": [ + "rate_limit_exceeded", + "invalid_api_key" + ], + "label": "AI Error Normalized (LLM)" + }, + "$ai_error_type": { + "description": "The type or class of the error from a failed AI event.", + "examples": [ + "RateLimitError", + "AuthenticationError" + ], + "label": "AI Error Type (LLM)" + }, "$ai_evaluation_allows_na": { "description": "Whether the evaluation allows N/A responses when the criteria doesn't apply.", "examples": [ @@ -288,6 +312,14 @@ ], "label": "AI input tokens (LLM)" }, + "$ai_is_error": { + "description": "Whether this AI event resulted in an error.", + "examples": [ + "true", + "false" + ], + "label": "AI Is Error (LLM)" + }, "$ai_latency": { "description": "The latency of the request made to the LLM API, in seconds.", "examples": [ @@ -414,6 +446,15 @@ ], "label": "AI Span Name (LLM)" }, + "$ai_stop_reason": { + "description": "The reason the LLM stopped generating tokens. Provider-specific values such as 'stop', 'end_turn', 'tool_use', 'length', 'max_tokens', 'STOP', 'SAFETY', etc.", + "examples": [ + "stop", + "end_turn", + "tool_use" + ], + "label": "AI stop reason (LLM)" + }, "$ai_stream": { "description": "Whether the response from the LLM API was streamed.", "examples": [ @@ -488,6 +529,14 @@ ], "label": "AI Trace ID (LLM)" }, + "$ai_trace_name": { + "description": "The name given to this AI trace. Deprecated in favor of $ai_span_name.", + "examples": [ + "summarize_text", + "chat_completion" + ], + "label": "AI Trace Name (LLM)" + }, "$ai_web_search_cost_usd": { "description": "The cost in USD of web searches performed during the LLM API request.", "examples": [ @@ -2894,6 +2943,30 @@ ], "label": "AI cost model source (LLM)" }, + "$ai_error": { + "description": "The error message from a failed AI event.", + "examples": [ + "Rate limit exceeded", + "Invalid API key" + ], + "label": "AI Error (LLM)" + }, + "$ai_error_normalized": { + "description": "A normalized version of the AI error message for grouping similar errors.", + "examples": [ + "rate_limit_exceeded", + "invalid_api_key" + ], + "label": "AI Error Normalized (LLM)" + }, + "$ai_error_type": { + "description": "The type or class of the error from a failed AI event.", + "examples": [ + "RateLimitError", + "AuthenticationError" + ], + "label": "AI Error Type (LLM)" + }, "$ai_evaluation_allows_na": { "description": "Whether the evaluation allows N/A responses when the criteria doesn't apply.", "examples": [ @@ -3028,6 +3101,14 @@ ], "label": "AI input tokens (LLM)" }, + "$ai_is_error": { + "description": "Whether this AI event resulted in an error.", + "examples": [ + "true", + "false" + ], + "label": "AI Is Error (LLM)" + }, "$ai_latency": { "description": "The latency of the request made to the LLM API, in seconds.", "examples": [ @@ -3154,6 +3235,15 @@ ], "label": "AI Span Name (LLM)" }, + "$ai_stop_reason": { + "description": "The reason the LLM stopped generating tokens. Provider-specific values such as 'stop', 'end_turn', 'tool_use', 'length', 'max_tokens', 'STOP', 'SAFETY', etc.", + "examples": [ + "stop", + "end_turn", + "tool_use" + ], + "label": "AI stop reason (LLM)" + }, "$ai_stream": { "description": "Whether the response from the LLM API was streamed.", "examples": [ @@ -3228,6 +3318,14 @@ ], "label": "AI Trace ID (LLM)" }, + "$ai_trace_name": { + "description": "The name given to this AI trace. Deprecated in favor of $ai_span_name.", + "examples": [ + "summarize_text", + "chat_completion" + ], + "label": "AI Trace Name (LLM)" + }, "$ai_web_search_cost_usd": { "description": "The cost in USD of web searches performed during the LLM API request.", "examples": [ diff --git a/nodejs/src/ingestion/ai/otel/middleware/traceloop.test.ts b/nodejs/src/ingestion/ai/otel/middleware/traceloop.test.ts index 7f29af5a1f58..41f777733510 100644 --- a/nodejs/src/ingestion/ai/otel/middleware/traceloop.test.ts +++ b/nodejs/src/ingestion/ai/otel/middleware/traceloop.test.ts @@ -204,6 +204,8 @@ describe('traceloop middleware', () => { expect(event.properties!['llm.usage.total_tokens']).toBeUndefined() expect(event.properties!['llm.response.finish_reason']).toBeUndefined() expect(event.properties!['llm.response.stop_reason']).toBeUndefined() + // stop_reason takes priority over finish_reason + expect(event.properties!['$ai_stop_reason']).toBe('end_turn') }) }) }) diff --git a/nodejs/src/ingestion/ai/otel/middleware/traceloop.ts b/nodejs/src/ingestion/ai/otel/middleware/traceloop.ts index e5260df709ae..d072c5c3f1bf 100644 --- a/nodejs/src/ingestion/ai/otel/middleware/traceloop.ts +++ b/nodejs/src/ingestion/ai/otel/middleware/traceloop.ts @@ -11,8 +11,6 @@ const STRIP_KEYS = [ 'traceloop.entity.output', 'llm.is_streaming', 'llm.usage.total_tokens', - 'llm.response.finish_reason', - 'llm.response.stop_reason', ] interface IndexedEntry { @@ -157,6 +155,16 @@ function process(event: PluginEvent, next: () => void): void { } } + // Map stop/finish reason to $ai_stop_reason before stripping + if (props['$ai_stop_reason'] === undefined) { + const stopReason = props['llm.response.stop_reason'] ?? props['llm.response.finish_reason'] + if (stopReason !== undefined) { + props['$ai_stop_reason'] = stopReason + } + } + delete props['llm.response.stop_reason'] + delete props['llm.response.finish_reason'] + props['$ai_lib'] = 'opentelemetry/traceloop' for (const key of STRIP_KEYS) { diff --git a/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.test.ts b/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.test.ts index fa53479e4ba1..6b8479e65a45 100644 --- a/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.test.ts +++ b/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.test.ts @@ -94,6 +94,7 @@ describe('vercel-ai middleware', () => { expect(key).not.toBe('operation.name') expect(key).not.toBe('resource.name') } + expect(event.properties!['$ai_stop_reason']).toBe('stop') }) }) diff --git a/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.ts b/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.ts index cb37071402e8..54aca212c6de 100644 --- a/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.ts +++ b/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.ts @@ -18,7 +18,6 @@ const STRIP_KEYS = [ 'ai.usage.promptTokens', 'ai.usage.completionTokens', 'ai.usage.tokens', - 'ai.response.finishReason', 'ai.response.id', 'ai.response.model', 'ai.response.timestamp', @@ -43,7 +42,6 @@ const STRIP_KEYS = [ 'resource.name', // Standard GenAI semantic convention attributes not mapped to $ai_* properties 'gen_ai.request.max_tokens', - 'gen_ai.response.finish_reasons', 'gen_ai.response.id', ] @@ -157,6 +155,19 @@ function process(event: PluginEvent, next: () => void): void { } } + // Map finish reason to $ai_stop_reason before stripping + if (props['$ai_stop_reason'] === undefined) { + const vercelReason = props['ai.response.finishReason'] + const genAiReasons = props['gen_ai.response.finish_reasons'] + if (vercelReason !== undefined) { + props['$ai_stop_reason'] = vercelReason + } else if (Array.isArray(genAiReasons) && genAiReasons.length > 0) { + props['$ai_stop_reason'] = genAiReasons[0] + } + } + delete props['ai.response.finishReason'] + delete props['gen_ai.response.finish_reasons'] + props['$ai_lib'] = 'opentelemetry/vercel-ai' for (const key of STRIP_KEYS) { diff --git a/posthog/hogql/ai.py b/posthog/hogql/ai.py index 1f6a7762ce0a..039cf24e9c7b 100644 --- a/posthog/hogql/ai.py +++ b/posthog/hogql/ai.py @@ -2595,6 +2595,11 @@ def hit_openai(messages, user, posthog_properties=None) -> tuple[str, int, int]: "description": "The number of tokens in the reasoning output from the LLM API.", "examples": [23], }, + "$ai_stop_reason": { + "label": "AI stop reason (LLM)", + "description": "The reason the LLM stopped generating tokens. Provider-specific values such as 'stop', 'end_turn', 'tool_use', 'length', 'max_tokens', 'STOP', 'SAFETY', etc.", + "examples": ["stop", "end_turn", "tool_use"], + }, "$ai_input_cost_usd": { "label": "AI input cost USD (LLM)", "description": "The cost in USD of the input tokens sent to the LLM API.", diff --git a/posthog/taxonomy/taxonomy.py b/posthog/taxonomy/taxonomy.py index facdfcde47aa..dd37f3b96127 100644 --- a/posthog/taxonomy/taxonomy.py +++ b/posthog/taxonomy/taxonomy.py @@ -1899,6 +1899,11 @@ class CoreFilterDefinition(TypedDict): "description": "The number of tokens in the reasoning output from the LLM API.", "examples": [23], }, + "$ai_stop_reason": { + "label": "AI stop reason (LLM)", + "description": "The reason the LLM stopped generating tokens. Provider-specific values such as 'stop', 'end_turn', 'tool_use', 'length', 'max_tokens', 'STOP', 'SAFETY', etc.", + "examples": ["stop", "end_turn", "tool_use"], + }, "$ai_input_cost_usd": { "label": "AI input cost USD (LLM)", "description": "The cost in USD of the input tokens sent to the LLM API.", From 89c0a1de30b45e84f5c6ad0ca2587b6842671334 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 10 Apr 2026 12:45:11 +0200 Subject: [PATCH 2/4] fix(llma): add missing OTel stop_reason test coverage - Test finish_reason-only path (no stop_reason) in traceloop middleware - Test gen_ai.response.finish_reasons array path in vercel-ai middleware - Test undefined when neither reason is present --- .../ai/otel/middleware/traceloop.test.ts | 20 +++++++++++++++++++ .../ai/otel/middleware/vercel-ai.test.ts | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/nodejs/src/ingestion/ai/otel/middleware/traceloop.test.ts b/nodejs/src/ingestion/ai/otel/middleware/traceloop.test.ts index 41f777733510..5f81fbc9974b 100644 --- a/nodejs/src/ingestion/ai/otel/middleware/traceloop.test.ts +++ b/nodejs/src/ingestion/ai/otel/middleware/traceloop.test.ts @@ -207,5 +207,25 @@ describe('traceloop middleware', () => { // stop_reason takes priority over finish_reason expect(event.properties!['$ai_stop_reason']).toBe('end_turn') }) + + it('maps finish_reason to $ai_stop_reason when stop_reason is absent', () => { + const event = createEvent('$ai_generation', { + 'llm.request.type': 'chat', + 'llm.response.finish_reason': 'stop', + }) + traceloop.process(event, () => mapOtelAttributes(event)) + + expect(event.properties!['llm.response.finish_reason']).toBeUndefined() + expect(event.properties!['$ai_stop_reason']).toBe('stop') + }) + + it('leaves $ai_stop_reason undefined when neither reason is present', () => { + const event = createEvent('$ai_generation', { + 'llm.request.type': 'chat', + }) + traceloop.process(event, () => mapOtelAttributes(event)) + + expect(event.properties!['$ai_stop_reason']).toBeUndefined() + }) }) }) diff --git a/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.test.ts b/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.test.ts index 6b8479e65a45..e18b8fa0f6d1 100644 --- a/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.test.ts +++ b/nodejs/src/ingestion/ai/otel/middleware/vercel-ai.test.ts @@ -96,6 +96,26 @@ describe('vercel-ai middleware', () => { } expect(event.properties!['$ai_stop_reason']).toBe('stop') }) + + it('maps gen_ai.response.finish_reasons array to $ai_stop_reason', () => { + const event = createEvent('$ai_generation', { + 'ai.operationId': 'ai.generateText.doGenerate', + 'gen_ai.response.finish_reasons': ['length'], + }) + convertOtelEvent(event) + + expect(event.properties!['gen_ai.response.finish_reasons']).toBeUndefined() + expect(event.properties!['$ai_stop_reason']).toBe('length') + }) + + it('leaves $ai_stop_reason undefined when no finish reason is present', () => { + const event = createEvent('$ai_generation', { + 'ai.operationId': 'ai.generateText.doGenerate', + }) + convertOtelEvent(event) + + expect(event.properties!['$ai_stop_reason']).toBeUndefined() + }) }) describe('$ai_trace (top-level span)', () => { From d5f34d43fec8ec12ac6d44428596279414d37021 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Mon, 20 Apr 2026 11:14:11 +0200 Subject: [PATCH 3/4] chore: revert hogql/ai.py taxonomy addition, will ship separately --- posthog/hogql/ai.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/posthog/hogql/ai.py b/posthog/hogql/ai.py index 039cf24e9c7b..1f6a7762ce0a 100644 --- a/posthog/hogql/ai.py +++ b/posthog/hogql/ai.py @@ -2595,11 +2595,6 @@ def hit_openai(messages, user, posthog_properties=None) -> tuple[str, int, int]: "description": "The number of tokens in the reasoning output from the LLM API.", "examples": [23], }, - "$ai_stop_reason": { - "label": "AI stop reason (LLM)", - "description": "The reason the LLM stopped generating tokens. Provider-specific values such as 'stop', 'end_turn', 'tool_use', 'length', 'max_tokens', 'STOP', 'SAFETY', etc.", - "examples": ["stop", "end_turn", "tool_use"], - }, "$ai_input_cost_usd": { "label": "AI input cost USD (LLM)", "description": "The cost in USD of the input tokens sent to the LLM API.", From 4baac50927c19c6d8a9903b87edcc4db37040253 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Mon, 20 Apr 2026 11:28:01 +0200 Subject: [PATCH 4/4] fix: remove duplicate $ai_stop_reason taxonomy entry --- posthog/taxonomy/taxonomy.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/posthog/taxonomy/taxonomy.py b/posthog/taxonomy/taxonomy.py index 253cc3568cea..42965aa037f9 100644 --- a/posthog/taxonomy/taxonomy.py +++ b/posthog/taxonomy/taxonomy.py @@ -1918,11 +1918,6 @@ class CoreFilterDefinition(TypedDict): "description": "The number of tokens in the reasoning output from the LLM API.", "examples": [23], }, - "$ai_stop_reason": { - "label": "AI stop reason (LLM)", - "description": "The reason the LLM stopped generating tokens. Provider-specific values such as 'stop', 'end_turn', 'tool_use', 'length', 'max_tokens', 'STOP', 'SAFETY', etc.", - "examples": ["stop", "end_turn", "tool_use"], - }, "$ai_input_cost_usd": { "label": "AI input cost USD (LLM)", "description": "The cost in USD of the input tokens sent to the LLM API.",