From f318ff15c2dc8160093e85ddad1db0041cbb3d76 Mon Sep 17 00:00:00 2001 From: Josh Snyder Date: Thu, 21 May 2026 14:17:14 +0000 Subject: [PATCH 1/4] feat(agent): route signal_report tasks to signals gateway product Attribute LLM gateway spend per task type so signals/report-generation cost can be tracked separately from background_agents and posthog_code: - internal task with origin_product "signal_report" -> "signals" - any other internal task -> "background_agents" - any other task -> "posthog_code" Extracted the decision into resolveGatewayProduct in @posthog/agent so the rule lives next to the GatewayProduct type and is independently testable. Generated-By: PostHog Code Task-Id: 31cea4e3-e0d7-49e7-bd01-d41bcafabe53 --- ...agent-server.configure-environment.test.ts | 33 ++++++++++++++- packages/agent/src/server/agent-server.ts | 13 +++--- packages/agent/src/types.ts | 3 +- packages/agent/src/utils/gateway.test.ts | 42 +++++++++++++++++++ packages/agent/src/utils/gateway.ts | 19 ++++++++- 5 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 packages/agent/src/utils/gateway.test.ts diff --git a/packages/agent/src/server/agent-server.configure-environment.test.ts b/packages/agent/src/server/agent-server.configure-environment.test.ts index e7d903b86..2a3832b69 100644 --- a/packages/agent/src/server/agent-server.configure-environment.test.ts +++ b/packages/agent/src/server/agent-server.configure-environment.test.ts @@ -2,7 +2,10 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { AgentServer } from "./agent-server"; interface TestableServer { - configureEnvironment(args?: { isInternal?: boolean }): void; + configureEnvironment(args?: { + isInternal?: boolean; + originProduct?: string | null; + }): void; } const ENV_KEYS_UNDER_TEST = [ @@ -85,6 +88,34 @@ describe("AgentServer.configureEnvironment", () => { expect(fromBackground).toBe("https://gateway.us.posthog.com/posthog_code"); }); + it("tags as signals when an internal task has origin_product 'signal_report'", () => { + buildServer("background").configureEnvironment({ + isInternal: true, + originProduct: "signal_report", + }); + + expect(process.env.LLM_GATEWAY_URL).toBe( + "https://gateway.us.posthog.com/signals", + ); + expect(process.env.ANTHROPIC_BASE_URL).toBe( + "https://gateway.us.posthog.com/signals", + ); + expect(process.env.OPENAI_BASE_URL).toBe( + "https://gateway.us.posthog.com/signals/v1", + ); + }); + + it("does not tag as signals when origin_product is 'signal_report' but the task is not internal", () => { + buildServer("background").configureEnvironment({ + isInternal: false, + originProduct: "signal_report", + }); + + expect(process.env.LLM_GATEWAY_URL).toBe( + "https://gateway.us.posthog.com/posthog_code", + ); + }); + it("respects the LLM_GATEWAY_URL override regardless of internal flag", () => { process.env.LLM_GATEWAY_URL = "http://ngrok.test/proxy"; diff --git a/packages/agent/src/server/agent-server.ts b/packages/agent/src/server/agent-server.ts index af4998a4a..253a4f59d 100644 --- a/packages/agent/src/server/agent-server.ts +++ b/packages/agent/src/server/agent-server.ts @@ -47,7 +47,7 @@ import type { } from "../types"; import { resourceLink } from "../utils/acp-content"; import { AsyncMutex } from "../utils/async-mutex"; -import { type GatewayProduct, getLlmGatewayUrl } from "../utils/gateway"; +import { getLlmGatewayUrl, resolveGatewayProduct } from "../utils/gateway"; import { Logger } from "../utils/logger"; import { logAgentshRuntimeInfo } from "./agentsh-runtime"; import { @@ -844,7 +844,10 @@ export class AgentServer { }), ]); - this.configureEnvironment({ isInternal: preTask?.internal === true }); + this.configureEnvironment({ + isInternal: preTask?.internal === true, + originProduct: preTask?.origin_product, + }); const prUrl = getTaskRunStateString(preTaskRun, "slack_notified_pr_url"); @@ -1804,13 +1807,13 @@ ${attributionInstructions} private configureEnvironment({ isInternal = false, + originProduct, }: { isInternal?: boolean; + originProduct?: string | null; } = {}): void { const { apiKey, apiUrl, projectId } = this.config; - const product: GatewayProduct = isInternal - ? "background_agents" - : "posthog_code"; + const product = resolveGatewayProduct({ isInternal, originProduct }); const gatewayUrl = process.env.LLM_GATEWAY_URL || getLlmGatewayUrl(apiUrl, product); const openaiBaseUrl = gatewayUrl.endsWith("/v1") diff --git a/packages/agent/src/types.ts b/packages/agent/src/types.ts index 18e5572c0..4a5864f6d 100644 --- a/packages/agent/src/types.ts +++ b/packages/agent/src/types.ts @@ -37,7 +37,8 @@ export interface Task { | "eval_clusters" | "user_created" | "support_queue" - | "session_summaries"; + | "session_summaries" + | "signal_report"; github_integration?: number | null; repository: string; // Format: "organization/repository" (e.g., "posthog/posthog-js") json_schema?: Record | null; // JSON schema for task output validation diff --git a/packages/agent/src/utils/gateway.test.ts b/packages/agent/src/utils/gateway.test.ts new file mode 100644 index 000000000..2047d8d03 --- /dev/null +++ b/packages/agent/src/utils/gateway.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { resolveGatewayProduct } from "./gateway"; + +describe("resolveGatewayProduct", () => { + it("returns posthog_code for non-internal tasks", () => { + expect(resolveGatewayProduct({ isInternal: false })).toBe("posthog_code"); + expect(resolveGatewayProduct()).toBe("posthog_code"); + }); + + it("returns posthog_code for non-internal tasks even when origin_product is signal_report", () => { + expect( + resolveGatewayProduct({ + isInternal: false, + originProduct: "signal_report", + }), + ).toBe("posthog_code"); + }); + + it("returns background_agents for internal tasks without origin_product", () => { + expect(resolveGatewayProduct({ isInternal: true })).toBe( + "background_agents", + ); + }); + + it("returns background_agents for internal tasks with a non-signal origin_product", () => { + expect( + resolveGatewayProduct({ + isInternal: true, + originProduct: "session_summaries", + }), + ).toBe("background_agents"); + }); + + it("returns signals for internal tasks with origin_product 'signal_report'", () => { + expect( + resolveGatewayProduct({ + isInternal: true, + originProduct: "signal_report", + }), + ).toBe("signals"); + }); +}); diff --git a/packages/agent/src/utils/gateway.ts b/packages/agent/src/utils/gateway.ts index 161247538..c5fbdfcba 100644 --- a/packages/agent/src/utils/gateway.ts +++ b/packages/agent/src/utils/gateway.ts @@ -1,4 +1,21 @@ -export type GatewayProduct = "posthog_code" | "background_agents"; +export type GatewayProduct = "posthog_code" | "background_agents" | "signals"; + +const SIGNAL_REPORT_ORIGIN_PRODUCT = "signal_report"; + +export function resolveGatewayProduct({ + isInternal, + originProduct, +}: { + isInternal?: boolean; + originProduct?: string | null; +} = {}): GatewayProduct { + if (isInternal) { + return originProduct === SIGNAL_REPORT_ORIGIN_PRODUCT + ? "signals" + : "background_agents"; + } + return "posthog_code"; +} function getGatewayBaseUrl(posthogHost: string): string { const url = new URL(posthogHost); From 145dcfc4d325b1928da0a9ebf55dde0f8dcb2dd0 Mon Sep 17 00:00:00 2001 From: Josh Snyder Date: Thu, 21 May 2026 14:25:49 +0000 Subject: [PATCH 2/4] feat(agent): forward task_origin_product and task_internal to gateway MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Send `x-posthog-property-task_origin_product` and `x-posthog-property-task_internal` to the LLM gateway via ANTHROPIC_CUSTOM_HEADERS. The gateway lifts any `x-posthog-property-*` header onto the captured `$ai_generation` event, so this lands task metadata on every generation we drive through Claude — letting LLM analytics break spend down by task origin without needing the gateway-product alone to encode it. The OpenAI/codex path has no equivalent custom-headers env var, so forwarding is Anthropic-only today. Also inline the one-shot `signal_report` constant — the literal only appears once now that `resolveGatewayProduct` is the single decision point. Generated-By: PostHog Code Task-Id: 31cea4e3-e0d7-49e7-bd01-d41bcafabe53 --- ...agent-server.configure-environment.test.ts | 20 +++++++++++ packages/agent/src/server/agent-server.ts | 15 +++++++- packages/agent/src/utils/gateway.test.ts | 36 ++++++++++++++++++- packages/agent/src/utils/gateway.ts | 23 +++++++++--- 4 files changed, 87 insertions(+), 7 deletions(-) diff --git a/packages/agent/src/server/agent-server.configure-environment.test.ts b/packages/agent/src/server/agent-server.configure-environment.test.ts index 2a3832b69..409e7c6bf 100644 --- a/packages/agent/src/server/agent-server.configure-environment.test.ts +++ b/packages/agent/src/server/agent-server.configure-environment.test.ts @@ -12,6 +12,7 @@ const ENV_KEYS_UNDER_TEST = [ "LLM_GATEWAY_URL", "ANTHROPIC_BASE_URL", "OPENAI_BASE_URL", + "ANTHROPIC_CUSTOM_HEADERS", ] as const; describe("AgentServer.configureEnvironment", () => { @@ -116,6 +117,25 @@ describe("AgentServer.configureEnvironment", () => { ); }); + it("forwards task_internal and task_origin_product as ANTHROPIC_CUSTOM_HEADERS", () => { + buildServer("background").configureEnvironment({ + isInternal: true, + originProduct: "signal_report", + }); + + expect(process.env.ANTHROPIC_CUSTOM_HEADERS).toBe( + "x-posthog-property-task_origin_product: signal_report\nx-posthog-property-task_internal: true", + ); + }); + + it("omits task_origin_product from ANTHROPIC_CUSTOM_HEADERS when not provided", () => { + buildServer("background").configureEnvironment({ isInternal: false }); + + expect(process.env.ANTHROPIC_CUSTOM_HEADERS).toBe( + "x-posthog-property-task_internal: false", + ); + }); + it("respects the LLM_GATEWAY_URL override regardless of internal flag", () => { process.env.LLM_GATEWAY_URL = "http://ngrok.test/proxy"; diff --git a/packages/agent/src/server/agent-server.ts b/packages/agent/src/server/agent-server.ts index 253a4f59d..1fddd4e1d 100644 --- a/packages/agent/src/server/agent-server.ts +++ b/packages/agent/src/server/agent-server.ts @@ -47,7 +47,11 @@ import type { } from "../types"; import { resourceLink } from "../utils/acp-content"; import { AsyncMutex } from "../utils/async-mutex"; -import { getLlmGatewayUrl, resolveGatewayProduct } from "../utils/gateway"; +import { + buildGatewayPropertyHeaders, + getLlmGatewayUrl, + resolveGatewayProduct, +} from "../utils/gateway"; import { Logger } from "../utils/logger"; import { logAgentshRuntimeInfo } from "./agentsh-runtime"; import { @@ -1819,6 +1823,14 @@ ${attributionInstructions} const openaiBaseUrl = gatewayUrl.endsWith("/v1") ? gatewayUrl : `${gatewayUrl}/v1`; + // Forward task metadata as `x-posthog-property-*` headers so the gateway + // lifts them onto the $ai_generation event. Routes through the Anthropic + // SDK's ANTHROPIC_CUSTOM_HEADERS env var; the OpenAI/codex path has no + // equivalent today. + const customHeaders = buildGatewayPropertyHeaders({ + task_origin_product: originProduct, + task_internal: isInternal, + }); Object.assign(process.env, { // PostHog @@ -1831,6 +1843,7 @@ ${attributionInstructions} ANTHROPIC_API_KEY: apiKey, ANTHROPIC_AUTH_TOKEN: apiKey, ANTHROPIC_BASE_URL: gatewayUrl, + ANTHROPIC_CUSTOM_HEADERS: customHeaders, // OpenAI (for models like GPT-4, o1, etc.) OPENAI_API_KEY: apiKey, OPENAI_BASE_URL: openaiBaseUrl, diff --git a/packages/agent/src/utils/gateway.test.ts b/packages/agent/src/utils/gateway.test.ts index 2047d8d03..0ffbcc2ae 100644 --- a/packages/agent/src/utils/gateway.test.ts +++ b/packages/agent/src/utils/gateway.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { resolveGatewayProduct } from "./gateway"; +import { buildGatewayPropertyHeaders, resolveGatewayProduct } from "./gateway"; describe("resolveGatewayProduct", () => { it("returns posthog_code for non-internal tasks", () => { @@ -40,3 +40,37 @@ describe("resolveGatewayProduct", () => { ).toBe("signals"); }); }); + +describe("buildGatewayPropertyHeaders", () => { + it("renders each property as an x-posthog-property header line", () => { + expect( + buildGatewayPropertyHeaders({ + task_origin_product: "signal_report", + task_internal: true, + }), + ).toBe( + "x-posthog-property-task_origin_product: signal_report\nx-posthog-property-task_internal: true", + ); + }); + + it("drops null and undefined values but keeps falsy primitives", () => { + expect( + buildGatewayPropertyHeaders({ + task_origin_product: null, + task_internal: false, + task_count: 0, + }), + ).toBe( + "x-posthog-property-task_internal: false\nx-posthog-property-task_count: 0", + ); + }); + + it("returns an empty string when no usable properties remain", () => { + expect( + buildGatewayPropertyHeaders({ + task_origin_product: null, + task_internal: undefined, + }), + ).toBe(""); + }); +}); diff --git a/packages/agent/src/utils/gateway.ts b/packages/agent/src/utils/gateway.ts index c5fbdfcba..ff27758b2 100644 --- a/packages/agent/src/utils/gateway.ts +++ b/packages/agent/src/utils/gateway.ts @@ -1,7 +1,5 @@ export type GatewayProduct = "posthog_code" | "background_agents" | "signals"; -const SIGNAL_REPORT_ORIGIN_PRODUCT = "signal_report"; - export function resolveGatewayProduct({ isInternal, originProduct, @@ -10,13 +8,28 @@ export function resolveGatewayProduct({ originProduct?: string | null; } = {}): GatewayProduct { if (isInternal) { - return originProduct === SIGNAL_REPORT_ORIGIN_PRODUCT - ? "signals" - : "background_agents"; + return originProduct === "signal_report" ? "signals" : "background_agents"; } return "posthog_code"; } +/** + * Build `x-posthog-property-: ` header lines that the LLM + * gateway lifts onto the `$ai_generation` event it captures for each call + * (see `services/llm-gateway/src/llm_gateway/request_context.py`). + * + * Returns a newline-joined string ready for `ANTHROPIC_CUSTOM_HEADERS`. + * `null`/`undefined` property values are dropped. + */ +export function buildGatewayPropertyHeaders( + properties: Record, +): string { + return Object.entries(properties) + .filter(([, value]) => value !== null && value !== undefined) + .map(([key, value]) => `x-posthog-property-${key}: ${value}`) + .join("\n"); +} + function getGatewayBaseUrl(posthogHost: string): string { const url = new URL(posthogHost); const hostname = url.hostname; From 4f14949e92c26ab15d1a106a533f45e63d5afe92 Mon Sep 17 00:00:00 2001 From: Josh Snyder Date: Thu, 21 May 2026 15:35:46 +0000 Subject: [PATCH 3/4] feat(agent): forward task_id, task_run_id, and task_user_id to gateway Extend ANTHROPIC_CUSTOM_HEADERS with the IDs the agent server already has in scope (from the JWT payload) so every $ai_generation captured by the gateway can be joined back to the task, run, and triggering user without an extra hop. Generated-By: PostHog Code Task-Id: 31cea4e3-e0d7-49e7-bd01-d41bcafabe53 --- .../agent-server.configure-environment.test.ts | 18 +++++++++++++++--- packages/agent/src/server/agent-server.ts | 12 ++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/agent/src/server/agent-server.configure-environment.test.ts b/packages/agent/src/server/agent-server.configure-environment.test.ts index 409e7c6bf..23e3ed33d 100644 --- a/packages/agent/src/server/agent-server.configure-environment.test.ts +++ b/packages/agent/src/server/agent-server.configure-environment.test.ts @@ -5,6 +5,9 @@ interface TestableServer { configureEnvironment(args?: { isInternal?: boolean; originProduct?: string | null; + taskId?: string | null; + taskRunId?: string | null; + taskUserId?: number | null; }): void; } @@ -117,18 +120,27 @@ describe("AgentServer.configureEnvironment", () => { ); }); - it("forwards task_internal and task_origin_product as ANTHROPIC_CUSTOM_HEADERS", () => { + it("forwards task metadata as ANTHROPIC_CUSTOM_HEADERS", () => { buildServer("background").configureEnvironment({ isInternal: true, originProduct: "signal_report", + taskId: "task-abc", + taskRunId: "run-xyz", + taskUserId: 42, }); expect(process.env.ANTHROPIC_CUSTOM_HEADERS).toBe( - "x-posthog-property-task_origin_product: signal_report\nx-posthog-property-task_internal: true", + [ + "x-posthog-property-task_origin_product: signal_report", + "x-posthog-property-task_internal: true", + "x-posthog-property-task_id: task-abc", + "x-posthog-property-task_run_id: run-xyz", + "x-posthog-property-task_user_id: 42", + ].join("\n"), ); }); - it("omits task_origin_product from ANTHROPIC_CUSTOM_HEADERS when not provided", () => { + it("omits optional task metadata from ANTHROPIC_CUSTOM_HEADERS when not provided", () => { buildServer("background").configureEnvironment({ isInternal: false }); expect(process.env.ANTHROPIC_CUSTOM_HEADERS).toBe( diff --git a/packages/agent/src/server/agent-server.ts b/packages/agent/src/server/agent-server.ts index 1fddd4e1d..58e7d9e61 100644 --- a/packages/agent/src/server/agent-server.ts +++ b/packages/agent/src/server/agent-server.ts @@ -851,6 +851,9 @@ export class AgentServer { this.configureEnvironment({ isInternal: preTask?.internal === true, originProduct: preTask?.origin_product, + taskId: payload.task_id, + taskRunId: payload.run_id, + taskUserId: payload.user_id, }); const prUrl = getTaskRunStateString(preTaskRun, "slack_notified_pr_url"); @@ -1812,9 +1815,15 @@ ${attributionInstructions} private configureEnvironment({ isInternal = false, originProduct, + taskId, + taskRunId, + taskUserId, }: { isInternal?: boolean; originProduct?: string | null; + taskId?: string | null; + taskRunId?: string | null; + taskUserId?: number | null; } = {}): void { const { apiKey, apiUrl, projectId } = this.config; const product = resolveGatewayProduct({ isInternal, originProduct }); @@ -1830,6 +1839,9 @@ ${attributionInstructions} const customHeaders = buildGatewayPropertyHeaders({ task_origin_product: originProduct, task_internal: isInternal, + task_id: taskId, + task_run_id: taskRunId, + task_user_id: taskUserId, }); Object.assign(process.env, { From b43925a1054ebf6d5a7a94ff129c5a3fb4741ce6 Mon Sep 17 00:00:00 2001 From: Josh Snyder Date: Thu, 21 May 2026 15:52:41 +0000 Subject: [PATCH 4/4] test(agent): parameterise resolveGatewayProduct cases with it.each Per review feedback, collapse the five near-identical resolveGatewayProduct tests into a single it.each table so the input/output contract is visible at a glance and each row reports independently on failure. Generated-By: PostHog Code Task-Id: 31cea4e3-e0d7-49e7-bd01-d41bcafabe53 --- packages/agent/src/utils/gateway.test.ts | 68 +++++++++++------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/packages/agent/src/utils/gateway.test.ts b/packages/agent/src/utils/gateway.test.ts index 0ffbcc2ae..5329a712d 100644 --- a/packages/agent/src/utils/gateway.test.ts +++ b/packages/agent/src/utils/gateway.test.ts @@ -2,43 +2,37 @@ import { describe, expect, it } from "vitest"; import { buildGatewayPropertyHeaders, resolveGatewayProduct } from "./gateway"; describe("resolveGatewayProduct", () => { - it("returns posthog_code for non-internal tasks", () => { - expect(resolveGatewayProduct({ isInternal: false })).toBe("posthog_code"); - expect(resolveGatewayProduct()).toBe("posthog_code"); - }); - - it("returns posthog_code for non-internal tasks even when origin_product is signal_report", () => { - expect( - resolveGatewayProduct({ - isInternal: false, - originProduct: "signal_report", - }), - ).toBe("posthog_code"); - }); - - it("returns background_agents for internal tasks without origin_product", () => { - expect(resolveGatewayProduct({ isInternal: true })).toBe( - "background_agents", - ); - }); - - it("returns background_agents for internal tasks with a non-signal origin_product", () => { - expect( - resolveGatewayProduct({ - isInternal: true, - originProduct: "session_summaries", - }), - ).toBe("background_agents"); - }); - - it("returns signals for internal tasks with origin_product 'signal_report'", () => { - expect( - resolveGatewayProduct({ - isInternal: true, - originProduct: "signal_report", - }), - ).toBe("signals"); - }); + it.each([ + { isInternal: false, originProduct: undefined, expected: "posthog_code" }, + { + isInternal: undefined, + originProduct: undefined, + expected: "posthog_code", + }, + { + isInternal: false, + originProduct: "signal_report", + expected: "posthog_code", + }, + { + isInternal: true, + originProduct: undefined, + expected: "background_agents", + }, + { + isInternal: true, + originProduct: "session_summaries", + expected: "background_agents", + }, + { isInternal: true, originProduct: "signal_report", expected: "signals" }, + ] as const)( + "isInternal=$isInternal originProduct=$originProduct -> $expected", + ({ isInternal, originProduct, expected }) => { + expect(resolveGatewayProduct({ isInternal, originProduct })).toBe( + expected, + ); + }, + ); }); describe("buildGatewayPropertyHeaders", () => {