diff --git a/sdk/typescript/packages/core/src/capture/common.ts b/sdk/typescript/packages/core/src/capture/common.ts index ebd18a2..a51baf9 100644 --- a/sdk/typescript/packages/core/src/capture/common.ts +++ b/sdk/typescript/packages/core/src/capture/common.ts @@ -1,18 +1,10 @@ import { randomUUID } from "node:crypto"; -import { currentConfig } from "../config.js"; import type { AdrianCallbackHandler } from "../handler.js"; -import { runWithInvocationId } from "../context.js"; -import { assertToolCallsAllowed } from "../policy.js"; -import { getWebSocketClient } from "../registry.js"; +import { getInvocationId, runWithInvocationId } from "../context.js"; import type { CallbackMetadata, ChatMessage, LlmEndData, TokenUsage, ToolArgs, ToolCallRecord } from "../types.js"; -/** Gate tool calls after the paired LLM event has been emitted (maps tool-call ids on the WS client). */ -export async function gateLlmEndData(end: LlmEndData): Promise { - await assertToolCallsAllowed( - end.toolCalls.map((call) => call.id), - getWebSocketClient(), - currentConfig()?.blockTimeout ?? 30, - ); +/** LLM end tool-call metadata is informational and never blockable. */ +export async function gateLlmEndData(_end: LlmEndData): Promise { } export interface LlmCaptureInput { @@ -33,7 +25,8 @@ export async function captureLlmCall( if (!handler) return execute(); const runId = randomUUID(); - return runWithInvocationId(randomUUID(), async () => { + const invocationId = getInvocationId(); + const run = async () => { await handler.handleChatModelStart({ name: input.model }, [input.messages], runId, input.parentRunId, { metadata: input.metadata }); try { const result = await execute(); @@ -45,7 +38,31 @@ export async function captureLlmCall( await handler.handleLLMError(error, runId); throw error; } - }); + }; + return invocationId === null ? run() : runWithInvocationId(invocationId, run); +} + +/** Wrap an LLM call that may fail before returning (e.g. streaming create). Records start+error, then re-throws. */ +export async function captureLlmExecute( + getHandler: () => AdrianCallbackHandler | null, + input: LlmCaptureInput, + execute: () => Promise, +): Promise { + try { + return await execute(); + } catch (error) { + const handler = getHandler(); + if (handler) { + const runId = randomUUID(); + const invocationId = getInvocationId(); + const run = async () => { + await handler.handleChatModelStart({ name: input.model }, [input.messages], runId, input.parentRunId, { metadata: input.metadata }); + await handler.handleLLMError(error, runId); + }; + await (invocationId === null ? run() : runWithInvocationId(invocationId, run)); + } + throw error; + } } export function captureLlmAsyncIterable( @@ -60,37 +77,94 @@ export function captureLlmAsyncIterable( if (!handler) return iterable; const runId = randomUUID(); - const invocationId = randomUUID(); - - async function* wrapped(): AsyncGenerator { - await handler?.handleChatModelStart({ name: input.model }, [input.messages], runId, input.parentRunId, { metadata: input.metadata }); - yield* runWithInvocationId(invocationId, async function* () { - let emitted = false; - let failed = false; - try { - for await (const chunk of iterable) { - aggregate(chunk); - yield chunk; - } - emitted = true; - const endData = await extractOutput(); - await handler?.handleLLMEnd(endData, runId); - await afterPairedEmit?.(endData); - } catch (error) { - failed = true; - await handler?.handleLLMError(error, runId); - throw error; - } finally { - if (!emitted && !failed) { + const invocationId = getInvocationId(); + const activeHandler = handler; + + const createIterator = (): AsyncIterator => { + async function* gen(): AsyncGenerator { + await activeHandler.handleChatModelStart({ name: input.model }, [input.messages], runId, input.parentRunId, { metadata: input.metadata }); + + const streamBody = async function* (): AsyncGenerator { + let emitted = false; + let failed = false; + try { + for await (const chunk of iterable) { + aggregate(chunk); + yield chunk; + } + emitted = true; const endData = await extractOutput(); - await handler?.handleLLMEnd(endData, runId); + await activeHandler.handleLLMEnd(endData, runId); await afterPairedEmit?.(endData); + } catch (error) { + failed = true; + await activeHandler.handleLLMError(error, runId); + throw error; + } finally { + if (!emitted && !failed) { + const endData = await extractOutput(); + await activeHandler.handleLLMEnd(endData, runId); + await afterPairedEmit?.(endData); + } } + }; + + if (invocationId === null) { + yield* streamBody(); + } else { + yield* runWithInvocationId(invocationId, streamBody); } - }); - } + } + return gen(); + }; + + return preserveStreamSurface(iterable, createIterator); +} + +/** Keep provider stream helpers (tee, toReadableStream, controller) while intercepting iteration. */ +function preserveStreamSurface( + source: AsyncIterable, + createIterator: () => AsyncIterator, +): AsyncIterable { + const iterable: AsyncIterable = { [Symbol.asyncIterator]: createIterator }; + if (!source || typeof source !== "object") return iterable; + + const stream = source as Record & AsyncIterable; + return new Proxy(iterable, { + get(_target, prop, receiver) { + if (prop === Symbol.asyncIterator) return createIterator; + if (prop === "tee") { + return () => teeCapturingStream(createIterator).map((branch) => preserveStreamSurface(stream, branch)); + } + const value = Reflect.get(stream, prop, stream); + if (typeof value === "function") return value.bind(receiver); + return value; + }, + }); +} + +/** Split one capturing iterator into two branches without restarting capture. */ +function teeCapturingStream( + createIterator: () => AsyncIterator, +): [() => AsyncIterator, () => AsyncIterator] { + const left: Array>> = []; + const right: Array>> = []; + const iterator = createIterator(); + + const branchIterator = (queue: Array>>) => (): AsyncIterator => ({ + next: () => { + if (queue.length === 0) { + const result = iterator.next(); + left.push(result); + right.push(result); + } + return queue.shift()!; + }, + return: (value) => iterator.return?.(value) ?? Promise.resolve({ done: true as const, value: undefined }), + throw: (error) => iterator.throw?.(error) ?? Promise.reject(error), + }); - return wrapped(); + return [branchIterator(left), branchIterator(right)]; } export function normalizeMessages(input: unknown): ChatMessage[] { diff --git a/sdk/typescript/packages/openai/README.md b/sdk/typescript/packages/openai/README.md new file mode 100644 index 0000000..4a666f4 --- /dev/null +++ b/sdk/typescript/packages/openai/README.md @@ -0,0 +1,5 @@ +# @secureagentics/adrian-openai + +OpenAI SDK instrumentation for Adrian security monitoring. + +Full documentation: [sdk/typescript/README.md](../../README.md) diff --git a/sdk/typescript/packages/openai/package.json b/sdk/typescript/packages/openai/package.json new file mode 100644 index 0000000..40cd21c --- /dev/null +++ b/sdk/typescript/packages/openai/package.json @@ -0,0 +1,52 @@ +{ + "name": "@secureagentics/adrian-openai", + "version": "1.0.0", + "description": "OpenAI SDK instrumentation for Adrian security monitoring.", + "license": "Apache-2.0", + "author": "Secure Agentics ", + "type": "module", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.cjs" + } + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "build": "tsup src/index.ts --format esm,cjs --clean && tsc -p tsconfig.build.json", + "typecheck": "tsc --noEmit", + "test": "vitest run" + }, + "keywords": [ + "openai", + "ai", + "agents", + "security", + "monitoring" + ], + "dependencies": { + "@secureagentics/adrian": "^1.0.0" + }, + "peerDependencies": { + "openai": ">=4.0.0" + }, + "peerDependenciesMeta": { + "openai": { + "optional": true + } + }, + "devDependencies": { + "@secureagentics/adrian": "file:../core", + "@types/node": "^25.9.1", + "tsup": "^8.5.1", + "typescript": "^6.0.3", + "vitest": "^4.1.7" + } +} diff --git a/sdk/typescript/packages/openai/src/index.ts b/sdk/typescript/packages/openai/src/index.ts new file mode 100644 index 0000000..80244aa --- /dev/null +++ b/sdk/typescript/packages/openai/src/index.ts @@ -0,0 +1,348 @@ +import { randomUUID } from "node:crypto"; +import { + BLOCKED_TOOL_MESSAGE, + currentConfig, + gateToolCallIds, + getHandler, + getInvocationId, + getWebSocketClient, + init, + runWithInvocationId, + shutdown, + version, + __version__, +} from "@secureagentics/adrian"; +import type { CallbackMetadata, EventData, InitOptions, LlmEndData, ToolCallRecord } from "@secureagentics/adrian"; +import { + captureLlmAsyncIterable, + captureLlmCall, + captureLlmExecute, + gateLlmEndData, + emptyLlmEnd, + messagesFromPromptLike, + normalizeMessages, + normalizeUsage, + parseToolArgs, + stringifyContent, +} from "@secureagentics/adrian/capture"; + +export interface AdrianOptions { + metadata?: CallbackMetadata | null; +} + +export interface ToolCallLike { + id: string; + type?: string; + function?: { + name?: string; + arguments?: string; + }; + name?: string; + arguments?: string; +} + +export interface ToolCaptureOptions { + metadata?: CallbackMetadata | null; + parentRunId?: string; +} + +/** Wrap manual OpenAI tool execution so Adrian captures tool events. */ +export async function captureTool( + toolCall: ToolCallLike, + execute: () => T | Promise, + options: ToolCaptureOptions = {}, +): Promise { + const handler = getHandler(); + if (!handler) return execute(); + + const runId = randomUUID(); + const toolName = String(toolCall.function?.name ?? toolCall.name ?? "unknown"); + const toolCallId = String(toolCall.id ?? ""); + const input = String(toolCall.function?.arguments ?? toolCall.arguments ?? ""); + const metadata = integrationMetadata(options.metadata, "openai.tool_call"); + + const gate = await gateToolCallIds(toolCallId ? [toolCallId] : [], getWebSocketClient(), currentConfig()?.blockTimeout ?? 30); + + // Match Python: only inherit an invocation that was established upstream. + const invocationId = getInvocationId(); + const run = async () => { + await handler.handleToolStart({ name: toolName }, input, runId, options.parentRunId, { metadata, tool_call_id: toolCallId }); + if (gate.action === "block") { + await handler.handleToolEnd(BLOCKED_TOOL_MESSAGE, runId); + return BLOCKED_TOOL_MESSAGE as T; + } + try { + const result = await execute(); + await handler.handleToolEnd(result, runId); + return result; + } catch (error) { + await handler.handleToolError(error, runId); + throw error; + } + }; + return invocationId === null ? run() : runWithInvocationId(invocationId, run); +} + +/** Public entry: `adrian.openai(new OpenAI())`. */ +function wrapOpenAI(client: T, options: AdrianOptions = {}): T { + // Top-level proxy routes into chat.completions and responses.create. + return new Proxy(client, { + get(target, prop, receiver) { + if (prop === "chat") return instrumentChat(Reflect.get(target, prop, receiver), options); + if (prop === "responses") return instrumentResponses(Reflect.get(target, prop, receiver), options); + return Reflect.get(target, prop, receiver); + }, + }); +} + +/** Proxy `client.chat` → `completions.create`. */ +function instrumentChat(chat: unknown, options: AdrianOptions): unknown { + if (!chat || typeof chat !== "object") return chat; + return new Proxy(chat as Record, { + get(target, prop, receiver) { + if (prop === "completions") return instrumentChatCompletions(Reflect.get(target, prop, receiver), options); + return Reflect.get(target, prop, receiver); + }, + }); +} + +/** + * Intercept `chat.completions.create`. + * Non-stream: one paired LLM event after the response resolves. + * Stream: wrap the async iterable; emit one event when the stream ends. + */ +function instrumentChatCompletions(completions: unknown, options: AdrianOptions): unknown { + if (!completions || typeof completions !== "object") return completions; + return new Proxy(completions as Record, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (prop !== "create" || typeof value !== "function") return value; + return async function adrianOpenAIChatCreate(this: unknown, body: Record = {}, ...rest: unknown[]) { + const model = String(body.model || "openai"); + const metadata = integrationMetadata(options.metadata, "openai.chat.completions"); + const messages = normalizeMessages(body.messages); + if (body.stream === true) { + return captureLlmExecute(getHandler, { model, messages, metadata }, async () => { + const result = await value.call(target, body, ...rest); + if (isAsyncIterable(result)) return captureChatCompletionStream(model, messages, metadata, result); + return result; + }); + } + // Non-stream path: core capture helper pairs handleChatModelStart/End around the call. + return captureLlmCall(getHandler, { model, messages, metadata }, () => Promise.resolve(value.call(target, body, ...rest)), extractChatCompletion, gateLlmEndData); + }; + }, + }); +} + +/** + * Intercept `responses.create`. + * Maps `input` / `instructions` into Adrian message shape via messagesFromPromptLike. + */ +function instrumentResponses(responses: unknown, options: AdrianOptions): unknown { + if (!responses || typeof responses !== "object") return responses; + return new Proxy(responses as Record, { + get(target, prop, receiver) { + const value = Reflect.get(target, prop, receiver); + if (prop !== "create" || typeof value !== "function") return value; + return async function adrianOpenAIResponsesCreate(this: unknown, body: Record = {}, ...rest: unknown[]) { + const model = String(body.model ?? "openai"); + const metadata = integrationMetadata(options.metadata, "openai.responses"); + const messages = messagesFromPromptLike({ + input: body.input, + instructions: body.instructions, + }); + if (body.stream === true) { + return captureLlmExecute(getHandler, { model, messages, metadata }, async () => { + const result = await Promise.resolve(value.call(target, body, ...rest)); + + if (isAsyncIterable(result)) return captureResponseStream(model, messages, metadata, result); + return result; + }); + } + return captureLlmCall(getHandler, { model, messages, metadata }, () => Promise.resolve(value.call(target, body, ...rest)), extractResponse, gateLlmEndData); + }; + }, + }); +} + +/** Aggregate Chat Completions stream chunks into one paired LLM event at the end. */ +function captureChatCompletionStream(model: string, messages: ReturnType, metadata: CallbackMetadata | null, stream: AsyncIterable): AsyncIterable { + let output = ""; + let usage: LlmEndData["usage"] = null; + // OpenAI streams tool calls by index; merge partial deltas before emit. + const toolCallParts = new Map(); + return captureLlmAsyncIterable(getHandler, { model, messages, metadata }, stream, (chunk) => { + const obj = chunk as Record; + // Usage arrives on the final chunk when stream_options.include_usage is set. + usage = normalizeUsage(obj.usage) ?? usage; + for (const choice of (obj.choices as any[] ?? [])) { + const delta = choice.delta; + output += stringifyContent(delta?.content); + + for (const call of (delta?.tool_calls ?? [])) { + const fn = call.function; + const current = toolCallParts.get(call.index) ?? { id: "", name: "", args: "" }; + toolCallParts.set(call.index, { + id: call.id ?? current.id, + name: fn?.name ?? current.name, + args: current.args + (fn?.arguments ?? ""), + }); + } + } + }, () => emptyLlmEnd(output, [...toolCallParts.values()].map((call) => ({ id: call.id, name: call.name, args: parseToolArgs(call.args) })), usage), gateLlmEndData); +} + +/** Aggregate Responses API stream events into one paired LLM event at the end. */ +function captureResponseStream(model: string, messages: ReturnType, metadata: CallbackMetadata | null, stream: AsyncIterable): AsyncIterable { + let output = ""; + let usage: LlmEndData["usage"] = null; + + const toolCallParts = new Map(); + + return captureLlmAsyncIterable( + getHandler, + { model, messages, metadata }, + stream, + (chunk) => { + const obj = chunk as Record; + switch (obj.type) { + case "response.output_text.delta": + output += obj.delta; + break; + + case "response.completed": + usage = normalizeUsage(obj.usage) ?? usage; + break; + } + collectResponseStreamToolCall(obj, toolCallParts); + }, + () => + emptyLlmEnd( + output, + [...toolCallParts.values()].map((call) => ({ + id: call.id, + name: call.name, + args: parseToolArgs(call.args), + })), + usage, + ), + gateLlmEndData, + ); +} + +/** Map a completed Chat Completions response object into Adrian LLM end data. */ +function extractChatCompletion(result: unknown): LlmEndData { + if (isAsyncIterable(result)) return emptyLlmEnd(); + const obj = result as Record; + const message = (obj.choices as Record[])[0] + ?.message as Record | undefined; + + return emptyLlmEnd( + stringifyContent(message?.content), + normalizeOpenAIToolCalls(message?.tool_calls), + normalizeUsage(obj.usage), + ); +} + +/** Map a completed Responses API object into Adrian LLM end data. */ +function extractResponse(result: unknown): LlmEndData { + if (isAsyncIterable(result)) return emptyLlmEnd(); + const obj = result as Record; + + return emptyLlmEnd( + typeof obj.output_text === "string" ? obj.output_text : stringifyContent(obj.output), + normalizeResponseToolCalls(obj.output), + normalizeUsage(obj.usage), + ); +} + +/** Normalise Chat Completions `message.tool_calls` into Adrian tool call records. */ +function normalizeOpenAIToolCalls(raw: unknown): ToolCallRecord[] { + if (!Array.isArray(raw)) return []; + return raw.map((call) => { + const obj = call as Record; + const fn = obj.function as {name: string, arguments: string} | undefined; + return { id: String(obj.id ?? ""), name: String(fn?.name ?? obj.name ?? ""), args: parseToolArgs(fn?.arguments) }; + }); +} + +/** Walk Responses `output` tree for function_call / tool_call items. */ +function normalizeResponseToolCalls(raw: unknown): ToolCallRecord[] { + if (!Array.isArray(raw)) return []; + return raw.flatMap((item) => { + const obj = item && typeof item === "object" ? item as Record : {}; + if (obj.type === "message" && Array.isArray(obj.content)) { + return obj.content.flatMap((part) => normalizeResponseToolCalls([part])); + } + if (obj.type !== "function_call" && obj.type !== "tool_call") return []; + return [{ id: String(obj.call_id ?? obj.id ?? ""), name: String(obj.name ?? ""), args: parseToolArgs(obj.arguments) }]; + }); +} + +/** + * Responses streaming emits tool metadata and argument deltas in separate events. + * Handles `response.output_item.added` items and `response.function_call_arguments.delta`. + */ +function collectResponseStreamToolCall(obj: Record, toolCallParts: Map): void { + switch (obj.type) { + case "response.output_item.added": + case "response.output_item.done": { + const item = obj.item as Record | undefined; + if (!item || (item.type !== "function_call" && item.type !== "tool_call")) break; + const key = String(item.id ?? item.call_id ?? obj.output_index ?? toolCallParts.size); + const current = toolCallParts.get(key) ?? { id: "", name: "", args: "" }; + toolCallParts.set(key, { + id: String(item.call_id ?? item.id ?? current.id), + name: String(item.name ?? current.name), + args: typeof item.arguments === "string" ? item.arguments : current.args, + }); + break; + } + + case "response.function_call_arguments.delta": { + if (typeof obj.delta !== "string") break; + const key = String(obj.item_id ?? obj.output_index ?? toolCallParts.size); + const current = toolCallParts.get(key) ?? { id: "", name: "", args: "" }; + toolCallParts.set(key, { ...current, args: current.args + obj.delta }); + break; + } + } +} + +/** Tag events with provider integration metadata for downstream filtering. */ +function integrationMetadata(metadata: CallbackMetadata | null | undefined, operation: string): CallbackMetadata { + return { ...(metadata ?? {}), adrianIntegration: "openai", operation }; +} + +function isAsyncIterable(value: unknown): value is AsyncIterable { + return Boolean(value && typeof value === "object" && Symbol.asyncIterator in value); +} + +/** + * Unified Adrian namespace for OpenAI apps. + * Prefer `import { adrian } from "@secureagentics/adrian-openai"` over named exports. + */ +export const adrian = { + init, + shutdown, + getHandler, + getWebSocketClient, + version, + __version__: __version__, + openai: wrapOpenAI, + captureTool, +}; + +export { + AdrianPolicyBlockedError, + BLOCKED_TOOL_MESSAGE, + getHandler, + getWebSocketClient, + init, + shutdown, + version, + __version__, +} from "@secureagentics/adrian"; + +export type { EventData, InitOptions }; diff --git a/sdk/typescript/packages/openai/tests/openai.test.ts b/sdk/typescript/packages/openai/tests/openai.test.ts new file mode 100644 index 0000000..bd6d455 --- /dev/null +++ b/sdk/typescript/packages/openai/tests/openai.test.ts @@ -0,0 +1,613 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as adrianCore from "@secureagentics/adrian"; +import { BLOCKED_TOOL_MESSAGE, Mode, type EventData, type PairedEvent, type Verdict, type WebSocketClient } from "@secureagentics/adrian"; +import { adrian } from "../src/index.js"; + +function mockOpenAIStream(chunks: T[]) { + const controller = new AbortController(); + async function* sourceIterator() { + for (const chunk of chunks) yield chunk; + } + + const stream = { + controller, + async *[Symbol.asyncIterator]() { + yield* sourceIterator(); + }, + tee() { + const left: Array>> = []; + const right: Array>> = []; + const iterator = sourceIterator(); + const branch = (queue: Array>>) => ({ + next: () => { + if (queue.length === 0) { + const result = iterator.next(); + left.push(result); + right.push(result); + } + return queue.shift()!; + }, + }); + const branchStream = (iter: () => AsyncIterator) => ({ + controller, + [Symbol.asyncIterator]: iter, + tee: stream.tee, + toReadableStream: stream.toReadableStream, + }); + return [branchStream(() => branch(left)), branchStream(() => branch(right))] as [typeof stream, typeof stream]; + }, + toReadableStream() { + let iter: AsyncIterator | undefined; + return new ReadableStream({ + pull: async (ctrl) => { + iter ??= (this as AsyncIterable)[Symbol.asyncIterator](); + const { value, done } = await iter.next(); + if (done) ctrl.close(); + else ctrl.enqueue(new TextEncoder().encode(`${JSON.stringify(value)}\n`)); + }, + }); + }, + }; + + return stream; +} + +interface StreamLike extends AsyncIterable { + controller: AbortController; + tee(): [StreamLike, StreamLike]; + toReadableStream(): ReadableStream; +} + +function mockWs(halt: boolean): WebSocketClient { + return { + waitForPolicyReady: async () => true, + policyActive: () => true, + blockTimeout: (seconds: number) => seconds, + waitForToolCallVerdict: async (toolCallId: string) => ({ + eventId: `event-${toolCallId}`, + sessionId: "sess", + madCode: "M3_TEST", + policy: { mode: Mode.MODE_BLOCK, policyM0: false, policyM2: false, policyM3: halt, policyM4: false }, + hitl: null, + } satisfies Verdict), + } as unknown as WebSocketClient; +} + +describe("OpenAI instrumentation", () => { + afterEach(async () => { + vi.restoreAllMocks(); + await adrian.shutdown(); + }); + + it("captures chat completion calls as paired LLM events", async () => { + const events: EventData[] = []; + const client = adrian.openai({ + chat: { + completions: { + create: async (_body: Record) => ({ + choices: [{ + message: { + content: "hello", + tool_calls: [{ + id: "call-1", + function: { name: "search", arguments: "{\"query\":\"docs\"}" }, + }], + }, + }], + usage: { prompt_tokens: 3, completion_tokens: 4, total_tokens: 7 }, + }), + }, + }, + }); + + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (_type, data) => { + events.push(data); + } }); + const result = await client.chat.completions.create({ + model: "gpt-4o-mini", + messages: [{ role: "system", content: "be brief" }, { role: "user", content: "hi" }], + }); + + expect(result.choices[0]?.message.content).toBe("hello"); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + kind: "llm", + model: "gpt-4o-mini", + output: "hello", + usage: { promptTokens: 3, completionTokens: 4, totalTokens: 7 }, + }); + expect("toolCalls" in events[0] && events[0].toolCalls[0]).toMatchObject({ id: "call-1", name: "search", args: { query: "docs" } }); + }); + + it("captures responses API calls", async () => { + const events: EventData[] = []; + const client = adrian.openai({ + responses: { + create: async (_body: Record) => ({ + output_text: "done", + output: [{ type: "function_call", call_id: "call-2", name: "lookup", arguments: "{\"id\":42}" }], + usage: { input_tokens: 5, output_tokens: 6, total_tokens: 11 }, + }), + }, + }); + + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (_type, data) => { + events.push(data); + } }); + await client.responses.create({ model: "gpt-4.1", input: "run lookup" }); + + expect(events[0]).toMatchObject({ + kind: "llm", + model: "gpt-4.1", + output: "done", + toolCalls: [{ id: "call-2", name: "lookup", args: { id: 42 } }], + }); + }); + + it("captures responses API streaming tool calls and text", async () => { + const events: EventData[] = []; + async function* stream() { + yield { type: "response.output_text.delta", delta: "The answer " }; + yield { type: "response.output_item.added", item: { id: "item-1", type: "function_call", call_id: "call-4", name: "lookup" } }; + yield { type: "response.function_call_arguments.delta", item_id: "item-1", delta: "{\"id\"" }; + yield { type: "response.function_call_arguments.delta", item_id: "item-1", delta: ":7}" }; + yield { type: "response.output_text.delta", delta: "is ready." }; + } + const client = adrian.openai({ + responses: { + create: async (_body: Record) => stream(), + }, + }); + + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (_type, data) => { + events.push(data); + } }); + const result = await client.responses.create({ model: "gpt-4.1", input: "lookup id 7", stream: true }); + for await (const _chunk of result) { + // consume the stream so Adrian can emit the paired event + } + + expect(events[0]).toMatchObject({ + kind: "llm", + model: "gpt-4.1", + output: "The answer is ready.", + toolCalls: [{ id: "call-4", name: "lookup", args: { id: 7 } }], + }); + }); + + it("captures Responses API instructions and array input for stream and non-stream", async () => { + const responseBody = { + instructions: "You are an autonomous assistant.", + input: [ + { role: "user", content: "Do the work." }, + { type: "function_call", call_id: "call-1", name: "get_weather", arguments: '{"city":"SF"}' }, + { type: "function_call_output", call_id: "call-1", output: '{"temp":58}' }, + ], + }; + + for (const stream of [false, true]) { + const events: EventData[] = []; + async function* responseStream() { + yield { type: "response.output_text.delta", delta: "Done." }; + } + const client = adrian.openai({ + responses: { + create: async (_body: Record) => ( + stream ? responseStream() : { output_text: "Done.", usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } } + ), + }, + }); + + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (_type, data) => { + events.push(data); + } }); + + const result = await client.responses.create({ + model: "gpt-4o-mini", + ...responseBody, + stream, + }); + + if (stream) { + for await (const _chunk of result as AsyncIterable) { + // consume stream + } + } + + expect(events[0]).toMatchObject({ + kind: "llm", + model: "gpt-4o-mini", + messages: [ + { role: "system", content: "You are an autonomous assistant." }, + { role: "user", content: "Do the work." }, + { role: "assistant", content: '[tool_call:get_weather] {"city":"SF"}' }, + { role: "tool", content: '{"temp":58}' }, + ], + }); + + await adrian.shutdown(); + } + }); + + it("preserves OpenAI stream helper methods when Adrian is enabled", async () => { + const events: EventData[] = []; + const source = mockOpenAIStream([ + { choices: [{ delta: { content: "hello" } }] }, + ]); + const client = adrian.openai({ + chat: { + completions: { + create: async (_body: Record) => source, + }, + }, + }); + + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (_type, data) => { + events.push(data); + } }); + + const result = await client.chat.completions.create({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "stream" }], + stream: true, + }) as StreamLike; + + expect(result.controller).toBe(source.controller); + expect(typeof result.tee).toBe("function"); + expect(typeof result.toReadableStream).toBe("function"); + + const reader = result.toReadableStream().getReader(); + const { value } = await reader.read(); + expect(new TextDecoder().decode(value)).toBe('{"choices":[{"delta":{"content":"hello"}}]}\n'); + while (true) { + const next = await reader.read(); + if (next.done) break; + } + + expect(events[0]).toMatchObject({ + kind: "llm", + model: "gpt-4o-mini", + output: "hello", + }); + }); + + it("preserves tee() while capturing a single paired LLM event", async () => { + const events: EventData[] = []; + const source = mockOpenAIStream([ + { choices: [{ delta: { content: "hello" } }] }, + { choices: [{ delta: { content: " world" } }] }, + ]); + const client = adrian.openai({ + chat: { + completions: { + create: async (_body: Record) => source, + }, + }, + }); + + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (_type, data) => { + events.push(data); + } }); + + const result = await client.chat.completions.create({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "stream" }], + stream: true, + }) as StreamLike; + + const [left, right] = result.tee(); + expect(left.controller).toBe(source.controller); + expect(typeof left.toReadableStream).toBe("function"); + + for await (const _chunk of left) { + // consume one tee branch + } + for await (const _chunk of right) { + // consume the other branch + } + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + kind: "llm", + model: "gpt-4o-mini", + output: "hello world", + }); + }); + + it("emits partial stream data when the consumer stops early", async () => { + const events: EventData[] = []; + async function* stream() { + yield { choices: [{ delta: { content: "first " } }] }; + yield { choices: [{ delta: { content: "second" } }] }; + } + const client = adrian.openai({ + chat: { + completions: { + create: async (_body: Record) => stream(), + }, + }, + }); + + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (_type, data) => { + events.push(data); + } }); + const result = await client.chat.completions.create({ model: "gpt-4o-mini", messages: [{ role: "user", content: "stream" }], stream: true }); + for await (const _chunk of result) { + break; + } + + expect(events[0]).toMatchObject({ + kind: "llm", + model: "gpt-4o-mini", + output: "first ", + }); + }); + + it("blocks captureTool when policy halts", async () => { + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, blockTimeout: 5 }); + vi.spyOn(adrianCore, "getWebSocketClient").mockReturnValue(mockWs(true)); + + let executed = false; + const result = await adrian.captureTool({ + id: "call-weather", + function: { name: "get_weather", arguments: "{}" }, + }, async () => { + executed = true; + return { ok: true }; + }); + + expect(result).toBe(BLOCKED_TOOL_MESSAGE); + expect(executed).toBe(false); + }); + + it("captures local OpenAI tool execution as a tool event", async () => { + const events: Array<{ type: string; data: EventData }> = []; + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (type, data) => { + events.push({ type, data }); + } }); + + const result = await adrian.captureTool({ + id: "call-weather", + type: "function", + function: { name: "get_weather", arguments: "{\"city\":\"San Francisco\"}" }, + }, async () => ({ temperatureF: 58, condition: "cloudy" })); + + expect(result).toEqual({ temperatureF: 58, condition: "cloudy" }); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + type: "tool", + data: { + kind: "tool", + toolName: "get_weather", + toolCallId: "call-weather", + input: "{\"city\":\"San Francisco\"}", + output: "{\"temperatureF\":58,\"condition\":\"cloudy\"}", + }, + }); + }); + + it("captures local OpenAI tool execution errors as tool events", async () => { + const events: Array<{ type: string; data: EventData }> = []; + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (type, data) => { + events.push({ type, data }); + } }); + + await expect(adrian.captureTool({ + id: "call-weather", + type: "function", + function: { name: "get_weather", arguments: "{\"city\":\"San Francisco\"}" }, + }, async () => { + throw new Error("weather API unavailable"); + })).rejects.toThrow("weather API unavailable"); + + expect(events[0]).toMatchObject({ + type: "tool", + data: { + kind: "tool", + toolName: "get_weather", + toolCallId: "call-weather", + output: "[ERROR] Error: weather API unavailable", + error: { name: "Error", message: "weather API unavailable" }, + }, + }); + }); + + it("emits no_invocation when OpenAI capture runs without an outer invocation", async () => { + const pairedEvents: PairedEvent[] = []; + const client = adrian.openai({ + chat: { + completions: { + create: async (_body: Record) => ({ + choices: [{ + message: { + content: "", + tool_calls: [{ + id: "call-weather", + function: { name: "get_weather", arguments: "{\"city\":\"San Francisco\"}" }, + }], + }, + }], + }), + }, + }, + }); + + await adrian.init({ + handlers: [{ + onPairedEvent(event) { + pairedEvents.push(event); + }, + close() {}, + }], + sessionId: "sess", + wsUrl: null, + }); + + const response = await client.chat.completions.create({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "What's the weather?" }], + }); + + await adrian.captureTool(response.choices[0].message.tool_calls[0], async () => ({ temperatureF: 58 })); + + expect(pairedEvents).toHaveLength(2); + expect(pairedEvents.map((event) => event.invocationId)).toEqual(["no_invocation", "no_invocation"]); + expect(pairedEvents.map((event) => event.pairType)).toEqual(["llm", "tool"]); + }); + + it("reuses the active invocation for LLM and tool captures", async () => { + const pairedEvents: PairedEvent[] = []; + const client = adrian.openai({ + chat: { + completions: { + create: async (_body: Record) => ({ + choices: [{ + message: { + content: "", + tool_calls: [{ + id: "call-weather", + function: { name: "get_weather", arguments: "{\"city\":\"San Francisco\"}" }, + }], + }, + }], + }), + }, + }, + }); + + await adrian.init({ + handlers: [{ + onPairedEvent(event) { + pairedEvents.push(event); + }, + close() {}, + }], + sessionId: "sess", + wsUrl: null, + }); + + await adrianCore.runWithInvocationId("inv-shared", async () => { + const response = await client.chat.completions.create({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "What's the weather?" }], + }); + + await adrian.captureTool(response.choices[0].message.tool_calls[0], async () => ({ temperatureF: 58 })); + }); + + expect(pairedEvents).toHaveLength(2); + expect(pairedEvents.map((event) => event.invocationId)).toEqual(["inv-shared", "inv-shared"]); + expect(pairedEvents.map((event) => event.pairType)).toEqual(["llm", "tool"]); + }); + + it("captures OpenAI request errors as LLM events", async () => { + const events: Array<{ type: string; data: EventData }> = []; + const client = adrian.openai({ + chat: { + completions: { + create: async (_body: Record) => { + throw new Error("rate limited"); + }, + }, + }, + }); + + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (type, data) => { + events.push({ type, data }); + } }); + + await expect(client.chat.completions.create({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "hi" }], + })).rejects.toThrow("rate limited"); + + expect(events[0]).toMatchObject({ + type: "llm", + data: { + kind: "llm", + model: "gpt-4o-mini", + output: "[ERROR] Error: rate limited", + error: { name: "Error", message: "rate limited" }, + }, + }); + }); + + it("captures streaming OpenAI request errors as LLM events", async () => { + const events: Array<{ type: string; data: EventData }> = []; + const client = adrian.openai({ + chat: { + completions: { + create: async (_body: Record) => { + throw new Error("rate limited"); + }, + }, + }, + responses: { + create: async (_body: Record) => { + throw new Error("responses rate limited"); + }, + }, + }); + + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (type, data) => { + events.push({ type, data }); + } }); + + await expect(client.chat.completions.create({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "hi" }], + stream: true, + })).rejects.toThrow("rate limited"); + + await expect(client.responses.create({ + model: "gpt-4.1", + input: "hi", + stream: true, + })).rejects.toThrow("responses rate limited"); + + expect(events).toHaveLength(2); + expect(events[0]).toMatchObject({ + type: "llm", + data: { + kind: "llm", + model: "gpt-4o-mini", + output: "[ERROR] Error: rate limited", + error: { name: "Error", message: "rate limited" }, + }, + }); + expect(events[1]).toMatchObject({ + type: "llm", + data: { + kind: "llm", + model: "gpt-4.1", + output: "[ERROR] Error: responses rate limited", + error: { name: "Error", message: "responses rate limited" }, + }, + }); + }); + + it("wraps OpenAI client via adrian.openai()", async () => { + const events: EventData[] = []; + const client = adrian.openai({ + chat: { + completions: { + create: async (_body: Record) => ({ + choices: [{ message: { content: "hello" } }], + }), + }, + }, + }); + + await adrian.init({ handlers: [], sessionId: "sess", wsUrl: null, onEvent: (_type, data) => { + events.push(data); + } }); + + await client.chat.completions.create({ + model: "gpt-4o", + messages: [{ role: "user", content: "hi" }], + }); + + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ model: "gpt-4o", output: "hello" }); + }); +}); diff --git a/sdk/typescript/packages/openai/tsconfig.build.json b/sdk/typescript/packages/openai/tsconfig.build.json new file mode 100644 index 0000000..d7b2c67 --- /dev/null +++ b/sdk/typescript/packages/openai/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true, + "outDir": "dist" + }, + "include": ["src/**/*.ts"], + "exclude": ["tests/**/*.ts"] +} diff --git a/sdk/typescript/packages/openai/tsconfig.json b/sdk/typescript/packages/openai/tsconfig.json new file mode 100644 index 0000000..6140ae3 --- /dev/null +++ b/sdk/typescript/packages/openai/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "." + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +}