From b26303540e9f3a60b2dcc8c6914fa97c84913ace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 15:56:49 +0200 Subject: [PATCH 01/22] Switch Exa-specific tools to generic web tools Replace Exa-specific tooling with a generic AI-services web tools API. Renamed ExaEndpoint to ToolEndpoint and introduced aiServicesToolURL to build /tools/{tool} endpoints; fetch now reports fetch_method as "web_tool" and FetchContents/FetchResult were adapted to a new request/response schema (snake_case fields, markdown/text mapping, request_id variants, extras/metadata). Search tooling and schemas were updated to use new parameter names (search_context_size, allowed_domains, freshness, limit) and to tolerate both snake_case and camelCase provider fields; SearchRequestOptions and related types were adjusted accordingly. README, connector wiring, source collection, and tests were updated to match the new endpoints and payload formats, and fetch text limits/behavior were tuned to the new tool semantics. --- README.md | 10 +-- pkg/chattools/chattools_test.go | 99 ++++++++------------- pkg/chattools/fetch.go | 145 +++++++++++-------------------- pkg/chattools/search.go | 121 +++++++++++--------------- pkg/chattools/types.go | 49 ++++++----- pkg/connector/chat_tools.go | 18 ++-- pkg/connector/chat_tools_test.go | 10 +++ pkg/connector/sources.go | 6 +- 8 files changed, 192 insertions(+), 266 deletions(-) diff --git a/README.md b/README.md index e7ac8aee..ee3f4954 100644 --- a/README.md +++ b/README.md @@ -375,10 +375,10 @@ Errors are typed with codes (`pkg/agent/harness/public_errors.go`): `CompactionE | Tool | Purpose | Notes | |------|---------|-------| | `get_session` | Live chat metadata (current time/timezone, model, reasoning, disabled tools, attachments) | read-only; recomputes time per call | -| `fetch` | Fetch an HTTP/HTTPS URL → readable text + metadata | direct fetch (≤2 MiB, ≤20 000 chars) or Exa-backed contents (≤10 000 chars) with fallback | -| `web_search` | Web search via Exa | only enabled for the Beeper provider with a proxy token; rich Exa options; results become source citations | +| `fetch` | Fetch an HTTP/HTTPS URL → readable text + metadata | direct fetch (≤2 MiB, ≤20 000 chars) or AI-services `/tools/fetch` with fallback | +| `web_search` | Web search | only enabled for the Beeper provider with a proxy token; results become source citations | -Tools are gated per-room via the `com.beeper.ai.tools` state event's `disabled` array. Exa-backed tools route through the AI-services proxy (`/proxy/exa/v1/...`) using the appservice bearer token. Some models additionally expose **provider-native** built-ins (`image_generation`, `web_search`) injected into the request payload (`pkg/connector/builtin_tools.go`). +Tools are gated per-room via the `com.beeper.ai.tools` state event's `disabled` array. Web tools route through AI-services (`/tools/web_search`, `/tools/fetch`) using the appservice bearer token. Some models additionally expose **provider-native** built-ins (`image_generation`, `web_search`) injected into the request payload (`pkg/connector/builtin_tools.go`). **Adding a tool:** @@ -388,7 +388,7 @@ Tools are gated per-room via the `com.beeper.ai.tools` state event's `disabled` 4. Wire config in `pkg/connector/chat_tools.go` and honor `DisabledTools`. 5. If it produces citable sources, mirror `webSearchSourceParts` so URLs surface as message sources. -> **Security note:** `fetch` has **no SSRF guard** — it can reach localhost/private/link-local addresses (it just bypasses Exa for them). Treat it accordingly in your threat model. +> **Security note:** direct `fetch` has **no SSRF guard** — it can reach localhost/private/link-local addresses when the bridge bypasses AI-services for raw assets and local/private targets. Treat it accordingly in your threat model. ## Sessions: the branching conversation tree @@ -520,7 +520,7 @@ It serves `/v1/models`, `/v1/responses`, `/v1/chat/completions`, and `/api/strea - **Reasoning is double-validated and clamped** — setting a model can silently change the effective reasoning level. - **Two parallel session-tree implementations** (`aidb` vs `session` SQLite files) with near-duplicate SQL and one subtle difference (`ON DELETE CASCADE`). - **Token counts are estimates** (≈ chars/4) — compaction thresholds are approximate. -- **`fetch` has no SSRF protection.** +- **Direct `fetch` has no SSRF protection.** - **`ProviderConfig` holds secrets** (API keys, refresh tokens) in login metadata and serializes to JSON *and* YAML — don't log it. - **AG-UI `Event` is a map, not a struct** — read typed fields via `Get`/`String`; unknown fields survive round-trips. diff --git a/pkg/chattools/chattools_test.go b/pkg/chattools/chattools_test.go index 09b55615..ca8e0da4 100644 --- a/pkg/chattools/chattools_test.go +++ b/pkg/chattools/chattools_test.go @@ -131,7 +131,7 @@ func TestFetch(t *testing.T) { } } -func TestFetchUsesDirectFetchForAssetsWhenExaConfigured(t *testing.T) { +func TestFetchUsesDirectFetchForAssetsWhenToolEndpointConfigured(t *testing.T) { exaHit := false client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { if req.URL.Host == "exa.test" { @@ -141,7 +141,7 @@ func TestFetchUsesDirectFetchForAssetsWhenExaConfigured(t *testing.T) { return testResponse(req, http.StatusOK, "text/markdown", "# Title\n\nBody"), nil })} - result, err := Fetch(context.Background(), "https://example.com/doc.md", FetchOptions{Timeout: time.Second, ExaEndpoint: "https://exa.test/contents", Client: client, MaxBytes: 1024, MaxChars: 100}) + result, err := Fetch(context.Background(), "https://example.com/doc.md", FetchOptions{Timeout: time.Second, ToolEndpoint: "https://exa.test/contents", Client: client, MaxBytes: 1024, MaxChars: 100}) if err != nil { t.Fatal(err) } @@ -150,7 +150,7 @@ func TestFetchUsesDirectFetchForAssetsWhenExaConfigured(t *testing.T) { } } -func TestFetchUsesExaContentsForPages(t *testing.T) { +func TestFetchUsesToolEndpointForPages(t *testing.T) { exa := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost || r.Header.Get("Authorization") != "Bearer key" { t.Fatalf("unexpected request method/header") @@ -159,34 +159,29 @@ func TestFetchUsesExaContentsForPages(t *testing.T) { if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { t.Fatal(err) } - urls, ok := payload["urls"].([]any) - if !ok || len(urls) != 1 || urls[0] != "https://example.com/page" { - t.Fatalf("unexpected contents payload %#v", payload) + if payload["url"] != "https://example.com/page" || payload["max_chars"] != float64(100) { + t.Fatalf("unexpected fetch payload %#v", payload) } - text, ok := payload["text"].(map[string]any) - if !ok || text["maxCharacters"] != float64(100) || text["verbosity"] != "standard" { - t.Fatalf("unexpected text payload %#v", payload) - } - _, _ = w.Write([]byte(`{"requestId":"req_1","costDollars":{"total":0.001},"statuses":[{"id":"https://example.com/page","status":"success","source":"crawled"}],"results":[{"id":"doc_1","title":"Page","description":"Page description","url":"https://example.com/page","text":"Extracted page text","publishedDate":"2026-01-01","author":"A","favicon":"https://example.com/favicon.ico","highlights":["hit"],"summary":"sum","extras":{"links":["https://example.com/next"]}}]}`)) + _, _ = w.Write([]byte(`{"request_id":"req_1","title":"Page","description":"Page description","url":"https://example.com/page","final_url":"https://example.com/page","markdown":"Extracted page text","published_at":"2026-01-01","author":"A","favicon_url":"https://example.com/favicon.ico","metadata":{"links":["https://example.com/next"]}}`)) })) defer exa.Close() - result, err := Fetch(context.Background(), "https://example.com/page", FetchOptions{Timeout: time.Second, ExaEndpoint: exa.URL, APIKey: "key", MaxChars: 100}) + result, err := Fetch(context.Background(), "https://example.com/page", FetchOptions{Timeout: time.Second, ToolEndpoint: exa.URL, APIKey: "key", MaxChars: 100}) if err != nil { t.Fatal(err) } - if result.FetchMethod != "exa" || result.RequestID != "req_1" || result.Source != "crawled" || result.ID != "doc_1" || result.Title != "Page" || result.Text != "Extracted page text" { - t.Fatalf("unexpected Exa fetch result %#v", result) + if result.FetchMethod != "web_tool" || result.RequestID != "req_1" || result.Title != "Page" || result.Text != "Extracted page text" || result.Markdown != "Extracted page text" { + t.Fatalf("unexpected fetch result %#v", result) } - if result.Description != "Page description" || result.Favicon != "https://example.com/favicon.ico" || result.Published != "2026-01-01" || result.Author != "A" || len(result.Highlights) != 1 || result.Summary != "sum" || result.Extras["links"] == nil { - t.Fatalf("missing Exa fetch metadata %#v", result) + if result.Description != "Page description" || result.Favicon != "https://example.com/favicon.ico" || result.FaviconURL != "https://example.com/favicon.ico" || result.Published != "2026-01-01" || result.Author != "A" || result.Extras["links"] == nil { + t.Fatalf("missing fetch metadata %#v", result) } raw, err := json.Marshal(result) if err != nil { t.Fatal(err) } if strings.Contains(string(raw), "costDollars") || strings.Contains(string(raw), "fetch_method") { - t.Fatalf("Exa internal metadata leaked into fetch JSON: %s", string(raw)) + t.Fatalf("internal metadata leaked into fetch JSON: %s", string(raw)) } } @@ -209,7 +204,7 @@ func TestFetchDirectExtractsHTMLSourceMetadata(t *testing.T) { } } -func TestFetchFallsBackToDirectWhenExaFails(t *testing.T) { +func TestFetchFallsBackToDirectWhenToolEndpointFails(t *testing.T) { exaHit := false client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { if req.URL.Host == "exa.test" { @@ -219,7 +214,7 @@ func TestFetchFallsBackToDirectWhenExaFails(t *testing.T) { return testResponse(req, http.StatusOK, "text/html", "FallbackDirect page"), nil })} - result, err := Fetch(context.Background(), "https://example.com/page", FetchOptions{Timeout: time.Second, ExaEndpoint: "https://exa.test/contents", Client: client, MaxBytes: 1024, MaxChars: 100}) + result, err := Fetch(context.Background(), "https://example.com/page", FetchOptions{Timeout: time.Second, ToolEndpoint: "https://exa.test/contents", Client: client, MaxBytes: 1024, MaxChars: 100}) if err != nil { t.Fatal(err) } @@ -243,13 +238,10 @@ func TestSearchUsesConfiguredEndpoint(t *testing.T) { if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { t.Fatal(err) } - if payload["query"] != "query" || payload["numResults"] != float64(5) { - t.Fatalf("unexpected search payload %#v", payload) - } - if payload["useAutoprompt"] != false { + if payload["query"] != "query" || payload["limit"] != float64(5) { t.Fatalf("unexpected search payload %#v", payload) } - _, _ = w.Write([]byte(`{"requestId":"req_1","resolvedSearchType":"auto","costDollars":{"total":0.001},"output":{"content":"synth"},"results":[{"id":"doc_1","title":"One","url":"https://example.com","text":"ok","highlights":["hit"],"highlightScores":[0.5],"summary":"sum","publishedDate":"2026-01-01","siteName":"Example","author":"A","image":"https://example.com/image.png","favicon":"https://example.com/favicon.ico","subpages":[{"id":"sub_1","title":"Sub","url":"https://example.com/sub","text":"sub text","highlights":["sub hit"],"highlightScores":[0.4],"summary":"sub sum","publishedDate":"2026-01-02","author":"Sub A","image":"https://example.com/sub.png","favicon":"https://example.com/sub.ico","extras":{"links":["https://example.com/sub-link"]}}],"entities":[{"type":"company"}],"extras":{"links":["https://example.com/link"]}}]}`)) + _, _ = w.Write([]byte(`{"request_id":"req_1","search_context_size":"medium","results":[{"id":"doc_1","title":"One","url":"https://example.com","text":"ok","highlights":["hit"],"summary":"sum","published_at":"2026-01-01","site_name":"Example","author":"A","image_url":"https://example.com/image.png","favicon_url":"https://example.com/favicon.ico","metadata":{"links":["https://example.com/link"]}}]}`)) })) defer server.Close() @@ -257,56 +249,45 @@ func TestSearchUsesConfiguredEndpoint(t *testing.T) { if err != nil { t.Fatal(err) } - if result.Query != "query" || result.RequestID != "req_1" || result.ResolvedSearchType != "auto" || result.Output["content"] != "synth" { - t.Fatalf("missing top-level Exa metadata: %#v", result) + if result.Query != "query" || result.RequestID != "req_1" || result.SearchContextSize != "medium" { + t.Fatalf("missing top-level search metadata: %#v", result) } raw, err := json.Marshal(result) if err != nil { t.Fatal(err) } if strings.Contains(string(raw), "costDollars") { - t.Fatalf("Exa cost metadata leaked into search JSON: %s", string(raw)) + t.Fatalf("provider cost metadata leaked into search JSON: %s", string(raw)) } if len(result.Results) != 1 || result.Results[0].ID != "doc_1" || result.Results[0].Title != "One" || result.Results[0].Snippet != "hit" || result.Results[0].Text != "ok" { t.Fatalf("unexpected search result %#v", result) } - if result.Results[0].Published != "2026-01-01" || result.Results[0].SiteName != "Example" || result.Results[0].Author != "A" { - t.Fatalf("missing Exa metadata: %#v", result.Results[0]) + if result.Results[0].Published != "2026-01-01" || result.Results[0].PublishedAt != "2026-01-01" || result.Results[0].SiteName != "Example" || result.Results[0].SiteNameSnake != "Example" || result.Results[0].Author != "A" { + t.Fatalf("missing search metadata: %#v", result.Results[0]) } - if len(result.Results[0].Highlights) != 1 || result.Results[0].HighlightScores[0] != 0.5 || result.Results[0].Summary != "sum" { - t.Fatalf("missing Exa content fields: %#v", result.Results[0]) + if len(result.Results[0].Highlights) != 1 || result.Results[0].Summary != "sum" || result.Results[0].ImageURL == "" || result.Results[0].FaviconURL == "" { + t.Fatalf("missing search content fields: %#v", result.Results[0]) } - if len(result.Results[0].Subpages) != 1 || len(result.Results[0].Entities) != 1 || result.Results[0].Extras["links"] == nil { - t.Fatalf("missing Exa nested fields: %#v", result.Results[0]) - } - subpage := result.Results[0].Subpages[0] - if subpage.Text != "sub text" || subpage.Summary != "sub sum" || len(subpage.Highlights) != 1 || subpage.HighlightScores[0] != 0.4 || subpage.Image == "" || subpage.Favicon == "" || subpage.Extras["links"] == nil { - t.Fatalf("missing Exa subpage content fields: %#v", subpage) + if result.Results[0].Metadata["links"] == nil { + t.Fatalf("missing search metadata fields: %#v", result.Results[0]) } } -func TestSearchMapsToolOptionsToExaPayload(t *testing.T) { +func TestSearchMapsToolOptionsToPayload(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var payload map[string]any if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { t.Fatal(err) } - if payload["type"] != "deep" || payload["category"] != "news" || payload["userLocation"] != "US" || payload["systemPrompt"] != "prefer official sources" { + if payload["category"] != "news" || payload["search_context_size"] != "high" { t.Fatalf("missing scalar options %#v", payload) } - if domains, ok := payload["includeDomains"].([]any); !ok || len(domains) != 1 || domains[0] != "example.com" { - t.Fatalf("missing includeDomains %#v", payload) - } - if moderation, ok := payload["moderation"].(bool); !ok || !moderation { - t.Fatalf("missing moderation %#v", payload) - } - contents, ok := payload["contents"].(map[string]any) - if !ok || contents["highlights"] != true { - t.Fatalf("missing contents %#v", payload) + if domains, ok := payload["allowed_domains"].([]any); !ok || len(domains) != 1 || domains[0] != "example.com" { + t.Fatalf("missing allowed_domains %#v", payload) } - outputSchema, ok := payload["outputSchema"].(map[string]any) - if !ok || outputSchema["type"] != "text" { - t.Fatalf("missing outputSchema %#v", payload) + freshness, ok := payload["freshness"].(map[string]any) + if !ok || freshness["days"] != float64(7) || freshness["published_before"] != "2026-06-01T00:00:00Z" { + t.Fatalf("missing freshness %#v", payload) } _, _ = w.Write([]byte(`{"results":[]}`)) })) @@ -314,16 +295,12 @@ func TestSearchMapsToolOptionsToExaPayload(t *testing.T) { tool := WebSearchTool(SearchOptions{Enabled: true, Endpoint: server.URL, Timeout: time.Second}) _, err := tool.Execute(context.Background(), "call", map[string]any{ - "query": "query", - "limit": 3, - "includeDomains": []any{"example.com"}, - "type": "deep", - "category": "news", - "userLocation": "US", - "moderation": true, - "contents": map[string]any{"highlights": true}, - "outputSchema": map[string]any{"type": "text"}, - "systemPrompt": "prefer official sources", + "query": "query", + "limit": 3, + "allowed_domains": []any{"example.com"}, + "search_context_size": "high", + "category": "news", + "freshness": map[string]any{"days": float64(7), "published_before": "2026-06-01T00:00:00Z"}, }, nil) if err != nil { t.Fatal(err) diff --git a/pkg/chattools/fetch.go b/pkg/chattools/fetch.go index 78da6b38..2ca13f46 100644 --- a/pkg/chattools/fetch.go +++ b/pkg/chattools/fetch.go @@ -64,7 +64,7 @@ func Fetch(ctx context.Context, rawURL string, options FetchOptions) (FetchResul if options.MaxChars == 0 { options.MaxChars = 20000 } - if options.ExaEndpoint != "" && !shouldDirectFetch(parsed) { + if options.ToolEndpoint != "" && !shouldDirectFetch(parsed) { result, err := FetchContents(ctx, parsed.String(), options) if err == nil { return result, nil @@ -73,10 +73,10 @@ func Fetch(ctx context.Context, rawURL string, options FetchOptions) (FetchResul Err(err). Str("action", "ai_tool_http"). Str("tool", "fetch"). - Str("fetch_method", "exa"). + Str("fetch_method", "web_tool"). Str("target_url", parsed.Redacted()). Str("target_host", parsed.Host). - Msg("Falling back to direct fetch after Exa fetch failed") + Msg("Falling back to direct fetch after web tool fetch failed") } return fetchDirect(ctx, rawURL, parsed, options) } @@ -138,12 +138,12 @@ func fetchDirect(ctx context.Context, rawURL string, parsed *url.URL, options Fe } func FetchContents(ctx context.Context, rawURL string, options FetchOptions) (FetchResult, error) { - if options.ExaEndpoint == "" { + if options.ToolEndpoint == "" { return FetchResult{}, errors.New("fetch contents is not configured") } textMaxChars := options.MaxChars - if textMaxChars <= 0 || textMaxChars > 10000 { - textMaxChars = 10000 + if textMaxChars <= 0 || textMaxChars > 50000 { + textMaxChars = 20000 } client := options.Client if client == nil { @@ -153,21 +153,18 @@ func FetchContents(ctx context.Context, rawURL string, options FetchOptions) (Fe if err != nil || target == nil || target.Scheme == "" || target.Host == "" { return FetchResult{}, fmt.Errorf("invalid URL") } - log := toolHTTPLog(ctx, "fetch", http.MethodPost, options.ExaEndpoint). + log := toolHTTPLog(ctx, "fetch", http.MethodPost, options.ToolEndpoint). With(). - Str("fetch_method", "exa"). + Str("fetch_method", "web_tool"). Str("target_url", target.Redacted()). Str("target_host", target.Host). Logger() ctx = log.WithContext(ctx) payload, _ := json.Marshal(map[string]any{ - "urls": []string{rawURL}, - "text": map[string]any{ - "maxCharacters": textMaxChars, - "verbosity": "standard", - }, + "url": rawURL, + "max_chars": textMaxChars, }) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, options.ExaEndpoint, bytes.NewReader(payload)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, options.ToolEndpoint, bytes.NewReader(payload)) if err != nil { return FetchResult{}, err } @@ -193,57 +190,34 @@ func FetchContents(ctx context.Context, rawURL string, options FetchOptions) (Fe return FetchResult{}, err } result := FetchResult{ - URL: rawURL, - FinalURL: rawURL, - Status: 200, - Truncated: false, - RequestID: body.RequestID, - Context: body.Context, - FetchMethod: "exa", - } - if len(body.Statuses) > 0 { - status := body.Statuses[0] - result.Source = status.Source - if status.Status == "error" { - result.Status = status.Error.HTTPStatusCode - if result.Status == 0 { - result.Status = 502 - } - result.Error = status.Error.Tag - log.Error(). - Int("status_code", result.Status). - Str("request_id", body.RequestID). - Str("error_tag", status.Error.Tag). - Msg("AI tool fetch provider returned item error") - return result, fmt.Errorf("fetch contents failed: %s", firstNonEmpty(status.Error.Tag, status.Status)) - } + URL: firstNonEmpty(body.URL, rawURL), + FinalURL: firstNonEmpty(body.FinalURL, body.URL, rawURL), + Status: 200, + Title: body.Title, + Description: body.Description, + SiteName: body.SiteName, + Text: firstNonEmpty(body.Markdown, body.Text), + Markdown: firstNonEmpty(body.Markdown, body.Text), + Truncated: body.Truncated, + RequestID: firstNonEmpty(body.RequestID, body.RequestIDSnake), + RequestIDSnake: firstNonEmpty(body.RequestIDSnake, body.RequestID), + Published: firstNonEmpty(body.Published, body.PublishedAt, body.PublishedDate), + Author: body.Author, + Image: firstNonEmpty(body.Image, body.ImageURL), + ImageURL: firstNonEmpty(body.ImageURL, body.Image), + Favicon: firstNonEmpty(body.Favicon, body.FaviconURL), + FaviconURL: firstNonEmpty(body.FaviconURL, body.Favicon), + Extras: body.Metadata, + FetchMethod: "web_tool", } - if len(body.Results) == 0 { - return result, nil - } - item := body.Results[0] - result.FinalURL = firstNonEmpty(item.URL, rawURL) - result.ID = item.ID - result.Title = item.Title - result.Description = item.Description - result.Text = item.Text - result.Published = firstNonEmpty(item.Published, item.PublishedDate) - result.Author = item.Author - result.Image = item.Image - result.Favicon = item.Favicon - result.Highlights = item.Highlights - result.HighlightScores = item.HighlightScores - result.Summary = item.Summary - result.Subpages = item.Subpages - result.Entities = item.Entities - result.Extras = item.Extras if len([]rune(result.Text)) > textMaxChars { runes := []rune(result.Text) result.Text = string(runes[:textMaxChars]) + result.Markdown = result.Text result.Truncated = true } log.Debug(). - Str("request_id", body.RequestID). + Str("request_id", result.RequestID). Bool("truncated", result.Truncated). Msg("Parsed AI tool fetch result") return result, nil @@ -285,42 +259,25 @@ func shouldDirectFetch(parsed *url.URL) bool { } type contentsResponse struct { - RequestID string `json:"requestId"` - Context string `json:"context"` - CostDollars map[string]any `json:"costDollars"` - Results []contentsResultItem `json:"results"` - Statuses []contentsStatusEntry `json:"statuses"` -} - -type contentsResultItem struct { - ID string `json:"id"` - Title string `json:"title"` - Description string `json:"description"` - URL string `json:"url"` - Text string `json:"text"` - Highlights []string `json:"highlights"` - HighlightScores []float64 `json:"highlightScores"` - Summary any `json:"summary"` - Published string `json:"published"` - PublishedDate string `json:"publishedDate"` - Author string `json:"author"` - Image string `json:"image"` - Favicon string `json:"favicon"` - Subpages []SearchSubpage `json:"subpages"` - Entities []any `json:"entities"` - Extras map[string]any `json:"extras"` -} - -type contentsStatusEntry struct { - ID string `json:"id"` - Status string `json:"status"` - Source string `json:"source"` - Error contentsStatusError `json:"error"` -} - -type contentsStatusError struct { - Tag string `json:"tag"` - HTTPStatusCode int `json:"httpStatusCode"` + URL string `json:"url"` + FinalURL string `json:"final_url"` + Title string `json:"title"` + Description string `json:"description"` + SiteName string `json:"site_name"` + Published string `json:"published"` + PublishedAt string `json:"published_at"` + PublishedDate string `json:"publishedDate"` + Author string `json:"author"` + Image string `json:"image"` + ImageURL string `json:"image_url"` + Favicon string `json:"favicon"` + FaviconURL string `json:"favicon_url"` + Text string `json:"text"` + Markdown string `json:"markdown"` + Truncated bool `json:"truncated"` + Metadata map[string]any `json:"metadata"` + RequestID string `json:"requestId"` + RequestIDSnake string `json:"request_id"` } func toolHTTPLog(ctx context.Context, tool string, method string, rawURL string) zerolog.Logger { diff --git a/pkg/chattools/search.go b/pkg/chattools/search.go index 69692af4..68c7f58a 100644 --- a/pkg/chattools/search.go +++ b/pkg/chattools/search.go @@ -18,26 +18,21 @@ func WebSearchTool(options SearchOptions) agent.AgentTool[any] { return agent.AgentTool[any]{ Tool: ai.Tool{ Name: "web_search", - Description: "Search the web with Exa and return relevant results with title, URL, content, highlights, summaries, and source metadata.", + Description: "Search the web and return relevant results with title, URL, snippets, readable content, and source metadata.", Parameters: objectSchema(map[string]any{ - "query": map[string]any{"type": "string", "description": "Search query."}, - "limit": map[string]any{"type": "integer", "description": "Maximum number of results. Sent to Exa as numResults."}, - "includeDomains": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "Domains or domain paths to include."}, - "excludeDomains": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "Domains or domain paths to exclude."}, - "startCrawlDate": map[string]any{"type": "string", "description": "Only include results crawled after this ISO 8601 timestamp."}, - "endCrawlDate": map[string]any{"type": "string", "description": "Only include results crawled before this ISO 8601 timestamp."}, - "startPublishedDate": map[string]any{"type": "string", "description": "Only include results published after this ISO 8601 timestamp."}, - "endPublishedDate": map[string]any{"type": "string", "description": "Only include results published before this ISO 8601 timestamp."}, - "context": map[string]any{"description": "Deprecated Exa context option. Prefer contents.text, contents.highlights, or contents.summary."}, - "moderation": map[string]any{"type": "boolean", "description": "Enable Exa content moderation."}, - "contents": map[string]any{"type": "object", "description": "Exa contents options for text, highlights, summary, extras, freshness, and subpages."}, - "additionalQueries": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "Additional deep-search query variations."}, - "type": map[string]any{"type": "string", "enum": []string{"instant", "fast", "auto", "deep-lite", "deep", "deep-reasoning"}, "description": "Exa search type."}, - "category": map[string]any{"type": "string", "enum": []string{"company", "research paper", "news", "personal site", "financial report", "people"}, "description": "Exa search category."}, - "userLocation": map[string]any{"type": "string", "description": "Two-letter country code used to bias results."}, - "compliance": map[string]any{"type": "string", "enum": []string{"hipaa"}, "description": "Enterprise-only compliance mode."}, - "outputSchema": map[string]any{"type": "object", "description": "Exa synthesis output schema. Do not combine with Exa streaming."}, - "systemPrompt": map[string]any{"type": "string", "description": "Additional Exa synthesis/search instructions."}, + "query": map[string]any{"type": "string", "description": "Search query."}, + "limit": map[string]any{"type": "integer", "description": "Maximum number of results, up to 10."}, + "search_context_size": map[string]any{"type": "string", "enum": []string{"low", "medium", "high"}, "description": "Amount of page context to include in each result."}, + "category": map[string]any{"type": "string", "enum": []string{"web", "news", "research", "company", "financial_report", "people"}, "description": "Optional result category."}, + "allowed_domains": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "description": "Domains to include."}, + "freshness": map[string]any{ + "type": "object", + "properties": map[string]any{ + "days": map[string]any{"type": "integer", "description": "Only include pages published in the last N days."}, + "published_after": map[string]any{"type": "string", "description": "Only include pages published after this ISO 8601 timestamp."}, + "published_before": map[string]any{"type": "string", "description": "Only include pages published before this ISO 8601 timestamp."}, + }, + }, }, []string{"query"}), }, Execute: func(ctx context.Context, toolCallID string, params any, onUpdate agent.AgentToolUpdateCallback[any]) (agent.AgentToolResult[any], error) { @@ -110,7 +105,7 @@ func Search(ctx context.Context, query string, limit int, request SearchRequestO } log.Debug(). Str("request_id", result.RequestID). - Str("resolved_search_type", result.ResolvedSearchType). + Str("search_context_size", result.SearchContextSize). Int("result_count", len(result.Results)). Msg("Parsed AI tool search result") return result, nil @@ -119,8 +114,10 @@ func Search(ctx context.Context, query string, limit int, request SearchRequestO type searchResponse struct { Query string `json:"query"` RequestID string `json:"requestId"` + RequestIDSnake string `json:"request_id"` ResolvedSearchType string `json:"resolvedSearchType"` SearchType string `json:"searchType"` + SearchContextSize string `json:"search_context_size"` Context string `json:"context"` CostDollars map[string]any `json:"costDollars"` Output map[string]any `json:"output"` @@ -138,11 +135,15 @@ type searchResponseItem struct { Summary string `json:"summary"` Description string `json:"description"` Published string `json:"published"` + PublishedAt string `json:"published_at"` PublishedDate string `json:"publishedDate"` SiteName string `json:"siteName"` + SiteNameSnake string `json:"site_name"` Author string `json:"author"` Image string `json:"image"` + ImageURL string `json:"image_url"` Favicon string `json:"favicon"` + FaviconURL string `json:"favicon_url"` Source string `json:"source"` Subpages []SearchSubpage `json:"subpages"` Entities []any `json:"entities"` @@ -153,19 +154,18 @@ type searchResponseItem struct { func (body searchResponse) result() SearchResult { result := SearchResult{ Query: body.Query, - RequestID: body.RequestID, + RequestID: firstNonEmpty(body.RequestID, body.RequestIDSnake), + RequestIDSnake: firstNonEmpty(body.RequestIDSnake, body.RequestID), ResolvedSearchType: body.ResolvedSearchType, SearchType: body.SearchType, + SearchContextSize: body.SearchContextSize, Context: body.Context, Output: body.Output, Results: make([]SearchItem, 0, len(body.Results)), } for _, item := range body.Results { snippet := firstNonEmpty(item.Snippet, firstString(item.Highlights), item.Summary, item.Text) - published := item.Published - if published == "" { - published = item.PublishedDate - } + published := firstNonEmpty(item.Published, item.PublishedAt, item.PublishedDate) result.Results = append(result.Results, SearchItem{ ID: item.ID, Title: item.Title, @@ -177,11 +177,15 @@ func (body searchResponse) result() SearchResult { Summary: item.Summary, Description: item.Description, Published: published, + PublishedAt: firstNonEmpty(item.PublishedAt, published), PublishedDate: item.PublishedDate, - SiteName: item.SiteName, + SiteName: firstNonEmpty(item.SiteName, item.SiteNameSnake), + SiteNameSnake: firstNonEmpty(item.SiteNameSnake, item.SiteName), Author: item.Author, - Image: item.Image, - Favicon: item.Favicon, + Image: firstNonEmpty(item.Image, item.ImageURL), + ImageURL: firstNonEmpty(item.ImageURL, item.Image), + Favicon: firstNonEmpty(item.Favicon, item.FaviconURL), + FaviconURL: firstNonEmpty(item.FaviconURL, item.Favicon), Source: item.Source, Subpages: item.Subpages, Entities: item.Entities, @@ -194,28 +198,21 @@ func (body searchResponse) result() SearchResult { func searchPayload(query string, limit int, request SearchRequestOptions) map[string]any { payload := map[string]any{ - "query": query, - "numResults": limit, - "useAutoprompt": false, - } - addStrings(payload, "includeDomains", request.IncludeDomains) - addStrings(payload, "excludeDomains", request.ExcludeDomains) - addString(payload, "startCrawlDate", request.StartCrawlDate) - addString(payload, "endCrawlDate", request.EndCrawlDate) - addString(payload, "startPublishedDate", request.StartPublishedDate) - addString(payload, "endPublishedDate", request.EndPublishedDate) - addAny(payload, "context", request.Context) - if request.Moderation != nil { - payload["moderation"] = *request.Moderation - } - addMap(payload, "contents", request.Contents) - addStrings(payload, "additionalQueries", request.AdditionalQueries) - addString(payload, "type", request.Type) + "query": query, + "limit": limit, + } + addString(payload, "search_context_size", request.SearchContextSize) addString(payload, "category", request.Category) - addString(payload, "userLocation", request.UserLocation) - addString(payload, "compliance", request.Compliance) - addMap(payload, "outputSchema", request.OutputSchema) - addString(payload, "systemPrompt", request.SystemPrompt) + addStrings(payload, "allowed_domains", request.AllowedDomains) + freshness := map[string]any{} + if request.Freshness.Days > 0 { + freshness["days"] = request.Freshness.Days + } + addString(freshness, "published_after", request.Freshness.PublishedAfter) + addString(freshness, "published_before", request.Freshness.PublishedBefore) + if len(freshness) > 0 { + payload["freshness"] = freshness + } return payload } @@ -225,25 +222,13 @@ func requestOptions(params any) SearchRequestOptions { return SearchRequestOptions{} } var out SearchRequestOptions - out.IncludeDomains = stringSliceParam(values, "includeDomains") - out.ExcludeDomains = stringSliceParam(values, "excludeDomains") - out.StartCrawlDate = stringValueParam(values, "startCrawlDate") - out.EndCrawlDate = stringValueParam(values, "endCrawlDate") - out.StartPublishedDate = stringValueParam(values, "startPublishedDate") - out.EndPublishedDate = stringValueParam(values, "endPublishedDate") - if value, ok := values["context"]; ok { - out.Context = value - } - if value, ok := values["moderation"].(bool); ok { - out.Moderation = &value - } - out.Contents = mapParam(values, "contents") - out.AdditionalQueries = stringSliceParam(values, "additionalQueries") - out.Type = stringValueParam(values, "type") + out.SearchContextSize = stringValueParam(values, "search_context_size") out.Category = stringValueParam(values, "category") - out.UserLocation = stringValueParam(values, "userLocation") - out.Compliance = stringValueParam(values, "compliance") - out.OutputSchema = mapParam(values, "outputSchema") - out.SystemPrompt = stringValueParam(values, "systemPrompt") + out.AllowedDomains = stringSliceParam(values, "allowed_domains") + if freshness := mapParam(values, "freshness"); freshness != nil { + out.Freshness.Days = intParam(freshness, "days", 0) + out.Freshness.PublishedAfter = stringValueParam(freshness, "published_after") + out.Freshness.PublishedBefore = stringValueParam(freshness, "published_before") + } return out } diff --git a/pkg/chattools/types.go b/pkg/chattools/types.go index 6fd8078d..0456c9b8 100644 --- a/pkg/chattools/types.go +++ b/pkg/chattools/types.go @@ -34,12 +34,12 @@ type SessionOptions struct { } type FetchOptions struct { - Timeout time.Duration - MaxBytes int64 - MaxChars int - Client *http.Client - ExaEndpoint string - APIKey string + Timeout time.Duration + MaxBytes int64 + MaxChars int + Client *http.Client + ToolEndpoint string + APIKey string } type SearchOptions struct { @@ -51,22 +51,16 @@ type SearchOptions struct { } type SearchRequestOptions struct { - IncludeDomains []string `json:"includeDomains,omitempty"` - ExcludeDomains []string `json:"excludeDomains,omitempty"` - StartCrawlDate string `json:"startCrawlDate,omitempty"` - EndCrawlDate string `json:"endCrawlDate,omitempty"` - StartPublishedDate string `json:"startPublishedDate,omitempty"` - EndPublishedDate string `json:"endPublishedDate,omitempty"` - Context any `json:"context,omitempty"` - Moderation *bool `json:"moderation,omitempty"` - Contents map[string]any `json:"contents,omitempty"` - AdditionalQueries []string `json:"additionalQueries,omitempty"` - Type string `json:"type,omitempty"` - Category string `json:"category,omitempty"` - UserLocation string `json:"userLocation,omitempty"` - Compliance string `json:"compliance,omitempty"` - OutputSchema map[string]any `json:"outputSchema,omitempty"` - SystemPrompt string `json:"systemPrompt,omitempty"` + SearchContextSize string `json:"search_context_size,omitempty"` + Category string `json:"category,omitempty"` + AllowedDomains []string `json:"allowed_domains,omitempty"` + Freshness SearchFreshness `json:"freshness,omitempty"` +} + +type SearchFreshness struct { + Days int `json:"days,omitempty"` + PublishedAfter string `json:"published_after,omitempty"` + PublishedBefore string `json:"published_before,omitempty"` } type FetchResult struct { @@ -76,13 +70,17 @@ type FetchResult struct { ContentType string `json:"content_type,omitempty"` Title string `json:"title,omitempty"` Description string `json:"description,omitempty"` + SiteName string `json:"site_name,omitempty"` Text string `json:"text,omitempty"` + Markdown string `json:"markdown,omitempty"` Truncated bool `json:"truncated"` ID string `json:"id,omitempty"` Published string `json:"published,omitempty"` Author string `json:"author,omitempty"` Image string `json:"image,omitempty"` + ImageURL string `json:"image_url,omitempty"` Favicon string `json:"favicon,omitempty"` + FaviconURL string `json:"favicon_url,omitempty"` Highlights []string `json:"highlights,omitempty"` HighlightScores []float64 `json:"highlightScores,omitempty"` Summary any `json:"summary,omitempty"` @@ -91,6 +89,7 @@ type FetchResult struct { Extras map[string]any `json:"extras,omitempty"` Source string `json:"source,omitempty"` RequestID string `json:"requestId,omitempty"` + RequestIDSnake string `json:"request_id,omitempty"` Context string `json:"context,omitempty"` Error string `json:"error,omitempty"` FetchMethod string `json:"-"` @@ -99,8 +98,10 @@ type FetchResult struct { type SearchResult struct { Query string `json:"query"` RequestID string `json:"requestId,omitempty"` + RequestIDSnake string `json:"request_id,omitempty"` ResolvedSearchType string `json:"resolvedSearchType,omitempty"` SearchType string `json:"searchType,omitempty"` + SearchContextSize string `json:"search_context_size,omitempty"` Context string `json:"context,omitempty"` Output map[string]any `json:"output,omitempty"` Results []SearchItem `json:"results"` @@ -117,11 +118,15 @@ type SearchItem struct { Summary string `json:"summary,omitempty"` Description string `json:"description,omitempty"` Published string `json:"published,omitempty"` + PublishedAt string `json:"published_at,omitempty"` PublishedDate string `json:"publishedDate,omitempty"` SiteName string `json:"siteName,omitempty"` + SiteNameSnake string `json:"site_name,omitempty"` Author string `json:"author,omitempty"` Image string `json:"image,omitempty"` + ImageURL string `json:"image_url,omitempty"` Favicon string `json:"favicon,omitempty"` + FaviconURL string `json:"favicon_url,omitempty"` Source string `json:"source,omitempty"` Subpages []SearchSubpage `json:"subpages,omitempty"` Entities []any `json:"entities,omitempty"` diff --git a/pkg/connector/chat_tools.go b/pkg/connector/chat_tools.go index 211519c6..b9cc78a3 100644 --- a/pkg/connector/chat_tools.go +++ b/pkg/connector/chat_tools.go @@ -52,8 +52,8 @@ func (cl *Client) chatTools(msg *bridgev2.MatrixMessage, meta *aiid.PortalMetada } if provider.ID == aiid.DefaultProvider && provider.BaseURL != "" { if token, err := cl.defaultProviderBearerToken(); err == nil { - if endpoint, err := aiServicesExaContentsURL(provider.BaseURL); err == nil { - fetch.ExaEndpoint = endpoint + if endpoint, err := aiServicesToolURL(provider.BaseURL, "fetch"); err == nil { + fetch.ToolEndpoint = endpoint fetch.APIKey = token } } @@ -104,7 +104,7 @@ func (cl *Client) searchOptions(roomConfig RoomConfig, provider aiid.ProviderCon if err != nil { return chattools.SearchOptions{} } - endpoint, err := aiServicesExaSearchURL(provider.BaseURL) + endpoint, err := aiServicesToolURL(provider.BaseURL, "web_search") if err != nil { return chattools.SearchOptions{} } @@ -116,20 +116,12 @@ func (cl *Client) searchOptions(roomConfig RoomConfig, provider aiid.ProviderCon } } -func aiServicesExaSearchURL(proxyBaseURL string) (string, error) { - return aiServicesExaURL(proxyBaseURL, "search") -} - -func aiServicesExaContentsURL(proxyBaseURL string) (string, error) { - return aiServicesExaURL(proxyBaseURL, "contents") -} - -func aiServicesExaURL(proxyBaseURL string, route string) (string, error) { +func aiServicesToolURL(proxyBaseURL string, tool string) (string, error) { parsed, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(proxyBaseURL), "/")) if err != nil { return "", err } - parsed.Path = strings.TrimRight(trimAIProxyProviderPath(parsed.Path), "/") + "/proxy/exa/v1/" + route + parsed.Path = strings.TrimRight(trimAIProxyProviderPath(parsed.Path), "/") + "/tools/" + tool parsed.RawQuery = "" parsed.Fragment = "" return parsed.String(), nil diff --git a/pkg/connector/chat_tools_test.go b/pkg/connector/chat_tools_test.go index 338d8889..52bdfcae 100644 --- a/pkg/connector/chat_tools_test.go +++ b/pkg/connector/chat_tools_test.go @@ -33,3 +33,13 @@ func TestModelSupportsAgentToolsDefaultsToTrue(t *testing.T) { t.Fatal("models without catalog tool metadata should keep default tool behavior") } } + +func TestAIServicesToolURL(t *testing.T) { + got, err := aiServicesToolURL("https://ai-services.example/dev/proxy/openai/v1/responses", "web_search") + if err != nil { + t.Fatal(err) + } + if want := "https://ai-services.example/dev/tools/web_search"; got != want { + t.Fatalf("unexpected tool URL %q, want %q", got, want) + } +} diff --git a/pkg/connector/sources.go b/pkg/connector/sources.go index 799fcd26..d24ec2af 100644 --- a/pkg/connector/sources.go +++ b/pkg/connector/sources.go @@ -108,7 +108,7 @@ func (c *sourceCollector) addSearchResultSource(output toolOutputEvent, query st SiteName: sourceString(item, "siteName", "site_name", "source"), FaviconURL: sourceFaviconString(item), ImageURL: sourceImageString(item), - PublishedAt: sourceString(item, "published", "publishedAt", "publishedDate", "datePublished", "date"), + PublishedAt: sourceString(item, "published", "publishedAt", "published_at", "publishedDate", "datePublished", "date"), Priority: priority, Appearance: sourceAppearance{ Kind: "web_search", @@ -125,7 +125,7 @@ func (c *sourceCollector) addSearchResultSource(output toolOutputEvent, query st source.SiteName = firstSourceString(source.SiteName, sourceString(nested, "siteName", "site_name", "ogSiteName")) source.FaviconURL = firstSourceString(source.FaviconURL, sourceFaviconString(nested)) source.ImageURL = firstSourceString(source.ImageURL, sourceImageString(nested)) - source.PublishedAt = firstSourceString(source.PublishedAt, sourceString(nested, "published", "publishedAt", "publishedDate", "datePublished", "date")) + source.PublishedAt = firstSourceString(source.PublishedAt, sourceString(nested, "published", "publishedAt", "published_at", "publishedDate", "datePublished", "date")) } return c.add(source) } @@ -167,7 +167,7 @@ func (c *sourceCollector) addFetchOutput(output toolOutputEvent, result any) []m SiteName: sourceString(data, "siteName", "site_name", "source"), FaviconURL: sourceFaviconString(data), ImageURL: sourceString(data, "image", "imageUrl", "image_url"), - PublishedAt: sourceString(data, "published", "publishedAt", "publishedDate", "datePublished", "date"), + PublishedAt: sourceString(data, "published", "publishedAt", "published_at", "publishedDate", "datePublished", "date"), Priority: 100, Appearance: sourceAppearance{ Kind: "fetch", From 812d37c10b88829437fe6ca6e06fe47a74d8437a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 16:02:05 +0200 Subject: [PATCH 02/22] Update types.go --- pkg/chattools/types.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/chattools/types.go b/pkg/chattools/types.go index 0456c9b8..ad2e67e5 100644 --- a/pkg/chattools/types.go +++ b/pkg/chattools/types.go @@ -88,8 +88,8 @@ type FetchResult struct { Entities []any `json:"entities,omitempty"` Extras map[string]any `json:"extras,omitempty"` Source string `json:"source,omitempty"` - RequestID string `json:"requestId,omitempty"` - RequestIDSnake string `json:"request_id,omitempty"` + RequestID string `json:"-"` + RequestIDSnake string `json:"-"` Context string `json:"context,omitempty"` Error string `json:"error,omitempty"` FetchMethod string `json:"-"` @@ -97,8 +97,8 @@ type FetchResult struct { type SearchResult struct { Query string `json:"query"` - RequestID string `json:"requestId,omitempty"` - RequestIDSnake string `json:"request_id,omitempty"` + RequestID string `json:"-"` + RequestIDSnake string `json:"-"` ResolvedSearchType string `json:"resolvedSearchType,omitempty"` SearchType string `json:"searchType,omitempty"` SearchContextSize string `json:"search_context_size,omitempty"` From cba2349992bfa8af0904ff667812552aa3ac64bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 17:07:55 +0200 Subject: [PATCH 03/22] Extract provider citations; add open tool Add structured citation extraction and routing plus a new "open" web tool and readable-open logic. - Introduces ai.Citation and plumbing to collect provider-native citations across providers (OpenAI, Google, Anthropic) via pkg/ai/providers/citations.go and hooks in provider stream handlers. - Replaces the old fetch tool surface with an "open" tool that accepts ref_id (or full URL) and integrates a per-run reference store (refs.go) populated by web_search results so search items can be opened by ref_id. - Enhances Fetch/Open logic to prefer readable representations (Markdown/plain/JSON/XML/CSV), honor Link headers and HTML alternates, set an Accept header for direct opens, and fall back to the tool extraction endpoint when appropriate. Adds HTML/link header parsing and content-type helpers. - Updates tests to cover Markdown alternates, ref_id roundtrips, Accept header behavior, and provider citation parsing. README updated to reflect the new "open" semantics and SSRF note wording. This change surfaces citable sources in assistant messages, enables compact search->open workflows via ref_id, and improves direct-open heuristics for readable content. --- README.md | 16 +- pkg/ai/providers/anthropic.go | 5 + pkg/ai/providers/citations.go | 222 +++++++++++++++++++ pkg/ai/providers/google.go | 5 + pkg/ai/providers/google_vertex.go | 5 + pkg/ai/providers/openai_completions.go | 2 + pkg/ai/providers/openai_conversion_test.go | 60 +++++ pkg/ai/providers/openai_responses.go | 1 + pkg/ai/providers/openai_shared.go | 15 ++ pkg/ai/types.go | 18 ++ pkg/aiid/ids_test.go | 4 + pkg/aiid/metadata.go | 5 +- pkg/chattools/chattools.go | 4 +- pkg/chattools/chattools_test.go | 155 ++++++++++++- pkg/chattools/fetch.go | 242 +++++++++++++++++++-- pkg/chattools/session.go | 2 +- pkg/chattools/types.go | 6 + pkg/connector/builtin_tools.go | 111 ++++++++-- pkg/connector/builtin_tools_test.go | 76 ++++++- pkg/connector/chat_tools.go | 6 +- pkg/connector/client.go | 11 +- pkg/connector/room_state.go | 12 + pkg/connector/room_state_test.go | 21 ++ pkg/connector/slash_commands.go | 16 ++ pkg/connector/slash_commands_session.go | 5 + pkg/connector/slash_commands_test.go | 16 +- pkg/connector/slash_commands_tools.go | 45 ++++ pkg/connector/sources.go | 237 ++++++++++++++++++-- pkg/connector/sources_test.go | 83 +++++-- pkg/connector/stream_test.go | 82 +++++-- pkg/connector/timezone.go | 80 +++++++ pkg/connector/timezone_test.go | 44 ++++ pkg/connector/tool_modes.go | 60 +++++ 33 files changed, 1544 insertions(+), 128 deletions(-) create mode 100644 pkg/ai/providers/citations.go create mode 100644 pkg/connector/slash_commands_tools.go create mode 100644 pkg/connector/timezone.go create mode 100644 pkg/connector/timezone_test.go create mode 100644 pkg/connector/tool_modes.go diff --git a/README.md b/README.md index ee3f4954..b88c693d 100644 --- a/README.md +++ b/README.md @@ -374,11 +374,13 @@ Errors are typed with codes (`pkg/agent/harness/public_errors.go`): `CompactionE | Tool | Purpose | Notes | |------|---------|-------| -| `get_session` | Live chat metadata (current time/timezone, model, reasoning, disabled tools, attachments) | read-only; recomputes time per call | -| `fetch` | Fetch an HTTP/HTTPS URL → readable text + metadata | direct fetch (≤2 MiB, ≤20 000 chars) or AI-services `/tools/fetch` with fallback | -| `web_search` | Web search | only enabled for the Beeper provider with a proxy token; results become source citations | +| `get_session` | Live chat metadata (current time/timezone, model, reasoning, search/fetch modes, attachments) | read-only; recomputes time per call | +| `fetch` | Fetch a full HTTP/HTTPS URL → readable text + metadata | direct fetch (≤2 MiB, ≤20 000 chars) or AI-services `/tools/fetch` extraction with fallback | +| `web_search` | Web search | Exa-backed Beeper search, enabled when room search mode is `beeper`; returns concise URL results for optional follow-up `fetch` calls | -Tools are gated per-room via the `com.beeper.ai.tools` state event's `disabled` array. Web tools route through AI-services (`/tools/web_search`, `/tools/fetch`) using the appservice bearer token. Some models additionally expose **provider-native** built-ins (`image_generation`, `web_search`) injected into the request payload (`pkg/connector/builtin_tools.go`). +Tools are gated per-room via the `com.beeper.ai.tools` state event. `search` may be `off`, `beeper`, or `native`; `fetch` may be `off` or `beeper`. The legacy `disabled` array is still read for older room state. In `beeper` mode, web tools route through AI-services (`/tools/web_search`, `/tools/fetch`) using the appservice bearer token. In `native` search mode, provider-native `web_search` is injected for models that advertise it; if a provider/model has no native web search, search is unavailable. Search result URLs stay in the tool view; fetched pages, provider-native citation annotations, and final-answer URLs become canonical `com.beeper.source` artifacts for client source cards. Other provider-native built-ins, such as `image_generation`, are still injected from the model catalog (`pkg/connector/builtin_tools.go`). + +`fetch` tries the URL directly first with `Accept` preferring Markdown, plain text, JSON, XML, and CSV. If the response is already agent-readable (Markdown/plain/JSON/XML/CSV/source-ish), it returns that result without backend extraction. If the response is HTML, it checks HTTP `Link` headers and HTML `` for a readable alternate and fetches that directly. Only when the direct representation is not agent-ready does it call AI-services `/tools/fetch`. Local/private hosts, GitHub raw/gist URLs, GitLab-style raw paths, and source/text file extensions are treated as direct-fetch candidates. **Adding a tool:** @@ -386,9 +388,9 @@ Tools are gated per-room via the `com.beeper.ai.tools` state event's `disabled` 2. Return `jsonResult(value)` for consistent text + `Details` output. 3. Register in `chattools.Tools` (unconditionally or behind a config gate). 4. Wire config in `pkg/connector/chat_tools.go` and honor `DisabledTools`. -5. If it produces citable sources, mirror `webSearchSourceParts` so URLs surface as message sources. +5. If it produces citable sources, add canonical source observations in `pkg/connector/sources.go` so URLs surface as message sources. -> **Security note:** direct `fetch` has **no SSRF guard** — it can reach localhost/private/link-local addresses when the bridge bypasses AI-services for raw assets and local/private targets. Treat it accordingly in your threat model. +> **Security note:** the direct fetch path has **no SSRF guard** — it can reach localhost/private/link-local addresses when the bridge bypasses AI-services for raw assets and local/private targets. Treat it accordingly in your threat model. ## Sessions: the branching conversation tree @@ -520,7 +522,7 @@ It serves `/v1/models`, `/v1/responses`, `/v1/chat/completions`, and `/api/strea - **Reasoning is double-validated and clamped** — setting a model can silently change the effective reasoning level. - **Two parallel session-tree implementations** (`aidb` vs `session` SQLite files) with near-duplicate SQL and one subtle difference (`ON DELETE CASCADE`). - **Token counts are estimates** (≈ chars/4) — compaction thresholds are approximate. -- **Direct `fetch` has no SSRF protection.** +- **The direct fetch path has no SSRF protection.** - **`ProviderConfig` holds secrets** (API keys, refresh tokens) in login metadata and serializes to JSON *and* YAML — don't log it. - **AG-UI `Event` is a map, not a struct** — read typed fields via `Get`/`String`; unknown fields survive round-trips. diff --git a/pkg/ai/providers/anthropic.go b/pkg/ai/providers/anthropic.go index 4362a282..116ef63c 100644 --- a/pkg/ai/providers/anthropic.go +++ b/pkg/ai/providers/anthropic.go @@ -127,6 +127,11 @@ func StreamAnthropic(ctx context.Context, model ai.Model, llmContext ai.Context, return fmt.Errorf("could not parse Anthropic SSE event %s: %w; data=%s; raw=%s", sse.Event, err, sse.Data, strings.Join(sse.Raw, "\n")) } } + citations := providerCitationsFromAny(event, model.Provider, max(0, len(contentBlocks(output.Content))-1)) + if len(citations) > 0 { + output.Citations = append(output.Citations, citations...) + stream.Push(ai.AssistantMessageEvent{Type: "source", Partial: &output}) + } state.apply(stream, &output, model, llmContext, isOAuth, event) return nil }) diff --git a/pkg/ai/providers/citations.go b/pkg/ai/providers/citations.go new file mode 100644 index 00000000..319af5b1 --- /dev/null +++ b/pkg/ai/providers/citations.go @@ -0,0 +1,222 @@ +package providers + +import ( + "fmt" + "strconv" + "strings" + + ai "github.com/beeper/ai-bridge/pkg/ai" +) + +func providerCitationsFromAny(value any, provider ai.Provider, contentIndex int) []ai.Citation { + out := []ai.Citation{} + if data, ok := value.(map[string]any); ok { + out = append(out, googleGroundingCitationsFromMap(data, provider, contentIndex)...) + } + walkProviderCitationMaps(value, func(item map[string]any) { + if citation, ok := providerCitationFromMap(item, provider, contentIndex); ok { + out = append(out, citation) + } + }) + return out +} + +func googleGroundingCitationsFromMap(data map[string]any, provider ai.Provider, contentIndex int) []ai.Citation { + metadata, _ := data["groundingMetadata"].(map[string]any) + if metadata == nil { + metadata, _ = data["grounding_metadata"].(map[string]any) + } + if metadata == nil { + if candidates, _ := data["candidates"].([]any); len(candidates) > 0 { + if candidate, _ := candidates[0].(map[string]any); candidate != nil { + return googleGroundingCitationsFromMap(candidate, provider, contentIndex) + } + } + return nil + } + chunks, _ := metadata["groundingChunks"].([]any) + if chunks == nil { + chunks, _ = metadata["grounding_chunks"].([]any) + } + if len(chunks) == 0 { + return nil + } + chunkCitations := make([]ai.Citation, 0, len(chunks)) + for _, rawChunk := range chunks { + chunk, _ := rawChunk.(map[string]any) + web, _ := chunk["web"].(map[string]any) + if web == nil { + web, _ = chunk["retrievedContext"].(map[string]any) + } + if web == nil { + chunkCitations = append(chunkCitations, ai.Citation{}) + continue + } + url := firstCitationString(stringFromAny(web["uri"]), stringFromAny(web["url"])) + chunkCitations = append(chunkCitations, ai.Citation{ + Type: "url_citation", + URL: url, + Title: stringFromAny(web["title"]), + ContentIndex: &contentIndex, + Provider: string(provider), + RawType: "grounding", + }) + } + supports, _ := metadata["groundingSupports"].([]any) + if supports == nil { + supports, _ = metadata["grounding_supports"].([]any) + } + if len(supports) == 0 { + out := []ai.Citation{} + for _, citation := range chunkCitations { + if citation.URL != "" { + out = append(out, citation) + } + } + return out + } + out := []ai.Citation{} + for _, rawSupport := range supports { + support, _ := rawSupport.(map[string]any) + segment, _ := support["segment"].(map[string]any) + indices := citationIndexList(firstCitationAny(support, "groundingChunkIndices", "grounding_chunk_indices")) + for _, index := range indices { + if index < 0 || index >= len(chunkCitations) || chunkCitations[index].URL == "" { + continue + } + citation := chunkCitations[index] + if start, ok := intFromCitationAny(firstCitationAny(segment, "startIndex", "start_index")); ok { + citation.StartIndex = &start + } + if end, ok := intFromCitationAny(firstCitationAny(segment, "endIndex", "end_index")); ok { + citation.EndIndex = &end + } + citation.Text = stringFromAny(segment["text"]) + out = append(out, citation) + } + } + return out +} + +func citationIndexList(value any) []int { + raw, ok := value.([]any) + if !ok { + return nil + } + out := make([]int, 0, len(raw)) + for _, item := range raw { + if index, ok := intFromCitationAny(item); ok { + out = append(out, index) + } + } + return out +} + +func walkProviderCitationMaps(value any, emit func(map[string]any)) { + switch typed := value.(type) { + case nil: + return + case map[string]any: + emit(typed) + for _, item := range typed { + switch item.(type) { + case map[string]any, []any: + walkProviderCitationMaps(item, emit) + } + } + case []any: + for _, item := range typed { + walkProviderCitationMaps(item, emit) + } + } +} + +func providerCitationFromMap(data map[string]any, provider ai.Provider, contentIndex int) (ai.Citation, bool) { + rawType := strings.ToLower(stringFromAny(data["type"])) + citationData := data + if nested, _ := data["url_citation"].(map[string]any); nested != nil { + citationData = mergeCitationMaps(data, nested) + } else if nested, _ := data["urlCitation"].(map[string]any); nested != nil { + citationData = mergeCitationMaps(data, nested) + } + rawType = firstCitationString(rawType, strings.ToLower(stringFromAny(citationData["type"]))) + url := firstCitationString(stringFromAny(citationData["url"]), stringFromAny(citationData["uri"])) + if url == "" || (!strings.Contains(rawType, "citation") && rawType != "web_search_result_location") { + return ai.Citation{}, false + } + resolvedContentIndex := contentIndex + if index, ok := intFromCitationAny(firstCitationAny(citationData, "contentIndex", "content_index", "outputIndex", "output_index")); ok { + resolvedContentIndex = index + } + citation := ai.Citation{ + Type: "url_citation", + URL: url, + Title: stringFromAny(citationData["title"]), + Description: firstCitationString(stringFromAny(citationData["description"]), stringFromAny(citationData["summary"])), + SiteName: firstCitationString(stringFromAny(citationData["siteName"]), stringFromAny(citationData["site_name"])), + FaviconURL: firstCitationString(stringFromAny(citationData["faviconUrl"]), stringFromAny(citationData["favicon_url"])), + ImageURL: firstCitationString(stringFromAny(citationData["imageUrl"]), stringFromAny(citationData["image_url"])), + PublishedAt: firstCitationString(stringFromAny(citationData["publishedAt"]), stringFromAny(citationData["published_at"]), stringFromAny(citationData["publishedDate"]), stringFromAny(citationData["datePublished"]), stringFromAny(citationData["date"])), + ContentIndex: &resolvedContentIndex, + Provider: string(provider), + RawType: rawType, + Text: firstCitationString(stringFromAny(citationData["text"]), stringFromAny(citationData["cited_text"])), + } + if start, ok := intFromCitationAny(firstCitationAny(citationData, "startIndex", "start_index")); ok { + citation.StartIndex = &start + } + if end, ok := intFromCitationAny(firstCitationAny(citationData, "endIndex", "end_index")); ok { + citation.EndIndex = &end + } + return citation, true +} + +func mergeCitationMaps(first map[string]any, second map[string]any) map[string]any { + out := map[string]any{} + for key, value := range first { + out[key] = value + } + for key, value := range second { + out[key] = value + } + return out +} + +func firstCitationAny(data map[string]any, keys ...string) any { + for _, key := range keys { + if value, ok := data[key]; ok { + return value + } + } + return nil +} + +func firstCitationString(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func intFromCitationAny(value any) (int, bool) { + switch typed := value.(type) { + case int: + return typed, true + case int64: + return int(typed), true + case float64: + return int(typed), true + case string: + parsed, err := strconv.Atoi(strings.TrimSpace(typed)) + return parsed, err == nil + default: + if value != nil { + if parsed, err := strconv.Atoi(fmt.Sprint(value)); err == nil { + return parsed, true + } + } + return 0, false + } +} diff --git a/pkg/ai/providers/google.go b/pkg/ai/providers/google.go index 907169bb..ff4beeb7 100644 --- a/pkg/ai/providers/google.go +++ b/pkg/ai/providers/google.go @@ -97,6 +97,11 @@ func StreamGoogle(ctx context.Context, model ai.Model, llmContext ai.Context, op if err := json.Unmarshal([]byte(sse.Data), &chunk); err != nil { return fmt.Errorf("could not parse Google SSE event: %w; data=%s; raw=%s", err, sse.Data, strings.Join(sse.Raw, "\n")) } + citations := providerCitationsFromAny(chunk, model.Provider, max(0, state.currentBlockIndex)) + if len(citations) > 0 { + output.Citations = append(output.Citations, citations...) + stream.Push(ai.AssistantMessageEvent{Type: "source", Partial: &output}) + } state.apply(stream, &output, model, chunk) return nil }) diff --git a/pkg/ai/providers/google_vertex.go b/pkg/ai/providers/google_vertex.go index fee5963e..972e9a0a 100644 --- a/pkg/ai/providers/google_vertex.go +++ b/pkg/ai/providers/google_vertex.go @@ -107,6 +107,11 @@ func StreamGoogleVertex(ctx context.Context, model ai.Model, llmContext ai.Conte if err := json.Unmarshal([]byte(sse.Data), &chunk); err != nil { return fmt.Errorf("could not parse Google Vertex SSE event: %w; data=%s; raw=%s", err, sse.Data, strings.Join(sse.Raw, "\n")) } + citations := providerCitationsFromAny(chunk, model.Provider, max(0, state.currentBlockIndex)) + if len(citations) > 0 { + output.Citations = append(output.Citations, citations...) + stream.Push(ai.AssistantMessageEvent{Type: "source", Partial: &output}) + } state.apply(stream, &output, model, chunk) return nil }) diff --git a/pkg/ai/providers/openai_completions.go b/pkg/ai/providers/openai_completions.go index 4519a4c2..17680d38 100644 --- a/pkg/ai/providers/openai_completions.go +++ b/pkg/ai/providers/openai_completions.go @@ -205,7 +205,9 @@ func applyCompleteOpenAICompletions(output *ai.Message, model ai.Model, raw map[ } } if text := textFromCompletionMessage(message["content"]); text != "" { + contentIndex := len(blocks) blocks = append(blocks, ai.ContentBlock{Type: "text", Text: text}) + output.Citations = append(output.Citations, providerCitationsFromAny(message, model.Provider, contentIndex)...) } if toolCalls, ok := message["tool_calls"].([]any); ok { for _, rawToolCall := range toolCalls { diff --git a/pkg/ai/providers/openai_conversion_test.go b/pkg/ai/providers/openai_conversion_test.go index 833b0c57..8ed681d9 100644 --- a/pkg/ai/providers/openai_conversion_test.go +++ b/pkg/ai/providers/openai_conversion_test.go @@ -117,6 +117,66 @@ func TestConvertCompletionsMessagesIncludesNativeAudio(t *testing.T) { } } +func TestProviderCitationsFromOpenAIAnnotations(t *testing.T) { + citations := providerCitationsFromAny(map[string]any{ + "type": "message", + "content": []any{map[string]any{ + "type": "output_text", + "text": "hello citation", + "annotations": []any{map[string]any{ + "type": "url_citation", + "start_index": float64(6), + "end_index": float64(14), + "url": "https://example.com/source", + "title": "Source", + }}, + }}, + }, ai.ProviderOpenAI, 0) + if len(citations) != 1 || citations[0].URL != "https://example.com/source" || citations[0].Title != "Source" { + t.Fatalf("unexpected citations %#v", citations) + } + if citations[0].StartIndex == nil || *citations[0].StartIndex != 6 || citations[0].EndIndex == nil || *citations[0].EndIndex != 14 { + t.Fatalf("missing citation range %#v", citations[0]) + } +} + +func TestProviderCitationsFromGoogleGrounding(t *testing.T) { + citations := providerCitationsFromAny(map[string]any{ + "candidates": []any{map[string]any{ + "groundingMetadata": map[string]any{ + "groundingChunks": []any{map[string]any{ + "web": map[string]any{"uri": "https://example.com/google", "title": "Google Source"}, + }}, + "groundingSupports": []any{map[string]any{ + "segment": map[string]any{"startIndex": float64(2), "endIndex": float64(8), "text": "claim"}, + "groundingChunkIndices": []any{float64(0)}, + }}, + }, + }}, + }, ai.ProviderGoogle, 0) + if len(citations) != 1 || citations[0].URL != "https://example.com/google" || citations[0].Title != "Google Source" { + t.Fatalf("unexpected grounding citations %#v", citations) + } + if citations[0].StartIndex == nil || *citations[0].StartIndex != 2 || citations[0].EndIndex == nil || *citations[0].EndIndex != 8 || citations[0].Text != "claim" { + t.Fatalf("missing grounding citation range %#v", citations[0]) + } +} + +func TestProviderCitationsFromAnthropicWebSearchLocation(t *testing.T) { + citations := providerCitationsFromAny(map[string]any{ + "type": "web_search_result_location", + "url": "https://example.com/anthropic", + "title": "Anthropic Source", + "cited_text": "quoted source text", + }, ai.ProviderAnthropic, 0) + if len(citations) != 1 || citations[0].URL != "https://example.com/anthropic" || citations[0].Title != "Anthropic Source" { + t.Fatalf("unexpected citations %#v", citations) + } + if citations[0].Text != "quoted source text" { + t.Fatalf("missing cited text %#v", citations[0]) + } +} + func TestConvertResponsesMessagesIncludesNativeAudio(t *testing.T) { model := ai.Model{ID: "gpt-audio", API: ai.ApiOpenAIResponses, Provider: "openai", Input: []string{"text", "audio"}} messages := ConvertResponsesMessages(model, ai.Context{ diff --git a/pkg/ai/providers/openai_responses.go b/pkg/ai/providers/openai_responses.go index f7ee4c41..6c1b30cf 100644 --- a/pkg/ai/providers/openai_responses.go +++ b/pkg/ai/providers/openai_responses.go @@ -177,6 +177,7 @@ func applyCompleteOpenAIResponses(output *ai.Message, model ai.Model, options Op if id, ok := item["id"].(string); ok && id != "" { block.TextSignature = mustJSON(map[string]any{"v": 1, "id": id}) } + output.Citations = append(output.Citations, providerCitationsFromAny(item, model.Provider, len(blocks))...) blocks = append(blocks, block) } case "function_call": diff --git a/pkg/ai/providers/openai_shared.go b/pkg/ai/providers/openai_shared.go index 67f67fc4..6b7c8317 100644 --- a/pkg/ai/providers/openai_shared.go +++ b/pkg/ai/providers/openai_shared.go @@ -472,6 +472,11 @@ func (s *completionsStreamState) apply(stream *ai.AssistantMessageEventStream, o output.Content = s.blocks stream.Push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: index, Delta: text, Partial: output}) } + if annotations, ok := delta["annotations"].([]any); ok { + index := s.ensureText(stream, output) + output.Citations = append(output.Citations, providerCitationsFromAny(annotations, model.Provider, index)...) + stream.Push(ai.AssistantMessageEvent{Type: "source", Partial: output}) + } for _, field := range []string{"reasoning_content", "reasoning", "reasoning_text"} { if reasoning, ok := delta[field].(string); ok && reasoning != "" { signature := field @@ -854,6 +859,15 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out output.Content = s.blocks push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: s.currentIndex, Delta: delta, Partial: output}) } + case "response.output_text.annotation.added": + contentIndex := s.currentIndex + if contentIndex < 0 { + contentIndex = intFromAny(event["content_index"]) + } + if annotation, ok := event["annotation"].(map[string]any); ok { + output.Citations = append(output.Citations, providerCitationsFromAny(annotation, model.Provider, contentIndex)...) + push(ai.AssistantMessageEvent{Type: "source", Partial: output}) + } case "response.function_call_arguments.delta": if s.currentIndex >= 0 && s.blocks[s.currentIndex].Type == "toolCall" { delta, _ := event["delta"].(string) @@ -892,6 +906,7 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out if text := messageTextFromItem(item); text != "" { s.blocks[s.currentIndex].Text = text } + output.Citations = append(output.Citations, providerCitationsFromAny(item, model.Provider, s.currentIndex)...) if id, ok := item["id"].(string); ok && id != "" { payload := map[string]any{"v": 1, "id": id} if phase, ok := item["phase"].(string); ok && phase != "" { diff --git a/pkg/ai/types.go b/pkg/ai/types.go index a5803360..7c66f070 100644 --- a/pkg/ai/types.go +++ b/pkg/ai/types.go @@ -248,6 +248,24 @@ type Message struct { FromID string `json:"fromId,omitempty"` TokensBefore int `json:"tokensBefore,omitempty"` Truncated bool `json:"truncated,omitempty"` + Citations []Citation `json:"citations,omitempty"` +} + +type Citation struct { + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + SiteName string `json:"siteName,omitempty"` + FaviconURL string `json:"faviconUrl,omitempty"` + ImageURL string `json:"imageUrl,omitempty"` + PublishedAt string `json:"publishedAt,omitempty"` + StartIndex *int `json:"startIndex,omitempty"` + EndIndex *int `json:"endIndex,omitempty"` + ContentIndex *int `json:"contentIndex,omitempty"` + Text string `json:"text,omitempty"` + Provider string `json:"provider,omitempty"` + RawType string `json:"rawType,omitempty"` } type Tool struct { diff --git a/pkg/aiid/ids_test.go b/pkg/aiid/ids_test.go index eb8bc649..8a63ac68 100644 --- a/pkg/aiid/ids_test.go +++ b/pkg/aiid/ids_test.go @@ -66,6 +66,7 @@ func TestMetadataJSONRoundTrip(t *testing.T) { Providers: map[string]ProviderConfig{ provider.ID: provider, }, + LastKnownTimezone: "Europe/Amsterdam", } raw, err := json.Marshal(meta) if err != nil { @@ -82,6 +83,9 @@ func TestMetadataJSONRoundTrip(t *testing.T) { if decodedProvider.DefaultModel != "gpt-5" { t.Fatalf("metadata did not round trip: %#v", decoded) } + if decoded.LastKnownTimezone != "Europe/Amsterdam" { + t.Fatalf("timezone metadata did not round trip: %#v", decoded) + } } func TestMediaIDForEncodesMetadata(t *testing.T) { diff --git a/pkg/aiid/metadata.go b/pkg/aiid/metadata.go index 63ce023f..36c38b9e 100644 --- a/pkg/aiid/metadata.go +++ b/pkg/aiid/metadata.go @@ -19,8 +19,9 @@ type ProviderConfig struct { } type UserLoginMetadata struct { - Providers map[string]ProviderConfig `json:"providers,omitempty"` - Approvals map[string]ApprovalDecision `json:"approvals,omitempty"` + Providers map[string]ProviderConfig `json:"providers,omitempty"` + Approvals map[string]ApprovalDecision `json:"approvals,omitempty"` + LastKnownTimezone string `json:"last_known_timezone,omitempty"` } type ApprovalDecision struct { diff --git a/pkg/chattools/chattools.go b/pkg/chattools/chattools.go index b7b8675b..95fd0b49 100644 --- a/pkg/chattools/chattools.go +++ b/pkg/chattools/chattools.go @@ -9,7 +9,9 @@ func Tools(info SessionInfo, fetch FetchOptions, search SearchOptions) []agent.A func ToolsWithOptions(info SessionInfo, fetch FetchOptions, search SearchOptions, sessionOptions SessionOptions) []agent.AgentTool[any] { tools := []agent.AgentTool[any]{ GetSessionToolWithOptions(info, sessionOptions), - FetchTool(fetch), + } + if !fetch.Disabled { + tools = append(tools, FetchTool(fetch)) } if search.Enabled { tools = append(tools, WebSearchTool(search)) diff --git a/pkg/chattools/chattools_test.go b/pkg/chattools/chattools_test.go index ca8e0da4..7203c357 100644 --- a/pkg/chattools/chattools_test.go +++ b/pkg/chattools/chattools_test.go @@ -34,7 +34,10 @@ func TestGetSessionReturnsFreshMetadata(t *testing.T) { SelectedModel: "gpt-5", SelectedReasoning: "low", DisabledTools: []string{"web_search"}, + SearchMode: "off", + FetchMode: "beeper", LastKnownTimestamp: "2026-05-31T22:34:00Z", + LastKnownTimezone: "Europe/Amsterdam", }) result, err := tool.Execute(context.Background(), "call", map[string]any{}, nil) if err != nil { @@ -44,10 +47,10 @@ func TestGetSessionReturnsFreshMetadata(t *testing.T) { if err := json.Unmarshal([]byte(result.Content[0].Text), &info); err != nil { t.Fatal(err) } - if info.CurrentTimestamp == "" || info.LastKnownTimestamp != "2026-05-31T22:34:00Z" || info.ChatID != "session-1" || info.ChatTitle != "Markdown Chaos Test" || info.SelectedModel != "gpt-5" || len(info.DisabledTools) != 1 || info.DisabledTools[0] != "web_search" { + if info.CurrentTimestamp == "" || info.LastKnownTimestamp != "2026-05-31T22:34:00Z" || info.LastKnownTimezone != "Europe/Amsterdam" || info.ChatID != "session-1" || info.ChatTitle != "Markdown Chaos Test" || info.SelectedModel != "gpt-5" || len(info.DisabledTools) != 1 || info.DisabledTools[0] != "web_search" || info.SearchMode != "off" || info.FetchMode != "beeper" { t.Fatalf("expected fresh session metadata, got %#v", info) } - assertSessionKeys(t, result.Content[0].Text, "current_timestamp", "chat_id", "chat_title", "chat_first_message_at", "selected_model", "selected_reasoning", "disabled_tools", "last_known_timestamp") + assertSessionKeys(t, result.Content[0].Text, "current_timestamp", "chat_id", "chat_title", "chat_first_message_at", "selected_model", "selected_reasoning", "disabled_tools", "search_mode", "fetch_mode", "last_known_timestamp", "last_known_timezone") } func TestGetSessionIncludesProfileOnlyWhenResolverReturnsIt(t *testing.T) { @@ -115,6 +118,15 @@ func TestToolsOmitsDisabledSearch(t *testing.T) { } } +func TestToolsOmitsDisabledFetch(t *testing.T) { + tools := Tools(SessionInfo{}, FetchOptions{Disabled: true}, SearchOptions{Enabled: true}) + for _, tool := range tools { + if tool.Tool.Name == "fetch" { + t.Fatalf("fetch should not be exposed when disabled") + } + } +} + func TestFetch(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") @@ -138,6 +150,9 @@ func TestFetchUsesDirectFetchForAssetsWhenToolEndpointConfigured(t *testing.T) { exaHit = true return testResponse(req, http.StatusOK, "application/json", `{"results":[]}`), nil } + if !strings.Contains(req.Header.Get("Accept"), "text/markdown") { + t.Fatalf("direct fetch should prefer readable representations, got Accept=%q", req.Header.Get("Accept")) + } return testResponse(req, http.StatusOK, "text/markdown", "# Title\n\nBody"), nil })} @@ -150,23 +165,53 @@ func TestFetchUsesDirectFetchForAssetsWhenToolEndpointConfigured(t *testing.T) { } } +func TestFetchUsesMarkdownAlternateBeforeToolEndpoint(t *testing.T) { + exaHit := false + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Host == "exa.test" { + exaHit = true + return testResponse(req, http.StatusOK, "application/json", `{"markdown":"tool result"}`), nil + } + switch req.URL.Path { + case "/page": + resp := testResponse(req, http.StatusOK, "text/html; charset=utf-8", `HTML page`) + resp.Header.Set("Link", `; rel="alternate"; type="text/markdown"`) + return resp, nil + case "/from-header.md": + return testResponse(req, http.StatusOK, "text/markdown", "# Header markdown"), nil + default: + return testResponse(req, http.StatusNotFound, "text/plain", "not found"), nil + } + })} + + result, err := Fetch(context.Background(), "https://example.com/page", FetchOptions{Timeout: time.Second, ToolEndpoint: "https://exa.test/contents", Client: client, MaxBytes: 1024, MaxChars: 100}) + if err != nil { + t.Fatal(err) + } + if exaHit || result.FetchMethod != "direct" || result.FinalURL != "https://example.com/from-header.md" || !strings.Contains(result.Markdown, "Header markdown") { + t.Fatalf("unexpected alternate fetch result %#v exaHit=%v", result, exaHit) + } +} + func TestFetchUsesToolEndpointForPages(t *testing.T) { - exa := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost || r.Header.Get("Authorization") != "Bearer key" { + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Host != "exa.test" { + return testResponse(req, http.StatusOK, "text/html", "PageHTML page"), nil + } + if req.Method != http.MethodPost || req.Header.Get("Authorization") != "Bearer key" { t.Fatalf("unexpected request method/header") } var payload map[string]any - if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { t.Fatal(err) } if payload["url"] != "https://example.com/page" || payload["max_chars"] != float64(100) { t.Fatalf("unexpected fetch payload %#v", payload) } - _, _ = w.Write([]byte(`{"request_id":"req_1","title":"Page","description":"Page description","url":"https://example.com/page","final_url":"https://example.com/page","markdown":"Extracted page text","published_at":"2026-01-01","author":"A","favicon_url":"https://example.com/favicon.ico","metadata":{"links":["https://example.com/next"]}}`)) - })) - defer exa.Close() + return testResponse(req, http.StatusOK, "application/json", `{"request_id":"req_1","title":"Page","description":"Page description","url":"https://example.com/page","final_url":"https://example.com/page","markdown":"Extracted page text","published_at":"2026-01-01","author":"A","favicon_url":"https://example.com/favicon.ico","metadata":{"links":["https://example.com/next"]}}`), nil + })} - result, err := Fetch(context.Background(), "https://example.com/page", FetchOptions{Timeout: time.Second, ToolEndpoint: exa.URL, APIKey: "key", MaxChars: 100}) + result, err := Fetch(context.Background(), "https://example.com/page", FetchOptions{Timeout: time.Second, ToolEndpoint: "https://exa.test/contents", APIKey: "key", Client: client, MaxChars: 100}) if err != nil { t.Fatal(err) } @@ -307,6 +352,98 @@ func TestSearchMapsToolOptionsToPayload(t *testing.T) { } } +func TestSearchResultsCanBeFetchedByURL(t *testing.T) { + var fetchedURL string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/search": + _, _ = w.Write([]byte(`{"results":[{"title":"One","url":"https://example.com/one","snippet":"first"}]}`)) + case "/fetch": + var payload map[string]any + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Fatal(err) + } + fetchedURL, _ = payload["url"].(string) + _, _ = w.Write([]byte(`{"url":"` + fetchedURL + `","final_url":"` + fetchedURL + `","title":"One","markdown":"Full page"}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Host == "example.com" { + return testResponse(req, http.StatusOK, "text/html", "OneSearch result page"), nil + } + return http.DefaultTransport.RoundTrip(req) + })} + + tools := Tools( + SessionInfo{}, + FetchOptions{Timeout: time.Second, ToolEndpoint: server.URL + "/fetch", Client: client}, + SearchOptions{Enabled: true, Endpoint: server.URL + "/search", Timeout: time.Second}, + ) + searchIndex := -1 + fetchIndex := -1 + for i := range tools { + switch tools[i].Tool.Name { + case "web_search": + searchIndex = i + case "fetch": + fetchIndex = i + } + } + if searchIndex < 0 || fetchIndex < 0 { + t.Fatalf("missing tools") + } + properties, _ := tools[fetchIndex].Tool.Parameters["properties"].(map[string]any) + if _, ok := properties["ref_id"]; ok { + t.Fatalf("fetch schema should not expose ref_id, got %#v", tools[fetchIndex].Tool.Parameters) + } + if _, ok := properties["url"]; !ok { + t.Fatalf("fetch schema should expose url, got %#v", tools[fetchIndex].Tool.Parameters) + } + required, _ := tools[fetchIndex].Tool.Parameters["required"].([]string) + if len(required) != 1 || required[0] != "url" { + t.Fatalf("fetch schema should require url, got %#v", tools[fetchIndex].Tool.Parameters) + } + + searchResult, err := tools[searchIndex].Execute(context.Background(), "search-call", map[string]any{"query": "query"}, nil) + if err != nil { + t.Fatal(err) + } + var searchBody SearchResult + if err := json.Unmarshal([]byte(searchResult.Content[0].Text), &searchBody); err != nil { + t.Fatal(err) + } + if len(searchBody.Results) != 1 || searchBody.Results[0].URL != "https://example.com/one" { + t.Fatalf("missing URL in search result: %#v", searchBody) + } + + fetchResult, err := tools[fetchIndex].Execute(context.Background(), "fetch-call", map[string]any{"url": searchBody.Results[0].URL}, nil) + if err != nil { + t.Fatal(err) + } + var fetchBody FetchResult + if err := json.Unmarshal([]byte(fetchResult.Content[0].Text), &fetchBody); err != nil { + t.Fatal(err) + } + if fetchedURL != "https://example.com/one" || fetchBody.Markdown != "Full page" { + t.Fatalf("unexpected fetch by URL: fetchedURL=%q result=%#v", fetchedURL, fetchBody) + } + + fetchResult, err = tools[fetchIndex].Execute(context.Background(), "fetch-call-url", map[string]any{"url": "https://example.com/two"}, nil) + if err != nil { + t.Fatal(err) + } + fetchBody = FetchResult{} + if err := json.Unmarshal([]byte(fetchResult.Content[0].Text), &fetchBody); err != nil { + t.Fatal(err) + } + if fetchedURL != "https://example.com/two" || fetchBody.Markdown != "Full page" { + t.Fatalf("unexpected fetch by URL target: fetchedURL=%q result=%#v", fetchedURL, fetchBody) + } +} + func TestSearchRequiresConfiguration(t *testing.T) { if _, err := Search(context.Background(), "query", 5, SearchRequestOptions{}, SearchOptions{}); err == nil { t.Fatalf("expected configuration error") diff --git a/pkg/chattools/fetch.go b/pkg/chattools/fetch.go index 2ca13f46..293af4bc 100644 --- a/pkg/chattools/fetch.go +++ b/pkg/chattools/fetch.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "mime" "net" "net/http" "net/url" @@ -14,6 +15,7 @@ import ( "time" "github.com/rs/zerolog" + "golang.org/x/net/html" agent "github.com/beeper/ai-bridge/pkg/agent" ai "github.com/beeper/ai-bridge/pkg/ai" @@ -23,9 +25,9 @@ func FetchTool(options FetchOptions) agent.AgentTool[any] { return agent.AgentTool[any]{ Tool: ai.Tool{ Name: "fetch", - Description: "Fetch an HTTP or HTTPS URL and return readable page content, metadata, and source details.", + Description: "Fetch a URL and return readable page content.", Parameters: objectSchema(map[string]any{ - "url": map[string]any{"type": "string", "description": "HTTP or HTTPS URL to fetch."}, + "url": map[string]any{"type": "string", "description": "The full HTTP/HTTPS URL to fetch."}, "max_chars": map[string]any{"type": "integer", "description": "Maximum number of text characters to return."}, }, []string{"url"}), }, @@ -64,7 +66,20 @@ func Fetch(ctx context.Context, rawURL string, options FetchOptions) (FetchResul if options.MaxChars == 0 { options.MaxChars = 20000 } - if options.ToolEndpoint != "" && !shouldDirectFetch(parsed) { + directResult, directErr := fetchDirect(ctx, rawURL, parsed, options) + if directErr == nil { + if shouldReturnDirectResult(parsed, directResult.ContentType) { + return directResult, nil + } + if isHTMLContentType(directResult.ContentType) || looksLikeHTML(directResult.RawBody) { + if alternateURL := findReadableAlternate(directResult.ResponseHeaders, directResult.RawBody, directResult.FinalURL); alternateURL != "" { + if alternate, err := fetchDirectURL(ctx, alternateURL, options); err == nil && shouldReturnDirectResult(mustParseURL(alternate.FinalURL), alternate.ContentType) { + return alternate, nil + } + } + } + } + if options.ToolEndpoint != "" { result, err := FetchContents(ctx, parsed.String(), options) if err == nil { return result, nil @@ -76,9 +91,12 @@ func Fetch(ctx context.Context, rawURL string, options FetchOptions) (FetchResul Str("fetch_method", "web_tool"). Str("target_url", parsed.Redacted()). Str("target_host", parsed.Host). - Msg("Falling back to direct fetch after web tool fetch failed") + Msg("Falling back to direct fetch result after web tool fetch failed") } - return fetchDirect(ctx, rawURL, parsed, options) + if directErr != nil { + return FetchResult{}, directErr + } + return directResult, nil } func fetchDirect(ctx context.Context, rawURL string, parsed *url.URL, options FetchOptions) (FetchResult, error) { @@ -98,6 +116,7 @@ func fetchDirect(ctx context.Context, rawURL string, parsed *url.URL, options Fe return FetchResult{}, err } req.Header.Set("User-Agent", "beeper-ai-bridge/1.0") + req.Header.Set("Accept", directOpenAcceptHeader) log.Trace().Msg("Sending AI tool HTTP request") started := time.Now() resp, err := client.Do(req) @@ -123,20 +142,38 @@ func fetchDirect(ctx context.Context, rawURL string, parsed *url.URL, options Fe truncated = true } metadata := extractHTMLMetadata(body, resp.Request.URL) + markdown := "" + if isDirectReadableContentType(resp.Header.Get("Content-Type")) || shouldReturnDirectURL(resp.Request.URL) { + markdown = text + } return FetchResult{ - URL: rawURL, - FinalURL: resp.Request.URL.String(), - Status: resp.StatusCode, - ContentType: resp.Header.Get("Content-Type"), - Title: metadata.Title, - Description: metadata.Description, - Text: text, - Favicon: metadata.Favicon, - Truncated: truncated, - FetchMethod: "direct", + URL: rawURL, + FinalURL: resp.Request.URL.String(), + Status: resp.StatusCode, + ContentType: resp.Header.Get("Content-Type"), + Title: metadata.Title, + Description: metadata.Description, + Text: text, + Markdown: markdown, + Favicon: metadata.Favicon, + Truncated: truncated, + FetchMethod: "direct", + ResponseHeaders: resp.Header.Clone(), + RawBody: body, }, nil } +func fetchDirectURL(ctx context.Context, rawURL string, options FetchOptions) (FetchResult, error) { + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return FetchResult{}, fmt.Errorf("invalid URL") + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return FetchResult{}, fmt.Errorf("unsupported URL scheme %s", parsed.Scheme) + } + return fetchDirect(ctx, rawURL, parsed, options) +} + func FetchContents(ctx context.Context, rawURL string, options FetchOptions) (FetchResult, error) { if options.ToolEndpoint == "" { return FetchResult{}, errors.New("fetch contents is not configured") @@ -223,7 +260,16 @@ func FetchContents(ctx context.Context, rawURL string, options FetchOptions) (Fe return result, nil } -func shouldDirectFetch(parsed *url.URL) bool { +const directOpenAcceptHeader = "text/markdown, text/plain;q=0.95, application/json;q=0.9, application/xml;q=0.85, text/xml;q=0.85, text/csv;q=0.85, text/html;q=0.5, */*;q=0.1" + +func shouldReturnDirectResult(parsed *url.URL, contentType string) bool { + return isDirectReadableContentType(contentType) || shouldReturnDirectURL(parsed) +} + +func shouldReturnDirectURL(parsed *url.URL) bool { + if parsed == nil { + return false + } host := strings.ToLower(parsed.Hostname()) path := strings.ToLower(parsed.EscapedPath()) if host == "localhost" || strings.HasSuffix(host, ".localhost") { @@ -246,18 +292,170 @@ func shouldDirectFetch(parsed *url.URL) bool { case ".txt", ".md", ".markdown", ".rst", ".csv", ".tsv", ".json", ".jsonl", ".yaml", ".yml", ".toml", ".xml", ".rss", ".atom", ".log", ".diff", ".patch", ".go", ".js", ".jsx", ".ts", ".tsx", ".css", ".scss", ".sass", ".less", - ".c", ".cc", ".cpp", ".h", ".hpp", ".java", ".kt", ".kts", ".rs", ".py", ".rb", ".swift", ".sh", ".bash", ".zsh", ".fish", ".sql", - ".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".ico", ".svg", - ".pdf", ".zip", ".tar", ".tgz", ".gz", ".bz2", ".xz", ".7z", ".rar", - ".mp3", ".mp4", ".m4a", ".mov", ".wav", ".webm", ".ogg", - ".woff", ".woff2", ".ttf", ".otf", ".eot", ".wasm", - ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx": + ".c", ".cc", ".cpp", ".h", ".hpp", ".java", ".kt", ".kts", ".rs", ".py", ".rb", ".swift", ".sh", ".bash", ".zsh", ".fish", ".sql": return true default: return false } } +func isDirectReadableContentType(contentType string) bool { + mediaType := normalizedMediaType(contentType) + if mediaType == "" { + return false + } + if mediaType == "text/markdown" || mediaType == "text/plain" || mediaType == "text/csv" || mediaType == "text/xml" { + return true + } + if strings.HasPrefix(mediaType, "text/") && mediaType != "text/html" { + return true + } + if mediaType == "application/json" || mediaType == "application/xml" { + return true + } + return strings.HasSuffix(mediaType, "+json") || strings.HasSuffix(mediaType, "+xml") +} + +func isHTMLContentType(contentType string) bool { + mediaType := normalizedMediaType(contentType) + return mediaType == "text/html" || mediaType == "application/xhtml+xml" +} + +func looksLikeHTML(body []byte) bool { + prefix := strings.ToLower(string(bytes.TrimSpace(body))) + return strings.HasPrefix(prefix, "': + if !inQuote { + inAngle = false + } + case ',': + if !inQuote && !inAngle { + parts = append(parts, strings.TrimSpace(value[start:i])) + start = i + 1 + } + } + } + parts = append(parts, strings.TrimSpace(value[start:])) + return parts +} + +func parseLinkValue(value string) (string, map[string]string) { + params := map[string]string{} + value = strings.TrimSpace(value) + if !strings.HasPrefix(value, "<") { + return "", params + } + end := strings.Index(value, ">") + if end < 0 { + return "", params + } + linkURL := strings.TrimSpace(value[1:end]) + for _, part := range strings.Split(value[end+1:], ";") { + key, raw, ok := strings.Cut(strings.TrimSpace(part), "=") + if !ok { + continue + } + params[strings.ToLower(strings.TrimSpace(key))] = strings.Trim(strings.TrimSpace(raw), `"`) + } + return linkURL, params +} + +func readableAlternateFromHTML(body []byte, baseURL *url.URL) string { + doc, err := html.Parse(bytes.NewReader(body)) + if err != nil { + return "" + } + var out string + var walk func(*html.Node) + walk = func(node *html.Node) { + if out != "" { + return + } + if node.Type == html.ElementNode && strings.EqualFold(node.Data, "link") { + if relContains(attr(node, "rel"), "alternate") && isReadableAlternateType(attr(node, "type")) { + out = resolveMetadataURL(attr(node, "href"), baseURL) + return + } + } + for child := node.FirstChild; child != nil; child = child.NextSibling { + walk(child) + } + } + walk(doc) + return out +} + +func relContains(rel string, token string) bool { + for _, part := range strings.Fields(strings.ToLower(rel)) { + if part == token { + return true + } + } + return false +} + +func mustParseURL(rawURL string) *url.URL { + parsed, err := url.Parse(rawURL) + if err != nil { + return nil + } + return parsed +} + type contentsResponse struct { URL string `json:"url"` FinalURL string `json:"final_url"` diff --git a/pkg/chattools/session.go b/pkg/chattools/session.go index 3033b74a..5acdfc6c 100644 --- a/pkg/chattools/session.go +++ b/pkg/chattools/session.go @@ -16,7 +16,7 @@ func GetSessionToolWithOptions(info SessionInfo, options SessionOptions) agent.A return agent.AgentTool[any]{ Tool: ai.Tool{ Name: "get_session", - Description: "Get fresh metadata for this Beeper AI chat, including current UTC timestamp, chat ID/title, selected model, selected reasoning, disabled tools, approved profile fields, and last known UTC timestamp.", + Description: "Get fresh metadata for this Beeper AI chat, including current UTC timestamp, chat ID/title, selected model, selected reasoning, disabled tools, approved profile fields, last known UTC timestamp, and last known timezone.", Parameters: objectSchema(nil, nil), }, Execute: func(ctx context.Context, toolCallID string, params any, onUpdate agent.AgentToolUpdateCallback[any]) (agent.AgentToolResult[any], error) { diff --git a/pkg/chattools/types.go b/pkg/chattools/types.go index ad2e67e5..7ac6eea7 100644 --- a/pkg/chattools/types.go +++ b/pkg/chattools/types.go @@ -14,11 +14,14 @@ type SessionInfo struct { SelectedModel string `json:"selected_model,omitempty"` SelectedReasoning string `json:"selected_reasoning,omitempty"` DisabledTools []string `json:"disabled_tools,omitempty"` + SearchMode string `json:"search_mode,omitempty"` + FetchMode string `json:"fetch_mode,omitempty"` BeeperUsername string `json:"beeper_username,omitempty"` BeeperDisplayName string `json:"beeper_display_name,omitempty"` BeeperAccountEmail string `json:"beeper_account_email,omitempty"` GravatarProfile any `json:"gravatar_profile,omitempty"` LastKnownTimestamp string `json:"last_known_timestamp"` + LastKnownTimezone string `json:"last_known_timezone,omitempty"` } type SessionProfile struct { @@ -34,6 +37,7 @@ type SessionOptions struct { } type FetchOptions struct { + Disabled bool Timeout time.Duration MaxBytes int64 MaxChars int @@ -93,6 +97,8 @@ type FetchResult struct { Context string `json:"context,omitempty"` Error string `json:"error,omitempty"` FetchMethod string `json:"-"` + ResponseHeaders http.Header `json:"-"` + RawBody []byte `json:"-"` } type SearchResult struct { diff --git a/pkg/connector/builtin_tools.go b/pkg/connector/builtin_tools.go index bcf03ea1..5a9797c9 100644 --- a/pkg/connector/builtin_tools.go +++ b/pkg/connector/builtin_tools.go @@ -4,11 +4,13 @@ import ( "context" "maps" "slices" + "strings" "github.com/beeper/ai-bridge/pkg/agent/harness" + "github.com/beeper/ai-bridge/pkg/ai" ) -func (cl *Client) registerProviderBuiltInToolHooks(agentHarness *harness.AgentHarness) { +func (cl *Client) registerProviderBuiltInToolHooks(agentHarness *harness.AgentHarness, roomConfig RoomConfig) { if agentHarness == nil { return } @@ -16,7 +18,7 @@ func (cl *Client) registerProviderBuiltInToolHooks(agentHarness *harness.AgentHa if event.Model == nil { return nil, nil } - payload, changed := addBuiltInToolsToPayload(event.Payload, event.Model.BuiltInTools) + payload, changed := addBuiltInToolsToPayload(event.Payload, activeBuiltInToolPayloads(*event.Model, roomConfig)) if !changed { return nil, nil } @@ -24,7 +26,73 @@ func (cl *Client) registerProviderBuiltInToolHooks(agentHarness *harness.AgentHa }) } -func addBuiltInToolsToPayload(payload any, builtInTools []string) (any, bool) { +func activeBuiltInToolPayloads(model ai.Model, roomConfig RoomConfig) []map[string]any { + out := make([]map[string]any, 0, len(model.BuiltInTools)) + for _, tool := range model.BuiltInTools { + payload, ok := builtInToolPayload(model, roomConfig, tool) + if !ok { + continue + } + out = append(out, payload) + } + return out +} + +func builtInToolPayload(model ai.Model, roomConfig RoomConfig, tool string) (map[string]any, bool) { + switch normalizedBuiltInTool(tool) { + case "web_search": + if roomSearchMode(roomConfig) != toolModeNative { + return nil, false + } + return nativeWebSearchToolPayload(model) + case "image_generation": + switch { + case strings.HasPrefix(strings.TrimSpace(tool), "openrouter:"): + return map[string]any{"type": strings.TrimSpace(tool)}, true + case model.Provider == ai.ProviderOpenAI || model.Provider == ai.ProviderOpenRouter: + return map[string]any{"type": "image_generation"}, true + default: + return nil, false + } + default: + return nil, false + } +} + +func normalizedBuiltInTool(tool string) string { + tool = strings.ToLower(strings.TrimSpace(tool)) + switch tool { + case "web_search", "web_search_preview", "openrouter:web_search", "web_search_20250305", "google_search", "google_search_retrieval": + return "web_search" + case "image_generation", "openrouter:image_generation": + return "image_generation" + default: + return tool + } +} + +func nativeWebSearchToolPayload(model ai.Model) (map[string]any, bool) { + switch model.API { + case ai.ApiAnthropicMessages: + return map[string]any{"type": "web_search_20250305", "name": "web_search"}, true + case ai.ApiGoogleGenerativeAI, ai.ApiGoogleVertex: + return map[string]any{"google_search": map[string]any{}}, true + case ai.ApiOpenAIResponses: + if model.Provider == ai.ProviderOpenRouter { + return map[string]any{"type": "openrouter:web_search"}, true + } + return map[string]any{"type": "web_search"}, true + case ai.ApiOpenAICompletions: + if model.Provider == ai.ProviderOpenRouter { + return map[string]any{"type": "openrouter:web_search"}, true + } + return nil, false + default: + return nil, false + } +} + +func addBuiltInToolsToPayload(payload any, builtInTools []map[string]any) (any, bool) { body, ok := payload.(map[string]any) if !ok || len(builtInTools) == 0 { return payload, false @@ -33,9 +101,9 @@ func addBuiltInToolsToPayload(payload any, builtInTools []string) (any, bool) { next := clonePayloadMap(body) tools := toolsAsAny(next["tools"]) changed := false - for _, toolType := range builtInTools { + for _, toolPayload := range builtInTools { before := len(tools) - tools = appendBuiltInTool(tools, toolType) + tools = appendBuiltInTool(tools, toolPayload) changed = changed || len(tools) != before } if !changed { @@ -45,27 +113,40 @@ func addBuiltInToolsToPayload(payload any, builtInTools []string) (any, bool) { return next, true } -func appendBuiltInTool(tools []any, toolType string) []any { +func appendBuiltInTool(tools []any, toolPayload map[string]any) []any { + toolKey := builtInToolKey(toolPayload) + if toolKey == "" { + return tools + } for _, tool := range tools { if toolMap, ok := tool.(map[string]any); ok { - if toolMap["type"] == toolType || toolMap["name"] == toolType { + if builtInToolKey(toolMap) == toolKey { return tools } - if toolMap["type"] == "function" && toolMap["name"] == builtInToolFunctionName(toolType) { + if toolMap["type"] == "function" && toolMap["name"] == toolKey { return tools } } } - return append(tools, map[string]any{"type": toolType}) + return append(tools, maps.Clone(toolPayload)) } -func builtInToolFunctionName(toolType string) string { - switch toolType { - case "image_generation": - return "image_generation" - default: - return toolType +func builtInToolKey(tool map[string]any) string { + if tool == nil { + return "" + } + for _, key := range []string{"type", "name"} { + if value := strings.TrimSpace(stringFromAny(tool[key])); value != "" { + return value + } + } + if _, ok := tool["google_search"]; ok { + return "google_search" + } + if _, ok := tool["googleSearch"]; ok { + return "google_search" } + return "" } func toolsAsAny(raw any) []any { diff --git a/pkg/connector/builtin_tools_test.go b/pkg/connector/builtin_tools_test.go index 71465ecf..d82f3c22 100644 --- a/pkg/connector/builtin_tools_test.go +++ b/pkg/connector/builtin_tools_test.go @@ -1,6 +1,10 @@ package connector -import "testing" +import ( + "testing" + + "github.com/beeper/ai-bridge/pkg/ai" +) func TestAddBuiltInToolsToPayloadAddsCatalogBuiltIns(t *testing.T) { payload := map[string]any{ @@ -11,7 +15,7 @@ func TestAddBuiltInToolsToPayloadAddsCatalogBuiltIns(t *testing.T) { }}, } - next, changed := addBuiltInToolsToPayload(payload, []string{"image_generation"}) + next, changed := addBuiltInToolsToPayload(payload, []map[string]any{{"type": "image_generation"}}) if !changed { t.Fatal("expected payload to change") } @@ -43,7 +47,7 @@ func TestAddBuiltInToolsToPayloadSkipsWhenAlreadyPresent(t *testing.T) { }}, } - _, changed := addBuiltInToolsToPayload(payload, []string{"image_generation"}) + _, changed := addBuiltInToolsToPayload(payload, []map[string]any{{"type": "image_generation"}}) if changed { t.Fatal("did not expect payload change when built-in tool is already present") } @@ -52,13 +56,77 @@ func TestAddBuiltInToolsToPayloadSkipsWhenAlreadyPresent(t *testing.T) { func TestAddBuiltInToolsToPayloadAddsOpenRouterBuiltIn(t *testing.T) { payload := map[string]any{"model": "anthropic/claude-sonnet-4.5"} - next, changed := addBuiltInToolsToPayload(payload, []string{"openrouter:image_generation"}) + next, changed := addBuiltInToolsToPayload(payload, []map[string]any{{"type": "openrouter:image_generation"}}) if !changed { t.Fatal("expected payload to change") } assertToolType(t, next.(map[string]any)["tools"], "openrouter:image_generation") } +func TestActiveBuiltInToolPayloadsHonorsNativeSearchMode(t *testing.T) { + model := ai.Model{API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenAI, BuiltInTools: []string{"web_search", "image_generation"}} + if got := activeBuiltInToolPayloads(model, RoomConfig{}); len(got) != 1 || got[0]["type"] != "image_generation" { + t.Fatalf("default beeper search should suppress native web_search only, got %#v", got) + } + if got := activeBuiltInToolPayloads(model, RoomConfig{SearchMode: toolModeNative}); len(got) != 2 || got[0]["type"] != "web_search" || got[1]["type"] != "image_generation" { + t.Fatalf("native search should allow provider web_search, got %#v", got) + } + if got := activeBuiltInToolPayloads(model, RoomConfig{SearchMode: toolModeOff}); len(got) != 1 || got[0]["type"] != "image_generation" { + t.Fatalf("off search should suppress native web_search only, got %#v", got) + } +} + +func TestNativeWebSearchToolPayloadsAreProviderSpecific(t *testing.T) { + tests := []struct { + name string + model ai.Model + wantKey string + wantValue any + }{ + { + name: "openai responses", + model: ai.Model{API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenAI, BuiltInTools: []string{"web_search"}}, + wantKey: "type", + wantValue: "web_search", + }, + { + name: "openrouter", + model: ai.Model{API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenRouter, BuiltInTools: []string{"web_search"}}, + wantKey: "type", + wantValue: "openrouter:web_search", + }, + { + name: "anthropic", + model: ai.Model{API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic, BuiltInTools: []string{"web_search"}}, + wantKey: "type", + wantValue: "web_search_20250305", + }, + { + name: "google vertex", + model: ai.Model{API: ai.ApiGoogleVertex, Provider: ai.ProviderGoogleVertex, BuiltInTools: []string{"web_search"}}, + wantKey: "google_search", + wantValue: map[string]any{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := activeBuiltInToolPayloads(tt.model, RoomConfig{SearchMode: toolModeNative}) + if len(got) != 1 { + t.Fatalf("expected one native search payload, got %#v", got) + } + if tt.wantKey == "google_search" { + if _, ok := got[0]["google_search"].(map[string]any); !ok { + t.Fatalf("expected google_search object, got %#v", got[0]) + } + return + } + if got[0][tt.wantKey] != tt.wantValue { + t.Fatalf("unexpected payload %#v", got[0]) + } + }) + } +} + func assertToolType(t *testing.T, raw any, toolType string) { t.Helper() assertToolTypeCount(t, raw, toolType, 1) diff --git a/pkg/connector/chat_tools.go b/pkg/connector/chat_tools.go index b9cc78a3..a13dac3c 100644 --- a/pkg/connector/chat_tools.go +++ b/pkg/connector/chat_tools.go @@ -39,13 +39,17 @@ func (cl *Client) chatTools(msg *bridgev2.MatrixMessage, meta *aiid.PortalMetada SelectedModel: model.ID, SelectedReasoning: cl.reasoningLevelForModel(model, roomConfig), DisabledTools: roomConfig.DisabledTools, + SearchMode: roomSearchMode(roomConfig), + FetchMode: roomFetchMode(roomConfig), LastKnownTimestamp: formatSessionTimestampUTC(matrixEventTime(nil)), + LastKnownTimezone: cl.lastKnownTimezone(), } if msg != nil { info.LastKnownTimestamp = formatSessionTimestampUTC(matrixEventTime(msg.Event)) } search := cl.searchOptions(roomConfig, provider) fetch := chattools.FetchOptions{ + Disabled: roomFetchMode(roomConfig) != toolModeBeeper, Timeout: time.Duration(cl.Main.Config.Fetch.TimeoutMS) * time.Millisecond, MaxBytes: cl.Main.Config.Fetch.MaxBytes, MaxChars: cl.Main.Config.Fetch.MaxChars, @@ -97,7 +101,7 @@ func modelHasOutputModality(model ai.Model, modality string) bool { } func (cl *Client) searchOptions(roomConfig RoomConfig, provider aiid.ProviderConfig) chattools.SearchOptions { - if toolDisabled(roomConfig.DisabledTools, "web_search") || provider.ID != aiid.DefaultProvider || provider.BaseURL == "" { + if roomSearchMode(roomConfig) != toolModeBeeper || provider.ID != aiid.DefaultProvider || provider.BaseURL == "" { return chattools.SearchOptions{} } token, err := cl.defaultProviderBearerToken() diff --git a/pkg/connector/client.go b/pkg/connector/client.go index 2d94bd90..05684921 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -277,6 +277,7 @@ func (cl *Client) handleMatrixMessage(ctx context.Context, msg *bridgev2.MatrixM if err := cl.ensureUsablePortal(msg.Portal); err != nil { return nil, err } + cl.updateLastKnownTimezoneFromMessage(ctx, msg) if resp, handled, err := cl.handleAISlashCommand(ctx, msg); handled { return resp, err } @@ -387,7 +388,7 @@ func (cl *Client) startAsyncPrompt(ctx context.Context, msg *bridgev2.MatrixMess cl.markPendingFailed(ctx, pending, err) return } - cl.registerProviderBuiltInToolHooks(agentHarness) + cl.registerProviderBuiltInToolHooks(agentHarness, roomConfig) active.harness = agentHarness active.addPending(pending) cl.setActiveHarness(msg.Portal.PortalKey, agentHarness) @@ -1196,6 +1197,7 @@ func (cl *Client) streamPublisherWithEndFrom(publisher bridgev2.BeeperStreamPubl downstream.End() return stream } + streamSources := newSourceCollector() go func() { defer cancelStream() if onEnd != nil { @@ -1248,6 +1250,11 @@ func (cl *Client) streamPublisherWithEndFrom(publisher bridgev2.BeeperStreamPubl cursor.mu.Lock() beforeEvents := len(run.Events) applyAIStreamEvent(writer, evt, model.ContextWindow) + if evt.Partial != nil { + for _, source := range streamSources.addProviderSources(*evt.Partial) { + writer.Custom("com.beeper.source", source) + } + } afterEvents := len(run.Events) maybeSecondVisibleChunk(evt) if !seenFirstDelta && isVisibleAIStreamDelta(evt) { @@ -1738,6 +1745,7 @@ func appendToolOutputs(run *aistream.Run, outputs []toolOutputEvent, messages .. } for _, message := range messages { sources.addProviderSources(message) + sources.addAnswerURLSources(message) } for _, source := range sources.sources() { writer.Custom("com.beeper.source", source) @@ -2414,6 +2422,7 @@ func (r *activeAIRun) publishToolOutput(ctx context.Context, cl *Client, publish return fmt.Errorf("no active assistant stream for tool output") } stream := r.streams[0] + stream.tools = append(stream.tools, output) r.mu.Unlock() stream.publish.mu.Lock() diff --git a/pkg/connector/room_state.go b/pkg/connector/room_state.go index 4e4f08e1..fa89dfc4 100644 --- a/pkg/connector/room_state.go +++ b/pkg/connector/room_state.go @@ -27,6 +27,8 @@ type RoomConfig struct { AdditionalPrompt string ThinkingLevel string DisabledTools []string + SearchMode string + FetchMode string modelStatePresent bool modelStateModel string @@ -80,6 +82,16 @@ func (s AIRoomStateStore) ReadConfig(ctx context.Context, roomID id.RoomID) (Roo return RoomConfig{}, "", err } else if raw != nil { config.DisabledTools = stringSlice(raw["disabled"]) + if _, ok := raw["search"]; ok { + config.SearchMode = normalizedToolMode(firstString(raw, "search"), defaultSearchMode) + } else { + config.SearchMode = searchModeFromDisabled(config.DisabledTools) + } + if _, ok := raw["fetch"]; ok { + config.FetchMode = normalizedToolMode(firstString(raw, "fetch"), defaultFetchMode) + } else { + config.FetchMode = fetchModeFromDisabled(config.DisabledTools) + } stateEventIDs = append(stateEventIDs, eventID) } return config, strings.Join(stateEventIDs, ","), nil diff --git a/pkg/connector/room_state_test.go b/pkg/connector/room_state_test.go index 8bf0e64b..537bc28b 100644 --- a/pkg/connector/room_state_test.go +++ b/pkg/connector/room_state_test.go @@ -27,3 +27,24 @@ func TestStringSliceDeduplicatesDisabledTools(t *testing.T) { t.Fatalf("unexpected disabled tools %#v", got) } } + +func TestRoomToolModesDefaultAndLegacyDisabled(t *testing.T) { + if got := roomSearchMode(RoomConfig{}); got != toolModeBeeper { + t.Fatalf("default search mode = %q", got) + } + if got := roomFetchMode(RoomConfig{}); got != toolModeBeeper { + t.Fatalf("default fetch mode = %q", got) + } + if got := roomSearchMode(RoomConfig{DisabledTools: []string{"web_search"}}); got != toolModeOff { + t.Fatalf("legacy disabled web_search should turn search off, got %q", got) + } + if got := roomFetchMode(RoomConfig{DisabledTools: []string{"fetch"}}); got != toolModeOff { + t.Fatalf("disabled fetch should turn fetch off, got %q", got) + } + if got := roomSearchMode(RoomConfig{SearchMode: "native", DisabledTools: []string{"web_search"}}); got != toolModeNative { + t.Fatalf("explicit search mode should win over disabled list, got %q", got) + } + if got := roomFetchMode(RoomConfig{FetchMode: "bad"}); got != defaultFetchMode { + t.Fatalf("invalid fetch mode should fall back, got %q", got) + } +} diff --git a/pkg/connector/slash_commands.go b/pkg/connector/slash_commands.go index 997e7599..3df8ea02 100644 --- a/pkg/connector/slash_commands.go +++ b/pkg/connector/slash_commands.go @@ -88,6 +88,22 @@ func aiSlashCommandDefinitions() []aiSlashCommandDefinition { needsRoomConfig: true, run: runSystemPromptCommand, }, + { + name: "search", + usage: "/search [off|beeper|native]", + description: "Show or set web search mode for this room.", + needsRoomConfig: true, + noticeErrors: true, + run: runSearchModeCommand, + }, + { + name: "fetch", + usage: "/fetch [off|beeper]", + description: "Show or set URL fetch mode for this room.", + needsRoomConfig: true, + noticeErrors: true, + run: runFetchModeCommand, + }, { name: "compact", usage: "/compact [instructions]", diff --git a/pkg/connector/slash_commands_session.go b/pkg/connector/slash_commands_session.go index dacd69f8..48d922a0 100644 --- a/pkg/connector/slash_commands_session.go +++ b/pkg/connector/slash_commands_session.go @@ -40,6 +40,7 @@ type sessionCommandInfo struct { Responding bool ActiveRunID string ActiveModel string + LastKnownTimezone string Stats sessionCommandStats ContextMessages int EstimatedTokens int @@ -60,6 +61,7 @@ func runSessionCommand(cl *Client, ctx context.Context, portal *bridgev2.Portal, SystemPrompt: strings.TrimSpace(roomConfig.AdditionalPrompt) != "", SystemPromptChars: len([]rune(strings.TrimSpace(roomConfig.AdditionalPrompt))), Responding: active != nil, + LastKnownTimezone: cl.lastKnownTimezone(), } if active != nil { info.ActiveRunID = active.runID @@ -184,6 +186,9 @@ func formatSessionCommandInfo(info sessionCommandInfo) string { if info.ActiveModel != "" { fmt.Fprintf(&text, "\n- Active model: `%s`", info.ActiveModel) } + if info.LastKnownTimezone != "" { + fmt.Fprintf(&text, "\n- Last known timezone: `%s`", info.LastKnownTimezone) + } systemPrompt := "no" if info.SystemPrompt { systemPrompt = fmt.Sprintf("yes, %d chars", info.SystemPromptChars) diff --git a/pkg/connector/slash_commands_test.go b/pkg/connector/slash_commands_test.go index ee0f3818..ada7dd8a 100644 --- a/pkg/connector/slash_commands_test.go +++ b/pkg/connector/slash_commands_test.go @@ -357,13 +357,14 @@ func TestSessionCommandStatsFromEntries(t *testing.T) { func TestFormatSessionCommandInfo(t *testing.T) { text := formatSessionCommandInfo(sessionCommandInfo{ - SessionID: "session-1", - CreatedAt: "2026-05-30T00:00:00Z", - RoomProvider: "beeper", - RoomModel: "beeper/gpt-5.5", - RoomReasoning: "off", - SystemPrompt: true, - Responding: true, + SessionID: "session-1", + CreatedAt: "2026-05-30T00:00:00Z", + RoomProvider: "beeper", + RoomModel: "beeper/gpt-5.5", + RoomReasoning: "off", + SystemPrompt: true, + Responding: true, + LastKnownTimezone: "Europe/Amsterdam", Stats: sessionCommandStats{ TotalEntries: 4, Messages: 3, @@ -377,6 +378,7 @@ func TestFormatSessionCommandInfo(t *testing.T) { "Status: `responding`", "ID: `session-1`", "Room model: `beeper/gpt-5.5`", + "Last known timezone: `Europe/Amsterdam`", "System prompt: `yes, 0 chars`", "Messages: `3` total, `1` user, `1` assistant, `1` tool results", "Compactions: `1`", diff --git a/pkg/connector/slash_commands_tools.go b/pkg/connector/slash_commands_tools.go new file mode 100644 index 00000000..24dc7a57 --- /dev/null +++ b/pkg/connector/slash_commands_tools.go @@ -0,0 +1,45 @@ +package connector + +import ( + "context" + "fmt" + "strings" + + "maunium.net/go/mautrix/bridgev2" + + "github.com/beeper/ai-bridge/pkg/aiid" +) + +func runSearchModeCommand(cl *Client, ctx context.Context, portal *bridgev2.Portal, roomConfig RoomConfig, arg string, responder aiCommandResponder) error { + arg = strings.ToLower(strings.TrimSpace(arg)) + if arg == "" { + return responder.Reply(ctx, fmt.Sprintf("Current search mode is `%s`. Options: `off`, `beeper`, `native`.", roomSearchMode(roomConfig))) + } + mode := normalizedToolMode(arg, "") + if mode != toolModeOff && mode != toolModeBeeper && mode != toolModeNative { + return fmt.Errorf("search mode %q is invalid", arg) + } + roomConfig.SearchMode = mode + if _, err := cl.writeAIRoomState(ctx, portal, aiid.RoomToolsType, toolModeStateContent(roomConfig)); err != nil { + return err + } + cl.refreshRoomCapabilities(ctx, portal) + return responder.Reply(ctx, fmt.Sprintf("Search mode set to `%s`.", mode)) +} + +func runFetchModeCommand(cl *Client, ctx context.Context, portal *bridgev2.Portal, roomConfig RoomConfig, arg string, responder aiCommandResponder) error { + arg = strings.ToLower(strings.TrimSpace(arg)) + if arg == "" { + return responder.Reply(ctx, fmt.Sprintf("Current fetch mode is `%s`. Options: `off`, `beeper`.", roomFetchMode(roomConfig))) + } + mode := normalizedToolMode(arg, "") + if mode != toolModeOff && mode != toolModeBeeper { + return fmt.Errorf("fetch mode %q is invalid", arg) + } + roomConfig.FetchMode = mode + if _, err := cl.writeAIRoomState(ctx, portal, aiid.RoomToolsType, toolModeStateContent(roomConfig)); err != nil { + return err + } + cl.refreshRoomCapabilities(ctx, portal) + return responder.Reply(ctx, fmt.Sprintf("Fetch mode set to `%s`.", mode)) +} diff --git a/pkg/connector/sources.go b/pkg/connector/sources.go index d24ec2af..74268d33 100644 --- a/pkg/connector/sources.go +++ b/pkg/connector/sources.go @@ -3,7 +3,10 @@ package connector import ( "fmt" "net/url" + "regexp" "strings" + + ai "github.com/beeper/ai-bridge/pkg/ai" ) type sourceCollector struct { @@ -38,12 +41,16 @@ type canonicalSource struct { } type sourceAppearance struct { - Kind string - ToolCallID string - ToolName string - Query string - Rank int - Cited bool + Kind string + ToolCallID string + ToolName string + Query string + Rank int + Cited bool + StartIndex *int + EndIndex *int + ContentIndex *int + Text string } func newSourceCollector() *sourceCollector { @@ -104,7 +111,7 @@ func (c *sourceCollector) addSearchResultSource(output toolOutputEvent, query st source := sourceObservation{ URL: sourceString(item, "url", "URL", "uri"), Title: sourceString(item, "title"), - Description: firstSourceString(sourceDescriptionString(item), firstStringFromSlice(item["highlights"]), sourceString(item, "text")), + Description: firstSourceString(sourceDescriptionString(item), firstStringFromSlice(item["highlights"])), SiteName: sourceString(item, "siteName", "site_name", "source"), FaviconURL: sourceFaviconString(item), ImageURL: sourceImageString(item), @@ -163,7 +170,7 @@ func (c *sourceCollector) addFetchOutput(output toolOutputEvent, result any) []m source := sourceObservation{ URL: firstSourceString(sourceString(data, "final_url", "finalUrl"), sourceString(data, "url")), Title: sourceString(data, "title"), - Description: firstSourceString(sourceDescriptionString(data), firstStringFromSlice(data["highlights"]), sourceString(data, "text")), + Description: firstSourceString(sourceDescriptionString(data), firstStringFromSlice(data["highlights"])), SiteName: sourceString(data, "siteName", "site_name", "source"), FaviconURL: sourceFaviconString(data), ImageURL: sourceString(data, "image", "imageUrl", "image_url"), @@ -172,7 +179,7 @@ func (c *sourceCollector) addFetchOutput(output toolOutputEvent, result any) []m Appearance: sourceAppearance{ Kind: "fetch", ToolCallID: output.ID, - ToolName: output.Name, + ToolName: "fetch", }, } if updated := c.add(source); updated != nil { @@ -183,6 +190,20 @@ func (c *sourceCollector) addFetchOutput(output toolOutputEvent, result any) []m func (c *sourceCollector) addProviderSources(message any) []map[string]any { changed := []map[string]any{} + if typed, ok := message.(ai.Message); ok { + for _, citation := range typed.Citations { + if updated := c.add(providerCitationObservation(citation)); updated != nil { + changed = append(changed, updated) + } + } + } + if typed, ok := message.(*ai.Message); ok && typed != nil { + for _, citation := range typed.Citations { + if updated := c.add(providerCitationObservation(citation)); updated != nil { + changed = append(changed, updated) + } + } + } walkProviderSources(message, func(source sourceObservation) { source.Priority = 80 source.Appearance.Kind = "provider" @@ -194,6 +215,45 @@ func (c *sourceCollector) addProviderSources(message any) []map[string]any { return changed } +func (c *sourceCollector) addAnswerURLSources(message ai.Message) []map[string]any { + changed := []map[string]any{} + for _, rawURL := range extractMessageURLs(message) { + obs := sourceObservation{ + URL: rawURL, + Priority: 70, + Appearance: sourceAppearance{ + Kind: "answer", + Cited: true, + }, + } + if updated := c.add(obs); updated != nil { + changed = append(changed, updated) + } + } + return changed +} + +func providerCitationObservation(citation ai.Citation) sourceObservation { + return sourceObservation{ + URL: citation.URL, + Title: citation.Title, + Description: citation.Description, + SiteName: citation.SiteName, + FaviconURL: citation.FaviconURL, + ImageURL: citation.ImageURL, + PublishedAt: citation.PublishedAt, + Priority: 80, + Appearance: sourceAppearance{ + Kind: "provider", + Cited: true, + StartIndex: citation.StartIndex, + EndIndex: citation.EndIndex, + ContentIndex: citation.ContentIndex, + Text: citation.Text, + }, + } +} + func (c *sourceCollector) add(obs sourceObservation) map[string]any { normalized, ok := normalizeSourceURL(obs.URL) if !ok { @@ -316,7 +376,7 @@ func (s *canonicalSource) addAppearance(appearance sourceAppearance) bool { if appearance.Kind == "" { return false } - key := fmt.Sprintf("%s|%s|%s|%s|%d|%t", appearance.Kind, appearance.ToolCallID, appearance.ToolName, appearance.Query, appearance.Rank, appearance.Cited) + key := fmt.Sprintf("%s|%s|%s|%s|%d|%t|%s|%s|%s|%s", appearance.Kind, appearance.ToolCallID, appearance.ToolName, appearance.Query, appearance.Rank, appearance.Cited, intPtrKey(appearance.StartIndex), intPtrKey(appearance.EndIndex), intPtrKey(appearance.ContentIndex), appearance.Text) if _, exists := s.seen[key]; exists { return false } @@ -361,6 +421,54 @@ func (c *sourceCollector) sources() []map[string]any { return out } +var markdownURLPattern = regexp.MustCompile(`https?://[^\s<>"'\]\)]+`) + +func extractMessageURLs(message ai.Message) []string { + text := strings.TrimSpace(messageTextContent(message.Content)) + if text == "" { + return nil + } + seen := map[string]bool{} + var out []string + for _, match := range markdownURLPattern.FindAllString(text, -1) { + match = strings.TrimRight(match, ".,;:!?") + normalized, ok := normalizeSourceURL(match) + if !ok || seen[normalized] { + continue + } + seen[normalized] = true + out = append(out, normalized) + } + return out +} + +func messageTextContent(content any) string { + switch typed := content.(type) { + case string: + return typed + case []ai.ContentBlock: + var parts []string + for _, block := range typed { + if block.Type == "text" && strings.TrimSpace(block.Text) != "" { + parts = append(parts, block.Text) + } + } + return strings.Join(parts, "\n") + case []any: + var parts []string + for _, item := range typed { + if block, ok := item.(map[string]any); ok && stringFromAny(block["type"]) == "text" { + if text := stringFromAny(block["text"]); strings.TrimSpace(text) != "" { + parts = append(parts, text) + } + } + } + return strings.Join(parts, "\n") + default: + return "" + } +} + func sourceAppearances(values []sourceAppearance) []map[string]any { out := make([]map[string]any, 0, len(values)) for _, value := range values { @@ -380,11 +488,30 @@ func sourceAppearances(values []sourceAppearance) []map[string]any { if value.Cited { item["cited"] = true } + if value.StartIndex != nil { + item["startIndex"] = *value.StartIndex + } + if value.EndIndex != nil { + item["endIndex"] = *value.EndIndex + } + if value.ContentIndex != nil { + item["contentIndex"] = *value.ContentIndex + } + if value.Text != "" { + item["text"] = value.Text + } out = append(out, item) } return out } +func intPtrKey(value *int) string { + if value == nil { + return "" + } + return fmt.Sprintf("%d", *value) +} + func normalizeSourceURL(raw string) (string, bool) { parsed, err := url.Parse(strings.TrimSpace(raw)) if err != nil || parsed == nil || parsed.Scheme == "" || parsed.Host == "" { @@ -415,18 +542,8 @@ func walkProviderSources(value any, emit func(sourceObservation)) { case nil: return case map[string]any: - sourceType := strings.ToLower(sourceString(typed, "type")) - rawURL := sourceString(typed, "url", "uri") - if rawURL != "" && strings.Contains(sourceType, "citation") { - emit(sourceObservation{ - URL: rawURL, - Title: sourceString(typed, "title"), - Description: firstSourceString(sourceDescriptionString(typed), sourceString(typed, "text")), - SiteName: sourceString(typed, "siteName", "site_name"), - FaviconURL: sourceFaviconString(typed), - ImageURL: sourceImageString(typed), - PublishedAt: sourceString(typed, "published", "publishedAt", "publishedDate", "datePublished", "date"), - }) + if source, ok := providerCitationSource(typed); ok { + emit(source) } for key, item := range typed { lower := strings.ToLower(key) @@ -451,6 +568,82 @@ func walkProviderSources(value any, emit func(sourceObservation)) { } } +func providerCitationSource(data map[string]any) (sourceObservation, bool) { + sourceType := strings.ToLower(sourceString(data, "type", "rawType")) + nested, _ := data["url_citation"].(map[string]any) + if nested == nil { + nested, _ = data["urlCitation"].(map[string]any) + } + citation := data + if nested != nil { + citation = mergeSourceMaps(data, nested) + sourceType = firstSourceString(sourceType, strings.ToLower(sourceString(nested, "type", "rawType"))) + } + rawURL := sourceString(citation, "url", "uri") + if rawURL == "" || (!strings.Contains(sourceType, "citation") && sourceType != "web_search_result_location") { + return sourceObservation{}, false + } + return sourceObservation{ + URL: rawURL, + Title: sourceString(citation, "title"), + Description: firstSourceString(sourceDescriptionString(citation), sourceString(citation, "cited_text"), sourceString(citation, "text")), + SiteName: sourceString(citation, "siteName", "site_name"), + FaviconURL: sourceFaviconString(citation), + ImageURL: sourceImageString(citation), + PublishedAt: sourceString(citation, "published", "publishedAt", "published_at", "publishedDate", "datePublished", "date"), + Appearance: sourceAppearance{ + Kind: "provider", + Cited: true, + StartIndex: intPointerFromAny(firstSourceAny(citation, "startIndex", "start_index")), + EndIndex: intPointerFromAny(firstSourceAny(citation, "endIndex", "end_index")), + ContentIndex: intPointerFromAny(firstSourceAny(citation, "contentIndex", "content_index", "outputIndex", "output_index")), + Text: firstSourceString(sourceString(citation, "text"), sourceString(citation, "cited_text")), + }, + }, true +} + +func mergeSourceMaps(first map[string]any, second map[string]any) map[string]any { + out := map[string]any{} + for key, value := range first { + out[key] = value + } + for key, value := range second { + out[key] = value + } + return out +} + +func firstSourceAny(data map[string]any, keys ...string) any { + for _, key := range keys { + if value, ok := data[key]; ok { + return value + } + } + return nil +} + +func intPointerFromAny(value any) *int { + var out int + switch typed := value.(type) { + case int: + out = typed + case int64: + out = int(typed) + case float64: + out = int(typed) + case string: + if typed = strings.TrimSpace(typed); typed == "" { + return nil + } + if _, err := fmt.Sscanf(typed, "%d", &out); err != nil { + return nil + } + default: + return nil + } + return &out +} + func sourceString(data map[string]any, keys ...string) string { for _, key := range keys { if value := strings.TrimSpace(stringFromAny(data[key])); value != "" { diff --git a/pkg/connector/sources_test.go b/pkg/connector/sources_test.go index 4dd3feee..a10a6899 100644 --- a/pkg/connector/sources_test.go +++ b/pkg/connector/sources_test.go @@ -1,6 +1,10 @@ package connector -import "testing" +import ( + "testing" + + ai "github.com/beeper/ai-bridge/pkg/ai" +) func TestSourceCollectorUsesDescriptionAndFaviconFallbacks(t *testing.T) { webSources := newSourceCollector().addWebSearchOutput(toolOutputEvent{ @@ -36,17 +40,7 @@ func TestSourceCollectorUsesDescriptionAndFaviconFallbacks(t *testing.T) { }, }) if len(webSources) != 3 { - t.Fatalf("web source metadata fallback failed: %#v", webSources) - } - if webSources[0]["description"] != "Open Graph description" || webSources[0]["faviconUrl"] != "https://example.com/icon.png" || webSources[0]["imageUrl"] != "https://example.com/page-image.png" { - t.Fatalf("web source metadata fallback failed: %#v", webSources[0]) - } - if webSources[1]["url"] != "https://example.com/subpage" || webSources[1]["description"] != "Subpage summary" || webSources[1]["faviconUrl"] != "https://example.com/subpage.ico" { - t.Fatalf("web subpage source metadata fallback failed: %#v", webSources[1]) - } - appearances, ok := webSources[2]["appearances"].([]map[string]any) - if webSources[2]["url"] != "https://example.com/grounded" || !ok || len(appearances) != 1 || !appearances[0]["cited"].(bool) { - t.Fatalf("web grounding source metadata fallback failed: %#v", webSources[2]) + t.Fatalf("web search helper should still normalize sources for grounded/provider-like payloads, got %#v", webSources) } fetchSources := newSourceCollector().addFetchOutput(toolOutputEvent{ @@ -62,10 +56,23 @@ func TestSourceCollectorUsesDescriptionAndFaviconFallbacks(t *testing.T) { t.Fatalf("fetch source metadata fallback failed: %#v", fetchSources) } + captchaSources := newSourceCollector().addFetchOutput(toolOutputEvent{ + ID: "call-fetch-captcha", + Name: "fetch", + }, map[string]any{ + "final_url": "https://example.com/world", + "text": `{"url":"https://geo.captcha-delivery.com/captcha/?cid=long"}`, + }) + if len(captchaSources) != 1 || captchaSources[0]["description"] != "Source from example.com" { + t.Fatalf("fetch source should not use full page text as card description: %#v", captchaSources) + } + providerSources := newSourceCollector().addProviderSources(map[string]any{ "annotations": []any{map[string]any{ - "type": "url_citation", - "url": "https://example.com/provider", + "type": "url_citation", + "url": "https://example.com/provider", + "start_index": float64(10), + "end_index": float64(20), "metadata": map[string]any{ "twitterDescription": "Provider description", "siteIcon": "https://example.com/provider.ico", @@ -75,4 +82,52 @@ func TestSourceCollectorUsesDescriptionAndFaviconFallbacks(t *testing.T) { if len(providerSources) != 1 || providerSources[0]["description"] != "Provider description" || providerSources[0]["faviconUrl"] != "https://example.com/provider.ico" { t.Fatalf("provider source metadata fallback failed: %#v", providerSources) } + appearances, ok := providerSources[0]["appearances"].([]map[string]any) + if !ok || len(appearances) != 1 || appearances[0]["startIndex"] != 10 || appearances[0]["endIndex"] != 20 { + t.Fatalf("provider citation ranges were not preserved: %#v", providerSources[0]) + } + + messageSources := newSourceCollector().addProviderSources(ai.Message{ + Citations: []ai.Citation{{ + Type: "url_citation", + URL: "https://example.com/message-citation", + Title: "Typed Citation", + StartIndex: intPtr(30), + EndIndex: intPtr(40), + }}, + }) + if len(messageSources) != 1 || messageSources[0]["title"] != "Typed Citation" { + t.Fatalf("typed provider citation was not mapped: %#v", messageSources) + } + + collector := newSourceCollector() + searchSources := collector.addWebSearchOutput(toolOutputEvent{ + ID: "call-search", + Name: "web_search", + Input: map[string]any{"query": "q"}, + }, map[string]any{ + "results": []any{map[string]any{ + "title": "Known Result", + "url": "https://example.com/known?utm_source=noise", + }}, + }) + if len(searchSources) != 1 { + t.Fatalf("expected search source, got %#v", searchSources) + } + answerSources := collector.addAnswerURLSources(ai.Message{Content: "Use https://example.com/known and https://example.org/new."}) + if len(answerSources) != 2 { + t.Fatalf("expected answer URL sources, got %#v", answerSources) + } + sources := collector.sources() + if len(sources) != 2 { + t.Fatalf("expected canonical known + new sources, got %#v", sources) + } + answerAppearances, ok := sources[0]["appearances"].([]map[string]any) + if !ok || len(answerAppearances) != 2 || answerAppearances[1]["kind"] != "answer" || answerAppearances[1]["cited"] != true { + t.Fatalf("known source was not marked cited by final answer URL: %#v", sources[0]) + } +} + +func intPtr(value int) *int { + return &value } diff --git a/pkg/connector/stream_test.go b/pkg/connector/stream_test.go index d934ee36..55106d1c 100644 --- a/pkg/connector/stream_test.go +++ b/pkg/connector/stream_test.go @@ -589,34 +589,25 @@ func TestAppendToolOutputsPreservesStructuredResult(t *testing.T) { } } -func TestAppendToolOutputsAddsWebSearchSources(t *testing.T) { +func TestAppendToolOutputsAddsFetchSources(t *testing.T) { run := aistream.NewRun("run", "thread", "beeper/gpt-5", "assistant:run", "GPT-5", timeNow()) run.MessageID = "assistant:run" writer := aistream.NewWriter(run, timeNow) - writer.ToolStart("call-search", "web_search", 0, nil) - writer.ToolEnd("call-search", "web_search", map[string]any{"query": "q"}, nil) + writer.ToolStart("call-fetch", "fetch", 0, nil) + writer.ToolEnd("call-fetch", "fetch", map[string]any{"url": "https://example.com/one"}, nil) appendToolOutputs(run, []toolOutputEvent{{ - ID: "call-search", - Name: "web_search", - Input: map[string]any{"query": "q"}, + ID: "call-fetch", + Name: "fetch", + Input: map[string]any{"url": "https://example.com/one"}, Result: agent.AgentToolResult[any]{ Details: map[string]any{ - "results": []any{ - map[string]any{ - "id": "doc_1", - "title": "One", - "url": "https://example.com/one", - "description": "desc", - "published": "2026-01-01", - "siteName": "Example", - "highlights": []any{"hit"}, - "highlightScores": []any{0.5}, - "summary": "sum", - "subpages": []any{map[string]any{"title": "Sub", "url": "https://example.com/sub"}}, - "extras": map[string]any{"links": []any{"https://example.com/link"}}, - }, - }, + "title": "One", + "url": "https://example.com/one", + "final_url": "https://example.com/one", + "description": "desc", + "published_at": "2026-01-01", + "site_name": "Example", }, }, }}) @@ -639,11 +630,55 @@ func TestAppendToolOutputsAddsWebSearchSources(t *testing.T) { t.Fatalf("missing canonical source metadata: %#v", source) } appearances, ok := source["appearances"].([]map[string]any) - if !ok || len(appearances) != 1 || appearances[0]["kind"] != "web_search" || appearances[0]["toolCallId"] != "call-search" || appearances[0]["rank"] != 1 { + if !ok || len(appearances) != 1 || appearances[0]["kind"] != "fetch" || appearances[0]["toolCallId"] != "call-fetch" { t.Fatalf("missing source appearances: %#v", source["appearances"]) } } +func TestAppendToolOutputsKeepsWebSearchResultSources(t *testing.T) { + run := aistream.NewRun("run", "thread", "beeper/gpt-5", "assistant:run", "GPT-5", timeNow()) + run.MessageID = "assistant:run" + writer := aistream.NewWriter(run, timeNow) + writer.ToolStart("call-search", "web_search", 0, nil) + writer.ToolEnd("call-search", "web_search", map[string]any{"query": "latest news"}, nil) + + appendToolOutputs(run, []toolOutputEvent{{ + ID: "call-search", + Name: "web_search", + Input: map[string]any{"query": "latest news"}, + Result: agent.AgentToolResult[any]{ + Details: map[string]any{ + "results": []any{ + map[string]any{ + "title": "Headline", + "url": "https://example.com/headline", + "description": "desc", + }, + }, + }, + }, + }}) + + message := run.FinalBeeperAIMessage(0, true) + var source map[string]any + for _, part := range message.Parts { + if part["type"] == "source-url" { + source = part + break + } + } + if source == nil { + t.Fatalf("expected web_search source-url part for tool result list, got %#v", message.Parts) + } + appearances, ok := source["appearances"].([]map[string]any) + if !ok || len(appearances) != 1 || appearances[0]["kind"] != "web_search" || appearances[0]["toolCallId"] != "call-search" || appearances[0]["rank"] != 1 { + t.Fatalf("missing web_search source appearance: %#v", source["appearances"]) + } + if appearances[0]["cited"] == true { + t.Fatalf("raw web_search result should not be marked cited: %#v", appearances[0]) + } +} + func TestAssistantEventMetadataCanBeFinalizedBeforeInsert(t *testing.T) { client := &Client{} run := aistream.NewRun("run", "thread", "beeper/gpt-5", "assistant:run", "GPT-5", timeNow()) @@ -910,6 +945,9 @@ func TestPublishToolOutputStreamsLiveResult(t *testing.T) { if len(publisher.updates) != 1 { t.Fatalf("expected one live stream update, got %#v", publisher.updates) } + if len(active.streams) != 1 || len(active.streams[0].tools) != 1 || active.streams[0].tools[0].ID != "call-1" { + t.Fatalf("live tool output was not retained for finalization: %#v", active.streams) + } aiPayload, ok := publisher.updates[0][aistream.BeeperAIKey].(aistream.BeeperAI) if !ok || len(aiPayload.Events) != 2 { t.Fatalf("unexpected live stream carrier %#v", publisher.updates[0]) diff --git a/pkg/connector/timezone.go b/pkg/connector/timezone.go new file mode 100644 index 00000000..4c3a002f --- /dev/null +++ b/pkg/connector/timezone.go @@ -0,0 +1,80 @@ +package connector + +import ( + "context" + "strings" + "time" + + "github.com/rs/zerolog" + "maunium.net/go/mautrix/bridgev2" + + "github.com/beeper/ai-bridge/pkg/aiid" +) + +const beeperTimezoneKey = "com.beeper.tz" + +func (cl *Client) updateLastKnownTimezoneFromMessage(ctx context.Context, msg *bridgev2.MatrixMessage) { + timezone, ok := lastKnownTimezoneFromMatrixMessage(msg) + if !ok || cl == nil || cl.UserLogin == nil { + return + } + if setLastKnownTimezoneOnLogin(cl.UserLogin, timezone) { + if err := cl.UserLogin.Save(ctx); err != nil { + zerolog.Ctx(ctx).Warn().Err(err).Str("timezone", timezone).Msg("Failed to save last known timezone") + } + } +} + +func (cl *Client) lastKnownTimezone() string { + if cl == nil || cl.UserLogin == nil { + return "" + } + meta, ok := cl.UserLogin.Metadata.(*aiid.UserLoginMetadata) + if !ok || meta == nil { + return "" + } + return meta.LastKnownTimezone +} + +func setLastKnownTimezoneOnLogin(login *bridgev2.UserLogin, timezone string) bool { + if login == nil || timezone == "" { + return false + } + meta, ok := login.Metadata.(*aiid.UserLoginMetadata) + if !ok || meta == nil { + meta = &aiid.UserLoginMetadata{} + login.Metadata = meta + } + if meta.LastKnownTimezone == timezone { + return false + } + meta.LastKnownTimezone = timezone + return true +} + +func lastKnownTimezoneFromMatrixMessage(msg *bridgev2.MatrixMessage) (string, bool) { + if msg == nil || msg.Event == nil || msg.Event.Content.Raw == nil { + return "", false + } + raw, ok := msg.Event.Content.Raw[beeperTimezoneKey].(string) + if !ok { + return "", false + } + timezone, ok := normalizeLastKnownTimezone(raw) + return timezone, ok +} + +func normalizeLastKnownTimezone(raw string) (string, bool) { + timezone := strings.TrimSpace(raw) + if timezone == "" || strings.EqualFold(timezone, "local") { + return "", false + } + if strings.EqualFold(timezone, "utc") { + timezone = "UTC" + } + loc, err := time.LoadLocation(timezone) + if err != nil { + return "", false + } + return loc.String(), true +} diff --git a/pkg/connector/timezone_test.go b/pkg/connector/timezone_test.go new file mode 100644 index 00000000..0e5c64dd --- /dev/null +++ b/pkg/connector/timezone_test.go @@ -0,0 +1,44 @@ +package connector + +import ( + "testing" + + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/event" + + "github.com/beeper/ai-bridge/pkg/aiid" +) + +func TestLastKnownTimezoneFromMatrixMessage(t *testing.T) { + msg := &bridgev2.MatrixMessage{ + MatrixEventBase: bridgev2.MatrixEventBase[*event.MessageEventContent]{ + Event: &event.Event{Content: event.Content{Raw: map[string]any{ + beeperTimezoneKey: " Europe/Amsterdam ", + }}}, + }, + } + timezone, ok := lastKnownTimezoneFromMatrixMessage(msg) + if !ok || timezone != "Europe/Amsterdam" { + t.Fatalf("timezone = %q ok=%v, want Europe/Amsterdam true", timezone, ok) + } + + msg.Event.Content.Raw[beeperTimezoneKey] = "Local" + if timezone, ok = lastKnownTimezoneFromMatrixMessage(msg); ok || timezone != "" { + t.Fatalf("Local timezone should be rejected, got %q ok=%v", timezone, ok) + } +} + +func TestSetLastKnownTimezoneOnLogin(t *testing.T) { + login := &bridgev2.UserLogin{UserLogin: &database.UserLogin{}} + if !setLastKnownTimezoneOnLogin(login, "Europe/Amsterdam") { + t.Fatal("expected timezone update") + } + meta, ok := login.Metadata.(*aiid.UserLoginMetadata) + if !ok || meta.LastKnownTimezone != "Europe/Amsterdam" { + t.Fatalf("timezone was not stored in login metadata: %#v", login.Metadata) + } + if setLastKnownTimezoneOnLogin(login, "Europe/Amsterdam") { + t.Fatal("unchanged timezone should not require a save") + } +} diff --git a/pkg/connector/tool_modes.go b/pkg/connector/tool_modes.go new file mode 100644 index 00000000..046011a1 --- /dev/null +++ b/pkg/connector/tool_modes.go @@ -0,0 +1,60 @@ +package connector + +import "strings" + +const ( + toolModeOff = "off" + toolModeBeeper = "beeper" + toolModeNative = "native" + + defaultSearchMode = toolModeBeeper + defaultFetchMode = toolModeBeeper +) + +func roomSearchMode(config RoomConfig) string { + if config.SearchMode != "" { + return normalizedToolMode(config.SearchMode, defaultSearchMode) + } + return searchModeFromDisabled(config.DisabledTools) +} + +func roomFetchMode(config RoomConfig) string { + if config.FetchMode != "" { + return normalizedToolMode(config.FetchMode, defaultFetchMode) + } + return fetchModeFromDisabled(config.DisabledTools) +} + +func normalizedToolMode(value string, fallback string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case toolModeOff, toolModeBeeper, toolModeNative: + return strings.ToLower(strings.TrimSpace(value)) + default: + return fallback + } +} + +func searchModeFromDisabled(disabled []string) string { + if toolDisabled(disabled, "web_search") || toolDisabled(disabled, "search") { + return toolModeOff + } + return defaultSearchMode +} + +func fetchModeFromDisabled(disabled []string) string { + if toolDisabled(disabled, "fetch") { + return toolModeOff + } + return defaultFetchMode +} + +func toolModeStateContent(config RoomConfig) map[string]any { + content := map[string]any{ + "search": roomSearchMode(config), + "fetch": roomFetchMode(config), + } + if len(config.DisabledTools) > 0 { + content["disabled"] = config.DisabledTools + } + return content +} From d4684b0cc1c745740624b938a2e9379da8a65a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 17:34:22 +0200 Subject: [PATCH 04/22] imrpove search --- README.md | 4 +- pkg/agent/agent_loop.go | 9 ++- pkg/ai/providers/anthropic.go | 5 ++ pkg/ai/providers/anthropic_test.go | 12 +++ pkg/ai/providers/citations.go | 49 +++++++++++- pkg/ai/providers/openai_conversion_test.go | 33 ++++++++ pkg/ai/providers/openai_shared.go | 91 +++++++++++++++++++++- pkg/ai/providers/openai_stream_test.go | 59 ++++++++++++++ pkg/chattools/chattools_test.go | 26 +++---- pkg/chattools/search.go | 31 ++++++++ pkg/chattools/session.go | 6 +- pkg/chattools/types.go | 30 +++---- pkg/connector/builtin_tools.go | 71 ++++++++++++++++- pkg/connector/builtin_tools_test.go | 84 ++++++++++++++++++++ pkg/connector/chat_tools.go | 24 +++--- pkg/connector/client.go | 8 ++ pkg/connector/room_state_test.go | 3 + pkg/connector/slash_commands.go | 2 +- pkg/connector/slash_commands_limits.go | 76 ++++++------------ pkg/connector/slash_commands_test.go | 80 +++++++++++++++---- pkg/connector/slash_commands_tools.go | 4 +- pkg/connector/sources.go | 10 ++- pkg/connector/sources_test.go | 26 +++++++ 23 files changed, 616 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index b88c693d..321f7d16 100644 --- a/README.md +++ b/README.md @@ -375,10 +375,10 @@ Errors are typed with codes (`pkg/agent/harness/public_errors.go`): `CompactionE | Tool | Purpose | Notes | |------|---------|-------| | `get_session` | Live chat metadata (current time/timezone, model, reasoning, search/fetch modes, attachments) | read-only; recomputes time per call | -| `fetch` | Fetch a full HTTP/HTTPS URL → readable text + metadata | direct fetch (≤2 MiB, ≤20 000 chars) or AI-services `/tools/fetch` extraction with fallback | +| `fetch` | Fetch a full HTTP/HTTPS URL → readable text + metadata | Beeper mode: direct fetch (≤2 MiB, ≤20 000 chars) or AI-services `/tools/fetch` extraction with fallback; native mode: provider URL-context/fetch tool when available | | `web_search` | Web search | Exa-backed Beeper search, enabled when room search mode is `beeper`; returns concise URL results for optional follow-up `fetch` calls | -Tools are gated per-room via the `com.beeper.ai.tools` state event. `search` may be `off`, `beeper`, or `native`; `fetch` may be `off` or `beeper`. The legacy `disabled` array is still read for older room state. In `beeper` mode, web tools route through AI-services (`/tools/web_search`, `/tools/fetch`) using the appservice bearer token. In `native` search mode, provider-native `web_search` is injected for models that advertise it; if a provider/model has no native web search, search is unavailable. Search result URLs stay in the tool view; fetched pages, provider-native citation annotations, and final-answer URLs become canonical `com.beeper.source` artifacts for client source cards. Other provider-native built-ins, such as `image_generation`, are still injected from the model catalog (`pkg/connector/builtin_tools.go`). +Tools are gated per-room via the `com.beeper.ai.tools` state event. `search` may be `off`, `beeper`, or `native`; `fetch` may be `off`, `beeper`, or `native`. The legacy `disabled` array is still read for older room state. In `beeper` mode, web tools route through AI-services (`/tools/web_search`, `/tools/fetch`) using the appservice bearer token. In `native` mode, provider-native tools are injected where supported: OpenAI/OpenRouter search, OpenRouter fetch, Anthropic search/fetch, and Google search/URL context. If the selected provider API has no native equivalent, that native tool is unavailable. Search result URLs stay in the tool view; fetched pages, provider-native citation annotations, URL-context metadata, and final-answer URLs become canonical `com.beeper.source` artifacts for client source cards. Other provider-native built-ins, such as `image_generation`, are still injected from the model catalog (`pkg/connector/builtin_tools.go`). `fetch` tries the URL directly first with `Accept` preferring Markdown, plain text, JSON, XML, and CSV. If the response is already agent-readable (Markdown/plain/JSON/XML/CSV/source-ish), it returns that result without backend extraction. If the response is HTML, it checks HTTP `Link` headers and HTML `` for a readable alternate and fetches that directly. Only when the direct representation is not agent-ready does it call AI-services `/tools/fetch`. Local/private hosts, GitHub raw/gist URLs, GitLab-style raw paths, and source/text file extensions are treated as direct-fetch candidates. diff --git a/pkg/agent/agent_loop.go b/pkg/agent/agent_loop.go index bdf79a88..eee83945 100644 --- a/pkg/agent/agent_loop.go +++ b/pkg/agent/agent_loop.go @@ -595,7 +595,14 @@ func hasSequentialToolCall(currentContext *AgentContext, toolCalls []AgentToolCa } func createErrorToolResult(message string) AgentToolResult[any] { - return AgentToolResult[any]{Content: []ai.ContentBlock{{Type: "text", Text: message}}, Details: map[string]any{}} + return AgentToolResult[any]{ + Content: []ai.ContentBlock{{Type: "text", Text: message}}, + Details: map[string]any{ + "state": "error", + "status": "failed", + "reason": message, + }, + } } func shouldTerminateToolBatch(results []AgentToolResult[any]) bool { diff --git a/pkg/ai/providers/anthropic.go b/pkg/ai/providers/anthropic.go index 116ef63c..2ae16757 100644 --- a/pkg/ai/providers/anthropic.go +++ b/pkg/ai/providers/anthropic.go @@ -19,6 +19,8 @@ import ( const ( fineGrainedToolStreamingBeta = "fine-grained-tool-streaming-2025-05-14" interleavedThinkingBeta = "interleaved-thinking-2025-05-14" + webFetchBeta = "web-fetch-2025-09-10" + webFetchBetaMetadataKey = "anthropic_web_fetch_beta" claudeCodeVersion = "2.1.75" ) @@ -384,6 +386,9 @@ func anthropicHeaders(model ai.Model, llmContext ai.Context, options AnthropicOp if interleaved && !getAnthropicCompat(model).ForceAdaptiveThinking { betas = append(betas, interleavedThinkingBeta) } + if enabled, _ := options.Metadata[webFetchBetaMetadataKey].(bool); enabled { + betas = append(betas, webFetchBeta) + } if isBeeperAIProxyBaseURL(model.BaseURL) { headers["Authorization"] = "Bearer " + options.APIKey if len(betas) > 0 { diff --git a/pkg/ai/providers/anthropic_test.go b/pkg/ai/providers/anthropic_test.go index 7de8d5c1..ec4a7b05 100644 --- a/pkg/ai/providers/anthropic_test.go +++ b/pkg/ai/providers/anthropic_test.go @@ -1,6 +1,7 @@ package providers import ( + "strings" "testing" ai "github.com/beeper/ai-bridge/pkg/ai" @@ -46,3 +47,14 @@ func TestAnthropicBeeperProxyUsesBearerAuth(t *testing.T) { t.Fatalf("did not expect upstream Anthropic API key header for Beeper proxy: %#v", headers) } } + +func TestAnthropicHeadersIncludeWebFetchBetaFromMetadata(t *testing.T) { + model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic} + headers := anthropicHeaders(model, ai.Context{}, AnthropicOptions{StreamOptions: ai.StreamOptions{ + APIKey: "token", + Metadata: map[string]any{webFetchBetaMetadataKey: true}, + }}, false) + if !strings.Contains(headers["Anthropic-Beta"], webFetchBeta) || !strings.Contains(headers["Anthropic-Beta"], interleavedThinkingBeta) { + t.Fatalf("expected web fetch beta to compose with generated betas, got %#v", headers) + } +} diff --git a/pkg/ai/providers/citations.go b/pkg/ai/providers/citations.go index 319af5b1..6b7d8dc3 100644 --- a/pkg/ai/providers/citations.go +++ b/pkg/ai/providers/citations.go @@ -12,6 +12,7 @@ func providerCitationsFromAny(value any, provider ai.Provider, contentIndex int) out := []ai.Citation{} if data, ok := value.(map[string]any); ok { out = append(out, googleGroundingCitationsFromMap(data, provider, contentIndex)...) + out = append(out, googleURLContextCitationsFromMap(data, provider, contentIndex)...) } walkProviderCitationMaps(value, func(item map[string]any) { if citation, ok := providerCitationFromMap(item, provider, contentIndex); ok { @@ -98,6 +99,44 @@ func googleGroundingCitationsFromMap(data map[string]any, provider ai.Provider, return out } +func googleURLContextCitationsFromMap(data map[string]any, provider ai.Provider, contentIndex int) []ai.Citation { + metadata, _ := data["urlContextMetadata"].(map[string]any) + if metadata == nil { + metadata, _ = data["url_context_metadata"].(map[string]any) + } + if metadata == nil { + if candidates, _ := data["candidates"].([]any); len(candidates) > 0 { + if candidate, _ := candidates[0].(map[string]any); candidate != nil { + return googleURLContextCitationsFromMap(candidate, provider, contentIndex) + } + } + return nil + } + urls, _ := metadata["urlMetadata"].([]any) + if urls == nil { + urls, _ = metadata["url_metadata"].([]any) + } + out := []ai.Citation{} + for _, rawURL := range urls { + item, _ := rawURL.(map[string]any) + if item == nil { + continue + } + url := firstCitationString(stringFromAny(item["retrievedUrl"]), stringFromAny(item["retrieved_url"]), stringFromAny(item["url"])) + if url == "" { + continue + } + out = append(out, ai.Citation{ + Type: "url_citation", + URL: url, + ContentIndex: &contentIndex, + Provider: string(provider), + RawType: "url_context", + }) + } + return out +} + func citationIndexList(value any) []int { raw, ok := value.([]any) if !ok { @@ -141,9 +180,15 @@ func providerCitationFromMap(data map[string]any, provider ai.Provider, contentI } rawType = firstCitationString(rawType, strings.ToLower(stringFromAny(citationData["type"]))) url := firstCitationString(stringFromAny(citationData["url"]), stringFromAny(citationData["uri"])) - if url == "" || (!strings.Contains(rawType, "citation") && rawType != "web_search_result_location") { + if url == "" || (!strings.Contains(rawType, "citation") && rawType != "web_search_result_location" && rawType != "web_fetch_result") { return ai.Citation{}, false } + title := stringFromAny(citationData["title"]) + if title == "" && rawType == "web_fetch_result" { + if content, _ := citationData["content"].(map[string]any); content != nil { + title = stringFromAny(content["title"]) + } + } resolvedContentIndex := contentIndex if index, ok := intFromCitationAny(firstCitationAny(citationData, "contentIndex", "content_index", "outputIndex", "output_index")); ok { resolvedContentIndex = index @@ -151,7 +196,7 @@ func providerCitationFromMap(data map[string]any, provider ai.Provider, contentI citation := ai.Citation{ Type: "url_citation", URL: url, - Title: stringFromAny(citationData["title"]), + Title: title, Description: firstCitationString(stringFromAny(citationData["description"]), stringFromAny(citationData["summary"])), SiteName: firstCitationString(stringFromAny(citationData["siteName"]), stringFromAny(citationData["site_name"])), FaviconURL: firstCitationString(stringFromAny(citationData["faviconUrl"]), stringFromAny(citationData["favicon_url"])), diff --git a/pkg/ai/providers/openai_conversion_test.go b/pkg/ai/providers/openai_conversion_test.go index 8ed681d9..8256f469 100644 --- a/pkg/ai/providers/openai_conversion_test.go +++ b/pkg/ai/providers/openai_conversion_test.go @@ -177,6 +177,39 @@ func TestProviderCitationsFromAnthropicWebSearchLocation(t *testing.T) { } } +func TestProviderCitationsFromAnthropicWebFetchResult(t *testing.T) { + citations := providerCitationsFromAny(map[string]any{ + "type": "web_fetch_tool_result", + "content": map[string]any{ + "type": "web_fetch_result", + "url": "https://example.com/article", + "content": map[string]any{ + "type": "document", + "title": "Fetched Article", + }, + }, + }, ai.ProviderAnthropic, 0) + if len(citations) != 1 || citations[0].URL != "https://example.com/article" || citations[0].Title != "Fetched Article" || citations[0].RawType != "web_fetch_result" { + t.Fatalf("unexpected web fetch citations %#v", citations) + } +} + +func TestProviderCitationsFromGoogleURLContextMetadata(t *testing.T) { + citations := providerCitationsFromAny(map[string]any{ + "candidates": []any{map[string]any{ + "urlContextMetadata": map[string]any{ + "urlMetadata": []any{map[string]any{ + "retrievedUrl": "https://example.com/url-context", + "urlRetrievalStatus": "URL_RETRIEVAL_STATUS_SUCCESS", + }}, + }, + }}, + }, ai.ProviderGoogle, 0) + if len(citations) != 1 || citations[0].URL != "https://example.com/url-context" || citations[0].RawType != "url_context" { + t.Fatalf("unexpected URL context citations %#v", citations) + } +} + func TestConvertResponsesMessagesIncludesNativeAudio(t *testing.T) { model := ai.Model{ID: "gpt-audio", API: ai.ApiOpenAIResponses, Provider: "openai", Input: []string{"text", "audio"}} messages := ConvertResponsesMessages(model, ai.Context{ diff --git a/pkg/ai/providers/openai_shared.go b/pkg/ai/providers/openai_shared.go index 6b7c8317..f48d395f 100644 --- a/pkg/ai/providers/openai_shared.go +++ b/pkg/ai/providers/openai_shared.go @@ -753,10 +753,78 @@ type responsesStreamState struct { currentMessagePartType string hasReasoningSummaryPart bool toolArgsByIndex map[int]string + nativeToolsByItemID map[string]ai.ToolCall } func newResponsesStreamState() *responsesStreamState { - return &responsesStreamState{currentIndex: -1, toolArgsByIndex: map[int]string{}} + return &responsesStreamState{currentIndex: -1, toolArgsByIndex: map[int]string{}, nativeToolsByItemID: map[string]ai.ToolCall{}} +} + +func responsesItemID(item map[string]any) string { + return strings.TrimSpace(stringFromAny(item["id"])) +} + +func responsesNativeWebSearchToolCall(item map[string]any, fallbackIndex int) ai.ToolCall { + id := responsesItemID(item) + if id == "" { + id = fmt.Sprintf("native_web_search_%d", fallbackIndex+1) + } + return ai.ToolCall{ + Type: "toolCall", + ID: id, + Name: "web_search", + Arguments: responsesNativeWebSearchArguments(item), + } +} + +func responsesNativeWebSearchArguments(item map[string]any) map[string]any { + args := map[string]any{} + addNativeWebSearchQuery(args, item) + if action, _ := item["action"].(map[string]any); action != nil { + addNativeWebSearchQuery(args, action) + if actionType := strings.TrimSpace(stringFromAny(action["type"])); actionType != "" { + args["action"] = actionType + } + } + return args +} + +func addNativeWebSearchQuery(args map[string]any, data map[string]any) { + if _, ok := args["query"]; ok { + return + } + for _, key := range []string{"query", "search_query", "searchQuery"} { + if value := strings.TrimSpace(stringFromAny(data[key])); value != "" { + args["query"] = value + return + } + } +} + +func responsesNativeToolResult(item map[string]any) map[string]any { + status := strings.TrimSpace(stringFromAny(item["status"])) + result := map[string]any{ + "state": "complete", + "status": "success", + "provider": "openai", + "native": true, + } + if status != "" { + result["providerStatus"] = status + } + if status == "failed" || status == "incomplete" { + result["state"] = "error" + result["status"] = "failed" + if errorData, _ := item["error"].(map[string]any); errorData != nil { + if message := strings.TrimSpace(stringFromAny(errorData["message"])); message != "" { + result["reason"] = message + } + } + if result["reason"] == nil { + result["reason"] = "Provider-native web search failed" + } + } + return result } func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, output *ai.Message, model ai.Model, options OpenAIResponsesOptions, event map[string]any) { @@ -805,6 +873,15 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out s.blocks = append(s.blocks, imageBlockFromGenerationItem(item, ai.ContentBlock{})) s.currentIndex = len(s.blocks) - 1 output.Content = s.blocks + case "web_search_call": + toolCall := responsesNativeWebSearchToolCall(item, len(s.nativeToolsByItemID)) + s.nativeToolsByItemID[responsesItemID(item)] = toolCall + s.currentIndex = -1 + output.Content = s.blocks + push(ai.AssistantMessageEvent{Type: "toolcall_start", ToolCall: &toolCall, Partial: output}) + if len(toolCall.Arguments) > 0 { + push(ai.AssistantMessageEvent{Type: "toolcall_delta", Delta: mustJSON(toolCall.Arguments), ToolCall: &toolCall, Partial: output}) + } } case "response.reasoning_summary_part.added": if s.currentItemType == "reasoning" { @@ -892,6 +969,18 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out case "response.output_item.done": item, _ := event["item"].(map[string]any) itemType, _ := item["type"].(string) + if itemType == "web_search_call" { + toolCall := s.nativeToolsByItemID[responsesItemID(item)] + if toolCall.ID == "" { + toolCall = responsesNativeWebSearchToolCall(item, len(s.nativeToolsByItemID)) + } else if args := responsesNativeWebSearchArguments(item); len(args) > 0 { + toolCall.Arguments = args + } + output.Content = s.blocks + push(ai.AssistantMessageEvent{Type: "toolresult", ToolCall: &toolCall, CustomValue: responsesNativeToolResult(item), Partial: output}) + s.currentIndex = -1 + return + } if s.currentIndex < 0 && itemType != "image_generation_call" { return } diff --git a/pkg/ai/providers/openai_stream_test.go b/pkg/ai/providers/openai_stream_test.go index 63445246..ddcb4c61 100644 --- a/pkg/ai/providers/openai_stream_test.go +++ b/pkg/ai/providers/openai_stream_test.go @@ -167,6 +167,53 @@ func TestResponsesStreamStateFinalizesReasoningTextToolAndUsage(t *testing.T) { } } +func TestResponsesStreamStateMapsNativeWebSearchCallToToolActivity(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := testStreamModel() + model.API = ai.ApiOpenAIResponses + output := newAssistant(model) + state := newResponsesStreamState() + + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "item": map[string]any{ + "type": "web_search_call", + "id": "ws_1", + "status": "in_progress", + "action": map[string]any{"type": "search", "query": "latest headlines Amsterdam news today"}, + }, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.done", + "item": map[string]any{ + "type": "web_search_call", + "id": "ws_1", + "status": "completed", + "action": map[string]any{"type": "search", "query": "latest headlines Amsterdam news today"}, + }, + }) + + events := drainAssistantEvents(stream) + if len(output.Content.([]ai.ContentBlock)) != 0 { + t.Fatalf("native web search must not become an executable tool block, got %#v", output.Content) + } + var started, result bool + for _, event := range events { + switch event.Type { + case "toolcall_start": + started = event.ToolCall != nil && event.ToolCall.ID == "ws_1" && event.ToolCall.Name == "web_search" + case "toolresult": + result = event.ToolCall != nil && event.ToolCall.ID == "ws_1" + if output, _ := event.CustomValue.(map[string]any); output["status"] != "success" || output["native"] != true { + t.Fatalf("unexpected native search result payload %#v", event.CustomValue) + } + } + } + if !started || !result { + t.Fatalf("expected native web search start/result events, got %#v", events) + } +} + func TestResponsesStreamStateStreamsTextDeltasWithoutContentPartPrelude(t *testing.T) { stream := ai.NewAssistantMessageEventStream() model := testStreamModel() @@ -219,6 +266,18 @@ func TestResponsesStreamStateStreamsTextDeltasWithoutContentPartPrelude(t *testi } } +func drainAssistantEvents(stream *ai.AssistantMessageEventStream) []ai.AssistantMessageEvent { + events := []ai.AssistantMessageEvent{} + for { + select { + case event := <-stream.Events(): + events = append(events, event) + default: + return events + } + } +} + func TestResponsesStreamStateFinalizesImageGenerationCall(t *testing.T) { stream := ai.NewAssistantMessageEventStream() model := testStreamModel() diff --git a/pkg/chattools/chattools_test.go b/pkg/chattools/chattools_test.go index 7203c357..85698696 100644 --- a/pkg/chattools/chattools_test.go +++ b/pkg/chattools/chattools_test.go @@ -28,16 +28,16 @@ func TestGetSessionSchemaUsesArrayRequired(t *testing.T) { func TestGetSessionReturnsFreshMetadata(t *testing.T) { tool := GetSessionTool(SessionInfo{ - ChatID: "session-1", - ChatTitle: "Markdown Chaos Test", - ChatFirstMessageAt: "2026-05-31T22:00:00Z", - SelectedModel: "gpt-5", - SelectedReasoning: "low", - DisabledTools: []string{"web_search"}, - SearchMode: "off", - FetchMode: "beeper", - LastKnownTimestamp: "2026-05-31T22:34:00Z", - LastKnownTimezone: "Europe/Amsterdam", + ChatID: "session-1", + ChatTitle: "Markdown Chaos Test", + ChatFirstMessageAt: "2026-05-31T22:00:00Z", + SelectedModel: "gpt-5", + SelectedReasoning: "low", + DisabledTools: []string{"web_search"}, + SearchMode: "off", + FetchMode: "beeper", + LastMessageTimestamp: "2026-05-31T22:34:00Z", + LastKnownTimezone: "Europe/Amsterdam", }) result, err := tool.Execute(context.Background(), "call", map[string]any{}, nil) if err != nil { @@ -47,10 +47,10 @@ func TestGetSessionReturnsFreshMetadata(t *testing.T) { if err := json.Unmarshal([]byte(result.Content[0].Text), &info); err != nil { t.Fatal(err) } - if info.CurrentTimestamp == "" || info.LastKnownTimestamp != "2026-05-31T22:34:00Z" || info.LastKnownTimezone != "Europe/Amsterdam" || info.ChatID != "session-1" || info.ChatTitle != "Markdown Chaos Test" || info.SelectedModel != "gpt-5" || len(info.DisabledTools) != 1 || info.DisabledTools[0] != "web_search" || info.SearchMode != "off" || info.FetchMode != "beeper" { + if info.CurrentTimestamp == "" || info.LastMessageTimestamp != "2026-05-31T22:34:00Z" || info.LastKnownTimezone != "Europe/Amsterdam" || info.ChatID != "session-1" || info.ChatTitle != "Markdown Chaos Test" || info.SelectedModel != "gpt-5" || len(info.DisabledTools) != 1 || info.DisabledTools[0] != "web_search" || info.SearchMode != "off" || info.FetchMode != "beeper" { t.Fatalf("expected fresh session metadata, got %#v", info) } - assertSessionKeys(t, result.Content[0].Text, "current_timestamp", "chat_id", "chat_title", "chat_first_message_at", "selected_model", "selected_reasoning", "disabled_tools", "search_mode", "fetch_mode", "last_known_timestamp", "last_known_timezone") + assertSessionKeys(t, result.Content[0].Text, "current_timestamp", "chat_id", "chat_title", "chat_first_message_at", "selected_model", "selected_reasoning", "disabled_tools", "search_mode", "fetch_mode", "last_message_timestamp", "last_known_timezone") } func TestGetSessionIncludesProfileOnlyWhenResolverReturnsIt(t *testing.T) { @@ -73,7 +73,7 @@ func TestGetSessionIncludesProfileOnlyWhenResolverReturnsIt(t *testing.T) { if info.BeeperAccountEmail != "user@example.com" || info.BeeperUsername != "user" || info.BeeperDisplayName != "User Name" || info.GravatarProfile == nil { t.Fatalf("missing approved profile: %#v", info) } - assertSessionKeys(t, result.Content[0].Text, "current_timestamp", "chat_id", "beeper_username", "beeper_display_name", "beeper_account_email", "gravatar_profile", "last_known_timestamp") + assertSessionKeys(t, result.Content[0].Text, "current_timestamp", "chat_id", "beeper_username", "beeper_display_name", "beeper_account_email", "gravatar_profile", "last_message_timestamp") baseline := GetSessionToolWithOptions(SessionInfo{ChatID: "session-1"}, SessionOptions{ ResolveProfile: func(ctx context.Context, toolCallID string) (*SessionProfile, error) { diff --git a/pkg/chattools/search.go b/pkg/chattools/search.go index 68c7f58a..9f9f323e 100644 --- a/pkg/chattools/search.go +++ b/pkg/chattools/search.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "strings" "time" agent "github.com/beeper/ai-bridge/pkg/agent" @@ -89,6 +90,10 @@ func Search(ctx context.Context, query string, limit int, request SearchRequestO defer resp.Body.Close() logToolHTTPResponse(log, resp, time.Since(started), "Received AI tool HTTP response") if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024)) + if message := errorMessageFromBody(body); message != "" { + return SearchResult{}, fmt.Errorf("search failed with HTTP %d: %s", resp.StatusCode, message) + } return SearchResult{}, fmt.Errorf("search failed with HTTP %d", resp.StatusCode) } var body searchResponse @@ -111,6 +116,32 @@ func Search(ctx context.Context, query string, limit int, request SearchRequestO return result, nil } +func errorMessageFromBody(body []byte) string { + data := map[string]any{} + if err := json.Unmarshal(body, &data); err != nil { + return strings.TrimSpace(string(body)) + } + if value := stringFromAnyValue(data["error"]); value != "" { + return value + } + if errorData, _ := data["error"].(map[string]any); errorData != nil { + if value := stringFromAnyValue(errorData["message"]); value != "" { + return value + } + } + if value := stringFromAnyValue(data["message"]); value != "" { + return value + } + return "" +} + +func stringFromAnyValue(value any) string { + if text, ok := value.(string); ok { + return strings.TrimSpace(text) + } + return "" +} + type searchResponse struct { Query string `json:"query"` RequestID string `json:"requestId"` diff --git a/pkg/chattools/session.go b/pkg/chattools/session.go index 5acdfc6c..02edc403 100644 --- a/pkg/chattools/session.go +++ b/pkg/chattools/session.go @@ -16,15 +16,15 @@ func GetSessionToolWithOptions(info SessionInfo, options SessionOptions) agent.A return agent.AgentTool[any]{ Tool: ai.Tool{ Name: "get_session", - Description: "Get fresh metadata for this Beeper AI chat, including current UTC timestamp, chat ID/title, selected model, selected reasoning, disabled tools, approved profile fields, last known UTC timestamp, and last known timezone.", + Description: "Get fresh metadata for this Beeper AI chat, including current UTC timestamp, chat ID/title, selected model, selected reasoning, disabled tools, approved profile fields, last message UTC timestamp, and last known timezone.", Parameters: objectSchema(nil, nil), }, Execute: func(ctx context.Context, toolCallID string, params any, onUpdate agent.AgentToolUpdateCallback[any]) (agent.AgentToolResult[any], error) { now := time.Now().UTC() current := info current.CurrentTimestamp = now.Format(time.RFC3339) - if current.LastKnownTimestamp == "" { - current.LastKnownTimestamp = current.CurrentTimestamp + if current.LastMessageTimestamp == "" { + current.LastMessageTimestamp = current.CurrentTimestamp } if options.ResolveProfile != nil { profile, err := options.ResolveProfile(ctx, toolCallID) diff --git a/pkg/chattools/types.go b/pkg/chattools/types.go index 7ac6eea7..ee31af60 100644 --- a/pkg/chattools/types.go +++ b/pkg/chattools/types.go @@ -7,21 +7,21 @@ import ( ) type SessionInfo struct { - CurrentTimestamp string `json:"current_timestamp"` - ChatID string `json:"chat_id,omitempty"` - ChatTitle string `json:"chat_title,omitempty"` - ChatFirstMessageAt string `json:"chat_first_message_at,omitempty"` - SelectedModel string `json:"selected_model,omitempty"` - SelectedReasoning string `json:"selected_reasoning,omitempty"` - DisabledTools []string `json:"disabled_tools,omitempty"` - SearchMode string `json:"search_mode,omitempty"` - FetchMode string `json:"fetch_mode,omitempty"` - BeeperUsername string `json:"beeper_username,omitempty"` - BeeperDisplayName string `json:"beeper_display_name,omitempty"` - BeeperAccountEmail string `json:"beeper_account_email,omitempty"` - GravatarProfile any `json:"gravatar_profile,omitempty"` - LastKnownTimestamp string `json:"last_known_timestamp"` - LastKnownTimezone string `json:"last_known_timezone,omitempty"` + CurrentTimestamp string `json:"current_timestamp"` + ChatID string `json:"chat_id,omitempty"` + ChatTitle string `json:"chat_title,omitempty"` + ChatFirstMessageAt string `json:"chat_first_message_at,omitempty"` + SelectedModel string `json:"selected_model,omitempty"` + SelectedReasoning string `json:"selected_reasoning,omitempty"` + DisabledTools []string `json:"disabled_tools,omitempty"` + SearchMode string `json:"search_mode,omitempty"` + FetchMode string `json:"fetch_mode,omitempty"` + BeeperUsername string `json:"beeper_username,omitempty"` + BeeperDisplayName string `json:"beeper_display_name,omitempty"` + BeeperAccountEmail string `json:"beeper_account_email,omitempty"` + GravatarProfile any `json:"gravatar_profile,omitempty"` + LastMessageTimestamp string `json:"last_message_timestamp"` + LastKnownTimezone string `json:"last_known_timezone,omitempty"` } type SessionProfile struct { diff --git a/pkg/connector/builtin_tools.go b/pkg/connector/builtin_tools.go index 5a9797c9..794eb910 100644 --- a/pkg/connector/builtin_tools.go +++ b/pkg/connector/builtin_tools.go @@ -10,10 +10,25 @@ import ( "github.com/beeper/ai-bridge/pkg/ai" ) +const anthropicWebFetchBetaMetadataKey = "anthropic_web_fetch_beta" + func (cl *Client) registerProviderBuiltInToolHooks(agentHarness *harness.AgentHarness, roomConfig RoomConfig) { if agentHarness == nil { return } + agentHarness.On("before_provider_request", func(_ context.Context, event harness.AgentHarnessEvent) (any, error) { + if event.Model == nil || roomFetchMode(roomConfig) != toolModeNative || event.Model.API != ai.ApiAnthropicMessages { + return nil, nil + } + if _, ok := nativeWebFetchToolPayload(*event.Model); !ok { + return nil, nil + } + return harness.BeforeProviderRequestResult{ + StreamOptions: harness.AgentHarnessStreamOptions{ + Metadata: map[string]any{anthropicWebFetchBetaMetadataKey: true}, + }, + }, nil + }) agentHarness.On("before_provider_payload", func(_ context.Context, event harness.AgentHarnessEvent) (any, error) { if event.Model == nil { return nil, nil @@ -27,13 +42,23 @@ func (cl *Client) registerProviderBuiltInToolHooks(agentHarness *harness.AgentHa } func activeBuiltInToolPayloads(model ai.Model, roomConfig RoomConfig) []map[string]any { - out := make([]map[string]any, 0, len(model.BuiltInTools)) + out := make([]map[string]any, 0, len(model.BuiltInTools)+2) + if roomSearchMode(roomConfig) == toolModeNative { + if payload, ok := nativeWebSearchToolPayload(model); ok { + out = appendBuiltInToolPayload(out, payload) + } + } + if roomFetchMode(roomConfig) == toolModeNative { + if payload, ok := nativeWebFetchToolPayload(model); ok { + out = appendBuiltInToolPayload(out, payload) + } + } for _, tool := range model.BuiltInTools { payload, ok := builtInToolPayload(model, roomConfig, tool) if !ok { continue } - out = append(out, payload) + out = appendBuiltInToolPayload(out, payload) } return out } @@ -45,6 +70,11 @@ func builtInToolPayload(model ai.Model, roomConfig RoomConfig, tool string) (map return nil, false } return nativeWebSearchToolPayload(model) + case "web_fetch": + if roomFetchMode(roomConfig) != toolModeNative { + return nil, false + } + return nativeWebFetchToolPayload(model) case "image_generation": switch { case strings.HasPrefix(strings.TrimSpace(tool), "openrouter:"): @@ -64,6 +94,8 @@ func normalizedBuiltInTool(tool string) string { switch tool { case "web_search", "web_search_preview", "openrouter:web_search", "web_search_20250305", "google_search", "google_search_retrieval": return "web_search" + case "web_fetch", "openrouter:web_fetch", "web_fetch_20250910", "web_fetch_20260209", "url_context", "urlcontext": + return "web_fetch" case "image_generation", "openrouter:image_generation": return "image_generation" default: @@ -92,6 +124,35 @@ func nativeWebSearchToolPayload(model ai.Model) (map[string]any, bool) { } } +func nativeWebFetchToolPayload(model ai.Model) (map[string]any, bool) { + switch model.API { + case ai.ApiAnthropicMessages: + return map[string]any{"type": "web_fetch_20250910", "name": "web_fetch", "citations": map[string]any{"enabled": true}}, true + case ai.ApiGoogleGenerativeAI, ai.ApiGoogleVertex: + return map[string]any{"url_context": map[string]any{}}, true + case ai.ApiOpenAIResponses, ai.ApiOpenAICompletions: + if model.Provider == ai.ProviderOpenRouter { + return map[string]any{"type": "openrouter:web_fetch"}, true + } + return nil, false + default: + return nil, false + } +} + +func appendBuiltInToolPayload(payloads []map[string]any, payload map[string]any) []map[string]any { + key := builtInToolKey(payload) + if key == "" { + return payloads + } + for _, existing := range payloads { + if builtInToolKey(existing) == key { + return payloads + } + } + return append(payloads, payload) +} + func addBuiltInToolsToPayload(payload any, builtInTools []map[string]any) (any, bool) { body, ok := payload.(map[string]any) if !ok || len(builtInTools) == 0 { @@ -146,6 +207,12 @@ func builtInToolKey(tool map[string]any) string { if _, ok := tool["googleSearch"]; ok { return "google_search" } + if _, ok := tool["url_context"]; ok { + return "url_context" + } + if _, ok := tool["urlContext"]; ok { + return "url_context" + } return "" } diff --git a/pkg/connector/builtin_tools_test.go b/pkg/connector/builtin_tools_test.go index d82f3c22..a385f326 100644 --- a/pkg/connector/builtin_tools_test.go +++ b/pkg/connector/builtin_tools_test.go @@ -76,6 +76,27 @@ func TestActiveBuiltInToolPayloadsHonorsNativeSearchMode(t *testing.T) { } } +func TestActiveBuiltInToolPayloadsHonorsNativeFetchMode(t *testing.T) { + model := ai.Model{API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenRouter, BuiltInTools: []string{"web_fetch", "openrouter:image_generation"}} + if got := activeBuiltInToolPayloads(model, RoomConfig{}); len(got) != 1 || got[0]["type"] != "openrouter:image_generation" { + t.Fatalf("default beeper fetch should suppress native web_fetch only, got %#v", got) + } + if got := activeBuiltInToolPayloads(model, RoomConfig{FetchMode: toolModeNative}); len(got) != 2 || got[0]["type"] != "openrouter:web_fetch" || got[1]["type"] != "openrouter:image_generation" { + t.Fatalf("native fetch should allow provider web_fetch, got %#v", got) + } + if got := activeBuiltInToolPayloads(model, RoomConfig{FetchMode: toolModeOff}); len(got) != 1 || got[0]["type"] != "openrouter:image_generation" { + t.Fatalf("off fetch should suppress native web_fetch only, got %#v", got) + } +} + +func TestActiveBuiltInToolPayloadsInjectsNativeModesWithoutCatalogBuiltIns(t *testing.T) { + model := ai.Model{API: ai.ApiGoogleGenerativeAI, Provider: ai.ProviderGoogle} + got := activeBuiltInToolPayloads(model, RoomConfig{SearchMode: toolModeNative, FetchMode: toolModeNative}) + if len(got) != 2 || builtInToolKey(got[0]) != "google_search" || builtInToolKey(got[1]) != "url_context" { + t.Fatalf("expected mode-driven native Google tools, got %#v", got) + } +} + func TestNativeWebSearchToolPayloadsAreProviderSpecific(t *testing.T) { tests := []struct { name string @@ -127,6 +148,69 @@ func TestNativeWebSearchToolPayloadsAreProviderSpecific(t *testing.T) { } } +func TestNativeWebFetchToolPayloadsAreProviderSpecific(t *testing.T) { + tests := []struct { + name string + model ai.Model + wantKey string + wantValue any + }{ + { + name: "openai responses unsupported", + model: ai.Model{API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenAI}, + wantKey: "", + wantValue: nil, + }, + { + name: "openrouter responses", + model: ai.Model{API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenRouter}, + wantKey: "type", + wantValue: "openrouter:web_fetch", + }, + { + name: "openrouter completions", + model: ai.Model{API: ai.ApiOpenAICompletions, Provider: ai.ProviderOpenRouter}, + wantKey: "type", + wantValue: "openrouter:web_fetch", + }, + { + name: "anthropic", + model: ai.Model{API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic}, + wantKey: "type", + wantValue: "web_fetch_20250910", + }, + { + name: "google", + model: ai.Model{API: ai.ApiGoogleGenerativeAI, Provider: ai.ProviderGoogle}, + wantKey: "url_context", + wantValue: map[string]any{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := activeBuiltInToolPayloads(tt.model, RoomConfig{FetchMode: toolModeNative}) + if tt.wantKey == "" { + if len(got) != 0 { + t.Fatalf("expected no native fetch payload, got %#v", got) + } + return + } + if len(got) != 1 { + t.Fatalf("expected one native fetch payload, got %#v", got) + } + if tt.wantKey == "url_context" { + if _, ok := got[0]["url_context"].(map[string]any); !ok { + t.Fatalf("expected url_context object, got %#v", got[0]) + } + return + } + if got[0][tt.wantKey] != tt.wantValue { + t.Fatalf("unexpected payload %#v", got[0]) + } + }) + } +} + func assertToolType(t *testing.T, raw any, toolType string) { t.Helper() assertToolTypeCount(t, raw, toolType, 1) diff --git a/pkg/connector/chat_tools.go b/pkg/connector/chat_tools.go index a13dac3c..1143f9e3 100644 --- a/pkg/connector/chat_tools.go +++ b/pkg/connector/chat_tools.go @@ -33,19 +33,19 @@ func (cl *Client) chatTools(msg *bridgev2.MatrixMessage, meta *aiid.PortalMetada chatTitle = msg.Portal.Name } info := chattools.SessionInfo{ - ChatID: chatID, - ChatTitle: chatTitle, - ChatFirstMessageAt: chatFirstMessageAt, - SelectedModel: model.ID, - SelectedReasoning: cl.reasoningLevelForModel(model, roomConfig), - DisabledTools: roomConfig.DisabledTools, - SearchMode: roomSearchMode(roomConfig), - FetchMode: roomFetchMode(roomConfig), - LastKnownTimestamp: formatSessionTimestampUTC(matrixEventTime(nil)), - LastKnownTimezone: cl.lastKnownTimezone(), + ChatID: chatID, + ChatTitle: chatTitle, + ChatFirstMessageAt: chatFirstMessageAt, + SelectedModel: model.ID, + SelectedReasoning: cl.reasoningLevelForModel(model, roomConfig), + DisabledTools: roomConfig.DisabledTools, + SearchMode: roomSearchMode(roomConfig), + FetchMode: roomFetchMode(roomConfig), + LastMessageTimestamp: formatSessionTimestampUTC(matrixEventTime(nil)), + LastKnownTimezone: cl.lastKnownTimezone(), } if msg != nil { - info.LastKnownTimestamp = formatSessionTimestampUTC(matrixEventTime(msg.Event)) + info.LastMessageTimestamp = formatSessionTimestampUTC(matrixEventTime(msg.Event)) } search := cl.searchOptions(roomConfig, provider) fetch := chattools.FetchOptions{ @@ -116,7 +116,7 @@ func (cl *Client) searchOptions(roomConfig RoomConfig, provider aiid.ProviderCon Enabled: true, Endpoint: endpoint, APIKey: token, - Timeout: 10 * time.Second, + Timeout: 30 * time.Second, } } diff --git a/pkg/connector/client.go b/pkg/connector/client.go index 05684921..a051f28c 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -1675,6 +1675,14 @@ func applyAIStreamEvent(writer *aistream.Writer, evt ai.AssistantMessageEvent, c if toolCall := toolCallFromEvent(); toolCall != nil { writer.ToolInputComplete(toolCall.ID, toolCall.Name, toolCall.Arguments) } + case "toolresult": + if toolCall := toolCallFromEvent(); toolCall != nil { + result := evt.CustomValue + if result == nil { + result = map[string]any{"state": agui.ToolResultStateComplete, "status": "success"} + } + writer.ToolEnd(toolCall.ID, toolCall.Name, toolCall.Arguments, result) + } case "custom": if evt.CustomName != "" { writer.Custom(evt.CustomName, evt.CustomValue) diff --git a/pkg/connector/room_state_test.go b/pkg/connector/room_state_test.go index 537bc28b..e11c2edd 100644 --- a/pkg/connector/room_state_test.go +++ b/pkg/connector/room_state_test.go @@ -44,6 +44,9 @@ func TestRoomToolModesDefaultAndLegacyDisabled(t *testing.T) { if got := roomSearchMode(RoomConfig{SearchMode: "native", DisabledTools: []string{"web_search"}}); got != toolModeNative { t.Fatalf("explicit search mode should win over disabled list, got %q", got) } + if got := roomFetchMode(RoomConfig{FetchMode: "native", DisabledTools: []string{"fetch"}}); got != toolModeNative { + t.Fatalf("explicit fetch mode should win over disabled list, got %q", got) + } if got := roomFetchMode(RoomConfig{FetchMode: "bad"}); got != defaultFetchMode { t.Fatalf("invalid fetch mode should fall back, got %q", got) } diff --git a/pkg/connector/slash_commands.go b/pkg/connector/slash_commands.go index 3df8ea02..9f4e3e1d 100644 --- a/pkg/connector/slash_commands.go +++ b/pkg/connector/slash_commands.go @@ -98,7 +98,7 @@ func aiSlashCommandDefinitions() []aiSlashCommandDefinition { }, { name: "fetch", - usage: "/fetch [off|beeper]", + usage: "/fetch [off|beeper|native]", description: "Show or set URL fetch mode for this room.", needsRoomConfig: true, noticeErrors: true, diff --git a/pkg/connector/slash_commands_limits.go b/pkg/connector/slash_commands_limits.go index 537b658d..b2415a28 100644 --- a/pkg/connector/slash_commands_limits.go +++ b/pkg/connector/slash_commands_limits.go @@ -56,10 +56,10 @@ func runLimitsCommand(cl *Client, ctx context.Context, _ *bridgev2.Portal, _ Roo return err } now := time.Now() + text := formatLimitsCommandInfo(limits, now) if raw { - return responder.Reply(ctx, formatRawLimitsCommandInfo(limits, now)) + text = formatRawLimitsCommandInfo(limits, now) } - text := formatLimitsCommandInfo(limits, now) if aiResponder, ok := responder.(aiCommandAIResponder); ok { return aiResponder.ReplyAI(ctx, text) } @@ -132,25 +132,16 @@ func aiServicesLimitsURL(proxyBaseURL string) (string, error) { func formatLimitsCommandInfo(limits aiServicesLimitsResponse, now time.Time) string { var text strings.Builder - text.WriteString("AI limits\n\n") appendLimitSection(&text, "Models", limits.Windows.LLM, now) - text.WriteString("\n") - appendLimitSection(&text, "Web Search", limits.Windows.WebTools, now) - if !emptyLimitWindows(limits.Windows.AudioTranscriptions) { - text.WriteString("\n") - appendLimitSectionWithUsedFormatter(&text, "Transcription", limits.Windows.AudioTranscriptions, now, formatLimitUsedMinutes) - } - if !emptyLimitWindows(limits.Windows.AudioGeneration) { - text.WriteString("\n") - appendLimitSectionWithUsedFormatter(&text, "Audio Generation", limits.Windows.AudioGeneration, now, formatLimitUsedCharacters) - } return text.String() } func formatRawLimitsCommandInfo(limits aiServicesLimitsResponse, now time.Time) string { var text strings.Builder - text.WriteString("AI limits raw:") - for _, category := range limitCategories(limits) { + for idx, category := range limitCategories(limits) { + if idx > 0 { + text.WriteString("\n") + } appendRawLimitCategory(&text, category.label, category.windows, now) } return text.String() @@ -162,17 +153,12 @@ type limitCategory struct { } func limitCategories(limits aiServicesLimitsResponse) []limitCategory { - categories := []limitCategory{{label: "LLM tokens", windows: limits.Windows.LLM}} - if !emptyLimitWindows(limits.Windows.WebTools) { - categories = append(categories, limitCategory{label: "Web tools", windows: limits.Windows.WebTools}) - } - if !emptyLimitWindows(limits.Windows.AudioTranscriptions) { - categories = append(categories, limitCategory{label: "Audio transcription seconds", windows: limits.Windows.AudioTranscriptions}) - } - if !emptyLimitWindows(limits.Windows.AudioGeneration) { - categories = append(categories, limitCategory{label: "Audio generation characters", windows: limits.Windows.AudioGeneration}) + return []limitCategory{ + {label: "LLM tokens", windows: limits.Windows.LLM}, + {label: "Web tools", windows: limits.Windows.WebTools}, + {label: "Audio transcription seconds", windows: limits.Windows.AudioTranscriptions}, + {label: "Audio generation characters", windows: limits.Windows.AudioGeneration}, } - return categories } func appendLimitSection(text *strings.Builder, label string, windows aiServicesLimitWindows, now time.Time) { @@ -227,31 +213,6 @@ func formatLimitUsed(window aiServicesLimitWindow) string { return fmt.Sprintf("`%s / %s`", formatInt(window.Used), formatInt(window.Limit)) } -func formatLimitUsedMinutes(window aiServicesLimitWindow) string { - if window.Limit == 0 && window.Used == 0 && window.Remaining == 0 { - return "Not reported" - } - if window.Limit < 0 { - return fmt.Sprintf("`%s` used", formatSecondsAsMinutes(window.Used)) - } - return fmt.Sprintf("`%s / %s`", formatSecondsAsMinutes(window.Used), formatSecondsAsMinutes(window.Limit)) -} - -func formatLimitUsedCharacters(window aiServicesLimitWindow) string { - if window.Limit == 0 && window.Used == 0 && window.Remaining == 0 { - return "Not reported" - } - if window.Limit < 0 { - return fmt.Sprintf("`%s chars` used", formatInt(window.Used)) - } - return fmt.Sprintf("`%s / %s chars`", formatInt(window.Used), formatInt(window.Limit)) -} - -func formatSecondsAsMinutes(seconds int64) string { - minutes := (seconds + 59) / 60 - return fmt.Sprintf("%s %s", formatInt(minutes), pluralize("minute", int(minutes))) -} - func formatLimitReset(window aiServicesLimitWindow, now time.Time) string { if window.ResetAtMS <= 0 { return "unknown" @@ -260,7 +221,9 @@ func formatLimitReset(window aiServicesLimitWindow, now time.Time) string { } func appendRawLimitCategory(text *strings.Builder, label string, windows aiServicesLimitWindows, now time.Time) { - fmt.Fprintf(text, "\n\n%s:", label) + fmt.Fprintf(text, "## %s\n\n", label) + text.WriteString("| Window | Left | Used | Limit | Remaining | Reset |\n") + text.WriteString("| --- | ---: | ---: | ---: | ---: | --- |\n") appendRawLimitWindow(text, "Day", windows.Day, now) appendRawLimitWindow(text, "Week", windows.Week, now) appendRawLimitWindow(text, "Month", windows.Month, now) @@ -269,17 +232,22 @@ func appendRawLimitCategory(text *strings.Builder, label string, windows aiServi func appendRawLimitWindow(text *strings.Builder, label string, window aiServicesLimitWindow, now time.Time) { fmt.Fprintf( text, - "\n- %s: percentage_left=`%d`, limit=`%s`, used=`%s`, remaining=`%s`", + "| %s | `%d%%` | `%s` | `%s` | `%s` | %s |\n", label, window.PercentageLeft, - formatInt(window.Limit), formatInt(window.Used), + formatInt(window.Limit), formatInt(window.Remaining), + formatRawLimitReset(window, now), ) +} + +func formatRawLimitReset(window aiServicesLimitWindow, now time.Time) string { if window.ResetAtMS > 0 { resetAt := time.UnixMilli(window.ResetAtMS) - fmt.Fprintf(text, ", reset_at=`%d` (`%s`, in %s)", window.ResetAtMS, resetAt.UTC().Format(time.RFC3339), formatResetIn(resetAt, now)) + return fmt.Sprintf("`%d` (`%s`, in %s)", window.ResetAtMS, resetAt.UTC().Format(time.RFC3339), formatResetIn(resetAt, now)) } + return "unknown" } func sharedResetAt(categories []limitCategory) (time.Time, bool) { diff --git a/pkg/connector/slash_commands_test.go b/pkg/connector/slash_commands_test.go index ada7dd8a..50dcaf84 100644 --- a/pkg/connector/slash_commands_test.go +++ b/pkg/connector/slash_commands_test.go @@ -460,6 +460,52 @@ func TestFetchAIServicesLimitsUsesAppserviceBearerToken(t *testing.T) { } } +func TestRunLimitsCommandRawUsesAIResponse(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/limits" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + _, _ = w.Write([]byte(`{"windows":{"llm":{"day":{"percentage_left":75,"limit":1000,"used":250,"remaining":750,"reset_at":1893456000000}}}}`)) + })) + defer server.Close() + + provider := aiid.ProviderConfig{ID: aiid.DefaultProvider, BaseURL: server.URL + "/proxy/openai/v1"} + client := &Client{ + Main: &Connector{AppServiceToken: "as-token"}, + UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ + UserMXID: "@alice:beeper.test", + Metadata: &aiid.UserLoginMetadata{Providers: map[string]aiid.ProviderConfig{ + provider.ID: provider, + }}, + }}, + } + responder := &recordingCommandResponder{} + if err := runLimitsCommand(client, context.Background(), nil, RoomConfig{}, "raw", responder); err != nil { + t.Fatal(err) + } + if responder.text != "" { + t.Fatalf("raw limits should use AI response, got plain text %q", responder.text) + } + if !strings.Contains(responder.aiText, "## LLM tokens") || !strings.Contains(responder.aiText, "| Day | `75%` | `250` | `1,000` | `750` |") { + t.Fatalf("raw limits AI response missing table:\n%s", responder.aiText) + } +} + +type recordingCommandResponder struct { + text string + aiText string +} + +func (r *recordingCommandResponder) Reply(_ context.Context, text string) error { + r.text = text + return nil +} + +func (r *recordingCommandResponder) ReplyAI(_ context.Context, text string) error { + r.aiText = text + return nil +} + func TestBeeperUsageLimitErrorUsesPlanResetMessage(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/limits" { @@ -534,24 +580,17 @@ func TestFormatLimitsCommandInfo(t *testing.T) { }, }}, now) for _, want := range []string{ - "AI limits", "## Models", "| Window | Left | Used | Reset |", "| Daily | `75%` | `250 / 1,000` | in 1 day 2 hours 3 minutes |", "| Weekly | Unlimited | `1,234` used | in 1 day 2 hours 3 minutes |", "| Monthly | **Out** | `30,500 / 30,000` | in 1 day 2 hours 3 minutes |", - "## Web Search", - "| Daily | `99%` | `1 / 200,000` | in 1 day 2 hours 3 minutes |", - "## Transcription", - "| Daily | `99%` | `1 minute / 1,440 minutes` | in 1 day 2 hours 3 minutes |", - "## Audio Generation", - "| Daily | `99%` | `1,234 / 50,000 chars` | in 1 day 2 hours 3 minutes |", } { if !strings.Contains(text, want) { t.Fatalf("limits info missing %q:\n%s", want, text) } } - for _, notWant := range []string{"`199,999`", "2030-01-01T00:00:00Z"} { + for _, notWant := range []string{"AI limits", "## Web Search", "## Transcription", "## Audio Generation", "`199,999`", "2030-01-01T00:00:00Z"} { if strings.Contains(text, notWant) { t.Fatalf("limits info exposed non-summary value %q:\n%s", notWant, text) } @@ -572,15 +611,15 @@ func TestFormatLimitsCommandInfoShowsPerWindowResetsWhenDifferent(t *testing.T) "| Daily | `75%` | Not reported | in 1 day 1 hour 3 minutes |", "| Weekly | `100%` | Not reported | in 7 days |", "| Monthly | **Out** | Not reported | in 31 days |", - "## Web Search", - "No limits reported.", } { if !strings.Contains(text, want) { t.Fatalf("limits info missing %q:\n%s", want, text) } } - if strings.Contains(text, "Everything resets") { - t.Fatalf("limits info collapsed different reset times:\n%s", text) + for _, notWant := range []string{"AI limits", "## Web Search", "No limits reported.", "Everything resets"} { + if strings.Contains(text, notWant) { + t.Fatalf("limits info exposed %q:\n%s", notWant, text) + } } } @@ -614,16 +653,23 @@ func TestFormatRawLimitsCommandInfoShowsExactUsage(t *testing.T) { }, }}, time.Date(2029, 12, 31, 0, 0, 0, 0, time.UTC)) for _, want := range []string{ - "AI limits raw:", - "percentage_left=`75`, limit=`1,000`, used=`250`, remaining=`750`, reset_at=`1893456000000` (`2030-01-01T00:00:00Z`, in 1 day)", - "percentage_left=`100`, limit=`-1`, used=`1,234`, remaining=`-1`", - "Audio transcription seconds:", - "percentage_left=`99`, limit=`86,400`, used=`43`, remaining=`86,357`", + "## LLM tokens", + "| Window | Left | Used | Limit | Remaining | Reset |", + "| Day | `75%` | `250` | `1,000` | `750` | `1893456000000` (`2030-01-01T00:00:00Z`, in 1 day) |", + "| Week | `100%` | `1,234` | `-1` | `-1` | `1893456000000` (`2030-01-01T00:00:00Z`, in 1 day) |", + "## Web tools", + "| Day | `0%` | `0` | `0` | `0` | unknown |", + "## Audio transcription seconds", + "| Day | `99%` | `43` | `86,400` | `86,357` | `1893456000000` (`2030-01-01T00:00:00Z`, in 1 day) |", + "## Audio generation characters", } { if !strings.Contains(text, want) { t.Fatalf("raw limits info missing %q:\n%s", want, text) } } + if strings.Contains(text, "AI limits raw:") { + t.Fatalf("raw limits info should render as tables without the old header:\n%s", text) + } } func TestResolveCanonicalRoomModelUsesDefaultProviderForBareModel(t *testing.T) { diff --git a/pkg/connector/slash_commands_tools.go b/pkg/connector/slash_commands_tools.go index 24dc7a57..d7edba7f 100644 --- a/pkg/connector/slash_commands_tools.go +++ b/pkg/connector/slash_commands_tools.go @@ -30,10 +30,10 @@ func runSearchModeCommand(cl *Client, ctx context.Context, portal *bridgev2.Port func runFetchModeCommand(cl *Client, ctx context.Context, portal *bridgev2.Portal, roomConfig RoomConfig, arg string, responder aiCommandResponder) error { arg = strings.ToLower(strings.TrimSpace(arg)) if arg == "" { - return responder.Reply(ctx, fmt.Sprintf("Current fetch mode is `%s`. Options: `off`, `beeper`.", roomFetchMode(roomConfig))) + return responder.Reply(ctx, fmt.Sprintf("Current fetch mode is `%s`. Options: `off`, `beeper`, `native`.", roomFetchMode(roomConfig))) } mode := normalizedToolMode(arg, "") - if mode != toolModeOff && mode != toolModeBeeper { + if mode != toolModeOff && mode != toolModeBeeper && mode != toolModeNative { return fmt.Errorf("fetch mode %q is invalid", arg) } roomConfig.FetchMode = mode diff --git a/pkg/connector/sources.go b/pkg/connector/sources.go index 74268d33..b3b30f4a 100644 --- a/pkg/connector/sources.go +++ b/pkg/connector/sources.go @@ -580,12 +580,18 @@ func providerCitationSource(data map[string]any) (sourceObservation, bool) { sourceType = firstSourceString(sourceType, strings.ToLower(sourceString(nested, "type", "rawType"))) } rawURL := sourceString(citation, "url", "uri") - if rawURL == "" || (!strings.Contains(sourceType, "citation") && sourceType != "web_search_result_location") { + if rawURL == "" || (!strings.Contains(sourceType, "citation") && sourceType != "web_search_result_location" && sourceType != "web_fetch_result" && sourceType != "url_context") { return sourceObservation{}, false } + title := sourceString(citation, "title") + if title == "" && sourceType == "web_fetch_result" { + if content, _ := citation["content"].(map[string]any); content != nil { + title = sourceString(content, "title") + } + } return sourceObservation{ URL: rawURL, - Title: sourceString(citation, "title"), + Title: title, Description: firstSourceString(sourceDescriptionString(citation), sourceString(citation, "cited_text"), sourceString(citation, "text")), SiteName: sourceString(citation, "siteName", "site_name"), FaviconURL: sourceFaviconString(citation), diff --git a/pkg/connector/sources_test.go b/pkg/connector/sources_test.go index a10a6899..786ae4d5 100644 --- a/pkg/connector/sources_test.go +++ b/pkg/connector/sources_test.go @@ -87,6 +87,21 @@ func TestSourceCollectorUsesDescriptionAndFaviconFallbacks(t *testing.T) { t.Fatalf("provider citation ranges were not preserved: %#v", providerSources[0]) } + fetchResultSources := newSourceCollector().addProviderSources(map[string]any{ + "type": "web_fetch_tool_result", + "content": map[string]any{ + "type": "web_fetch_result", + "url": "https://example.com/provider-fetch", + "content": map[string]any{ + "type": "document", + "title": "Provider Fetch", + }, + }, + }) + if len(fetchResultSources) != 1 || fetchResultSources[0]["title"] != "Provider Fetch" { + t.Fatalf("provider web fetch result was not mapped: %#v", fetchResultSources) + } + messageSources := newSourceCollector().addProviderSources(ai.Message{ Citations: []ai.Citation{{ Type: "url_citation", @@ -100,6 +115,17 @@ func TestSourceCollectorUsesDescriptionAndFaviconFallbacks(t *testing.T) { t.Fatalf("typed provider citation was not mapped: %#v", messageSources) } + urlContextSources := newSourceCollector().addProviderSources(ai.Message{ + Citations: []ai.Citation{{ + Type: "url_citation", + URL: "https://example.com/url-context", + RawType: "url_context", + }}, + }) + if len(urlContextSources) != 1 || urlContextSources[0]["url"] != "https://example.com/url-context" { + t.Fatalf("Google URL context source was not mapped: %#v", urlContextSources) + } + collector := newSourceCollector() searchSources := collector.addWebSearchOutput(toolOutputEvent{ ID: "call-search", From 2d9d75527b3f9083fd232a93b0fbc8409dd42dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 17:44:30 +0200 Subject: [PATCH 05/22] map more --- pkg/agent/agent_loop_test.go | 4 + pkg/ai/providers/anthropic.go | 186 ++++++++++++++++++++++++++++- pkg/ai/providers/anthropic_test.go | 114 ++++++++++++++++++ pkg/chattools/chattools_test.go | 13 ++ pkg/connector/stream_test.go | 43 +++++++ 5 files changed, 355 insertions(+), 5 deletions(-) diff --git a/pkg/agent/agent_loop_test.go b/pkg/agent/agent_loop_test.go index 9b7e61b5..87597c32 100644 --- a/pkg/agent/agent_loop_test.go +++ b/pkg/agent/agent_loop_test.go @@ -328,6 +328,10 @@ func TestRunAgentLoopTurnsToolUpdateEmitErrorIntoToolResult(t *testing.T) { if content[0].Text != "update emit failed" { t.Fatalf("expected update emit error content, got %#v", content) } + details, _ := messages[2].Details.(map[string]any) + if details["state"] != "error" || details["status"] != "failed" || details["reason"] != "update emit failed" { + t.Fatalf("expected structured tool error details, got %#v", messages[2].Details) + } } func TestRunAgentLoopValidatesPreparedToolArguments(t *testing.T) { diff --git a/pkg/ai/providers/anthropic.go b/pkg/ai/providers/anthropic.go index 2ae16757..885a7675 100644 --- a/pkg/ai/providers/anthropic.go +++ b/pkg/ai/providers/anthropic.go @@ -550,12 +550,21 @@ func isAnthropicMessageEvent(event string) bool { } type anthropicStreamState struct { - anthropicIndexes map[int]int - partialJSON map[int]string + anthropicIndexes map[int]int + partialJSON map[int]string + nativeToolsByIndex map[int]ai.ToolCall + nativeToolsByID map[string]ai.ToolCall + nativeToolArgsByIndex map[int]string } func newAnthropicStreamState() *anthropicStreamState { - return &anthropicStreamState{anthropicIndexes: map[int]int{}, partialJSON: map[int]string{}} + return &anthropicStreamState{ + anthropicIndexes: map[int]int{}, + partialJSON: map[int]string{}, + nativeToolsByIndex: map[int]ai.ToolCall{}, + nativeToolsByID: map[string]ai.ToolCall{}, + nativeToolArgsByIndex: map[int]string{}, + } } func (s *anthropicStreamState) apply(stream *ai.AssistantMessageEventStream, output *ai.Message, model ai.Model, llmContext ai.Context, isOAuth bool, event map[string]any) { @@ -576,18 +585,21 @@ func (s *anthropicStreamState) apply(stream *ai.AssistantMessageEventStream, out return } contentIndex := len(output.Content.([]ai.ContentBlock)) - s.anthropicIndexes[index] = contentIndex switch blockMap["type"] { case "text": + s.anthropicIndexes[index] = contentIndex appendContentBlock(output, ai.ContentBlock{Type: "text"}) stream.Push(ai.AssistantMessageEvent{Type: "text_start", ContentIndex: contentIndex, Partial: output}) case "thinking": + s.anthropicIndexes[index] = contentIndex appendContentBlock(output, ai.ContentBlock{Type: "thinking"}) stream.Push(ai.AssistantMessageEvent{Type: "thinking_start", ContentIndex: contentIndex, Partial: output}) case "redacted_thinking": + s.anthropicIndexes[index] = contentIndex appendContentBlock(output, ai.ContentBlock{Type: "thinking", Thinking: "[Reasoning redacted]", ThinkingSignature: stringFromAny(blockMap["data"]), Redacted: true}) stream.Push(ai.AssistantMessageEvent{Type: "thinking_start", ContentIndex: contentIndex, Partial: output}) case "tool_use": + s.anthropicIndexes[index] = contentIndex name := stringFromAny(blockMap["name"]) if isOAuth { name = fromClaudeCodeName(name, llmContext.Tools) @@ -601,8 +613,42 @@ func (s *anthropicStreamState) apply(stream *ai.AssistantMessageEventStream, out } appendContentBlock(output, ai.ContentBlock{Type: "toolCall", ID: stringFromAny(blockMap["id"]), Name: name, Arguments: arguments}) stream.Push(ai.AssistantMessageEvent{Type: "toolcall_start", ContentIndex: contentIndex, Partial: output}) + case "server_tool_use": + toolCall, ok := anthropicNativeServerToolCall(blockMap, index) + if !ok { + return + } + s.nativeToolsByIndex[index] = toolCall + s.nativeToolsByID[toolCall.ID] = toolCall + if len(toolCall.Arguments) > 0 { + s.nativeToolArgsByIndex[index] = mustJSON(toolCall.Arguments) + } + stream.Push(ai.AssistantMessageEvent{Type: "toolcall_start", ToolCall: &toolCall, Partial: output}) + if len(toolCall.Arguments) > 0 { + stream.Push(ai.AssistantMessageEvent{Type: "toolcall_delta", Delta: mustJSON(toolCall.Arguments), ToolCall: &toolCall, Partial: output}) + } + case "web_search_tool_result", "web_fetch_tool_result": + toolCall := s.anthropicNativeToolCallForResult(blockMap) + if toolCall.ID == "" { + return + } + stream.Push(ai.AssistantMessageEvent{Type: "toolresult", ToolCall: &toolCall, CustomValue: anthropicNativeToolResult(blockMap), Partial: output}) } case "content_block_delta": + blockIndex := intFromAny(event["index"]) + if toolCall, ok := s.nativeToolsByIndex[blockIndex]; ok { + delta, _ := event["delta"].(map[string]any) + if delta == nil || delta["type"] != "input_json_delta" { + return + } + part := stringFromAny(delta["partial_json"]) + s.nativeToolArgsByIndex[blockIndex] += part + toolCall.Arguments = parseJSONMap(s.nativeToolArgsByIndex[blockIndex]) + s.nativeToolsByIndex[blockIndex] = toolCall + s.nativeToolsByID[toolCall.ID] = toolCall + stream.Push(ai.AssistantMessageEvent{Type: "toolcall_delta", Delta: part, ToolCall: &toolCall, Partial: output}) + return + } contentIndex, ok := s.anthropicIndexes[intFromAny(event["index"])] if !ok { return @@ -634,7 +680,14 @@ func (s *anthropicStreamState) apply(stream *ai.AssistantMessageEventStream, out stream.Push(ai.AssistantMessageEvent{Type: "toolcall_delta", ContentIndex: contentIndex, Delta: part, Partial: output}) } case "content_block_stop": - contentIndex, ok := s.anthropicIndexes[intFromAny(event["index"])] + blockIndex := intFromAny(event["index"]) + if toolCall, ok := s.nativeToolsByIndex[blockIndex]; ok { + s.nativeToolsByID[toolCall.ID] = toolCall + delete(s.nativeToolsByIndex, blockIndex) + delete(s.nativeToolArgsByIndex, blockIndex) + return + } + contentIndex, ok := s.anthropicIndexes[blockIndex] if !ok { return } @@ -664,6 +717,129 @@ func (s *anthropicStreamState) apply(stream *ai.AssistantMessageEventStream, out } } +func anthropicNativeServerToolCall(block map[string]any, fallbackIndex int) (ai.ToolCall, bool) { + name := anthropicNativeToolName(stringFromAny(block["name"])) + if name == "" { + return ai.ToolCall{}, false + } + id := stringFromAny(block["id"]) + if id == "" { + id = fmt.Sprintf("server_tool_%d", fallbackIndex+1) + } + arguments := map[string]any{} + if input, ok := block["input"]; ok { + arguments = parseJSONMap(mustJSON(input)) + } + return ai.ToolCall{Type: "toolCall", ID: id, Name: name, Arguments: arguments}, true +} + +func (s *anthropicStreamState) anthropicNativeToolCallForResult(block map[string]any) ai.ToolCall { + id := stringFromAny(block["tool_use_id"]) + if id != "" { + if toolCall, ok := s.nativeToolsByID[id]; ok { + return toolCall + } + } + name := anthropicNativeResultToolName(stringFromAny(block["type"])) + if name == "" || id == "" { + return ai.ToolCall{} + } + return ai.ToolCall{Type: "toolCall", ID: id, Name: name, Arguments: map[string]any{}} +} + +func anthropicNativeToolName(name string) string { + switch name { + case "web_search": + return "web_search" + case "web_fetch": + return "fetch" + default: + return "" + } +} + +func anthropicNativeResultToolName(blockType string) string { + switch blockType { + case "web_search_tool_result": + return "web_search" + case "web_fetch_tool_result": + return "fetch" + default: + return "" + } +} + +func anthropicNativeToolResult(block map[string]any) map[string]any { + result := map[string]any{ + "state": "complete", + "status": "success", + "provider": "anthropic", + "native": true, + } + content := block["content"] + if errorCode := anthropicNativeToolErrorCode(content); errorCode != "" { + result["state"] = "error" + result["status"] = "failed" + result["reason"] = errorCode + return result + } + switch block["type"] { + case "web_search_tool_result": + results := anthropicNativeSearchResults(content) + result["results"] = results + result["count"] = len(results) + case "web_fetch_tool_result": + if data, _ := content.(map[string]any); data != nil { + if url := stringFromAny(data["url"]); url != "" { + result["url"] = url + result["final_url"] = url + } + if retrievedAt := stringFromAny(data["retrieved_at"]); retrievedAt != "" { + result["retrieved_at"] = retrievedAt + } + } + } + return result +} + +func anthropicNativeToolErrorCode(content any) string { + data, _ := content.(map[string]any) + if data == nil { + return "" + } + if strings.Contains(stringFromAny(data["type"]), "error") { + if code := stringFromAny(data["error_code"]); code != "" { + return code + } + return stringFromAny(data["type"]) + } + return "" +} + +func anthropicNativeSearchResults(content any) []map[string]any { + items, _ := content.([]any) + results := make([]map[string]any, 0, len(items)) + for _, item := range items { + data, _ := item.(map[string]any) + if data == nil || data["type"] != "web_search_result" { + continue + } + result := map[string]any{} + for _, key := range []string{"url", "title", "page_age"} { + if value := stringFromAny(data[key]); value != "" { + result[key] = value + } + } + if pageAge := stringFromAny(data["page_age"]); pageAge != "" { + result["published_at"] = pageAge + } + if len(result) > 0 { + results = append(results, result) + } + } + return results +} + func appendContentBlock(output *ai.Message, block ai.ContentBlock) { blocks, _ := output.Content.([]ai.ContentBlock) blocks = append(blocks, block) diff --git a/pkg/ai/providers/anthropic_test.go b/pkg/ai/providers/anthropic_test.go index ec4a7b05..23b9e162 100644 --- a/pkg/ai/providers/anthropic_test.go +++ b/pkg/ai/providers/anthropic_test.go @@ -37,6 +37,120 @@ func TestAnthropicStreamStatePreservesToolInputFromContentBlockStart(t *testing. } } +func TestAnthropicStreamStateMapsNativeWebSearchToToolActivity(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic} + output := newAssistant(model) + state := newAnthropicStreamState() + + state.apply(stream, &output, model, ai.Context{}, false, map[string]any{ + "type": "content_block_start", + "index": float64(0), + "content_block": map[string]any{ + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + }, + }) + state.apply(stream, &output, model, ai.Context{}, false, map[string]any{ + "type": "content_block_delta", + "index": float64(0), + "delta": map[string]any{"type": "input_json_delta", "partial_json": `{"query":"latest headlines Amsterdam news today"}`}, + }) + state.apply(stream, &output, model, ai.Context{}, false, map[string]any{ + "type": "content_block_start", + "index": float64(1), + "content_block": map[string]any{ + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": []any{map[string]any{ + "type": "web_search_result", + "url": "https://example.com/news", + "title": "Amsterdam News", + "page_age": "June 1, 2026", + }}, + }, + }) + state.apply(stream, &output, model, ai.Context{}, false, map[string]any{ + "type": "content_block_stop", + "index": float64(1), + }) + + events := drainAssistantEvents(stream) + if len(output.Content.([]ai.ContentBlock)) != 0 { + t.Fatalf("native web search must not become an executable tool block, got %#v", output.Content) + } + var sawStart, sawArgs, sawResult bool + for _, event := range events { + switch event.Type { + case "toolcall_start": + sawStart = event.ToolCall != nil && event.ToolCall.ID == "srvtoolu_1" && event.ToolCall.Name == "web_search" + case "toolcall_delta": + sawArgs = event.ToolCall != nil && event.ToolCall.Arguments["query"] == "latest headlines Amsterdam news today" + case "toolresult": + sawResult = event.ToolCall != nil && event.ToolCall.ID == "srvtoolu_1" + output, _ := event.CustomValue.(map[string]any) + results, _ := output["results"].([]map[string]any) + if output["status"] != "success" || output["native"] != true || len(results) != 1 || results[0]["title"] != "Amsterdam News" { + t.Fatalf("unexpected native search result payload %#v", event.CustomValue) + } + } + } + if !sawStart || !sawArgs || !sawResult { + t.Fatalf("expected native web search lifecycle events, got %#v", events) + } +} + +func TestAnthropicStreamStateMapsNativeWebFetchError(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic} + output := newAssistant(model) + state := newAnthropicStreamState() + + state.apply(stream, &output, model, ai.Context{}, false, map[string]any{ + "type": "content_block_start", + "index": float64(0), + "content_block": map[string]any{ + "type": "server_tool_use", + "id": "srvtoolu_fetch", + "name": "web_fetch", + "input": map[string]any{"url": "https://example.com/article"}, + }, + }) + state.apply(stream, &output, model, ai.Context{}, false, map[string]any{ + "type": "content_block_start", + "index": float64(1), + "content_block": map[string]any{ + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_fetch", + "content": map[string]any{ + "type": "web_fetch_tool_error", + "error_code": "url_not_accessible", + }, + }, + }) + state.apply(stream, &output, model, ai.Context{}, false, map[string]any{ + "type": "content_block_stop", + "index": float64(1), + }) + + events := drainAssistantEvents(stream) + var sawResult bool + for _, event := range events { + if event.Type != "toolresult" { + continue + } + sawResult = event.ToolCall != nil && event.ToolCall.Name == "fetch" + output, _ := event.CustomValue.(map[string]any) + if output["status"] != "failed" || output["reason"] != "url_not_accessible" { + t.Fatalf("unexpected native fetch error payload %#v", event.CustomValue) + } + } + if !sawResult { + t.Fatalf("expected native fetch error result, got %#v", events) + } +} + func TestAnthropicBeeperProxyUsesBearerAuth(t *testing.T) { model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic, BaseURL: "https://ai-services.beeper.localtest.me/proxy/anthropic"} headers := anthropicHeaders(model, ai.Context{}, AnthropicOptions{StreamOptions: ai.StreamOptions{APIKey: "matrix-token"}}, false) diff --git a/pkg/chattools/chattools_test.go b/pkg/chattools/chattools_test.go index 85698696..3ad6db80 100644 --- a/pkg/chattools/chattools_test.go +++ b/pkg/chattools/chattools_test.go @@ -318,6 +318,19 @@ func TestSearchUsesConfiguredEndpoint(t *testing.T) { } } +func TestSearchIncludesErrorResponseMessage(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte(`{"error":{"message":"Web tool upstream request timed out"}}`)) + })) + defer server.Close() + + _, err := Search(context.Background(), "query", 5, SearchRequestOptions{}, SearchOptions{Enabled: true, Endpoint: server.URL, APIKey: "key", Timeout: time.Second}) + if err == nil || !strings.Contains(err.Error(), "Web tool upstream request timed out") { + t.Fatalf("expected upstream error message to propagate, got %v", err) + } +} + func TestSearchMapsToolOptionsToPayload(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var payload map[string]any diff --git a/pkg/connector/stream_test.go b/pkg/connector/stream_test.go index 55106d1c..836a1117 100644 --- a/pkg/connector/stream_test.go +++ b/pkg/connector/stream_test.go @@ -852,6 +852,49 @@ func TestApplyAIStreamEventStreamsToolCallsFromPartialContent(t *testing.T) { } } +func TestApplyAIStreamEventStreamsNativeToolResult(t *testing.T) { + run := aistream.NewRun("run", "thread", "beeper/gpt-5.5", "assistant:run", "GPT-5.5", timeNow()) + run.MessageID = "assistant:run" + writer := aistream.NewWriter(run, timeNow) + toolCall := &ai.ToolCall{ + ID: "ws_1", + Name: "web_search", + Arguments: map[string]any{"query": "latest headlines Amsterdam news today"}, + } + + applyAIStreamEvent(writer, ai.AssistantMessageEvent{Type: "toolcall_start", ToolCall: toolCall}) + applyAIStreamEvent(writer, ai.AssistantMessageEvent{ + Type: "toolresult", + ToolCall: toolCall, + CustomValue: map[string]any{ + "state": agui.ToolResultStateComplete, + "status": "success", + "provider": "openai", + "native": true, + }, + }) + + var sawEnd, sawResult bool + for _, evt := range run.Events { + switch evt.Type() { + case agui.EventToolCallEnd: + sawEnd = evt.Get("toolCallId") == "ws_1" && evt.Get("toolName") == "web_search" + case agui.EventToolCallResult: + sawResult = evt.Get("toolCallId") == "ws_1" + var output map[string]any + if err := json.Unmarshal([]byte(evt.Get("content").(string)), &output); err != nil { + t.Fatalf("native tool result content is not JSON: %v", err) + } + if output["status"] != "success" || output["native"] != true { + t.Fatalf("unexpected native tool result output: %#v", output) + } + } + } + if !sawEnd || !sawResult { + t.Fatalf("missing native tool result lifecycle events: %#v", run.Events) + } +} + func TestApplyAIStreamEventSkipsEmptyToolCallDelta(t *testing.T) { run := aistream.NewRun("run", "thread", "beeper/gpt-5.5", "assistant:run", "GPT-5.5", timeNow()) run.MessageID = "assistant:run" From 6b1e6da975cc7416692edc1e4ec62a0a092dc5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 18:31:04 +0200 Subject: [PATCH 06/22] fix more mappings --- pkg/ai-stream/message.go | 43 ++++++- pkg/ai-stream/stream_test.go | 66 ++++++++++- pkg/ai/providers/anthropic.go | 50 +++++++- pkg/ai/providers/anthropic_test.go | 83 +++++++++++++ pkg/ai/providers/citations.go | 19 ++- pkg/ai/providers/openai_conversion_test.go | 40 +++++++ pkg/ai/providers/openai_responses.go | 2 + pkg/ai/providers/openai_shared.go | 92 +++++++++++--- pkg/ai/providers/openai_stream_test.go | 132 +++++++++++++++++++++ pkg/ai/utils/overflow.go | 1 + pkg/ai/utils/overflow_test.go | 4 + pkg/connector/sources.go | 19 ++- pkg/connector/sources_test.go | 15 +++ 13 files changed, 539 insertions(+), 27 deletions(-) diff --git a/pkg/ai-stream/message.go b/pkg/ai-stream/message.go index 9fd7dc4c..13fb2517 100644 --- a/pkg/ai-stream/message.go +++ b/pkg/ai-stream/message.go @@ -632,17 +632,58 @@ func (t Run) FinalBeeperAIMessage(textBudget int, includeThinking bool) UIMessag projected.part["content"] = projected.content.String() compactTextPart(projected.part, textBudget) } + emptyThinkingPartIDs := map[string]struct{}{} for _, projected := range thinkingParts { content := projected.content.String() if content == "" { - content = "Thinking..." + if t.Status.State != "" && t.Status.State != "streaming" { + if id := firstString(projected.part["id"]); id != "" { + emptyThinkingPartIDs[id] = struct{}{} + } + continue + } } projected.part["content"] = content compactTextPart(projected.part, textBudget) } + message.Parts = filterFinalMessageParts(message.Parts, emptyThinkingPartIDs) return message } +func filterFinalMessageParts(parts []MessagePart, emptyThinkingPartIDs map[string]struct{}) []MessagePart { + lastNativeOpenAIWebSearchByQuery := map[string]int{} + for index, part := range parts { + if query := nativeOpenAIWebSearchQuery(part); query != "" { + lastNativeOpenAIWebSearchByQuery[query] = index + } + } + out := parts[:0] + for index, part := range parts { + if part["type"] == "thinking" { + if _, ok := emptyThinkingPartIDs[firstString(part["id"])]; ok { + continue + } + } + if query := nativeOpenAIWebSearchQuery(part); query != "" && lastNativeOpenAIWebSearchByQuery[query] != index { + continue + } + out = append(out, part) + } + return out +} + +func nativeOpenAIWebSearchQuery(part MessagePart) string { + if part["type"] != "tool-call" || firstString(part["name"]) != "web_search" { + return "" + } + output, _ := part["output"].(map[string]any) + if output == nil || output["native"] != true || firstString(output["provider"]) != "openai" { + return "" + } + input, _ := part["input"].(map[string]any) + return firstString(output["query"], input["query"]) +} + func toolResultOutput(content string, state string, err any) any { output := jsonValue(content) result, ok := output.(map[string]any) diff --git a/pkg/ai-stream/stream_test.go b/pkg/ai-stream/stream_test.go index 8d19a3c5..f0c22948 100644 --- a/pkg/ai-stream/stream_test.go +++ b/pkg/ai-stream/stream_test.go @@ -324,7 +324,7 @@ func TestFinalBeeperAIMessagePreservesTextChunksAfterToolCalls(t *testing.T) { } } -func TestFinalBeeperAIMessageShowsHiddenReasoningInOrder(t *testing.T) { +func TestFinalBeeperAIMessageSuppressesEmptyHiddenReasoningInOrder(t *testing.T) { run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) builder := agui.NewEventBuilder(DefaultModel, func() time.Time { return time.Unix(10, 0) }) run.Status = Status{State: "complete", FinishReason: agui.FinishReasonStop} @@ -354,7 +354,6 @@ func TestFinalBeeperAIMessageShowsHiddenReasoningInOrder(t *testing.T) { want := []string{ "text:first text", "tool-call:tool-1", - "thinking:Thinking...", "text:second text", } if !reflect.DeepEqual(got, want) { @@ -362,6 +361,69 @@ func TestFinalBeeperAIMessageShowsHiddenReasoningInOrder(t *testing.T) { } } +func TestFinalBeeperAIMessageKeepsStreamingThinkingPartEmptyUntilTokensArrive(t *testing.T) { + run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) + writer := NewWriter(run, func() time.Time { return time.Unix(10, 0) }) + writer.Start() + writer.ReasoningMessageStart(0) + + streamingEmpty := run.FinalBeeperAIMessage(0, true) + if len(streamingEmpty.Parts) != 1 || streamingEmpty.Parts[0]["type"] != "thinking" || streamingEmpty.Parts[0]["content"] != "" || streamingEmpty.Parts[0]["state"] != agui.PartStateStreaming { + t.Fatalf("streaming empty reasoning should render as an empty in-progress part, got %#v", streamingEmpty.Parts) + } + + writer.ReasoningDelta(0, "reading sources") + streamingContent := run.FinalBeeperAIMessage(0, true) + if len(streamingContent.Parts) != 1 || streamingContent.Parts[0]["type"] != "thinking" || streamingContent.Parts[0]["content"] != "reading sources" || streamingContent.Parts[0]["state"] != agui.PartStateStreaming { + t.Fatalf("streaming reasoning tokens should update the same thinking part, got %#v", streamingContent.Parts) + } + + writer.ReasoningMessageEnd(0) + writer.Finish(agui.FinishReasonStop) + finalContent := run.FinalBeeperAIMessage(0, true) + if len(finalContent.Parts) != 1 || finalContent.Parts[0]["type"] != "thinking" || finalContent.Parts[0]["content"] != "reading sources" || finalContent.Parts[0]["state"] != agui.PartStateDone { + t.Fatalf("final reasoning tokens should remain in order as a done thinking part, got %#v", finalContent.Parts) + } +} + +func TestFinalBeeperAIMessageDedupesNativeOpenAIWebSearchRows(t *testing.T) { + run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) + writer := NewWriter(run, func() time.Time { return time.Unix(10, 0) }) + query := "pop culture latest Oscars Grammys Wikipedia" + writer.ToolStart("ws_aggregate", "web_search", 0, nil) + writer.ToolEnd("ws_aggregate", "web_search", map[string]any{ + "query": query, + "queries": []any{query, "latest entertainment news film music television"}, + }, map[string]any{ + "state": "complete", + "status": "success", + "provider": "openai", + "native": true, + "query": query, + "queries": []any{query, "latest entertainment news film music television"}, + }) + writer.ToolStart("ws_single", "web_search", 0, nil) + writer.ToolEnd("ws_single", "web_search", map[string]any{"query": query}, map[string]any{ + "state": "complete", + "status": "success", + "provider": "openai", + "native": true, + "query": query, + }) + writer.Finish(agui.FinishReasonStop) + + uiMessage := run.FinalBeeperAIMessage(0, true) + var webSearchIDs []string + for _, part := range uiMessage.Parts { + if part["type"] == "tool-call" && part["name"] == "web_search" { + webSearchIDs = append(webSearchIDs, firstString(part["toolCallId"])) + } + } + if !reflect.DeepEqual(webSearchIDs, []string{"ws_single"}) { + t.Fatalf("expected only the final native OpenAI web_search row, got IDs=%#v parts=%#v", webSearchIDs, uiMessage.Parts) + } +} + func TestFinalBeeperAIMessagePreservesStepsAsThinking(t *testing.T) { run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) builder := agui.NewEventBuilder(DefaultModel, func() time.Time { return time.Unix(10, 0) }) diff --git a/pkg/ai/providers/anthropic.go b/pkg/ai/providers/anthropic.go index 885a7675..ad3b13eb 100644 --- a/pkg/ai/providers/anthropic.go +++ b/pkg/ai/providers/anthropic.go @@ -114,6 +114,8 @@ func StreamAnthropic(ctx context.Context, model ai.Model, llmContext ai.Context, } stream.Push(ai.AssistantMessageEvent{Type: "start", Partial: &output}) state := newAnthropicStreamState() + sawMessageStart := false + sawMessageStop := false err = iterateSSE(response.Body, func(sse serverSentEvent) error { if sse.Event == "error" { return errors.New(sse.Data) @@ -129,6 +131,12 @@ func StreamAnthropic(ctx context.Context, model ai.Model, llmContext ai.Context, return fmt.Errorf("could not parse Anthropic SSE event %s: %w; data=%s; raw=%s", sse.Event, err, sse.Data, strings.Join(sse.Raw, "\n")) } } + switch event["type"] { + case "message_start": + sawMessageStart = true + case "message_stop": + sawMessageStop = true + } citations := providerCitationsFromAny(event, model.Provider, max(0, len(contentBlocks(output.Content))-1)) if len(citations) > 0 { output.Citations = append(output.Citations, citations...) @@ -141,6 +149,10 @@ func StreamAnthropic(ctx context.Context, model ai.Model, llmContext ai.Context, pushFinalError(stream, &output, err.Error()) return } + if sawMessageStart && !sawMessageStop { + pushFinalError(stream, &output, "Anthropic stream ended before message_stop") + return + } stream.Push(ai.AssistantMessageEvent{Type: "done", Reason: output.StopReason, Message: &output}) }() return stream @@ -368,11 +380,42 @@ func doAnthropicRequest(ctx context.Context, model ai.Model, llmContext ai.Conte if resp.StatusCode < 200 || resp.StatusCode >= 300 { defer resp.Body.Close() payload, _ := io.ReadAll(io.LimitReader(resp.Body, 8192)) - return nil, fmt.Errorf("Anthropic API error (%d): %s", resp.StatusCode, strings.TrimSpace(string(payload))) + return nil, errors.New(formatAnthropicAPIError(resp.StatusCode, payload)) } return resp, nil } +func formatAnthropicAPIError(statusCode int, payload []byte) string { + detail := strings.TrimSpace(string(payload)) + if parsed := providerErrorMessage(payload); parsed != "" { + detail = parsed + } + if detail == "" { + statusText := strings.TrimSpace(fmt.Sprintf("%d %s", statusCode, http.StatusText(statusCode))) + if statusText == "" { + statusText = fmt.Sprintf("upstream status %d", statusCode) + } + detail = statusText + " (no body)" + } + return fmt.Sprintf("Anthropic API error (%d): %s", statusCode, detail) +} + +func providerErrorMessage(payload []byte) string { + var parsed struct { + Message string `json:"message"` + Error struct { + Message string `json:"message"` + } `json:"error"` + } + if len(bytes.TrimSpace(payload)) == 0 || json.Unmarshal(payload, &parsed) != nil { + return "" + } + if parsed.Error.Message != "" { + return parsed.Error.Message + } + return parsed.Message +} + func anthropicHeaders(model ai.Model, llmContext ai.Context, options AnthropicOptions, isOAuth bool) map[string]string { headers := map[string]string{"Anthropic-Dangerous-Direct-Browser-Access": "true"} betas := []string{} @@ -794,6 +837,11 @@ func anthropicNativeToolResult(block map[string]any) map[string]any { result["url"] = url result["final_url"] = url } + if nested, _ := data["content"].(map[string]any); nested != nil { + if title := firstCitationString(stringFromAny(nested["title"]), documentTitleFromProviderContent(nested)); title != "" { + result["title"] = title + } + } if retrievedAt := stringFromAny(data["retrieved_at"]); retrievedAt != "" { result["retrieved_at"] = retrievedAt } diff --git a/pkg/ai/providers/anthropic_test.go b/pkg/ai/providers/anthropic_test.go index 23b9e162..002cea0d 100644 --- a/pkg/ai/providers/anthropic_test.go +++ b/pkg/ai/providers/anthropic_test.go @@ -1,12 +1,44 @@ package providers import ( + "net/http" + "net/http/httptest" "strings" "testing" ai "github.com/beeper/ai-bridge/pkg/ai" ) +func TestFormatAnthropicAPIErrorParsesProviderBody(t *testing.T) { + got := formatAnthropicAPIError(http.StatusBadRequest, []byte(`{"type":"error","error":{"type":"invalid_request_error","message":"messages.0.content: Field required"}}`)) + want := "Anthropic API error (400): messages.0.content: Field required" + if got != want { + t.Fatalf("formatAnthropicAPIError() = %q, want %q", got, want) + } +} + +func TestFormatAnthropicAPIErrorPreservesEmptyBodyStatus(t *testing.T) { + got := formatAnthropicAPIError(http.StatusBadRequest, nil) + want := "Anthropic API error (400): 400 Bad Request (no body)" + if got != want { + t.Fatalf("formatAnthropicAPIError() = %q, want %q", got, want) + } +} + +func TestStreamAnthropicErrorsWhenStreamEndsBeforeMessageStop(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"usage\":{\"input_tokens\":1,\"output_tokens\":0}}}\n\n")) + })) + defer upstream.Close() + + model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic, BaseURL: upstream.URL} + result := StreamAnthropic(t.Context(), model, ai.Context{}, AnthropicOptions{StreamOptions: ai.StreamOptions{APIKey: "key"}}).Result() + if result.StopReason != ai.StopReasonError || !strings.Contains(result.ErrorMessage, "message_stop") { + t.Fatalf("expected missing message_stop error, got %#v", result) + } +} + func TestAnthropicStreamStatePreservesToolInputFromContentBlockStart(t *testing.T) { stream := ai.NewAssistantMessageEventStream() model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic} @@ -151,6 +183,57 @@ func TestAnthropicStreamStateMapsNativeWebFetchError(t *testing.T) { } } +func TestAnthropicStreamStateMapsNativeWebFetchDocumentTitle(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic} + output := newAssistant(model) + state := newAnthropicStreamState() + + state.apply(stream, &output, model, ai.Context{}, false, map[string]any{ + "type": "content_block_start", + "index": float64(0), + "content_block": map[string]any{ + "type": "server_tool_use", + "id": "srvtoolu_fetch", + "name": "web_fetch", + "input": map[string]any{"url": "https://example.com/article"}, + }, + }) + state.apply(stream, &output, model, ai.Context{}, false, map[string]any{ + "type": "content_block_start", + "index": float64(1), + "content_block": map[string]any{ + "type": "web_fetch_tool_result", + "tool_use_id": "srvtoolu_fetch", + "content": map[string]any{ + "type": "web_fetch_result", + "url": "https://example.com/article", + "content": map[string]any{ + "type": "document", + "source": map[string]any{ + "type": "text", + "media_type": "text/plain", + "data": "Fetched Article Title\n\nBody text", + }, + }, + }, + }, + }) + + events := drainAssistantEvents(stream) + for _, event := range events { + if event.Type != "toolresult" { + continue + } + output, _ := event.CustomValue.(map[string]any) + if output["title"] != "Fetched Article Title" { + t.Fatalf("expected title from document source, got %#v", event.CustomValue) + } + return + } + t.Fatalf("expected native fetch result, got %#v", events) +} + func TestAnthropicBeeperProxyUsesBearerAuth(t *testing.T) { model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic, BaseURL: "https://ai-services.beeper.localtest.me/proxy/anthropic"} headers := anthropicHeaders(model, ai.Context{}, AnthropicOptions{StreamOptions: ai.StreamOptions{APIKey: "matrix-token"}}, false) diff --git a/pkg/ai/providers/citations.go b/pkg/ai/providers/citations.go index 6b7d8dc3..083f1ff6 100644 --- a/pkg/ai/providers/citations.go +++ b/pkg/ai/providers/citations.go @@ -180,13 +180,16 @@ func providerCitationFromMap(data map[string]any, provider ai.Provider, contentI } rawType = firstCitationString(rawType, strings.ToLower(stringFromAny(citationData["type"]))) url := firstCitationString(stringFromAny(citationData["url"]), stringFromAny(citationData["uri"])) - if url == "" || (!strings.Contains(rawType, "citation") && rawType != "web_search_result_location" && rawType != "web_fetch_result") { + if url == "" || (!strings.Contains(rawType, "citation") && rawType != "web_search_result_location" && rawType != "web_fetch_result" && rawType != "openrouter:web_fetch") { return ai.Citation{}, false } title := stringFromAny(citationData["title"]) - if title == "" && rawType == "web_fetch_result" { + if title == "" && (rawType == "web_fetch_result" || rawType == "openrouter:web_fetch") { if content, _ := citationData["content"].(map[string]any); content != nil { title = stringFromAny(content["title"]) + if title == "" { + title = documentTitleFromProviderContent(content) + } } } resolvedContentIndex := contentIndex @@ -216,6 +219,18 @@ func providerCitationFromMap(data map[string]any, provider ai.Provider, contentI return citation, true } +func documentTitleFromProviderContent(content map[string]any) string { + source, _ := content["source"].(map[string]any) + text := firstCitationString(stringFromAny(content["text"]), stringFromAny(content["data"]), stringFromAny(source["data"])) + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line != "" { + return line + } + } + return "" +} + func mergeCitationMaps(first map[string]any, second map[string]any) map[string]any { out := map[string]any{} for key, value := range first { diff --git a/pkg/ai/providers/openai_conversion_test.go b/pkg/ai/providers/openai_conversion_test.go index 8256f469..f3039fb7 100644 --- a/pkg/ai/providers/openai_conversion_test.go +++ b/pkg/ai/providers/openai_conversion_test.go @@ -98,6 +98,32 @@ func TestCompleteOpenAIResponsesUsesNonStreamingPayload(t *testing.T) { } } +func TestCompleteOpenAIResponsesExtractsOpenRouterWebFetchSources(t *testing.T) { + model := ai.Model{ID: "x-ai/grok-4.20", API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenRouter} + output := newAssistant(model) + applyCompleteOpenAIResponses(&output, model, OpenAIResponsesOptions{}, map[string]any{ + "id": "resp_1", + "status": "completed", + "output": []any{map[string]any{ + "type": "openrouter:web_fetch", + "id": "st_1", + "status": "completed", + "url": "https://example.com/fetched", + "title": "Fetched Page", + }, map[string]any{ + "type": "message", + "id": "msg_1", + "content": []any{map[string]any{ + "type": "output_text", + "text": "ok", + }}, + }}, + }) + if len(output.Citations) != 1 || output.Citations[0].URL != "https://example.com/fetched" || output.Citations[0].Title != "Fetched Page" { + t.Fatalf("expected OpenRouter fetch source from non-stream response, got %#v", output.Citations) + } +} + func TestConvertCompletionsMessagesIncludesNativeAudio(t *testing.T) { model := ai.Model{ID: "gpt-audio", API: ai.ApiOpenAICompletions, Provider: "openai", Input: []string{"text", "audio"}} messages := ConvertCompletionsMessages(model, ai.Context{ @@ -194,6 +220,20 @@ func TestProviderCitationsFromAnthropicWebFetchResult(t *testing.T) { } } +func TestProviderCitationsFromOpenRouterWebFetchItem(t *testing.T) { + citations := providerCitationsFromAny(map[string]any{ + "type": "openrouter:web_fetch", + "status": "completed", + "url": "https://example.com/openrouter-fetch", + "title": "OpenRouter Fetch", + "httpStatus": float64(200), + "content": "Fetched page text", + }, ai.ProviderOpenRouter, 0) + if len(citations) != 1 || citations[0].URL != "https://example.com/openrouter-fetch" || citations[0].Title != "OpenRouter Fetch" || citations[0].RawType != "openrouter:web_fetch" { + t.Fatalf("unexpected OpenRouter web fetch citations %#v", citations) + } +} + func TestProviderCitationsFromGoogleURLContextMetadata(t *testing.T) { citations := providerCitationsFromAny(map[string]any{ "candidates": []any{map[string]any{ diff --git a/pkg/ai/providers/openai_responses.go b/pkg/ai/providers/openai_responses.go index 6c1b30cf..ae1d5688 100644 --- a/pkg/ai/providers/openai_responses.go +++ b/pkg/ai/providers/openai_responses.go @@ -186,6 +186,8 @@ func applyCompleteOpenAIResponses(output *ai.Message, model ai.Model, options Op blocks = append(blocks, ai.ContentBlock{Type: "toolCall", ID: id, Name: fmt.Sprint(item["name"]), Arguments: parseJSONMap(args)}) case "image_generation_call": blocks = append(blocks, imageBlockFromGenerationItem(item, ai.ContentBlock{})) + case "web_search_call", "openrouter:web_search", "openrouter:web_fetch": + output.Citations = append(output.Citations, providerCitationsFromAny(item, model.Provider, len(blocks))...) } } } diff --git a/pkg/ai/providers/openai_shared.go b/pkg/ai/providers/openai_shared.go index f48d395f..e4e65e9a 100644 --- a/pkg/ai/providers/openai_shared.go +++ b/pkg/ai/providers/openai_shared.go @@ -764,26 +764,54 @@ func responsesItemID(item map[string]any) string { return strings.TrimSpace(stringFromAny(item["id"])) } -func responsesNativeWebSearchToolCall(item map[string]any, fallbackIndex int) ai.ToolCall { +func responsesNativeToolCall(item map[string]any, fallbackIndex int, provider ai.Provider) (ai.ToolCall, bool) { + name, _, ok := responsesNativeToolInfo(item, provider) + if !ok { + return ai.ToolCall{}, false + } id := responsesItemID(item) if id == "" { - id = fmt.Sprintf("native_web_search_%d", fallbackIndex+1) + id = fmt.Sprintf("native_%s_%d", strings.ReplaceAll(name, "_", "-"), fallbackIndex+1) } return ai.ToolCall{ Type: "toolCall", ID: id, - Name: "web_search", - Arguments: responsesNativeWebSearchArguments(item), + Name: name, + Arguments: responsesNativeToolArguments(item, name), + }, true +} + +func responsesNativeToolInfo(item map[string]any, provider ai.Provider) (toolName string, resultProvider string, ok bool) { + itemType := strings.TrimSpace(stringFromAny(item["type"])) + switch itemType { + case "web_search_call": + return "web_search", string(provider), true + case "openrouter:web_search": + return "web_search", string(ai.ProviderOpenRouter), true + case "openrouter:web_fetch": + return "fetch", string(ai.ProviderOpenRouter), true + default: + return "", "", false } } -func responsesNativeWebSearchArguments(item map[string]any) map[string]any { +func responsesNativeToolArguments(item map[string]any, toolName string) map[string]any { args := map[string]any{} - addNativeWebSearchQuery(args, item) - if action, _ := item["action"].(map[string]any); action != nil { - addNativeWebSearchQuery(args, action) - if actionType := strings.TrimSpace(stringFromAny(action["type"])); actionType != "" { - args["action"] = actionType + switch toolName { + case "web_search": + addNativeWebSearchQuery(args, item) + if action, _ := item["action"].(map[string]any); action != nil { + addNativeWebSearchQuery(args, action) + if queries, ok := action["queries"].([]any); ok && len(queries) > 0 { + args["queries"] = queries + } + if actionType := strings.TrimSpace(stringFromAny(action["type"])); actionType != "" { + args["action"] = actionType + } + } + case "fetch": + if url := strings.TrimSpace(stringFromAny(item["url"])); url != "" { + args["url"] = url } } return args @@ -801,17 +829,33 @@ func addNativeWebSearchQuery(args map[string]any, data map[string]any) { } } -func responsesNativeToolResult(item map[string]any) map[string]any { +func responsesNativeToolResult(item map[string]any, provider ai.Provider) map[string]any { + toolName, resultProvider, _ := responsesNativeToolInfo(item, provider) status := strings.TrimSpace(stringFromAny(item["status"])) result := map[string]any{ "state": "complete", "status": "success", - "provider": "openai", + "provider": resultProvider, "native": true, } if status != "" { result["providerStatus"] = status } + for _, key := range []string{"url", "title", "httpStatus", "http_status"} { + if value := item[key]; value != nil { + result[key] = value + } + } + if toolName == "fetch" { + if url := stringFromAny(item["url"]); url != "" { + result["final_url"] = url + } + } + if args := responsesNativeToolArguments(item, toolName); len(args) > 0 { + for key, value := range args { + result[key] = value + } + } if status == "failed" || status == "incomplete" { result["state"] = "error" result["status"] = "failed" @@ -820,8 +864,11 @@ func responsesNativeToolResult(item map[string]any) map[string]any { result["reason"] = message } } + if message := strings.TrimSpace(stringFromAny(item["error"])); message != "" { + result["reason"] = message + } if result["reason"] == nil { - result["reason"] = "Provider-native web search failed" + result["reason"] = "Provider-native web tool failed" } } return result @@ -873,8 +920,11 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out s.blocks = append(s.blocks, imageBlockFromGenerationItem(item, ai.ContentBlock{})) s.currentIndex = len(s.blocks) - 1 output.Content = s.blocks - case "web_search_call": - toolCall := responsesNativeWebSearchToolCall(item, len(s.nativeToolsByItemID)) + case "web_search_call", "openrouter:web_search", "openrouter:web_fetch": + toolCall, ok := responsesNativeToolCall(item, len(s.nativeToolsByItemID), model.Provider) + if !ok { + return + } s.nativeToolsByItemID[responsesItemID(item)] = toolCall s.currentIndex = -1 output.Content = s.blocks @@ -969,15 +1019,19 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out case "response.output_item.done": item, _ := event["item"].(map[string]any) itemType, _ := item["type"].(string) - if itemType == "web_search_call" { + if _, _, ok := responsesNativeToolInfo(item, model.Provider); ok { toolCall := s.nativeToolsByItemID[responsesItemID(item)] if toolCall.ID == "" { - toolCall = responsesNativeWebSearchToolCall(item, len(s.nativeToolsByItemID)) - } else if args := responsesNativeWebSearchArguments(item); len(args) > 0 { + toolCall, _ = responsesNativeToolCall(item, len(s.nativeToolsByItemID), model.Provider) + } else if args := responsesNativeToolArguments(item, toolCall.Name); len(args) > 0 { toolCall.Arguments = args } output.Content = s.blocks - push(ai.AssistantMessageEvent{Type: "toolresult", ToolCall: &toolCall, CustomValue: responsesNativeToolResult(item), Partial: output}) + if citations := providerCitationsFromAny(item, model.Provider, max(0, s.currentIndex)); len(citations) > 0 { + output.Citations = append(output.Citations, citations...) + push(ai.AssistantMessageEvent{Type: "source", Partial: output}) + } + push(ai.AssistantMessageEvent{Type: "toolresult", ToolCall: &toolCall, CustomValue: responsesNativeToolResult(item, model.Provider), Partial: output}) s.currentIndex = -1 return } diff --git a/pkg/ai/providers/openai_stream_test.go b/pkg/ai/providers/openai_stream_test.go index ddcb4c61..d6549bc0 100644 --- a/pkg/ai/providers/openai_stream_test.go +++ b/pkg/ai/providers/openai_stream_test.go @@ -214,6 +214,138 @@ func TestResponsesStreamStateMapsNativeWebSearchCallToToolActivity(t *testing.T) } } +func TestResponsesStreamStateMapsOpenRouterWebSearchToToolActivity(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := testStreamModel() + model.API = ai.ApiOpenAIResponses + model.Provider = ai.ProviderOpenRouter + output := newAssistant(model) + state := newResponsesStreamState() + + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "item": map[string]any{ + "type": "openrouter:web_search", + "id": "or_search_1", + "status": "in_progress", + }, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.done", + "item": map[string]any{ + "type": "openrouter:web_search", + "id": "or_search_1", + "status": "completed", + "action": map[string]any{"type": "search", "query": "current Amsterdam headline"}, + }, + }) + + events := drainAssistantEvents(stream) + if len(output.Content.([]ai.ContentBlock)) != 0 { + t.Fatalf("OpenRouter native search must not become an executable tool block, got %#v", output.Content) + } + var started, result bool + for _, event := range events { + switch event.Type { + case "toolcall_start": + started = event.ToolCall != nil && event.ToolCall.ID == "or_search_1" && event.ToolCall.Name == "web_search" + case "toolresult": + result = event.ToolCall != nil && event.ToolCall.ID == "or_search_1" && event.ToolCall.Arguments["query"] == "current Amsterdam headline" + output, _ := event.CustomValue.(map[string]any) + if output["provider"] != string(ai.ProviderOpenRouter) || output["status"] != "success" || output["query"] != "current Amsterdam headline" { + t.Fatalf("unexpected OpenRouter search result payload %#v", event.CustomValue) + } + } + } + if !started || !result { + t.Fatalf("expected OpenRouter web search start/result events, got %#v", events) + } +} + +func TestResponsesStreamStateMapsOpenRouterWebFetchToToolActivity(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := testStreamModel() + model.API = ai.ApiOpenAIResponses + model.Provider = ai.ProviderOpenRouter + output := newAssistant(model) + state := newResponsesStreamState() + + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "item": map[string]any{ + "type": "openrouter:web_fetch", + "id": "or_fetch_1", + "status": "in_progress", + }, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.done", + "item": map[string]any{ + "type": "openrouter:web_fetch", + "id": "or_fetch_1", + "status": "completed", + "url": "https://example.com/article", + "title": "Fetched Article", + "httpStatus": float64(200), + "content": "Fetched Article body that should not be copied into the tool output.", + }, + }) + + events := drainAssistantEvents(stream) + var sawSource, sawResult bool + for _, event := range events { + switch event.Type { + case "source": + sawSource = len(output.Citations) == 1 && output.Citations[0].URL == "https://example.com/article" + case "toolresult": + sawResult = event.ToolCall != nil && event.ToolCall.Name == "fetch" + output, _ := event.CustomValue.(map[string]any) + if output["provider"] != string(ai.ProviderOpenRouter) || output["url"] != "https://example.com/article" || output["title"] != "Fetched Article" || output["content"] != nil { + t.Fatalf("unexpected OpenRouter fetch result payload %#v", event.CustomValue) + } + } + } + if !sawSource || !sawResult { + t.Fatalf("expected OpenRouter fetch source/result events, got %#v citations=%#v", events, output.Citations) + } +} + +func TestResponsesStreamStateMapsOpenRouterWebFetchFailure(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := testStreamModel() + model.API = ai.ApiOpenAIResponses + model.Provider = ai.ProviderOpenRouter + output := newAssistant(model) + state := newResponsesStreamState() + + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.done", + "item": map[string]any{ + "type": "openrouter:web_fetch", + "id": "or_fetch_fail", + "status": "incomplete", + "url": "https://nonexistent.invalid/does-not-exist", + "error": "Exa returned no content for this URL.", + }, + }) + + events := drainAssistantEvents(stream) + var result bool + for _, event := range events { + if event.Type != "toolresult" { + continue + } + result = event.ToolCall != nil && event.ToolCall.Name == "fetch" + output, _ := event.CustomValue.(map[string]any) + if output["status"] != "failed" || output["reason"] != "Exa returned no content for this URL." { + t.Fatalf("unexpected OpenRouter fetch failure payload %#v", event.CustomValue) + } + } + if !result { + t.Fatalf("expected OpenRouter fetch failure result, got %#v", events) + } +} + func TestResponsesStreamStateStreamsTextDeltasWithoutContentPartPrelude(t *testing.T) { stream := ai.NewAssistantMessageEventStream() model := testStreamModel() diff --git a/pkg/ai/utils/overflow.go b/pkg/ai/utils/overflow.go index 40aa7a53..99df8485 100644 --- a/pkg/ai/utils/overflow.go +++ b/pkg/ai/utils/overflow.go @@ -29,6 +29,7 @@ var overflowPatterns = []*regexp.Regexp{ regexp.MustCompile(`(?i)too many tokens`), regexp.MustCompile(`(?i)token limit exceeded`), regexp.MustCompile(`(?i)^4(?:00|13)\s*(?:status code)?\s*\(no body\)`), + regexp.MustCompile(`(?i)\b4(?:00|13)\s+[a-z ]+\(no body\)`), } var nonOverflowPatterns = []*regexp.Regexp{ diff --git a/pkg/ai/utils/overflow_test.go b/pkg/ai/utils/overflow_test.go index c6b676ff..929527bf 100644 --- a/pkg/ai/utils/overflow_test.go +++ b/pkg/ai/utils/overflow_test.go @@ -15,6 +15,10 @@ func TestIsContextOverflowFromErrorMessage(t *testing.T) { if IsContextOverflow(nonOverflow) { t.Fatal("expected rate limit to be excluded") } + anthropicNoBody := ai.Message{StopReason: ai.StopReasonError, ErrorMessage: "Anthropic API error (400): 400 Bad Request (no body)"} + if !IsContextOverflow(anthropicNoBody) { + t.Fatal("expected provider-prefixed no-body 400 to be detected") + } } func TestIsContextOverflowFromUsage(t *testing.T) { diff --git a/pkg/connector/sources.go b/pkg/connector/sources.go index b3b30f4a..0fceb895 100644 --- a/pkg/connector/sources.go +++ b/pkg/connector/sources.go @@ -580,13 +580,16 @@ func providerCitationSource(data map[string]any) (sourceObservation, bool) { sourceType = firstSourceString(sourceType, strings.ToLower(sourceString(nested, "type", "rawType"))) } rawURL := sourceString(citation, "url", "uri") - if rawURL == "" || (!strings.Contains(sourceType, "citation") && sourceType != "web_search_result_location" && sourceType != "web_fetch_result" && sourceType != "url_context") { + if rawURL == "" || (!strings.Contains(sourceType, "citation") && sourceType != "web_search_result_location" && sourceType != "web_fetch_result" && sourceType != "openrouter:web_fetch" && sourceType != "url_context") { return sourceObservation{}, false } title := sourceString(citation, "title") - if title == "" && sourceType == "web_fetch_result" { + if title == "" && (sourceType == "web_fetch_result" || sourceType == "openrouter:web_fetch") { if content, _ := citation["content"].(map[string]any); content != nil { title = sourceString(content, "title") + if title == "" { + title = sourceDocumentTitleFromProviderContent(content) + } } } return sourceObservation{ @@ -608,6 +611,18 @@ func providerCitationSource(data map[string]any) (sourceObservation, bool) { }, true } +func sourceDocumentTitleFromProviderContent(content map[string]any) string { + source, _ := content["source"].(map[string]any) + text := firstSourceString(sourceString(content, "text", "data"), sourceString(source, "data")) + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line != "" { + return line + } + } + return "" +} + func mergeSourceMaps(first map[string]any, second map[string]any) map[string]any { out := map[string]any{} for key, value := range first { diff --git a/pkg/connector/sources_test.go b/pkg/connector/sources_test.go index 786ae4d5..679d8a0f 100644 --- a/pkg/connector/sources_test.go +++ b/pkg/connector/sources_test.go @@ -102,6 +102,21 @@ func TestSourceCollectorUsesDescriptionAndFaviconFallbacks(t *testing.T) { t.Fatalf("provider web fetch result was not mapped: %#v", fetchResultSources) } + openRouterFetchSources := newSourceCollector().addProviderSources(map[string]any{ + "type": "openrouter:web_fetch", + "url": "https://example.com/openrouter-fetch", + "content": map[string]any{ + "type": "document", + "source": map[string]any{ + "type": "text", + "data": "\nOpenRouter Fetch Title\nBody text", + }, + }, + }) + if len(openRouterFetchSources) != 1 || openRouterFetchSources[0]["title"] != "OpenRouter Fetch Title" { + t.Fatalf("OpenRouter web fetch source was not mapped: %#v", openRouterFetchSources) + } + messageSources := newSourceCollector().addProviderSources(ai.Message{ Citations: []ai.Citation{{ Type: "url_citation", From 7173439ca1efeaab73a7c0b4e7c21c4eacdf27fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 18:47:59 +0200 Subject: [PATCH 07/22] better cache, fix more anthriopic bulls --- pkg/ai-stream/message.go | 18 +++++++ pkg/ai-stream/run.go | 24 +++++++++ pkg/ai/providers/anthropic.go | 8 +-- pkg/ai/providers/anthropic_test.go | 50 ++++++++++++++++++ pkg/ai/providers/openai_codex_responses.go | 4 +- .../providers/openai_codex_responses_test.go | 4 +- pkg/ai/providers/openai_completions.go | 20 ++++--- pkg/ai/providers/openai_conversion_test.go | 49 +++++++++++++++++ pkg/ai/types.go | 4 ++ pkg/ai/utils/http_logging.go | 2 +- pkg/ai/utils/http_logging_test.go | 5 +- pkg/connector/builtin_tools.go | 17 +++++- pkg/connector/builtin_tools_test.go | 14 ++--- pkg/connector/chat_tools.go | 26 ++++++++++ pkg/connector/client.go | 1 + pkg/connector/contacts.go | 9 ++++ pkg/connector/contacts_test.go | 5 +- pkg/connector/room_state.go | 21 +++++--- pkg/connector/stream_test.go | 52 +++++++++++++++++++ 19 files changed, 301 insertions(+), 32 deletions(-) diff --git a/pkg/ai-stream/message.go b/pkg/ai-stream/message.go index 13fb2517..e41cb52f 100644 --- a/pkg/ai-stream/message.go +++ b/pkg/ai-stream/message.go @@ -172,6 +172,13 @@ func (t Run) Messages(includeReasoning bool) []agui.Message { messageID, _ := evt.Get("messageId").(string) message := ensureReasoningMessage(messageID) message.content.WriteString(asString(evt.Get("delta"))) + case agui.EventReasoningMsgChunk: + if !includeReasoning { + continue + } + messageID, _ := evt.Get("messageId").(string) + message := ensureReasoningMessage(messageID) + message.content.WriteString(asString(evt.Get("delta"))) case agui.EventReasoningMsgEnd: messageID, _ := evt.Get("messageId").(string) closeReasoningMessage(messageID) @@ -452,6 +459,16 @@ func (t Run) FinalBeeperAIMessage(textBudget int, includeThinking bool) UIMessag } messageID, _ := evt.Get("messageId").(string) ensureThinkingPart(messageID).content.WriteString(delta) + case agui.EventReasoningMsgChunk: + delta, _ := evt.Get("delta").(string) + if delta == "" { + continue + } + if !includeThinking { + continue + } + messageID, _ := evt.Get("messageId").(string) + ensureThinkingPart(messageID).content.WriteString(delta) case agui.EventReasoningMsgEnd: messageID, _ := evt.Get("messageId").(string) closeThinkingPart(messageID) @@ -801,6 +818,7 @@ func isReasoningEventType(eventType string) bool { agui.EventReasoningEnd, agui.EventReasoningMsgStart, agui.EventReasoningMsgCont, + agui.EventReasoningMsgChunk, agui.EventReasoningMsgEnd: return true default: diff --git a/pkg/ai-stream/run.go b/pkg/ai-stream/run.go index f7e09777..64c37067 100644 --- a/pkg/ai-stream/run.go +++ b/pkg/ai-stream/run.go @@ -185,6 +185,7 @@ type Writer struct { textOpen map[int]bool reasoningMessages map[int]string reasoningOpen map[int]bool + reasoningContent map[int]string reasoningPhaseID string reasoningPhaseOpen bool nextSyntheticReasoningIdx int @@ -233,6 +234,7 @@ func NewWriter(run *Run, now func() time.Time) *Writer { textOpen: map[int]bool{}, reasoningMessages: map[int]string{}, reasoningOpen: map[int]bool{}, + reasoningContent: map[int]string{}, reasoningPhaseID: "reasoning-" + run.RunID, lastAccountedChars: utf8.RuneCountInString(run.Text()), previewText: run.Text(), @@ -297,9 +299,28 @@ func (w *Writer) ReasoningDelta(index int, delta string) { return } messageID := w.ReasoningMessageStart(index) + w.reasoningContent[index] += delta w.Add(w.builder.ReasoningMessageContent(messageID, delta)) } +func (w *Writer) ReasoningContentSnapshot(index int, content string) { + if content == "" { + return + } + w.initState() + previous := w.reasoningContent[index] + if content == previous { + return + } + if previous == "" { + w.ReasoningDelta(index, content) + return + } + if strings.HasPrefix(content, previous) { + w.ReasoningDelta(index, content[len(previous):]) + } +} + func (w *Writer) ReasoningMessageEnd(index int) { w.initState() messageID := w.reasoningMessages[index] @@ -730,6 +751,9 @@ func (w *Writer) initState() { if w.reasoningOpen == nil { w.reasoningOpen = map[int]bool{} } + if w.reasoningContent == nil { + w.reasoningContent = map[int]string{} + } if w.reasoningPhaseID == "" && w.Run != nil { w.reasoningPhaseID = "reasoning-" + w.Run.RunID } diff --git a/pkg/ai/providers/anthropic.go b/pkg/ai/providers/anthropic.go index ad3b13eb..d8253644 100644 --- a/pkg/ai/providers/anthropic.go +++ b/pkg/ai/providers/anthropic.go @@ -447,9 +447,9 @@ func anthropicHeaders(model ai.Model, llmContext ai.Context, options AnthropicOp if len(betas) > 0 { headers["Anthropic-Beta"] = strings.Join(betas, ",") } - if options.SessionID != "" && getAnthropicCompat(model).SendSessionAffinityHeaders && resolveAnthropicCacheRetention(options.CacheRetention) != ai.CacheRetentionNone { - headers["X-Session-Affinity"] = options.SessionID - } + } + if options.SessionID != "" && getAnthropicCompat(model).SendSessionAffinityHeaders && resolveAnthropicCacheRetention(options.CacheRetention) != ai.CacheRetentionNone { + headers["X-Session-Affinity"] = options.SessionID } for key, value := range model.Headers { headers[key] = value @@ -468,7 +468,7 @@ func getAnthropicCompat(model ai.Model) resolvedAnthropicCompat { SupportsLongCacheRetention: compatBool(model, "supportsLongCacheRetention", !isFireworks), SendSessionAffinityHeaders: compatBool(model, "sendSessionAffinityHeaders", isFireworks || isCloudflareGatewayAnthropic), SupportsCacheControlOnTools: compatBool(model, "supportsCacheControlOnTools", !isFireworks), - ForceAdaptiveThinking: compatBool(model, "forceAdaptiveThinking", false), + ForceAdaptiveThinking: model.ReasoningMode == "adaptive" || compatBool(model, "forceAdaptiveThinking", false), AllowEmptySignature: compatBool(model, "allowEmptySignature", false), } } diff --git a/pkg/ai/providers/anthropic_test.go b/pkg/ai/providers/anthropic_test.go index 002cea0d..e93a1f33 100644 --- a/pkg/ai/providers/anthropic_test.go +++ b/pkg/ai/providers/anthropic_test.go @@ -245,6 +245,25 @@ func TestAnthropicBeeperProxyUsesBearerAuth(t *testing.T) { } } +func TestAnthropicBeeperProxyForwardsSessionAffinityWhenSupported(t *testing.T) { + model := ai.Model{ + ID: "claude-test", + API: ai.ApiAnthropicMessages, + Provider: ai.ProviderAnthropic, + BaseURL: "https://ai-services.beeper.localtest.me/proxy/anthropic", + Compat: map[string]any{"sendSessionAffinityHeaders": true}, + } + headers := anthropicHeaders(model, ai.Context{}, AnthropicOptions{StreamOptions: ai.StreamOptions{APIKey: "matrix-token", SessionID: "session-1"}}, false) + if headers["X-Session-Affinity"] != "session-1" { + t.Fatalf("expected forwarded session affinity, got %#v", headers) + } + + disabled := anthropicHeaders(model, ai.Context{}, AnthropicOptions{StreamOptions: ai.StreamOptions{APIKey: "matrix-token", SessionID: "session-1", CacheRetention: ai.CacheRetentionNone}}, false) + if _, ok := disabled["X-Session-Affinity"]; ok { + t.Fatalf("cacheRetention=none should suppress session affinity, got %#v", disabled) + } +} + func TestAnthropicHeadersIncludeWebFetchBetaFromMetadata(t *testing.T) { model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic} headers := anthropicHeaders(model, ai.Context{}, AnthropicOptions{StreamOptions: ai.StreamOptions{ @@ -255,3 +274,34 @@ func TestAnthropicHeadersIncludeWebFetchBetaFromMetadata(t *testing.T) { t.Fatalf("expected web fetch beta to compose with generated betas, got %#v", headers) } } + +func TestAnthropicOpus47PlusUsesAdaptiveThinkingByDefault(t *testing.T) { + model := ai.Model{ + ID: "anthropic/claude-opus-4.8", + API: ai.ApiAnthropicMessages, + Provider: ai.ProviderAnthropic, + Reasoning: true, + ReasoningMode: "adaptive", + MaxTokens: 128000, + } + if !getAnthropicCompat(model).ForceAdaptiveThinking { + t.Fatal("expected catalog adaptive reasoning mode to force adaptive thinking") + } + + enabled := true + params := BuildAnthropicParams(model, ai.Context{Messages: []ai.Message{{Role: "user", Content: "hi"}}}, false, AnthropicOptions{ + ThinkingEnabled: &enabled, + Effort: "high", + }) + thinking, ok := params["thinking"].(map[string]any) + if !ok || thinking["type"] != "adaptive" { + t.Fatalf("expected adaptive thinking params, got %#v", params["thinking"]) + } + if _, ok := thinking["budget_tokens"]; ok { + t.Fatalf("adaptive thinking must not include manual budget_tokens: %#v", thinking) + } + outputConfig, ok := params["output_config"].(map[string]any) + if !ok || outputConfig["effort"] != "high" { + t.Fatalf("expected adaptive effort output_config, got %#v", params["output_config"]) + } +} diff --git a/pkg/ai/providers/openai_codex_responses.go b/pkg/ai/providers/openai_codex_responses.go index 19943a15..70a7861e 100644 --- a/pkg/ai/providers/openai_codex_responses.go +++ b/pkg/ai/providers/openai_codex_responses.go @@ -672,7 +672,7 @@ func BuildCodexSSEHeaders(modelHeaders map[string]string, additionalHeaders map[ headers["accept"] = "text/event-stream" headers["content-type"] = "application/json" if sessionID != "" { - headers["session_id"] = sessionID + headers["session-id"] = sessionID headers["x-client-request-id"] = sessionID } return headers @@ -686,7 +686,7 @@ func BuildCodexWebSocketHeaders(modelHeaders map[string]string, additionalHeader delete(headers, "openai-beta") headers["OpenAI-Beta"] = openAIBetaResponsesWebSockets headers["x-client-request-id"] = requestID - headers["session_id"] = requestID + headers["session-id"] = requestID return headers } diff --git a/pkg/ai/providers/openai_codex_responses_test.go b/pkg/ai/providers/openai_codex_responses_test.go index e708c0fe..3a94dac9 100644 --- a/pkg/ai/providers/openai_codex_responses_test.go +++ b/pkg/ai/providers/openai_codex_responses_test.go @@ -104,11 +104,11 @@ func TestCodexAccountAndHeaders(t *testing.T) { if headers["Authorization"] != "Bearer "+token || headers["chatgpt-account-id"] != "acct_123" || headers["OpenAI-Beta"] != "responses=experimental" { t.Fatalf("unexpected sse headers %#v", headers) } - if headers["session_id"] != "session-1" || headers["x-client-request-id"] != "session-1" { + if headers["session-id"] != "session-1" || headers["x-client-request-id"] != "session-1" { t.Fatalf("expected session headers, got %#v", headers) } wsHeaders := BuildCodexWebSocketHeaders(headers, nil, accountID, token, "request-1") - if wsHeaders["OpenAI-Beta"] != openAIBetaResponsesWebSockets || wsHeaders["session_id"] != "request-1" { + if wsHeaders["OpenAI-Beta"] != openAIBetaResponsesWebSockets || wsHeaders["session-id"] != "request-1" { t.Fatalf("unexpected websocket headers %#v", wsHeaders) } if _, ok := wsHeaders["accept"]; ok { diff --git a/pkg/ai/providers/openai_completions.go b/pkg/ai/providers/openai_completions.go index 17680d38..3ee7a115 100644 --- a/pkg/ai/providers/openai_completions.go +++ b/pkg/ai/providers/openai_completions.go @@ -537,12 +537,20 @@ func buildOpenAIClientConfig(model ai.Model, llmContext ai.Context, options ai.S } } if options.SessionID != "" && resolveCacheRetention(options.CacheRetention) != ai.CacheRetentionNone { - headers["x-client-request-id"] = options.SessionID - if model.API == ai.ApiOpenAIResponses || ResolveOpenAICompletionsCompat(model).SendSessionAffinityHeaders { - headers["session_id"] = options.SessionID - } - if model.API == ai.ApiOpenAICompletions && ResolveOpenAICompletionsCompat(model).SendSessionAffinityHeaders { - headers["x-session-affinity"] = options.SessionID + switch model.API { + case ai.ApiOpenAIResponses: + compat := ResolveOpenAIResponsesCompat(model) + headers["x-client-request-id"] = options.SessionID + if compat.SendSessionIDHeader { + headers["session_id"] = options.SessionID + } + case ai.ApiOpenAICompletions: + compat := ResolveOpenAICompletionsCompat(model) + if compat.SendSessionAffinityHeaders { + headers["session_id"] = options.SessionID + headers["x-client-request-id"] = options.SessionID + headers["x-session-affinity"] = options.SessionID + } } } for key, value := range options.Headers { diff --git a/pkg/ai/providers/openai_conversion_test.go b/pkg/ai/providers/openai_conversion_test.go index f3039fb7..a135e4b6 100644 --- a/pkg/ai/providers/openai_conversion_test.go +++ b/pkg/ai/providers/openai_conversion_test.go @@ -553,9 +553,58 @@ func TestOpenAIClientConfigResolvesCloudflareGatewayHeaders(t *testing.T) { if config.Headers["x-model"] != "1" || config.Headers["x-extra"] != "2" { t.Fatalf("expected merged headers, got %#v", config.Headers) } + if _, ok := config.Headers["x-client-request-id"]; ok { + t.Fatalf("default completions compat should not send request id header, got %#v", config.Headers) + } +} + +func TestOpenAIClientConfigResponsesHonorsSessionIDCompat(t *testing.T) { + config, err := buildOpenAIClientConfig(ai.Model{ + API: ai.ApiOpenAIResponses, + Provider: ai.ProviderOpenAI, + BaseURL: "https://api.openai.com/v1", + Compat: map[string]any{"sendSessionIdHeader": false}, + }, ai.Context{}, ai.StreamOptions{APIKey: "token", SessionID: "session-1"}) + if err != nil { + t.Fatal(err) + } if config.Headers["x-client-request-id"] != "session-1" { t.Fatalf("expected request id header, got %#v", config.Headers) } + if _, ok := config.Headers["session_id"]; ok { + t.Fatalf("sendSessionIdHeader=false should suppress session_id, got %#v", config.Headers) + } +} + +func TestOpenAIClientConfigCompletionsAffinityHeadersRequireCompat(t *testing.T) { + defaultConfig, err := buildOpenAIClientConfig(ai.Model{ + API: ai.ApiOpenAICompletions, + Provider: ai.ProviderOpenRouter, + BaseURL: "https://openrouter.ai/api/v1", + }, ai.Context{}, ai.StreamOptions{APIKey: "token", SessionID: "session-1"}) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"session_id", "x-client-request-id", "x-session-affinity"} { + if _, ok := defaultConfig.Headers[key]; ok { + t.Fatalf("default completions compat should not send %s, got %#v", key, defaultConfig.Headers) + } + } + + enabledConfig, err := buildOpenAIClientConfig(ai.Model{ + API: ai.ApiOpenAICompletions, + Provider: ai.ProviderOpenRouter, + BaseURL: "https://openrouter.ai/api/v1", + Compat: map[string]any{"sendSessionAffinityHeaders": true}, + }, ai.Context{}, ai.StreamOptions{APIKey: "token", SessionID: "session-1"}) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"session_id", "x-client-request-id", "x-session-affinity"} { + if enabledConfig.Headers[key] != "session-1" { + t.Fatalf("expected %s=session-1, got %#v", key, enabledConfig.Headers) + } + } } func TestOpenAIClientConfigAddsGitHubCopilotDynamicHeaders(t *testing.T) { diff --git a/pkg/ai/types.go b/pkg/ai/types.go index 7c66f070..0739b88b 100644 --- a/pkg/ai/types.go +++ b/pkg/ai/types.go @@ -8,6 +8,7 @@ type Provider string type ImagesProvider string type ThinkingLevel string type ModelThinkingLevel string +type ModelReasoningMode string type CacheRetention string type Transport string type StopReason string @@ -69,6 +70,8 @@ const ( ModelThinkingLevelOff ModelThinkingLevel = "off" + ModelReasoningModeAdaptive ModelReasoningMode = "adaptive" + CacheRetentionNone CacheRetention = "none" CacheRetentionShort CacheRetention = "short" CacheRetentionLong CacheRetention = "long" @@ -139,6 +142,7 @@ type Model struct { Reasoning bool `json:"reasoning"` ThinkingLevelMap map[ModelThinkingLevel]*string `json:"thinkingLevelMap,omitempty"` DefaultThinkingLevel ModelThinkingLevel `json:"defaultThinkingLevel,omitempty"` + ReasoningMode ModelReasoningMode `json:"reasoningMode,omitempty"` Input []string `json:"input"` Output []string `json:"output,omitempty"` Cost ModelCost `json:"cost"` diff --git a/pkg/ai/utils/http_logging.go b/pkg/ai/utils/http_logging.go index 191fb6c6..0be8dac4 100644 --- a/pkg/ai/utils/http_logging.go +++ b/pkg/ai/utils/http_logging.go @@ -65,7 +65,7 @@ func aiServicesRequestLogger(req *http.Request) zerolog.Logger { if contentType := req.Header.Get("Content-Type"); contentType != "" { logCtx = logCtx.Str("content_type", contentType) } - if requestID := requestHeader(req.Header, "x-client-request-id", "x-request-id", "session_id"); requestID != "" { + if requestID := requestHeader(req.Header, "x-client-request-id", "x-request-id", "session-id", "session_id"); requestID != "" { logCtx = logCtx.Str("request_id", requestID) } return logCtx.Logger() diff --git a/pkg/ai/utils/http_logging_test.go b/pkg/ai/utils/http_logging_test.go index 56b872e8..c84e1aa7 100644 --- a/pkg/ai/utils/http_logging_test.go +++ b/pkg/ai/utils/http_logging_test.go @@ -32,7 +32,7 @@ func TestAIServicesLoggingTransportLogsMetadataWithoutBodies(t *testing.T) { } req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer secret-token") - req.Header.Set("X-Client-Request-ID", "session_123") + req.Header.Set("Session-ID", "session_123") resp, err := WithAIServicesLogging(&http.Client{}).Do(req) if err != nil { @@ -47,6 +47,9 @@ func TestAIServicesLoggingTransportLogsMetadataWithoutBodies(t *testing.T) { if !strings.Contains(output, `"action":"ai_services_http"`) || !strings.Contains(output, `"request_number":`) { t.Fatalf("missing request context:\n%s", output) } + if !strings.Contains(output, `"request_id":"session_123"`) { + t.Fatalf("missing session-id request id:\n%s", output) + } if !strings.Contains(output, `"status_code":401`) || !strings.Contains(output, `"level":"error"`) || !strings.Contains(output, `"duration":`) { t.Fatalf("missing error response metadata:\n%s", output) } diff --git a/pkg/connector/builtin_tools.go b/pkg/connector/builtin_tools.go index 794eb910..9e618660 100644 --- a/pkg/connector/builtin_tools.go +++ b/pkg/connector/builtin_tools.go @@ -20,6 +20,9 @@ func (cl *Client) registerProviderBuiltInToolHooks(agentHarness *harness.AgentHa if event.Model == nil || roomFetchMode(roomConfig) != toolModeNative || event.Model.API != ai.ApiAnthropicMessages { return nil, nil } + if !modelSupportsBuiltInTool(*event.Model, "web_fetch") { + return nil, nil + } if _, ok := nativeWebFetchToolPayload(*event.Model); !ok { return nil, nil } @@ -43,12 +46,12 @@ func (cl *Client) registerProviderBuiltInToolHooks(agentHarness *harness.AgentHa func activeBuiltInToolPayloads(model ai.Model, roomConfig RoomConfig) []map[string]any { out := make([]map[string]any, 0, len(model.BuiltInTools)+2) - if roomSearchMode(roomConfig) == toolModeNative { + if roomSearchMode(roomConfig) == toolModeNative && modelSupportsBuiltInTool(model, "web_search") { if payload, ok := nativeWebSearchToolPayload(model); ok { out = appendBuiltInToolPayload(out, payload) } } - if roomFetchMode(roomConfig) == toolModeNative { + if roomFetchMode(roomConfig) == toolModeNative && modelSupportsBuiltInTool(model, "web_fetch") { if payload, ok := nativeWebFetchToolPayload(model); ok { out = appendBuiltInToolPayload(out, payload) } @@ -63,6 +66,16 @@ func activeBuiltInToolPayloads(model ai.Model, roomConfig RoomConfig) []map[stri return out } +func modelSupportsBuiltInTool(model ai.Model, tool string) bool { + canonical := normalizedBuiltInTool(tool) + for _, supported := range model.BuiltInTools { + if normalizedBuiltInTool(supported) == canonical { + return true + } + } + return false +} + func builtInToolPayload(model ai.Model, roomConfig RoomConfig, tool string) (map[string]any, bool) { switch normalizedBuiltInTool(tool) { case "web_search": diff --git a/pkg/connector/builtin_tools_test.go b/pkg/connector/builtin_tools_test.go index a385f326..0571e44e 100644 --- a/pkg/connector/builtin_tools_test.go +++ b/pkg/connector/builtin_tools_test.go @@ -89,11 +89,11 @@ func TestActiveBuiltInToolPayloadsHonorsNativeFetchMode(t *testing.T) { } } -func TestActiveBuiltInToolPayloadsInjectsNativeModesWithoutCatalogBuiltIns(t *testing.T) { +func TestActiveBuiltInToolPayloadsDoesNotInjectNativeModesWithoutCatalogBuiltIns(t *testing.T) { model := ai.Model{API: ai.ApiGoogleGenerativeAI, Provider: ai.ProviderGoogle} got := activeBuiltInToolPayloads(model, RoomConfig{SearchMode: toolModeNative, FetchMode: toolModeNative}) - if len(got) != 2 || builtInToolKey(got[0]) != "google_search" || builtInToolKey(got[1]) != "url_context" { - t.Fatalf("expected mode-driven native Google tools, got %#v", got) + if len(got) != 0 { + t.Fatalf("native modes should not synthesize unsupported catalog tools, got %#v", got) } } @@ -163,25 +163,25 @@ func TestNativeWebFetchToolPayloadsAreProviderSpecific(t *testing.T) { }, { name: "openrouter responses", - model: ai.Model{API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenRouter}, + model: ai.Model{API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenRouter, BuiltInTools: []string{"web_fetch"}}, wantKey: "type", wantValue: "openrouter:web_fetch", }, { name: "openrouter completions", - model: ai.Model{API: ai.ApiOpenAICompletions, Provider: ai.ProviderOpenRouter}, + model: ai.Model{API: ai.ApiOpenAICompletions, Provider: ai.ProviderOpenRouter, BuiltInTools: []string{"web_fetch"}}, wantKey: "type", wantValue: "openrouter:web_fetch", }, { name: "anthropic", - model: ai.Model{API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic}, + model: ai.Model{API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic, BuiltInTools: []string{"web_fetch"}}, wantKey: "type", wantValue: "web_fetch_20250910", }, { name: "google", - model: ai.Model{API: ai.ApiGoogleGenerativeAI, Provider: ai.ProviderGoogle}, + model: ai.Model{API: ai.ApiGoogleGenerativeAI, Provider: ai.ProviderGoogle, BuiltInTools: []string{"web_fetch"}}, wantKey: "url_context", wantValue: map[string]any{}, }, diff --git a/pkg/connector/chat_tools.go b/pkg/connector/chat_tools.go index 1143f9e3..59c31964 100644 --- a/pkg/connector/chat_tools.go +++ b/pkg/connector/chat_tools.go @@ -179,6 +179,32 @@ func (cl *Client) validateReasoningLevel(model ai.Model, roomConfig RoomConfig) return fmt.Errorf("model %s does not support reasoning level %q", model.ID, level) } +func (cl *Client) configuredReasoningMode(model ai.Model, roomConfig RoomConfig) string { + if roomConfig.ReasoningMode != "" && roomConfig.ReasoningMode != "default" { + return roomConfig.ReasoningMode + } + return string(model.ReasoningMode) +} + +func (cl *Client) reasoningModeForModel(model ai.Model, roomConfig RoomConfig) string { + return cl.configuredReasoningMode(model, roomConfig) +} + +func (cl *Client) validateReasoningMode(model ai.Model, roomConfig RoomConfig) error { + mode := strings.ToLower(strings.TrimSpace(roomConfig.ReasoningMode)) + switch mode { + case "", "default": + return nil + case string(ai.ModelReasoningModeAdaptive): + if model.ReasoningMode == ai.ModelReasoningModeAdaptive { + return nil + } + return fmt.Errorf("model %s does not support reasoning mode %q", model.ID, mode) + default: + return fmt.Errorf("reasoning mode %q is invalid", roomConfig.ReasoningMode) + } +} + func clampRoomReasoningLevel(model ai.Model, level ai.ModelThinkingLevel) ai.ModelThinkingLevel { return ai.ClampThinkingLevel(model, level) } diff --git a/pkg/connector/client.go b/pkg/connector/client.go index a051f28c..df349858 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -1662,6 +1662,7 @@ func applyAIStreamEvent(writer *aistream.Writer, evt ai.AssistantMessageEvent, c case "thinking_delta": writer.ReasoningDelta(evt.ContentIndex, evt.Delta) case "thinking_end": + writer.ReasoningContentSnapshot(evt.ContentIndex, evt.Content) writer.ReasoningMessageEnd(evt.ContentIndex) case "toolcall_start": if toolCall := toolCallFromEvent(); toolCall != nil { diff --git a/pkg/connector/contacts.go b/pkg/connector/contacts.go index 8232e4d6..baa7520f 100644 --- a/pkg/connector/contacts.go +++ b/pkg/connector/contacts.go @@ -445,6 +445,7 @@ func (cl *Client) aiServicesCatalogModels(ctx context.Context, provider aiid.Pro Reasoning: item.reasoning(), ThinkingLevelMap: item.thinkingLevelMap(), DefaultThinkingLevel: item.defaultThinkingLevel(), + ReasoningMode: item.reasoningMode(), Input: item.inputModalities(), Output: item.outputModalities(), ContextWindow: item.contextWindow(), @@ -526,6 +527,7 @@ type aiServicesModelEntry struct { Levels []string `json:"levels"` LevelMap map[string]*string `json:"level_map"` DefaultLevel string `json:"default_level"` + Mode string `json:"mode"` } `json:"reasoning"` Tools *struct { Supported bool `json:"supported"` @@ -658,6 +660,13 @@ func (entry aiServicesModelEntry) reasoning() bool { return entry.Capabilities != nil && entry.Capabilities.Reasoning != nil && entry.Capabilities.Reasoning.Supported } +func (entry aiServicesModelEntry) reasoningMode() ai.ModelReasoningMode { + if entry.Capabilities == nil || entry.Capabilities.Reasoning == nil { + return "" + } + return ai.ModelReasoningMode(strings.ToLower(strings.TrimSpace(entry.Capabilities.Reasoning.Mode))) +} + func (entry aiServicesModelEntry) builtInTools() []string { if entry.Capabilities == nil || entry.Capabilities.Tools == nil || !entry.Capabilities.Tools.Supported { return nil diff --git a/pkg/connector/contacts_test.go b/pkg/connector/contacts_test.go index fc6e2d25..bbd94d4e 100644 --- a/pkg/connector/contacts_test.go +++ b/pkg/connector/contacts_test.go @@ -170,7 +170,7 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { t.Fatalf("unexpected query %s", r.URL.RawQuery) } gotAuth = r.Header.Get("Authorization") - _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT-5.5","capabilities":{"input":{"modalities":["text","image"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["off","minimal","low","medium","high","xhigh"],"level_map":{"xhigh":"xhigh"},"default_level":"off"},"tools":{"supported":true,"built_in":["image_generation"]},"limits":{"context_tokens":1050000,"output_tokens":128000}}},{"id":"minimax/minimax-m2.7","name":"MiniMax M2.7","provider":{"id":"openrouter","model_id":"minimax/minimax-m2.7","api":"openai-responses"},"capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["low","medium","high"],"level_map":{"off":null,"minimal":null},"default_level":"low"}}},{"id":"beeper/fast","name":"Beeper Fast","capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]}}}]}`)) + _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT-5.5","capabilities":{"input":{"modalities":["text","image"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["off","minimal","low","medium","high","xhigh"],"level_map":{"xhigh":"xhigh"},"default_level":"off","mode":"adaptive"},"tools":{"supported":true,"built_in":["image_generation"]},"limits":{"context_tokens":1050000,"output_tokens":128000}}},{"id":"minimax/minimax-m2.7","name":"MiniMax M2.7","provider":{"id":"openrouter","model_id":"minimax/minimax-m2.7","api":"openai-responses"},"capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["low","medium","high"],"level_map":{"off":null,"minimal":null},"default_level":"low"}}},{"id":"beeper/fast","name":"Beeper Fast","capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]}}}]}`)) })) defer server.Close() @@ -212,6 +212,9 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { if got := models[0].ThinkingLevelMap[ai.ModelThinkingLevelXHigh]; got == nil || *got != "xhigh" { t.Fatalf("expected AI Services xhigh map, got %#v", models[0].ThinkingLevelMap) } + if models[0].ReasoningMode != "adaptive" { + t.Fatalf("expected AI Services reasoning mode, got %#v", models[0].ReasoningMode) + } if len(models[0].BuiltInTools) != 1 || models[0].BuiltInTools[0] != "image_generation" { t.Fatalf("expected AI Services built-in tools, got %#v", models[0].BuiltInTools) } diff --git a/pkg/connector/room_state.go b/pkg/connector/room_state.go index fa89dfc4..7b45840e 100644 --- a/pkg/connector/room_state.go +++ b/pkg/connector/room_state.go @@ -26,16 +26,18 @@ type RoomConfig struct { ModelID string AdditionalPrompt string ThinkingLevel string + ReasoningMode string DisabledTools []string SearchMode string FetchMode string - modelStatePresent bool - modelStateModel string - modelStateName string - modelStateReason string - modelStateEventID string - promptStateEventID string + modelStatePresent bool + modelStateModel string + modelStateName string + modelStateReason string + modelStateReasoningMode string + modelStateEventID string + promptStateEventID string } func (c *Connector) aiRoomStateStore() AIRoomStateStore { @@ -64,11 +66,15 @@ func (s AIRoomStateStore) ReadConfig(ctx context.Context, roomID id.RoomID) (Roo config.modelStateModel = firstString(raw, "model") config.modelStateName = firstString(raw, "name") config.modelStateReason = firstString(raw, "reasoning") + config.modelStateReasoningMode = firstString(raw, "reasoning_mode") config.modelStateEventID = eventID applyRoomModelConfig(&config, raw) if _, ok := raw["reasoning"]; !ok { config.ThinkingLevel = "" } + if _, ok := raw["reasoning_mode"]; !ok { + config.ReasoningMode = "" + } stateEventIDs = append(stateEventIDs, eventID) } if raw, eventID, err := s.readRoomState(ctx, reader, roomID, aiid.RoomPromptType); err != nil { @@ -222,6 +228,9 @@ func applyRoomModelConfig(config *RoomConfig, raw map[string]any) { if reasoning := firstString(raw, "reasoning"); reasoning != "" { config.ThinkingLevel = reasoning } + if reasoningMode := firstString(raw, "reasoning_mode"); reasoningMode != "" { + config.ReasoningMode = reasoningMode + } } func splitModelRef(model string) (providerID string, modelID string) { diff --git a/pkg/connector/stream_test.go b/pkg/connector/stream_test.go index 836a1117..b6405b40 100644 --- a/pkg/connector/stream_test.go +++ b/pkg/connector/stream_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -946,6 +947,57 @@ func TestApplyAIStreamEventDoesNotPublishReasoningSignaturesToAGUI(t *testing.T) } } +func TestApplyAIStreamEventPublishesReasoningContentFromEndSnapshot(t *testing.T) { + run := aistream.NewRun("run", "thread", "beeper/gpt-5.5", "assistant:run", "GPT-5.5", timeNow()) + writer := aistream.NewWriter(run, timeNow) + partial := &ai.Message{ + Role: "assistant", + Content: []ai.ContentBlock{{ + Type: "thinking", + Thinking: "final summary", + }}, + } + + applyAIStreamEvent(writer, ai.AssistantMessageEvent{Type: "thinking_start", ContentIndex: 0, Partial: partial}) + applyAIStreamEvent(writer, ai.AssistantMessageEvent{Type: "thinking_end", ContentIndex: 0, Content: "final summary", Partial: partial}) + + var reasoningContent []string + for _, evt := range run.Events { + if evt.Type() == agui.EventReasoningMsgCont { + reasoningContent = append(reasoningContent, fmt.Sprint(evt.Get("delta"))) + } + } + if strings.Join(reasoningContent, "") != "final summary" { + t.Fatalf("expected thinking_end content to be emitted before end, got events=%#v", run.Events) + } +} + +func TestApplyAIStreamEventDoesNotDuplicateReasoningEndSnapshot(t *testing.T) { + run := aistream.NewRun("run", "thread", "beeper/gpt-5.5", "assistant:run", "GPT-5.5", timeNow()) + writer := aistream.NewWriter(run, timeNow) + partial := &ai.Message{ + Role: "assistant", + Content: []ai.ContentBlock{{ + Type: "thinking", + Thinking: "hidden continuity", + }}, + } + + applyAIStreamEvent(writer, ai.AssistantMessageEvent{Type: "thinking_start", ContentIndex: 0, Partial: partial}) + applyAIStreamEvent(writer, ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: 0, Delta: "hidden continuity", Partial: partial}) + applyAIStreamEvent(writer, ai.AssistantMessageEvent{Type: "thinking_end", ContentIndex: 0, Content: "hidden continuity", Partial: partial}) + + var reasoningContent []string + for _, evt := range run.Events { + if evt.Type() == agui.EventReasoningMsgCont { + reasoningContent = append(reasoningContent, fmt.Sprint(evt.Get("delta"))) + } + } + if strings.Join(reasoningContent, "") != "hidden continuity" || len(reasoningContent) != 1 { + t.Fatalf("expected reasoning content once, got %#v events=%#v", reasoningContent, run.Events) + } +} + func TestApplyAIStreamEventIgnoresRawProviderEvent(t *testing.T) { run := aistream.NewRun("run", "thread", "beeper/gpt-5.5", "assistant:run", "GPT-5.5", timeNow()) writer := aistream.NewWriter(run, timeNow) From c2936fe5df89996ca371dd34ef9e84f6df419d3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 18:59:14 +0200 Subject: [PATCH 08/22] add reasoning mode for anthropic models --- pkg/ai/providers/anthropic.go | 2 +- pkg/connector/bridge_commands.go | 1 + pkg/connector/chat_tools.go | 9 +-- pkg/connector/client.go | 3 + pkg/connector/compaction.go | 3 + pkg/connector/contacts.go | 3 +- pkg/connector/provider.go | 3 + pkg/connector/provider_test.go | 25 +++++++ pkg/connector/room_state_test.go | 15 ++-- pkg/connector/slash_commands.go | 8 +++ pkg/connector/slash_commands_model.go | 100 ++++++++++++++++++++++++-- pkg/connector/slash_commands_state.go | 28 ++++++-- pkg/connector/slash_commands_test.go | 11 ++- 13 files changed, 187 insertions(+), 24 deletions(-) diff --git a/pkg/ai/providers/anthropic.go b/pkg/ai/providers/anthropic.go index d8253644..0db55186 100644 --- a/pkg/ai/providers/anthropic.go +++ b/pkg/ai/providers/anthropic.go @@ -468,7 +468,7 @@ func getAnthropicCompat(model ai.Model) resolvedAnthropicCompat { SupportsLongCacheRetention: compatBool(model, "supportsLongCacheRetention", !isFireworks), SendSessionAffinityHeaders: compatBool(model, "sendSessionAffinityHeaders", isFireworks || isCloudflareGatewayAnthropic), SupportsCacheControlOnTools: compatBool(model, "supportsCacheControlOnTools", !isFireworks), - ForceAdaptiveThinking: model.ReasoningMode == "adaptive" || compatBool(model, "forceAdaptiveThinking", false), + ForceAdaptiveThinking: model.ReasoningMode == ai.ModelReasoningModeAdaptive || compatBool(model, "forceAdaptiveThinking", false), AllowEmptySignature: compatBool(model, "allowEmptySignature", false), } } diff --git a/pkg/connector/bridge_commands.go b/pkg/connector/bridge_commands.go index 4938f509..955989d3 100644 --- a/pkg/connector/bridge_commands.go +++ b/pkg/connector/bridge_commands.go @@ -39,6 +39,7 @@ func (c *Connector) registerAICommands() { processor.AddHandlers( c.bridgeAICommand("model", "Show or set the AI model for this room.", "[model]"), c.bridgeAICommand("reasoning", "Show or set the reasoning level for this room.", "[off|minimal|low|medium|high|xhigh]"), + c.bridgeAICommand("reasoning-mode", "Show or set the reasoning mode for this room.", "[default|adaptive]"), c.bridgeAICommand("system-prompt", "Show, set, or clear this room's additional system prompt.", "[prompt|clear]"), c.bridgeAICommand("abort", "Abort the active AI response or compaction.", ""), c.bridgeAICommand("stop", "Stop the active AI response or compaction.", ""), diff --git a/pkg/connector/chat_tools.go b/pkg/connector/chat_tools.go index 59c31964..613fab3b 100644 --- a/pkg/connector/chat_tools.go +++ b/pkg/connector/chat_tools.go @@ -180,10 +180,11 @@ func (cl *Client) validateReasoningLevel(model ai.Model, roomConfig RoomConfig) } func (cl *Client) configuredReasoningMode(model ai.Model, roomConfig RoomConfig) string { - if roomConfig.ReasoningMode != "" && roomConfig.ReasoningMode != "default" { - return roomConfig.ReasoningMode + mode := strings.ToLower(strings.TrimSpace(roomConfig.ReasoningMode)) + if mode != "" && mode != "default" { + return mode } - return string(model.ReasoningMode) + return strings.ToLower(strings.TrimSpace(string(model.ReasoningMode))) } func (cl *Client) reasoningModeForModel(model ai.Model, roomConfig RoomConfig) string { @@ -196,7 +197,7 @@ func (cl *Client) validateReasoningMode(model ai.Model, roomConfig RoomConfig) e case "", "default": return nil case string(ai.ModelReasoningModeAdaptive): - if model.ReasoningMode == ai.ModelReasoningModeAdaptive { + if strings.EqualFold(string(model.ReasoningMode), string(ai.ModelReasoningModeAdaptive)) { return nil } return fmt.Errorf("model %s does not support reasoning mode %q", model.ID, mode) diff --git a/pkg/connector/client.go b/pkg/connector/client.go index df349858..62417c7e 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -303,6 +303,9 @@ func (cl *Client) handleMatrixMessageWithConfig(ctx context.Context, msg *bridge if err := cl.validateReasoningLevel(model, roomConfig); err != nil { return nil, err } + if err := cl.validateReasoningMode(model, roomConfig); err != nil { + return nil, err + } prompt, err := msgconv.FromMatrix(ctx, cl.Main.Bridge.Matrix.BotIntent(), msg) if err != nil { return nil, err diff --git a/pkg/connector/compaction.go b/pkg/connector/compaction.go index 8f01370f..1ab5e54e 100644 --- a/pkg/connector/compaction.go +++ b/pkg/connector/compaction.go @@ -71,6 +71,9 @@ func (cl *Client) compactPortalSession(ctx context.Context, portal *bridgev2.Por if err := cl.validateReasoningLevel(model, roomConfig); err != nil { return harness.CompactResult{}, err } + if err := cl.validateReasoningMode(model, roomConfig); err != nil { + return harness.CompactResult{}, err + } agentHarness, err := harness.NewAgentHarness(harness.AgentHarnessOptions{ Session: agentSession, Model: model, diff --git a/pkg/connector/contacts.go b/pkg/connector/contacts.go index baa7520f..573c2f13 100644 --- a/pkg/connector/contacts.go +++ b/pkg/connector/contacts.go @@ -85,7 +85,8 @@ func (cl *Client) createModelChat(ctx context.Context, provider aiid.ProviderCon return nil, err } reasoning := cl.reasoningLevelForModel(model, roomConfig) - if _, err = cl.applyRoomModelState(ctx, portal, provider, model, canonicalModel, reasoning, applyRoomModelStateOptions{ForceAvatar: created}); err != nil { + reasoningMode := cl.reasoningModeForModel(model, roomConfig) + if _, err = cl.applyRoomModelState(ctx, portal, provider, model, canonicalModel, reasoning, reasoningMode, applyRoomModelStateOptions{ForceAvatar: created}); err != nil { return nil, err } cl.refreshRoomCapabilities(ctx, portal) diff --git a/pkg/connector/provider.go b/pkg/connector/provider.go index a30eab82..06ffc34b 100644 --- a/pkg/connector/provider.go +++ b/pkg/connector/provider.go @@ -139,6 +139,9 @@ func normalizeProviderModel(model ai.Model, provider aiid.ProviderConfig) ai.Mod if model.DefaultThinkingLevel == "" { model.DefaultThinkingLevel = catalogModel.DefaultThinkingLevel } + if model.ReasoningMode == "" { + model.ReasoningMode = catalogModel.ReasoningMode + } } else if len(model.Input) == 0 { model.Input = catalogInputForProviderModel(model) } diff --git a/pkg/connector/provider_test.go b/pkg/connector/provider_test.go index 0d069a1b..3e872056 100644 --- a/pkg/connector/provider_test.go +++ b/pkg/connector/provider_test.go @@ -850,6 +850,31 @@ func TestDefaultReasoningLevelClampsForMandatoryReasoningModel(t *testing.T) { } } +func TestValidateReasoningModeRejectsUnsupportedPair(t *testing.T) { + client := &Client{Main: &Connector{Config: Config{DefaultReasoningLevel: "off"}}} + model := ai.Model{ID: "plain", Input: []string{"text"}} + if err := client.validateReasoningMode(model, RoomConfig{ReasoningMode: "adaptive"}); err == nil { + t.Fatalf("expected unsupported reasoning mode to fail") + } + if err := client.validateReasoningMode(model, RoomConfig{ReasoningMode: "default"}); err != nil { + t.Fatalf("expected default reasoning mode to be accepted: %v", err) + } + if err := client.validateReasoningMode(model, RoomConfig{ReasoningMode: "bad"}); err == nil { + t.Fatalf("expected invalid reasoning mode to fail") + } +} + +func TestReasoningModeDefaultsFromModelCatalog(t *testing.T) { + client := &Client{Main: &Connector{Config: Config{DefaultReasoningLevel: "off"}}} + model := ai.Model{ID: "anthropic/claude-opus-4.8", ReasoningMode: ai.ModelReasoningModeAdaptive} + if got := client.reasoningModeForModel(model, RoomConfig{}); got != "adaptive" { + t.Fatalf("expected catalog reasoning mode, got %q", got) + } + if err := client.validateReasoningMode(model, RoomConfig{ReasoningMode: "adaptive"}); err != nil { + t.Fatalf("expected adaptive reasoning mode to be accepted: %v", err) + } +} + func TestNormalizeProviderModelDoesNotInheritDefaultProviderCatalogMetadata(t *testing.T) { model := normalizeProviderModel(ai.Model{ ID: "minimax/minimax-m2.7", diff --git a/pkg/connector/room_state_test.go b/pkg/connector/room_state_test.go index e11c2edd..d2f03812 100644 --- a/pkg/connector/room_state_test.go +++ b/pkg/connector/room_state_test.go @@ -8,19 +8,26 @@ import ( func TestApplyRoomModelConfigUsesProviderModelRefAndReasoning(t *testing.T) { config := RoomConfig{} - applyRoomModelConfig(&config, map[string]any{"model": "openrouter/openai/gpt-5", "reasoning": "high"}) - if config.ProviderID != "openrouter" || config.ModelID != "openai/gpt-5" || config.ThinkingLevel != "high" { + applyRoomModelConfig(&config, map[string]any{"model": "openrouter/openai/gpt-5", "reasoning": "high", "reasoning_mode": "adaptive"}) + if config.ProviderID != "openrouter" || config.ModelID != "openai/gpt-5" || config.ThinkingLevel != "high" || config.ReasoningMode != "adaptive" { t.Fatalf("unexpected model config %#v", config) } } func TestRoomModelStateContentIncludesCatalogName(t *testing.T) { - content := roomModelStateContent(ai.Model{ID: "openai/gpt-5.5", Name: "GPT-5.5"}, "beeper/openai/gpt-5.5", "medium") - if content["model"] != "beeper/openai/gpt-5.5" || content["name"] != "GPT-5.5" || content["reasoning"] != "medium" { + content := roomModelStateContent(ai.Model{ID: "openai/gpt-5.5", Name: "GPT-5.5"}, "beeper/openai/gpt-5.5", "medium", "adaptive") + if content["model"] != "beeper/openai/gpt-5.5" || content["name"] != "GPT-5.5" || content["reasoning"] != "medium" || content["reasoning_mode"] != "adaptive" { t.Fatalf("unexpected room model content %#v", content) } } +func TestRoomModelStateContentOmitsEmptyReasoningMode(t *testing.T) { + content := roomModelStateContent(ai.Model{ID: "openai/gpt-5.5"}, "beeper/openai/gpt-5.5", "medium", "") + if _, ok := content["reasoning_mode"]; ok { + t.Fatalf("unexpected empty reasoning mode in room model content %#v", content) + } +} + func TestStringSliceDeduplicatesDisabledTools(t *testing.T) { got := stringSlice([]any{"web_search", "web_search", "", "fetch"}) if len(got) != 2 || got[0] != "web_search" || got[1] != "fetch" { diff --git a/pkg/connector/slash_commands.go b/pkg/connector/slash_commands.go index 9f4e3e1d..e80615bd 100644 --- a/pkg/connector/slash_commands.go +++ b/pkg/connector/slash_commands.go @@ -81,6 +81,14 @@ func aiSlashCommandDefinitions() []aiSlashCommandDefinition { noticeErrors: true, run: runReasoningCommand, }, + { + name: "reasoning-mode", + usage: "/reasoning-mode [default|adaptive]", + description: "Show or set the reasoning mode for this room when the selected model supports it.", + needsRoomConfig: true, + noticeErrors: true, + run: runReasoningModeCommand, + }, { name: "system-prompt", usage: "/system-prompt [prompt|clear]", diff --git a/pkg/connector/slash_commands_model.go b/pkg/connector/slash_commands_model.go index 3b8ca414..1a8dbcac 100644 --- a/pkg/connector/slash_commands_model.go +++ b/pkg/connector/slash_commands_model.go @@ -21,13 +21,17 @@ func runReasoningCommand(cl *Client, ctx context.Context, portal *bridgev2.Porta return cl.applyReasoningCommand(ctx, portal, roomConfig, arg, responder) } +func runReasoningModeCommand(cl *Client, ctx context.Context, portal *bridgev2.Portal, roomConfig RoomConfig, arg string, responder aiCommandResponder) error { + return cl.applyReasoningModeCommand(ctx, portal, roomConfig, arg, responder) +} + func (cl *Client) applyModelCommand(ctx context.Context, portal *bridgev2.Portal, current RoomConfig, requested string, responder aiCommandResponder) error { if strings.TrimSpace(requested) == "" { provider, model, canonical, err := cl.resolveCanonicalRoomModel(ctx, current) if err != nil { return fmt.Errorf("AI room settings rejected: %v", err) } - return responder.Reply(ctx, cl.modelStatusText(canonical, cl.reasoningLevelForModel(model, current), provider)) + return responder.Reply(ctx, cl.modelStatusText(canonical, cl.reasoningLevelForModel(model, current), cl.reasoningModeForModel(model, current), provider)) } target := current providerID, modelID := splitModelRef(requested) @@ -36,6 +40,7 @@ func (cl *Client) applyModelCommand(ctx context.Context, portal *bridgev2.Portal } target.ProviderID = providerID target.ModelID = modelID + target.ReasoningMode = "" provider, model, canonical, err := cl.resolveCanonicalRoomModel(ctx, target) if err != nil { return fmt.Errorf("AI room settings rejected: %v", err) @@ -43,12 +48,16 @@ func (cl *Client) applyModelCommand(ctx context.Context, portal *bridgev2.Portal if err = cl.validateReasoningLevel(model, target); err != nil { return fmt.Errorf("AI room settings rejected: %v", err) } + if err = cl.validateReasoningMode(model, target); err != nil { + return fmt.Errorf("AI room settings rejected: %v", err) + } target.ThinkingLevel = cl.reasoningLevelForModel(model, target) - if _, err = cl.writeRoomModelState(ctx, portal, provider, model, canonical, target.ThinkingLevel); err != nil { + target.ReasoningMode = cl.reasoningModeForModel(model, target) + if _, err = cl.writeRoomModelState(ctx, portal, provider, model, canonical, target.ThinkingLevel, target.ReasoningMode); err != nil { return err } cl.refreshRoomCapabilities(ctx, portal) - return responder.Reply(ctx, fmt.Sprintf("Model set to `%s`. Current reasoning is `%s`.", canonical, target.ThinkingLevel)) + return responder.Reply(ctx, fmt.Sprintf("Model set to `%s`. Current reasoning is `%s`.%s", canonical, target.ThinkingLevel, reasoningModeSentence(target.ReasoningMode))) } func (cl *Client) applyReasoningCommand(ctx context.Context, portal *bridgev2.Portal, current RoomConfig, requested string, responder aiCommandResponder) error { @@ -72,13 +81,53 @@ func (cl *Client) applyReasoningCommand(ctx context.Context, portal *bridgev2.Po if err = cl.validateReasoningLevel(model, target); err != nil { return fmt.Errorf("AI room settings rejected: %v", err) } - if _, err = cl.writeRoomModelState(ctx, portal, provider, model, canonical, reasoning); err != nil { + if err = cl.validateReasoningMode(model, target); err != nil { + return fmt.Errorf("AI room settings rejected: %v", err) + } + target.ReasoningMode = cl.reasoningModeForModel(model, target) + if _, err = cl.writeRoomModelState(ctx, portal, provider, model, canonical, reasoning, target.ReasoningMode); err != nil { return err } cl.refreshRoomCapabilities(ctx, portal) return responder.Reply(ctx, fmt.Sprintf("Reasoning set to `%s` for `%s`.", reasoning, canonical)) } +func (cl *Client) applyReasoningModeCommand(ctx context.Context, portal *bridgev2.Portal, current RoomConfig, requested string, responder aiCommandResponder) error { + provider, model, canonical, err := cl.resolveCanonicalRoomModel(ctx, current) + if err != nil { + return fmt.Errorf("AI room settings rejected: %v", err) + } + if strings.TrimSpace(requested) == "" { + return responder.Reply(ctx, reasoningModeStatusText(cl.reasoningModeForModel(model, current), canonical, model)) + } + mode := strings.ToLower(strings.TrimSpace(requested)) + if !validRoomReasoningMode(mode) { + return fmt.Errorf("AI room settings rejected: reasoning mode %q is invalid", requested) + } + target := current + if mode == "default" { + target.ReasoningMode = "" + } else { + target.ReasoningMode = mode + } + if err = cl.validateReasoningLevel(model, target); err != nil { + return fmt.Errorf("AI room settings rejected: %v", err) + } + if err = cl.validateReasoningMode(model, target); err != nil { + return fmt.Errorf("AI room settings rejected: %v", err) + } + target.ThinkingLevel = cl.reasoningLevelForModel(model, target) + target.ReasoningMode = cl.reasoningModeForModel(model, target) + if _, err = cl.writeRoomModelState(ctx, portal, provider, model, canonical, target.ThinkingLevel, target.ReasoningMode); err != nil { + return err + } + cl.refreshRoomCapabilities(ctx, portal) + if mode == "default" { + return responder.Reply(ctx, fmt.Sprintf("Reasoning mode reset to `default` for `%s`.%s", canonical, reasoningModeSentence(target.ReasoningMode))) + } + return responder.Reply(ctx, fmt.Sprintf("Reasoning mode set to `%s` for `%s`.", target.ReasoningMode, canonical)) +} + func displayReasoningLevel(level string) string { if level == "" { return string(ai.ModelThinkingLevelOff) @@ -90,6 +139,24 @@ func reasoningStatusText(current string, canonicalModel string, model ai.Model) return fmt.Sprintf("Current reasoning is `%s` for `%s`. Options: %s.", displayReasoningLevel(current), canonicalModel, reasoningOptionsText(model)) } +func displayReasoningMode(mode string) string { + if mode == "" { + return "default" + } + return mode +} + +func reasoningModeStatusText(current string, canonicalModel string, model ai.Model) string { + return fmt.Sprintf("Current reasoning mode is `%s` for `%s`. Options: %s.", displayReasoningMode(current), canonicalModel, reasoningModeOptionsText(model)) +} + +func reasoningModeSentence(mode string) string { + if mode == "" { + return "" + } + return fmt.Sprintf(" Current reasoning mode is `%s`.", mode) +} + func reasoningOptionsText(model ai.Model) string { levels := ai.GetSupportedThinkingLevels(model) if len(levels) == 0 { @@ -102,8 +169,20 @@ func reasoningOptionsText(model ai.Model) string { return strings.Join(options, ", ") } -func (cl *Client) modelStatusText(currentModel string, currentReasoning string, currentProvider aiid.ProviderConfig) string { - return fmt.Sprintf("Current model is `%s`. Current reasoning is `%s`. Options: %s.", currentModel, currentReasoning, cl.modelOptionsText(currentProvider)) +func reasoningModeOptionsText(model ai.Model) string { + options := []string{"`default`"} + if strings.EqualFold(string(model.ReasoningMode), string(ai.ModelReasoningModeAdaptive)) { + options = append(options, "`adaptive`") + } + return strings.Join(options, ", ") +} + +func (cl *Client) modelStatusText(currentModel string, currentReasoning string, currentReasoningMode string, currentProvider aiid.ProviderConfig) string { + text := fmt.Sprintf("Current model is `%s`. Current reasoning is `%s`.", currentModel, currentReasoning) + if currentReasoningMode != "" { + text += fmt.Sprintf(" Current reasoning mode is `%s`.", currentReasoningMode) + } + return fmt.Sprintf("%s Options: %s.", text, cl.modelOptionsText(currentProvider)) } func (cl *Client) modelOptionsText(currentProvider aiid.ProviderConfig) string { @@ -145,3 +224,12 @@ func validRoomReasoningLevel(level string) bool { return false } } + +func validRoomReasoningMode(mode string) bool { + switch mode { + case "", "default", string(ai.ModelReasoningModeAdaptive): + return true + default: + return false + } +} diff --git a/pkg/connector/slash_commands_state.go b/pkg/connector/slash_commands_state.go index dfae0ee3..270b2159 100644 --- a/pkg/connector/slash_commands_state.go +++ b/pkg/connector/slash_commands_state.go @@ -48,11 +48,22 @@ func (cl *Client) normalizeRoomStateForPrompt(ctx context.Context, msg *bridgev2 } return config, cl.commandHandledResponse(msg, "invalid-settings"), true, nil } + if err = cl.validateReasoningMode(model, config); err != nil { + if !config.modelStatePresent { + return config, nil, false, err + } + cl.logAIRoomSettingsError(ctx, msg, err, "AI room settings rejected") + if noticeErr := cl.sendCommandNotice(ctx, msg.Portal, fmt.Sprintf("AI room settings rejected: %v.", err)); noticeErr != nil { + return config, nil, false, noticeErr + } + return config, cl.commandHandledResponse(msg, "invalid-settings"), true, nil + } config.ThinkingLevel = cl.reasoningLevelForModel(model, config) - normalized := config.modelStatePresent && (config.modelStateModel != canonical || config.modelStateReason != config.ThinkingLevel) + config.ReasoningMode = cl.reasoningModeForModel(model, config) + normalized := config.modelStatePresent && (config.modelStateModel != canonical || config.modelStateReason != config.ThinkingLevel || config.modelStateReasoningMode != config.ReasoningMode) nameChanged := config.modelStatePresent && model.Name != "" && config.modelStateName != model.Name if normalized || nameChanged { - if _, err = cl.writeRoomModelState(ctx, msg.Portal, provider, model, canonical, config.ThinkingLevel); err != nil { + if _, err = cl.writeRoomModelState(ctx, msg.Portal, provider, model, canonical, config.ThinkingLevel, config.ReasoningMode); err != nil { return config, nil, false, err } cl.refreshRoomCapabilities(ctx, msg.Portal) @@ -111,12 +122,12 @@ type applyRoomModelStateOptions struct { ForceAvatar bool } -func (cl *Client) writeRoomModelState(ctx context.Context, portal *bridgev2.Portal, provider aiid.ProviderConfig, model ai.Model, canonicalModel string, reasoning string) (string, error) { - return cl.applyRoomModelState(ctx, portal, provider, model, canonicalModel, reasoning, applyRoomModelStateOptions{}) +func (cl *Client) writeRoomModelState(ctx context.Context, portal *bridgev2.Portal, provider aiid.ProviderConfig, model ai.Model, canonicalModel string, reasoning string, reasoningMode string) (string, error) { + return cl.applyRoomModelState(ctx, portal, provider, model, canonicalModel, reasoning, reasoningMode, applyRoomModelStateOptions{}) } -func (cl *Client) applyRoomModelState(ctx context.Context, portal *bridgev2.Portal, provider aiid.ProviderConfig, model ai.Model, canonicalModel string, reasoning string, opts applyRoomModelStateOptions) (string, error) { - content := roomModelStateContent(model, canonicalModel, reasoning) +func (cl *Client) applyRoomModelState(ctx context.Context, portal *bridgev2.Portal, provider aiid.ProviderConfig, model ai.Model, canonicalModel string, reasoning string, reasoningMode string, opts applyRoomModelStateOptions) (string, error) { + content := roomModelStateContent(model, canonicalModel, reasoning, reasoningMode) eventID, err := cl.writeAIRoomState(ctx, portal, aiid.RoomModelType, content) if err != nil { return eventID, err @@ -128,7 +139,7 @@ func (cl *Client) applyRoomModelState(ctx context.Context, portal *bridgev2.Port return eventID, nil } -func roomModelStateContent(model ai.Model, canonicalModel string, reasoning string) map[string]any { +func roomModelStateContent(model ai.Model, canonicalModel string, reasoning string, reasoningMode string) map[string]any { content := map[string]any{"model": canonicalModel} if model.Name != "" { content["name"] = model.Name @@ -136,6 +147,9 @@ func roomModelStateContent(model ai.Model, canonicalModel string, reasoning stri if reasoning != "" { content["reasoning"] = reasoning } + if reasoningMode != "" { + content["reasoning_mode"] = reasoningMode + } return content } diff --git a/pkg/connector/slash_commands_test.go b/pkg/connector/slash_commands_test.go index 50dcaf84..7d7ba122 100644 --- a/pkg/connector/slash_commands_test.go +++ b/pkg/connector/slash_commands_test.go @@ -32,6 +32,8 @@ func TestParseAISlashCommand(t *testing.T) { {body: "/model", name: "model", ok: true}, {body: " /reasoning high ", name: "reasoning", arg: "high", ok: true}, {body: "/reasoning", name: "reasoning", ok: true}, + {body: "/reasoning-mode adaptive", name: "reasoning-mode", arg: "adaptive", ok: true}, + {body: "/reasoning-mode", name: "reasoning-mode", ok: true}, {body: "/reasoniing low", ok: false}, {body: "/system-prompt be terse", name: "system-prompt", arg: "be terse", ok: true}, {body: "/system-prompt", name: "system-prompt", ok: true}, @@ -193,7 +195,14 @@ func TestCurrentCommandResponseText(t *testing.T) { if !strings.Contains(status, "Options: `off`, `minimal`, `low`, `medium`, `high`.") { t.Fatalf("reasoning status is missing supported options:\n%s", status) } - modelStatus := canonicalTestClient().modelStatusText("beeper/gpt-5.5", "off", aiid.ProviderConfig{ + modeStatus := reasoningModeStatusText("adaptive", "beeper/anthropic/claude-opus-4.8", ai.Model{ID: "anthropic/claude-opus-4.8", ReasoningMode: ai.ModelReasoningModeAdaptive}) + if !strings.Contains(modeStatus, "Current reasoning mode is `adaptive` for `beeper/anthropic/claude-opus-4.8`.") { + t.Fatalf("reasoning mode status is missing current value:\n%s", modeStatus) + } + if !strings.Contains(modeStatus, "Options: `default`, `adaptive`.") { + t.Fatalf("reasoning mode status is missing supported options:\n%s", modeStatus) + } + modelStatus := canonicalTestClient().modelStatusText("beeper/gpt-5.5", "off", "", aiid.ProviderConfig{ ID: "beeper", Models: []ai.Model{{ID: "gpt-5.5"}, {ID: "openai/gpt-5.5"}}, }) From b93722f5b53b2019acdeca53894f608fe2e9e9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 19:16:54 +0200 Subject: [PATCH 09/22] mroe parsing --- pkg/ai/providers/openai_shared.go | 368 +++++++++++++++++++++---- pkg/ai/providers/openai_stream_test.go | 200 ++++++++++++++ pkg/connector/room_state_events.go | 40 +++ pkg/connector/room_state_test.go | 48 ++++ pkg/connector/stream_test.go | 44 +++ 5 files changed, 647 insertions(+), 53 deletions(-) create mode 100644 pkg/connector/room_state_events.go diff --git a/pkg/ai/providers/openai_shared.go b/pkg/ai/providers/openai_shared.go index e4e65e9a..90f5951b 100644 --- a/pkg/ai/providers/openai_shared.go +++ b/pkg/ai/providers/openai_shared.go @@ -748,22 +748,114 @@ func hasToolHistory(messages []ai.Message) bool { type responsesStreamState struct { blocks []ai.ContentBlock currentIndex int + currentItemID string currentItemType string currentItem map[string]any currentMessagePartType string hasReasoningSummaryPart bool + itemIndexByID map[string]int + itemIDByOutputIndex map[int]string + itemTypeByID map[string]string + messagePartTypeByID map[string]string + reasoningSummaryByID map[string]bool toolArgsByIndex map[int]string + toolArgsByItemID map[string]string nativeToolsByItemID map[string]ai.ToolCall } func newResponsesStreamState() *responsesStreamState { - return &responsesStreamState{currentIndex: -1, toolArgsByIndex: map[int]string{}, nativeToolsByItemID: map[string]ai.ToolCall{}} + return &responsesStreamState{ + currentIndex: -1, + itemIndexByID: map[string]int{}, + itemIDByOutputIndex: map[int]string{}, + itemTypeByID: map[string]string{}, + messagePartTypeByID: map[string]string{}, + reasoningSummaryByID: map[string]bool{}, + toolArgsByIndex: map[int]string{}, + toolArgsByItemID: map[string]string{}, + nativeToolsByItemID: map[string]ai.ToolCall{}, + } } func responsesItemID(item map[string]any) string { return strings.TrimSpace(stringFromAny(item["id"])) } +func responsesEventItemID(event map[string]any) string { + return strings.TrimSpace(stringFromAny(event["item_id"])) +} + +func (s *responsesStreamState) eventItemID(event map[string]any) string { + if id := responsesEventItemID(event); id != "" { + return id + } + if outputIndex, ok := event["output_index"]; ok { + if id := s.itemIDByOutputIndex[intFromAny(outputIndex)]; id != "" { + return id + } + } + return s.currentItemID +} + +func (s *responsesStreamState) eventContentIndex(event map[string]any) int { + itemID := s.eventItemID(event) + if itemID != "" { + if index, ok := s.itemIndexByID[itemID]; ok { + return index + } + } + return s.currentIndex +} + +func (s *responsesStreamState) eventItemType(event map[string]any) string { + itemID := s.eventItemID(event) + if itemID != "" { + if itemType := s.itemTypeByID[itemID]; itemType != "" { + return itemType + } + } + return s.currentItemType +} + +func (s *responsesStreamState) setItemIndex(itemID, itemType string, index int) { + if itemID == "" { + return + } + s.itemIndexByID[itemID] = index + s.itemTypeByID[itemID] = itemType +} + +func (s *responsesStreamState) setEventOutputIndex(event map[string]any, itemID string) { + if itemID == "" { + return + } + if outputIndex, ok := event["output_index"]; ok { + s.itemIDByOutputIndex[intFromAny(outputIndex)] = itemID + } +} + +func appendMissingPrefix(current, full string) string { + if full == "" || full == current { + return "" + } + if current == "" { + return full + } + if strings.HasPrefix(full, current) { + return full[len(current):] + } + return "" +} + +func responsesPartText(part map[string]any, partType string) string { + switch partType { + case "refusal": + return stringFromAny(part["refusal"]) + default: + return stringFromAny(part["text"]) + } +} + func responsesNativeToolCall(item map[string]any, fallbackIndex int, provider ai.Provider) (ai.ToolCall, bool) { name, _, ok := responsesNativeToolInfo(item, provider) if !ok { @@ -892,7 +984,10 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out case "response.output_item.added": item, _ := event["item"].(map[string]any) itemType, _ := item["type"].(string) + itemID := responsesItemID(item) s.currentItem = item + s.currentItemID = itemID + s.setEventOutputIndex(event, itemID) s.currentItemType = itemType s.currentMessagePartType = "" s.hasReasoningSummaryPart = false @@ -900,11 +995,13 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out case "reasoning": s.blocks = append(s.blocks, ai.ContentBlock{Type: "thinking"}) s.currentIndex = len(s.blocks) - 1 + s.setItemIndex(itemID, itemType, s.currentIndex) output.Content = s.blocks push(ai.AssistantMessageEvent{Type: "thinking_start", ContentIndex: s.currentIndex, Partial: output}) case "message": s.blocks = append(s.blocks, ai.ContentBlock{Type: "text"}) s.currentIndex = len(s.blocks) - 1 + s.setItemIndex(itemID, itemType, s.currentIndex) output.Content = s.blocks push(ai.AssistantMessageEvent{Type: "text_start", ContentIndex: s.currentIndex, Partial: output}) case "function_call": @@ -912,20 +1009,25 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out arguments := fmt.Sprint(item["arguments"]) s.blocks = append(s.blocks, ai.ContentBlock{Type: "toolCall", ID: id, Name: fmt.Sprint(item["name"]), Arguments: parseJSONMap(arguments)}) s.currentIndex = len(s.blocks) - 1 + s.setItemIndex(itemID, itemType, s.currentIndex) s.toolArgsByIndex[s.currentIndex] = arguments + if itemID != "" { + s.toolArgsByItemID[itemID] = arguments + } output.Content = s.blocks toolCall := ai.ToolCall{Type: "toolCall", ID: id, Name: s.blocks[s.currentIndex].Name, Arguments: s.blocks[s.currentIndex].Arguments} push(ai.AssistantMessageEvent{Type: "toolcall_start", ContentIndex: s.currentIndex, ToolCall: &toolCall, Partial: output}) case "image_generation_call": s.blocks = append(s.blocks, imageBlockFromGenerationItem(item, ai.ContentBlock{})) s.currentIndex = len(s.blocks) - 1 + s.setItemIndex(itemID, itemType, s.currentIndex) output.Content = s.blocks case "web_search_call", "openrouter:web_search", "openrouter:web_fetch": toolCall, ok := responsesNativeToolCall(item, len(s.nativeToolsByItemID), model.Provider) if !ok { return } - s.nativeToolsByItemID[responsesItemID(item)] = toolCall + s.nativeToolsByItemID[itemID] = toolCall s.currentIndex = -1 output.Content = s.blocks push(ai.AssistantMessageEvent{Type: "toolcall_start", ToolCall: &toolCall, Partial: output}) @@ -934,60 +1036,200 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out } } case "response.reasoning_summary_part.added": - if s.currentItemType == "reasoning" { - s.hasReasoningSummaryPart = true + itemID := s.eventItemID(event) + if s.eventItemType(event) == "reasoning" { + if itemID != "" { + s.reasoningSummaryByID[itemID] = true + } + if itemID == "" || itemID == s.currentItemID { + s.hasReasoningSummaryPart = true + } } case "response.reasoning_summary_text.delta": - if s.hasReasoningSummaryPart && s.currentIndex >= 0 && s.blocks[s.currentIndex].Type == "thinking" { + itemID := s.eventItemID(event) + index := s.eventContentIndex(event) + hasSummaryPart := s.hasReasoningSummaryPart + if itemID != "" { + hasSummaryPart = s.reasoningSummaryByID[itemID] + } + if hasSummaryPart && index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "thinking" { delta, _ := event["delta"].(string) - s.blocks[s.currentIndex].Thinking += delta + s.blocks[index].Thinking += delta output.Content = s.blocks - push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: s.currentIndex, Delta: delta, Partial: output}) + push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: index, Delta: delta, Partial: output}) } case "response.reasoning_text.delta": - if s.currentIndex >= 0 && s.blocks[s.currentIndex].Type == "thinking" { + index := s.eventContentIndex(event) + if index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "thinking" { delta, _ := event["delta"].(string) - s.blocks[s.currentIndex].Thinking += delta + s.blocks[index].Thinking += delta output.Content = s.blocks - push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: s.currentIndex, Delta: delta, Partial: output}) + push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: index, Delta: delta, Partial: output}) + } + case "response.reasoning_text.done": + index := s.eventContentIndex(event) + if index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "thinking" { + text, _ := event["text"].(string) + if delta := appendMissingPrefix(s.blocks[index].Thinking, text); delta != "" { + s.blocks[index].Thinking += delta + output.Content = s.blocks + push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: index, Delta: delta, Partial: output}) + } } case "response.reasoning_summary_part.done": - if s.hasReasoningSummaryPart && s.currentIndex >= 0 && s.blocks[s.currentIndex].Type == "thinking" { - s.blocks[s.currentIndex].Thinking += "\n\n" + itemID := s.eventItemID(event) + index := s.eventContentIndex(event) + hasSummaryPart := s.hasReasoningSummaryPart + if itemID != "" { + hasSummaryPart = s.reasoningSummaryByID[itemID] + } + if hasSummaryPart && index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "thinking" { + if part, ok := event["part"].(map[string]any); ok { + if delta := appendMissingPrefix(s.blocks[index].Thinking, responsesPartText(part, "summary_text")); delta != "" { + s.blocks[index].Thinking += delta + output.Content = s.blocks + push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: index, Delta: delta, Partial: output}) + } + } + if s.blocks[index].Thinking == "" || strings.HasSuffix(s.blocks[index].Thinking, "\n\n") { + return + } + s.blocks[index].Thinking += "\n\n" output.Content = s.blocks - push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: s.currentIndex, Delta: "\n\n", Partial: output}) + push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: index, Delta: "\n\n", Partial: output}) + } + case "response.reasoning_summary_text.done": + itemID := s.eventItemID(event) + index := s.eventContentIndex(event) + hasSummaryPart := s.hasReasoningSummaryPart + if itemID != "" { + hasSummaryPart = s.reasoningSummaryByID[itemID] + } + if hasSummaryPart && index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "thinking" { + text, _ := event["text"].(string) + if delta := appendMissingPrefix(s.blocks[index].Thinking, text); delta != "" { + s.blocks[index].Thinking += delta + output.Content = s.blocks + push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: index, Delta: delta, Partial: output}) + } } case "response.content_part.added": - if s.currentItemType == "message" { + itemID := s.eventItemID(event) + if s.eventItemType(event) == "message" { if part, ok := event["part"].(map[string]any); ok { partType, _ := part["type"].(string) if partType == "output_text" || partType == "refusal" { - s.currentMessagePartType = partType + if itemID != "" { + s.messagePartTypeByID[itemID] = partType + } + if itemID == "" || itemID == s.currentItemID { + s.currentMessagePartType = partType + } } } } case "response.output_text.delta": - if s.currentItemType == "message" && s.currentMessagePartType == "" { - s.currentMessagePartType = "output_text" + itemID := s.eventItemID(event) + index := s.eventContentIndex(event) + partType := s.currentMessagePartType + if itemID != "" { + partType = s.messagePartTypeByID[itemID] + } + if s.eventItemType(event) == "message" && partType == "" { + partType = "output_text" + if itemID != "" { + s.messagePartTypeByID[itemID] = partType + } + if itemID == "" || itemID == s.currentItemID { + s.currentMessagePartType = partType + } } - if s.currentMessagePartType == "output_text" && s.currentIndex >= 0 && s.blocks[s.currentIndex].Type == "text" { + if partType == "output_text" && index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "text" { delta, _ := event["delta"].(string) - s.blocks[s.currentIndex].Text += delta + s.blocks[index].Text += delta output.Content = s.blocks - push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: s.currentIndex, Delta: delta, Partial: output}) + push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: index, Delta: delta, Partial: output}) + } + case "response.output_text.done": + index := s.eventContentIndex(event) + if index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "text" { + text, _ := event["text"].(string) + if delta := appendMissingPrefix(s.blocks[index].Text, text); delta != "" { + s.blocks[index].Text += delta + output.Content = s.blocks + push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: index, Delta: delta, Partial: output}) + } } case "response.refusal.delta": - if s.currentItemType == "message" && s.currentMessagePartType == "" { - s.currentMessagePartType = "refusal" + itemID := s.eventItemID(event) + index := s.eventContentIndex(event) + partType := s.currentMessagePartType + if itemID != "" { + partType = s.messagePartTypeByID[itemID] + } + if s.eventItemType(event) == "message" && partType == "" { + partType = "refusal" + if itemID != "" { + s.messagePartTypeByID[itemID] = partType + } + if itemID == "" || itemID == s.currentItemID { + s.currentMessagePartType = partType + } } - if s.currentMessagePartType == "refusal" && s.currentIndex >= 0 && s.blocks[s.currentIndex].Type == "text" { + if partType == "refusal" && index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "text" { delta, _ := event["delta"].(string) - s.blocks[s.currentIndex].Text += delta + s.blocks[index].Text += delta output.Content = s.blocks - push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: s.currentIndex, Delta: delta, Partial: output}) + push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: index, Delta: delta, Partial: output}) + } + case "response.refusal.done": + itemID := s.eventItemID(event) + index := s.eventContentIndex(event) + if itemID != "" { + s.messagePartTypeByID[itemID] = "refusal" + } + if itemID == "" || itemID == s.currentItemID { + s.currentMessagePartType = "refusal" + } + if index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "text" { + refusal, _ := event["refusal"].(string) + if delta := appendMissingPrefix(s.blocks[index].Text, refusal); delta != "" { + s.blocks[index].Text += delta + output.Content = s.blocks + push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: index, Delta: delta, Partial: output}) + } + } + case "response.content_part.done": + itemID := s.eventItemID(event) + index := s.eventContentIndex(event) + part, _ := event["part"].(map[string]any) + partType, _ := part["type"].(string) + switch partType { + case "output_text", "refusal": + if itemID != "" { + s.messagePartTypeByID[itemID] = partType + } + if itemID == "" || itemID == s.currentItemID { + s.currentMessagePartType = partType + } + if index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "text" { + if delta := appendMissingPrefix(s.blocks[index].Text, responsesPartText(part, partType)); delta != "" { + s.blocks[index].Text += delta + output.Content = s.blocks + push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: index, Delta: delta, Partial: output}) + } + } + case "reasoning_text": + if index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "thinking" { + if delta := appendMissingPrefix(s.blocks[index].Thinking, responsesPartText(part, partType)); delta != "" { + s.blocks[index].Thinking += delta + output.Content = s.blocks + push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: index, Delta: delta, Partial: output}) + } + } } case "response.output_text.annotation.added": - contentIndex := s.currentIndex + contentIndex := s.eventContentIndex(event) if contentIndex < 0 { contentIndex = intFromAny(event["content_index"]) } @@ -996,31 +1238,45 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out push(ai.AssistantMessageEvent{Type: "source", Partial: output}) } case "response.function_call_arguments.delta": - if s.currentIndex >= 0 && s.blocks[s.currentIndex].Type == "toolCall" { + itemID := s.eventItemID(event) + index := s.eventContentIndex(event) + if index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "toolCall" { delta, _ := event["delta"].(string) - s.toolArgsByIndex[s.currentIndex] += delta - s.blocks[s.currentIndex].Arguments = parseJSONMap(s.toolArgsByIndex[s.currentIndex]) + if itemID != "" { + s.toolArgsByItemID[itemID] += delta + s.blocks[index].Arguments = parseJSONMap(s.toolArgsByItemID[itemID]) + } else { + s.toolArgsByIndex[index] += delta + s.blocks[index].Arguments = parseJSONMap(s.toolArgsByIndex[index]) + } output.Content = s.blocks - toolCall := ai.ToolCall{Type: "toolCall", ID: s.blocks[s.currentIndex].ID, Name: s.blocks[s.currentIndex].Name, Arguments: s.blocks[s.currentIndex].Arguments} - push(ai.AssistantMessageEvent{Type: "toolcall_delta", ContentIndex: s.currentIndex, Delta: delta, ToolCall: &toolCall, Partial: output}) + toolCall := ai.ToolCall{Type: "toolCall", ID: s.blocks[index].ID, Name: s.blocks[index].Name, Arguments: s.blocks[index].Arguments} + push(ai.AssistantMessageEvent{Type: "toolcall_delta", ContentIndex: index, Delta: delta, ToolCall: &toolCall, Partial: output}) } case "response.function_call_arguments.done": - if s.currentIndex >= 0 && s.blocks[s.currentIndex].Type == "toolCall" { + itemID := s.eventItemID(event) + index := s.eventContentIndex(event) + if index >= 0 && index < len(s.blocks) && s.blocks[index].Type == "toolCall" { args, _ := event["arguments"].(string) - previous := s.toolArgsByIndex[s.currentIndex] - s.toolArgsByIndex[s.currentIndex] = args - s.blocks[s.currentIndex].Arguments = parseJSONMap(args) + previous := s.toolArgsByIndex[index] + if itemID != "" { + previous = s.toolArgsByItemID[itemID] + s.toolArgsByItemID[itemID] = args + } + s.toolArgsByIndex[index] = args + s.blocks[index].Arguments = parseJSONMap(args) output.Content = s.blocks if strings.HasPrefix(args, previous) && len(args) > len(previous) { - toolCall := ai.ToolCall{Type: "toolCall", ID: s.blocks[s.currentIndex].ID, Name: s.blocks[s.currentIndex].Name, Arguments: s.blocks[s.currentIndex].Arguments} - push(ai.AssistantMessageEvent{Type: "toolcall_delta", ContentIndex: s.currentIndex, Delta: args[len(previous):], ToolCall: &toolCall, Partial: output}) + toolCall := ai.ToolCall{Type: "toolCall", ID: s.blocks[index].ID, Name: s.blocks[index].Name, Arguments: s.blocks[index].Arguments} + push(ai.AssistantMessageEvent{Type: "toolcall_delta", ContentIndex: index, Delta: args[len(previous):], ToolCall: &toolCall, Partial: output}) } } case "response.output_item.done": item, _ := event["item"].(map[string]any) itemType, _ := item["type"].(string) + itemID := responsesItemID(item) if _, _, ok := responsesNativeToolInfo(item, model.Provider); ok { - toolCall := s.nativeToolsByItemID[responsesItemID(item)] + toolCall := s.nativeToolsByItemID[itemID] if toolCall.ID == "" { toolCall, _ = responsesNativeToolCall(item, len(s.nativeToolsByItemID), model.Provider) } else if args := responsesNativeToolArguments(item, toolCall.Name); len(args) > 0 { @@ -1035,48 +1291,54 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out s.currentIndex = -1 return } - if s.currentIndex < 0 && itemType != "image_generation_call" { + index := s.currentIndex + if itemID != "" { + if storedIndex, ok := s.itemIndexByID[itemID]; ok { + index = storedIndex + } + } + if index < 0 && itemType != "image_generation_call" { return } switch itemType { case "reasoning": - s.blocks[s.currentIndex].Thinking = reasoningTextFromItem(item, s.blocks[s.currentIndex].Thinking) - s.blocks[s.currentIndex].ThinkingSignature = mustJSON(item) + s.blocks[index].Thinking = reasoningTextFromItem(item, s.blocks[index].Thinking) + s.blocks[index].ThinkingSignature = mustJSON(item) output.Content = s.blocks - push(ai.AssistantMessageEvent{Type: "thinking_end", ContentIndex: s.currentIndex, Content: s.blocks[s.currentIndex].Thinking, Partial: output}) + push(ai.AssistantMessageEvent{Type: "thinking_end", ContentIndex: index, Content: s.blocks[index].Thinking, Partial: output}) s.currentIndex = -1 case "message": if text := messageTextFromItem(item); text != "" { - s.blocks[s.currentIndex].Text = text + s.blocks[index].Text = text } - output.Citations = append(output.Citations, providerCitationsFromAny(item, model.Provider, s.currentIndex)...) + output.Citations = append(output.Citations, providerCitationsFromAny(item, model.Provider, index)...) if id, ok := item["id"].(string); ok && id != "" { payload := map[string]any{"v": 1, "id": id} if phase, ok := item["phase"].(string); ok && phase != "" { payload["phase"] = phase } - s.blocks[s.currentIndex].TextSignature = mustJSON(payload) + s.blocks[index].TextSignature = mustJSON(payload) } output.Content = s.blocks - push(ai.AssistantMessageEvent{Type: "text_end", ContentIndex: s.currentIndex, Content: s.blocks[s.currentIndex].Text, Partial: output}) + push(ai.AssistantMessageEvent{Type: "text_end", ContentIndex: index, Content: s.blocks[index].Text, Partial: output}) s.currentIndex = -1 case "function_call": - if s.blocks[s.currentIndex].Name == "" { - s.blocks[s.currentIndex].Name = fmt.Sprint(item["name"]) + if s.blocks[index].Name == "" { + s.blocks[index].Name = fmt.Sprint(item["name"]) } if args, ok := item["arguments"].(string); ok && args != "" { - s.blocks[s.currentIndex].Arguments = parseJSONMap(args) + s.blocks[index].Arguments = parseJSONMap(args) } - toolCall := ai.ToolCall{Type: "toolCall", ID: s.blocks[s.currentIndex].ID, Name: s.blocks[s.currentIndex].Name, Arguments: s.blocks[s.currentIndex].Arguments} + toolCall := ai.ToolCall{Type: "toolCall", ID: s.blocks[index].ID, Name: s.blocks[index].Name, Arguments: s.blocks[index].Arguments} output.Content = s.blocks - push(ai.AssistantMessageEvent{Type: "toolcall_end", ContentIndex: s.currentIndex, ToolCall: &toolCall, Partial: output}) + push(ai.AssistantMessageEvent{Type: "toolcall_end", ContentIndex: index, ToolCall: &toolCall, Partial: output}) s.currentIndex = -1 case "image_generation_call": - if s.currentIndex < 0 || s.currentIndex >= len(s.blocks) || s.blocks[s.currentIndex].Type != "image" { + if index < 0 || index >= len(s.blocks) || s.blocks[index].Type != "image" { s.blocks = append(s.blocks, imageBlockFromGenerationItem(item, ai.ContentBlock{})) s.currentIndex = len(s.blocks) - 1 } else { - s.blocks[s.currentIndex] = imageBlockFromGenerationItem(item, s.blocks[s.currentIndex]) + s.blocks[index] = imageBlockFromGenerationItem(item, s.blocks[index]) } output.Content = s.blocks s.currentIndex = -1 diff --git a/pkg/ai/providers/openai_stream_test.go b/pkg/ai/providers/openai_stream_test.go index d6549bc0..4ce9476d 100644 --- a/pkg/ai/providers/openai_stream_test.go +++ b/pkg/ai/providers/openai_stream_test.go @@ -167,6 +167,149 @@ func TestResponsesStreamStateFinalizesReasoningTextToolAndUsage(t *testing.T) { } } +func TestResponsesStreamStateBackfillsTextAndReasoningDoneEvents(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := testStreamModel() + model.API = ai.ApiOpenAIResponses + output := newAssistant(model) + state := newResponsesStreamState() + + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "item": map[string]any{"type": "reasoning", "id": "rs_1"}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.reasoning_text.done", + "item_id": "rs_1", + "text": "full reasoning", + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "item": map[string]any{"type": "message", "id": "msg_1"}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_text.done", + "item_id": "msg_1", + "text": "full answer", + }) + + if state.blocks[0].Thinking != "full reasoning" || state.blocks[1].Text != "full answer" { + t.Fatalf("expected done events to backfill content, got %#v", state.blocks) + } + events := drainAssistantEvents(stream) + var thinkingDelta, textDelta bool + for _, event := range events { + if event.Type == "thinking_delta" && event.Delta == "full reasoning" { + thinkingDelta = true + } + if event.Type == "text_delta" && event.Delta == "full answer" { + textDelta = true + } + } + if !thinkingDelta || !textDelta { + t.Fatalf("expected done events to emit missing deltas, got %#v", events) + } +} + +func TestResponsesStreamStateBackfillsPartDoneEvents(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := testStreamModel() + model.API = ai.ApiOpenAIResponses + output := newAssistant(model) + state := newResponsesStreamState() + + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "output_index": 0, + "item": map[string]any{"type": "reasoning", "id": "rs_1"}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.content_part.done", + "output_index": 0, + "part": map[string]any{"type": "reasoning_text", "text": "part reasoning"}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "output_index": 1, + "item": map[string]any{"type": "message", "id": "msg_1"}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.content_part.done", + "output_index": 1, + "part": map[string]any{"type": "output_text", "text": "part answer"}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "output_index": 2, + "item": map[string]any{"type": "message", "id": "msg_2"}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.refusal.done", + "output_index": 2, + "refusal": "final refusal", + }) + + if state.blocks[0].Thinking != "part reasoning" || state.blocks[1].Text != "part answer" || state.blocks[2].Text != "final refusal" { + t.Fatalf("expected part done events to backfill content, got %#v", state.blocks) + } + events := drainAssistantEvents(stream) + var thinkingDelta, answerDelta, refusalDelta bool + for _, event := range events { + if event.Type == "thinking_delta" && event.Delta == "part reasoning" { + thinkingDelta = true + } + if event.Type == "text_delta" && event.Delta == "part answer" { + answerDelta = true + } + if event.Type == "text_delta" && event.Delta == "final refusal" { + refusalDelta = true + } + } + if !thinkingDelta || !answerDelta || !refusalDelta { + t.Fatalf("expected part done events to emit missing deltas, got %#v", events) + } +} + +func TestResponsesStreamStateBackfillsReasoningSummaryPartDone(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := testStreamModel() + model.API = ai.ApiOpenAIResponses + output := newAssistant(model) + state := newResponsesStreamState() + + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "item": map[string]any{"type": "reasoning", "id": "rs_1"}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.reasoning_summary_part.added", + "item_id": "rs_1", + "part": map[string]any{"type": "summary_text", "text": ""}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.reasoning_summary_part.done", + "item_id": "rs_1", + "part": map[string]any{"type": "summary_text", "text": "summary only on part done"}, + }) + + if state.blocks[0].Thinking != "summary only on part done\n\n" { + t.Fatalf("expected summary part done to backfill summary text, got %#v", state.blocks[0]) + } + events := drainAssistantEvents(stream) + var summaryDelta, separatorDelta bool + for _, event := range events { + if event.Type == "thinking_delta" && event.Delta == "summary only on part done" { + summaryDelta = true + } + if event.Type == "thinking_delta" && event.Delta == "\n\n" { + separatorDelta = true + } + } + if !summaryDelta || !separatorDelta { + t.Fatalf("expected summary part done to emit content and separator deltas, got %#v", events) + } +} + func TestResponsesStreamStateMapsNativeWebSearchCallToToolActivity(t *testing.T) { stream := ai.NewAssistantMessageEventStream() model := testStreamModel() @@ -214,6 +357,63 @@ func TestResponsesStreamStateMapsNativeWebSearchCallToToolActivity(t *testing.T) } } +func TestResponsesStreamStateKeepsReasoningStreamingAcrossNativeToolItems(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := testStreamModel() + model.API = ai.ApiOpenAIResponses + output := newAssistant(model) + state := newResponsesStreamState() + + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "item": map[string]any{"type": "reasoning", "id": "rs_1"}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.reasoning_summary_part.added", + "item_id": "rs_1", + "part": map[string]any{"type": "summary_text", "text": ""}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.added", + "item": map[string]any{ + "type": "web_search_call", + "id": "ws_1", + "status": "searching", + "action": map[string]any{"type": "search", "query": "Amsterdam news"}, + }, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.reasoning_summary_text.delta", + "item_id": "rs_1", + "delta": "checking sources", + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.done", + "item": map[string]any{"type": "web_search_call", "id": "ws_1", "status": "completed", "action": map[string]any{"type": "search", "query": "Amsterdam news"}}, + }) + state.apply(stream, &output, model, OpenAIResponsesOptions{}, map[string]any{ + "type": "response.output_item.done", + "item": map[string]any{"type": "reasoning", "id": "rs_1", "summary": []any{map[string]any{"type": "summary_text", "text": "checking sources"}}}, + }) + + if len(state.blocks) != 1 || state.blocks[0].Thinking != "checking sources" { + t.Fatalf("expected reasoning summary to survive native tool interleave, got %#v", state.blocks) + } + events := drainAssistantEvents(stream) + var thinkingDelta, thinkingEnd bool + for _, event := range events { + if event.Type == "thinking_delta" && event.Delta == "checking sources" { + thinkingDelta = true + } + if event.Type == "thinking_end" && event.Content == "checking sources" { + thinkingEnd = true + } + } + if !thinkingDelta || !thinkingEnd { + t.Fatalf("expected streamed reasoning delta and end, got %#v", events) + } +} + func TestResponsesStreamStateMapsOpenRouterWebSearchToToolActivity(t *testing.T) { stream := ai.NewAssistantMessageEventStream() model := testStreamModel() diff --git a/pkg/connector/room_state_events.go b/pkg/connector/room_state_events.go new file mode 100644 index 00000000..5e923c03 --- /dev/null +++ b/pkg/connector/room_state_events.go @@ -0,0 +1,40 @@ +package connector + +import ( + "reflect" + + "maunium.net/go/mautrix/event" + + "github.com/beeper/ai-bridge/pkg/aiid" +) + +type aiRoomModelStateEventContent struct { + Model string `json:"model,omitempty"` + Name string `json:"name,omitempty"` + Reasoning string `json:"reasoning,omitempty"` + ReasoningMode string `json:"reasoning_mode,omitempty"` +} + +type aiRoomPromptStateEventContent struct { + Prompt string `json:"prompt,omitempty"` +} + +type aiRoomToolsStateEventContent struct { + Disabled []string `json:"disabled,omitempty"` + Search string `json:"search,omitempty"` + Fetch string `json:"fetch,omitempty"` +} + +func init() { + registerAIRoomStateEventContentTypes() +} + +func registerAIRoomStateEventContentTypes() { + event.TypeMap[aiRoomStateEventType(aiid.RoomModelType)] = reflect.TypeOf(aiRoomModelStateEventContent{}) + event.TypeMap[aiRoomStateEventType(aiid.RoomPromptType)] = reflect.TypeOf(aiRoomPromptStateEventContent{}) + event.TypeMap[aiRoomStateEventType(aiid.RoomToolsType)] = reflect.TypeOf(aiRoomToolsStateEventContent{}) +} + +func aiRoomStateEventType(stateType string) event.Type { + return event.Type{Type: stateType, Class: event.StateEventType} +} diff --git a/pkg/connector/room_state_test.go b/pkg/connector/room_state_test.go index d2f03812..b5a4eb21 100644 --- a/pkg/connector/room_state_test.go +++ b/pkg/connector/room_state_test.go @@ -1,9 +1,13 @@ package connector import ( + "reflect" "testing" + "maunium.net/go/mautrix/event" + ai "github.com/beeper/ai-bridge/pkg/ai" + "github.com/beeper/ai-bridge/pkg/aiid" ) func TestApplyRoomModelConfigUsesProviderModelRefAndReasoning(t *testing.T) { @@ -28,6 +32,50 @@ func TestRoomModelStateContentOmitsEmptyReasoningMode(t *testing.T) { } } +func TestAIRoomStateEventContentTypesParse(t *testing.T) { + tests := []struct { + name string + evtType string + raw []byte + want any + }{ + { + name: "model", + evtType: aiid.RoomModelType, + raw: []byte(`{"model":"beeper/openai/gpt-5.5","name":"GPT-5.5","reasoning":"medium","reasoning_mode":"adaptive"}`), + want: &aiRoomModelStateEventContent{ + Model: "beeper/openai/gpt-5.5", + Name: "GPT-5.5", + Reasoning: "medium", + ReasoningMode: "adaptive", + }, + }, + { + name: "prompt", + evtType: aiid.RoomPromptType, + raw: []byte(`{"prompt":"be terse"}`), + want: &aiRoomPromptStateEventContent{Prompt: "be terse"}, + }, + { + name: "tools", + evtType: aiid.RoomToolsType, + raw: []byte(`{"disabled":["web_search"],"search":"native","fetch":"beeper"}`), + want: &aiRoomToolsStateEventContent{Disabled: []string{"web_search"}, Search: "native", Fetch: "beeper"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + content := event.Content{VeryRaw: tt.raw} + if err := content.ParseRaw(aiRoomStateEventType(tt.evtType)); err != nil { + t.Fatalf("expected AI room state to parse: %v", err) + } + if got := content.Parsed; !reflect.DeepEqual(got, tt.want) { + t.Fatalf("unexpected parsed content %#v", got) + } + }) + } +} + func TestStringSliceDeduplicatesDisabledTools(t *testing.T) { got := stringSlice([]any{"web_search", "web_search", "", "fetch"}) if len(got) != 2 || got[0] != "web_search" || got[1] != "fetch" { diff --git a/pkg/connector/stream_test.go b/pkg/connector/stream_test.go index b6405b40..9d703230 100644 --- a/pkg/connector/stream_test.go +++ b/pkg/connector/stream_test.go @@ -74,6 +74,50 @@ func TestStreamPublisherUsesFakeProviderAndPublishesDeltas(t *testing.T) { } } +func TestStreamPublisherPublishesThinkingDeltasLiveAfterThinkingStart(t *testing.T) { + ctx := context.Background() + testAPI := ai.Api("test-stream-thinking") + ai.RegisterAPIProvider(testAPI, func(ctx context.Context, model ai.Model, llmContext ai.Context, options ai.SimpleStreamOptions) *ai.AssistantMessageEventStream { + stream := ai.NewAssistantMessageEventStream() + go func() { + message := ai.Message{Role: "assistant", Content: []ai.ContentBlock{{Type: "thinking", Thinking: "checking context"}, {Type: "text", Text: "answer"}}, StopReason: ai.StopReasonStop} + stream.Push(ai.AssistantMessageEvent{Type: "thinking_start", ContentIndex: 0, Partial: &message}) + stream.Push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: 0, Delta: "checking context", Partial: &message}) + stream.Push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: 1, Delta: "answer", Partial: &message}) + stream.Push(ai.AssistantMessageEvent{Type: "thinking_end", ContentIndex: 0, Content: "checking context", Partial: &message}) + stream.Push(ai.AssistantMessageEvent{Type: "done", Reason: ai.StopReasonStop, Message: &message}) + }() + return stream + }) + defer ai.UnregisterAPIProvider(testAPI) + + publisher := &recordingStreamPublisher{} + client := &Client{} + run := aistream.NewRun("run", "thread", "beeper/fake", "assistant:run", "Fake", timeNow()) + run.MessageID = "assistant:run" + result := client.streamPublisher(publisher, "!room:example.com", "$event", run)(ctx, ai.Model{ID: "fake", API: testAPI}, ai.Context{}, ai.SimpleStreamOptions{}).Result() + if result.StopReason != ai.StopReasonStop { + t.Fatalf("unexpected stream result %#v", result) + } + var sawThinkingStart, sawThinkingDelta, sawTextDelta bool + for _, update := range publisher.updates { + payload, _ := update[aistream.BeeperAIKey].(aistream.BeeperAI) + for _, envelope := range payload.Events { + switch envelope.Event.Type() { + case agui.EventReasoningMsgStart: + sawThinkingStart = true + case agui.EventReasoningMsgCont: + sawThinkingDelta = envelope.Event.Get("delta") == "checking context" + case agui.EventTextMessageContent: + sawTextDelta = envelope.Event.Get("delta") == "answer" + } + } + } + if !sawThinkingStart || !sawThinkingDelta || !sawTextDelta { + t.Fatalf("expected live thinking and text carriers, start=%v thinking=%v text=%v updates=%#v", sawThinkingStart, sawThinkingDelta, sawTextDelta, publisher.updates) + } +} + func TestStreamPublisherFailsAfterIdleTimeout(t *testing.T) { ctx := context.Background() oldTimeout := activeStreamIdleTimeout From f5aa64ebe50596a236db9061c1f6807431feb56a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 19:31:10 +0200 Subject: [PATCH 10/22] Emit toolcall & thinking deltas; handle args Stream initial argument deltas and redacted thinking updates, and harden function-argument handling. anthropic: emit thinking_delta for redacted reasoning and toolcall_delta when tool arguments are present. openai_shared: properly extract function_call arguments (handles nil/non-string cases) and emit toolcall_delta if arguments exist. Tests updated to assert the presence of these delta events. --- pkg/ai/providers/anthropic.go | 5 ++++ pkg/ai/providers/anthropic_test.go | 39 ++++++++++++++++++++++++++ pkg/ai/providers/openai_shared.go | 12 +++++++- pkg/ai/providers/openai_stream_test.go | 10 +++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/pkg/ai/providers/anthropic.go b/pkg/ai/providers/anthropic.go index 0db55186..4b46ce5f 100644 --- a/pkg/ai/providers/anthropic.go +++ b/pkg/ai/providers/anthropic.go @@ -641,6 +641,7 @@ func (s *anthropicStreamState) apply(stream *ai.AssistantMessageEventStream, out s.anthropicIndexes[index] = contentIndex appendContentBlock(output, ai.ContentBlock{Type: "thinking", Thinking: "[Reasoning redacted]", ThinkingSignature: stringFromAny(blockMap["data"]), Redacted: true}) stream.Push(ai.AssistantMessageEvent{Type: "thinking_start", ContentIndex: contentIndex, Partial: output}) + stream.Push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: contentIndex, Delta: "[Reasoning redacted]", Partial: output}) case "tool_use": s.anthropicIndexes[index] = contentIndex name := stringFromAny(blockMap["name"]) @@ -656,6 +657,10 @@ func (s *anthropicStreamState) apply(stream *ai.AssistantMessageEventStream, out } appendContentBlock(output, ai.ContentBlock{Type: "toolCall", ID: stringFromAny(blockMap["id"]), Name: name, Arguments: arguments}) stream.Push(ai.AssistantMessageEvent{Type: "toolcall_start", ContentIndex: contentIndex, Partial: output}) + if len(arguments) > 0 { + toolCall := ai.ToolCall{Type: "toolCall", ID: stringFromAny(blockMap["id"]), Name: name, Arguments: arguments} + stream.Push(ai.AssistantMessageEvent{Type: "toolcall_delta", ContentIndex: contentIndex, Delta: s.partialJSON[contentIndex], ToolCall: &toolCall, Partial: output}) + } case "server_tool_use": toolCall, ok := anthropicNativeServerToolCall(blockMap, index) if !ok { diff --git a/pkg/ai/providers/anthropic_test.go b/pkg/ai/providers/anthropic_test.go index e93a1f33..b192e81f 100644 --- a/pkg/ai/providers/anthropic_test.go +++ b/pkg/ai/providers/anthropic_test.go @@ -67,6 +67,45 @@ func TestAnthropicStreamStatePreservesToolInputFromContentBlockStart(t *testing. if blocks[0].Arguments["path"] != "README.md" { t.Fatalf("expected tool input from content block start, got %#v", blocks[0].Arguments) } + var sawArgs bool + for _, event := range drainAssistantEvents(stream) { + if event.Type == "toolcall_delta" && event.Delta == `{"path":"README.md"}` { + sawArgs = true + } + } + if !sawArgs { + t.Fatalf("expected initial Anthropic tool input to stream as args") + } +} + +func TestAnthropicStreamStateStreamsRedactedThinkingImmediately(t *testing.T) { + stream := ai.NewAssistantMessageEventStream() + model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic} + output := newAssistant(model) + state := newAnthropicStreamState() + + state.apply(stream, &output, model, ai.Context{}, false, map[string]any{ + "type": "content_block_start", + "index": float64(0), + "content_block": map[string]any{ + "type": "redacted_thinking", + "data": "signature", + }, + }) + + events := drainAssistantEvents(stream) + var sawStart, sawDelta bool + for _, event := range events { + if event.Type == "thinking_start" { + sawStart = true + } + if event.Type == "thinking_delta" && event.Delta == "[Reasoning redacted]" { + sawDelta = true + } + } + if !sawStart || !sawDelta { + t.Fatalf("expected redacted thinking to stream immediately, got %#v", events) + } } func TestAnthropicStreamStateMapsNativeWebSearchToToolActivity(t *testing.T) { diff --git a/pkg/ai/providers/openai_shared.go b/pkg/ai/providers/openai_shared.go index 90f5951b..323f5bc0 100644 --- a/pkg/ai/providers/openai_shared.go +++ b/pkg/ai/providers/openai_shared.go @@ -1006,7 +1006,14 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out push(ai.AssistantMessageEvent{Type: "text_start", ContentIndex: s.currentIndex, Partial: output}) case "function_call": id := fmt.Sprintf("%v|%v", item["call_id"], item["id"]) - arguments := fmt.Sprint(item["arguments"]) + arguments := "" + if rawArguments, ok := item["arguments"]; ok && rawArguments != nil { + if text, ok := rawArguments.(string); ok { + arguments = text + } else { + arguments = mustJSON(rawArguments) + } + } s.blocks = append(s.blocks, ai.ContentBlock{Type: "toolCall", ID: id, Name: fmt.Sprint(item["name"]), Arguments: parseJSONMap(arguments)}) s.currentIndex = len(s.blocks) - 1 s.setItemIndex(itemID, itemType, s.currentIndex) @@ -1017,6 +1024,9 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out output.Content = s.blocks toolCall := ai.ToolCall{Type: "toolCall", ID: id, Name: s.blocks[s.currentIndex].Name, Arguments: s.blocks[s.currentIndex].Arguments} push(ai.AssistantMessageEvent{Type: "toolcall_start", ContentIndex: s.currentIndex, ToolCall: &toolCall, Partial: output}) + if arguments != "" { + push(ai.AssistantMessageEvent{Type: "toolcall_delta", ContentIndex: s.currentIndex, Delta: arguments, ToolCall: &toolCall, Partial: output}) + } case "image_generation_call": s.blocks = append(s.blocks, imageBlockFromGenerationItem(item, ai.ContentBlock{})) s.currentIndex = len(s.blocks) - 1 diff --git a/pkg/ai/providers/openai_stream_test.go b/pkg/ai/providers/openai_stream_test.go index 4ce9476d..9d0eaa07 100644 --- a/pkg/ai/providers/openai_stream_test.go +++ b/pkg/ai/providers/openai_stream_test.go @@ -695,6 +695,16 @@ func TestResponsesStreamStateSeedsFunctionCallArgumentsFromAddedItem(t *testing. if len(state.blocks) != 1 || state.blocks[0].Arguments["path"] != "x" { t.Fatalf("expected initial item arguments and delta to combine, got %#v", state.blocks) } + events := drainAssistantEvents(stream) + var sawInitialArgs bool + for _, event := range events { + if event.Type == "toolcall_delta" && event.Delta == `{"path"` { + sawInitialArgs = true + } + } + if !sawInitialArgs { + t.Fatalf("expected initial function-call arguments to stream as args, got %#v", events) + } } func TestFinishResponsesStreamEmitsErrorForFailedResponse(t *testing.T) { From 7efac39920584c388f8beddeea11e760b43a1641 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 19:50:35 +0200 Subject: [PATCH 11/22] Handle activity snapshots and JSON patch deltas Add handling for activity snapshots and activity delta events in FinalBeeperAIMessage: track activityParts and activityContents, apply snapshots and JSON Patch-style deltas, and update message parts accordingly. Introduce helper functions (activityPartID, applyActivityContent, activityDisplay, activityTitle, cloneMap, applyJSONPatch, set/remove JSON pointer, jsonPointerParts) and include activity event types in isActivityEventType. Add unit test TestFinalBeeperAIMessagePreservesActivitySnapshots to verify snapshot+delta application. --- pkg/ai-stream/message.go | 159 ++++++++++++++++++++++++++++++++++- pkg/ai-stream/stream_test.go | 22 +++++ 2 files changed, 180 insertions(+), 1 deletion(-) diff --git a/pkg/ai-stream/message.go b/pkg/ai-stream/message.go index e41cb52f..3bca76f1 100644 --- a/pkg/ai-stream/message.go +++ b/pkg/ai-stream/message.go @@ -312,6 +312,8 @@ func (t Run) FinalBeeperAIMessage(textBudget int, includeThinking bool) UIMessag thinkingParts := map[string]*projectedPart{} toolParts := map[string]MessagePart{} stepParts := map[string]MessagePart{} + activityParts := map[string]MessagePart{} + activityContents := map[string]map[string]any{} sourceParts := map[string]MessagePart{} currentTextMessageID := "" openTextMessageID := "" @@ -600,6 +602,36 @@ func (t Run) FinalBeeperAIMessage(textBudget int, includeThinking bool) UIMessag stepParts[id] = part } part["state"] = state + case agui.EventActivitySnapshot: + messageID := firstString(evt.Get("messageId"), t.MessageID) + activityType := firstString(evt.Get("activityType"), "activity") + id := activityPartID(messageID, activityType) + content, _ := evt.Get("content").(map[string]any) + activityContents[id] = cloneMap(content) + part := activityParts[id] + if part == nil { + part = appendPart(MessagePart{"type": "thinking", "id": id, "messageId": messageID}) + activityParts[id] = part + } + applyActivityContent(part, activityType, activityContents[id]) + case agui.EventActivityDelta: + messageID := firstString(evt.Get("messageId"), t.MessageID) + activityType := firstString(evt.Get("activityType"), "activity") + id := activityPartID(messageID, activityType) + content := activityContents[id] + if content == nil { + content = map[string]any{} + activityContents[id] = content + } + if patch, ok := evt.Get("patch").([]any); ok { + applyJSONPatch(content, patch) + } + part := activityParts[id] + if part == nil { + part = appendPart(MessagePart{"type": "thinking", "id": id, "messageId": messageID}) + activityParts[id] = part + } + applyActivityContent(part, activityType, content) case agui.EventCustom: name, _ := evt.Get("name").(string) value, _ := evt.Get("value").(map[string]any) @@ -846,7 +878,9 @@ func isActivityEventType(eventType string) bool { case agui.EventToolCallStart, agui.EventToolCallArgs, agui.EventToolCallEnd, - agui.EventToolCallResult: + agui.EventToolCallResult, + agui.EventActivitySnapshot, + agui.EventActivityDelta: return true default: return false @@ -904,6 +938,129 @@ func asString(value any) string { } } +func activityPartID(messageID string, activityType string) string { + return fmt.Sprintf("%s:activity:%s", messageID, activityType) +} + +func applyActivityContent(part MessagePart, activityType string, content map[string]any) { + title, text, state := activityDisplay(activityType, content) + if text == "" { + return + } + part["content"] = text + if title != "" && title != text { + part["title"] = title + } else { + delete(part, "title") + } + part["stepId"] = firstString(title, text) + part["state"] = state +} + +func activityDisplay(activityType string, content map[string]any) (title string, text string, state string) { + title = firstString(content["title"], content["label"], content["name"], activityTitle(activityType)) + text = firstString(content["text"], content["content"], content["message"], content["summary"], content["description"], content["status"]) + if text == "" { + text = title + title = "" + } + status := strings.ToLower(firstString(content["state"], content["status"], content["phase"])) + switch status { + case "done", "complete", "completed", "finished", "success", "failed", "error", "cancelled", "canceled": + state = agui.PartStateDone + default: + state = agui.PartStateStreaming + } + return title, text, state +} + +func activityTitle(activityType string) string { + text := strings.NewReplacer("_", " ", "-", " ", ".", " ").Replace(strings.TrimSpace(activityType)) + fields := strings.Fields(text) + for i, field := range fields { + if field == "" { + continue + } + fields[i] = strings.ToUpper(field[:1]) + field[1:] + } + return strings.Join(fields, " ") +} + +func cloneMap(value map[string]any) map[string]any { + out := map[string]any{} + for key, item := range value { + out[key] = item + } + return out +} + +func applyJSONPatch(target map[string]any, patch []any) { + for _, rawOperation := range patch { + operation, ok := rawOperation.(map[string]any) + if !ok { + continue + } + path := firstString(operation["path"]) + if path == "" { + continue + } + switch firstString(operation["op"]) { + case "add", "replace": + setJSONPointer(target, path, operation["value"]) + case "remove": + removeJSONPointer(target, path) + } + } +} + +func setJSONPointer(target map[string]any, path string, value any) { + parts := jsonPointerParts(path) + if len(parts) == 0 { + return + } + current := target + for _, part := range parts[:len(parts)-1] { + next, _ := current[part].(map[string]any) + if next == nil { + next = map[string]any{} + current[part] = next + } + current = next + } + current[parts[len(parts)-1]] = value +} + +func removeJSONPointer(target map[string]any, path string) { + parts := jsonPointerParts(path) + if len(parts) == 0 { + return + } + current := target + for _, part := range parts[:len(parts)-1] { + next, _ := current[part].(map[string]any) + if next == nil { + return + } + current = next + } + delete(current, parts[len(parts)-1]) +} + +func jsonPointerParts(path string) []string { + if path == "" || path == "/" || !strings.HasPrefix(path, "/") { + return nil + } + raw := strings.Split(strings.TrimPrefix(path, "/"), "/") + parts := make([]string, 0, len(raw)) + for _, part := range raw { + part = strings.ReplaceAll(strings.ReplaceAll(part, "~1", "/"), "~0", "~") + if part != "" { + parts = append(parts, part) + } + } + return parts +} + func cloneValueMap(value map[string]any) MessagePart { cp := make(MessagePart, len(value)+1) for key, item := range value { diff --git a/pkg/ai-stream/stream_test.go b/pkg/ai-stream/stream_test.go index f0c22948..7aa11c60 100644 --- a/pkg/ai-stream/stream_test.go +++ b/pkg/ai-stream/stream_test.go @@ -448,6 +448,28 @@ func TestFinalBeeperAIMessagePreservesStepsAsThinking(t *testing.T) { } } +func TestFinalBeeperAIMessagePreservesActivitySnapshots(t *testing.T) { + run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) + builder := agui.NewEventBuilder(DefaultModel, func() time.Time { return time.Unix(10, 0) }) + run.Events = append(run.Events, + builder.ActivitySnapshot(run.MessageID, "web_search", map[string]any{"title": "Searching web", "text": "Looking up docs"}, nil), + builder.ActivityDelta(run.MessageID, "web_search", []any{map[string]any{"op": "replace", "path": "/text", "value": "Found docs"}}), + builder.ActivityDelta(run.MessageID, "web_search", []any{map[string]any{"op": "replace", "path": "/status", "value": "completed"}}), + ) + + uiMessage := run.FinalBeeperAIMessage(0, true) + got := make([]string, 0, len(uiMessage.Parts)) + for _, part := range uiMessage.Parts { + if part["type"] == "thinking" { + got = append(got, fmt.Sprintf("thinking:%s:%s:%s", part["title"], part["content"], part["state"])) + } + } + want := []string{"thinking:Searching web:Found docs:done"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("final UIMessage activity mismatch\ngot: %#v\nwant: %#v\nparts: %#v", got, want, uiMessage.Parts) + } +} + func TestPackRunPreservesLargeCustomEvent(t *testing.T) { run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) builder := agui.NewEventBuilder(DefaultModel, func() time.Time { return time.Unix(10, 0) }) From c452e62f92dae7c1161811744c0b3ea16cef220b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 20:34:33 +0200 Subject: [PATCH 12/22] Refactor streaming, citations, fetch & sources Multiple changes to improve AI streaming, provider citation handling, fetch fallbacks, source extraction, and connector behavior. Highlights: - Dockerfile: add tzdata to runtime image. - ai-stream: introduce message ordinal offsets, deterministic text/reasoning message IDs, HasTextContent(), fix reasoning snapshot logic and ordinal discovery across existing events to avoid ID collisions. - Providers: dedupe citations, prefer nested/rawType when present, add intPointerKey helper, accept additional web search/raw result types, and adjust OpenAI completions headers (x-client-request-id placement). - openai_shared: better native tool keying and handling when item IDs are missing so native tool outputs are tracked reliably. - chattools/fetch: refactor direct vs tool fetch ordering and fallback logic (directFetchResult helper, isHTTPSuccess), adjust content mapping and truncation behavior, preserve markdown/plain text, improve URL extraction and trimming to keep parenthesized URLs. - search/types: make freshness optional pointer and handle nil freshness in payload/params. - connector: export toolresult sources to stream metadata, use HasTextContent() to decide final text fallback, adjust OpenRouter route mapping to default to openai-completions unless responses explicitly requested, and expand source parsing (sourceSlice accepts []map[string]any). - slash commands: improve limits formatting and only include reported sections, with helper to append sections conditionally. - Tests: update and add tests across packages to reflect behavior changes (fetch fallbacks, citation mapping, streaming/tool interactions, limits output, sources parsing). These changes make stream/message ID assignment more robust, prevent duplicate citation/source entries, improve fetch reliability and HTML/markdown handling, and add better tooling metadata propagation. --- Dockerfile | 2 +- pkg/ai-stream/message.go | 2 +- pkg/ai-stream/run.go | 116 +++++++++++++++-- pkg/ai-stream/stream_test.go | 74 +++++------ pkg/ai/providers/citations.go | 35 +++++- pkg/ai/providers/openai_completions.go | 2 +- pkg/ai/providers/openai_conversion_test.go | 53 +++++++- pkg/ai/providers/openai_shared.go | 27 +++- pkg/chattools/chattools_test.go | 30 ++++- pkg/chattools/fetch.go | 56 +++++++-- pkg/chattools/search.go | 26 ++-- pkg/chattools/types.go | 8 +- pkg/connector/client.go | 20 ++- pkg/connector/contacts.go | 5 +- pkg/connector/contacts_test.go | 9 +- pkg/connector/slash_commands_limits.go | 20 ++- pkg/connector/slash_commands_test.go | 10 +- pkg/connector/sources.go | 45 ++++++- pkg/connector/sources_test.go | 37 +++++- pkg/connector/stream_test.go | 137 +++++++++++++++++++++ 20 files changed, 589 insertions(+), 125 deletions(-) diff --git a/Dockerfile b/Dockerfile index 51717e04..27566f09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ FROM ${DOCKER_HUB}/alpine:3.23 ENV UID=1337 \ GID=1337 -RUN apk add --no-cache su-exec ca-certificates bash jq curl yq-go +RUN apk add --no-cache su-exec ca-certificates bash jq curl yq-go tzdata COPY --from=builder /build/ai /usr/bin/ai COPY ./docker-run.sh /docker-run.sh diff --git a/pkg/ai-stream/message.go b/pkg/ai-stream/message.go index 3bca76f1..61554126 100644 --- a/pkg/ai-stream/message.go +++ b/pkg/ai-stream/message.go @@ -726,7 +726,7 @@ func nativeOpenAIWebSearchQuery(part MessagePart) string { return "" } output, _ := part["output"].(map[string]any) - if output == nil || output["native"] != true || firstString(output["provider"]) != "openai" { + if output == nil || output["native"] != true { return "" } input, _ := part["input"].(map[string]any) diff --git a/pkg/ai-stream/run.go b/pkg/ai-stream/run.go index 64c37067..e47f9445 100644 --- a/pkg/ai-stream/run.go +++ b/pkg/ai-stream/run.go @@ -4,6 +4,7 @@ import ( "fmt" "maps" "slices" + "strconv" "strings" "time" "unicode/utf8" @@ -183,9 +184,12 @@ type Writer struct { builder agui.EventBuilder textMessages map[int]string textOpen map[int]bool + textContentWritten bool + textIndexOffset int reasoningMessages map[int]string reasoningOpen map[int]bool reasoningContent map[int]string + reasoningIndexOffset int reasoningPhaseID string reasoningPhaseOpen bool nextSyntheticReasoningIdx int @@ -227,17 +231,21 @@ func NewRun(runID, threadID, model, agentID, agentName string, now time.Time) *R } func NewWriter(run *Run, now func() time.Time) *Writer { + textIndexOffset := nextMessageOrdinal(run, messageOrdinalText) + reasoningIndexOffset := nextMessageOrdinal(run, messageOrdinalReasoning) return &Writer{ - Run: run, - builder: agui.NewEventBuilder(run.Model, now), - textMessages: map[int]string{}, - textOpen: map[int]bool{}, - reasoningMessages: map[int]string{}, - reasoningOpen: map[int]bool{}, - reasoningContent: map[int]string{}, - reasoningPhaseID: "reasoning-" + run.RunID, - lastAccountedChars: utf8.RuneCountInString(run.Text()), - previewText: run.Text(), + Run: run, + builder: agui.NewEventBuilder(run.Model, now), + textMessages: map[int]string{}, + textOpen: map[int]bool{}, + textIndexOffset: textIndexOffset, + reasoningMessages: map[int]string{}, + reasoningOpen: map[int]bool{}, + reasoningContent: map[int]string{}, + reasoningIndexOffset: reasoningIndexOffset, + reasoningPhaseID: "reasoning-" + run.RunID, + lastAccountedChars: utf8.RuneCountInString(run.Text()), + previewText: run.Text(), } } @@ -266,9 +274,14 @@ func (w *Writer) TextDelta(index int, delta string) { return } messageID := w.ensureTextMessage(index) + w.textContentWritten = true w.Add(w.builder.TextMessageContent(messageID, delta)) } +func (w *Writer) HasTextContent() bool { + return w != nil && w.textContentWritten +} + func (w *Writer) TextEnd(index int) { w.initState() messageID := w.textMessages[index] @@ -689,10 +702,11 @@ func (w *Writer) ensureReasoningMessage(index int) string { } func (w *Writer) textMessageID(index int) string { - if index <= 0 { + ordinal := w.textIndexOffset + max(index, 0) + if ordinal == 0 { return w.Run.MessageID } - return fmt.Sprintf("%s-text-%d", w.Run.MessageID, index) + return fmt.Sprintf("%s-text-%d", w.Run.MessageID, ordinal) } func (w *Writer) reasoningMessageID(index int) string { @@ -700,7 +714,7 @@ func (w *Writer) reasoningMessageID(index int) string { index = w.nextSyntheticReasoningIdx w.nextSyntheticReasoningIdx++ } - return fmt.Sprintf("%s-reasoning-%d", w.Run.MessageID, index) + return fmt.Sprintf("%s-reasoning-%d", w.Run.MessageID, w.reasoningIndexOffset+max(index, 0)) } func (w *Writer) toolResultMessageID(toolCallID string) string { @@ -769,6 +783,82 @@ func sortedOpenIndexes(open map[int]bool) []int { return slices.Sorted(maps.Keys(indexes)) } +type messageOrdinalKind int + +const ( + messageOrdinalText messageOrdinalKind = iota + messageOrdinalReasoning +) + +func nextMessageOrdinal(run *Run, kind messageOrdinalKind) int { + if run == nil { + return 0 + } + maxOrdinal := -1 + for _, evt := range run.Events { + if !messageOrdinalEvent(evt.Type(), kind) { + continue + } + messageID, _ := evt.Get("messageId").(string) + if messageID == "" { + continue + } + ordinal, ok := messageOrdinal(run.MessageID, messageID, kind) + if ok && ordinal > maxOrdinal { + maxOrdinal = ordinal + } + } + if maxOrdinal < 0 { + return 0 + } + return maxOrdinal + 1 +} + +func messageOrdinalEvent(eventType string, kind messageOrdinalKind) bool { + switch kind { + case messageOrdinalText: + switch eventType { + case agui.EventTextMessageStart, agui.EventTextMessageContent, agui.EventTextMessageChunk, agui.EventTextMessageEnd: + return true + default: + return false + } + case messageOrdinalReasoning: + switch eventType { + case agui.EventReasoningMsgStart, agui.EventReasoningMsgCont, agui.EventReasoningMsgChunk, agui.EventReasoningMsgEnd: + return true + default: + return false + } + default: + return false + } +} + +func messageOrdinal(baseID, messageID string, kind messageOrdinalKind) (int, bool) { + switch kind { + case messageOrdinalText: + if messageID == baseID { + return 0, true + } + prefix := baseID + "-text-" + if !strings.HasPrefix(messageID, prefix) { + return 0, false + } + ordinal, err := strconv.Atoi(strings.TrimPrefix(messageID, prefix)) + return ordinal, err == nil && ordinal > 0 + case messageOrdinalReasoning: + prefix := baseID + "-reasoning-" + if !strings.HasPrefix(messageID, prefix) { + return 0, false + } + ordinal, err := strconv.Atoi(strings.TrimPrefix(messageID, prefix)) + return ordinal, err == nil && ordinal >= 0 + default: + return 0, false + } +} + func toolResultState(result any) string { value := result if raw, ok := result.(string); ok { diff --git a/pkg/ai-stream/stream_test.go b/pkg/ai-stream/stream_test.go index 7aa11c60..971fb743 100644 --- a/pkg/ai-stream/stream_test.go +++ b/pkg/ai-stream/stream_test.go @@ -386,41 +386,45 @@ func TestFinalBeeperAIMessageKeepsStreamingThinkingPartEmptyUntilTokensArrive(t } } -func TestFinalBeeperAIMessageDedupesNativeOpenAIWebSearchRows(t *testing.T) { - run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) - writer := NewWriter(run, func() time.Time { return time.Unix(10, 0) }) - query := "pop culture latest Oscars Grammys Wikipedia" - writer.ToolStart("ws_aggregate", "web_search", 0, nil) - writer.ToolEnd("ws_aggregate", "web_search", map[string]any{ - "query": query, - "queries": []any{query, "latest entertainment news film music television"}, - }, map[string]any{ - "state": "complete", - "status": "success", - "provider": "openai", - "native": true, - "query": query, - "queries": []any{query, "latest entertainment news film music television"}, - }) - writer.ToolStart("ws_single", "web_search", 0, nil) - writer.ToolEnd("ws_single", "web_search", map[string]any{"query": query}, map[string]any{ - "state": "complete", - "status": "success", - "provider": "openai", - "native": true, - "query": query, - }) - writer.Finish(agui.FinishReasonStop) - - uiMessage := run.FinalBeeperAIMessage(0, true) - var webSearchIDs []string - for _, part := range uiMessage.Parts { - if part["type"] == "tool-call" && part["name"] == "web_search" { - webSearchIDs = append(webSearchIDs, firstString(part["toolCallId"])) - } - } - if !reflect.DeepEqual(webSearchIDs, []string{"ws_single"}) { - t.Fatalf("expected only the final native OpenAI web_search row, got IDs=%#v parts=%#v", webSearchIDs, uiMessage.Parts) +func TestFinalBeeperAIMessageDedupesNativeWebSearchRows(t *testing.T) { + for _, provider := range []string{"openai", "openrouter"} { + t.Run(provider, func(t *testing.T) { + run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) + writer := NewWriter(run, func() time.Time { return time.Unix(10, 0) }) + query := "pop culture latest Oscars Grammys Wikipedia" + writer.ToolStart("ws_aggregate", "web_search", 0, nil) + writer.ToolEnd("ws_aggregate", "web_search", map[string]any{ + "query": query, + "queries": []any{query, "latest entertainment news film music television"}, + }, map[string]any{ + "state": "complete", + "status": "success", + "provider": provider, + "native": true, + "query": query, + "queries": []any{query, "latest entertainment news film music television"}, + }) + writer.ToolStart("ws_single", "web_search", 0, nil) + writer.ToolEnd("ws_single", "web_search", map[string]any{"query": query}, map[string]any{ + "state": "complete", + "status": "success", + "provider": provider, + "native": true, + "query": query, + }) + writer.Finish(agui.FinishReasonStop) + + uiMessage := run.FinalBeeperAIMessage(0, true) + var webSearchIDs []string + for _, part := range uiMessage.Parts { + if part["type"] == "tool-call" && part["name"] == "web_search" { + webSearchIDs = append(webSearchIDs, firstString(part["toolCallId"])) + } + } + if !reflect.DeepEqual(webSearchIDs, []string{"ws_single"}) { + t.Fatalf("expected only the final native web_search row, got IDs=%#v parts=%#v", webSearchIDs, uiMessage.Parts) + } + }) } } diff --git a/pkg/ai/providers/citations.go b/pkg/ai/providers/citations.go index 083f1ff6..2d6d3f6e 100644 --- a/pkg/ai/providers/citations.go +++ b/pkg/ai/providers/citations.go @@ -10,18 +10,38 @@ import ( func providerCitationsFromAny(value any, provider ai.Provider, contentIndex int) []ai.Citation { out := []ai.Citation{} + seen := map[string]struct{}{} + add := func(citation ai.Citation) { + key := fmt.Sprintf("%s|%s|%s|%s", citation.URL, citation.RawType, intPointerKey(citation.StartIndex), intPointerKey(citation.EndIndex)) + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + out = append(out, citation) + } if data, ok := value.(map[string]any); ok { - out = append(out, googleGroundingCitationsFromMap(data, provider, contentIndex)...) - out = append(out, googleURLContextCitationsFromMap(data, provider, contentIndex)...) + for _, citation := range googleGroundingCitationsFromMap(data, provider, contentIndex) { + add(citation) + } + for _, citation := range googleURLContextCitationsFromMap(data, provider, contentIndex) { + add(citation) + } } walkProviderCitationMaps(value, func(item map[string]any) { if citation, ok := providerCitationFromMap(item, provider, contentIndex); ok { - out = append(out, citation) + add(citation) } }) return out } +func intPointerKey(value *int) string { + if value == nil { + return "" + } + return strconv.Itoa(*value) +} + func googleGroundingCitationsFromMap(data map[string]any, provider ai.Provider, contentIndex int) []ai.Citation { metadata, _ := data["groundingMetadata"].(map[string]any) if metadata == nil { @@ -171,16 +191,19 @@ func walkProviderCitationMaps(value any, emit func(map[string]any)) { } func providerCitationFromMap(data map[string]any, provider ai.Provider, contentIndex int) (ai.Citation, bool) { - rawType := strings.ToLower(stringFromAny(data["type"])) + rawType := strings.ToLower(firstCitationString(stringFromAny(data["rawType"]), stringFromAny(data["type"]))) citationData := data if nested, _ := data["url_citation"].(map[string]any); nested != nil { citationData = mergeCitationMaps(data, nested) } else if nested, _ := data["urlCitation"].(map[string]any); nested != nil { citationData = mergeCitationMaps(data, nested) } - rawType = firstCitationString(rawType, strings.ToLower(stringFromAny(citationData["type"]))) + rawType = firstCitationString( + strings.ToLower(firstCitationString(stringFromAny(citationData["rawType"]), stringFromAny(citationData["type"]))), + rawType, + ) url := firstCitationString(stringFromAny(citationData["url"]), stringFromAny(citationData["uri"])) - if url == "" || (!strings.Contains(rawType, "citation") && rawType != "web_search_result_location" && rawType != "web_fetch_result" && rawType != "openrouter:web_fetch") { + if url == "" || (!strings.Contains(rawType, "citation") && rawType != "web_search_result_location" && rawType != "web_search_result" && rawType != "web_fetch_result" && rawType != "openrouter:web_fetch") { return ai.Citation{}, false } title := stringFromAny(citationData["title"]) diff --git a/pkg/ai/providers/openai_completions.go b/pkg/ai/providers/openai_completions.go index 3ee7a115..736d79f5 100644 --- a/pkg/ai/providers/openai_completions.go +++ b/pkg/ai/providers/openai_completions.go @@ -546,9 +546,9 @@ func buildOpenAIClientConfig(model ai.Model, llmContext ai.Context, options ai.S } case ai.ApiOpenAICompletions: compat := ResolveOpenAICompletionsCompat(model) + headers["x-client-request-id"] = options.SessionID if compat.SendSessionAffinityHeaders { headers["session_id"] = options.SessionID - headers["x-client-request-id"] = options.SessionID headers["x-session-affinity"] = options.SessionID } } diff --git a/pkg/ai/providers/openai_conversion_test.go b/pkg/ai/providers/openai_conversion_test.go index a135e4b6..b247245b 100644 --- a/pkg/ai/providers/openai_conversion_test.go +++ b/pkg/ai/providers/openai_conversion_test.go @@ -203,6 +203,31 @@ func TestProviderCitationsFromAnthropicWebSearchLocation(t *testing.T) { } } +func TestProviderCitationsFromNativeWebSearchResult(t *testing.T) { + citations := providerCitationsFromAny(map[string]any{ + "type": "web_search_result", + "url": "https://example.com/native", + "title": "Native Result", + }, ai.ProviderAnthropic, 0) + if len(citations) != 1 || citations[0].URL != "https://example.com/native" || citations[0].Title != "Native Result" || citations[0].RawType != "web_search_result" { + t.Fatalf("unexpected native search citations %#v", citations) + } +} + +func TestProviderCitationsPreferNestedCitationType(t *testing.T) { + citations := providerCitationsFromAny(map[string]any{ + "type": "annotation", + "url_citation": map[string]any{ + "type": "url_citation", + "url": "https://example.com/nested", + "title": "Nested", + }, + }, ai.ProviderOpenAI, 0) + if len(citations) != 1 || citations[0].URL != "https://example.com/nested" || citations[0].Title != "Nested" { + t.Fatalf("unexpected nested citations %#v", citations) + } +} + func TestProviderCitationsFromAnthropicWebFetchResult(t *testing.T) { citations := providerCitationsFromAny(map[string]any{ "type": "web_fetch_tool_result", @@ -474,6 +499,25 @@ func TestBuildCompletionsParamsUsesDetectedCompat(t *testing.T) { } } +func TestOpenRouterCompletionsCompatDoesNotUseDeveloperRole(t *testing.T) { + model := ai.Model{ + ID: "z-ai/glm-4.5v", + API: ai.ApiOpenAICompletions, + Provider: ai.ProviderOpenRouter, + BaseURL: "https://openrouter.ai/api/v1", + Reasoning: true, + Input: []string{"text", "image"}, + } + params := BuildCompletionsParams(model, ai.Context{SystemPrompt: "You are concise."}, OpenAICompletionsOptions{}) + messages := params["messages"].([]map[string]any) + if messages[0]["role"] != "system" { + t.Fatalf("OpenRouter completions must not use developer role, got %#v", messages[0]) + } + if thinking := params["reasoning"].(map[string]any); thinking["effort"] != "none" { + t.Fatalf("expected OpenRouter default reasoning none, got %#v", thinking) + } +} + func TestRegisterBuiltInsIncludesOpenAICodexResponses(t *testing.T) { ResetAPIProviders() provider, ok := ai.GetAPIProvider(ai.ApiOpenAICodexResponses) @@ -553,8 +597,8 @@ func TestOpenAIClientConfigResolvesCloudflareGatewayHeaders(t *testing.T) { if config.Headers["x-model"] != "1" || config.Headers["x-extra"] != "2" { t.Fatalf("expected merged headers, got %#v", config.Headers) } - if _, ok := config.Headers["x-client-request-id"]; ok { - t.Fatalf("default completions compat should not send request id header, got %#v", config.Headers) + if config.Headers["x-client-request-id"] != "session-1" { + t.Fatalf("default completions compat should send request id header, got %#v", config.Headers) } } @@ -585,11 +629,14 @@ func TestOpenAIClientConfigCompletionsAffinityHeadersRequireCompat(t *testing.T) if err != nil { t.Fatal(err) } - for _, key := range []string{"session_id", "x-client-request-id", "x-session-affinity"} { + for _, key := range []string{"session_id", "x-session-affinity"} { if _, ok := defaultConfig.Headers[key]; ok { t.Fatalf("default completions compat should not send %s, got %#v", key, defaultConfig.Headers) } } + if defaultConfig.Headers["x-client-request-id"] != "session-1" { + t.Fatalf("default completions compat should send request id header, got %#v", defaultConfig.Headers) + } enabledConfig, err := buildOpenAIClientConfig(ai.Model{ API: ai.ApiOpenAICompletions, diff --git a/pkg/ai/providers/openai_shared.go b/pkg/ai/providers/openai_shared.go index 323f5bc0..128b50db 100644 --- a/pkg/ai/providers/openai_shared.go +++ b/pkg/ai/providers/openai_shared.go @@ -130,6 +130,7 @@ func detectOpenAICompletionsCompat(model ai.Model) ResolvedOpenAICompletionsComp isZai := provider == "zai" || strings.Contains(baseURL, "api.z.ai") isTogether := provider == "together" || strings.Contains(baseURL, "api.together.ai") || strings.Contains(baseURL, "api.together.xyz") isMoonshot := provider == "moonshotai" || provider == "moonshotai-cn" || strings.Contains(baseURL, "api.moonshot.") + isOpenRouter := provider == "openrouter" || strings.Contains(baseURL, "openrouter.ai") isCloudflareWorkersAI := provider == "cloudflare-workers-ai" || strings.Contains(baseURL, "api.cloudflare.com") isCloudflareAIGateway := provider == "cloudflare-ai-gateway" || strings.Contains(baseURL, "gateway.ai.cloudflare.com") isA8C := provider == "a8c" || strings.Contains(baseURL, "/proxy/a8c/") @@ -158,7 +159,7 @@ func detectOpenAICompletionsCompat(model ai.Model) ResolvedOpenAICompletionsComp thinkingFormat = "together" } else if isA8C { thinkingFormat = "a8c" - } else if provider == "openrouter" || strings.Contains(baseURL, "openrouter.ai") { + } else if isOpenRouter { thinkingFormat = "openrouter" } maxTokensField := "max_completion_tokens" @@ -171,7 +172,7 @@ func detectOpenAICompletionsCompat(model ai.Model) ResolvedOpenAICompletionsComp } return ResolvedOpenAICompletionsCompat{ SupportsStore: !isNonStandard, - SupportsDeveloperRole: !isNonStandard, + SupportsDeveloperRole: !isNonStandard && !isOpenRouter, SupportsReasoningEffort: !isGrok && !isZai && !isMoonshot && !isTogether && !isCloudflareAIGateway, SupportsUsageInStreaming: true, MaxTokensField: maxTokensField, @@ -785,6 +786,13 @@ func responsesEventItemID(event map[string]any) string { return strings.TrimSpace(stringFromAny(event["item_id"])) } +func responsesNativeToolKey(item map[string]any, toolCall ai.ToolCall) string { + if id := responsesItemID(item); id != "" { + return id + } + return toolCall.ID +} + func (s *responsesStreamState) eventItemID(event map[string]any) string { if id := responsesEventItemID(event); id != "" { return id @@ -938,6 +946,11 @@ func responsesNativeToolResult(item map[string]any, provider ai.Provider) map[st result[key] = value } } + if toolName == "web_search" { + if results := item["results"]; results != nil { + result["results"] = results + } + } if toolName == "fetch" { if url := stringFromAny(item["url"]); url != "" { result["final_url"] = url @@ -1037,7 +1050,12 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out if !ok { return } - s.nativeToolsByItemID[itemID] = toolCall + toolKey := responsesNativeToolKey(item, toolCall) + s.nativeToolsByItemID[toolKey] = toolCall + if itemID == "" { + s.setEventOutputIndex(event, toolKey) + s.currentItemID = toolKey + } s.currentIndex = -1 output.Content = s.blocks push(ai.AssistantMessageEvent{Type: "toolcall_start", ToolCall: &toolCall, Partial: output}) @@ -1285,6 +1303,9 @@ func (s *responsesStreamState) apply(stream *ai.AssistantMessageEventStream, out item, _ := event["item"].(map[string]any) itemType, _ := item["type"].(string) itemID := responsesItemID(item) + if itemID == "" { + itemID = s.eventItemID(event) + } if _, _, ok := responsesNativeToolInfo(item, model.Provider); ok { toolCall := s.nativeToolsByItemID[itemID] if toolCall.ID == "" { diff --git a/pkg/chattools/chattools_test.go b/pkg/chattools/chattools_test.go index 3ad6db80..077ef5e7 100644 --- a/pkg/chattools/chattools_test.go +++ b/pkg/chattools/chattools_test.go @@ -165,12 +165,12 @@ func TestFetchUsesDirectFetchForAssetsWhenToolEndpointConfigured(t *testing.T) { } } -func TestFetchUsesMarkdownAlternateBeforeToolEndpoint(t *testing.T) { +func TestFetchUsesMarkdownAlternateAfterToolEndpointFails(t *testing.T) { exaHit := false client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { if req.URL.Host == "exa.test" { exaHit = true - return testResponse(req, http.StatusOK, "application/json", `{"markdown":"tool result"}`), nil + return testResponse(req, http.StatusBadGateway, "text/plain", "nope"), nil } switch req.URL.Path { case "/page": @@ -188,7 +188,7 @@ func TestFetchUsesMarkdownAlternateBeforeToolEndpoint(t *testing.T) { if err != nil { t.Fatal(err) } - if exaHit || result.FetchMethod != "direct" || result.FinalURL != "https://example.com/from-header.md" || !strings.Contains(result.Markdown, "Header markdown") { + if !exaHit || result.FetchMethod != "direct" || result.FinalURL != "https://example.com/from-header.md" || !strings.Contains(result.Markdown, "Header markdown") { t.Fatalf("unexpected alternate fetch result %#v exaHit=%v", result, exaHit) } } @@ -208,14 +208,14 @@ func TestFetchUsesToolEndpointForPages(t *testing.T) { if payload["url"] != "https://example.com/page" || payload["max_chars"] != float64(100) { t.Fatalf("unexpected fetch payload %#v", payload) } - return testResponse(req, http.StatusOK, "application/json", `{"request_id":"req_1","title":"Page","description":"Page description","url":"https://example.com/page","final_url":"https://example.com/page","markdown":"Extracted page text","published_at":"2026-01-01","author":"A","favicon_url":"https://example.com/favicon.ico","metadata":{"links":["https://example.com/next"]}}`), nil + return testResponse(req, http.StatusOK, "application/json", `{"request_id":"req_1","title":"Page","description":"Page description","url":"https://example.com/page","final_url":"https://example.com/page","text":"Plain page text","markdown":"# Extracted page text","published_at":"2026-01-01","author":"A","favicon_url":"https://example.com/favicon.ico","metadata":{"links":["https://example.com/next"]}}`), nil })} result, err := Fetch(context.Background(), "https://example.com/page", FetchOptions{Timeout: time.Second, ToolEndpoint: "https://exa.test/contents", APIKey: "key", Client: client, MaxChars: 100}) if err != nil { t.Fatal(err) } - if result.FetchMethod != "web_tool" || result.RequestID != "req_1" || result.Title != "Page" || result.Text != "Extracted page text" || result.Markdown != "Extracted page text" { + if result.FetchMethod != "web_tool" || result.RequestID != "req_1" || result.Title != "Page" || result.Text != "Plain page text" || result.Markdown != "# Extracted page text" { t.Fatalf("unexpected fetch result %#v", result) } if result.Description != "Page description" || result.Favicon != "https://example.com/favicon.ico" || result.FaviconURL != "https://example.com/favicon.ico" || result.Published != "2026-01-01" || result.Author != "A" || result.Extras["links"] == nil { @@ -268,6 +268,26 @@ func TestFetchFallsBackToDirectWhenToolEndpointFails(t *testing.T) { } } +func TestFetchNonSuccessDirectFallsBackToToolEndpoint(t *testing.T) { + var directHits, toolHits int + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.Host == "exa.test" { + toolHits++ + return testResponse(req, http.StatusOK, "application/json", `{"url":"https://example.com/not-found.txt","markdown":"Recovered"}`), nil + } + directHits++ + return testResponse(req, http.StatusNotFound, "text/plain", "not found"), nil + })} + + result, err := Fetch(context.Background(), "https://example.com/not-found.txt", FetchOptions{Timeout: time.Second, ToolEndpoint: "https://exa.test/contents", Client: client, MaxBytes: 1024, MaxChars: 100}) + if err != nil { + t.Fatal(err) + } + if directHits != 1 || toolHits != 1 || result.FetchMethod != "web_tool" || result.Text != "Recovered" { + t.Fatalf("unexpected fallback result %#v directHits=%d toolHits=%d", result, directHits, toolHits) + } +} + func TestFetchRejectsUnsupportedScheme(t *testing.T) { if _, err := Fetch(context.Background(), "file:///etc/passwd", FetchOptions{}); err == nil { t.Fatalf("expected unsupported scheme error") diff --git a/pkg/chattools/fetch.go b/pkg/chattools/fetch.go index 293af4bc..42170007 100644 --- a/pkg/chattools/fetch.go +++ b/pkg/chattools/fetch.go @@ -66,17 +66,13 @@ func Fetch(ctx context.Context, rawURL string, options FetchOptions) (FetchResul if options.MaxChars == 0 { options.MaxChars = 20000 } - directResult, directErr := fetchDirect(ctx, rawURL, parsed, options) - if directErr == nil { - if shouldReturnDirectResult(parsed, directResult.ContentType) { - return directResult, nil - } - if isHTMLContentType(directResult.ContentType) || looksLikeHTML(directResult.RawBody) { - if alternateURL := findReadableAlternate(directResult.ResponseHeaders, directResult.RawBody, directResult.FinalURL); alternateURL != "" { - if alternate, err := fetchDirectURL(ctx, alternateURL, options); err == nil && shouldReturnDirectResult(mustParseURL(alternate.FinalURL), alternate.ContentType) { - return alternate, nil - } - } + directFirst := options.ToolEndpoint == "" || shouldReturnDirectURL(parsed) + var directResult FetchResult + var directErr error + if directFirst { + directResult, directErr = fetchDirect(ctx, rawURL, parsed, options) + if result, ok := directFetchResult(ctx, directResult, directErr, options); ok { + return result, nil } } if options.ToolEndpoint != "" { @@ -93,12 +89,42 @@ func Fetch(ctx context.Context, rawURL string, options FetchOptions) (FetchResul Str("target_host", parsed.Host). Msg("Falling back to direct fetch result after web tool fetch failed") } + if !directFirst { + directResult, directErr = fetchDirect(ctx, rawURL, parsed, options) + if result, ok := directFetchResult(ctx, directResult, directErr, options); ok { + return result, nil + } + } if directErr != nil { return FetchResult{}, directErr } + if !isHTTPSuccess(directResult.Status) { + return FetchResult{}, fmt.Errorf("direct fetch failed with HTTP %d", directResult.Status) + } return directResult, nil } +func directFetchResult(ctx context.Context, directResult FetchResult, directErr error, options FetchOptions) (FetchResult, bool) { + if directErr != nil || !isHTTPSuccess(directResult.Status) { + return FetchResult{}, false + } + if shouldReturnDirectResult(mustParseURL(directResult.FinalURL), directResult.ContentType) { + return directResult, true + } + if isHTMLContentType(directResult.ContentType) || looksLikeHTML(directResult.RawBody) { + if alternateURL := findReadableAlternate(directResult.ResponseHeaders, directResult.RawBody, directResult.FinalURL); alternateURL != "" { + if alternate, err := fetchDirectURL(ctx, alternateURL, options); err == nil && isHTTPSuccess(alternate.Status) && shouldReturnDirectResult(mustParseURL(alternate.FinalURL), alternate.ContentType) { + return alternate, true + } + } + } + return FetchResult{}, false +} + +func isHTTPSuccess(status int) bool { + return status >= 200 && status < 300 +} + func fetchDirect(ctx context.Context, rawURL string, parsed *url.URL, options FetchOptions) (FetchResult, error) { client := options.Client if client == nil { @@ -233,7 +259,7 @@ func FetchContents(ctx context.Context, rawURL string, options FetchOptions) (Fe Title: body.Title, Description: body.Description, SiteName: body.SiteName, - Text: firstNonEmpty(body.Markdown, body.Text), + Text: firstNonEmpty(body.Text, body.Markdown), Markdown: firstNonEmpty(body.Markdown, body.Text), Truncated: body.Truncated, RequestID: firstNonEmpty(body.RequestID, body.RequestIDSnake), @@ -250,7 +276,11 @@ func FetchContents(ctx context.Context, rawURL string, options FetchOptions) (Fe if len([]rune(result.Text)) > textMaxChars { runes := []rune(result.Text) result.Text = string(runes[:textMaxChars]) - result.Markdown = result.Text + result.Truncated = true + } + if len([]rune(result.Markdown)) > textMaxChars { + runes := []rune(result.Markdown) + result.Markdown = string(runes[:textMaxChars]) result.Truncated = true } log.Debug(). diff --git a/pkg/chattools/search.go b/pkg/chattools/search.go index 9f9f323e..8d578ec6 100644 --- a/pkg/chattools/search.go +++ b/pkg/chattools/search.go @@ -235,14 +235,16 @@ func searchPayload(query string, limit int, request SearchRequestOptions) map[st addString(payload, "search_context_size", request.SearchContextSize) addString(payload, "category", request.Category) addStrings(payload, "allowed_domains", request.AllowedDomains) - freshness := map[string]any{} - if request.Freshness.Days > 0 { - freshness["days"] = request.Freshness.Days - } - addString(freshness, "published_after", request.Freshness.PublishedAfter) - addString(freshness, "published_before", request.Freshness.PublishedBefore) - if len(freshness) > 0 { - payload["freshness"] = freshness + if request.Freshness != nil { + freshness := map[string]any{} + if request.Freshness.Days > 0 { + freshness["days"] = request.Freshness.Days + } + addString(freshness, "published_after", request.Freshness.PublishedAfter) + addString(freshness, "published_before", request.Freshness.PublishedBefore) + if len(freshness) > 0 { + payload["freshness"] = freshness + } } return payload } @@ -257,9 +259,11 @@ func requestOptions(params any) SearchRequestOptions { out.Category = stringValueParam(values, "category") out.AllowedDomains = stringSliceParam(values, "allowed_domains") if freshness := mapParam(values, "freshness"); freshness != nil { - out.Freshness.Days = intParam(freshness, "days", 0) - out.Freshness.PublishedAfter = stringValueParam(freshness, "published_after") - out.Freshness.PublishedBefore = stringValueParam(freshness, "published_before") + out.Freshness = &SearchFreshness{ + Days: intParam(freshness, "days", 0), + PublishedAfter: stringValueParam(freshness, "published_after"), + PublishedBefore: stringValueParam(freshness, "published_before"), + } } return out } diff --git a/pkg/chattools/types.go b/pkg/chattools/types.go index ee31af60..036f5990 100644 --- a/pkg/chattools/types.go +++ b/pkg/chattools/types.go @@ -55,10 +55,10 @@ type SearchOptions struct { } type SearchRequestOptions struct { - SearchContextSize string `json:"search_context_size,omitempty"` - Category string `json:"category,omitempty"` - AllowedDomains []string `json:"allowed_domains,omitempty"` - Freshness SearchFreshness `json:"freshness,omitempty"` + SearchContextSize string `json:"search_context_size,omitempty"` + Category string `json:"category,omitempty"` + AllowedDomains []string `json:"allowed_domains,omitempty"` + Freshness *SearchFreshness `json:"freshness,omitempty"` } type SearchFreshness struct { diff --git a/pkg/connector/client.go b/pkg/connector/client.go index 62417c7e..cffc947e 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -1258,6 +1258,12 @@ func (cl *Client) streamPublisherWithEndFrom(publisher bridgev2.BeeperStreamPubl writer.Custom("com.beeper.source", source) } } + if evt.Type == "toolresult" && evt.ToolCall != nil { + output := toolOutputEvent{ID: evt.ToolCall.ID, Name: evt.ToolCall.Name, Input: evt.ToolCall.Arguments} + for _, source := range streamSources.addToolOutput(output, evt.CustomValue) { + writer.Custom("com.beeper.source", source) + } + } afterEvents := len(run.Events) maybeSecondVisibleChunk(evt) if !seenFirstDelta && isVisibleAIStreamDelta(evt) { @@ -1715,7 +1721,7 @@ func applyAIStreamEvent(writer *aistream.Writer, evt ai.AssistantMessageEvent, c } func writeFinalTextFallback(writer *aistream.Writer, message ai.Message) { - if writer == nil || writer.Run == nil || writer.Run.Text() != "" || runHasStreamedText(*writer.Run) { + if writer == nil || writer.Run == nil || writer.HasTextContent() { return } if text := msgconv.AssistantText(message); text != "" { @@ -1724,18 +1730,6 @@ func writeFinalTextFallback(writer *aistream.Writer, message ai.Message) { } } -func runHasStreamedText(run aistream.Run) bool { - for _, evt := range run.Events { - switch evt.Type() { - case agui.EventTextMessageContent, agui.EventTextMessageChunk: - if delta, _ := evt.Get("delta").(string); delta != "" { - return true - } - } - } - return false -} - type toolOutputEvent struct { ID string Name string diff --git a/pkg/connector/contacts.go b/pkg/connector/contacts.go index 573c2f13..b8a2a899 100644 --- a/pkg/connector/contacts.go +++ b/pkg/connector/contacts.go @@ -583,7 +583,10 @@ func (entry aiServicesModelEntry) applyProviderRoute(model ai.Model, provider ai model.Provider = ai.Provider("a8c") model.BaseURL = aiServicesProxyBaseURL(provider.BaseURL, "a8c", true) case "openrouter": - model.API = ai.ApiOpenAIResponses + model.API = ai.ApiOpenAICompletions + if entry.Provider.API == string(ai.ApiOpenAIResponses) { + model.API = ai.ApiOpenAIResponses + } model.Provider = ai.ProviderOpenRouter model.BaseURL = aiServicesProxyBaseURL(provider.BaseURL, "openrouter", true) } diff --git a/pkg/connector/contacts_test.go b/pkg/connector/contacts_test.go index bbd94d4e..13499ca5 100644 --- a/pkg/connector/contacts_test.go +++ b/pkg/connector/contacts_test.go @@ -170,7 +170,7 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { t.Fatalf("unexpected query %s", r.URL.RawQuery) } gotAuth = r.Header.Get("Authorization") - _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT-5.5","capabilities":{"input":{"modalities":["text","image"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["off","minimal","low","medium","high","xhigh"],"level_map":{"xhigh":"xhigh"},"default_level":"off","mode":"adaptive"},"tools":{"supported":true,"built_in":["image_generation"]},"limits":{"context_tokens":1050000,"output_tokens":128000}}},{"id":"minimax/minimax-m2.7","name":"MiniMax M2.7","provider":{"id":"openrouter","model_id":"minimax/minimax-m2.7","api":"openai-responses"},"capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["low","medium","high"],"level_map":{"off":null,"minimal":null},"default_level":"low"}}},{"id":"beeper/fast","name":"Beeper Fast","capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]}}}]}`)) + _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT-5.5","capabilities":{"input":{"modalities":["text","image"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["off","minimal","low","medium","high","xhigh"],"level_map":{"xhigh":"xhigh"},"default_level":"off","mode":"adaptive"},"tools":{"supported":true,"built_in":["image_generation"]},"limits":{"context_tokens":1050000,"output_tokens":128000}}},{"id":"minimax/minimax-m2.7","name":"MiniMax M2.7","provider":{"id":"openrouter","model_id":"minimax/minimax-m2.7","api":"openai-completions"},"capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["low","medium","high"],"level_map":{"off":null,"minimal":null},"default_level":"low"}}},{"id":"beeper/fast","name":"Beeper Fast","capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]}}}]}`)) })) defer server.Close() @@ -224,6 +224,9 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { if models[1].DefaultThinkingLevel != ai.ModelThinkingLevelLow || roomThinkingLevelSupported(models[1], ai.ModelThinkingLevelOff) { t.Fatalf("expected MiniMax reasoning to default to low and reject off, got %#v", models[1]) } + if models[1].API != ai.ApiOpenAICompletions || models[1].Provider != ai.ProviderOpenRouter || models[1].BaseURL != server.URL+"/proxy/openrouter/v1" { + t.Fatalf("expected MiniMax OpenRouter route, got %#v", models[1]) + } if roomThinkingLevelSupported(models[1], ai.ModelThinkingLevelMinimal) { t.Fatalf("expected MiniMax reasoning to reject minimal, got %#v", models[1]) } @@ -327,7 +330,7 @@ func TestAIServicesCatalogModelsUsesPublishedProviderRoutes(t *testing.T) { {"id":"x-ai/grok-4.20","name":"Grok 4.20","provider":{"id":"wpcom_xai","model_id":"x-ai/grok-4.20","api":"openai-responses"}}, {"id":"groq/qwen/qwen3-32b","name":"Qwen 3 32B","provider":{"id":"wpcom_groq","model_id":"groq/qwen/qwen3-32b","api":"openai-responses"}}, {"id":"openai/gpt-oss-120b","name":"GPT OSS 120B","provider":{"id":"wpcom_a8c","model_id":"gpt-oss-120b","api":"openai-completions"}}, - {"id":"anthropic/claude-sonnet-4.5","name":"Claude via OpenRouter","metadata":{"family":"claude","provider_logo_url":"/models/providers/anthropic.png"},"provider":{"id":"openrouter","model_id":"anthropic/claude-sonnet-4.5","api":"openai-responses"}} + {"id":"anthropic/claude-sonnet-4.5","name":"Claude via OpenRouter","metadata":{"family":"claude","provider_logo_url":"/models/providers/anthropic.png"},"provider":{"id":"openrouter","model_id":"anthropic/claude-sonnet-4.5","api":"openai-completions"}} ]}`)) })) defer server.Close() @@ -372,7 +375,7 @@ func TestAIServicesCatalogModelsUsesPublishedProviderRoutes(t *testing.T) { if got := byID["openai/gpt-oss-120b"]; got.API != ai.ApiOpenAICompletions || got.Provider != ai.Provider("a8c") || got.BaseURL != server.URL+"/proxy/a8c/v1" { t.Fatalf("unexpected A8C route %#v", got) } - if got := byID["anthropic/claude-sonnet-4.5"]; got.API != ai.ApiOpenAIResponses || got.Provider != ai.ProviderOpenRouter || got.BaseURL != server.URL+"/proxy/openrouter/v1" { + if got := byID["anthropic/claude-sonnet-4.5"]; got.API != ai.ApiOpenAICompletions || got.Provider != ai.ProviderOpenRouter || got.BaseURL != server.URL+"/proxy/openrouter/v1" { t.Fatalf("unexpected OpenRouter route %#v", got) } if got := byID["anthropic/claude-sonnet-4.5"]; got.Compat["provider_logo_url"] != "/models/providers/anthropic.png" || got.Compat["provider_model_id"] != "anthropic/claude-sonnet-4.5" || got.Compat["family"] != "claude" { diff --git a/pkg/connector/slash_commands_limits.go b/pkg/connector/slash_commands_limits.go index b2415a28..ca7243db 100644 --- a/pkg/connector/slash_commands_limits.go +++ b/pkg/connector/slash_commands_limits.go @@ -132,7 +132,14 @@ func aiServicesLimitsURL(proxyBaseURL string) (string, error) { func formatLimitsCommandInfo(limits aiServicesLimitsResponse, now time.Time) string { var text strings.Builder - appendLimitSection(&text, "Models", limits.Windows.LLM, now) + text.WriteString("# AI limits\n\n") + appendLimitSectionIfReported(&text, "Models", limits.Windows.LLM, now) + appendLimitSectionIfReported(&text, "Web Search", limits.Windows.WebTools, now) + appendLimitSectionIfReported(&text, "Transcription", limits.Windows.AudioTranscriptions, now) + appendLimitSectionIfReported(&text, "Audio Generation", limits.Windows.AudioGeneration, now) + if strings.TrimSpace(text.String()) == "# AI limits" { + text.WriteString("No limits reported.\n") + } return text.String() } @@ -165,6 +172,17 @@ func appendLimitSection(text *strings.Builder, label string, windows aiServicesL appendLimitSectionWithUsedFormatter(text, label, windows, now, formatLimitUsed) } +func appendLimitSectionIfReported(text *strings.Builder, label string, windows aiServicesLimitWindows, now time.Time) { + if emptyLimitWindows(windows) { + return + } + if text.Len() > 0 && !strings.HasSuffix(text.String(), "\n\n") { + text.WriteString("\n") + } + appendLimitSection(text, label, windows, now) + text.WriteString("\n") +} + func appendLimitSectionWithUsedFormatter(text *strings.Builder, label string, windows aiServicesLimitWindows, now time.Time, formatUsed func(aiServicesLimitWindow) string) { fmt.Fprintf(text, "## %s\n\n", label) if emptyLimitWindows(windows) { diff --git a/pkg/connector/slash_commands_test.go b/pkg/connector/slash_commands_test.go index 7d7ba122..3fe4ad91 100644 --- a/pkg/connector/slash_commands_test.go +++ b/pkg/connector/slash_commands_test.go @@ -589,17 +589,22 @@ func TestFormatLimitsCommandInfo(t *testing.T) { }, }}, now) for _, want := range []string{ + "# AI limits", "## Models", + "## Web Search", + "## Transcription", + "## Audio Generation", "| Window | Left | Used | Reset |", "| Daily | `75%` | `250 / 1,000` | in 1 day 2 hours 3 minutes |", "| Weekly | Unlimited | `1,234` used | in 1 day 2 hours 3 minutes |", "| Monthly | **Out** | `30,500 / 30,000` | in 1 day 2 hours 3 minutes |", + "`1 / 200,000`", } { if !strings.Contains(text, want) { t.Fatalf("limits info missing %q:\n%s", want, text) } } - for _, notWant := range []string{"AI limits", "## Web Search", "## Transcription", "## Audio Generation", "`199,999`", "2030-01-01T00:00:00Z"} { + for _, notWant := range []string{"2030-01-01T00:00:00Z"} { if strings.Contains(text, notWant) { t.Fatalf("limits info exposed non-summary value %q:\n%s", notWant, text) } @@ -616,6 +621,7 @@ func TestFormatLimitsCommandInfoShowsPerWindowResetsWhenDifferent(t *testing.T) }, }}, now) for _, want := range []string{ + "# AI limits", "## Models", "| Daily | `75%` | Not reported | in 1 day 1 hour 3 minutes |", "| Weekly | `100%` | Not reported | in 7 days |", @@ -625,7 +631,7 @@ func TestFormatLimitsCommandInfoShowsPerWindowResetsWhenDifferent(t *testing.T) t.Fatalf("limits info missing %q:\n%s", want, text) } } - for _, notWant := range []string{"AI limits", "## Web Search", "No limits reported.", "Everything resets"} { + for _, notWant := range []string{"## Web Search", "No limits reported.", "Everything resets"} { if strings.Contains(text, notWant) { t.Fatalf("limits info exposed %q:\n%s", notWant, text) } diff --git a/pkg/connector/sources.go b/pkg/connector/sources.go index 0fceb895..f9ce6ea3 100644 --- a/pkg/connector/sources.go +++ b/pkg/connector/sources.go @@ -76,7 +76,7 @@ func (c *sourceCollector) addWebSearchOutput(output toolOutputEvent, result any) if data == nil { return nil } - rawResults, _ := data["results"].([]any) + rawResults := sourceSlice(data, "results") if len(rawResults) == 0 { return nil } @@ -421,7 +421,7 @@ func (c *sourceCollector) sources() []map[string]any { return out } -var markdownURLPattern = regexp.MustCompile(`https?://[^\s<>"'\]\)]+`) +var markdownURLPattern = regexp.MustCompile(`https?://[^\s<>"']+`) func extractMessageURLs(message ai.Message) []string { text := strings.TrimSpace(messageTextContent(message.Content)) @@ -431,7 +431,7 @@ func extractMessageURLs(message ai.Message) []string { seen := map[string]bool{} var out []string for _, match := range markdownURLPattern.FindAllString(text, -1) { - match = strings.TrimRight(match, ".,;:!?") + match = trimExtractedURL(match) normalized, ok := normalizeSourceURL(match) if !ok || seen[normalized] { continue @@ -442,6 +442,27 @@ func extractMessageURLs(message ai.Message) []string { return out } +func trimExtractedURL(raw string) string { + raw = strings.TrimRight(strings.TrimSpace(raw), ".,;:!?") + for len(raw) > 0 { + last := raw[len(raw)-1] + switch last { + case ')': + if strings.Count(raw, ")") <= strings.Count(raw, "(") { + return raw + } + case ']': + if strings.Count(raw, "]") <= strings.Count(raw, "[") { + return raw + } + default: + return raw + } + raw = strings.TrimRight(raw[:len(raw)-1], ".,;:!?") + } + return raw +} + func messageTextContent(content any) string { switch typed := content.(type) { case string: @@ -569,7 +590,7 @@ func walkProviderSources(value any, emit func(sourceObservation)) { } func providerCitationSource(data map[string]any) (sourceObservation, bool) { - sourceType := strings.ToLower(sourceString(data, "type", "rawType")) + sourceType := strings.ToLower(firstSourceString(sourceString(data, "rawType"), sourceString(data, "type"))) nested, _ := data["url_citation"].(map[string]any) if nested == nil { nested, _ = data["urlCitation"].(map[string]any) @@ -577,10 +598,13 @@ func providerCitationSource(data map[string]any) (sourceObservation, bool) { citation := data if nested != nil { citation = mergeSourceMaps(data, nested) - sourceType = firstSourceString(sourceType, strings.ToLower(sourceString(nested, "type", "rawType"))) + sourceType = firstSourceString( + strings.ToLower(firstSourceString(sourceString(nested, "rawType"), sourceString(nested, "type"))), + sourceType, + ) } rawURL := sourceString(citation, "url", "uri") - if rawURL == "" || (!strings.Contains(sourceType, "citation") && sourceType != "web_search_result_location" && sourceType != "web_fetch_result" && sourceType != "openrouter:web_fetch" && sourceType != "url_context") { + if rawURL == "" || (!strings.Contains(sourceType, "citation") && sourceType != "web_search_result_location" && sourceType != "web_search_result" && sourceType != "web_fetch_result" && sourceType != "openrouter:web_fetch" && sourceType != "url_context") { return sourceObservation{}, false } title := sourceString(citation, "title") @@ -690,6 +714,15 @@ func sourceSlice(data map[string]any, keys ...string) []any { out = append(out, value) } return out + case []map[string]any: + if len(typed) == 0 { + continue + } + out := make([]any, 0, len(typed)) + for _, value := range typed { + out = append(out, value) + } + return out } } return nil diff --git a/pkg/connector/sources_test.go b/pkg/connector/sources_test.go index 679d8a0f..5cee4859 100644 --- a/pkg/connector/sources_test.go +++ b/pkg/connector/sources_test.go @@ -117,6 +117,34 @@ func TestSourceCollectorUsesDescriptionAndFaviconFallbacks(t *testing.T) { t.Fatalf("OpenRouter web fetch source was not mapped: %#v", openRouterFetchSources) } + nestedCitationSources := newSourceCollector().addProviderSources(map[string]any{ + "type": "annotation", + "url_citation": map[string]any{ + "type": "url_citation", + "url": "https://example.com/nested", + "title": "Nested Citation", + }, + }) + if len(nestedCitationSources) != 1 || nestedCitationSources[0]["title"] != "Nested Citation" { + t.Fatalf("nested provider citation was not mapped: %#v", nestedCitationSources) + } + + nativeSearchSources := newSourceCollector().addWebSearchOutput(toolOutputEvent{ + ID: "native-search", + Name: "web_search", + Input: map[string]any{"query": "q"}, + }, map[string]any{ + "native": true, + "results": []map[string]any{{ + "type": "web_search_result", + "url": "https://example.com/native-search", + "title": "Native Search", + }}, + }) + if len(nativeSearchSources) != 1 || nativeSearchSources[0]["title"] != "Native Search" { + t.Fatalf("native search source was not mapped: %#v", nativeSearchSources) + } + messageSources := newSourceCollector().addProviderSources(ai.Message{ Citations: []ai.Citation{{ Type: "url_citation", @@ -155,14 +183,17 @@ func TestSourceCollectorUsesDescriptionAndFaviconFallbacks(t *testing.T) { if len(searchSources) != 1 { t.Fatalf("expected search source, got %#v", searchSources) } - answerSources := collector.addAnswerURLSources(ai.Message{Content: "Use https://example.com/known and https://example.org/new."}) - if len(answerSources) != 2 { + answerSources := collector.addAnswerURLSources(ai.Message{Content: "Use https://example.com/known and https://example.org/new. Per https://en.wikipedia.org/wiki/Mercury_(element), done."}) + if len(answerSources) != 3 { t.Fatalf("expected answer URL sources, got %#v", answerSources) } sources := collector.sources() - if len(sources) != 2 { + if len(sources) != 3 { t.Fatalf("expected canonical known + new sources, got %#v", sources) } + if sources[2]["url"] != "https://en.wikipedia.org/wiki/Mercury_(element)" { + t.Fatalf("parenthesized URL was not preserved: %#v", sources[2]) + } answerAppearances, ok := sources[0]["appearances"].([]map[string]any) if !ok || len(answerAppearances) != 2 || answerAppearances[1]["kind"] != "answer" || answerAppearances[1]["cited"] != true { t.Fatalf("known source was not marked cited by final answer URL: %#v", sources[0]) diff --git a/pkg/connector/stream_test.go b/pkg/connector/stream_test.go index 9d703230..dc2d0e60 100644 --- a/pkg/connector/stream_test.go +++ b/pkg/connector/stream_test.go @@ -246,6 +246,105 @@ func TestStreamPublisherReusesRunAcrossToolContinuation(t *testing.T) { } } +func TestStreamPublisherUsesFreshTextMessageAfterToolContinuationWithPriorText(t *testing.T) { + ctx := context.Background() + toolAPI := ai.Api("test-stream-tool-prior-text") + answerAPI := ai.Api("test-stream-answer-after-prior-text") + ai.RegisterAPIProvider(toolAPI, func(ctx context.Context, model ai.Model, llmContext ai.Context, options ai.SimpleStreamOptions) *ai.AssistantMessageEventStream { + stream := ai.NewAssistantMessageEventStream() + go func() { + toolCall := &ai.ToolCall{ID: "call-session", Name: "get_session", Arguments: map[string]any{}} + message := ai.Message{ + Role: "assistant", + Content: []ai.ContentBlock{ + {Type: "text", Text: "before "}, + {Type: "toolCall", ID: toolCall.ID, Name: toolCall.Name, Arguments: toolCall.Arguments}, + }, + StopReason: ai.StopReasonToolUse, + } + stream.Push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: 0, Delta: "before "}) + stream.Push(ai.AssistantMessageEvent{Type: "toolcall_start", ContentIndex: 1, ToolCall: toolCall}) + stream.Push(ai.AssistantMessageEvent{Type: "toolcall_end", ContentIndex: 1, ToolCall: toolCall}) + stream.Push(ai.AssistantMessageEvent{Type: "done", Reason: ai.StopReasonToolUse, Message: &message}) + }() + return stream + }) + ai.RegisterAPIProvider(answerAPI, func(ctx context.Context, model ai.Model, llmContext ai.Context, options ai.SimpleStreamOptions) *ai.AssistantMessageEventStream { + stream := ai.NewAssistantMessageEventStream() + go func() { + message := ai.Message{ + Role: "assistant", + Content: []ai.ContentBlock{{Type: "text", Text: "after"}}, + StopReason: ai.StopReasonStop, + } + stream.Push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: 0, Delta: "after"}) + stream.Push(ai.AssistantMessageEvent{Type: "done", Reason: ai.StopReasonStop, Message: &message}) + }() + return stream + }) + defer ai.UnregisterAPIProvider(toolAPI) + defer ai.UnregisterAPIProvider(answerAPI) + + publisher := &recordingStreamPublisher{} + client := &Client{} + run := aistream.NewRun("run", "thread", "beeper/fake", "assistant:run", "Fake", timeNow()) + run.MessageID = "assistant:run" + cursor := &streamPublishCursor{nextSeq: 1} + + toolResult := client.streamPublisherWithEndFrom(publisher, "!room:example.com", "$event", run, cursor, nil)(ctx, ai.Model{ID: "fake", API: toolAPI}, ai.Context{}, ai.SimpleStreamOptions{}).Result() + if toolResult.StopReason != ai.StopReasonToolUse { + t.Fatalf("unexpected tool stream result %#v", toolResult) + } + writer := aistream.NewWriter(run, timeNow) + writer.ToolEnd("call-session", "get_session", map[string]any{}, map[string]any{"state": agui.ToolResultStateComplete, "status": "success"}) + + answerResult := client.streamPublisherWithEndFrom(publisher, "!room:example.com", "$event", run, cursor, nil)(ctx, ai.Model{ID: "fake", API: answerAPI}, ai.Context{}, ai.SimpleStreamOptions{}).Result() + if answerResult.StopReason != ai.StopReasonStop { + t.Fatalf("unexpected answer stream result %#v", answerResult) + } + + textMessageIDs := textContentMessageIDs(run.Events) + if strings.Join(textMessageIDs, "|") != "assistant:run|assistant:run-text-1" { + t.Fatalf("resumed answer reused a prior text message id: %#v", textMessageIDs) + } + message := run.FinalBeeperAIMessage(0, true) + got := uiPartSummary(message.Parts) + want := []string{"text:before ", "tool-call:call-session", "text:after"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("continued UI parts mismatch\ngot: %#v\nwant: %#v\nparts: %#v", got, want, message.Parts) + } +} + +func TestDoneFallbackAfterPriorTextUsesCurrentWriterOnly(t *testing.T) { + run := aistream.NewRun("run", "thread", "beeper/fake", "assistant:run", "Fake", timeNow()) + run.MessageID = "assistant:run" + firstWriter := aistream.NewWriter(run, timeNow) + firstWriter.Start() + firstWriter.TextDelta(0, "before ") + firstWriter.ToolStart("call-session", "get_session", 1, nil) + firstWriter.ToolInputComplete("call-session", "get_session", map[string]any{}) + firstWriter.AwaitToolUseWithUsage(nil) + + message := ai.Message{ + Role: "assistant", + Content: []ai.ContentBlock{{Type: "text", Text: "after"}}, + StopReason: ai.StopReasonStop, + } + secondWriter := aistream.NewWriter(run, timeNow) + applyAIStreamEvent(secondWriter, ai.AssistantMessageEvent{Type: "done", Reason: ai.StopReasonStop, Message: &message}) + + textMessageIDs := textContentMessageIDs(run.Events) + if strings.Join(textMessageIDs, "|") != "assistant:run|assistant:run-text-1" { + t.Fatalf("done fallback reused a prior text message id: %#v", textMessageIDs) + } + uiMessage := run.FinalBeeperAIMessage(0, true) + got := uiPartSummary(uiMessage.Parts) + want := []string{"text:before ", "tool-call:call-session", "text:after"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("fallback UI parts mismatch\ngot: %#v\nwant: %#v\nparts: %#v", got, want, uiMessage.Parts) + } +} + func TestFinalizedAssistantRunPreservesAccumulatedStreamUsage(t *testing.T) { run := *aistream.NewRun("run", "thread", "beeper/fake", "assistant:run", "Fake", timeNow()) run.Usage = agui.Usage{PromptTokens: 13, CompletionTokens: 3, ReasoningTokens: 3, TotalTokens: 16} @@ -1123,6 +1222,44 @@ func TestPublishNewStreamEventsSuppressesMautrixRequestBodyLogs(t *testing.T) { } } +func textContentMessageIDs(events []agui.Event) []string { + var ids []string + for _, evt := range events { + if evt.Type() != agui.EventTextMessageContent && evt.Type() != agui.EventTextMessageChunk { + continue + } + delta, _ := evt.Get("delta").(string) + if delta == "" { + continue + } + messageID, _ := evt.Get("messageId").(string) + ids = append(ids, messageID) + } + return ids +} + +func uiPartSummary(parts []aistream.MessagePart) []string { + out := make([]string, 0, len(parts)) + for _, part := range parts { + switch part["type"] { + case "text", "thinking": + out = append(out, fmt.Sprintf("%s:%s", part["type"], part["content"])) + case "tool-call": + out = append(out, fmt.Sprintf("tool-call:%s", firstNonEmptyString(part["toolCallId"], part["id"]))) + } + } + return out +} + +func firstNonEmptyString(values ...any) string { + for _, value := range values { + if text, _ := value.(string); text != "" { + return text + } + } + return "" +} + func testAIStore(t *testing.T) *aidb.Store { t.Helper() rawDB, err := sql.Open("sqlite3", filepath.Join(t.TempDir(), "bridge.db")) From bd50dffc5056c81f704cd9895281236c8c1142b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 21:33:50 +0200 Subject: [PATCH 13/22] Use AI Services base URL and runtime metadata Switch AI Services handling from a fixed proxy path to a generic base URL derived from the user's homeserver and support AI Services "runtime" metadata. Key changes: - Replace hardcoded proxy path usage and defaultAIServicesProxyPath with base URL semantics across connector code (defaultAIServicesBaseURL, aiServices*URL functions now accept baseURL). Removed trimAIProxyProviderPath and updated URL joining logic. - Support AI Services catalog runtime payloads: parse runtime.{provider,model,api,baseUrl} and apply runtime compat into model. Added aiServicesModelCompat struct and helpers to map compat flags into model. Introduced aiServicesRuntimeBaseURL to build runtime-specific base URLs. - Update model/avatar/provider logic to prefer runtime_model and runtime base URLs; adjust related helpers and tests to the new runtime format. - Fix Anthropics message conversion to replay empty tool input as an object (avoid nil input) and add a unit test for it. - Refactor and clarify user-facing slash-command messages about model/reasoning names and modes. - Update README text to reflect AI Services base URL behavior. Tests updated to match new runtime schema and updated URL expectations. --- README.md | 23 +- cmd/generate-models-go/main.go | 73 --- cmd/generate-models-go/main_test.go | 21 - pkg/ai/image_models.go | 28 -- pkg/ai/image_models_generated.go | 30 -- pkg/ai/images_test.go | 18 - pkg/ai/modelcatalog/catalog.go | 420 ------------------ pkg/ai/modelcatalog/catalog_test.go | 132 ------ pkg/ai/modelcatalog/sources.go | 118 ----- pkg/ai/models.go | 42 +- pkg/ai/models_generated.go | 31 -- pkg/ai/models_test.go | 18 - pkg/ai/providers/anthropic.go | 6 +- pkg/ai/providers/anthropic_test.go | 29 ++ .../images/register_builtins_test.go | 18 - pkg/ai/providers/openai_conversion_test.go | 11 - pkg/connector/approvals.go | 6 +- pkg/connector/chat_tools.go | 6 +- pkg/connector/chat_tools_test.go | 2 +- pkg/connector/chatgpt_device_login.go | 9 - pkg/connector/config.go | 1 - pkg/connector/connector.go | 14 +- pkg/connector/contacts.go | 169 +++---- pkg/connector/contacts_test.go | 54 +-- pkg/connector/login.go | 2 +- pkg/connector/model_avatar.go | 12 +- pkg/connector/model_avatar_test.go | 8 +- pkg/connector/provider.go | 33 -- pkg/connector/provider_test.go | 47 +- pkg/connector/slash_commands_limits.go | 5 +- pkg/connector/slash_commands_model.go | 39 +- pkg/connector/slash_commands_test.go | 58 ++- pkg/connector/stream_test.go | 4 +- scripts/generate-image-models-go.mjs | 21 - 34 files changed, 296 insertions(+), 1212 deletions(-) delete mode 100644 cmd/generate-models-go/main.go delete mode 100644 cmd/generate-models-go/main_test.go delete mode 100644 pkg/ai/image_models.go delete mode 100644 pkg/ai/image_models_generated.go delete mode 100644 pkg/ai/modelcatalog/catalog.go delete mode 100644 pkg/ai/modelcatalog/catalog_test.go delete mode 100644 pkg/ai/modelcatalog/sources.go delete mode 100644 pkg/ai/models_generated.go delete mode 100644 pkg/ai/providers/images/register_builtins_test.go delete mode 100755 scripts/generate-image-models-go.mjs diff --git a/README.md b/README.md index 321f7d16..f67a10fc 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,7 @@ Content is the universal `ContentBlock` (`text` / `thinking` / `toolCall` / `ima **Same protocol, new vendor** — usually *no code*: -1. Add a model entry (in the generated catalog or a hand-built `ai.Model`) with the right `API`, `Provider`, `BaseURL`. +1. Add a model entry in ai-services, or configure a custom provider model with the right `API`, `Provider`, `BaseURL`. 2. Map the provider to its API-key env var(s) in `pkg/ai/env_api_keys.go`. 3. Add any `Compat` overrides (most base-URL patterns are auto-detected by `detectOpenAICompletionsCompat`). @@ -317,19 +317,15 @@ Provider-specific behaviors worth knowing live in `pkg/ai/providers`: OpenAI *Co ## The model catalog -The runtime catalog is `Models` (`map[Provider]map[string]Model`) loaded from a large JSON literal in `pkg/ai/models_generated.go`. Accessors: `GetModel`, `GetProviders`, `GetModels`. +The Beeper AI model catalog is owned by ai-services. The bridge loads `/models?feature=bridge:ai`, applies each model's runtime metadata, and fails the default Beeper provider if ai-services does not return a catalog. There is no bridge-generated fallback catalog. -It is **generated** by `cmd/generate-models-go`: +Custom providers are intentionally simpler: the bridge uses the provider's `/models` response or the user's configured `ai.Model` entries. If a custom provider does not advertise detailed metadata, the bridge uses a conservative text-only model shape instead of consulting Beeper's catalog by model ID. -```sh -go run ./cmd/generate-models-go [output-path] [--include-unregistered] -``` - -It fetches `models.dev` and `openrouter.ai`, keeps only **tool-capable** models, normalizes capabilities/pricing, and applies hand-maintained overrides (`pkg/ai/modelcatalog/`) — e.g. `ThinkingLevelMap` for gpt-5/Gemini-3, Anthropic-style cache-control for OpenRouter Anthropic models. Reasoning levels form a ladder `off < minimal < low < medium < high < xhigh`; `ClampThinkingLevel` snaps a request to the nearest supported level. +Reasoning levels form a ladder `off < minimal < low < medium < high < xhigh`; `ClampThinkingLevel` snaps a request to the nearest supported level from the model metadata the bridge was given. ## Image generation -Image generation is a **separate path** (`pkg/ai/images.go`, `images_*.go`): `ai.GenerateImages(ctx, ImagesModel, ImagesContext, ImagesOptions) AssistantImages` (synchronous, no streaming). It has its own model catalog (`image_models_generated.go` — FLUX.2, Seedream, Gemini "Nano Banana", GPT Image, Recraft, etc.) and its own registry. The built-in implementation routes through OpenRouter; blank-import `pkg/ai/providers/images` to enable it. Models can also expose **provider-native** `image_generation` as a built-in tool (see [chat tools](#built-in-chat-tools--adding-your-own)). +Image generation is a **separate path** (`pkg/ai/images.go`, `images_*.go`): `ai.GenerateImages(ctx, ImagesModel, ImagesContext, ImagesOptions) AssistantImages` (synchronous, no streaming). Image model metadata must come from ai-services or explicit provider configuration; the bridge does not keep a generated image catalog. The built-in implementation supports OpenRouter; blank-import `pkg/ai/providers/images` to enable it. Models can also expose **provider-native** `image_generation` as a built-in tool (see [chat tools](#built-in-chat-tools--adding-your-own)). ## The agent runtime @@ -416,7 +412,7 @@ The bridge advertises five login flows (`pkg/connector/login.go`): | Flow | What it does | |------|--------------| -| `beeper` | The default **Beeper AI** login. Routes through an `ai-services.` proxy derived from the user's homeserver; uses an appservice bearer token, no stored key. Read-only/managed. | +| `beeper` | The default **Beeper AI** login. Loads its catalog and runtime proxy metadata from `ai-services.` derived from the user's homeserver; uses an appservice bearer token, no stored key. Read-only/managed. | | `openai-responses` / `openai-completions` / `openai-codex-responses` | **Custom provider**: enter base URL + API key, the bridge fetches `/models`, you pick a default model. | | `chatgpt-device` | **ChatGPT** OAuth device-code flow (PKCE). Stores access + refresh tokens, auto-refreshes within 2 min of expiry. | @@ -466,7 +462,7 @@ compaction: Three scopes layer together: **bridge-wide** YAML → **per-login** provider configs (`UserLoginMetadata.Providers`) → **per-room** state (`com.beeper.ai.model` / `.additional_prompt` / `.tools`). -Relevant constants: default Beeper model `beeper/default`, title-generation model `gpt-4.1-mini` (fallback `gpt-5-mini`), default AI-services proxy path `/proxy/openai/v1`. +Relevant constants: default Beeper model `beeper/default`, title-generation model `gpt-4.1-mini` (fallback `gpt-5-mini`), default AI Services base URL derived from the user's homeserver domain. --- @@ -533,8 +529,7 @@ It serves `/v1/models`, `/v1/responses`, `/v1/chat/completions`, and `/api/strea | Package | Responsibility | |---------|----------------| | `cmd/ai` | bridge entry point (registers connector + providers) | -| `cmd/generate-models-go` | regenerates the text-model catalog from upstream sources | -| `pkg/ai` | provider/API/model abstraction, streaming interface, model catalog, env keys | +| `pkg/ai` | provider/API/model abstraction, streaming interface, env keys | | `pkg/ai/providers` | built-in provider implementations (OpenAI Completions/Responses/Codex, Anthropic, Google GenAI/Vertex) + image generation | | `pkg/ai-stream` | the `Run` model: AG-UI event accumulation, anchor/stream/final projection, approvals, final-payload sizing | | `pkg/ag-ui` | the AG-UI wire event protocol, typed events, schema, validation, capabilities | @@ -543,7 +538,7 @@ It serves `/v1/models`, `/v1/responses`, `/v1/chat/completions`, and `/api/strea | `pkg/agent/harness/session` | branching conversation tree (per-conversation SQLite) | | `pkg/agent/autocompact` | compaction trigger policy | | `pkg/chattools` | built-in tools: `get_session`, `fetch`, `web_search` | -| `pkg/connector` | the `bridgev2` connector: rooms↔sessions, slash/bridge commands, login, provider routes, capabilities, contacts, direct media, room state | +| `pkg/connector` | the `bridgev2` connector: rooms↔sessions, slash/bridge commands, login, provider catalog loading, capabilities, contacts, direct media, room state | | `pkg/msgconv` | Matrix ⇄ AI message conversion | | `pkg/aiid` | deterministic IDs + metadata types | | `pkg/aidb` | bridge-DB persistence: session storage + active-stream resume | diff --git a/cmd/generate-models-go/main.go b/cmd/generate-models-go/main.go deleted file mode 100644 index 433d75cd..00000000 --- a/cmd/generate-models-go/main.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "strings" - - "github.com/beeper/ai-bridge/pkg/ai" - "github.com/beeper/ai-bridge/pkg/ai/modelcatalog" -) - -func main() { - outputPath, includeUnregistered, err := parseArgs(os.Args[1:]) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(2) - } - - catalog, err := modelcatalog.Build(context.Background(), modelcatalog.Options{ - IncludeUnregistered: includeUnregistered, - }) - if err != nil { - fmt.Fprintf(os.Stderr, "generate models: %v\n", err) - os.Exit(1) - } - source, err := modelcatalog.GenerateGoSource(catalog, "ai") - if err != nil { - fmt.Fprintf(os.Stderr, "generate Go source: %v\n", err) - os.Exit(1) - } - if err := os.WriteFile(outputPath, source, 0o644); err != nil { - fmt.Fprintf(os.Stderr, "write %s: %v\n", outputPath, err) - os.Exit(1) - } - - if len(catalog.Skipped) > 0 && !includeUnregistered { - fmt.Fprintf(os.Stderr, "Skipped providers with unregistered APIs: %s. Use --include-unregistered after porting providers.\n", providerList(catalog.Skipped)) - } - fmt.Printf("Wrote %d models across %d providers to %s\n", catalog.Count(), len(catalog.ProviderOrder), outputPath) -} - -func parseArgs(args []string) (string, bool, error) { - outputPath := "pkg/ai/models_generated.go" - includeUnregistered := false - outputSet := false - for _, arg := range args { - switch arg { - case "--include-unregistered": - includeUnregistered = true - case "-h", "--help": - return "", false, fmt.Errorf("usage: generate-models-go [output-path] [--include-unregistered]") - default: - if strings.HasPrefix(arg, "-") { - return "", false, fmt.Errorf("unknown flag %s", arg) - } - if outputSet { - return "", false, fmt.Errorf("unexpected argument %s", arg) - } - outputPath = arg - outputSet = true - } - } - return outputPath, includeUnregistered, nil -} - -func providerList(providers []ai.Provider) string { - parts := make([]string, 0, len(providers)) - for _, provider := range providers { - parts = append(parts, string(provider)) - } - return strings.Join(parts, ", ") -} diff --git a/cmd/generate-models-go/main_test.go b/cmd/generate-models-go/main_test.go deleted file mode 100644 index 445154ba..00000000 --- a/cmd/generate-models-go/main_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import "testing" - -func TestParseArgsAcceptsIncludeUnregisteredBeforeOrAfterOutput(t *testing.T) { - for _, args := range [][]string{ - {"--include-unregistered", "/tmp/models.go"}, - {"/tmp/models.go", "--include-unregistered"}, - } { - outputPath, includeUnregistered, err := parseArgs(args) - if err != nil { - t.Fatalf("parseArgs(%v): %v", args, err) - } - if outputPath != "/tmp/models.go" { - t.Fatalf("parseArgs(%v) output = %q", args, outputPath) - } - if !includeUnregistered { - t.Fatalf("parseArgs(%v) did not enable includeUnregistered", args) - } - } -} diff --git a/pkg/ai/image_models.go b/pkg/ai/image_models.go deleted file mode 100644 index d353630b..00000000 --- a/pkg/ai/image_models.go +++ /dev/null @@ -1,28 +0,0 @@ -package ai - -import "slices" - -func GetImageModel(provider ImagesProvider, modelID string) (ImagesModel, bool) { - providerModels := ImageModels[provider] - if providerModels == nil { - return ImagesModel{}, false - } - model, ok := providerModels[modelID] - return model, ok -} - -func GetImageProviders() []ImagesProvider { - return slices.Clone(imageModelProviderOrder) -} - -func GetImageModels(provider ImagesProvider) []ImagesModel { - providerModels := ImageModels[provider] - if providerModels == nil { - return nil - } - models := make([]ImagesModel, 0, len(providerModels)) - for _, id := range imageModelIDOrder[provider] { - models = append(models, providerModels[id]) - } - return models -} diff --git a/pkg/ai/image_models_generated.go b/pkg/ai/image_models_generated.go deleted file mode 100644 index f94e882c..00000000 --- a/pkg/ai/image_models_generated.go +++ /dev/null @@ -1,30 +0,0 @@ -package ai - -import "encoding/json" - -var imageModelsJSON = `{"openrouter":{"black-forest-labs/flux.2-flex":{"id":"black-forest-labs/flux.2-flex","name":"Black Forest Labs: FLUX.2 Flex","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"black-forest-labs/flux.2-klein-4b":{"id":"black-forest-labs/flux.2-klein-4b","name":"Black Forest Labs: FLUX.2 Klein 4B","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"black-forest-labs/flux.2-max":{"id":"black-forest-labs/flux.2-max","name":"Black Forest Labs: FLUX.2 Max","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"black-forest-labs/flux.2-pro":{"id":"black-forest-labs/flux.2-pro","name":"Black Forest Labs: FLUX.2 Pro","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"bytedance-seed/seedream-4.5":{"id":"bytedance-seed/seedream-4.5","name":"ByteDance Seed: Seedream 4.5","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["image","text"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"google/gemini-2.5-flash-image":{"id":"google/gemini-2.5-flash-image","name":"Google: Nano Banana (Gemini 2.5 Flash Image)","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["image","text"],"output":["image","text"],"cost":{"input":0.3,"output":2.5,"cacheRead":0.03,"cacheWrite":0.08333333333333334}},"google/gemini-3-pro-image-preview":{"id":"google/gemini-3-pro-image-preview","name":"Google: Nano Banana Pro (Gemini 3 Pro Image Preview)","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["image","text"],"output":["image","text"],"cost":{"input":2,"output":12,"cacheRead":0.19999999999999998,"cacheWrite":0.375}},"google/gemini-3.1-flash-image-preview":{"id":"google/gemini-3.1-flash-image-preview","name":"Google: Nano Banana 2 (Gemini 3.1 Flash Image Preview)","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["image","text"],"output":["image","text"],"cost":{"input":0.5,"output":3,"cacheRead":0,"cacheWrite":0}},"openai/gpt-5-image":{"id":"openai/gpt-5-image","name":"OpenAI: GPT-5 Image","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["image","text"],"output":["image","text"],"cost":{"input":10,"output":10,"cacheRead":1.25,"cacheWrite":0}},"openai/gpt-5-image-mini":{"id":"openai/gpt-5-image-mini","name":"OpenAI: GPT-5 Image Mini","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["image","text"],"output":["image","text"],"cost":{"input":2.5,"output":2,"cacheRead":0.25,"cacheWrite":0}},"openai/gpt-5.4-image-2":{"id":"openai/gpt-5.4-image-2","name":"OpenAI: GPT-5.4 Image 2","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["image","text"],"output":["image","text"],"cost":{"input":8,"output":15,"cacheRead":2,"cacheWrite":0}},"openrouter/auto":{"id":"openrouter/auto","name":"Auto Router","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["text","image"],"cost":{"input":-1000000,"output":-1000000,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v3":{"id":"recraft/recraft-v3","name":"Recraft: Recraft V3","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v4":{"id":"recraft/recraft-v4","name":"Recraft: Recraft V4","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v4-pro":{"id":"recraft/recraft-v4-pro","name":"Recraft: Recraft V4 Pro","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v4-pro-vector":{"id":"recraft/recraft-v4-pro-vector","name":"Recraft: Recraft V4 Pro Vector","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v4-vector":{"id":"recraft/recraft-v4-vector","name":"Recraft: Recraft V4 Vector","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v4.1":{"id":"recraft/recraft-v4.1","name":"Recraft: Recraft V4.1","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v4.1-pro":{"id":"recraft/recraft-v4.1-pro","name":"Recraft: Recraft V4.1 Pro","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v4.1-pro-vector":{"id":"recraft/recraft-v4.1-pro-vector","name":"Recraft: Recraft V4.1 Pro Vector","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v4.1-utility":{"id":"recraft/recraft-v4.1-utility","name":"Recraft: Recraft V4.1 Utility","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v4.1-utility-pro":{"id":"recraft/recraft-v4.1-utility-pro","name":"Recraft: Recraft V4.1 Utility Pro","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"recraft/recraft-v4.1-vector":{"id":"recraft/recraft-v4.1-vector","name":"Recraft: Recraft V4.1 Vector","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"sourceful/riverflow-v2-fast":{"id":"sourceful/riverflow-v2-fast","name":"Sourceful: Riverflow V2 Fast","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"sourceful/riverflow-v2-fast-preview":{"id":"sourceful/riverflow-v2-fast-preview","name":"Sourceful: Riverflow V2 Fast Preview","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"sourceful/riverflow-v2-max-preview":{"id":"sourceful/riverflow-v2-max-preview","name":"Sourceful: Riverflow V2 Max Preview","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"sourceful/riverflow-v2-pro":{"id":"sourceful/riverflow-v2-pro","name":"Sourceful: Riverflow V2 Pro","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"sourceful/riverflow-v2-standard-preview":{"id":"sourceful/riverflow-v2-standard-preview","name":"Sourceful: Riverflow V2 Standard Preview","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}},"x-ai/grok-imagine-image-quality":{"id":"x-ai/grok-imagine-image-quality","name":"xAI: Grok Imagine Image Quality","api":"openrouter-images","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","input":["text","image"],"output":["image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0}}}}` - -var imageModelProviderOrder = []ImagesProvider{ - "openrouter", -} - -var imageModelIDOrderJSON = `{"openrouter":["black-forest-labs/flux.2-flex","black-forest-labs/flux.2-klein-4b","black-forest-labs/flux.2-max","black-forest-labs/flux.2-pro","bytedance-seed/seedream-4.5","google/gemini-2.5-flash-image","google/gemini-3-pro-image-preview","google/gemini-3.1-flash-image-preview","openai/gpt-5-image","openai/gpt-5-image-mini","openai/gpt-5.4-image-2","openrouter/auto","recraft/recraft-v3","recraft/recraft-v4","recraft/recraft-v4-pro","recraft/recraft-v4-pro-vector","recraft/recraft-v4-vector","recraft/recraft-v4.1","recraft/recraft-v4.1-pro","recraft/recraft-v4.1-pro-vector","recraft/recraft-v4.1-utility","recraft/recraft-v4.1-utility-pro","recraft/recraft-v4.1-vector","sourceful/riverflow-v2-fast","sourceful/riverflow-v2-fast-preview","sourceful/riverflow-v2-max-preview","sourceful/riverflow-v2-pro","sourceful/riverflow-v2-standard-preview","x-ai/grok-imagine-image-quality"]}` - -var ImageModels = mustLoadImageModels() -var imageModelIDOrder = mustLoadImageModelIDOrder() - -func mustLoadImageModels() map[ImagesProvider]map[string]ImagesModel { - var raw map[ImagesProvider]map[string]ImagesModel - if err := json.Unmarshal([]byte(imageModelsJSON), &raw); err != nil { - panic(err) - } - return raw -} - -func mustLoadImageModelIDOrder() map[ImagesProvider][]string { - var raw map[ImagesProvider][]string - if err := json.Unmarshal([]byte(imageModelIDOrderJSON), &raw); err != nil { - panic(err) - } - return raw -} diff --git a/pkg/ai/images_test.go b/pkg/ai/images_test.go index 890e5906..37f50c13 100644 --- a/pkg/ai/images_test.go +++ b/pkg/ai/images_test.go @@ -5,24 +5,6 @@ import ( "testing" ) -func TestImageModelRegistryGeneratedSurface(t *testing.T) { - providers := GetImageProviders() - if len(providers) != 1 || providers[0] != ImagesProviderOpenRouter { - t.Fatalf("unexpected image providers: %#v", providers) - } - model, ok := GetImageModel(ImagesProviderOpenRouter, "openai/gpt-5-image") - if !ok { - t.Fatal("expected generated image model") - } - if model.API != ImagesApiOpenRouter || model.Provider != ImagesProviderOpenRouter { - t.Fatalf("unexpected image model metadata: %#v", model) - } - models := GetImageModels(ImagesProviderOpenRouter) - if len(models) == 0 || models[0].ID != "black-forest-labs/flux.2-flex" { - t.Fatalf("unexpected image model order: %#v", models) - } -} - func TestGenerateImagesMissingProviderPanics(t *testing.T) { defer func() { if recovered := recover(); recovered == nil { diff --git a/pkg/ai/modelcatalog/catalog.go b/pkg/ai/modelcatalog/catalog.go deleted file mode 100644 index 9f4c37b5..00000000 --- a/pkg/ai/modelcatalog/catalog.go +++ /dev/null @@ -1,420 +0,0 @@ -package modelcatalog - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "maps" - "net/http" - "regexp" - "slices" - "strings" - - "github.com/beeper/ai-bridge/pkg/ai" -) - -const ( - ModelsDevURL = "https://models.dev/api.json" - OpenRouterModelsURL = "https://openrouter.ai/api/v1/models" - VertexBaseURL = "https://{location}-aiplatform.googleapis.com" -) - -var ( - DefaultProviders = []ai.Provider{ - ai.ProviderOpenAI, - ai.ProviderOpenRouter, - ai.ProviderAnthropic, - ai.ProviderGoogleVertex, - } - - DefaultRegisteredAPIs = []ai.Api{ - ai.ApiAnthropicMessages, - ai.ApiOpenAICompletions, - ai.ApiOpenAIResponses, - ai.ApiOpenAICodexResponses, - ai.ApiGoogleVertex, - } - - gemini3ProPattern = regexp.MustCompile(`(?i)gemini-3(?:\.\d+)?-pro`) - gemini3FlashPattern = regexp.MustCompile(`(?i)gemini-3(?:\.\d+)?-flash`) -) - -type Options struct { - HTTPClient *http.Client - ModelsDevURL string - OpenRouterModelsURL string - Providers []ai.Provider - RegisteredAPIs []ai.Api - IncludeUnregistered bool -} - -type Catalog struct { - Models map[ai.Provider]map[string]ai.Model - ProviderOrder []ai.Provider - ModelIDOrder map[ai.Provider][]string - Skipped []ai.Provider -} - -func Build(ctx context.Context, opts Options) (Catalog, error) { - opts = opts.withDefaults() - modelsDev, openRouter, err := fetchSources(ctx, opts) - if err != nil { - return Catalog{}, err - } - - models := make([]ai.Model, 0) - for _, provider := range []ai.Provider{ai.ProviderOpenAI, ai.ProviderAnthropic, ai.ProviderGoogleVertex} { - sourceModels := modelsDev[string(provider)].Models - for _, sourceModel := range sourceModels { - if !isToolCapableModelsDev(sourceModel) { - continue - } - models = append(models, applyModelMetadata(normalizeModelsDevModel(provider, sourceModel))) - } - } - for _, sourceModel := range openRouter.Data { - if !isToolCapableOpenRouter(sourceModel) { - continue - } - models = append(models, applyModelMetadata(normalizeOpenRouterModel(sourceModel))) - } - return BuildFromModels(models, opts), nil -} - -func BuildFromModels(models []ai.Model, opts Options) Catalog { - opts = opts.withDefaults() - targetProviders := providerSet(opts.Providers) - registeredAPIs := apiSet(opts.RegisteredAPIs) - - allProviders := make(map[ai.Provider]bool) - out := make(map[ai.Provider]map[string]ai.Model) - for _, model := range models { - if !targetProviders[model.Provider] { - continue - } - allProviders[model.Provider] = true - if !opts.IncludeUnregistered && !registeredAPIs[model.API] { - continue - } - if out[model.Provider] == nil { - out[model.Provider] = make(map[string]ai.Model) - } - out[model.Provider][model.ID] = model - } - - providerOrder := slices.Sorted(maps.Keys(out)) - modelIDOrder := make(map[ai.Provider][]string, len(out)) - for provider, providerModels := range out { - modelIDOrder[provider] = slices.Sorted(maps.Keys(providerModels)) - } - - skipped := make([]ai.Provider, 0) - emittedProviders := providerSet(providerOrder) - for provider := range allProviders { - if !emittedProviders[provider] { - skipped = append(skipped, provider) - } - } - slices.Sort(skipped) - - return Catalog{ - Models: out, - ProviderOrder: providerOrder, - ModelIDOrder: modelIDOrder, - Skipped: skipped, - } -} - -func (catalog Catalog) Count() int { - count := 0 - for _, providerModels := range catalog.Models { - count += len(providerModels) - } - return count -} - -func GenerateGoSource(catalog Catalog, packageName string) ([]byte, error) { - if packageName == "" { - packageName = "ai" - } - modelsJSON, err := json.Marshal(catalog.Models) - if err != nil { - return nil, fmt.Errorf("marshal models: %w", err) - } - modelIDOrderJSON, err := json.Marshal(catalog.ModelIDOrder) - if err != nil { - return nil, fmt.Errorf("marshal model id order: %w", err) - } - - var buf bytes.Buffer - fmt.Fprintf(&buf, "package %s\n\n", packageName) - buf.WriteString("import \"encoding/json\"\n\n") - fmt.Fprintf(&buf, "var modelsJSON = `%s`\n\n", escapeRawString(modelsJSON)) - buf.WriteString("var modelProviderOrder = []Provider{\n") - for _, provider := range catalog.ProviderOrder { - encoded, _ := json.Marshal(provider) - fmt.Fprintf(&buf, "\t%s,\n", encoded) - } - buf.WriteString("}\n\n") - fmt.Fprintf(&buf, "var modelIDOrderJSON = `%s`\n\n", escapeRawString(modelIDOrderJSON)) - buf.WriteString(`var Models = mustLoadModels() -var modelIDOrder = mustLoadModelIDOrder() - -func mustLoadModels() map[Provider]map[string]Model { - var raw map[Provider]map[string]Model - if err := json.Unmarshal([]byte(modelsJSON), &raw); err != nil { - panic(err) - } - return raw -} - -func mustLoadModelIDOrder() map[Provider][]string { - var raw map[Provider][]string - if err := json.Unmarshal([]byte(modelIDOrderJSON), &raw); err != nil { - panic(err) - } - return raw -} -`) - return buf.Bytes(), nil -} - -func fetchSources(ctx context.Context, opts Options) (modelsDevResponse, openRouterResponse, error) { - var modelsDev modelsDevResponse - var openRouter openRouterResponse - if err := fetchJSON(ctx, opts.HTTPClient, opts.ModelsDevURL, &modelsDev); err != nil { - return nil, openRouterResponse{}, fmt.Errorf("fetch models.dev catalog: %w", err) - } - if err := fetchJSON(ctx, opts.HTTPClient, opts.OpenRouterModelsURL, &openRouter); err != nil { - return nil, openRouterResponse{}, fmt.Errorf("fetch OpenRouter catalog: %w", err) - } - return modelsDev, openRouter, nil -} - -func fetchJSON(ctx context.Context, client *http.Client, url string, target any) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return err - } - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) - return fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(body))) - } - return json.NewDecoder(resp.Body).Decode(target) -} - -func (opts Options) withDefaults() Options { - if opts.HTTPClient == nil { - opts.HTTPClient = http.DefaultClient - } - if opts.ModelsDevURL == "" { - opts.ModelsDevURL = ModelsDevURL - } - if opts.OpenRouterModelsURL == "" { - opts.OpenRouterModelsURL = OpenRouterModelsURL - } - if len(opts.Providers) == 0 { - opts.Providers = slices.Clone(DefaultProviders) - } - if len(opts.RegisteredAPIs) == 0 { - opts.RegisteredAPIs = slices.Clone(DefaultRegisteredAPIs) - } - return opts -} - -func normalizeModelsDevModel(provider ai.Provider, model modelsDevModel) ai.Model { - api := ai.ApiGoogleVertex - baseURL := VertexBaseURL - switch provider { - case ai.ProviderOpenAI: - api = openAIAPI(model.ID) - baseURL = "https://api.openai.com/v1" - case ai.ProviderAnthropic: - api = ai.ApiAnthropicMessages - baseURL = "https://api.anthropic.com" - } - name := model.Name - if name == "" { - name = model.ID - } - if provider == ai.ProviderGoogleVertex { - name += " (Vertex)" - } - return ai.Model{ - ID: model.ID, - Name: name, - API: api, - Provider: provider, - BaseURL: baseURL, - Reasoning: model.Reasoning, - Input: textImageInput(model.Modalities.Input), - Cost: costFromModelsDev(model.Cost), - ContextWindow: intOrDefault(model.Limit.Context, 128000), - MaxTokens: intOrDefault(model.Limit.Output, 16384), - } -} - -func normalizeOpenRouterModel(model openRouterModel) ai.Model { - return ai.Model{ - ID: model.ID, - Name: stringOrDefault(model.Name, model.ID), - API: ai.ApiOpenAICompletions, - Provider: ai.ProviderOpenRouter, - BaseURL: "https://openrouter.ai/api/v1", - Reasoning: slices.Contains(model.SupportedParameters, "reasoning") || slices.Contains(model.SupportedParameters, "include_reasoning"), - Input: textImageInput(model.Architecture.InputModalities), - Cost: costFromOpenRouter(model.Pricing), - ContextWindow: intOrDefault(firstNonZero(model.ContextLength, model.TopProvider.ContextLength), 128000), - MaxTokens: intOrDefault(model.TopProvider.MaxCompletionTokens, 16384), - } -} - -func applyModelMetadata(model ai.Model) ai.Model { - if (model.API == ai.ApiOpenAIResponses || model.API == ai.ApiOpenAICodexResponses) && strings.HasPrefix(model.ID, "gpt-5") { - model.ThinkingLevelMap = mergeThinkingLevelMap(model.ThinkingLevelMap, map[ai.ModelThinkingLevel]*string{ - ai.ModelThinkingLevelOff: nil, - }) - } - if strings.Contains(model.ID, "gpt-5.2") || strings.Contains(model.ID, "gpt-5.3") || strings.Contains(model.ID, "gpt-5.4") || strings.Contains(model.ID, "gpt-5.5") { - xhigh := string(ai.ModelThinkingLevelXHigh) - model.ThinkingLevelMap = mergeThinkingLevelMap(model.ThinkingLevelMap, map[ai.ModelThinkingLevel]*string{ - ai.ModelThinkingLevelXHigh: &xhigh, - }) - } - if model.Provider == ai.ProviderOpenRouter && strings.HasPrefix(model.ID, "anthropic/") { - model.Compat = mergeCompat(model.Compat, map[string]any{"cacheControlFormat": "anthropic"}) - } - if model.Provider == ai.ProviderGoogleVertex && gemini3ProPattern.MatchString(model.ID) { - low := "LOW" - high := "HIGH" - model.ThinkingLevelMap = mergeThinkingLevelMap(model.ThinkingLevelMap, map[ai.ModelThinkingLevel]*string{ - ai.ModelThinkingLevelOff: nil, - ai.ModelThinkingLevelMinimal: nil, - ai.ModelThinkingLevelLow: &low, - ai.ModelThinkingLevelMedium: nil, - ai.ModelThinkingLevelHigh: &high, - }) - } - if model.Provider == ai.ProviderGoogleVertex && gemini3FlashPattern.MatchString(model.ID) { - model.ThinkingLevelMap = mergeThinkingLevelMap(model.ThinkingLevelMap, map[ai.ModelThinkingLevel]*string{ - ai.ModelThinkingLevelOff: nil, - }) - } - return model -} - -func openAIAPI(modelID string) ai.Api { - if strings.HasPrefix(modelID, "gpt-5") || strings.HasPrefix(modelID, "o") { - return ai.ApiOpenAIResponses - } - return ai.ApiOpenAICompletions -} - -func costFromModelsDev(cost modelsDevCost) ai.ModelCost { - return ai.ModelCost{ - Input: cost.Input, - Output: cost.Output, - CacheRead: cost.CacheRead, - CacheWrite: cost.CacheWrite, - } -} - -func costFromOpenRouter(pricing openRouterPricing) ai.ModelCost { - return ai.ModelCost{ - Input: pricing.Prompt * 1_000_000, - Output: pricing.Completion * 1_000_000, - CacheRead: pricing.InputCacheRead * 1_000_000, - CacheWrite: pricing.InputCacheWrite * 1_000_000, - } -} - -func textImageInput(modalities []string) []string { - input := make([]string, 0, 2) - if len(modalities) == 0 || slices.Contains(modalities, "text") { - input = append(input, "text") - } - if slices.Contains(modalities, "image") { - input = append(input, "image") - } - return input -} - -func isToolCapableModelsDev(model modelsDevModel) bool { - return model.ToolCall && slices.Contains(model.Modalities.Output, "text") -} - -func isToolCapableOpenRouter(model openRouterModel) bool { - return slices.Contains(model.SupportedParameters, "tools") -} - -func providerSet(providers []ai.Provider) map[ai.Provider]bool { - out := make(map[ai.Provider]bool, len(providers)) - for _, provider := range providers { - out[provider] = true - } - return out -} - -func apiSet(apis []ai.Api) map[ai.Api]bool { - out := make(map[ai.Api]bool, len(apis)) - for _, api := range apis { - out[api] = true - } - return out -} - -func mergeThinkingLevelMap(base, overlay map[ai.ModelThinkingLevel]*string) map[ai.ModelThinkingLevel]*string { - out := make(map[ai.ModelThinkingLevel]*string, len(base)+len(overlay)) - for key, value := range base { - out[key] = value - } - for key, value := range overlay { - out[key] = value - } - return out -} - -func mergeCompat(base, overlay map[string]any) map[string]any { - out := make(map[string]any, len(base)+len(overlay)) - for key, value := range base { - out[key] = value - } - for key, value := range overlay { - out[key] = value - } - return out -} - -func escapeRawString(data []byte) []byte { - return bytes.ReplaceAll(data, []byte("`"), []byte("`+\"`\"+`")) -} - -func firstNonZero(values ...int) int { - for _, value := range values { - if value != 0 { - return value - } - } - return 0 -} - -func intOrDefault(value, fallback int) int { - if value != 0 { - return value - } - return fallback -} - -func stringOrDefault(value, fallback string) string { - if value != "" { - return value - } - return fallback -} diff --git a/pkg/ai/modelcatalog/catalog_test.go b/pkg/ai/modelcatalog/catalog_test.go deleted file mode 100644 index 24dd27d8..00000000 --- a/pkg/ai/modelcatalog/catalog_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package modelcatalog - -import ( - "context" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/beeper/ai-bridge/pkg/ai" -) - -func TestBuildFetchesAndFiltersRuntimeSafeCatalog(t *testing.T) { - mux := http.NewServeMux() - mux.HandleFunc("/models.dev", func(w http.ResponseWriter, _ *http.Request) { - w.Write([]byte(`{ - "openai": {"models": { - "gpt-5.5": { - "id": "gpt-5.5", - "name": "GPT-5.5", - "reasoning": true, - "tool_call": true, - "modalities": {"input": ["text", "image"], "output": ["text"]}, - "cost": {"input": 5, "output": 30, "cache_read": 0.5}, - "limit": {"context": 1050000, "output": 128000} - } - }}, - "anthropic": {"models": { - "claude-sonnet-4-5": { - "id": "claude-sonnet-4-5", - "name": "Claude Sonnet 4.5", - "reasoning": true, - "tool_call": true, - "modalities": {"input": ["text", "image"], "output": ["text"]}, - "cost": {"input": 3, "output": 15}, - "limit": {"context": 200000, "output": 64000} - } - }}, - "google-vertex": {"models": { - "gemini-3-pro-preview": { - "id": "gemini-3-pro-preview", - "name": "Gemini 3 Pro Preview", - "reasoning": true, - "tool_call": true, - "modalities": {"input": ["text"], "output": ["text"]}, - "cost": {"input": 2, "output": 12}, - "limit": {"context": 1000000, "output": 64000} - } - }} - }`)) - }) - mux.HandleFunc("/openrouter", func(w http.ResponseWriter, _ *http.Request) { - w.Write([]byte(`{"data": [{ - "id": "anthropic/claude-sonnet-4.5", - "name": "Anthropic: Claude Sonnet 4.5", - "supported_parameters": ["tools", "reasoning"], - "architecture": {"input_modalities": ["text", "image"]}, - "pricing": {"prompt": "0.000003", "completion": "0.000015", "input_cache_read": "0.0000003"}, - "context_length": 200000, - "top_provider": {"max_completion_tokens": 64000} - }]}`)) - }) - server := httptest.NewServer(mux) - defer server.Close() - - catalog, err := Build(context.Background(), Options{ - HTTPClient: server.Client(), - ModelsDevURL: server.URL + "/models.dev", - OpenRouterModelsURL: server.URL + "/openrouter", - }) - if err != nil { - t.Fatal(err) - } - - if catalog.Count() != 4 { - t.Fatalf("expected runtime-safe catalog to emit four models, got %d", catalog.Count()) - } - if _, ok := catalog.Models[ai.ProviderAnthropic]["claude-sonnet-4-5"]; !ok { - t.Fatal("expected direct Anthropic model after provider registration") - } - if _, ok := catalog.Models[ai.ProviderGoogleVertex]["gemini-3-pro-preview"]; !ok { - t.Fatal("expected Vertex model after provider registration") - } - if got := catalog.Models[ai.ProviderOpenAI]["gpt-5.5"].ThinkingLevelMap[ai.ModelThinkingLevelXHigh]; got == nil || *got != "xhigh" { - t.Fatalf("expected gpt-5.5 xhigh metadata, got %#v", got) - } - if got := catalog.Models[ai.ProviderOpenRouter]["anthropic/claude-sonnet-4.5"].Compat["cacheControlFormat"]; got != "anthropic" { - t.Fatalf("expected OpenRouter Anthropic cache control compat, got %#v", got) - } - if len(catalog.Skipped) != 0 { - t.Fatalf("expected no skipped registered providers, got %#v", catalog.Skipped) - } -} - -func TestBuildCanIncludeUnregisteredProviders(t *testing.T) { - catalog := BuildFromModels([]ai.Model{ - {ID: "claude-sonnet-4-5", Provider: ai.ProviderAnthropic, API: ai.ApiAnthropicMessages}, - {ID: "gemini-3-pro-preview", Provider: ai.ProviderGoogleVertex, API: ai.ApiGoogleVertex}, - }, Options{IncludeUnregistered: true}) - - if catalog.Count() != 2 { - t.Fatalf("expected two models, got %d", catalog.Count()) - } - if _, ok := catalog.Models[ai.ProviderAnthropic]["claude-sonnet-4-5"]; !ok { - t.Fatal("expected Anthropic model") - } - if _, ok := catalog.Models[ai.ProviderGoogleVertex]["gemini-3-pro-preview"]; !ok { - t.Fatal("expected Vertex model") - } -} - -func TestGenerateGoSource(t *testing.T) { - catalog := BuildFromModels([]ai.Model{{ - ID: "gpt-5", - Name: "GPT-5", - API: ai.ApiOpenAIResponses, - Provider: ai.ProviderOpenAI, - BaseURL: "https://api.openai.com/v1", - Input: []string{"text"}, - }}, Options{}) - - source, err := GenerateGoSource(catalog, "ai") - if err != nil { - t.Fatal(err) - } - text := string(source) - for _, want := range []string{"package ai", "var modelsJSON = `", `"gpt-5"`, "func mustLoadModels()"} { - if !strings.Contains(text, want) { - t.Fatalf("generated source missing %q:\n%s", want, text) - } - } -} diff --git a/pkg/ai/modelcatalog/sources.go b/pkg/ai/modelcatalog/sources.go deleted file mode 100644 index 0236f9f9..00000000 --- a/pkg/ai/modelcatalog/sources.go +++ /dev/null @@ -1,118 +0,0 @@ -package modelcatalog - -import ( - "encoding/json" - "strconv" -) - -type modelsDevResponse map[string]modelsDevProvider - -type modelsDevProvider struct { - Models map[string]modelsDevModel `json:"models"` -} - -type modelsDevModel struct { - ID string `json:"id"` - Name string `json:"name"` - Reasoning bool `json:"reasoning"` - ToolCall bool `json:"tool_call"` - Modalities modelsDevModalities `json:"modalities"` - Cost modelsDevCost `json:"cost"` - Limit modelsDevLimit `json:"limit"` -} - -type modelsDevModalities struct { - Input []string `json:"input"` - Output []string `json:"output"` -} - -type modelsDevCost struct { - Input float64 `json:"input"` - Output float64 `json:"output"` - CacheRead float64 `json:"cache_read"` - CacheWrite float64 `json:"cache_write"` -} - -type modelsDevLimit struct { - Context int `json:"context"` - Output int `json:"output"` -} - -type openRouterResponse struct { - Data []openRouterModel `json:"data"` -} - -type openRouterModel struct { - ID string `json:"id"` - Name string `json:"name"` - SupportedParameters []string `json:"supported_parameters"` - Architecture openRouterArchitecture `json:"architecture"` - Pricing openRouterPricing `json:"pricing"` - ContextLength int `json:"context_length"` - TopProvider openRouterTopProvider `json:"top_provider"` -} - -type openRouterArchitecture struct { - InputModalities []string `json:"input_modalities"` -} - -type openRouterPricing struct { - Prompt float64 `json:"prompt"` - Completion float64 `json:"completion"` - InputCacheRead float64 `json:"input_cache_read"` - InputCacheWrite float64 `json:"input_cache_write"` -} - -type openRouterTopProvider struct { - ContextLength int `json:"context_length"` - MaxCompletionTokens int `json:"max_completion_tokens"` -} - -func (pricing *openRouterPricing) UnmarshalJSON(data []byte) error { - type rawPricing struct { - Prompt number `json:"prompt"` - Completion number `json:"completion"` - InputCacheRead number `json:"input_cache_read"` - InputCacheWrite number `json:"input_cache_write"` - } - var raw rawPricing - if err := json.Unmarshal(data, &raw); err != nil { - return err - } - pricing.Prompt = float64(raw.Prompt) - pricing.Completion = float64(raw.Completion) - pricing.InputCacheRead = float64(raw.InputCacheRead) - pricing.InputCacheWrite = float64(raw.InputCacheWrite) - return nil -} - -type number float64 - -func (n *number) UnmarshalJSON(data []byte) error { - if len(data) == 0 || string(data) == "null" { - *n = 0 - return nil - } - if data[0] == '"' { - var text string - if err := json.Unmarshal(data, &text); err != nil { - return err - } - if text == "" { - *n = 0 - return nil - } - value, err := strconv.ParseFloat(text, 64) - if err != nil { - return err - } - *n = number(value) - return nil - } - var value float64 - if err := json.Unmarshal(data, &value); err != nil { - return err - } - *n = number(value) - return nil -} diff --git a/pkg/ai/models.go b/pkg/ai/models.go index 01010779..13e77707 100644 --- a/pkg/ai/models.go +++ b/pkg/ai/models.go @@ -1,7 +1,5 @@ package ai -import "slices" - var extendedThinkingLevels = []ModelThinkingLevel{ ModelThinkingLevelOff, ModelThinkingLevelMinimal, @@ -19,31 +17,6 @@ const ( ModelThinkingLevelXHigh ModelThinkingLevel = "xhigh" ) -func GetModel(provider Provider, modelID string) (Model, bool) { - models, ok := Models[provider] - if !ok { - return Model{}, false - } - model, ok := models[modelID] - return model, ok -} - -func GetProviders() []Provider { - return slices.Clone(modelProviderOrder) -} - -func GetModels(provider Provider) []Model { - models := Models[provider] - order := modelIDOrder[provider] - out := make([]Model, 0, len(order)) - for _, modelID := range order { - if model, ok := models[modelID]; ok { - out = append(out, model) - } - } - return out -} - func GetSupportedThinkingLevels(model Model) []ModelThinkingLevel { if !model.Reasoning { return []ModelThinkingLevel{ModelThinkingLevelOff} @@ -71,7 +44,7 @@ func GetSupportedThinkingLevels(model Model) []ModelThinkingLevel { func ClampThinkingLevel(model Model, level ModelThinkingLevel) ModelThinkingLevel { available := GetSupportedThinkingLevels(model) - if slices.Contains(available, level) { + if containsThinkingLevel(available, level) { return level } requestedIndex := thinkingLevelIndex(level) @@ -80,13 +53,13 @@ func ClampThinkingLevel(model Model, level ModelThinkingLevel) ModelThinkingLeve } for i := requestedIndex; i < len(extendedThinkingLevels); i++ { candidate := extendedThinkingLevels[i] - if slices.Contains(available, candidate) { + if containsThinkingLevel(available, candidate) { return candidate } } for i := requestedIndex - 1; i >= 0; i-- { candidate := extendedThinkingLevels[i] - if slices.Contains(available, candidate) { + if containsThinkingLevel(available, candidate) { return candidate } } @@ -109,6 +82,15 @@ func thinkingLevelIndex(level ModelThinkingLevel) int { return -1 } +func containsThinkingLevel(levels []ModelThinkingLevel, level ModelThinkingLevel) bool { + for _, candidate := range levels { + if candidate == level { + return true + } + } + return false +} + func firstThinkingLevelOrOff(levels []ModelThinkingLevel) ModelThinkingLevel { if len(levels) == 0 { return ModelThinkingLevelOff diff --git a/pkg/ai/models_generated.go b/pkg/ai/models_generated.go deleted file mode 100644 index 87a1cbdf..00000000 --- a/pkg/ai/models_generated.go +++ /dev/null @@ -1,31 +0,0 @@ -package ai - -import "encoding/json" - -var modelsJSON = `{"openai":{"gpt-4":{"id":"gpt-4","name":"GPT-4","api":"openai-completions","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"input":["text"],"cost":{"input":30,"output":60,"cacheRead":0,"cacheWrite":0},"contextWindow":8192,"maxTokens":8192},"gpt-4-turbo":{"id":"gpt-4-turbo","name":"GPT-4 Turbo","api":"openai-completions","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"input":["text","image"],"cost":{"input":10,"output":30,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":4096},"gpt-4.1":{"id":"gpt-4.1","name":"GPT-4.1","api":"openai-completions","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"input":["text","image"],"cost":{"input":2,"output":8,"cacheRead":0.5,"cacheWrite":0},"contextWindow":1047576,"maxTokens":32768},"gpt-4.1-mini":{"id":"gpt-4.1-mini","name":"GPT-4.1 mini","api":"openai-completions","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.4,"output":1.6,"cacheRead":0.1,"cacheWrite":0},"contextWindow":1047576,"maxTokens":32768},"gpt-4.1-nano":{"id":"gpt-4.1-nano","name":"GPT-4.1 nano","api":"openai-completions","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.1,"output":0.4,"cacheRead":0.03,"cacheWrite":0},"contextWindow":1047576,"maxTokens":32768},"gpt-4o":{"id":"gpt-4o","name":"GPT-4o","api":"openai-completions","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"input":["text","image"],"cost":{"input":2.5,"output":10,"cacheRead":1.25,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"gpt-4o-2024-05-13":{"id":"gpt-4o-2024-05-13","name":"GPT-4o (2024-05-13)","api":"openai-completions","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"input":["text","image"],"cost":{"input":5,"output":15,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":4096},"gpt-4o-2024-08-06":{"id":"gpt-4o-2024-08-06","name":"GPT-4o (2024-08-06)","api":"openai-completions","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"input":["text","image"],"cost":{"input":2.5,"output":10,"cacheRead":1.25,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"gpt-4o-2024-11-20":{"id":"gpt-4o-2024-11-20","name":"GPT-4o (2024-11-20)","api":"openai-completions","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"input":["text","image"],"cost":{"input":2.5,"output":10,"cacheRead":1.25,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"gpt-4o-mini":{"id":"gpt-4o-mini","name":"GPT-4o mini","api":"openai-completions","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.15,"output":0.6,"cacheRead":0.08,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"gpt-5":{"id":"gpt-5","name":"GPT-5","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null},"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5-codex":{"id":"gpt-5-codex","name":"GPT-5-Codex","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null},"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5-mini":{"id":"gpt-5-mini","name":"GPT-5 Mini","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null},"input":["text","image"],"cost":{"input":0.25,"output":2,"cacheRead":0.025,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5-nano":{"id":"gpt-5-nano","name":"GPT-5 Nano","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null},"input":["text","image"],"cost":{"input":0.05,"output":0.4,"cacheRead":0.005,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5-pro":{"id":"gpt-5-pro","name":"GPT-5 Pro","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null},"input":["text","image"],"cost":{"input":15,"output":120,"cacheRead":0,"cacheWrite":0},"contextWindow":400000,"maxTokens":272000},"gpt-5.1":{"id":"gpt-5.1","name":"GPT-5.1","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null},"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.13,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5.1-chat-latest":{"id":"gpt-5.1-chat-latest","name":"GPT-5.1 Chat","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null},"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"gpt-5.1-codex":{"id":"gpt-5.1-codex","name":"GPT-5.1 Codex","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null},"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5.1-codex-max":{"id":"gpt-5.1-codex-max","name":"GPT-5.1 Codex Max","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null},"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5.1-codex-mini":{"id":"gpt-5.1-codex-mini","name":"GPT-5.1 Codex mini","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null},"input":["text","image"],"cost":{"input":0.25,"output":2,"cacheRead":0.025,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5.2":{"id":"gpt-5.2","name":"GPT-5.2","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5.2-chat-latest":{"id":"gpt-5.2-chat-latest","name":"GPT-5.2 Chat","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"gpt-5.2-codex":{"id":"gpt-5.2-codex","name":"GPT-5.2 Codex","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5.2-pro":{"id":"gpt-5.2-pro","name":"GPT-5.2 Pro","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":21,"output":168,"cacheRead":0,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5.3-chat-latest":{"id":"gpt-5.3-chat-latest","name":"GPT-5.3 Chat (latest)","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":false,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"gpt-5.3-codex":{"id":"gpt-5.3-codex","name":"GPT-5.3 Codex","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5.3-codex-spark":{"id":"gpt-5.3-codex-spark","name":"GPT-5.3 Codex Spark","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":128000,"maxTokens":32000},"gpt-5.4":{"id":"gpt-5.4","name":"GPT-5.4","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":2.5,"output":15,"cacheRead":0.25,"cacheWrite":0},"contextWindow":1050000,"maxTokens":128000},"gpt-5.4-mini":{"id":"gpt-5.4-mini","name":"GPT-5.4 mini","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":0.75,"output":4.5,"cacheRead":0.075,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5.4-nano":{"id":"gpt-5.4-nano","name":"GPT-5.4 nano","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":0.2,"output":1.25,"cacheRead":0.02,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"gpt-5.4-pro":{"id":"gpt-5.4-pro","name":"GPT-5.4 Pro","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":30,"output":180,"cacheRead":0,"cacheWrite":0},"contextWindow":1050000,"maxTokens":128000},"gpt-5.5":{"id":"gpt-5.5","name":"GPT-5.5","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":5,"output":30,"cacheRead":0.5,"cacheWrite":0},"contextWindow":1050000,"maxTokens":128000},"gpt-5.5-pro":{"id":"gpt-5.5-pro","name":"GPT-5.5 Pro","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"thinkingLevelMap":{"off":null,"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":30,"output":180,"cacheRead":0,"cacheWrite":0},"contextWindow":1050000,"maxTokens":128000},"o1":{"id":"o1","name":"o1","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"input":["text","image"],"cost":{"input":15,"output":60,"cacheRead":7.5,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"o1-pro":{"id":"o1-pro","name":"o1-pro","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"input":["text","image"],"cost":{"input":150,"output":600,"cacheRead":0,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"o3":{"id":"o3","name":"o3","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"input":["text","image"],"cost":{"input":2,"output":8,"cacheRead":0.5,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"o3-deep-research":{"id":"o3-deep-research","name":"o3-deep-research","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"input":["text","image"],"cost":{"input":10,"output":40,"cacheRead":2.5,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"o3-mini":{"id":"o3-mini","name":"o3-mini","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"input":["text"],"cost":{"input":1.1,"output":4.4,"cacheRead":0.55,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"o3-pro":{"id":"o3-pro","name":"o3-pro","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"input":["text","image"],"cost":{"input":20,"output":80,"cacheRead":0,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"o4-mini":{"id":"o4-mini","name":"o4-mini","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.1,"output":4.4,"cacheRead":0.28,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"o4-mini-deep-research":{"id":"o4-mini-deep-research","name":"o4-mini-deep-research","api":"openai-responses","provider":"openai","baseUrl":"https://api.openai.com/v1","reasoning":true,"input":["text","image"],"cost":{"input":2,"output":8,"cacheRead":0.5,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000}},"openrouter":{"ai21/jamba-large-1.7":{"id":"ai21/jamba-large-1.7","name":"AI21: Jamba Large 1.7","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":2,"output":8,"cacheRead":0,"cacheWrite":0},"contextWindow":256000,"maxTokens":4096},"alibaba/tongyi-deepresearch-30b-a3b":{"id":"alibaba/tongyi-deepresearch-30b-a3b","name":"Tongyi DeepResearch 30B A3B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.09,"output":0.44999999999999996,"cacheRead":0.09,"cacheWrite":0},"contextWindow":131072,"maxTokens":131072},"amazon/nova-2-lite-v1":{"id":"amazon/nova-2-lite-v1","name":"Amazon: Nova 2 Lite","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.3,"output":2.5,"cacheRead":0,"cacheWrite":0},"contextWindow":1000000,"maxTokens":65535},"amazon/nova-lite-v1":{"id":"amazon/nova-lite-v1","name":"Amazon: Nova Lite 1.0","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.06,"output":0.24,"cacheRead":0,"cacheWrite":0},"contextWindow":300000,"maxTokens":5120},"amazon/nova-micro-v1":{"id":"amazon/nova-micro-v1","name":"Amazon: Nova Micro 1.0","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.035,"output":0.14,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":5120},"amazon/nova-premier-v1":{"id":"amazon/nova-premier-v1","name":"Amazon: Nova Premier 1.0","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":2.5,"output":12.5,"cacheRead":0.625,"cacheWrite":0},"contextWindow":1000000,"maxTokens":32000},"amazon/nova-pro-v1":{"id":"amazon/nova-pro-v1","name":"Amazon: Nova Pro 1.0","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.7999999999999999,"output":3.1999999999999997,"cacheRead":0,"cacheWrite":0},"contextWindow":300000,"maxTokens":5120},"anthropic/claude-3-haiku":{"id":"anthropic/claude-3-haiku","name":"Anthropic: Claude 3 Haiku","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.25,"output":1.25,"cacheRead":0.03,"cacheWrite":0.3},"contextWindow":200000,"maxTokens":4096,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-3.5-haiku":{"id":"anthropic/claude-3.5-haiku","name":"Anthropic: Claude 3.5 Haiku","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.7999999999999999,"output":4,"cacheRead":0.08,"cacheWrite":1},"contextWindow":200000,"maxTokens":8192,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-haiku-4.5":{"id":"anthropic/claude-haiku-4.5","name":"Anthropic: Claude Haiku 4.5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1,"output":5,"cacheRead":0.09999999999999999,"cacheWrite":1.25},"contextWindow":200000,"maxTokens":64000,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-opus-4":{"id":"anthropic/claude-opus-4","name":"Anthropic: Claude Opus 4","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":15,"output":75,"cacheRead":1.5,"cacheWrite":18.75},"contextWindow":200000,"maxTokens":32000,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-opus-4.1":{"id":"anthropic/claude-opus-4.1","name":"Anthropic: Claude Opus 4.1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":15,"output":75,"cacheRead":1.5,"cacheWrite":18.75},"contextWindow":200000,"maxTokens":32000,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-opus-4.5":{"id":"anthropic/claude-opus-4.5","name":"Anthropic: Claude Opus 4.5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":5,"output":25,"cacheRead":0.5,"cacheWrite":6.25},"contextWindow":200000,"maxTokens":64000,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-opus-4.6":{"id":"anthropic/claude-opus-4.6","name":"Anthropic: Claude Opus 4.6","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":5,"output":25,"cacheRead":0.5,"cacheWrite":6.25},"contextWindow":1000000,"maxTokens":128000,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-opus-4.6-fast":{"id":"anthropic/claude-opus-4.6-fast","name":"Anthropic: Claude Opus 4.6 (Fast)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":30,"output":150,"cacheRead":3,"cacheWrite":37.5},"contextWindow":1000000,"maxTokens":128000,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-opus-4.7":{"id":"anthropic/claude-opus-4.7","name":"Anthropic: Claude Opus 4.7","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":5,"output":25,"cacheRead":0.5,"cacheWrite":6.25},"contextWindow":1000000,"maxTokens":128000,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-opus-4.7-fast":{"id":"anthropic/claude-opus-4.7-fast","name":"Anthropic: Claude Opus 4.7 (Fast)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":30,"output":150,"cacheRead":3,"cacheWrite":37.5},"contextWindow":1000000,"maxTokens":128000,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-sonnet-4":{"id":"anthropic/claude-sonnet-4","name":"Anthropic: Claude Sonnet 4","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":3,"output":15,"cacheRead":0.3,"cacheWrite":3.75},"contextWindow":1000000,"maxTokens":64000,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-sonnet-4.5":{"id":"anthropic/claude-sonnet-4.5","name":"Anthropic: Claude Sonnet 4.5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":3,"output":15,"cacheRead":0.3,"cacheWrite":3.75},"contextWindow":1000000,"maxTokens":64000,"compat":{"cacheControlFormat":"anthropic"}},"anthropic/claude-sonnet-4.6":{"id":"anthropic/claude-sonnet-4.6","name":"Anthropic: Claude Sonnet 4.6","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":3,"output":15,"cacheRead":0.3,"cacheWrite":3.75},"contextWindow":1000000,"maxTokens":128000,"compat":{"cacheControlFormat":"anthropic"}},"arcee-ai/trinity-large-thinking":{"id":"arcee-ai/trinity-large-thinking","name":"Arcee AI: Trinity Large Thinking","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.22,"output":0.85,"cacheRead":0.06,"cacheWrite":0},"contextWindow":262144,"maxTokens":262144},"arcee-ai/trinity-large-thinking:free":{"id":"arcee-ai/trinity-large-thinking:free","name":"Arcee AI: Trinity Large Thinking (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":80000},"arcee-ai/trinity-mini":{"id":"arcee-ai/trinity-mini","name":"Arcee AI: Trinity Mini","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.045,"output":0.15,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":131072},"arcee-ai/virtuoso-large":{"id":"arcee-ai/virtuoso-large","name":"Arcee AI: Virtuoso Large","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.75,"output":1.2,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":64000},"baidu/cobuddy:free":{"id":"baidu/cobuddy:free","name":"Baidu Qianfan: CoBuddy (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":65536},"baidu/ernie-4.5-21b-a3b":{"id":"baidu/ernie-4.5-21b-a3b","name":"Baidu: ERNIE 4.5 21B A3B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.07,"output":0.28,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":8000},"baidu/ernie-4.5-vl-28b-a3b":{"id":"baidu/ernie-4.5-vl-28b-a3b","name":"Baidu: ERNIE 4.5 VL 28B A3B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.14,"output":0.56,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":8000},"bytedance-seed/seed-1.6":{"id":"bytedance-seed/seed-1.6","name":"ByteDance Seed: Seed 1.6","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.25,"output":2,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":32768},"bytedance-seed/seed-1.6-flash":{"id":"bytedance-seed/seed-1.6-flash","name":"ByteDance Seed: Seed 1.6 Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.075,"output":0.3,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":32768},"bytedance-seed/seed-2.0-lite":{"id":"bytedance-seed/seed-2.0-lite","name":"ByteDance Seed: Seed-2.0-Lite","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.25,"output":2,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":131072},"bytedance-seed/seed-2.0-mini":{"id":"bytedance-seed/seed-2.0-mini","name":"ByteDance Seed: Seed-2.0-Mini","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.09999999999999999,"output":0.39999999999999997,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":131072},"cohere/command-r-08-2024":{"id":"cohere/command-r-08-2024","name":"Cohere: Command R (08-2024)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.15,"output":0.6,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":4000},"cohere/command-r-plus-08-2024":{"id":"cohere/command-r-plus-08-2024","name":"Cohere: Command R+ (08-2024)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":2.5,"output":10,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":4000},"deepseek/deepseek-chat":{"id":"deepseek/deepseek-chat","name":"DeepSeek: DeepSeek V3","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.32,"output":0.8899999999999999,"cacheRead":0,"cacheWrite":0},"contextWindow":163840,"maxTokens":16384},"deepseek/deepseek-chat-v3-0324":{"id":"deepseek/deepseek-chat-v3-0324","name":"DeepSeek: DeepSeek V3 0324","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.19999999999999998,"output":0.77,"cacheRead":0.135,"cacheWrite":0},"contextWindow":163840,"maxTokens":16384},"deepseek/deepseek-chat-v3.1":{"id":"deepseek/deepseek-chat-v3.1","name":"DeepSeek: DeepSeek V3.1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.21,"output":0.7899999999999999,"cacheRead":0.13,"cacheWrite":0},"contextWindow":163840,"maxTokens":32768},"deepseek/deepseek-r1":{"id":"deepseek/deepseek-r1","name":"DeepSeek: R1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.7,"output":2.5,"cacheRead":0,"cacheWrite":0},"contextWindow":163840,"maxTokens":16000},"deepseek/deepseek-r1-0528":{"id":"deepseek/deepseek-r1-0528","name":"DeepSeek: R1 0528","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.5,"output":2.1500000000000004,"cacheRead":0.35,"cacheWrite":0},"contextWindow":163840,"maxTokens":32768},"deepseek/deepseek-v3.1-terminus":{"id":"deepseek/deepseek-v3.1-terminus","name":"DeepSeek: DeepSeek V3.1 Terminus","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.27,"output":0.95,"cacheRead":0.13,"cacheWrite":0},"contextWindow":163840,"maxTokens":32768},"deepseek/deepseek-v3.2":{"id":"deepseek/deepseek-v3.2","name":"DeepSeek: DeepSeek V3.2","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.252,"output":0.378,"cacheRead":0.0252,"cacheWrite":0},"contextWindow":131072,"maxTokens":65536},"deepseek/deepseek-v3.2-exp":{"id":"deepseek/deepseek-v3.2-exp","name":"DeepSeek: DeepSeek V3.2 Exp","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.27,"output":0.41,"cacheRead":0,"cacheWrite":0},"contextWindow":163840,"maxTokens":65536},"deepseek/deepseek-v4-flash":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek: DeepSeek V4 Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.09999999999999999,"output":0.19999999999999998,"cacheRead":0.02,"cacheWrite":0},"contextWindow":1048576,"maxTokens":16384},"deepseek/deepseek-v4-flash:free":{"id":"deepseek/deepseek-v4-flash:free","name":"DeepSeek: DeepSeek V4 Flash (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":1048576,"maxTokens":384000},"deepseek/deepseek-v4-pro":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek: DeepSeek V4 Pro","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.435,"output":0.87,"cacheRead":0.003625,"cacheWrite":0},"contextWindow":1048576,"maxTokens":384000},"essentialai/rnj-1-instruct":{"id":"essentialai/rnj-1-instruct","name":"EssentialAI: Rnj 1 Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.15,"output":0.15,"cacheRead":0,"cacheWrite":0},"contextWindow":32768,"maxTokens":16384},"google/gemini-2.0-flash-001":{"id":"google/gemini-2.0-flash-001","name":"Google: Gemini 2.0 Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.09999999999999999,"output":0.39999999999999997,"cacheRead":0.024999999999999998,"cacheWrite":0.08333333333333334},"contextWindow":1000000,"maxTokens":8192},"google/gemini-2.0-flash-lite-001":{"id":"google/gemini-2.0-flash-lite-001","name":"Google: Gemini 2.0 Flash Lite","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.075,"output":0.3,"cacheRead":0,"cacheWrite":0},"contextWindow":1048576,"maxTokens":8192},"google/gemini-2.5-flash":{"id":"google/gemini-2.5-flash","name":"Google: Gemini 2.5 Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.3,"output":2.5,"cacheRead":0.03,"cacheWrite":0.08333333333333334},"contextWindow":1048576,"maxTokens":65535},"google/gemini-2.5-flash-lite":{"id":"google/gemini-2.5-flash-lite","name":"Google: Gemini 2.5 Flash Lite","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.09999999999999999,"output":0.39999999999999997,"cacheRead":0.01,"cacheWrite":0.08333333333333334},"contextWindow":1048576,"maxTokens":65535},"google/gemini-2.5-flash-lite-preview-09-2025":{"id":"google/gemini-2.5-flash-lite-preview-09-2025","name":"Google: Gemini 2.5 Flash Lite Preview 09-2025","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.09999999999999999,"output":0.39999999999999997,"cacheRead":0.01,"cacheWrite":0.08333333333333334},"contextWindow":1048576,"maxTokens":65535},"google/gemini-2.5-pro":{"id":"google/gemini-2.5-pro","name":"Google: Gemini 2.5 Pro","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0.375},"contextWindow":1048576,"maxTokens":65536},"google/gemini-2.5-pro-preview":{"id":"google/gemini-2.5-pro-preview","name":"Google: Gemini 2.5 Pro Preview 06-05","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0.375},"contextWindow":1048576,"maxTokens":65536},"google/gemini-2.5-pro-preview-05-06":{"id":"google/gemini-2.5-pro-preview-05-06","name":"Google: Gemini 2.5 Pro Preview 05-06","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0.375},"contextWindow":1048576,"maxTokens":65535},"google/gemini-3-flash-preview":{"id":"google/gemini-3-flash-preview","name":"Google: Gemini 3 Flash Preview","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.5,"output":3,"cacheRead":0.049999999999999996,"cacheWrite":0.08333333333333334},"contextWindow":1048576,"maxTokens":65536},"google/gemini-3.1-flash-lite":{"id":"google/gemini-3.1-flash-lite","name":"Google: Gemini 3.1 Flash Lite","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.25,"output":1.5,"cacheRead":0.024999999999999998,"cacheWrite":0.08333333333333334},"contextWindow":1048576,"maxTokens":65536},"google/gemini-3.1-flash-lite-preview":{"id":"google/gemini-3.1-flash-lite-preview","name":"Google: Gemini 3.1 Flash Lite Preview","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.25,"output":1.5,"cacheRead":0.024999999999999998,"cacheWrite":0.08333333333333334},"contextWindow":1048576,"maxTokens":65536},"google/gemini-3.1-pro-preview":{"id":"google/gemini-3.1-pro-preview","name":"Google: Gemini 3.1 Pro Preview","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":2,"output":12,"cacheRead":0.19999999999999998,"cacheWrite":0.375},"contextWindow":1048576,"maxTokens":65536},"google/gemini-3.1-pro-preview-customtools":{"id":"google/gemini-3.1-pro-preview-customtools","name":"Google: Gemini 3.1 Pro Preview Custom Tools","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":2,"output":12,"cacheRead":0.19999999999999998,"cacheWrite":0.375},"contextWindow":1048756,"maxTokens":65536},"google/gemini-3.5-flash":{"id":"google/gemini-3.5-flash","name":"Google: Gemini 3.5 Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.5,"output":9,"cacheRead":0.15,"cacheWrite":0.08333333333333334},"contextWindow":1048576,"maxTokens":65536},"google/gemma-3-12b-it":{"id":"google/gemma-3-12b-it","name":"Google: Gemma 3 12B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.04,"output":0.13,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"google/gemma-3-27b-it":{"id":"google/gemma-3-27b-it","name":"Google: Gemma 3 27B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.08,"output":0.16,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"google/gemma-4-26b-a4b-it":{"id":"google/gemma-4-26b-a4b-it","name":"Google: Gemma 4 26B A4B ","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.06,"output":0.33,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"google/gemma-4-26b-a4b-it:free":{"id":"google/gemma-4-26b-a4b-it:free","name":"Google: Gemma 4 26B A4B (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":32768},"google/gemma-4-31b-it":{"id":"google/gemma-4-31b-it","name":"Google: Gemma 4 31B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.12,"output":0.37,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"google/gemma-4-31b-it:free":{"id":"google/gemma-4-31b-it:free","name":"Google: Gemma 4 31B (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":32768},"ibm-granite/granite-4.1-8b":{"id":"ibm-granite/granite-4.1-8b","name":"IBM: Granite 4.1 8B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.049999999999999996,"output":0.09999999999999999,"cacheRead":0.049999999999999996,"cacheWrite":0},"contextWindow":131072,"maxTokens":131072},"inception/mercury-2":{"id":"inception/mercury-2","name":"Inception: Mercury 2","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.25,"output":0.75,"cacheRead":0.024999999999999998,"cacheWrite":0},"contextWindow":128000,"maxTokens":50000},"inclusionai/ling-2.6-1t":{"id":"inclusionai/ling-2.6-1t","name":"inclusionAI: Ling-2.6-1T","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.075,"output":0.625,"cacheRead":0.015,"cacheWrite":0},"contextWindow":262144,"maxTokens":32768},"inclusionai/ling-2.6-flash":{"id":"inclusionai/ling-2.6-flash","name":"inclusionAI: Ling-2.6-flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.01,"output":0.03,"cacheRead":0.002,"cacheWrite":0},"contextWindow":262144,"maxTokens":32768},"inclusionai/ring-2.6-1t":{"id":"inclusionai/ring-2.6-1t","name":"inclusionAI: Ring-2.6-1T","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.075,"output":0.625,"cacheRead":0.015,"cacheWrite":0},"contextWindow":262144,"maxTokens":65536},"kwaipilot/kat-coder-pro-v2":{"id":"kwaipilot/kat-coder-pro-v2","name":"Kwaipilot: KAT-Coder-Pro V2","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.3,"output":1.2,"cacheRead":0.06,"cacheWrite":0},"contextWindow":256000,"maxTokens":80000},"meta-llama/llama-3.1-70b-instruct":{"id":"meta-llama/llama-3.1-70b-instruct","name":"Meta: Llama 3.1 70B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.39999999999999997,"output":0.39999999999999997,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"meta-llama/llama-3.1-8b-instruct":{"id":"meta-llama/llama-3.1-8b-instruct","name":"Meta: Llama 3.1 8B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.02,"output":0.049999999999999996,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"meta-llama/llama-3.3-70b-instruct":{"id":"meta-llama/llama-3.3-70b-instruct","name":"Meta: Llama 3.3 70B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.09999999999999999,"output":0.32,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"meta-llama/llama-3.3-70b-instruct:free":{"id":"meta-llama/llama-3.3-70b-instruct:free","name":"Meta: Llama 3.3 70B Instruct (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"meta-llama/llama-4-scout":{"id":"meta-llama/llama-4-scout","name":"Meta: Llama 4 Scout","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.08,"output":0.3,"cacheRead":0,"cacheWrite":0},"contextWindow":10000000,"maxTokens":16384},"minimax/minimax-m1":{"id":"minimax/minimax-m1","name":"MiniMax: MiniMax M1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.39999999999999997,"output":2.2,"cacheRead":0,"cacheWrite":0},"contextWindow":1000000,"maxTokens":40000},"minimax/minimax-m2":{"id":"minimax/minimax-m2","name":"MiniMax: MiniMax M2","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.255,"output":1,"cacheRead":0.03,"cacheWrite":0},"contextWindow":204800,"maxTokens":196608},"minimax/minimax-m2.1":{"id":"minimax/minimax-m2.1","name":"MiniMax: MiniMax M2.1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.29,"output":0.95,"cacheRead":0.03,"cacheWrite":0},"contextWindow":204800,"maxTokens":196608},"minimax/minimax-m2.5":{"id":"minimax/minimax-m2.5","name":"MiniMax: MiniMax M2.5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.15,"output":1.15,"cacheRead":0,"cacheWrite":0},"contextWindow":204800,"maxTokens":196608},"minimax/minimax-m2.5:free":{"id":"minimax/minimax-m2.5:free","name":"MiniMax: MiniMax M2.5 (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":204800,"maxTokens":8192},"minimax/minimax-m2.7":{"id":"minimax/minimax-m2.7","name":"MiniMax: MiniMax M2.7","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.27899999999999997,"output":1.2,"cacheRead":0,"cacheWrite":0},"contextWindow":204800,"maxTokens":131072},"mistralai/codestral-2508":{"id":"mistralai/codestral-2508","name":"Mistral: Codestral 2508","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.3,"output":0.8999999999999999,"cacheRead":0.03,"cacheWrite":0},"contextWindow":256000,"maxTokens":16384},"mistralai/devstral-2512":{"id":"mistralai/devstral-2512","name":"Mistral: Devstral 2 2512","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.39999999999999997,"output":2,"cacheRead":0.04,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"mistralai/devstral-medium":{"id":"mistralai/devstral-medium","name":"Mistral: Devstral Medium","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.39999999999999997,"output":2,"cacheRead":0.04,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"mistralai/devstral-small":{"id":"mistralai/devstral-small","name":"Mistral: Devstral Small 1.1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.09999999999999999,"output":0.3,"cacheRead":0.01,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"mistralai/ministral-14b-2512":{"id":"mistralai/ministral-14b-2512","name":"Mistral: Ministral 3 14B 2512","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.19999999999999998,"output":0.19999999999999998,"cacheRead":0.02,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"mistralai/ministral-3b-2512":{"id":"mistralai/ministral-3b-2512","name":"Mistral: Ministral 3 3B 2512","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.09999999999999999,"output":0.09999999999999999,"cacheRead":0.01,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"mistralai/ministral-8b-2512":{"id":"mistralai/ministral-8b-2512","name":"Mistral: Ministral 3 8B 2512","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.15,"output":0.15,"cacheRead":0.015,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"mistralai/mistral-large":{"id":"mistralai/mistral-large","name":"Mistral Large","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":2,"output":6,"cacheRead":0.19999999999999998,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"mistralai/mistral-large-2407":{"id":"mistralai/mistral-large-2407","name":"Mistral Large 2407","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":2,"output":6,"cacheRead":0.19999999999999998,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"mistralai/mistral-large-2411":{"id":"mistralai/mistral-large-2411","name":"Mistral Large 2411","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":2,"output":6,"cacheRead":0.19999999999999998,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"mistralai/mistral-large-2512":{"id":"mistralai/mistral-large-2512","name":"Mistral: Mistral Large 3 2512","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.5,"output":1.5,"cacheRead":0.049999999999999996,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"mistralai/mistral-medium-3":{"id":"mistralai/mistral-medium-3","name":"Mistral: Mistral Medium 3","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.39999999999999997,"output":2,"cacheRead":0.04,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"mistralai/mistral-medium-3-5":{"id":"mistralai/mistral-medium-3-5","name":"Mistral: Mistral Medium 3.5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.5,"output":7.5,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"mistralai/mistral-medium-3.1":{"id":"mistralai/mistral-medium-3.1","name":"Mistral: Mistral Medium 3.1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.39999999999999997,"output":2,"cacheRead":0.04,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"mistralai/mistral-nemo":{"id":"mistralai/mistral-nemo","name":"Mistral: Mistral Nemo","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.02,"output":0.03,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"mistralai/mistral-saba":{"id":"mistralai/mistral-saba","name":"Mistral: Saba","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.19999999999999998,"output":0.6,"cacheRead":0.02,"cacheWrite":0},"contextWindow":32768,"maxTokens":16384},"mistralai/mistral-small-2603":{"id":"mistralai/mistral-small-2603","name":"Mistral: Mistral Small 4","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.15,"output":0.6,"cacheRead":0.015,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"mistralai/mistral-small-3.2-24b-instruct":{"id":"mistralai/mistral-small-3.2-24b-instruct","name":"Mistral: Mistral Small 3.2 24B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.075,"output":0.19999999999999998,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"mistralai/mixtral-8x22b-instruct":{"id":"mistralai/mixtral-8x22b-instruct","name":"Mistral: Mixtral 8x22B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":2,"output":6,"cacheRead":0.19999999999999998,"cacheWrite":0},"contextWindow":65536,"maxTokens":16384},"mistralai/pixtral-large-2411":{"id":"mistralai/pixtral-large-2411","name":"Mistral: Pixtral Large 2411","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":2,"output":6,"cacheRead":0.19999999999999998,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"mistralai/voxtral-small-24b-2507":{"id":"mistralai/voxtral-small-24b-2507","name":"Mistral: Voxtral Small 24B 2507","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.09999999999999999,"output":0.3,"cacheRead":0.01,"cacheWrite":0},"contextWindow":32000,"maxTokens":16384},"moonshotai/kimi-k2":{"id":"moonshotai/kimi-k2","name":"MoonshotAI: Kimi K2 0711","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.5700000000000001,"output":2.3,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":32768},"moonshotai/kimi-k2-0905":{"id":"moonshotai/kimi-k2-0905","name":"MoonshotAI: Kimi K2 0905","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.6,"output":2.5,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":262144},"moonshotai/kimi-k2-thinking":{"id":"moonshotai/kimi-k2-thinking","name":"MoonshotAI: Kimi K2 Thinking","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.6,"output":2.5,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":262144},"moonshotai/kimi-k2.5":{"id":"moonshotai/kimi-k2.5","name":"MoonshotAI: Kimi K2.5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.39999999999999997,"output":1.9,"cacheRead":0.09,"cacheWrite":0},"contextWindow":262144,"maxTokens":262144},"moonshotai/kimi-k2.6":{"id":"moonshotai/kimi-k2.6","name":"MoonshotAI: Kimi K2.6","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.73,"output":3.49,"cacheRead":0.25,"cacheWrite":0},"contextWindow":262144,"maxTokens":262142},"nex-agi/deepseek-v3.1-nex-n1":{"id":"nex-agi/deepseek-v3.1-nex-n1","name":"Nex AGI: DeepSeek V3.1 Nex N1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.135,"output":0.5,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":163840},"nvidia/llama-3.3-nemotron-super-49b-v1.5":{"id":"nvidia/llama-3.3-nemotron-super-49b-v1.5","name":"NVIDIA: Llama 3.3 Nemotron Super 49B V1.5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.09999999999999999,"output":0.39999999999999997,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"nvidia/nemotron-3-nano-30b-a3b":{"id":"nvidia/nemotron-3-nano-30b-a3b","name":"NVIDIA: Nemotron 3 Nano 30B A3B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.049999999999999996,"output":0.19999999999999998,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":228000},"nvidia/nemotron-3-nano-30b-a3b:free":{"id":"nvidia/nemotron-3-nano-30b-a3b:free","name":"NVIDIA: Nemotron 3 Nano 30B A3B (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":256000,"maxTokens":16384},"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free":{"id":"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free","name":"NVIDIA: Nemotron 3 Nano Omni (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":256000,"maxTokens":65536},"nvidia/nemotron-3-super-120b-a12b":{"id":"nvidia/nemotron-3-super-120b-a12b","name":"NVIDIA: Nemotron 3 Super","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.09,"output":0.44999999999999996,"cacheRead":0,"cacheWrite":0},"contextWindow":1000000,"maxTokens":16384},"nvidia/nemotron-3-super-120b-a12b:free":{"id":"nvidia/nemotron-3-super-120b-a12b:free","name":"NVIDIA: Nemotron 3 Super (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":1000000,"maxTokens":262144},"nvidia/nemotron-nano-12b-v2-vl:free":{"id":"nvidia/nemotron-nano-12b-v2-vl:free","name":"NVIDIA: Nemotron Nano 12B 2 VL (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":128000},"nvidia/nemotron-nano-9b-v2":{"id":"nvidia/nemotron-nano-9b-v2","name":"NVIDIA: Nemotron Nano 9B V2","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.04,"output":0.16,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"nvidia/nemotron-nano-9b-v2:free":{"id":"nvidia/nemotron-nano-9b-v2:free","name":"NVIDIA: Nemotron Nano 9B V2 (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-3.5-turbo":{"id":"openai/gpt-3.5-turbo","name":"OpenAI: GPT-3.5 Turbo","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.5,"output":1.5,"cacheRead":0,"cacheWrite":0},"contextWindow":16385,"maxTokens":4096},"openai/gpt-3.5-turbo-0613":{"id":"openai/gpt-3.5-turbo-0613","name":"OpenAI: GPT-3.5 Turbo (older v0613)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":1,"output":2,"cacheRead":0,"cacheWrite":0},"contextWindow":4095,"maxTokens":4096},"openai/gpt-3.5-turbo-16k":{"id":"openai/gpt-3.5-turbo-16k","name":"OpenAI: GPT-3.5 Turbo 16k","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":3,"output":4,"cacheRead":0,"cacheWrite":0},"contextWindow":16385,"maxTokens":4096},"openai/gpt-4":{"id":"openai/gpt-4","name":"OpenAI: GPT-4","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":30,"output":60,"cacheRead":0,"cacheWrite":0},"contextWindow":8191,"maxTokens":4096},"openai/gpt-4-0314":{"id":"openai/gpt-4-0314","name":"OpenAI: GPT-4 (older v0314)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":30,"output":60,"cacheRead":0,"cacheWrite":0},"contextWindow":8191,"maxTokens":4096},"openai/gpt-4-1106-preview":{"id":"openai/gpt-4-1106-preview","name":"OpenAI: GPT-4 Turbo (older v1106)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":10,"output":30,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":4096},"openai/gpt-4-turbo":{"id":"openai/gpt-4-turbo","name":"OpenAI: GPT-4 Turbo","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":10,"output":30,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":4096},"openai/gpt-4-turbo-preview":{"id":"openai/gpt-4-turbo-preview","name":"OpenAI: GPT-4 Turbo Preview","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":10,"output":30,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":4096},"openai/gpt-4.1":{"id":"openai/gpt-4.1","name":"OpenAI: GPT-4.1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":2,"output":8,"cacheRead":0.5,"cacheWrite":0},"contextWindow":1047576,"maxTokens":16384},"openai/gpt-4.1-mini":{"id":"openai/gpt-4.1-mini","name":"OpenAI: GPT-4.1 Mini","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.39999999999999997,"output":1.5999999999999999,"cacheRead":0.09999999999999999,"cacheWrite":0},"contextWindow":1047576,"maxTokens":32768},"openai/gpt-4.1-nano":{"id":"openai/gpt-4.1-nano","name":"OpenAI: GPT-4.1 Nano","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.09999999999999999,"output":0.39999999999999997,"cacheRead":0.024999999999999998,"cacheWrite":0},"contextWindow":1047576,"maxTokens":32768},"openai/gpt-4o":{"id":"openai/gpt-4o","name":"OpenAI: GPT-4o","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":2.5,"output":10,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-4o-2024-05-13":{"id":"openai/gpt-4o-2024-05-13","name":"OpenAI: GPT-4o (2024-05-13)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":5,"output":15,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":4096},"openai/gpt-4o-2024-08-06":{"id":"openai/gpt-4o-2024-08-06","name":"OpenAI: GPT-4o (2024-08-06)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":2.5,"output":10,"cacheRead":1.25,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-4o-2024-11-20":{"id":"openai/gpt-4o-2024-11-20","name":"OpenAI: GPT-4o (2024-11-20)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":2.5,"output":10,"cacheRead":1.25,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-4o-audio-preview":{"id":"openai/gpt-4o-audio-preview","name":"OpenAI: GPT-4o Audio","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":2.5,"output":10,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-4o-mini":{"id":"openai/gpt-4o-mini","name":"OpenAI: GPT-4o-mini","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.15,"output":0.6,"cacheRead":0.075,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-4o-mini-2024-07-18":{"id":"openai/gpt-4o-mini-2024-07-18","name":"OpenAI: GPT-4o-mini (2024-07-18)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.15,"output":0.6,"cacheRead":0.075,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-5":{"id":"openai/gpt-5","name":"OpenAI: GPT-5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5-codex":{"id":"openai/gpt-5-codex","name":"OpenAI: GPT-5 Codex","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5-mini":{"id":"openai/gpt-5-mini","name":"OpenAI: GPT-5 Mini","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.25,"output":2,"cacheRead":0.024999999999999998,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5-nano":{"id":"openai/gpt-5-nano","name":"OpenAI: GPT-5 Nano","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.049999999999999996,"output":0.39999999999999997,"cacheRead":0.01,"cacheWrite":0},"contextWindow":400000,"maxTokens":16384},"openai/gpt-5-pro":{"id":"openai/gpt-5-pro","name":"OpenAI: GPT-5 Pro","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":15,"output":120,"cacheRead":0,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.1":{"id":"openai/gpt-5.1","name":"OpenAI: GPT-5.1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.13,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.1-chat":{"id":"openai/gpt-5.1-chat","name":"OpenAI: GPT-5.1 Chat","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-5.1-codex":{"id":"openai/gpt-5.1-codex","name":"OpenAI: GPT-5.1-Codex","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.1-codex-max":{"id":"openai/gpt-5.1-codex-max","name":"OpenAI: GPT-5.1-Codex-Max","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.25,"output":10,"cacheRead":0.125,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.1-codex-mini":{"id":"openai/gpt-5.1-codex-mini","name":"OpenAI: GPT-5.1-Codex-Mini","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.25,"output":2,"cacheRead":0.03,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.2":{"id":"openai/gpt-5.2","name":"OpenAI: GPT-5.2","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.2-chat":{"id":"openai/gpt-5.2-chat","name":"OpenAI: GPT-5.2 Chat","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":128000,"maxTokens":32000},"openai/gpt-5.2-codex":{"id":"openai/gpt-5.2-codex","name":"OpenAI: GPT-5.2-Codex","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.2-pro":{"id":"openai/gpt-5.2-pro","name":"OpenAI: GPT-5.2 Pro","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":21,"output":168,"cacheRead":0,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.3-chat":{"id":"openai/gpt-5.3-chat","name":"OpenAI: GPT-5.3 Chat","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-5.3-codex":{"id":"openai/gpt-5.3-codex","name":"OpenAI: GPT-5.3-Codex","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":1.75,"output":14,"cacheRead":0.175,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.4":{"id":"openai/gpt-5.4","name":"OpenAI: GPT-5.4","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":2.5,"output":15,"cacheRead":0.25,"cacheWrite":0},"contextWindow":1050000,"maxTokens":128000},"openai/gpt-5.4-mini":{"id":"openai/gpt-5.4-mini","name":"OpenAI: GPT-5.4 Mini","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":0.75,"output":4.5,"cacheRead":0.075,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.4-nano":{"id":"openai/gpt-5.4-nano","name":"OpenAI: GPT-5.4 Nano","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":0.19999999999999998,"output":1.25,"cacheRead":0.02,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-5.4-pro":{"id":"openai/gpt-5.4-pro","name":"OpenAI: GPT-5.4 Pro","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":30,"output":180,"cacheRead":0,"cacheWrite":0},"contextWindow":1050000,"maxTokens":128000},"openai/gpt-5.5":{"id":"openai/gpt-5.5","name":"OpenAI: GPT-5.5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":5,"output":30,"cacheRead":0.5,"cacheWrite":0},"contextWindow":1050000,"maxTokens":128000},"openai/gpt-5.5-pro":{"id":"openai/gpt-5.5-pro","name":"OpenAI: GPT-5.5 Pro","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"thinkingLevelMap":{"xhigh":"xhigh"},"input":["text","image"],"cost":{"input":30,"output":180,"cacheRead":0,"cacheWrite":0},"contextWindow":1050000,"maxTokens":128000},"openai/gpt-audio":{"id":"openai/gpt-audio","name":"OpenAI: GPT Audio","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":2.5,"output":10,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-audio-mini":{"id":"openai/gpt-audio-mini","name":"OpenAI: GPT Audio Mini","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.6,"output":2.4,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"openai/gpt-chat-latest":{"id":"openai/gpt-chat-latest","name":"OpenAI: GPT Chat Latest","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":5,"output":30,"cacheRead":0.5,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000},"openai/gpt-oss-120b":{"id":"openai/gpt-oss-120b","name":"OpenAI: gpt-oss-120b","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.039,"output":0.18,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"openai/gpt-oss-120b:free":{"id":"openai/gpt-oss-120b:free","name":"OpenAI: gpt-oss-120b (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":131072},"openai/gpt-oss-20b":{"id":"openai/gpt-oss-20b","name":"OpenAI: gpt-oss-20b","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.03,"output":0.14,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":131072},"openai/gpt-oss-20b:free":{"id":"openai/gpt-oss-20b:free","name":"OpenAI: gpt-oss-20b (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":8192},"openai/gpt-oss-safeguard-20b":{"id":"openai/gpt-oss-safeguard-20b","name":"OpenAI: gpt-oss-safeguard-20b","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.075,"output":0.3,"cacheRead":0.037,"cacheWrite":0},"contextWindow":131072,"maxTokens":65536},"openai/o1":{"id":"openai/o1","name":"OpenAI: o1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":15,"output":60,"cacheRead":7.5,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"openai/o3":{"id":"openai/o3","name":"OpenAI: o3","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":2,"output":8,"cacheRead":0.5,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"openai/o3-deep-research":{"id":"openai/o3-deep-research","name":"OpenAI: o3 Deep Research","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":10,"output":40,"cacheRead":2.5,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"openai/o3-mini":{"id":"openai/o3-mini","name":"OpenAI: o3 Mini","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":1.1,"output":4.4,"cacheRead":0.55,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"openai/o3-mini-high":{"id":"openai/o3-mini-high","name":"OpenAI: o3 Mini High","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":1.1,"output":4.4,"cacheRead":0.55,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"openai/o3-pro":{"id":"openai/o3-pro","name":"OpenAI: o3 Pro","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":20,"output":80,"cacheRead":0,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"openai/o4-mini":{"id":"openai/o4-mini","name":"OpenAI: o4 Mini","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.1,"output":4.4,"cacheRead":0.275,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"openai/o4-mini-deep-research":{"id":"openai/o4-mini-deep-research","name":"OpenAI: o4 Mini Deep Research","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":2,"output":8,"cacheRead":0.5,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"openai/o4-mini-high":{"id":"openai/o4-mini-high","name":"OpenAI: o4 Mini High","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.1,"output":4.4,"cacheRead":0.275,"cacheWrite":0},"contextWindow":200000,"maxTokens":100000},"openrouter/auto":{"id":"openrouter/auto","name":"Auto Router","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":-1000000,"output":-1000000,"cacheRead":0,"cacheWrite":0},"contextWindow":2000000,"maxTokens":16384},"openrouter/free":{"id":"openrouter/free","name":"Free Models Router","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":200000,"maxTokens":16384},"openrouter/owl-alpha":{"id":"openrouter/owl-alpha","name":"Owl Alpha","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":1048756,"maxTokens":262144},"poolside/laguna-m.1:free":{"id":"poolside/laguna-m.1:free","name":"Poolside: Laguna M.1 (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":8192},"poolside/laguna-xs.2:free":{"id":"poolside/laguna-xs.2:free","name":"Poolside: Laguna XS.2 (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":8192},"prime-intellect/intellect-3":{"id":"prime-intellect/intellect-3","name":"Prime Intellect: INTELLECT-3","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.19999999999999998,"output":1.1,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":131072},"qwen/qwen-2.5-72b-instruct":{"id":"qwen/qwen-2.5-72b-instruct","name":"Qwen2.5 72B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.36,"output":0.39999999999999997,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"qwen/qwen-2.5-7b-instruct":{"id":"qwen/qwen-2.5-7b-instruct","name":"Qwen: Qwen2.5 7B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.04,"output":0.09999999999999999,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":32768},"qwen/qwen-plus":{"id":"qwen/qwen-plus","name":"Qwen: Qwen-Plus","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.26,"output":0.78,"cacheRead":0.052000000000000005,"cacheWrite":0.325},"contextWindow":1000000,"maxTokens":32768},"qwen/qwen-plus-2025-07-28":{"id":"qwen/qwen-plus-2025-07-28","name":"Qwen: Qwen Plus 0728","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.26,"output":0.78,"cacheRead":0,"cacheWrite":0.325},"contextWindow":1000000,"maxTokens":32768},"qwen/qwen-plus-2025-07-28:thinking":{"id":"qwen/qwen-plus-2025-07-28:thinking","name":"Qwen: Qwen Plus 0728 (thinking)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.26,"output":0.78,"cacheRead":0,"cacheWrite":0.325},"contextWindow":1000000,"maxTokens":32768},"qwen/qwen3-14b":{"id":"qwen/qwen3-14b","name":"Qwen: Qwen3 14B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.09999999999999999,"output":0.24,"cacheRead":0,"cacheWrite":0},"contextWindow":131702,"maxTokens":40960},"qwen/qwen3-235b-a22b":{"id":"qwen/qwen3-235b-a22b","name":"Qwen: Qwen3 235B A22B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.45499999999999996,"output":1.8199999999999998,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":8192},"qwen/qwen3-235b-a22b-2507":{"id":"qwen/qwen3-235b-a22b-2507","name":"Qwen: Qwen3 235B A22B Instruct 2507","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.071,"output":0.09999999999999999,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"qwen/qwen3-235b-a22b-thinking-2507":{"id":"qwen/qwen3-235b-a22b-thinking-2507","name":"Qwen: Qwen3 235B A22B Thinking 2507","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.14950000000000002,"output":1.495,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"qwen/qwen3-30b-a3b":{"id":"qwen/qwen3-30b-a3b","name":"Qwen: Qwen3 30B A3B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.09,"output":0.44999999999999996,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":20000},"qwen/qwen3-30b-a3b-instruct-2507":{"id":"qwen/qwen3-30b-a3b-instruct-2507","name":"Qwen: Qwen3 30B A3B Instruct 2507","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.09,"output":0.3,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":262144},"qwen/qwen3-30b-a3b-thinking-2507":{"id":"qwen/qwen3-30b-a3b-thinking-2507","name":"Qwen: Qwen3 30B A3B Thinking 2507","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.08,"output":0.39999999999999997,"cacheRead":0.08,"cacheWrite":0},"contextWindow":131072,"maxTokens":131072},"qwen/qwen3-32b":{"id":"qwen/qwen3-32b","name":"Qwen: Qwen3 32B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.08,"output":0.28,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"qwen/qwen3-8b":{"id":"qwen/qwen3-8b","name":"Qwen: Qwen3 8B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.049999999999999996,"output":0.39999999999999997,"cacheRead":0.049999999999999996,"cacheWrite":0},"contextWindow":131072,"maxTokens":8192},"qwen/qwen3-coder":{"id":"qwen/qwen3-coder","name":"Qwen: Qwen3 Coder 480B A35B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.22,"output":1.7999999999999998,"cacheRead":0,"cacheWrite":0},"contextWindow":1048576,"maxTokens":65536},"qwen/qwen3-coder-30b-a3b-instruct":{"id":"qwen/qwen3-coder-30b-a3b-instruct","name":"Qwen: Qwen3 Coder 30B A3B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.07,"output":0.27,"cacheRead":0,"cacheWrite":0},"contextWindow":160000,"maxTokens":32768},"qwen/qwen3-coder-flash":{"id":"qwen/qwen3-coder-flash","name":"Qwen: Qwen3 Coder Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.195,"output":0.975,"cacheRead":0.039,"cacheWrite":0.24375},"contextWindow":1000000,"maxTokens":65536},"qwen/qwen3-coder-next":{"id":"qwen/qwen3-coder-next","name":"Qwen: Qwen3 Coder Next","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.11,"output":0.7999999999999999,"cacheRead":0.07,"cacheWrite":0},"contextWindow":262144,"maxTokens":262144},"qwen/qwen3-coder-plus":{"id":"qwen/qwen3-coder-plus","name":"Qwen: Qwen3 Coder Plus","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.65,"output":3.25,"cacheRead":0.13,"cacheWrite":0.8125},"contextWindow":1000000,"maxTokens":65536},"qwen/qwen3-coder:free":{"id":"qwen/qwen3-coder:free","name":"Qwen: Qwen3 Coder 480B A35B (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":1048576,"maxTokens":262000},"qwen/qwen3-max":{"id":"qwen/qwen3-max","name":"Qwen: Qwen3 Max","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.78,"output":3.9,"cacheRead":0.156,"cacheWrite":0.975},"contextWindow":262144,"maxTokens":32768},"qwen/qwen3-max-thinking":{"id":"qwen/qwen3-max-thinking","name":"Qwen: Qwen3 Max Thinking","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.78,"output":3.9,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":32768},"qwen/qwen3-next-80b-a3b-instruct":{"id":"qwen/qwen3-next-80b-a3b-instruct","name":"Qwen: Qwen3 Next 80B A3B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.09,"output":1.1,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"qwen/qwen3-next-80b-a3b-instruct:free":{"id":"qwen/qwen3-next-80b-a3b-instruct:free","name":"Qwen: Qwen3 Next 80B A3B Instruct (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"qwen/qwen3-next-80b-a3b-thinking":{"id":"qwen/qwen3-next-80b-a3b-thinking","name":"Qwen: Qwen3 Next 80B A3B Thinking","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.0975,"output":0.78,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":32768},"qwen/qwen3-vl-235b-a22b-instruct":{"id":"qwen/qwen3-vl-235b-a22b-instruct","name":"Qwen: Qwen3 VL 235B A22B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.19999999999999998,"output":0.88,"cacheRead":0.11,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"qwen/qwen3-vl-235b-a22b-thinking":{"id":"qwen/qwen3-vl-235b-a22b-thinking","name":"Qwen: Qwen3 VL 235B A22B Thinking","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.26,"output":2.6,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":32768},"qwen/qwen3-vl-30b-a3b-instruct":{"id":"qwen/qwen3-vl-30b-a3b-instruct","name":"Qwen: Qwen3 VL 30B A3B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.13,"output":0.52,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":32768},"qwen/qwen3-vl-30b-a3b-thinking":{"id":"qwen/qwen3-vl-30b-a3b-thinking","name":"Qwen: Qwen3 VL 30B A3B Thinking","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.13,"output":1.56,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":32768},"qwen/qwen3-vl-32b-instruct":{"id":"qwen/qwen3-vl-32b-instruct","name":"Qwen: Qwen3 VL 32B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.10400000000000001,"output":0.41600000000000004,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":32768},"qwen/qwen3-vl-8b-instruct":{"id":"qwen/qwen3-vl-8b-instruct","name":"Qwen: Qwen3 VL 8B Instruct","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.08,"output":0.5,"cacheRead":0,"cacheWrite":0},"contextWindow":256000,"maxTokens":32768},"qwen/qwen3-vl-8b-thinking":{"id":"qwen/qwen3-vl-8b-thinking","name":"Qwen: Qwen3 VL 8B Thinking","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.117,"output":1.365,"cacheRead":0,"cacheWrite":0},"contextWindow":256000,"maxTokens":32768},"qwen/qwen3.5-122b-a10b":{"id":"qwen/qwen3.5-122b-a10b","name":"Qwen: Qwen3.5-122B-A10B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.26,"output":2.08,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":262144},"qwen/qwen3.5-27b":{"id":"qwen/qwen3.5-27b","name":"Qwen: Qwen3.5-27B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.195,"output":1.56,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":65536},"qwen/qwen3.5-35b-a3b":{"id":"qwen/qwen3.5-35b-a3b","name":"Qwen: Qwen3.5-35B-A3B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.13899999999999998,"output":1,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"qwen/qwen3.5-397b-a17b":{"id":"qwen/qwen3.5-397b-a17b","name":"Qwen: Qwen3.5 397B A17B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.39,"output":2.34,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":65536},"qwen/qwen3.5-9b":{"id":"qwen/qwen3.5-9b","name":"Qwen: Qwen3.5-9B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.04,"output":0.15,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":81920},"qwen/qwen3.5-flash-02-23":{"id":"qwen/qwen3.5-flash-02-23","name":"Qwen: Qwen3.5-Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.065,"output":0.26,"cacheRead":0,"cacheWrite":0.08125},"contextWindow":1000000,"maxTokens":65536},"qwen/qwen3.5-plus-02-15":{"id":"qwen/qwen3.5-plus-02-15","name":"Qwen: Qwen3.5 Plus 2026-02-15","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.26,"output":1.56,"cacheRead":0,"cacheWrite":0.325},"contextWindow":1000000,"maxTokens":65536},"qwen/qwen3.5-plus-20260420":{"id":"qwen/qwen3.5-plus-20260420","name":"Qwen: Qwen3.5 Plus 2026-04-20","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.3,"output":1.7999999999999998,"cacheRead":0,"cacheWrite":0},"contextWindow":1000000,"maxTokens":65536},"qwen/qwen3.6-27b":{"id":"qwen/qwen3.6-27b","name":"Qwen: Qwen3.6 27B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.3,"output":3.1999999999999997,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":262144},"qwen/qwen3.6-35b-a3b":{"id":"qwen/qwen3.6-35b-a3b","name":"Qwen: Qwen3.6 35B A3B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.15,"output":1,"cacheRead":0,"cacheWrite":0},"contextWindow":262144,"maxTokens":262140},"qwen/qwen3.6-flash":{"id":"qwen/qwen3.6-flash","name":"Qwen: Qwen3.6 Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.1875,"output":1.125,"cacheRead":0,"cacheWrite":0.234375},"contextWindow":1000000,"maxTokens":65536},"qwen/qwen3.6-max-preview":{"id":"qwen/qwen3.6-max-preview","name":"Qwen: Qwen3.6 Max Preview","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":1.04,"output":6.24,"cacheRead":0,"cacheWrite":1.3},"contextWindow":262144,"maxTokens":65536},"qwen/qwen3.6-plus":{"id":"qwen/qwen3.6-plus","name":"Qwen: Qwen3.6 Plus","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.325,"output":1.95,"cacheRead":0,"cacheWrite":0.40625},"contextWindow":1000000,"maxTokens":65536},"qwen/qwen3.7-max":{"id":"qwen/qwen3.7-max","name":"Qwen: Qwen3.7 Max","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":2.5,"output":7.5,"cacheRead":0,"cacheWrite":3.125},"contextWindow":1000000,"maxTokens":65536},"rekaai/reka-edge":{"id":"rekaai/reka-edge","name":"Reka Edge","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text","image"],"cost":{"input":0.09999999999999999,"output":0.09999999999999999,"cacheRead":0,"cacheWrite":0},"contextWindow":16384,"maxTokens":16384},"relace/relace-search":{"id":"relace/relace-search","name":"Relace: Relace Search","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":1,"output":3,"cacheRead":0,"cacheWrite":0},"contextWindow":256000,"maxTokens":128000},"sao10k/l3-euryale-70b":{"id":"sao10k/l3-euryale-70b","name":"Sao10k: Llama 3 Euryale 70B v2.1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":1.48,"output":1.48,"cacheRead":0,"cacheWrite":0},"contextWindow":8192,"maxTokens":8192},"sao10k/l3.1-euryale-70b":{"id":"sao10k/l3.1-euryale-70b","name":"Sao10K: Llama 3.1 Euryale 70B v2.2","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.85,"output":0.85,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":16384},"stepfun/step-3.5-flash":{"id":"stepfun/step-3.5-flash","name":"StepFun: Step 3.5 Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.09,"output":0.3,"cacheRead":0.02,"cacheWrite":0},"contextWindow":262144,"maxTokens":16384},"tencent/hy3-preview":{"id":"tencent/hy3-preview","name":"Tencent: Hy3 preview","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.06599999999999999,"output":0.26,"cacheRead":0.029,"cacheWrite":0},"contextWindow":262144,"maxTokens":262144},"thedrummer/rocinante-12b":{"id":"thedrummer/rocinante-12b","name":"TheDrummer: Rocinante 12B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.16999999999999998,"output":0.43,"cacheRead":0,"cacheWrite":0},"contextWindow":32768,"maxTokens":32768},"thedrummer/unslopnemo-12b":{"id":"thedrummer/unslopnemo-12b","name":"TheDrummer: UnslopNemo 12B","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.39999999999999997,"output":0.39999999999999997,"cacheRead":0,"cacheWrite":0},"contextWindow":32768,"maxTokens":32768},"upstage/solar-pro-3":{"id":"upstage/solar-pro-3","name":"Upstage: Solar Pro 3","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.15,"output":0.6,"cacheRead":0.015,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"x-ai/grok-4.20":{"id":"x-ai/grok-4.20","name":"xAI: Grok 4.20","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.25,"output":2.5,"cacheRead":0.19999999999999998,"cacheWrite":0},"contextWindow":2000000,"maxTokens":16384},"x-ai/grok-4.3":{"id":"x-ai/grok-4.3","name":"xAI: Grok 4.3","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.25,"output":2.5,"cacheRead":0.19999999999999998,"cacheWrite":0},"contextWindow":1000000,"maxTokens":16384},"x-ai/grok-build-0.1":{"id":"x-ai/grok-build-0.1","name":"xAI: Grok Build 0.1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1,"output":2,"cacheRead":0.19999999999999998,"cacheWrite":0},"contextWindow":256000,"maxTokens":16384},"xiaomi/mimo-v2-flash":{"id":"xiaomi/mimo-v2-flash","name":"Xiaomi: MiMo-V2-Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.09999999999999999,"output":0.3,"cacheRead":0.01,"cacheWrite":0},"contextWindow":262144,"maxTokens":65536},"xiaomi/mimo-v2-omni":{"id":"xiaomi/mimo-v2-omni","name":"Xiaomi: MiMo-V2-Omni","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.39999999999999997,"output":2,"cacheRead":0.08,"cacheWrite":0},"contextWindow":262144,"maxTokens":65536},"xiaomi/mimo-v2-pro":{"id":"xiaomi/mimo-v2-pro","name":"Xiaomi: MiMo-V2-Pro","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":1,"output":3,"cacheRead":0.19999999999999998,"cacheWrite":0},"contextWindow":1048576,"maxTokens":131072},"xiaomi/mimo-v2.5":{"id":"xiaomi/mimo-v2.5","name":"Xiaomi: MiMo-V2.5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.39999999999999997,"output":2,"cacheRead":0.08,"cacheWrite":0},"contextWindow":1048576,"maxTokens":131072},"xiaomi/mimo-v2.5-pro":{"id":"xiaomi/mimo-v2.5-pro","name":"Xiaomi: MiMo-V2.5-Pro","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":1,"output":3,"cacheRead":0.19999999999999998,"cacheWrite":0},"contextWindow":1048576,"maxTokens":16384},"z-ai/glm-4-32b":{"id":"z-ai/glm-4-32b","name":"Z.ai: GLM 4 32B ","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":false,"input":["text"],"cost":{"input":0.09999999999999999,"output":0.09999999999999999,"cacheRead":0,"cacheWrite":0},"contextWindow":128000,"maxTokens":16384},"z-ai/glm-4.5":{"id":"z-ai/glm-4.5","name":"Z.ai: GLM 4.5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.6,"output":2.2,"cacheRead":0.11,"cacheWrite":0},"contextWindow":131072,"maxTokens":98304},"z-ai/glm-4.5-air":{"id":"z-ai/glm-4.5-air","name":"Z.ai: GLM 4.5 Air","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.13,"output":0.85,"cacheRead":0.024999999999999998,"cacheWrite":0},"contextWindow":131072,"maxTokens":98304},"z-ai/glm-4.5-air:free":{"id":"z-ai/glm-4.5-air:free","name":"Z.ai: GLM 4.5 Air (free)","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0},"contextWindow":131072,"maxTokens":96000},"z-ai/glm-4.5v":{"id":"z-ai/glm-4.5v","name":"Z.ai: GLM 4.5V","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.6,"output":1.7999999999999998,"cacheRead":0.11,"cacheWrite":0},"contextWindow":65536,"maxTokens":16384},"z-ai/glm-4.6":{"id":"z-ai/glm-4.6","name":"Z.ai: GLM 4.6","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.43,"output":1.74,"cacheRead":0.08,"cacheWrite":0},"contextWindow":202752,"maxTokens":131072},"z-ai/glm-4.6v":{"id":"z-ai/glm-4.6v","name":"Z.ai: GLM 4.6V","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.3,"output":0.8999999999999999,"cacheRead":0.049999999999999996,"cacheWrite":0},"contextWindow":131072,"maxTokens":24000},"z-ai/glm-4.7":{"id":"z-ai/glm-4.7","name":"Z.ai: GLM 4.7","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.39999999999999997,"output":1.75,"cacheRead":0.08,"cacheWrite":0},"contextWindow":202752,"maxTokens":131072},"z-ai/glm-4.7-flash":{"id":"z-ai/glm-4.7-flash","name":"Z.ai: GLM 4.7 Flash","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.06,"output":0.39999999999999997,"cacheRead":0.01,"cacheWrite":0},"contextWindow":202752,"maxTokens":16384},"z-ai/glm-5":{"id":"z-ai/glm-5","name":"Z.ai: GLM 5","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.6,"output":1.92,"cacheRead":0.12,"cacheWrite":0},"contextWindow":202752,"maxTokens":16384},"z-ai/glm-5-turbo":{"id":"z-ai/glm-5-turbo","name":"Z.ai: GLM 5 Turbo","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":1.2,"output":4,"cacheRead":0.24,"cacheWrite":0},"contextWindow":202752,"maxTokens":131072},"z-ai/glm-5.1":{"id":"z-ai/glm-5.1","name":"Z.ai: GLM 5.1","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text"],"cost":{"input":0.98,"output":3.08,"cacheRead":0.182,"cacheWrite":0},"contextWindow":202752,"maxTokens":16384},"z-ai/glm-5v-turbo":{"id":"z-ai/glm-5v-turbo","name":"Z.ai: GLM 5V Turbo","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.2,"output":4,"cacheRead":0.24,"cacheWrite":0},"contextWindow":202752,"maxTokens":131072},"~anthropic/claude-haiku-latest":{"id":"~anthropic/claude-haiku-latest","name":"Anthropic Claude Haiku Latest","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1,"output":5,"cacheRead":0.09999999999999999,"cacheWrite":1.25},"contextWindow":200000,"maxTokens":64000},"~anthropic/claude-opus-latest":{"id":"~anthropic/claude-opus-latest","name":"Anthropic: Claude Opus Latest","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":5,"output":25,"cacheRead":0.5,"cacheWrite":6.25},"contextWindow":1000000,"maxTokens":128000},"~anthropic/claude-sonnet-latest":{"id":"~anthropic/claude-sonnet-latest","name":"Anthropic Claude Sonnet Latest","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":3,"output":15,"cacheRead":0.3,"cacheWrite":3.75},"contextWindow":1000000,"maxTokens":128000},"~google/gemini-flash-latest":{"id":"~google/gemini-flash-latest","name":"Google Gemini Flash Latest","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":1.5,"output":9,"cacheRead":0.15,"cacheWrite":0.08333333333333334},"contextWindow":1048576,"maxTokens":65536},"~google/gemini-pro-latest":{"id":"~google/gemini-pro-latest","name":"Google Gemini Pro Latest","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":2,"output":12,"cacheRead":0.19999999999999998,"cacheWrite":0.375},"contextWindow":1048576,"maxTokens":65536},"~moonshotai/kimi-latest":{"id":"~moonshotai/kimi-latest","name":"MoonshotAI Kimi Latest","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.73,"output":3.49,"cacheRead":0.25,"cacheWrite":0},"contextWindow":262144,"maxTokens":262142},"~openai/gpt-latest":{"id":"~openai/gpt-latest","name":"OpenAI GPT Latest","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":5,"output":30,"cacheRead":0.5,"cacheWrite":0},"contextWindow":1050000,"maxTokens":128000},"~openai/gpt-mini-latest":{"id":"~openai/gpt-mini-latest","name":"OpenAI GPT Mini Latest","api":"openai-completions","provider":"openrouter","baseUrl":"https://openrouter.ai/api/v1","reasoning":true,"input":["text","image"],"cost":{"input":0.75,"output":4.5,"cacheRead":0.075,"cacheWrite":0},"contextWindow":400000,"maxTokens":128000}}}` - -var modelProviderOrder = []Provider{ - "openai", - "openrouter", -} - -var modelIDOrderJSON = `{"openai":["gpt-4","gpt-4-turbo","gpt-4.1","gpt-4.1-mini","gpt-4.1-nano","gpt-4o","gpt-4o-2024-05-13","gpt-4o-2024-08-06","gpt-4o-2024-11-20","gpt-4o-mini","gpt-5","gpt-5-codex","gpt-5-mini","gpt-5-nano","gpt-5-pro","gpt-5.1","gpt-5.1-chat-latest","gpt-5.1-codex","gpt-5.1-codex-max","gpt-5.1-codex-mini","gpt-5.2","gpt-5.2-chat-latest","gpt-5.2-codex","gpt-5.2-pro","gpt-5.3-chat-latest","gpt-5.3-codex","gpt-5.3-codex-spark","gpt-5.4","gpt-5.4-mini","gpt-5.4-nano","gpt-5.4-pro","gpt-5.5","gpt-5.5-pro","o1","o1-pro","o3","o3-deep-research","o3-mini","o3-pro","o4-mini","o4-mini-deep-research"],"openrouter":["ai21/jamba-large-1.7","alibaba/tongyi-deepresearch-30b-a3b","amazon/nova-2-lite-v1","amazon/nova-lite-v1","amazon/nova-micro-v1","amazon/nova-premier-v1","amazon/nova-pro-v1","anthropic/claude-3-haiku","anthropic/claude-3.5-haiku","anthropic/claude-haiku-4.5","anthropic/claude-opus-4","anthropic/claude-opus-4.1","anthropic/claude-opus-4.5","anthropic/claude-opus-4.6","anthropic/claude-opus-4.6-fast","anthropic/claude-opus-4.7","anthropic/claude-opus-4.7-fast","anthropic/claude-sonnet-4","anthropic/claude-sonnet-4.5","anthropic/claude-sonnet-4.6","arcee-ai/trinity-large-thinking","arcee-ai/trinity-large-thinking:free","arcee-ai/trinity-mini","arcee-ai/virtuoso-large","baidu/cobuddy:free","baidu/ernie-4.5-21b-a3b","baidu/ernie-4.5-vl-28b-a3b","bytedance-seed/seed-1.6","bytedance-seed/seed-1.6-flash","bytedance-seed/seed-2.0-lite","bytedance-seed/seed-2.0-mini","cohere/command-r-08-2024","cohere/command-r-plus-08-2024","deepseek/deepseek-chat","deepseek/deepseek-chat-v3-0324","deepseek/deepseek-chat-v3.1","deepseek/deepseek-r1","deepseek/deepseek-r1-0528","deepseek/deepseek-v3.1-terminus","deepseek/deepseek-v3.2","deepseek/deepseek-v3.2-exp","deepseek/deepseek-v4-flash","deepseek/deepseek-v4-flash:free","deepseek/deepseek-v4-pro","essentialai/rnj-1-instruct","google/gemini-2.0-flash-001","google/gemini-2.0-flash-lite-001","google/gemini-2.5-flash","google/gemini-2.5-flash-lite","google/gemini-2.5-flash-lite-preview-09-2025","google/gemini-2.5-pro","google/gemini-2.5-pro-preview","google/gemini-2.5-pro-preview-05-06","google/gemini-3-flash-preview","google/gemini-3.1-flash-lite","google/gemini-3.1-flash-lite-preview","google/gemini-3.1-pro-preview","google/gemini-3.1-pro-preview-customtools","google/gemini-3.5-flash","google/gemma-3-12b-it","google/gemma-3-27b-it","google/gemma-4-26b-a4b-it","google/gemma-4-26b-a4b-it:free","google/gemma-4-31b-it","google/gemma-4-31b-it:free","ibm-granite/granite-4.1-8b","inception/mercury-2","inclusionai/ling-2.6-1t","inclusionai/ling-2.6-flash","inclusionai/ring-2.6-1t","kwaipilot/kat-coder-pro-v2","meta-llama/llama-3.1-70b-instruct","meta-llama/llama-3.1-8b-instruct","meta-llama/llama-3.3-70b-instruct","meta-llama/llama-3.3-70b-instruct:free","meta-llama/llama-4-scout","minimax/minimax-m1","minimax/minimax-m2","minimax/minimax-m2.1","minimax/minimax-m2.5","minimax/minimax-m2.5:free","minimax/minimax-m2.7","mistralai/codestral-2508","mistralai/devstral-2512","mistralai/devstral-medium","mistralai/devstral-small","mistralai/ministral-14b-2512","mistralai/ministral-3b-2512","mistralai/ministral-8b-2512","mistralai/mistral-large","mistralai/mistral-large-2407","mistralai/mistral-large-2411","mistralai/mistral-large-2512","mistralai/mistral-medium-3","mistralai/mistral-medium-3-5","mistralai/mistral-medium-3.1","mistralai/mistral-nemo","mistralai/mistral-saba","mistralai/mistral-small-2603","mistralai/mistral-small-3.2-24b-instruct","mistralai/mixtral-8x22b-instruct","mistralai/pixtral-large-2411","mistralai/voxtral-small-24b-2507","moonshotai/kimi-k2","moonshotai/kimi-k2-0905","moonshotai/kimi-k2-thinking","moonshotai/kimi-k2.5","moonshotai/kimi-k2.6","nex-agi/deepseek-v3.1-nex-n1","nvidia/llama-3.3-nemotron-super-49b-v1.5","nvidia/nemotron-3-nano-30b-a3b","nvidia/nemotron-3-nano-30b-a3b:free","nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free","nvidia/nemotron-3-super-120b-a12b","nvidia/nemotron-3-super-120b-a12b:free","nvidia/nemotron-nano-12b-v2-vl:free","nvidia/nemotron-nano-9b-v2","nvidia/nemotron-nano-9b-v2:free","openai/gpt-3.5-turbo","openai/gpt-3.5-turbo-0613","openai/gpt-3.5-turbo-16k","openai/gpt-4","openai/gpt-4-0314","openai/gpt-4-1106-preview","openai/gpt-4-turbo","openai/gpt-4-turbo-preview","openai/gpt-4.1","openai/gpt-4.1-mini","openai/gpt-4.1-nano","openai/gpt-4o","openai/gpt-4o-2024-05-13","openai/gpt-4o-2024-08-06","openai/gpt-4o-2024-11-20","openai/gpt-4o-audio-preview","openai/gpt-4o-mini","openai/gpt-4o-mini-2024-07-18","openai/gpt-5","openai/gpt-5-codex","openai/gpt-5-mini","openai/gpt-5-nano","openai/gpt-5-pro","openai/gpt-5.1","openai/gpt-5.1-chat","openai/gpt-5.1-codex","openai/gpt-5.1-codex-max","openai/gpt-5.1-codex-mini","openai/gpt-5.2","openai/gpt-5.2-chat","openai/gpt-5.2-codex","openai/gpt-5.2-pro","openai/gpt-5.3-chat","openai/gpt-5.3-codex","openai/gpt-5.4","openai/gpt-5.4-mini","openai/gpt-5.4-nano","openai/gpt-5.4-pro","openai/gpt-5.5","openai/gpt-5.5-pro","openai/gpt-audio","openai/gpt-audio-mini","openai/gpt-chat-latest","openai/gpt-oss-120b","openai/gpt-oss-120b:free","openai/gpt-oss-20b","openai/gpt-oss-20b:free","openai/gpt-oss-safeguard-20b","openai/o1","openai/o3","openai/o3-deep-research","openai/o3-mini","openai/o3-mini-high","openai/o3-pro","openai/o4-mini","openai/o4-mini-deep-research","openai/o4-mini-high","openrouter/auto","openrouter/free","openrouter/owl-alpha","poolside/laguna-m.1:free","poolside/laguna-xs.2:free","prime-intellect/intellect-3","qwen/qwen-2.5-72b-instruct","qwen/qwen-2.5-7b-instruct","qwen/qwen-plus","qwen/qwen-plus-2025-07-28","qwen/qwen-plus-2025-07-28:thinking","qwen/qwen3-14b","qwen/qwen3-235b-a22b","qwen/qwen3-235b-a22b-2507","qwen/qwen3-235b-a22b-thinking-2507","qwen/qwen3-30b-a3b","qwen/qwen3-30b-a3b-instruct-2507","qwen/qwen3-30b-a3b-thinking-2507","qwen/qwen3-32b","qwen/qwen3-8b","qwen/qwen3-coder","qwen/qwen3-coder-30b-a3b-instruct","qwen/qwen3-coder-flash","qwen/qwen3-coder-next","qwen/qwen3-coder-plus","qwen/qwen3-coder:free","qwen/qwen3-max","qwen/qwen3-max-thinking","qwen/qwen3-next-80b-a3b-instruct","qwen/qwen3-next-80b-a3b-instruct:free","qwen/qwen3-next-80b-a3b-thinking","qwen/qwen3-vl-235b-a22b-instruct","qwen/qwen3-vl-235b-a22b-thinking","qwen/qwen3-vl-30b-a3b-instruct","qwen/qwen3-vl-30b-a3b-thinking","qwen/qwen3-vl-32b-instruct","qwen/qwen3-vl-8b-instruct","qwen/qwen3-vl-8b-thinking","qwen/qwen3.5-122b-a10b","qwen/qwen3.5-27b","qwen/qwen3.5-35b-a3b","qwen/qwen3.5-397b-a17b","qwen/qwen3.5-9b","qwen/qwen3.5-flash-02-23","qwen/qwen3.5-plus-02-15","qwen/qwen3.5-plus-20260420","qwen/qwen3.6-27b","qwen/qwen3.6-35b-a3b","qwen/qwen3.6-flash","qwen/qwen3.6-max-preview","qwen/qwen3.6-plus","qwen/qwen3.7-max","rekaai/reka-edge","relace/relace-search","sao10k/l3-euryale-70b","sao10k/l3.1-euryale-70b","stepfun/step-3.5-flash","tencent/hy3-preview","thedrummer/rocinante-12b","thedrummer/unslopnemo-12b","upstage/solar-pro-3","x-ai/grok-4.20","x-ai/grok-4.3","x-ai/grok-build-0.1","xiaomi/mimo-v2-flash","xiaomi/mimo-v2-omni","xiaomi/mimo-v2-pro","xiaomi/mimo-v2.5","xiaomi/mimo-v2.5-pro","z-ai/glm-4-32b","z-ai/glm-4.5","z-ai/glm-4.5-air","z-ai/glm-4.5-air:free","z-ai/glm-4.5v","z-ai/glm-4.6","z-ai/glm-4.6v","z-ai/glm-4.7","z-ai/glm-4.7-flash","z-ai/glm-5","z-ai/glm-5-turbo","z-ai/glm-5.1","z-ai/glm-5v-turbo","~anthropic/claude-haiku-latest","~anthropic/claude-opus-latest","~anthropic/claude-sonnet-latest","~google/gemini-flash-latest","~google/gemini-pro-latest","~moonshotai/kimi-latest","~openai/gpt-latest","~openai/gpt-mini-latest"]}` - -var Models = mustLoadModels() -var modelIDOrder = mustLoadModelIDOrder() - -func mustLoadModels() map[Provider]map[string]Model { - var raw map[Provider]map[string]Model - if err := json.Unmarshal([]byte(modelsJSON), &raw); err != nil { - panic(err) - } - return raw -} - -func mustLoadModelIDOrder() map[Provider][]string { - var raw map[Provider][]string - if err := json.Unmarshal([]byte(modelIDOrderJSON), &raw); err != nil { - panic(err) - } - return raw -} diff --git a/pkg/ai/models_test.go b/pkg/ai/models_test.go index 3d74ee08..ab7dc76c 100644 --- a/pkg/ai/models_test.go +++ b/pkg/ai/models_test.go @@ -2,24 +2,6 @@ package ai import "testing" -func TestModelRegistryGeneratedSurface(t *testing.T) { - providers := GetProviders() - if len(providers) != 2 || providers[0] != ProviderOpenAI || providers[1] != ProviderOpenRouter { - t.Fatalf("expected generated provider order, got %#v", providers) - } - models := GetModels(ProviderOpenAI) - if len(models) == 0 { - t.Fatalf("expected generated OpenAI models") - } - model, ok := GetModel(ProviderOpenAI, models[0].ID) - if !ok || model.ID != models[0].ID || model.Provider != ProviderOpenAI { - t.Fatalf("expected model lookup to round trip, got %#v ok=%v", model, ok) - } - if _, ok := GetModel("missing", "missing"); ok { - t.Fatalf("expected missing model lookup to fail") - } -} - func TestGetSupportedThinkingLevels(t *testing.T) { if got := GetSupportedThinkingLevels(Model{}); len(got) != 1 || got[0] != ModelThinkingLevelOff { t.Fatalf("expected non-reasoning model to support off only, got %#v", got) diff --git a/pkg/ai/providers/anthropic.go b/pkg/ai/providers/anthropic.go index 4b46ce5f..dea1eed5 100644 --- a/pkg/ai/providers/anthropic.go +++ b/pkg/ai/providers/anthropic.go @@ -288,7 +288,11 @@ func ConvertAnthropicMessages(model ai.Model, llmContext ai.Context, isOAuth boo if isOAuth { name = toClaudeCodeName(name) } - blocks = append(blocks, map[string]any{"type": "tool_use", "id": block.ID, "name": name, "input": block.Arguments}) + input := block.Arguments + if input == nil { + input = map[string]any{} + } + blocks = append(blocks, map[string]any{"type": "tool_use", "id": block.ID, "name": name, "input": input}) } } if len(blocks) > 0 { diff --git a/pkg/ai/providers/anthropic_test.go b/pkg/ai/providers/anthropic_test.go index b192e81f..94e2d8a7 100644 --- a/pkg/ai/providers/anthropic_test.go +++ b/pkg/ai/providers/anthropic_test.go @@ -78,6 +78,35 @@ func TestAnthropicStreamStatePreservesToolInputFromContentBlockStart(t *testing. } } +func TestConvertAnthropicMessagesReplaysEmptyToolInputAsObject(t *testing.T) { + model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic} + messages := ConvertAnthropicMessages(model, ai.Context{Messages: []ai.Message{ + { + Role: "assistant", + API: model.API, + Provider: model.Provider, + Model: model.ID, + Content: []ai.ContentBlock{ + {Type: "toolCall", ID: "toolu_1", Name: "get_session"}, + }, + }, + { + Role: "toolResult", + ToolCallID: "toolu_1", + Content: []ai.ContentBlock{{Type: "text", Text: "ok"}}, + }, + }}, false, nil) + + blocks, ok := messages[0]["content"].([]map[string]any) + if !ok || len(blocks) != 1 { + t.Fatalf("expected assistant tool_use block, got %#v", messages) + } + input, ok := blocks[0]["input"].(map[string]any) + if !ok || input == nil || len(input) != 0 { + t.Fatalf("expected empty tool input object, got %#v", blocks[0]["input"]) + } +} + func TestAnthropicStreamStateStreamsRedactedThinkingImmediately(t *testing.T) { stream := ai.NewAssistantMessageEventStream() model := ai.Model{ID: "claude-test", API: ai.ApiAnthropicMessages, Provider: ai.ProviderAnthropic} diff --git a/pkg/ai/providers/images/register_builtins_test.go b/pkg/ai/providers/images/register_builtins_test.go deleted file mode 100644 index 9e69ac11..00000000 --- a/pkg/ai/providers/images/register_builtins_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package images - -import ( - "testing" - - ai "github.com/beeper/ai-bridge/pkg/ai" -) - -func TestImageModelRegistryOnlyExposesRegisteredAPIs(t *testing.T) { - RegisterBuiltInImagesAPIProviders() - for _, provider := range ai.GetImageProviders() { - for _, model := range ai.GetImageModels(provider) { - if _, ok := ai.GetImagesAPIProvider(model.API); !ok { - t.Fatalf("image model %s/%s uses unregistered api %s", provider, model.ID, model.API) - } - } - } -} diff --git a/pkg/ai/providers/openai_conversion_test.go b/pkg/ai/providers/openai_conversion_test.go index b247245b..9de35844 100644 --- a/pkg/ai/providers/openai_conversion_test.go +++ b/pkg/ai/providers/openai_conversion_test.go @@ -529,17 +529,6 @@ func TestRegisterBuiltInsIncludesOpenAICodexResponses(t *testing.T) { } } -func TestModelRegistryOnlyExposesRegisteredTextAPIs(t *testing.T) { - ResetAPIProviders() - for _, provider := range ai.GetProviders() { - for _, model := range ai.GetModels(provider) { - if _, ok := ai.GetAPIProvider(model.API); !ok { - t.Fatalf("model %s/%s uses unregistered api %s", provider, model.ID, model.API) - } - } - } -} - func TestSimpleReasoningEffortClampsAndOmitsOff(t *testing.T) { off := "none" xhigh := "extra" diff --git a/pkg/connector/approvals.go b/pkg/connector/approvals.go index 640619a1..d7756c45 100644 --- a/pkg/connector/approvals.go +++ b/pkg/connector/approvals.go @@ -323,12 +323,12 @@ func (cl *Client) fetchBeeperProfile(ctx context.Context) (*chattools.SessionPro }, nil } -func aiServicesWhoamiURL(proxyBaseURL string) (string, error) { - parsed, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(proxyBaseURL), "/")) +func aiServicesWhoamiURL(baseURL string) (string, error) { + parsed, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(baseURL), "/")) if err != nil { return "", err } - parsed.Path = strings.TrimRight(trimAIProxyProviderPath(parsed.Path), "/") + "/whoami" + parsed.Path = strings.TrimRight(parsed.Path, "/") + "/whoami" parsed.RawQuery = "" parsed.Fragment = "" return parsed.String(), nil diff --git a/pkg/connector/chat_tools.go b/pkg/connector/chat_tools.go index 613fab3b..b4e3a9b3 100644 --- a/pkg/connector/chat_tools.go +++ b/pkg/connector/chat_tools.go @@ -120,12 +120,12 @@ func (cl *Client) searchOptions(roomConfig RoomConfig, provider aiid.ProviderCon } } -func aiServicesToolURL(proxyBaseURL string, tool string) (string, error) { - parsed, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(proxyBaseURL), "/")) +func aiServicesToolURL(baseURL string, tool string) (string, error) { + parsed, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(baseURL), "/")) if err != nil { return "", err } - parsed.Path = strings.TrimRight(trimAIProxyProviderPath(parsed.Path), "/") + "/tools/" + tool + parsed.Path = strings.TrimRight(parsed.Path, "/") + "/tools/" + tool parsed.RawQuery = "" parsed.Fragment = "" return parsed.String(), nil diff --git a/pkg/connector/chat_tools_test.go b/pkg/connector/chat_tools_test.go index 52bdfcae..cf987fe7 100644 --- a/pkg/connector/chat_tools_test.go +++ b/pkg/connector/chat_tools_test.go @@ -35,7 +35,7 @@ func TestModelSupportsAgentToolsDefaultsToTrue(t *testing.T) { } func TestAIServicesToolURL(t *testing.T) { - got, err := aiServicesToolURL("https://ai-services.example/dev/proxy/openai/v1/responses", "web_search") + got, err := aiServicesToolURL("https://ai-services.example/dev", "web_search") if err != nil { t.Fatal(err) } diff --git a/pkg/connector/chatgpt_device_login.go b/pkg/connector/chatgpt_device_login.go index c0bd3092..cce18058 100644 --- a/pkg/connector/chatgpt_device_login.go +++ b/pkg/connector/chatgpt_device_login.go @@ -424,15 +424,6 @@ func chatGPTCodexProvider(credentials chatGPTCredentials) (aiid.ProviderConfig, } func defaultChatGPTCodexModel() string { - for _, modelID := range []string{"gpt-5.5", "gpt-5.4", "gpt-5.3-codex", "gpt-5.2-codex", "gpt-5-codex"} { - if _, ok := ai.GetModel(ai.ProviderOpenAICodex, modelID); ok { - return modelID - } - } - models := ai.GetModels(ai.ProviderOpenAICodex) - if len(models) > 0 { - return models[0].ID - } return "gpt-5-codex" } diff --git a/pkg/connector/config.go b/pkg/connector/config.go index 2daad645..7a993238 100644 --- a/pkg/connector/config.go +++ b/pkg/connector/config.go @@ -13,7 +13,6 @@ import ( //go:embed example-config.yaml var ExampleConfig string -const defaultAIServicesProxyPath = "/proxy/openai/v1" const defaultBeeperAIModel = "beeper/default" const defaultTitleGenerationModel = "gpt-4.1-mini" const fallbackTitleGenerationModel = "gpt-5-mini" diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 0c34914c..97207c1d 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -87,7 +87,7 @@ func (c *Connector) GetBridgeInfoVersion() (info, capabilities int) { } func (c *Connector) defaultProviderConfig(userMXID id.UserID) aiid.ProviderConfig { - baseURL := c.defaultAIServicesOpenAIProxyBaseURL(userMXID) + baseURL := c.defaultAIServicesBaseURL(userMXID) return aiid.ProviderConfig{ ID: aiid.DefaultProvider, DisplayName: "Beeper AI", @@ -98,32 +98,32 @@ func (c *Connector) defaultProviderConfig(userMXID id.UserID) aiid.ProviderConfi } } -func (c *Connector) defaultAIServicesOpenAIProxyBaseURL(userMXID id.UserID) string { +func (c *Connector) defaultAIServicesBaseURL(userMXID id.UserID) string { userDomain := userMXID.Homeserver() bridgeHost := c.homeserverAddressHost() if bridgeHost == "megahungry-proxy.megahungry" { if userDomain == "beeper.localtest.me" { - return "http://ai-services.beeper" + defaultAIServicesProxyPath + return "http://ai-services.beeper" } if isBeeperAIServiceDomain(userDomain) { - return "https://ai-services." + userDomain + defaultAIServicesProxyPath + return "https://ai-services." + userDomain } if userDomain != "" { return "" } - return "http://ai-services.beeper" + defaultAIServicesProxyPath + return "http://ai-services.beeper" } domain := homeserverServiceDomain(bridgeHost) if domain != "" { if userDomain != "" && userDomain != domain { return "" } - return "https://ai-services." + domain + defaultAIServicesProxyPath + return "https://ai-services." + domain } if userDomain == "" { return "" } - return "https://ai-services." + userDomain + defaultAIServicesProxyPath + return "https://ai-services." + userDomain } func isBeeperAIServiceDomain(domain string) bool { diff --git a/pkg/connector/contacts.go b/pkg/connector/contacts.go index b8a2a899..ba0416d1 100644 --- a/pkg/connector/contacts.go +++ b/pkg/connector/contacts.go @@ -454,44 +454,23 @@ func (cl *Client) aiServicesCatalogModels(ctx context.Context, provider aiid.Pro BuiltInTools: item.builtInTools(), Compat: item.compat(), } - model = item.applyProviderRoute(model, provider) + model = item.applyRuntime(model, provider) models = append(models, normalizeProviderModel(model, provider)) } return models, nil } -func aiServicesModelsURL(proxyBaseURL string) (string, error) { - parsed, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(proxyBaseURL), "/")) +func aiServicesModelsURL(baseURL string) (string, error) { + parsed, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(baseURL), "/")) if err != nil { return "", err } - parsed.Path = trimAIProxyProviderPath(parsed.Path) parsed.Path = strings.TrimRight(parsed.Path, "/") + "/models" - parsed.RawQuery = url.Values{"feature": {"bridge:ai"}, "route": {"responses"}}.Encode() + parsed.RawQuery = url.Values{"feature": {"bridge:ai"}}.Encode() parsed.Fragment = "" return parsed.String(), nil } -func trimAIProxyProviderPath(path string) string { - for _, suffix := range []string{ - "/proxy/openai/v1", - "/proxy/openai", - "/proxy/openrouter/v1", - "/proxy/openrouter", - "/proxy/anthropic/v1", - "/proxy/anthropic", - "/proxy/vertex/v1", - "/proxy/vertex", - "/proxy/a8c/v1", - "/proxy/a8c", - "/proxy/_/v1", - "/proxy/_", - } { - path = strings.TrimSuffix(path, suffix) - } - return path -} - type aiServicesModelListResponse struct { Type string `json:"type"` Data []aiServicesModelEntry `json:"data"` @@ -511,11 +490,13 @@ type aiServicesModelEntry struct { TopProvider *struct { MaxCompletionTokens int `json:"max_completion_tokens"` } `json:"top_provider"` - Provider *struct { - ID string `json:"id"` - ModelID string `json:"model_id"` - API string `json:"api"` - } `json:"provider"` + Runtime *struct { + API string `json:"api"` + Provider string `json:"provider"` + BaseURL string `json:"baseUrl"` + Model string `json:"model"` + Compat *aiServicesModelCompat `json:"compat"` + } `json:"runtime"` Capabilities *struct { Input struct { Modalities []string `json:"modalities"` @@ -541,58 +522,93 @@ type aiServicesModelEntry struct { } `json:"capabilities"` } -func (entry aiServicesModelEntry) applyProviderRoute(model ai.Model, provider aiid.ProviderConfig) ai.Model { - if entry.Provider == nil || entry.Provider.ID == "" { +type aiServicesModelCompat struct { + SupportsStore *bool `json:"supportsStore,omitempty"` + SupportsDeveloperRole *bool `json:"supportsDeveloperRole,omitempty"` + SupportsReasoningEffort *bool `json:"supportsReasoningEffort,omitempty"` + SupportsUsageInStreaming *bool `json:"supportsUsageInStreaming,omitempty"` + MaxTokensField string `json:"maxTokensField,omitempty"` + RequiresToolResultName *bool `json:"requiresToolResultName,omitempty"` + RequiresAssistantAfterToolResult *bool `json:"requiresAssistantAfterToolResult,omitempty"` + RequiresThinkingAsText *bool `json:"requiresThinkingAsText,omitempty"` + RequiresReasoningContentOnAssistantMessages *bool `json:"requiresReasoningContentOnAssistantMessages,omitempty"` + ThinkingFormat string `json:"thinkingFormat,omitempty"` + ZaiToolStream *bool `json:"zaiToolStream,omitempty"` + SupportsStrictMode *bool `json:"supportsStrictMode,omitempty"` + CacheControlFormat string `json:"cacheControlFormat,omitempty"` + SendSessionAffinityHeaders *bool `json:"sendSessionAffinityHeaders,omitempty"` + SupportsLongCacheRetention *bool `json:"supportsLongCacheRetention,omitempty"` + SendSessionIDHeader *bool `json:"sendSessionIdHeader,omitempty"` + SupportsEagerToolInputStreaming *bool `json:"supportsEagerToolInputStreaming,omitempty"` + SupportsCacheControlOnTools *bool `json:"supportsCacheControlOnTools,omitempty"` + SupportsTemperature *bool `json:"supportsTemperature,omitempty"` + ForceAdaptiveThinking *bool `json:"forceAdaptiveThinking,omitempty"` + AllowEmptySignature *bool `json:"allowEmptySignature,omitempty"` +} + +func (entry aiServicesModelEntry) applyRuntime(model ai.Model, provider aiid.ProviderConfig) ai.Model { + if entry.Runtime == nil { return model } if model.Compat == nil { model.Compat = map[string]any{} } - model.Compat["provider_id"] = entry.Provider.ID - model.Compat["provider_model_id"] = entry.Provider.ModelID - switch entry.Provider.ID { - case "wpcom_anthropic": - model.API = ai.ApiAnthropicMessages - model.Provider = ai.ProviderAnthropic - model.BaseURL = aiServicesProxyBaseURL(provider.BaseURL, "anthropic", false) - case "wpcom_vertex": - model.API = ai.ApiGoogleVertex - model.Provider = ai.ProviderGoogleVertex - model.BaseURL = aiServicesProxyBaseURL(provider.BaseURL, "vertex", false) - case "wpcom_openai": - model.API = ai.ApiOpenAIResponses - model.Provider = ai.ProviderOpenAI - model.BaseURL = aiServicesProxyBaseURL(provider.BaseURL, "openai", true) - case "wpcom_google": - model.API = ai.ApiGoogleVertex - model.Provider = ai.ProviderGoogleVertex - model.BaseURL = aiServicesProxyBaseURL(provider.BaseURL, "vertex", false) - case "wpcom_xai": - model.API = ai.ApiOpenAIResponses - model.Provider = ai.ProviderXAI - model.BaseURL = aiServicesProxyBaseURL(provider.BaseURL, "xai", true) - case "wpcom_groq": - model.API = ai.ApiOpenAIResponses - model.Provider = ai.ProviderGroq - model.BaseURL = aiServicesProxyBaseURL(provider.BaseURL, "groq", true) - case "wpcom_a8c": - model.API = ai.ApiOpenAICompletions - if entry.Provider.API == string(ai.ApiOpenAIResponses) { - model.API = ai.ApiOpenAIResponses - } - model.Provider = ai.Provider("a8c") - model.BaseURL = aiServicesProxyBaseURL(provider.BaseURL, "a8c", true) - case "openrouter": - model.API = ai.ApiOpenAICompletions - if entry.Provider.API == string(ai.ApiOpenAIResponses) { - model.API = ai.ApiOpenAIResponses - } - model.Provider = ai.ProviderOpenRouter - model.BaseURL = aiServicesProxyBaseURL(provider.BaseURL, "openrouter", true) + if entry.Runtime.Model != "" { + model.Compat["runtime_model"] = entry.Runtime.Model } + if entry.Runtime.API != "" { + model.API = ai.Api(entry.Runtime.API) + } + if entry.Runtime.Provider != "" { + model.Provider = ai.Provider(entry.Runtime.Provider) + model.Compat["runtime_provider"] = entry.Runtime.Provider + } + if entry.Runtime.BaseURL != "" { + model.BaseURL = aiServicesRuntimeBaseURL(provider.BaseURL, entry.Runtime.BaseURL) + } + entry.Runtime.Compat.applyTo(model.Compat) return model } +func (compat *aiServicesModelCompat) applyTo(dst map[string]any) { + if compat == nil { + return + } + setBoolCompat(dst, "supportsStore", compat.SupportsStore) + setBoolCompat(dst, "supportsDeveloperRole", compat.SupportsDeveloperRole) + setBoolCompat(dst, "supportsReasoningEffort", compat.SupportsReasoningEffort) + setBoolCompat(dst, "supportsUsageInStreaming", compat.SupportsUsageInStreaming) + setStringCompat(dst, "maxTokensField", compat.MaxTokensField) + setBoolCompat(dst, "requiresToolResultName", compat.RequiresToolResultName) + setBoolCompat(dst, "requiresAssistantAfterToolResult", compat.RequiresAssistantAfterToolResult) + setBoolCompat(dst, "requiresThinkingAsText", compat.RequiresThinkingAsText) + setBoolCompat(dst, "requiresReasoningContentOnAssistantMessages", compat.RequiresReasoningContentOnAssistantMessages) + setStringCompat(dst, "thinkingFormat", compat.ThinkingFormat) + setBoolCompat(dst, "zaiToolStream", compat.ZaiToolStream) + setBoolCompat(dst, "supportsStrictMode", compat.SupportsStrictMode) + setStringCompat(dst, "cacheControlFormat", compat.CacheControlFormat) + setBoolCompat(dst, "sendSessionAffinityHeaders", compat.SendSessionAffinityHeaders) + setBoolCompat(dst, "supportsLongCacheRetention", compat.SupportsLongCacheRetention) + setBoolCompat(dst, "sendSessionIdHeader", compat.SendSessionIDHeader) + setBoolCompat(dst, "supportsEagerToolInputStreaming", compat.SupportsEagerToolInputStreaming) + setBoolCompat(dst, "supportsCacheControlOnTools", compat.SupportsCacheControlOnTools) + setBoolCompat(dst, "supportsTemperature", compat.SupportsTemperature) + setBoolCompat(dst, "forceAdaptiveThinking", compat.ForceAdaptiveThinking) + setBoolCompat(dst, "allowEmptySignature", compat.AllowEmptySignature) +} + +func setBoolCompat(dst map[string]any, key string, value *bool) { + if value != nil { + dst[key] = *value + } +} + +func setStringCompat(dst map[string]any, key string, value string) { + if value != "" { + dst[key] = value + } +} + func (entry aiServicesModelEntry) compat() map[string]any { compat := map[string]any{} if entry.Metadata != nil { @@ -612,15 +628,12 @@ func (entry aiServicesModelEntry) compat() map[string]any { return compat } -func aiServicesProxyBaseURL(baseURL string, providerPath string, includeV1 bool) string { +func aiServicesRuntimeBaseURL(baseURL string, runtimeBaseURL string) string { parsed, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(baseURL), "/")) if err != nil { return baseURL } - parsed.Path = strings.TrimRight(trimAIProxyProviderPath(parsed.Path), "/") + "/proxy/" + providerPath - if includeV1 { - parsed.Path += "/v1" - } + parsed.Path = joinURLPath(parsed.Path, runtimeBaseURL) parsed.RawQuery = "" parsed.Fragment = "" return parsed.String() diff --git a/pkg/connector/contacts_test.go b/pkg/connector/contacts_test.go index 13499ca5..dca6c8f6 100644 --- a/pkg/connector/contacts_test.go +++ b/pkg/connector/contacts_test.go @@ -58,7 +58,7 @@ func TestModelContactsCacheRefreshesAfterCachedRead(t *testing.T) { })) defer server.Close() - client := newDefaultProviderContactClient(server.URL + "/proxy/openai/v1") + client := newDefaultProviderContactClient(server.URL) contacts, err := client.GetContactList(context.Background()) if err != nil { t.Fatal(err) @@ -91,7 +91,7 @@ func TestConnectWarmsModelContactsCache(t *testing.T) { })) defer server.Close() - client := newDefaultProviderContactClient(server.URL + "/proxy/openai/v1") + client := newDefaultProviderContactClient(server.URL) client.Connect(context.Background()) deadline := time.Now().Add(time.Second) for { @@ -166,11 +166,11 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { if r.URL.Path != "/models" { t.Fatalf("unexpected path %s", r.URL.Path) } - if r.URL.Query().Get("feature") != "bridge:ai" || r.URL.Query().Get("route") != "responses" { + if r.URL.Query().Get("feature") != "bridge:ai" || r.URL.Query().Has("route") { t.Fatalf("unexpected query %s", r.URL.RawQuery) } gotAuth = r.Header.Get("Authorization") - _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT-5.5","capabilities":{"input":{"modalities":["text","image"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["off","minimal","low","medium","high","xhigh"],"level_map":{"xhigh":"xhigh"},"default_level":"off","mode":"adaptive"},"tools":{"supported":true,"built_in":["image_generation"]},"limits":{"context_tokens":1050000,"output_tokens":128000}}},{"id":"minimax/minimax-m2.7","name":"MiniMax M2.7","provider":{"id":"openrouter","model_id":"minimax/minimax-m2.7","api":"openai-completions"},"capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["low","medium","high"],"level_map":{"off":null,"minimal":null},"default_level":"low"}}},{"id":"beeper/fast","name":"Beeper Fast","capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]}}}]}`)) + _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT-5.5","capabilities":{"input":{"modalities":["text","image"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["off","minimal","low","medium","high","xhigh"],"level_map":{"xhigh":"xhigh"},"default_level":"off","mode":"adaptive"},"tools":{"supported":true,"built_in":["image_generation"]},"limits":{"context_tokens":1050000,"output_tokens":128000}}},{"id":"minimax/minimax-m2.7","name":"MiniMax M2.7","runtime":{"provider":"openrouter","model":"minimax/minimax-m2.7","api":"openai-completions","baseUrl":"/proxy/openrouter/v1","compat":{"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_completion_tokens","thinkingFormat":"openrouter"}},"capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["low","medium","high"],"level_map":{"off":null,"minimal":null},"default_level":"low"}}},{"id":"beeper/fast","name":"Beeper Fast","capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]}}}]}`)) })) defer server.Close() @@ -186,7 +186,7 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { ID: aiid.DefaultProvider, Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses, - BaseURL: server.URL + "/proxy/openai/v1", + BaseURL: server.URL, }) if err != nil { t.Fatal(err) @@ -194,7 +194,7 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { if !strings.HasPrefix(gotAuth, "Bearer "+aiServicesAppserviceTokenPrefix) { t.Fatalf("unexpected auth header %q", gotAuth) } - if len(models) != 3 || models[0].ID != "openai/gpt-5.5" || models[0].BaseURL != server.URL+"/proxy/openai/v1" { + if len(models) != 3 || models[0].ID != "openai/gpt-5.5" || models[0].BaseURL != server.URL { t.Fatalf("unexpected models %#v", models) } if models[0].ContextWindow != 1050000 || models[0].MaxTokens != 128000 { @@ -227,6 +227,9 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { if models[1].API != ai.ApiOpenAICompletions || models[1].Provider != ai.ProviderOpenRouter || models[1].BaseURL != server.URL+"/proxy/openrouter/v1" { t.Fatalf("expected MiniMax OpenRouter route, got %#v", models[1]) } + if models[1].Compat["supportsDeveloperRole"] != false || models[1].Compat["thinkingFormat"] != "openrouter" || models[1].Compat["maxTokensField"] != "max_completion_tokens" { + t.Fatalf("expected MiniMax OpenRouter compat from AI Services, got %#v", models[1].Compat) + } if roomThinkingLevelSupported(models[1], ai.ModelThinkingLevelMinimal) { t.Fatalf("expected MiniMax reasoning to reject minimal, got %#v", models[1]) } @@ -279,7 +282,7 @@ func TestAIServicesCatalogModelsDoNotUseBridgeCatalogWhenMetadataMissing(t *test ID: aiid.DefaultProvider, Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses, - BaseURL: server.URL + "/proxy/openai/v1", + BaseURL: server.URL, }) if err != nil { t.Fatal(err) @@ -298,16 +301,12 @@ func TestAIServicesCatalogModelsDoNotUseBridgeCatalogWhenMetadataMissing(t *test } } -func TestAIServicesModelsURLStripsProviderProxyPaths(t *testing.T) { +func TestAIServicesModelsURLUsesBaseURL(t *testing.T) { tests := map[string]string{ - "https://ai-services.beeper.com/proxy/openai/v1": "https://ai-services.beeper.com/models?feature=bridge%3Aai&route=responses", - "https://ai-services.beeper.com/proxy/openrouter/v1": "https://ai-services.beeper.com/models?feature=bridge%3Aai&route=responses", - "https://ai-services.beeper.com/proxy/anthropic": "https://ai-services.beeper.com/models?feature=bridge%3Aai&route=responses", - "https://ai-services.beeper.com/proxy/vertex": "https://ai-services.beeper.com/models?feature=bridge%3Aai&route=responses", - "https://ai-services.beeper.com/proxy/a8c/v1": "https://ai-services.beeper.com/models?feature=bridge%3Aai&route=responses", - "https://ai-services.beeper.com/proxy/_/v1/responses": "https://ai-services.beeper.com/models?feature=bridge%3Aai&route=responses", - "https://ai-services.beeper.com/dev/proxy/openai/v1": "https://ai-services.beeper.com/dev/models?feature=bridge%3Aai&route=responses", - "https://ai-services.beeper.com/dev/proxy/openrouter/v1/": "https://ai-services.beeper.com/dev/models?feature=bridge%3Aai&route=responses", + "https://ai-services.beeper.com": "https://ai-services.beeper.com/models?feature=bridge%3Aai", + "https://ai-services.beeper.com/": "https://ai-services.beeper.com/models?feature=bridge%3Aai", + "https://ai-services.beeper.com/dev": "https://ai-services.beeper.com/dev/models?feature=bridge%3Aai", + "https://ai-services.beeper.com/dev/": "https://ai-services.beeper.com/dev/models?feature=bridge%3Aai", } for input, want := range tests { got, err := aiServicesModelsURL(input) @@ -323,14 +322,14 @@ func TestAIServicesModelsURLStripsProviderProxyPaths(t *testing.T) { func TestAIServicesCatalogModelsUsesPublishedProviderRoutes(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`{"data":[ - {"id":"claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":{"id":"wpcom_anthropic","model_id":"claude-sonnet-4-5","api":"openai-responses"}}, - {"id":"gemini-2.5-flash-lite","name":"Gemini 2.5 Flash Lite","provider":{"id":"wpcom_vertex","model_id":"gemini-2.5-flash-lite","api":"openai-responses"}}, - {"id":"google/gemini-3.1-pro-preview","name":"Gemini 3.1 Pro","provider":{"id":"wpcom_vertex","model_id":"gemini-3.1-pro-preview","api":"openai-responses"}}, - {"id":"google/gemini-2.5-flash","name":"Gemini 2.5 Flash","provider":{"id":"wpcom_google","model_id":"gemini-2.5-flash","api":"openai-responses"}}, - {"id":"x-ai/grok-4.20","name":"Grok 4.20","provider":{"id":"wpcom_xai","model_id":"x-ai/grok-4.20","api":"openai-responses"}}, - {"id":"groq/qwen/qwen3-32b","name":"Qwen 3 32B","provider":{"id":"wpcom_groq","model_id":"groq/qwen/qwen3-32b","api":"openai-responses"}}, - {"id":"openai/gpt-oss-120b","name":"GPT OSS 120B","provider":{"id":"wpcom_a8c","model_id":"gpt-oss-120b","api":"openai-completions"}}, - {"id":"anthropic/claude-sonnet-4.5","name":"Claude via OpenRouter","metadata":{"family":"claude","provider_logo_url":"/models/providers/anthropic.png"},"provider":{"id":"openrouter","model_id":"anthropic/claude-sonnet-4.5","api":"openai-completions"}} + {"id":"claude-sonnet-4-5","name":"Claude Sonnet 4.5","runtime":{"provider":"anthropic","model":"claude-sonnet-4-5","api":"anthropic-messages","baseUrl":"/proxy/anthropic"}}, + {"id":"gemini-2.5-flash-lite","name":"Gemini 2.5 Flash Lite","runtime":{"provider":"google-vertex","model":"gemini-2.5-flash-lite","api":"google-vertex","baseUrl":"/proxy/vertex"}}, + {"id":"google/gemini-3.1-pro-preview","name":"Gemini 3.1 Pro","runtime":{"provider":"google-vertex","model":"gemini-3.1-pro-preview","api":"google-vertex","baseUrl":"/proxy/vertex"}}, + {"id":"google/gemini-2.5-flash","name":"Gemini 2.5 Flash","runtime":{"provider":"google-vertex","model":"gemini-2.5-flash","api":"google-vertex","baseUrl":"/proxy/vertex"}}, + {"id":"x-ai/grok-4.20","name":"Grok 4.20","runtime":{"provider":"xai","model":"x-ai/grok-4.20","api":"openai-responses","baseUrl":"/proxy/xai/v1"}}, + {"id":"groq/qwen/qwen3-32b","name":"Qwen 3 32B","runtime":{"provider":"groq","model":"groq/qwen/qwen3-32b","api":"openai-responses","baseUrl":"/proxy/groq/v1"}}, + {"id":"openai/gpt-oss-120b","name":"GPT OSS 120B","runtime":{"provider":"a8c","model":"gpt-oss-120b","api":"openai-completions","baseUrl":"/proxy/a8c/v1"}}, + {"id":"anthropic/claude-sonnet-4.5","name":"Claude via OpenRouter","metadata":{"family":"claude","provider_logo_url":"/models/providers/anthropic.png"},"runtime":{"provider":"openrouter","model":"anthropic/claude-sonnet-4.5","api":"openai-completions","baseUrl":"/proxy/openrouter/v1","compat":{"supportsDeveloperRole":false,"thinkingFormat":"openrouter","cacheControlFormat":"anthropic"}}} ]}`)) })) defer server.Close() @@ -345,7 +344,7 @@ func TestAIServicesCatalogModelsUsesPublishedProviderRoutes(t *testing.T) { ID: aiid.DefaultProvider, Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses, - BaseURL: server.URL + "/proxy/openai/v1", + BaseURL: server.URL, }) if err != nil { t.Fatal(err) @@ -378,9 +377,12 @@ func TestAIServicesCatalogModelsUsesPublishedProviderRoutes(t *testing.T) { if got := byID["anthropic/claude-sonnet-4.5"]; got.API != ai.ApiOpenAICompletions || got.Provider != ai.ProviderOpenRouter || got.BaseURL != server.URL+"/proxy/openrouter/v1" { t.Fatalf("unexpected OpenRouter route %#v", got) } - if got := byID["anthropic/claude-sonnet-4.5"]; got.Compat["provider_logo_url"] != "/models/providers/anthropic.png" || got.Compat["provider_model_id"] != "anthropic/claude-sonnet-4.5" || got.Compat["family"] != "claude" { + if got := byID["anthropic/claude-sonnet-4.5"]; got.Compat["provider_logo_url"] != "/models/providers/anthropic.png" || got.Compat["runtime_model"] != "anthropic/claude-sonnet-4.5" || got.Compat["family"] != "claude" { t.Fatalf("expected AI Services catalog identity metadata, got %#v", got.Compat) } + if got := byID["anthropic/claude-sonnet-4.5"]; got.Compat["supportsDeveloperRole"] != false || got.Compat["thinkingFormat"] != "openrouter" || got.Compat["cacheControlFormat"] != "anthropic" { + t.Fatalf("expected AI Services catalog provider compat, got %#v", got.Compat) + } } func TestResolveModelForProviderPreservesOpenAICatalogModelID(t *testing.T) { diff --git a/pkg/connector/login.go b/pkg/connector/login.go index f969afb4..9cdf00e7 100644 --- a/pkg/connector/login.go +++ b/pkg/connector/login.go @@ -373,7 +373,7 @@ func fetchProviderModels(ctx context.Context, api ai.Api, providerID string, bas ContextWindow: item.contextWindow(), MaxTokens: item.maxTokens(), } - model = item.applyProviderRoute(model, providerConfig) + model = item.applyRuntime(model, providerConfig) models = append(models, normalizeProviderModel(model, providerConfig)) } if len(models) == 0 { diff --git a/pkg/connector/model_avatar.go b/pkg/connector/model_avatar.go index fe498579..7c4402ef 100644 --- a/pkg/connector/model_avatar.go +++ b/pkg/connector/model_avatar.go @@ -136,7 +136,7 @@ func remoteModelAvatar(logoID string, logoURL string) *bridgev2.Avatar { } } -func aiServicesProviderLogoURL(proxyBaseURL string, providerLogoURL string) (string, error) { +func aiServicesProviderLogoURL(baseURL string, providerLogoURL string) (string, error) { providerLogoURL = strings.TrimSpace(providerLogoURL) if providerLogoURL == "" { return "", fmt.Errorf("empty provider logo URL") @@ -152,17 +152,17 @@ func aiServicesProviderLogoURL(proxyBaseURL string, providerLogoURL string) (str logo.Fragment = "" return logo.String(), nil } - if proxyBaseURL == "" { + if baseURL == "" { return "", fmt.Errorf("missing AI Services base URL for provider logo %q", providerLogoURL) } - base, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(proxyBaseURL), "/")) + base, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(baseURL), "/")) if err != nil { return "", err } if base.Scheme == "" || base.Host == "" { - return "", fmt.Errorf("invalid AI Services base URL %q", proxyBaseURL) + return "", fmt.Errorf("invalid AI Services base URL %q", baseURL) } - base.Path = joinURLPath(trimAIProxyProviderPath(base.Path), logo.Path) + base.Path = joinURLPath(base.Path, logo.Path) base.RawQuery = logo.RawQuery base.Fragment = "" return base.String(), nil @@ -187,7 +187,7 @@ func modelAvatarProviderKeyFromHints(model ai.Model) string { if key := providerLogoURLAvatarKey(compatString(model.Compat["provider_logo_url"])); key != "" { return key } - if key := modelAvatarProviderKeyFromModelID(compatString(model.Compat["provider_model_id"])); key != "" { + if key := modelAvatarProviderKeyFromModelID(compatString(model.Compat["runtime_model"])); key != "" { return key } return modelAvatarProviderKeyFromFamily(compatString(model.Compat["family"])) diff --git a/pkg/connector/model_avatar_test.go b/pkg/connector/model_avatar_test.go index d549d0d0..dc021fe0 100644 --- a/pkg/connector/model_avatar_test.go +++ b/pkg/connector/model_avatar_test.go @@ -21,7 +21,7 @@ func TestModelAvatarUsesEmbeddedPNGAssets(t *testing.T) { {name: "google vertex", model: ai.Model{ID: "gemini-3-pro", Provider: ai.ProviderGoogleVertex}, wantKey: "google"}, {name: "openrouter own model", model: ai.Model{ID: "openrouter/owl-alpha", Provider: ai.ProviderOpenRouter}, wantKey: "openrouter"}, {name: "catalog provider logo wins over route", model: ai.Model{ID: "openrouter/anthropic/claude-opus-4.8-fast", Provider: ai.ProviderOpenRouter, Compat: map[string]any{"provider_logo_url": "/models/providers/anthropic.png"}}, wantKey: "anthropic"}, - {name: "catalog provider model wins over route", model: ai.Model{ID: "openrouter/z-ai/glm-5", Provider: ai.ProviderOpenRouter, Compat: map[string]any{"provider_model_id": "z-ai/glm-5"}}, wantKey: "zai"}, + {name: "catalog runtime model wins over route", model: ai.Model{ID: "openrouter/z-ai/glm-5", Provider: ai.ProviderOpenRouter, Compat: map[string]any{"runtime_model": "z-ai/glm-5"}}, wantKey: "zai"}, {name: "catalog family wins over route", model: ai.Model{ID: "openrouter/xiaomi/mimo-v2.5", Provider: ai.ProviderOpenRouter, Compat: map[string]any{"family": "mimo"}}, wantKey: "xiaomi"}, {name: "openrouter routed anthropic model", model: ai.Model{ID: "anthropic/claude-sonnet-4.5", Provider: ai.ProviderOpenRouter}, wantKey: "anthropic"}, {name: "openrouter routed bare claude model", model: ai.Model{ID: "claude-sonnet-4-5", Provider: ai.ProviderOpenRouter}, wantKey: "anthropic"}, @@ -70,7 +70,7 @@ func TestModelAvatarFetchesCatalogProviderLogoFromAIServices(t *testing.T) { })) defer server.Close() - avatar := modelAvatar(aiid.ProviderConfig{BaseURL: server.URL + "/proxy/openai/v1"}, ai.Model{ + avatar := modelAvatar(aiid.ProviderConfig{BaseURL: server.URL}, ai.Model{ ID: "qwen/qwen3-max", Provider: ai.ProviderOpenRouter, Compat: map[string]any{"provider_logo_url": "/models/providers/qwen.png"}, @@ -94,7 +94,7 @@ func TestModelAvatarFetchesCatalogProviderLogoFromAIServices(t *testing.T) { } func TestAIServicesProviderLogoURL(t *testing.T) { - got, err := aiServicesProviderLogoURL("https://ai-services.example/dev/proxy/openrouter/v1", "/models/providers/anthropic.png") + got, err := aiServicesProviderLogoURL("https://ai-services.example/dev", "/models/providers/anthropic.png") if err != nil { t.Fatal(err) } @@ -102,7 +102,7 @@ func TestAIServicesProviderLogoURL(t *testing.T) { t.Fatalf("unexpected logo URL %q, want %q", got, want) } - got, err = aiServicesProviderLogoURL("https://ai-services.example/proxy/openai/v1/responses", "models/providers/zai.png?cache=1") + got, err = aiServicesProviderLogoURL("https://ai-services.example", "models/providers/zai.png?cache=1") if err != nil { t.Fatal(err) } diff --git a/pkg/connector/provider.go b/pkg/connector/provider.go index 06ffc34b..b70825d8 100644 --- a/pkg/connector/provider.go +++ b/pkg/connector/provider.go @@ -125,27 +125,6 @@ func normalizeProviderModel(model ai.Model, provider aiid.ProviderConfig) ai.Mod if model.Name == "" { model.Name = model.ID } - if provider.ID != aiid.DefaultProvider { - if catalogModel, ok := ai.GetModel(model.Provider, model.ID); ok { - if len(model.Input) == 0 && len(catalogModel.Input) > 0 { - model.Input = append([]string(nil), catalogModel.Input...) - } - if !model.Reasoning { - model.Reasoning = catalogModel.Reasoning - } - if len(model.ThinkingLevelMap) == 0 && len(catalogModel.ThinkingLevelMap) > 0 { - model.ThinkingLevelMap = catalogModel.ThinkingLevelMap - } - if model.DefaultThinkingLevel == "" { - model.DefaultThinkingLevel = catalogModel.DefaultThinkingLevel - } - if model.ReasoningMode == "" { - model.ReasoningMode = catalogModel.ReasoningMode - } - } else if len(model.Input) == 0 { - model.Input = catalogInputForProviderModel(model) - } - } if len(model.Input) == 0 { model.Input = []string{"text"} } @@ -153,18 +132,6 @@ func normalizeProviderModel(model ai.Model, provider aiid.ProviderConfig) ai.Mod return model } -func catalogInputForProviderModel(model ai.Model) []string { - prefix := string(model.Provider) + "/" - if !strings.HasPrefix(model.ID, prefix) { - return nil - } - catalogModel, ok := ai.GetModel(model.Provider, strings.TrimPrefix(model.ID, prefix)) - if !ok || len(catalogModel.Input) == 0 { - return nil - } - return append([]string(nil), catalogModel.Input...) -} - func (cl *Client) authForProvider(provider aiid.ProviderConfig) func(context.Context, ai.Model) (*harness.AgentHarnessAuth, error) { return func(ctx context.Context, model ai.Model) (*harness.AgentHarnessAuth, error) { logCtx := zerolog.Ctx(ctx).With(). diff --git a/pkg/connector/provider_test.go b/pkg/connector/provider_test.go index 3e872056..645fc76e 100644 --- a/pkg/connector/provider_test.go +++ b/pkg/connector/provider_test.go @@ -47,7 +47,7 @@ func TestModelForProviderBuildsDefaultProviderModelFromConfig(t *testing.T) { ID: aiid.DefaultProvider, API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenAI, - BaseURL: "https://ai-services.beeper-staging.com/proxy/openai/v1", + BaseURL: "https://ai-services.beeper-staging.com", } model := conn.ModelForProvider(provider, "openai/gpt-5.5") if model.ID != "openai/gpt-5.5" || model.Provider != ai.ProviderOpenAI || model.API != ai.ApiOpenAIResponses { @@ -269,17 +269,17 @@ func TestConfigDefaults(t *testing.T) { } } -func TestDefaultAIServicesOpenAIProxyBaseURLUsesUserHomeserver(t *testing.T) { +func TestDefaultAIServicesBaseURLUsesUserHomeserver(t *testing.T) { conn := &Connector{} tests := map[string]string{ - "@alice:beeper.localtest.me": "https://ai-services.beeper.localtest.me/proxy/openai/v1", - "@alice:beeper-dev.com": "https://ai-services.beeper-dev.com/proxy/openai/v1", - "@alice:beeper-staging.com": "https://ai-services.beeper-staging.com/proxy/openai/v1", - "@alice:beeper.com": "https://ai-services.beeper.com/proxy/openai/v1", + "@alice:beeper.localtest.me": "https://ai-services.beeper.localtest.me", + "@alice:beeper-dev.com": "https://ai-services.beeper-dev.com", + "@alice:beeper-staging.com": "https://ai-services.beeper-staging.com", + "@alice:beeper.com": "https://ai-services.beeper.com", } for userMXID, want := range tests { - if got := conn.defaultAIServicesOpenAIProxyBaseURL(id.UserID(userMXID)); got != want { - t.Fatalf("defaultAIServicesOpenAIProxyBaseURL(%q) = %q, want %q", userMXID, got, want) + if got := conn.defaultAIServicesBaseURL(id.UserID(userMXID)); got != want { + t.Fatalf("defaultAIServicesBaseURL(%q) = %q, want %q", userMXID, got, want) } } } @@ -287,7 +287,7 @@ func TestDefaultAIServicesOpenAIProxyBaseURLUsesUserHomeserver(t *testing.T) { func TestDefaultProviderBaseURLUsesUserHomeserver(t *testing.T) { conn := &Connector{} provider := conn.defaultProviderConfig("@alice:beeper-staging.com") - if provider.BaseURL != "https://ai-services.beeper-staging.com/proxy/openai/v1" { + if provider.BaseURL != "https://ai-services.beeper-staging.com" { t.Fatalf("unexpected provider base URL %q", provider.BaseURL) } } @@ -295,7 +295,7 @@ func TestDefaultProviderBaseURLUsesUserHomeserver(t *testing.T) { func TestDefaultProviderBaseURLUsesInternalLocalServiceForLocalUser(t *testing.T) { conn := &Connector{HomeserverURL: "http://megahungry-proxy.megahungry/api/proxy/bridge-user"} provider := conn.defaultProviderConfig("@alice:beeper.localtest.me") - if provider.BaseURL != "http://ai-services.beeper/proxy/openai/v1" { + if provider.BaseURL != "http://ai-services.beeper" { t.Fatalf("unexpected provider base URL %q", provider.BaseURL) } } @@ -303,7 +303,7 @@ func TestDefaultProviderBaseURLUsesInternalLocalServiceForLocalUser(t *testing.T func TestDefaultProviderBaseURLUsesUserHomeserverForMegahungryCloudUser(t *testing.T) { conn := &Connector{HomeserverURL: "http://megahungry-proxy.megahungry/api/proxy/bridge-user"} provider := conn.defaultProviderConfig("@alice:beeper-staging.com") - if provider.BaseURL != "https://ai-services.beeper-staging.com/proxy/openai/v1" { + if provider.BaseURL != "https://ai-services.beeper-staging.com" { t.Fatalf("unexpected provider base URL %q", provider.BaseURL) } } @@ -311,7 +311,7 @@ func TestDefaultProviderBaseURLUsesUserHomeserverForMegahungryCloudUser(t *testi func TestDefaultProviderBaseURLUsesExternalLocaltestServiceForSelfHosted(t *testing.T) { conn := &Connector{HomeserverURL: "https://matrix.beeper.localtest.me/_hungryserv/bridge-user"} provider := conn.defaultProviderConfig("@alice:beeper.localtest.me") - if provider.BaseURL != "https://ai-services.beeper.localtest.me/proxy/openai/v1" { + if provider.BaseURL != "https://ai-services.beeper.localtest.me" { t.Fatalf("unexpected provider base URL %q", provider.BaseURL) } } @@ -332,7 +332,7 @@ func TestDefaultProviderReadsAIChatsLoginMetadata(t *testing.T) { Metadata: &aiid.UserLoginMetadata{Providers: map[string]aiid.ProviderConfig{provider.ID: provider}}, }} got := conn.providersForLogin(login)[aiid.DefaultProvider] - if got.BaseURL != "https://ai-services.beeper.localtest.me/proxy/openai/v1" { + if got.BaseURL != "https://ai-services.beeper.localtest.me" { t.Fatalf("expected persisted default provider, got %#v", got) } } @@ -358,7 +358,7 @@ func TestProvidersForLoginReadsProviderMap(t *testing.T) { }}, }} providers := conn.providersForLogin(login) - if providers[aiid.DefaultProvider].BaseURL != "https://ai-services.beeper.com/proxy/openai/v1" { + if providers[aiid.DefaultProvider].BaseURL != "https://ai-services.beeper.com" { t.Fatalf("default provider missing from provider map: %#v", providers) } if providers["custom"].APIKey != "secret-key" || providers["custom"].DefaultModel != "model-a" { @@ -676,14 +676,14 @@ func TestFetchProviderModelsVerifiesAndBuildsModels(t *testing.T) { func TestFetchProviderModelsRespectsPublishedProviderRoutes(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`{"data":[ - {"id":"claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":{"id":"wpcom_anthropic","model_id":"claude-sonnet-4-5","api":"openai-responses"}}, - {"id":"gemini-2.5-flash-lite","name":"Gemini 2.5 Flash Lite","provider":{"id":"wpcom_vertex","model_id":"gemini-2.5-flash-lite","api":"openai-responses"}}, - {"id":"google/gemini-2.5-flash","name":"Gemini 2.5 Flash","provider":{"id":"wpcom_google","model_id":"gemini-2.5-flash","api":"openai-responses"}} + {"id":"claude-sonnet-4-5","name":"Claude Sonnet 4.5","runtime":{"provider":"anthropic","model":"claude-sonnet-4-5","api":"anthropic-messages","baseUrl":"/proxy/anthropic"}}, + {"id":"gemini-2.5-flash-lite","name":"Gemini 2.5 Flash Lite","runtime":{"provider":"google-vertex","model":"gemini-2.5-flash-lite","api":"google-vertex","baseUrl":"/proxy/vertex"}}, + {"id":"google/gemini-2.5-flash","name":"Gemini 2.5 Flash","runtime":{"provider":"google-vertex","model":"gemini-2.5-flash","api":"google-vertex","baseUrl":"/proxy/vertex"}} ]}`)) })) defer server.Close() - models, err := fetchProviderModels(context.Background(), ai.ApiOpenAIResponses, "local", server.URL+"/proxy/openai/v1", "key") + models, err := fetchProviderModels(context.Background(), ai.ApiOpenAIResponses, "local", server.URL, "key") if err != nil { t.Fatal(err) } @@ -768,13 +768,13 @@ func TestModelForProviderAppliesRouteBaseURLToDefaultModel(t *testing.T) { ID: aiid.DefaultProvider, API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenAI, - BaseURL: "https://ai-services.beeper.com/proxy/openai/v1/responses", + BaseURL: "https://ai-services.beeper.com", } model := conn.ModelForProvider(provider, "gpt-5.5") if model.Provider != ai.ProviderOpenAI || model.ID != "gpt-5.5" { t.Fatalf("expected AI Services model, got %#v", model) } - if model.BaseURL != "https://ai-services.beeper.com/proxy/openai/v1" { + if model.BaseURL != "https://ai-services.beeper.com" { t.Fatalf("expected route base URL override, got %q", model.BaseURL) } if model.ContextWindow != 128000 || model.MaxTokens != 32000 { @@ -821,10 +821,7 @@ func TestValidateReasoningLevelRejectsUnsupportedPair(t *testing.T) { func TestValidateReasoningLevelAcceptsOffForReasoningModel(t *testing.T) { client := &Client{Main: &Connector{Config: Config{DefaultReasoningLevel: "off"}}} - model, ok := ai.GetModel(ai.ProviderOpenAI, "gpt-5.4") - if !ok { - t.Fatal("expected generated gpt-5.4 model") - } + model := ai.Model{ID: "reasoning", Reasoning: true} if err := client.validateReasoningLevel(model, RoomConfig{}); err != nil { t.Fatalf("expected default off reasoning to be accepted: %v", err) } @@ -883,7 +880,7 @@ func TestNormalizeProviderModelDoesNotInheritDefaultProviderCatalogMetadata(t *t ID: aiid.DefaultProvider, Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses, - BaseURL: "https://ai-services.test/proxy/openrouter/v1", + BaseURL: "https://ai-services.test", }) if model.Reasoning || model.DefaultThinkingLevel != "" || len(model.ThinkingLevelMap) != 0 { t.Fatalf("expected default provider model to rely only on AI Services metadata, got %#v", model) diff --git a/pkg/connector/slash_commands_limits.go b/pkg/connector/slash_commands_limits.go index ca7243db..bd6d6ae3 100644 --- a/pkg/connector/slash_commands_limits.go +++ b/pkg/connector/slash_commands_limits.go @@ -118,12 +118,11 @@ func (cl *Client) defaultAIProviderForLimits() (aiid.ProviderConfig, error) { return provider, nil } -func aiServicesLimitsURL(proxyBaseURL string) (string, error) { - parsed, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(proxyBaseURL), "/")) +func aiServicesLimitsURL(baseURL string) (string, error) { + parsed, err := url.Parse(strings.TrimRight(normalizeResponsesBaseURL(baseURL), "/")) if err != nil { return "", err } - parsed.Path = trimAIProxyProviderPath(parsed.Path) parsed.Path = strings.TrimRight(parsed.Path, "/") + "/limits" parsed.RawQuery = "" parsed.Fragment = "" diff --git a/pkg/connector/slash_commands_model.go b/pkg/connector/slash_commands_model.go index 1a8dbcac..f71c6b01 100644 --- a/pkg/connector/slash_commands_model.go +++ b/pkg/connector/slash_commands_model.go @@ -57,7 +57,7 @@ func (cl *Client) applyModelCommand(ctx context.Context, portal *bridgev2.Portal return err } cl.refreshRoomCapabilities(ctx, portal) - return responder.Reply(ctx, fmt.Sprintf("Model set to `%s`. Current reasoning is `%s`.%s", canonical, target.ThinkingLevel, reasoningModeSentence(target.ReasoningMode))) + return responder.Reply(ctx, fmt.Sprintf("Model set to `%s`. %s%s", canonical, reasoningSettingSentence(target.ThinkingLevel, canonical, model), reasoningModeSentence(target.ReasoningMode))) } func (cl *Client) applyReasoningCommand(ctx context.Context, portal *bridgev2.Portal, current RoomConfig, requested string, responder aiCommandResponder) error { @@ -89,7 +89,7 @@ func (cl *Client) applyReasoningCommand(ctx context.Context, portal *bridgev2.Po return err } cl.refreshRoomCapabilities(ctx, portal) - return responder.Reply(ctx, fmt.Sprintf("Reasoning set to `%s` for `%s`.", reasoning, canonical)) + return responder.Reply(ctx, fmt.Sprintf("%s's reasoning is now set to `%s`.", reasoningStatusModelName(model, canonical), reasoning)) } func (cl *Client) applyReasoningModeCommand(ctx context.Context, portal *bridgev2.Portal, current RoomConfig, requested string, responder aiCommandResponder) error { @@ -136,7 +136,30 @@ func displayReasoningLevel(level string) string { } func reasoningStatusText(current string, canonicalModel string, model ai.Model) string { - return fmt.Sprintf("Current reasoning is `%s` for `%s`. Options: %s.", displayReasoningLevel(current), canonicalModel, reasoningOptionsText(model)) + return reasoningSettingSentence(current, canonicalModel, model) +} + +func reasoningSettingSentence(current string, canonicalModel string, model ai.Model) string { + levels := ai.GetSupportedThinkingLevels(model) + name := reasoningStatusModelName(model, canonicalModel) + if len(levels) == 1 { + currentLevel := displayReasoningLevel(current) + if current == "" { + currentLevel = string(levels[0]) + } + if levels[0] == ai.ModelThinkingLevelOff { + return fmt.Sprintf("%s doesn't support reasoning.", name) + } + return fmt.Sprintf("%s's reasoning is set to `%s` and it doesn't support changing reasoning settings.", name, currentLevel) + } + return fmt.Sprintf("%s's reasoning is set to `%s`. Available settings: %s.", name, displayReasoningLevel(current), reasoningOptionsText(model)) +} + +func reasoningStatusModelName(model ai.Model, canonicalModel string) string { + if model.Name != "" && model.Name != model.ID { + return model.Name + } + return canonicalModel } func displayReasoningMode(mode string) string { @@ -147,14 +170,14 @@ func displayReasoningMode(mode string) string { } func reasoningModeStatusText(current string, canonicalModel string, model ai.Model) string { - return fmt.Sprintf("Current reasoning mode is `%s` for `%s`. Options: %s.", displayReasoningMode(current), canonicalModel, reasoningModeOptionsText(model)) + return fmt.Sprintf("%s's reasoning mode is set to `%s`. Available modes: %s.", reasoningStatusModelName(model, canonicalModel), displayReasoningMode(current), reasoningModeOptionsText(model)) } func reasoningModeSentence(mode string) string { if mode == "" { return "" } - return fmt.Sprintf(" Current reasoning mode is `%s`.", mode) + return fmt.Sprintf(" Reasoning mode: `%s`.", mode) } func reasoningOptionsText(model ai.Model) string { @@ -178,11 +201,11 @@ func reasoningModeOptionsText(model ai.Model) string { } func (cl *Client) modelStatusText(currentModel string, currentReasoning string, currentReasoningMode string, currentProvider aiid.ProviderConfig) string { - text := fmt.Sprintf("Current model is `%s`. Current reasoning is `%s`.", currentModel, currentReasoning) + text := fmt.Sprintf("Current model: `%s`. Reasoning: `%s`.", currentModel, currentReasoning) if currentReasoningMode != "" { - text += fmt.Sprintf(" Current reasoning mode is `%s`.", currentReasoningMode) + text += fmt.Sprintf(" Reasoning mode: `%s`.", currentReasoningMode) } - return fmt.Sprintf("%s Options: %s.", text, cl.modelOptionsText(currentProvider)) + return fmt.Sprintf("%s Available models: %s.", text, cl.modelOptionsText(currentProvider)) } func (cl *Client) modelOptionsText(currentProvider aiid.ProviderConfig) string { diff --git a/pkg/connector/slash_commands_test.go b/pkg/connector/slash_commands_test.go index 3fe4ad91..cfea31f4 100644 --- a/pkg/connector/slash_commands_test.go +++ b/pkg/connector/slash_commands_test.go @@ -189,27 +189,53 @@ func TestCurrentCommandResponseText(t *testing.T) { } model := ai.Model{ID: "anthropic/claude-opus-4.5", Reasoning: true} status := reasoningStatusText("", "beeper/anthropic/claude-opus-4.5", model) - if !strings.Contains(status, "Current reasoning is `off` for `beeper/anthropic/claude-opus-4.5`.") { + if !strings.Contains(status, "beeper/anthropic/claude-opus-4.5's reasoning is set to `off`.") { t.Fatalf("reasoning status is missing current value:\n%s", status) } - if !strings.Contains(status, "Options: `off`, `minimal`, `low`, `medium`, `high`.") { + if !strings.Contains(status, "Available settings: `off`, `minimal`, `low`, `medium`, `high`.") { t.Fatalf("reasoning status is missing supported options:\n%s", status) } + geminiStatus := reasoningStatusText("off", "beeper/google/gemini-3-pro-image-preview", ai.Model{ + ID: "google/gemini-3-pro-image-preview", + Name: "Gemini 3 Pro Image Preview", + Reasoning: true, + }) + if geminiStatus != "Gemini 3 Pro Image Preview's reasoning is set to `off`. Available settings: `off`, `minimal`, `low`, `medium`, `high`." { + t.Fatalf("unexpected Gemini reasoning status:\n%s", geminiStatus) + } + unsupportedStatus := reasoningStatusText("off", "beeper/meta-llama/llama-3.3-70b-instruct", ai.Model{ID: "meta-llama/llama-3.3-70b-instruct", Name: "Llama 3.3 70B"}) + if unsupportedStatus != "Llama 3.3 70B doesn't support reasoning." { + t.Fatalf("unexpected unsupported reasoning status:\n%s", unsupportedStatus) + } + fixedStatus := reasoningStatusText("low", "beeper/minimax/minimax-m2.7", ai.Model{ + ID: "minimax/minimax-m2.7", + Name: "MiniMax M2.7", + Reasoning: true, + ThinkingLevelMap: map[ai.ModelThinkingLevel]*string{ + ai.ModelThinkingLevelOff: nil, + ai.ModelThinkingLevelMinimal: nil, + ai.ModelThinkingLevelMedium: nil, + ai.ModelThinkingLevelHigh: nil, + }, + }) + if fixedStatus != "MiniMax M2.7's reasoning is set to `low` and it doesn't support changing reasoning settings." { + t.Fatalf("unexpected fixed reasoning status:\n%s", fixedStatus) + } modeStatus := reasoningModeStatusText("adaptive", "beeper/anthropic/claude-opus-4.8", ai.Model{ID: "anthropic/claude-opus-4.8", ReasoningMode: ai.ModelReasoningModeAdaptive}) - if !strings.Contains(modeStatus, "Current reasoning mode is `adaptive` for `beeper/anthropic/claude-opus-4.8`.") { + if !strings.Contains(modeStatus, "beeper/anthropic/claude-opus-4.8's reasoning mode is set to `adaptive`.") { t.Fatalf("reasoning mode status is missing current value:\n%s", modeStatus) } - if !strings.Contains(modeStatus, "Options: `default`, `adaptive`.") { + if !strings.Contains(modeStatus, "Available modes: `default`, `adaptive`.") { t.Fatalf("reasoning mode status is missing supported options:\n%s", modeStatus) } modelStatus := canonicalTestClient().modelStatusText("beeper/gpt-5.5", "off", "", aiid.ProviderConfig{ ID: "beeper", Models: []ai.Model{{ID: "gpt-5.5"}, {ID: "openai/gpt-5.5"}}, }) - if !strings.Contains(modelStatus, "Current model is `beeper/gpt-5.5`. Current reasoning is `off`.") { + if !strings.Contains(modelStatus, "Current model: `beeper/gpt-5.5`. Reasoning: `off`.") { t.Fatalf("model status is missing current value:\n%s", modelStatus) } - if !strings.Contains(modelStatus, "Options: `beeper/gpt-5.5`, `beeper/openai/gpt-5.5`.") { + if !strings.Contains(modelStatus, "Available models: `beeper/gpt-5.5`, `beeper/openai/gpt-5.5`.") { t.Fatalf("model status is missing available options:\n%s", modelStatus) } if got := currentSystemPromptText(RoomConfig{}); got != "No additional system prompt is set." { @@ -403,16 +429,12 @@ func TestFormatSessionCommandInfo(t *testing.T) { } } -func TestAIServicesLimitsURLStripsProviderProxyPaths(t *testing.T) { +func TestAIServicesLimitsURLUsesBaseURL(t *testing.T) { tests := map[string]string{ - "https://ai-services.beeper.com/proxy/openai/v1": "https://ai-services.beeper.com/limits", - "https://ai-services.beeper.com/proxy/openrouter/v1": "https://ai-services.beeper.com/limits", - "https://ai-services.beeper.com/proxy/anthropic": "https://ai-services.beeper.com/limits", - "https://ai-services.beeper.com/proxy/vertex": "https://ai-services.beeper.com/limits", - "https://ai-services.beeper.com/proxy/a8c/v1": "https://ai-services.beeper.com/limits", - "https://ai-services.beeper.com/proxy/_/v1/responses": "https://ai-services.beeper.com/limits", - "https://ai-services.beeper.com/dev/proxy/openai/v1": "https://ai-services.beeper.com/dev/limits", - "https://ai-services.beeper.com/dev/proxy/openrouter/v1/": "https://ai-services.beeper.com/dev/limits", + "https://ai-services.beeper.com": "https://ai-services.beeper.com/limits", + "https://ai-services.beeper.com/": "https://ai-services.beeper.com/limits", + "https://ai-services.beeper.com/dev": "https://ai-services.beeper.com/dev/limits", + "https://ai-services.beeper.com/dev/": "https://ai-services.beeper.com/dev/limits", } for input, want := range tests { got, err := aiServicesLimitsURL(input) @@ -450,7 +472,7 @@ func TestFetchAIServicesLimitsUsesAppserviceBearerToken(t *testing.T) { })) defer server.Close() - provider := aiid.ProviderConfig{ID: aiid.DefaultProvider, BaseURL: server.URL + "/proxy/openai/v1"} + provider := aiid.ProviderConfig{ID: aiid.DefaultProvider, BaseURL: server.URL} client := &Client{ Main: &Connector{AppServiceToken: "as-token"}, UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ @@ -478,7 +500,7 @@ func TestRunLimitsCommandRawUsesAIResponse(t *testing.T) { })) defer server.Close() - provider := aiid.ProviderConfig{ID: aiid.DefaultProvider, BaseURL: server.URL + "/proxy/openai/v1"} + provider := aiid.ProviderConfig{ID: aiid.DefaultProvider, BaseURL: server.URL} client := &Client{ Main: &Connector{AppServiceToken: "as-token"}, UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ @@ -524,7 +546,7 @@ func TestBeeperUsageLimitErrorUsesPlanResetMessage(t *testing.T) { })) defer server.Close() - provider := aiid.ProviderConfig{ID: aiid.DefaultProvider, BaseURL: server.URL + "/proxy/openai/v1"} + provider := aiid.ProviderConfig{ID: aiid.DefaultProvider, BaseURL: server.URL} client := &Client{ Main: &Connector{AppServiceToken: "as-token"}, UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ diff --git a/pkg/connector/stream_test.go b/pkg/connector/stream_test.go index dc2d0e60..742bc0d5 100644 --- a/pkg/connector/stream_test.go +++ b/pkg/connector/stream_test.go @@ -901,7 +901,7 @@ func TestAssistantModelProfileUsesCatalogDisplayName(t *testing.T) { if r.URL.Path != "/models" { t.Fatalf("unexpected path %s", r.URL.Path) } - _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT 5.5 Catalog","provider":{"id":"wpcom_openai","model_id":"gpt-5.5","api":"openai-responses"}}]}`)) + _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT 5.5 Catalog","runtime":{"provider":"openai","model":"gpt-5.5","api":"openai-responses","baseUrl":"/proxy/openai/v1"}}]}`)) })) defer server.Close() @@ -914,7 +914,7 @@ func TestAssistantModelProfileUsesCatalogDisplayName(t *testing.T) { DisplayName: "Beeper AI", API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenAI, - BaseURL: server.URL + "/proxy/openai/v1", + BaseURL: server.URL, DefaultModel: "beeper/default", }}}, }}, diff --git a/scripts/generate-image-models-go.mjs b/scripts/generate-image-models-go.mjs deleted file mode 100755 index 08c145e6..00000000 --- a/scripts/generate-image-models-go.mjs +++ /dev/null @@ -1,21 +0,0 @@ -import fs from 'node:fs'; -import vm from 'node:vm'; - -const sourcePath = process.argv[2] ?? '../pi/packages/ai/src/image-models.generated.ts'; -const outputPath = process.argv[3] ?? 'pkg/ai/image_models_generated.go'; -const input = fs.readFileSync(sourcePath, 'utf8'); -let code = input - .replace(/^\s*\/\/.*$/gm, '') - .replace(/import type[^;]+;\s*/g, '') - .replace(/export const IMAGE_MODELS\s*=\s*/, 'const IMAGE_MODELS = ') - .replace(/\s+satisfies\s+ImagesModel<[^>]+>/g, '') - .replace(/\s+as const satisfies Record>>;?/, ';'); -code += '\nIMAGE_MODELS;'; -const models = vm.runInNewContext(code, {}); -const json = JSON.stringify(models).replace(/`/g, '`+"`"+`'); -const providerOrder = Object.keys(models); -const idOrder = Object.fromEntries(Object.entries(models).map(([provider, providerModels]) => [provider, Object.keys(providerModels)])); -const providerOrderGo = providerOrder.map((provider) => `\t${JSON.stringify(provider)},`).join('\n'); -const idOrderJSON = JSON.stringify(idOrder).replace(/`/g, '`+"`"+`'); -const out = `package ai\n\nimport \"encoding/json\"\n\nvar imageModelsJSON = \`${json}\`\n\nvar imageModelProviderOrder = []ImagesProvider{\n${providerOrderGo}\n}\n\nvar imageModelIDOrderJSON = \`${idOrderJSON}\`\n\nvar ImageModels = mustLoadImageModels()\nvar imageModelIDOrder = mustLoadImageModelIDOrder()\n\nfunc mustLoadImageModels() map[ImagesProvider]map[string]ImagesModel {\n\tvar raw map[ImagesProvider]map[string]ImagesModel\n\tif err := json.Unmarshal([]byte(imageModelsJSON), &raw); err != nil {\n\t\tpanic(err)\n\t}\n\treturn raw\n}\n\nfunc mustLoadImageModelIDOrder() map[ImagesProvider][]string {\n\tvar raw map[ImagesProvider][]string\n\tif err := json.Unmarshal([]byte(imageModelIDOrderJSON), &raw); err != nil {\n\t\tpanic(err)\n\t}\n\treturn raw\n}\n`; -fs.writeFileSync(outputPath, out); From fc39119e6098f7738c48f628050e3244de331d4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 21:38:31 +0200 Subject: [PATCH 14/22] nuke some code just use ai services dont duplicate ffs --- pkg/connector/bridge_commands.go | 8 +- pkg/connector/contacts.go | 86 ++++--------- pkg/connector/login.go | 168 +++++++++----------------- pkg/connector/provider.go | 35 +++--- pkg/connector/provider_lifecycle.go | 74 +++++++++--- pkg/connector/provider_routes.go | 8 +- pkg/connector/provider_test.go | 94 +++++--------- pkg/connector/room_state.go | 8 +- pkg/connector/slash_commands_model.go | 4 - 9 files changed, 191 insertions(+), 294 deletions(-) diff --git a/pkg/connector/bridge_commands.go b/pkg/connector/bridge_commands.go index 955989d3..963b7dd6 100644 --- a/pkg/connector/bridge_commands.go +++ b/pkg/connector/bridge_commands.go @@ -223,14 +223,14 @@ func (c *Connector) handleBridgeProviderUpsert(ce *commands.Event, action string if len(fields) == 5 { input.DefaultModel = fields[4] } - provider, err := c.VerifyProviderConfig(ce.Ctx, input) + login, err := c.loginForBridgeProviderCommand(ce) if err != nil { - ce.Reply("Provider rejected: %v", err) + ce.Reply(err.Error()) return } - login, err := c.loginForBridgeProviderCommand(ce) + provider, err := c.VerifyProviderConfig(ce.Ctx, login, input) if err != nil { - ce.Reply(err.Error()) + ce.Reply("Provider rejected: %v", err) return } if err = c.SaveProviderConfig(ce.Ctx, login, provider); err != nil { diff --git a/pkg/connector/contacts.go b/pkg/connector/contacts.go index ba0416d1..697e3b34 100644 --- a/pkg/connector/contacts.go +++ b/pkg/connector/contacts.go @@ -168,22 +168,6 @@ func (cl *Client) modelContacts(ctx context.Context, query string) []*bridgev2.R seen[contact.UserID] = true filtered = append(filtered, contact) } - providers := cl.providers() - for _, provider := range providers { - if !providerAllowsArbitraryModels(provider) { - continue - } - model, ok := arbitraryModelForProvider(provider, query) - if !ok { - continue - } - userID := aiid.ModelContactID(provider.ID, model.ID) - if seen[userID] { - continue - } - seen[userID] = true - filtered = append(filtered, modelContactWithGhost(ctx, cl.bridge(), provider, model)) - } return filtered } @@ -309,9 +293,6 @@ func (cl *Client) providerWithCatalogModels(ctx context.Context, provider aiid.P } func (cl *Client) providerForModelContacts(ctx context.Context, provider aiid.ProviderConfig, refresh bool) (aiid.ProviderConfig, bool) { - if provider.ID != aiid.DefaultProvider { - return provider, true - } refreshed, err := cl.providerWithCatalogModelsStrictWithRefresh(ctx, provider, refresh) if err != nil { zerolog.Ctx(ctx).Warn(). @@ -336,12 +317,6 @@ func (cl *Client) providerWithCatalogModelsStrict(ctx context.Context, provider } func (cl *Client) providerWithCatalogModelsStrictWithRefresh(ctx context.Context, provider aiid.ProviderConfig, refresh bool) (aiid.ProviderConfig, error) { - if provider.ID != aiid.DefaultProvider { - return provider, nil - } - if len(provider.Models) > 0 && !refresh { - return provider, nil - } models, err := cl.cachedAIServicesCatalogModels(ctx, provider, refresh) if err != nil { return provider, err @@ -402,10 +377,17 @@ func listedProviderModelContacts(ctx context.Context, br *bridgev2.Bridge, provi } func (cl *Client) aiServicesCatalogModels(ctx context.Context, provider aiid.ProviderConfig) ([]ai.Model, error) { - if cl == nil || cl.Main == nil || provider.ID != aiid.DefaultProvider || provider.BaseURL == "" || cl.Main.AppServiceToken == "" { + if cl == nil || cl.Main == nil || cl.UserLogin == nil || cl.Main.AppServiceToken == "" { return nil, nil } - modelsURL, err := aiServicesModelsURL(provider.BaseURL) + catalogProvider := provider + if provider.ID != aiid.DefaultProvider { + catalogProvider = cl.Main.defaultProviderConfig(cl.UserLogin.UserMXID) + if catalogProvider.BaseURL == "" { + return nil, fmt.Errorf("AI Services is not available for %s", cl.UserLogin.UserMXID.Homeserver()) + } + } + modelsURL, err := aiServicesModelsURL(catalogProvider.BaseURL) if err != nil { return nil, err } @@ -433,6 +415,9 @@ func (cl *Client) aiServicesCatalogModels(ctx context.Context, provider aiid.Pro } models := make([]ai.Model, 0, len(body.Data)) for _, item := range body.Data { + if provider.ID != aiid.DefaultProvider && !item.matchesProvider(provider.Provider) { + continue + } modelID := strings.TrimSpace(item.ID) if modelID == "" { continue @@ -454,7 +439,7 @@ func (cl *Client) aiServicesCatalogModels(ctx context.Context, provider aiid.Pro BuiltInTools: item.builtInTools(), Compat: item.compat(), } - model = item.applyRuntime(model, provider) + model = item.applyRuntime(model, provider, provider.ID == aiid.DefaultProvider) models = append(models, normalizeProviderModel(model, provider)) } return models, nil @@ -546,7 +531,14 @@ type aiServicesModelCompat struct { AllowEmptySignature *bool `json:"allowEmptySignature,omitempty"` } -func (entry aiServicesModelEntry) applyRuntime(model ai.Model, provider aiid.ProviderConfig) ai.Model { +func (entry aiServicesModelEntry) matchesProvider(provider ai.Provider) bool { + if entry.Runtime == nil || entry.Runtime.Provider == "" { + return provider == "" + } + return ai.Provider(entry.Runtime.Provider) == provider +} + +func (entry aiServicesModelEntry) applyRuntime(model ai.Model, provider aiid.ProviderConfig, useRuntimeBaseURL bool) ai.Model { if entry.Runtime == nil { return model } @@ -563,7 +555,7 @@ func (entry aiServicesModelEntry) applyRuntime(model ai.Model, provider aiid.Pro model.Provider = ai.Provider(entry.Runtime.Provider) model.Compat["runtime_provider"] = entry.Runtime.Provider } - if entry.Runtime.BaseURL != "" { + if useRuntimeBaseURL && entry.Runtime.BaseURL != "" { model.BaseURL = aiServicesRuntimeBaseURL(provider.BaseURL, entry.Runtime.BaseURL) } entry.Runtime.Compat.applyTo(model.Compat) @@ -820,16 +812,6 @@ func resolveModelForProvider(provider aiid.ProviderConfig, identifier string) (a return model, true } } - if modelID, ok := strings.CutPrefix(identifier, provider.ID+"/"); ok { - if model, ok := arbitraryModelForProvider(provider, modelID); ok && providerAllowsArbitraryModels(provider) { - return model, true - } - } - if !strings.Contains(identifier, "/") && providerAllowsArbitraryModels(provider) { - if model, ok := arbitraryModelForProvider(provider, identifier); ok { - return model, true - } - } return ai.Model{}, false } @@ -883,30 +865,6 @@ func contactModels(provider aiid.ProviderConfig) []ai.Model { return []ai.Model{normalizeProviderModel(modelForProviderConfig(provider, provider.DefaultModel), provider)} } -func arbitraryModelForProvider(provider aiid.ProviderConfig, query string) (ai.Model, bool) { - modelID := strings.TrimSpace(query) - if modelID == "" { - return ai.Model{}, false - } - if stripped, ok := strings.CutPrefix(modelID, provider.ID+"/"); ok { - modelID = strings.TrimSpace(stripped) - } - if modelID == "" { - return ai.Model{}, false - } - model := normalizeProviderModel(modelForProviderConfig(provider, modelID), provider) - displayName := provider.DisplayName - if displayName == "" { - displayName = providerDisplayName(provider.ID) - } - model.Name = displayName + ": " + modelID - return model, true -} - -func providerAllowsArbitraryModels(provider aiid.ProviderConfig) bool { - return provider.ID != aiid.DefaultProvider -} - func cloneModelContacts(contacts []*bridgev2.ResolveIdentifierResponse) []*bridgev2.ResolveIdentifierResponse { if contacts == nil { return nil diff --git a/pkg/connector/login.go b/pkg/connector/login.go index 9cdf00e7..e4a5507a 100644 --- a/pkg/connector/login.go +++ b/pkg/connector/login.go @@ -2,12 +2,9 @@ package connector import ( "context" - "encoding/json" "fmt" - "net/http" "net/url" "strings" - "time" "github.com/rs/zerolog" "maunium.net/go/mautrix/bridgev2" @@ -22,6 +19,8 @@ const ( loginFlowOpenAIResponses = "openai-responses" loginFlowOpenAICompletions = "openai-completions" loginFlowOpenAICodexResponses = "openai-codex-responses" + loginFlowAnthropicMessages = "anthropic-messages" + loginFlowGoogleVertex = "google-vertex" loginFlowChatGPTDevice = "chatgpt-device" loginStepBeeper = "com.beeper.ai.login.beeper" loginStepProviderConfig = "com.beeper.ai.login.provider.config" @@ -46,6 +45,14 @@ func (c *Connector) GetLoginFlows() []bridgev2.LoginFlow { Name: "OpenAI Codex Responses", Description: "Add a provider using the OpenAI Codex Responses API", ID: loginFlowOpenAICodexResponses, + }, { + Name: "Anthropic", + Description: "Add a provider using the Anthropic Messages API", + ID: loginFlowAnthropicMessages, + }, { + Name: "Google Vertex", + Description: "Add a provider using the Google Vertex API", + ID: loginFlowGoogleVertex, }, { Name: "ChatGPT", Description: "Log in with ChatGPT using a browser device code", @@ -63,6 +70,10 @@ func (c *Connector) CreateLogin(ctx context.Context, user *bridgev2.User, flowID return &CustomProviderLogin{Main: c, User: user, config: providerLoginConfig{API: ai.ApiOpenAICompletions}}, nil case loginFlowOpenAICodexResponses: return &CustomProviderLogin{Main: c, User: user, config: providerLoginConfig{API: ai.ApiOpenAICodexResponses}}, nil + case loginFlowAnthropicMessages: + return &CustomProviderLogin{Main: c, User: user, config: providerLoginConfig{API: ai.ApiAnthropicMessages}}, nil + case loginFlowGoogleVertex: + return &CustomProviderLogin{Main: c, User: user, config: providerLoginConfig{API: ai.ApiGoogleVertex}}, nil case loginFlowChatGPTDevice: return &ChatGPTDeviceLogin{Main: c, User: user}, nil default: @@ -162,18 +173,24 @@ func (l *CustomProviderLogin) submitProviderConfig(ctx context.Context, input ma log.Err(err).Msg("Provider login config rejected") return nil, err } - models, err := fetchProviderModels(ctx, l.config.API, providerID, baseURL, apiKey) + provider, err := l.Main.providerConfigFromInput(ctx, l.User.MXID, ProviderInput{ + ID: providerID, + API: l.config.API, + BaseURL: baseURL, + APIKey: apiKey, + DefaultModel: "", + }) if err != nil { - log.Err(err).Msg("Failed to fetch provider models during login") + log.Err(err).Msg("Failed to load provider models during login") return nil, err } - log.Debug().Int("model_count", len(models)).Msg("Fetched provider models during login") - l.config.ProviderID = providerID - l.config.BaseURL = baseURL - l.config.APIKey = apiKey - l.config.Models = models - options := make([]string, 0, len(models)) - for _, model := range models { + log.Debug().Int("model_count", len(provider.Models)).Msg("Loaded provider models during login") + l.config.ProviderID = provider.ID + l.config.BaseURL = provider.BaseURL + l.config.APIKey = provider.APIKey + l.config.Models = provider.Models + options := make([]string, 0, len(provider.Models)) + for _, model := range provider.Models { options = append(options, model.ID) } if len(options) == 0 { @@ -255,7 +272,7 @@ type providerLoginConfig struct { func supportedProviderLoginAPI(api ai.Api) bool { switch api { - case ai.ApiOpenAIResponses, ai.ApiOpenAICompletions, ai.ApiOpenAICodexResponses: + case ai.ApiOpenAIResponses, ai.ApiOpenAICompletions, ai.ApiOpenAICodexResponses, ai.ApiAnthropicMessages, ai.ApiGoogleVertex: return true default: return false @@ -263,7 +280,16 @@ func supportedProviderLoginAPI(api ai.Api) bool { } func defaultBaseURLForAPI(api ai.Api) string { - return "https://api.openai.com/v1" + switch api { + case ai.ApiAnthropicMessages: + return "https://api.anthropic.com" + case ai.ApiGoogleVertex: + return "https://aiplatform.googleapis.com" + case ai.ApiOpenAICompletions, ai.ApiOpenAIResponses, ai.ApiOpenAICodexResponses: + return "https://api.openai.com/v1" + default: + return "" + } } func providerDisplayName(providerID string) string { @@ -272,9 +298,11 @@ func providerDisplayName(providerID string) string { return "Provider" } known := map[string]string{ - "openai": "OpenAI", - "openrouter": "OpenRouter", - "lmstudio": "LM Studio", + "anthropic": "Anthropic", + "google-vertex": "Google Vertex", + "openai": "OpenAI", + "openrouter": "OpenRouter", + "vertex": "Google Vertex", } if name, ok := known[strings.ToLower(providerID)]; ok { return name @@ -291,98 +319,6 @@ func providerDisplayName(providerID string) string { return strings.Join(parts, " ") } -func fetchProviderModels(ctx context.Context, api ai.Api, providerID string, baseURL string, apiKey string) ([]ai.Model, error) { - if !supportedProviderLoginAPI(api) { - return nil, fmt.Errorf("unsupported API type %s", api) - } - modelURL, err := url.JoinPath(baseURL, "models") - if err != nil { - return nil, fmt.Errorf("invalid base URL: %w", err) - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, modelURL, nil) - if err != nil { - return nil, err - } - if apiKey != "" { - req.Header.Set("Authorization", "Bearer "+resolveConfiguredAPIKey(apiKey)) - } - client := &http.Client{Timeout: 20 * time.Second} - logCtx := zerolog.Ctx(ctx).With(). - Str("action", "ai_provider_models_http"). - Str("provider_id", providerID). - Str("api", string(api)). - Str("method", http.MethodGet) - if parsed, parseErr := url.Parse(modelURL); parseErr == nil { - logCtx = logCtx.Str("url", parsed.Redacted()).Str("host", parsed.Host).Str("path", parsed.EscapedPath()) - } else { - logCtx = logCtx.Str("url", modelURL) - } - log := logCtx.Logger() - ctx = log.WithContext(ctx) - log.Trace().Msg("Sending provider models HTTP request") - started := time.Now() - resp, err := client.Do(req) - if err != nil { - log.Err(err).Dur("duration", time.Since(started)).Msg("Provider models HTTP request failed") - return nil, fmt.Errorf("failed to fetch models: %w", err) - } - defer resp.Body.Close() - logEvent := log.Debug() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - logEvent = log.Error() - } - logEvent.Dur("duration", time.Since(started)). - Int("status_code", resp.StatusCode). - Str("status", resp.Status). - Int64("response_content_length", resp.ContentLength). - Str("response_content_type", resp.Header.Get("Content-Type")). - Msg("Received provider models HTTP response") - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("failed to fetch models: provider returned HTTP %d", resp.StatusCode) - } - var body aiServicesModelListResponse - if err = json.NewDecoder(resp.Body).Decode(&body); err != nil { - return nil, fmt.Errorf("failed to parse models response: %w", err) - } - models := make([]ai.Model, 0, len(body.Data)) - seen := map[string]bool{} - provider, inferredAPI := inferProviderRoute(providerID, baseURL) - if api != "" { - inferredAPI = api - } - providerConfig := aiid.ProviderConfig{ - ID: providerID, - API: inferredAPI, - Provider: configuredModelProvider(providerID, provider), - BaseURL: baseURL, - } - for _, item := range body.Data { - modelID := strings.TrimSpace(item.ID) - if modelID == "" || seen[modelID] { - continue - } - seen[modelID] = true - model := ai.Model{ - ID: modelID, - Name: item.Name, - API: inferredAPI, - Provider: providerConfig.Provider, - BaseURL: baseURL, - Reasoning: item.reasoning(), - Input: item.inputModalities(), - ContextWindow: item.contextWindow(), - MaxTokens: item.maxTokens(), - } - model = item.applyRuntime(model, providerConfig) - models = append(models, normalizeProviderModel(model, providerConfig)) - } - if len(models) == 0 { - return nil, fmt.Errorf("provider returned no models") - } - log.Debug().Int("model_count", len(models)).Msg("Parsed provider models") - return models, nil -} - func customProviderConfig(providerID string, displayName string, baseURL string, apiKey string, defaultModel string, modelList string) aiid.ProviderConfig { provider, api := inferProviderRoute(providerID, baseURL) modelIDs := providerModelIDs(modelList, defaultModel) @@ -404,7 +340,16 @@ func inferProviderRoute(providerID string, baseURL string) (ai.Provider, ai.Api) if providerID == string(ai.ProviderOpenRouter) || strings.Contains(baseURL, "openrouter.ai") { return ai.ProviderOpenRouter, ai.ApiOpenAICompletions } - return ai.ProviderOpenAI, ai.ApiOpenAIResponses + if providerID == string(ai.ProviderAnthropic) || strings.Contains(baseURL, "anthropic.com") { + return ai.ProviderAnthropic, ai.ApiAnthropicMessages + } + if providerID == string(ai.ProviderGoogleVertex) || providerID == "vertex" || strings.Contains(baseURL, "aiplatform.googleapis.com") { + return ai.ProviderGoogleVertex, ai.ApiGoogleVertex + } + if providerID == string(ai.ProviderOpenAI) || strings.Contains(baseURL, "api.openai.com") { + return ai.ProviderOpenAI, ai.ApiOpenAIResponses + } + return "", "" } func providerModels(modelList string, defaultModel string, providerID string, baseURL string) []ai.Model { @@ -452,9 +397,6 @@ func providerModelsFromIDs(modelIDs []string, providerID string, provider ai.Pro } func configuredModelProvider(providerID string, provider ai.Provider) ai.Provider { - if providerID != string(ai.ProviderOpenAI) && providerID != string(ai.ProviderOpenRouter) { - return ai.Provider(providerID) - } return provider } diff --git a/pkg/connector/provider.go b/pkg/connector/provider.go index b70825d8..2ede84cc 100644 --- a/pkg/connector/provider.go +++ b/pkg/connector/provider.go @@ -10,6 +10,7 @@ import ( "time" "github.com/rs/zerolog" + "maunium.net/go/mautrix/id" "github.com/beeper/ai-bridge/pkg/agent/harness" ai "github.com/beeper/ai-bridge/pkg/ai" @@ -49,22 +50,14 @@ func (cl *Client) resolveProvider(ctx context.Context, roomConfig RoomConfig) (a log.Err(err).Msg("Failed to resolve AI provider") return aiid.ProviderConfig{}, "", err } - if provider.ID != aiid.DefaultProvider { - log.Debug(). - Str("provider_id", provider.ID). - Str("provider", string(provider.Provider)). - Str("model_id", modelID). - Msg("Resolved AI provider") - return provider, modelID, nil - } provider, err = cl.providerWithCatalogModelsStrict(ctx, provider) if err != nil { - log.Err(err).Str("provider_id", provider.ID).Msg("Failed to load default AI provider model catalog") + log.Err(err).Str("provider_id", provider.ID).Msg("Failed to load AI provider model catalog") return aiid.ProviderConfig{}, "", err } if len(provider.Models) == 0 { - err := fmt.Errorf("Beeper AI model catalog is unavailable") - log.Err(err).Str("provider_id", provider.ID).Msg("Default AI provider model catalog is empty") + err := fmt.Errorf("AI model catalog is unavailable for provider %s", provider.ID) + log.Err(err).Str("provider_id", provider.ID).Msg("AI provider model catalog is empty") return aiid.ProviderConfig{}, "", err } if resolvedModelID, ok := resolveProviderModelID(provider, modelID); ok { @@ -175,15 +168,22 @@ func (cl *Client) defaultProviderBearerToken() (string, error) { if cl == nil || cl.Main == nil { return "", fmt.Errorf("missing connector for default provider") } - if cl.Main.AppServiceToken == "" { + if cl.UserLogin == nil { + return "", fmt.Errorf("missing user login for default provider") + } + return cl.Main.defaultProviderBearerToken(cl.UserLogin.UserMXID) +} + +func (c *Connector) defaultProviderBearerToken(userMXID id.UserID) (string, error) { + if c == nil || c.AppServiceToken == "" { return "", fmt.Errorf("missing appservice token for default provider") } - username := cl.defaultProviderUsername() + username := userMXID.Localpart() if username == "" { return "", fmt.Errorf("missing Beeper username for default provider") } payload, err := json.Marshal(aiServicesAppserviceToken{ - ASToken: cl.Main.AppServiceToken, + ASToken: c.AppServiceToken, Username: username, }) if err != nil { @@ -192,13 +192,6 @@ func (cl *Client) defaultProviderBearerToken() (string, error) { return aiServicesAppserviceTokenPrefix + base64.RawURLEncoding.EncodeToString(payload), nil } -func (cl *Client) defaultProviderUsername() string { - if cl == nil || cl.UserLogin == nil || cl.UserLogin.UserLogin == nil { - return "" - } - return cl.UserLogin.UserMXID.Localpart() -} - func (cl *Client) refreshProviderIfNeeded(ctx context.Context, provider aiid.ProviderConfig) (aiid.ProviderConfig, error) { if provider.ID != chatGPTProviderID || provider.RefreshToken == "" || provider.ExpiresAtMS == 0 { return provider, nil diff --git a/pkg/connector/provider_lifecycle.go b/pkg/connector/provider_lifecycle.go index 472d5f2f..a1402766 100644 --- a/pkg/connector/provider_lifecycle.go +++ b/pkg/connector/provider_lifecycle.go @@ -10,6 +10,8 @@ import ( "strings" "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/id" ai "github.com/beeper/ai-bridge/pkg/ai" "github.com/beeper/ai-bridge/pkg/aiid" @@ -40,7 +42,14 @@ type ProviderResponse struct { ReadOnly bool `json:"read_only,omitempty"` } -func (c *Connector) VerifyProviderConfig(ctx context.Context, input ProviderInput) (aiid.ProviderConfig, error) { +func (c *Connector) VerifyProviderConfig(ctx context.Context, login *bridgev2.UserLogin, input ProviderInput) (aiid.ProviderConfig, error) { + if login == nil { + return aiid.ProviderConfig{}, fmt.Errorf("login is required") + } + return c.providerConfigFromInput(ctx, login.UserMXID, input) +} + +func (c *Connector) providerConfigFromInput(ctx context.Context, userMXID id.UserID, input ProviderInput) (aiid.ProviderConfig, error) { input.ID = strings.TrimSpace(input.ID) input.DisplayName = strings.TrimSpace(input.DisplayName) input.BaseURL = normalizeResponsesBaseURL(strings.TrimSpace(input.BaseURL)) @@ -52,30 +61,23 @@ func (c *Connector) VerifyProviderConfig(ctx context.Context, input ProviderInpu if input.ID == aiid.DefaultProvider { return aiid.ProviderConfig{}, fmt.Errorf("provider %q is managed by Beeper AI", aiid.DefaultProvider) } + provider, inferredAPI := inferProviderRoute(input.ID, input.BaseURL) if input.API == "" { - provider, api := inferProviderRoute(input.ID, input.BaseURL) - input.API = api + input.API = inferredAPI if input.DisplayName == "" { input.DisplayName = providerDisplayName(string(provider)) } } + if !supportedProviderLoginAPI(input.API) || !providerAPIAllowed(provider, input.API) { + return aiid.ProviderConfig{}, fmt.Errorf("provider %s with API %s is not supported", input.ID, input.API) + } if input.BaseURL == "" || input.APIKey == "" { return aiid.ProviderConfig{}, fmt.Errorf("base_url and api_key are required") } - models, err := fetchProviderModels(ctx, input.API, input.ID, input.BaseURL, input.APIKey) - if err != nil { - return aiid.ProviderConfig{}, err - } - if input.DefaultModel == "" { - input.DefaultModel = models[0].ID - } else if !providerHasModel(aiid.ProviderConfig{Models: models}, input.DefaultModel) { - return aiid.ProviderConfig{}, fmt.Errorf("model %s was not returned by provider %s", input.DefaultModel, input.ID) - } - provider, _ := inferProviderRoute(input.ID, input.BaseURL) if input.DisplayName == "" { input.DisplayName = providerDisplayName(input.ID) } - return aiid.ProviderConfig{ + providerConfig := aiid.ProviderConfig{ ID: input.ID, DisplayName: input.DisplayName, API: input.API, @@ -83,8 +85,48 @@ func (c *Connector) VerifyProviderConfig(ctx context.Context, input ProviderInpu BaseURL: input.BaseURL, APIKey: input.APIKey, DefaultModel: input.DefaultModel, - Models: models, - }, nil + } + models, err := c.aiServicesCatalogModelsForUserProvider(ctx, userMXID, providerConfig) + if err != nil { + return aiid.ProviderConfig{}, err + } + if len(models) == 0 { + return aiid.ProviderConfig{}, fmt.Errorf("AI Services catalog has no models for provider %s", providerConfig.Provider) + } + providerConfig.Models = models + if providerConfig.DefaultModel == "" { + providerConfig.DefaultModel = models[0].ID + } else if resolved, ok := resolveProviderModelID(aiid.ProviderConfig{Models: models}, providerConfig.DefaultModel); ok { + providerConfig.DefaultModel = resolved + } else { + return aiid.ProviderConfig{}, fmt.Errorf("model %s was not returned by AI Services for provider %s", providerConfig.DefaultModel, input.ID) + } + return providerConfig, nil +} + +func (c *Connector) aiServicesCatalogModelsForUserProvider(ctx context.Context, userMXID id.UserID, provider aiid.ProviderConfig) ([]ai.Model, error) { + client := &Client{ + Main: c, + UserLogin: &bridgev2.UserLogin{ + UserLogin: &database.UserLogin{UserMXID: userMXID}, + }, + } + return client.aiServicesCatalogModels(ctx, provider) +} + +func providerAPIAllowed(provider ai.Provider, api ai.Api) bool { + switch provider { + case ai.ProviderOpenAI: + return api == ai.ApiOpenAIResponses || api == ai.ApiOpenAICompletions || api == ai.ApiOpenAICodexResponses + case ai.ProviderOpenRouter: + return api == ai.ApiOpenAICompletions + case ai.ProviderAnthropic: + return api == ai.ApiAnthropicMessages + case ai.ProviderGoogleVertex: + return api == ai.ApiGoogleVertex + default: + return false + } } func (c *Connector) SaveProviderConfig(ctx context.Context, login *bridgev2.UserLogin, provider aiid.ProviderConfig) error { diff --git a/pkg/connector/provider_routes.go b/pkg/connector/provider_routes.go index ef50f9ef..cc6606c3 100644 --- a/pkg/connector/provider_routes.go +++ b/pkg/connector/provider_routes.go @@ -96,14 +96,14 @@ func (c *Connector) handleProviderUpsert(w http.ResponseWriter, r *http.Request, } input.ID = routeProviderID } - provider, err := c.VerifyProviderConfig(r.Context(), input) + login, err := c.loginForProviderRequest(r.Context(), provisioning.GetUser(r), r.URL.Query().Get("login_id")) if err != nil { - writeProviderError(w, http.StatusBadRequest, err) + writeProviderError(w, providerErrorStatus(err), err) return } - login, err := c.loginForProviderRequest(r.Context(), provisioning.GetUser(r), r.URL.Query().Get("login_id")) + provider, err := c.VerifyProviderConfig(r.Context(), login, input) if err != nil { - writeProviderError(w, providerErrorStatus(err), err) + writeProviderError(w, http.StatusBadRequest, err) return } if err = c.SaveProviderConfig(r.Context(), login, provider); err != nil { diff --git a/pkg/connector/provider_test.go b/pkg/connector/provider_test.go index 645fc76e..f6ebe2bb 100644 --- a/pkg/connector/provider_test.go +++ b/pkg/connector/provider_test.go @@ -647,69 +647,41 @@ func TestProviderModelsParsesOptionalModelList(t *testing.T) { } } -func TestFetchProviderModelsVerifiesAndBuildsModels(t *testing.T) { - var gotAuth string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v1/models" { - t.Fatalf("unexpected path %s", r.URL.Path) - } - gotAuth = r.Header.Get("Authorization") - _, _ = w.Write([]byte(`{"data":[{"id":"model-a"},{"id":"model-b"},{"id":"model-a"}]}`)) - })) - defer server.Close() - - models, err := fetchProviderModels(context.Background(), ai.ApiOpenAIResponses, "local", server.URL+"/v1", "key") - if err != nil { - t.Fatal(err) - } - if gotAuth != "Bearer key" { - t.Fatalf("unexpected auth header %q", gotAuth) - } - if len(models) != 2 || models[0].ID != "model-a" || models[1].ID != "model-b" { - t.Fatalf("unexpected models %#v", models) - } - if models[0].API != ai.ApiOpenAIResponses || models[0].Provider != "local" || models[0].BaseURL != server.URL+"/v1" { - t.Fatalf("unexpected model route %#v", models[0]) - } -} - -func TestFetchProviderModelsRespectsPublishedProviderRoutes(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{"data":[ - {"id":"claude-sonnet-4-5","name":"Claude Sonnet 4.5","runtime":{"provider":"anthropic","model":"claude-sonnet-4-5","api":"anthropic-messages","baseUrl":"/proxy/anthropic"}}, - {"id":"gemini-2.5-flash-lite","name":"Gemini 2.5 Flash Lite","runtime":{"provider":"google-vertex","model":"gemini-2.5-flash-lite","api":"google-vertex","baseUrl":"/proxy/vertex"}}, - {"id":"google/gemini-2.5-flash","name":"Gemini 2.5 Flash","runtime":{"provider":"google-vertex","model":"gemini-2.5-flash","api":"google-vertex","baseUrl":"/proxy/vertex"}} - ]}`)) - })) - defer server.Close() - - models, err := fetchProviderModels(context.Background(), ai.ApiOpenAIResponses, "local", server.URL, "key") - if err != nil { - t.Fatal(err) - } - byID := map[string]ai.Model{} - for _, model := range models { - byID[model.ID] = model - } - if got := byID["claude-sonnet-4-5"]; got.API != ai.ApiAnthropicMessages || got.Provider != ai.ProviderAnthropic || got.BaseURL != server.URL+"/proxy/anthropic" { - t.Fatalf("unexpected Anthropic route %#v", got) - } - if got := byID["gemini-2.5-flash-lite"]; got.API != ai.ApiGoogleVertex || got.Provider != ai.ProviderGoogleVertex || got.BaseURL != server.URL+"/proxy/vertex" { - t.Fatalf("unexpected Vertex route %#v", got) +func TestCatalogRuntimeForCustomProviderUsesCustomBaseURL(t *testing.T) { + entry := aiServicesModelEntry{ + ID: "z-ai/glm-4.5v", + Name: "GLM 4.5V", + Runtime: &struct { + API string `json:"api"` + Provider string `json:"provider"` + BaseURL string `json:"baseUrl"` + Model string `json:"model"` + Compat *aiServicesModelCompat `json:"compat"` + }{ + API: string(ai.ApiOpenAICompletions), + Provider: string(ai.ProviderOpenRouter), + BaseURL: "/proxy/openrouter/v1", + Model: "z-ai/glm-4.5v", + }, } - if got := byID["google/gemini-2.5-flash"]; got.API != ai.ApiGoogleVertex || got.Provider != ai.ProviderGoogleVertex || got.BaseURL != server.URL+"/proxy/vertex" { - t.Fatalf("unexpected legacy Google route %#v", got) + provider := aiid.ProviderConfig{ + ID: "openrouter", + API: ai.ApiOpenAICompletions, + Provider: ai.ProviderOpenRouter, + BaseURL: "https://openrouter.ai/api/v1", } -} - -func TestFetchProviderModelsRejectsFailedVerification(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "bad key", http.StatusUnauthorized) - })) - defer server.Close() - - if _, err := fetchProviderModels(context.Background(), ai.ApiOpenAICompletions, "local", server.URL, "bad"); err == nil { - t.Fatalf("expected failed verification to return an error") + model := entry.applyRuntime(ai.Model{ + ID: entry.ID, + Name: entry.Name, + API: provider.API, + Provider: provider.Provider, + BaseURL: provider.BaseURL, + }, provider, false) + if model.API != ai.ApiOpenAICompletions || model.Provider != ai.ProviderOpenRouter || model.BaseURL != provider.BaseURL { + t.Fatalf("unexpected custom provider runtime %#v", model) + } + if model.Compat["runtime_model"] != "z-ai/glm-4.5v" || model.Compat["runtime_provider"] != string(ai.ProviderOpenRouter) { + t.Fatalf("expected runtime metadata, got %#v", model.Compat) } } diff --git a/pkg/connector/room_state.go b/pkg/connector/room_state.go index 7b45840e..f9cb9076 100644 --- a/pkg/connector/room_state.go +++ b/pkg/connector/room_state.go @@ -139,9 +139,6 @@ func (c *Connector) ResolveProvider(ctx context.Context, login *bridgev2.UserLog if resolvedModelID, ok := resolveProviderModelID(provider, modelID); ok { return provider, resolvedModelID, nil } - if !providerAllowsModel(provider, modelID) { - return aiid.ProviderConfig{}, "", fmt.Errorf("model %s is not available for provider %s", modelID, providerID) - } return provider, modelID, nil } @@ -149,10 +146,7 @@ func providerAllowsModel(provider aiid.ProviderConfig, modelID string) bool { if _, ok := resolveProviderModelID(provider, modelID); ok { return true } - if len(provider.Models) > 0 { - return false - } - return strings.TrimSpace(modelID) != "" + return false } func providerHasModel(provider aiid.ProviderConfig, modelID string) bool { diff --git a/pkg/connector/slash_commands_model.go b/pkg/connector/slash_commands_model.go index f71c6b01..5836778c 100644 --- a/pkg/connector/slash_commands_model.go +++ b/pkg/connector/slash_commands_model.go @@ -225,10 +225,6 @@ func (cl *Client) modelOptionsText(currentProvider aiid.ProviderConfig) string { options := []string{} for _, providerID := range providerIDs { provider := providers[providerID] - if len(provider.Models) == 0 && providerAllowsArbitraryModels(provider) { - options = append(options, fmt.Sprintf("`%s/`", provider.ID)) - continue - } for _, model := range contactModels(provider) { options = append(options, fmt.Sprintf("`%s/%s`", provider.ID, model.ID)) } From 3110ae67c657efb9ca4e995bf1f9283ff48076fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 21:40:35 +0200 Subject: [PATCH 15/22] wip --- README.md | 6 +-- pkg/connector/contacts_test.go | 22 ++++------ pkg/connector/provider_test.go | 78 +++++++++------------------------- 3 files changed, 32 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index f67a10fc..4e610037 100644 --- a/README.md +++ b/README.md @@ -317,9 +317,9 @@ Provider-specific behaviors worth knowing live in `pkg/ai/providers`: OpenAI *Co ## The model catalog -The Beeper AI model catalog is owned by ai-services. The bridge loads `/models?feature=bridge:ai`, applies each model's runtime metadata, and fails the default Beeper provider if ai-services does not return a catalog. There is no bridge-generated fallback catalog. +The model catalog is owned by ai-services. The bridge loads `/models?feature=bridge:ai`, applies each model's runtime metadata, and fails provider resolution if ai-services does not return a catalog. There is no bridge-generated fallback catalog. -Custom providers are intentionally simpler: the bridge uses the provider's `/models` response or the user's configured `ai.Model` entries. If a custom provider does not advertise detailed metadata, the bridge uses a conservative text-only model shape instead of consulting Beeper's catalog by model ID. +Custom providers use the same ai-services catalog for model metadata. The bridge filters that catalog to the supported provider runtime (`openai`, `openrouter`, `anthropic`, or `google-vertex`) and then uses the user's configured base URL and API key for execution. Arbitrary model IDs and generic OpenAI-compatible providers are not accepted. Reasoning levels form a ladder `off < minimal < low < medium < high < xhigh`; `ClampThinkingLevel` snaps a request to the nearest supported level from the model metadata the bridge was given. @@ -413,7 +413,7 @@ The bridge advertises five login flows (`pkg/connector/login.go`): | Flow | What it does | |------|--------------| | `beeper` | The default **Beeper AI** login. Loads its catalog and runtime proxy metadata from `ai-services.` derived from the user's homeserver; uses an appservice bearer token, no stored key. Read-only/managed. | -| `openai-responses` / `openai-completions` / `openai-codex-responses` | **Custom provider**: enter base URL + API key, the bridge fetches `/models`, you pick a default model. | +| `openai-responses` / `openai-completions` / `openai-codex-responses` / `anthropic-messages` / `google-vertex` | **Custom provider**: enter base URL + API key, the bridge loads matching model metadata from ai-services, then you pick a default model. | | `chatgpt-device` | **ChatGPT** OAuth device-code flow (PKCE). Stores access + refresh tokens, auto-refreshes within 2 min of expiry. | One Matrix user can hold multiple AI logins; there's a canonical "AI Chats" login per user. Provider configs (with secrets) live in `UserLoginMetadata.Providers`. API keys support `env:NAME` indirection. The `beeper` provider is special and **read-only** — it can't be added/updated/deleted. diff --git a/pkg/connector/contacts_test.go b/pkg/connector/contacts_test.go index dca6c8f6..8a30b155 100644 --- a/pkg/connector/contacts_test.go +++ b/pkg/connector/contacts_test.go @@ -402,20 +402,18 @@ func TestResolveModelForProviderPreservesOpenAICatalogModelID(t *testing.T) { } } -func TestResolveModelForProviderAcceptsArbitraryCustomModelID(t *testing.T) { +func TestResolveModelForProviderRejectsUnlistedCustomModelID(t *testing.T) { provider := aiid.ProviderConfig{ ID: "custom-openai", Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses, Models: []ai.Model{{ID: "gpt-5.5", Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses}}, } - model, ok := resolveModelForProvider(provider, "custom-openai/openai/gpt-5.5") - if !ok || model.ID != "openai/gpt-5.5" { - t.Fatalf("expected arbitrary model ID to resolve, got ok=%v model=%#v", ok, model) + if model, ok := resolveModelForProvider(provider, "custom-openai/openai/gpt-5.5"); ok { + t.Fatalf("expected unlisted model ID to be rejected, got %#v", model) } - model, ok = resolveModelForProvider(provider, "whateveristyped") - if !ok || model.ID != "whateveristyped" { - t.Fatalf("expected bare arbitrary model ID to resolve, got ok=%v model=%#v", ok, model) + if model, ok := resolveModelForProvider(provider, "whateveristyped"); ok { + t.Fatalf("expected arbitrary model ID to be rejected, got %#v", model) } } @@ -515,7 +513,7 @@ func TestSearchUsersFiltersModelContacts(t *testing.T) { } } -func TestSearchUsersAddsArbitraryModelContact(t *testing.T) { +func TestSearchUsersRejectsArbitraryModelContact(t *testing.T) { provider := aiid.ProviderConfig{ ID: "local", DisplayName: "Local", @@ -531,12 +529,8 @@ func TestSearchUsersAddsArbitraryModelContact(t *testing.T) { if err != nil { t.Fatal(err) } - if len(results) != 1 { - t.Fatalf("expected arbitrary model contact, got %#v", results) - } - name := results[0].UserInfo.Name - if name == nil || *name != "Local: whateveristyped" { - t.Fatalf("unexpected arbitrary model contact name %#v", results[0].UserInfo) + if len(results) != 0 { + t.Fatalf("expected no arbitrary model contact, got %#v", results) } } diff --git a/pkg/connector/provider_test.go b/pkg/connector/provider_test.go index f6ebe2bb..5e561335 100644 --- a/pkg/connector/provider_test.go +++ b/pkg/connector/provider_test.go @@ -4,8 +4,6 @@ import ( "context" "encoding/base64" "encoding/json" - "net/http" - "net/http/httptest" "strings" "testing" @@ -79,7 +77,7 @@ func TestModelForProviderFillsCustomListedModelInputFromCatalog(t *testing.T) { API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenAI, BaseURL: "https://custom.test/v1", - Models: []ai.Model{{ID: "gpt-5.5", Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses}}, + Models: []ai.Model{{ID: "gpt-5.5", Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses, Input: []string{"text", "image"}}}, } model := conn.ModelForProvider(provider, "gpt-5.5") if !isImageModel(model) { @@ -94,7 +92,7 @@ func TestModelForProviderFillsCustomPrefixedOpenAIInputFromCatalog(t *testing.T) API: ai.ApiOpenAIResponses, Provider: ai.ProviderOpenAI, BaseURL: "https://custom.test/v1", - Models: []ai.Model{{ID: "openai/gpt-5.5", Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses}}, + Models: []ai.Model{{ID: "openai/gpt-5.5", Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses, Input: []string{"text", "image"}}}, } model := conn.ModelForProvider(provider, "openai/gpt-5.5") if !isImageModel(model) { @@ -135,7 +133,7 @@ func TestTitleGenerationModelUsesDefaultMini(t *testing.T) { client := &Client{Main: conn} provider := conn.defaultProviderConfig("@alice:beeper-staging.com") model := client.titleGenerationModel(provider, ai.Model{ID: defaultBeeperAIModel}) - if model.ID != defaultTitleGenerationModel || model.Provider != ai.ProviderOpenAI || model.API != ai.ApiOpenAIResponses { + if model.ID != defaultBeeperAIModel { t.Fatalf("unexpected title model %#v", model) } } @@ -238,8 +236,8 @@ func TestDefaultProviderAuthRequiresBeeperUsername(t *testing.T) { }, } _, err := client.authForProvider(aiid.ProviderConfig{ID: aiid.DefaultProvider})(context.Background(), ai.Model{}) - if err == nil || !strings.Contains(err.Error(), "Beeper username") { - t.Fatalf("expected Beeper username error, got %v", err) + if err == nil || !strings.Contains(err.Error(), "user login") { + t.Fatalf("expected user login error, got %v", err) } } @@ -685,53 +683,12 @@ func TestCatalogRuntimeForCustomProviderUsesCustomBaseURL(t *testing.T) { } } -func TestCustomProviderLoginStagesAPIConfigAndDefaultModel(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{"data":[{"id":"gpt-test"},{"id":"gpt-other"}]}`)) - })) - defer server.Close() - +func TestCustomProviderLoginConfigStep(t *testing.T) { login := &CustomProviderLogin{config: providerLoginConfig{API: ai.ApiOpenAICompletions}} step := login.providerConfigStep() if step.StepID != loginStepProviderConfig || len(step.UserInputParams.Fields) != 3 { t.Fatalf("unexpected config step %#v", step) } - - step, err := login.submitProviderConfig(context.Background(), map[string]string{ - "provider_id": "local-ai", - "base_url": server.URL, - "api_key": "key", - }) - if err != nil { - t.Fatal(err) - } - if step.StepID != loginStepProviderDefault { - t.Fatalf("expected default model step, got %#v", step) - } - field := step.UserInputParams.Fields[0] - if field.Type != bridgev2.LoginInputFieldTypeSelect || field.DefaultValue != "gpt-test" || len(field.Options) != 2 { - t.Fatalf("unexpected default model field %#v", field) - } - if login.config.ProviderID != "local-ai" || login.config.API != ai.ApiOpenAICompletions || len(login.config.Models) != 2 { - t.Fatalf("login config was not retained: %#v", login.config) - } -} - -func TestCustomProviderLoginRejectsEmptyModelList(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte(`{"data":[]}`)) - })) - defer server.Close() - - login := &CustomProviderLogin{config: providerLoginConfig{API: ai.ApiOpenAICompletions}} - _, err := login.submitProviderConfig(context.Background(), map[string]string{ - "provider_id": "local-ai", - "base_url": server.URL, - "api_key": "key", - }) - if err == nil || !strings.Contains(err.Error(), "no models") { - t.Fatalf("expected empty model list error, got %v", err) - } } func TestModelForProviderAppliesRouteBaseURLToDefaultModel(t *testing.T) { @@ -754,7 +711,7 @@ func TestModelForProviderAppliesRouteBaseURLToDefaultModel(t *testing.T) { } } -func TestResolveProviderRequiresListedModelWhenModelListExists(t *testing.T) { +func TestResolveProviderDefersModelValidationToCatalogLoad(t *testing.T) { conn := &Connector{} provider := aiid.ProviderConfig{ ID: "custom", @@ -767,11 +724,14 @@ func TestResolveProviderRequiresListedModelWhenModelListExists(t *testing.T) { ID: "login", Metadata: &aiid.UserLoginMetadata{Providers: map[string]aiid.ProviderConfig{provider.ID: provider}}, }} - _, _, err := conn.ResolveProvider(context.Background(), login, RoomConfig{ProviderID: "custom", ModelID: "missing"}) - if err == nil { - t.Fatal("expected missing model to be rejected") + _, modelID, err := conn.ResolveProvider(context.Background(), login, RoomConfig{ProviderID: "custom", ModelID: "missing"}) + if err != nil { + t.Fatal(err) + } + if modelID != "missing" { + t.Fatalf("unexpected model ID %q", modelID) } - _, modelID, err := conn.ResolveProvider(context.Background(), login, RoomConfig{ProviderID: "custom", ModelID: "allowed"}) + _, modelID, err = conn.ResolveProvider(context.Background(), login, RoomConfig{ProviderID: "custom", ModelID: "allowed"}) if err != nil { t.Fatal(err) } @@ -864,8 +824,12 @@ func TestNormalizeProviderModelDoesNotInheritDefaultProviderCatalogMetadata(t *t func TestNormalizeProviderModelInheritsCustomProviderCatalogReasoningMetadata(t *testing.T) { model := normalizeProviderModel(ai.Model{ - ID: "gpt-5.5", - Provider: ai.ProviderOpenAI, + ID: "gpt-5.5", + Provider: ai.ProviderOpenAI, + Reasoning: true, + ThinkingLevelMap: map[ai.ModelThinkingLevel]*string{ + ai.ModelThinkingLevelOff: nil, + }, }, aiid.ProviderConfig{ ID: "custom-openai", Provider: ai.ProviderOpenAI, @@ -873,7 +837,7 @@ func TestNormalizeProviderModelInheritsCustomProviderCatalogReasoningMetadata(t BaseURL: "https://custom.test/v1", }) if !model.Reasoning || len(model.ThinkingLevelMap) == 0 { - t.Fatalf("expected custom provider model to inherit local catalog reasoning metadata, got %#v", model) + t.Fatalf("expected custom provider model to preserve catalog reasoning metadata, got %#v", model) } if roomThinkingLevelSupported(model, ai.ModelThinkingLevelOff) { t.Fatalf("expected normalized GPT-5.5 custom provider model to reject off reasoning") From 00ad8f248939b31bde09f17848669663c2436704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 21:43:06 +0200 Subject: [PATCH 16/22] Update login.go --- pkg/connector/login.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/connector/login.go b/pkg/connector/login.go index e4a5507a..968013b3 100644 --- a/pkg/connector/login.go +++ b/pkg/connector/login.go @@ -155,7 +155,7 @@ func (l *CustomProviderLogin) providerConfigStep() *bridgev2.LoginStep { Instructions: "Enter provider connection details", UserInputParams: &bridgev2.LoginUserInputParams{Fields: []bridgev2.LoginInputDataField{ {Type: bridgev2.LoginInputFieldTypeUsername, ID: "provider_id", Name: "ID", Description: "Stable provider ID"}, - {Type: bridgev2.LoginInputFieldTypeURL, ID: "base_url", Name: "Base URL", Description: "OpenAI-compatible API base URL", DefaultValue: defaultBaseURLForAPI(l.config.API)}, + {Type: bridgev2.LoginInputFieldTypeURL, ID: "base_url", Name: "Base URL", Description: "Provider API base URL", DefaultValue: defaultBaseURLForAPI(l.config.API)}, {Type: bridgev2.LoginInputFieldTypeToken, ID: "api_key", Name: "API key", Description: "Provider API key"}, }}, } From f43ee1929566de749dfdb7fe9a43118705542791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 22:00:32 +0200 Subject: [PATCH 17/22] wip --- pkg/ai-stream/run.go | 55 ++++++++++++++++++-- pkg/ai-stream/stream_test.go | 85 +++++++++++++++++++++++++++++++ pkg/connector/contacts.go | 45 +++++++++-------- pkg/connector/contacts_test.go | 18 +++---- pkg/connector/provider_test.go | 3 +- pkg/connector/stream_test.go | 92 +++++++++++++++++++++++++++++++++- 6 files changed, 261 insertions(+), 37 deletions(-) diff --git a/pkg/ai-stream/run.go b/pkg/ai-stream/run.go index e47f9445..991f5032 100644 --- a/pkg/ai-stream/run.go +++ b/pkg/ai-stream/run.go @@ -186,6 +186,7 @@ type Writer struct { textOpen map[int]bool textContentWritten bool textIndexOffset int + currentAssistantMessageID string reasoningMessages map[int]string reasoningOpen map[int]bool reasoningContent map[int]string @@ -233,7 +234,7 @@ func NewRun(runID, threadID, model, agentID, agentName string, now time.Time) *R func NewWriter(run *Run, now func() time.Time) *Writer { textIndexOffset := nextMessageOrdinal(run, messageOrdinalText) reasoningIndexOffset := nextMessageOrdinal(run, messageOrdinalReasoning) - return &Writer{ + writer := &Writer{ Run: run, builder: agui.NewEventBuilder(run.Model, now), textMessages: map[int]string{}, @@ -247,6 +248,8 @@ func NewWriter(run *Run, now func() time.Time) *Writer { lastAccountedChars: utf8.RuneCountInString(run.Text()), previewText: run.Text(), } + writer.currentAssistantMessageID = writer.textMessageID(0) + return writer } func (w *Writer) Add(evt agui.Event) { @@ -303,6 +306,7 @@ func (w *Writer) Thinking(delta string) { } func (w *Writer) ReasoningMessageStart(index int) string { + w.ensureCurrentAssistantMessage() w.ensureReasoningPhase() return w.ensureReasoningMessage(index) } @@ -345,11 +349,13 @@ func (w *Writer) ReasoningMessageEnd(index int) { } func (w *Writer) StepStart(stepID string) { - w.Add(w.builder.StepStarted(w.Run.MessageID, stepID)) + messageID := w.ensureCurrentAssistantMessage() + w.Add(w.builder.StepStarted(messageID, stepID)) } func (w *Writer) StepFinish(stepID string) { - w.Add(w.builder.StepFinished(w.Run.MessageID, stepID)) + messageID := w.ensureCurrentAssistantMessage() + w.Add(w.builder.StepFinished(messageID, stepID)) } func (w *Writer) ToolStart(toolCallID, name string, index int, approval *ToolApproval) { @@ -358,7 +364,7 @@ func (w *Writer) ToolStart(toolCallID, name string, index int, approval *ToolApp func (w *Writer) ToolStartWithMetadata(toolCallID, name string, index int, approval *ToolApproval, metadata map[string]any) { idx := index - w.Add(w.builder.ToolCallStartWithMetadata(w.Run.MessageID, toolCallID, name, &idx, metadata)) + w.Add(w.builder.ToolCallStartWithMetadata(w.currentAssistantMessage(), toolCallID, name, &idx, metadata)) if approval != nil { w.recordApprovalRequest(toolCallID, name, approval) } @@ -661,6 +667,7 @@ func (w *Writer) finishText() { func (w *Writer) ensureTextMessage(index int) string { w.initState() if messageID := w.textMessages[index]; messageID != "" { + w.currentAssistantMessageID = messageID if !w.textOpen[index] { w.Add(w.builder.TextMessageStart(messageID, agui.RoleAssistant)) w.textOpen[index] = true @@ -669,6 +676,34 @@ func (w *Writer) ensureTextMessage(index int) string { } messageID := w.textMessageID(index) w.textMessages[index] = messageID + w.currentAssistantMessageID = messageID + w.Add(w.builder.TextMessageStart(messageID, agui.RoleAssistant)) + w.textOpen[index] = true + return messageID +} + +func (w *Writer) currentAssistantMessage() string { + w.initState() + if w.currentAssistantMessageID == "" { + w.currentAssistantMessageID = w.textMessageID(0) + } + return w.currentAssistantMessageID +} + +func (w *Writer) ensureCurrentAssistantMessage() string { + messageID := w.currentAssistantMessage() + for index, existingID := range w.textMessages { + if existingID != messageID { + continue + } + if !w.textOpen[index] { + w.Add(w.builder.TextMessageStart(messageID, agui.RoleAssistant)) + w.textOpen[index] = true + } + return messageID + } + index := w.textIndexForMessageID(messageID) + w.textMessages[index] = messageID w.Add(w.builder.TextMessageStart(messageID, agui.RoleAssistant)) w.textOpen[index] = true return messageID @@ -717,6 +752,18 @@ func (w *Writer) reasoningMessageID(index int) string { return fmt.Sprintf("%s-reasoning-%d", w.Run.MessageID, w.reasoningIndexOffset+max(index, 0)) } +func (w *Writer) textIndexForMessageID(messageID string) int { + ordinal, ok := messageOrdinal(w.Run.MessageID, messageID, messageOrdinalText) + if !ok { + return 0 + } + index := ordinal - w.textIndexOffset + if index < 0 { + return 0 + } + return index +} + func (w *Writer) toolResultMessageID(toolCallID string) string { toolCallID = strings.TrimSpace(toolCallID) if toolCallID == "" { diff --git a/pkg/ai-stream/stream_test.go b/pkg/ai-stream/stream_test.go index 971fb743..2c6f6c1c 100644 --- a/pkg/ai-stream/stream_test.go +++ b/pkg/ai-stream/stream_test.go @@ -195,6 +195,78 @@ func TestWriterKeepsReasoningMessagesSeparate(t *testing.T) { } } +func TestWriterAnchorsContinuationReasoningBeforeTextToFreshAssistantMessage(t *testing.T) { + run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) + writer := NewWriter(run, func() time.Time { return time.Unix(10, 0) }) + writer.Start() + writer.Text("before ") + writer.ToolStart("tool-1", "ask_approval", 1, nil) + writer.AwaitToolUseWithUsage(nil) + + continuation := NewWriter(run, func() time.Time { return time.Unix(11, 0) }) + continuation.ReasoningMessageStart(0) + continuation.ReasoningDelta(0, "checking approval") + continuation.Text("after") + + var ordered []string + for _, evt := range run.Events { + switch evt.Type() { + case agui.EventTextMessageStart: + ordered = append(ordered, "text-start:"+asString(evt.Get("messageId"))) + case agui.EventReasoningMsgStart: + ordered = append(ordered, "thinking-start:"+asString(evt.Get("messageId"))) + case agui.EventReasoningMsgCont: + ordered = append(ordered, "thinking:"+asString(evt.Get("delta"))) + case agui.EventTextMessageContent: + if delta := asString(evt.Get("delta")); delta != "" { + ordered = append(ordered, "text:"+asString(evt.Get("messageId"))+":"+delta) + } + } + } + got := strings.Join(ordered, "|") + if !strings.Contains(got, "text-start:msg-run-1-text-1|thinking-start:msg-run-1-reasoning-0|thinking:checking approval|text:msg-run-1-text-1:after") { + t.Fatalf("continuation reasoning was not anchored to the fresh assistant message:\n%s", got) + } + + uiMessage := run.FinalBeeperAIMessage(0, true) + gotParts := uiPartSummary(uiMessage.Parts) + wantParts := []string{"text:before ", "tool-call:tool-1", "thinking:checking approval", "text:after"} + if !reflect.DeepEqual(gotParts, wantParts) { + t.Fatalf("continued reasoning UI parts mismatch\ngot: %#v\nwant: %#v\nparts: %#v", gotParts, wantParts, uiMessage.Parts) + } +} + +func TestWriterParentsContinuationToolsToCurrentAssistantMessage(t *testing.T) { + run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) + writer := NewWriter(run, func() time.Time { return time.Unix(10, 0) }) + writer.Start() + writer.Text("before ") + writer.ToolStart("tool-1", "ask_approval", 1, nil) + writer.AwaitToolUseWithUsage(nil) + + continuation := NewWriter(run, func() time.Time { return time.Unix(11, 0) }) + continuation.Text("after ") + continuation.ToolStart("tool-2", "second_tool", 1, nil) + + parentByTool := map[string]string{} + for _, evt := range run.Events { + if evt.Type() != agui.EventToolCallStart { + continue + } + parentByTool[asString(evt.Get("toolCallId"))] = asString(evt.Get("parentMessageId")) + } + if parentByTool["tool-2"] != "msg-run-1-text-1" { + t.Fatalf("second continuation tool used wrong parent: %#v", parentByTool) + } + + uiMessage := run.FinalBeeperAIMessage(0, true) + gotParts := uiPartSummary(uiMessage.Parts) + wantParts := []string{"text:before ", "tool-call:tool-1", "text:after ", "tool-call:tool-2"} + if !reflect.DeepEqual(gotParts, wantParts) { + t.Fatalf("continued tool UI parts mismatch\ngot: %#v\nwant: %#v\nparts: %#v", gotParts, wantParts, uiMessage.Parts) + } +} + func TestInterleavedReasoningContentStaysSeparateInFinalProjections(t *testing.T) { run := NewRun("run-1", "thread-1", DefaultModel, "ai", "AI", time.Unix(10, 0)) builder := agui.NewEventBuilder(DefaultModel, func() time.Time { return time.Unix(10, 0) }) @@ -1033,3 +1105,16 @@ func TestApprovalQueueKeepsOneActiveInterruptAndTimeouts(t *testing.T) { t.Fatalf("bad timed-out tool result: %#v", result) } } + +func uiPartSummary(parts []MessagePart) []string { + out := make([]string, 0, len(parts)) + for _, part := range parts { + switch part["type"] { + case "text", "thinking": + out = append(out, fmt.Sprintf("%s:%s", part["type"], part["content"])) + case "tool-call": + out = append(out, fmt.Sprintf("tool-call:%s", firstString(part["toolCallId"], part["id"]))) + } + } + return out +} diff --git a/pkg/connector/contacts.go b/pkg/connector/contacts.go index 697e3b34..5de3f294 100644 --- a/pkg/connector/contacts.go +++ b/pkg/connector/contacts.go @@ -478,8 +478,9 @@ type aiServicesModelEntry struct { Runtime *struct { API string `json:"api"` Provider string `json:"provider"` - BaseURL string `json:"baseUrl"` + BaseURL string `json:"base_url"` Model string `json:"model"` + Endpoint string `json:"endpoint"` Compat *aiServicesModelCompat `json:"compat"` } `json:"runtime"` Capabilities *struct { @@ -508,27 +509,27 @@ type aiServicesModelEntry struct { } type aiServicesModelCompat struct { - SupportsStore *bool `json:"supportsStore,omitempty"` - SupportsDeveloperRole *bool `json:"supportsDeveloperRole,omitempty"` - SupportsReasoningEffort *bool `json:"supportsReasoningEffort,omitempty"` - SupportsUsageInStreaming *bool `json:"supportsUsageInStreaming,omitempty"` - MaxTokensField string `json:"maxTokensField,omitempty"` - RequiresToolResultName *bool `json:"requiresToolResultName,omitempty"` - RequiresAssistantAfterToolResult *bool `json:"requiresAssistantAfterToolResult,omitempty"` - RequiresThinkingAsText *bool `json:"requiresThinkingAsText,omitempty"` - RequiresReasoningContentOnAssistantMessages *bool `json:"requiresReasoningContentOnAssistantMessages,omitempty"` - ThinkingFormat string `json:"thinkingFormat,omitempty"` - ZaiToolStream *bool `json:"zaiToolStream,omitempty"` - SupportsStrictMode *bool `json:"supportsStrictMode,omitempty"` - CacheControlFormat string `json:"cacheControlFormat,omitempty"` - SendSessionAffinityHeaders *bool `json:"sendSessionAffinityHeaders,omitempty"` - SupportsLongCacheRetention *bool `json:"supportsLongCacheRetention,omitempty"` - SendSessionIDHeader *bool `json:"sendSessionIdHeader,omitempty"` - SupportsEagerToolInputStreaming *bool `json:"supportsEagerToolInputStreaming,omitempty"` - SupportsCacheControlOnTools *bool `json:"supportsCacheControlOnTools,omitempty"` - SupportsTemperature *bool `json:"supportsTemperature,omitempty"` - ForceAdaptiveThinking *bool `json:"forceAdaptiveThinking,omitempty"` - AllowEmptySignature *bool `json:"allowEmptySignature,omitempty"` + SupportsStore *bool `json:"supports_store,omitempty"` + SupportsDeveloperRole *bool `json:"supports_developer_role,omitempty"` + SupportsReasoningEffort *bool `json:"supports_reasoning_effort,omitempty"` + SupportsUsageInStreaming *bool `json:"supports_usage_in_streaming,omitempty"` + MaxTokensField string `json:"max_tokens_field,omitempty"` + RequiresToolResultName *bool `json:"requires_tool_result_name,omitempty"` + RequiresAssistantAfterToolResult *bool `json:"requires_assistant_after_tool_result,omitempty"` + RequiresThinkingAsText *bool `json:"requires_thinking_as_text,omitempty"` + RequiresReasoningContentOnAssistantMessages *bool `json:"requires_reasoning_content_on_assistant_messages,omitempty"` + ThinkingFormat string `json:"thinking_format,omitempty"` + ZaiToolStream *bool `json:"zai_tool_stream,omitempty"` + SupportsStrictMode *bool `json:"supports_strict_mode,omitempty"` + CacheControlFormat string `json:"cache_control_format,omitempty"` + SendSessionAffinityHeaders *bool `json:"send_session_affinity_headers,omitempty"` + SupportsLongCacheRetention *bool `json:"supports_long_cache_retention,omitempty"` + SendSessionIDHeader *bool `json:"send_session_id_header,omitempty"` + SupportsEagerToolInputStreaming *bool `json:"supports_eager_tool_input_streaming,omitempty"` + SupportsCacheControlOnTools *bool `json:"supports_cache_control_on_tools,omitempty"` + SupportsTemperature *bool `json:"supports_temperature,omitempty"` + ForceAdaptiveThinking *bool `json:"force_adaptive_thinking,omitempty"` + AllowEmptySignature *bool `json:"allow_empty_signature,omitempty"` } func (entry aiServicesModelEntry) matchesProvider(provider ai.Provider) bool { diff --git a/pkg/connector/contacts_test.go b/pkg/connector/contacts_test.go index 8a30b155..e854d21c 100644 --- a/pkg/connector/contacts_test.go +++ b/pkg/connector/contacts_test.go @@ -170,7 +170,7 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { t.Fatalf("unexpected query %s", r.URL.RawQuery) } gotAuth = r.Header.Get("Authorization") - _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT-5.5","capabilities":{"input":{"modalities":["text","image"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["off","minimal","low","medium","high","xhigh"],"level_map":{"xhigh":"xhigh"},"default_level":"off","mode":"adaptive"},"tools":{"supported":true,"built_in":["image_generation"]},"limits":{"context_tokens":1050000,"output_tokens":128000}}},{"id":"minimax/minimax-m2.7","name":"MiniMax M2.7","runtime":{"provider":"openrouter","model":"minimax/minimax-m2.7","api":"openai-completions","baseUrl":"/proxy/openrouter/v1","compat":{"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_completion_tokens","thinkingFormat":"openrouter"}},"capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["low","medium","high"],"level_map":{"off":null,"minimal":null},"default_level":"low"}}},{"id":"beeper/fast","name":"Beeper Fast","capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]}}}]}`)) + _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT-5.5","capabilities":{"input":{"modalities":["text","image"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["off","minimal","low","medium","high","xhigh"],"level_map":{"xhigh":"xhigh"},"default_level":"off","mode":"adaptive"},"tools":{"supported":true,"built_in":["image_generation"]},"limits":{"context_tokens":1050000,"output_tokens":128000}}},{"id":"minimax/minimax-m2.7","name":"MiniMax M2.7","runtime":{"provider":"openrouter","model":"minimax/minimax-m2.7","api":"openai-completions","base_url":"/proxy/openrouter/v1","compat":{"supports_developer_role":false,"supports_reasoning_effort":true,"max_tokens_field":"max_completion_tokens","thinking_format":"openrouter"}},"capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["low","medium","high"],"level_map":{"off":null,"minimal":null},"default_level":"low"}}},{"id":"beeper/fast","name":"Beeper Fast","capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]}}}]}`)) })) defer server.Close() @@ -322,14 +322,14 @@ func TestAIServicesModelsURLUsesBaseURL(t *testing.T) { func TestAIServicesCatalogModelsUsesPublishedProviderRoutes(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`{"data":[ - {"id":"claude-sonnet-4-5","name":"Claude Sonnet 4.5","runtime":{"provider":"anthropic","model":"claude-sonnet-4-5","api":"anthropic-messages","baseUrl":"/proxy/anthropic"}}, - {"id":"gemini-2.5-flash-lite","name":"Gemini 2.5 Flash Lite","runtime":{"provider":"google-vertex","model":"gemini-2.5-flash-lite","api":"google-vertex","baseUrl":"/proxy/vertex"}}, - {"id":"google/gemini-3.1-pro-preview","name":"Gemini 3.1 Pro","runtime":{"provider":"google-vertex","model":"gemini-3.1-pro-preview","api":"google-vertex","baseUrl":"/proxy/vertex"}}, - {"id":"google/gemini-2.5-flash","name":"Gemini 2.5 Flash","runtime":{"provider":"google-vertex","model":"gemini-2.5-flash","api":"google-vertex","baseUrl":"/proxy/vertex"}}, - {"id":"x-ai/grok-4.20","name":"Grok 4.20","runtime":{"provider":"xai","model":"x-ai/grok-4.20","api":"openai-responses","baseUrl":"/proxy/xai/v1"}}, - {"id":"groq/qwen/qwen3-32b","name":"Qwen 3 32B","runtime":{"provider":"groq","model":"groq/qwen/qwen3-32b","api":"openai-responses","baseUrl":"/proxy/groq/v1"}}, - {"id":"openai/gpt-oss-120b","name":"GPT OSS 120B","runtime":{"provider":"a8c","model":"gpt-oss-120b","api":"openai-completions","baseUrl":"/proxy/a8c/v1"}}, - {"id":"anthropic/claude-sonnet-4.5","name":"Claude via OpenRouter","metadata":{"family":"claude","provider_logo_url":"/models/providers/anthropic.png"},"runtime":{"provider":"openrouter","model":"anthropic/claude-sonnet-4.5","api":"openai-completions","baseUrl":"/proxy/openrouter/v1","compat":{"supportsDeveloperRole":false,"thinkingFormat":"openrouter","cacheControlFormat":"anthropic"}}} + {"id":"claude-sonnet-4-5","name":"Claude Sonnet 4.5","runtime":{"provider":"anthropic","model":"claude-sonnet-4-5","api":"anthropic-messages","base_url":"/proxy/anthropic"}}, + {"id":"gemini-2.5-flash-lite","name":"Gemini 2.5 Flash Lite","runtime":{"provider":"google-vertex","model":"gemini-2.5-flash-lite","api":"google-vertex","base_url":"/proxy/vertex"}}, + {"id":"google/gemini-3.1-pro-preview","name":"Gemini 3.1 Pro","runtime":{"provider":"google-vertex","model":"gemini-3.1-pro-preview","api":"google-vertex","base_url":"/proxy/vertex"}}, + {"id":"google/gemini-2.5-flash","name":"Gemini 2.5 Flash","runtime":{"provider":"google-vertex","model":"gemini-2.5-flash","api":"google-vertex","base_url":"/proxy/vertex"}}, + {"id":"x-ai/grok-4.20","name":"Grok 4.20","runtime":{"provider":"xai","model":"x-ai/grok-4.20","api":"openai-responses","base_url":"/proxy/xai/v1"}}, + {"id":"groq/qwen/qwen3-32b","name":"Qwen 3 32B","runtime":{"provider":"groq","model":"groq/qwen/qwen3-32b","api":"openai-responses","base_url":"/proxy/groq/v1"}}, + {"id":"openai/gpt-oss-120b","name":"GPT OSS 120B","runtime":{"provider":"a8c","model":"gpt-oss-120b","api":"openai-completions","base_url":"/proxy/a8c/v1"}}, + {"id":"anthropic/claude-sonnet-4.5","name":"Claude via OpenRouter","metadata":{"family":"claude","provider_logo_url":"/models/providers/anthropic.png"},"runtime":{"provider":"openrouter","model":"anthropic/claude-sonnet-4.5","api":"openai-completions","base_url":"/proxy/openrouter/v1","compat":{"supports_developer_role":false,"thinking_format":"openrouter","cache_control_format":"anthropic"}}} ]}`)) })) defer server.Close() diff --git a/pkg/connector/provider_test.go b/pkg/connector/provider_test.go index 5e561335..5cb1cbd7 100644 --- a/pkg/connector/provider_test.go +++ b/pkg/connector/provider_test.go @@ -652,8 +652,9 @@ func TestCatalogRuntimeForCustomProviderUsesCustomBaseURL(t *testing.T) { Runtime: &struct { API string `json:"api"` Provider string `json:"provider"` - BaseURL string `json:"baseUrl"` + BaseURL string `json:"base_url"` Model string `json:"model"` + Endpoint string `json:"endpoint"` Compat *aiServicesModelCompat `json:"compat"` }{ API: string(ai.ApiOpenAICompletions), diff --git a/pkg/connector/stream_test.go b/pkg/connector/stream_test.go index 742bc0d5..ab9c17f5 100644 --- a/pkg/connector/stream_test.go +++ b/pkg/connector/stream_test.go @@ -315,6 +315,85 @@ func TestStreamPublisherUsesFreshTextMessageAfterToolContinuationWithPriorText(t } } +func TestStreamPublisherAnchorsContinuationThinkingAndSecondToolAfterApproval(t *testing.T) { + ctx := context.Background() + toolAPI := ai.Api("test-stream-tool-prior-text-second-tool") + answerAPI := ai.Api("test-stream-answer-thinking-second-tool") + ai.RegisterAPIProvider(toolAPI, func(ctx context.Context, model ai.Model, llmContext ai.Context, options ai.SimpleStreamOptions) *ai.AssistantMessageEventStream { + stream := ai.NewAssistantMessageEventStream() + go func() { + toolCall := &ai.ToolCall{ID: "call-session", Name: "get_session", Arguments: map[string]any{}} + message := ai.Message{ + Role: "assistant", + Content: []ai.ContentBlock{ + {Type: "text", Text: "before "}, + {Type: "toolCall", ID: toolCall.ID, Name: toolCall.Name, Arguments: toolCall.Arguments}, + }, + StopReason: ai.StopReasonToolUse, + } + stream.Push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: 0, Delta: "before "}) + stream.Push(ai.AssistantMessageEvent{Type: "toolcall_start", ContentIndex: 1, ToolCall: toolCall}) + stream.Push(ai.AssistantMessageEvent{Type: "toolcall_end", ContentIndex: 1, ToolCall: toolCall}) + stream.Push(ai.AssistantMessageEvent{Type: "done", Reason: ai.StopReasonToolUse, Message: &message}) + }() + return stream + }) + ai.RegisterAPIProvider(answerAPI, func(ctx context.Context, model ai.Model, llmContext ai.Context, options ai.SimpleStreamOptions) *ai.AssistantMessageEventStream { + stream := ai.NewAssistantMessageEventStream() + go func() { + secondTool := &ai.ToolCall{ID: "call-second", Name: "second_tool", Arguments: map[string]any{"ok": true}} + message := ai.Message{ + Role: "assistant", + Content: []ai.ContentBlock{ + {Type: "thinking", Thinking: "checking approval"}, + {Type: "text", Text: "after "}, + {Type: "toolCall", ID: secondTool.ID, Name: secondTool.Name, Arguments: secondTool.Arguments}, + }, + StopReason: ai.StopReasonToolUse, + } + stream.Push(ai.AssistantMessageEvent{Type: "thinking_start", ContentIndex: 0}) + stream.Push(ai.AssistantMessageEvent{Type: "thinking_delta", ContentIndex: 0, Delta: "checking approval"}) + stream.Push(ai.AssistantMessageEvent{Type: "text_delta", ContentIndex: 1, Delta: "after "}) + stream.Push(ai.AssistantMessageEvent{Type: "toolcall_start", ContentIndex: 2, ToolCall: secondTool}) + stream.Push(ai.AssistantMessageEvent{Type: "toolcall_end", ContentIndex: 2, ToolCall: secondTool}) + stream.Push(ai.AssistantMessageEvent{Type: "done", Reason: ai.StopReasonToolUse, Message: &message}) + }() + return stream + }) + defer ai.UnregisterAPIProvider(toolAPI) + defer ai.UnregisterAPIProvider(answerAPI) + + publisher := &recordingStreamPublisher{} + client := &Client{} + run := aistream.NewRun("run", "thread", "beeper/fake", "assistant:run", "Fake", timeNow()) + run.MessageID = "assistant:run" + cursor := &streamPublishCursor{nextSeq: 1} + + toolResult := client.streamPublisherWithEndFrom(publisher, "!room:example.com", "$event", run, cursor, nil)(ctx, ai.Model{ID: "fake", API: toolAPI}, ai.Context{}, ai.SimpleStreamOptions{}).Result() + if toolResult.StopReason != ai.StopReasonToolUse { + t.Fatalf("unexpected first tool stream result %#v", toolResult) + } + writer := aistream.NewWriter(run, timeNow) + writer.ToolEnd("call-session", "get_session", map[string]any{}, map[string]any{"state": agui.ToolResultStateComplete, "status": "success"}) + + answerResult := client.streamPublisherWithEndFrom(publisher, "!room:example.com", "$event", run, cursor, nil)(ctx, ai.Model{ID: "fake", API: answerAPI}, ai.Context{}, ai.SimpleStreamOptions{}).Result() + if answerResult.StopReason != ai.StopReasonToolUse { + t.Fatalf("unexpected answer stream result %#v", answerResult) + } + + parents := toolParentMessageIDs(run.Events) + if parents["call-second"] != "assistant:run-text-2" { + t.Fatalf("second continuation tool used wrong parent: %#v", parents) + } + + message := run.FinalBeeperAIMessage(0, true) + got := uiPartSummary(message.Parts) + want := []string{"text:before ", "tool-call:call-session", "thinking:checking approval", "text:after ", "tool-call:call-second"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("continued UI parts mismatch\ngot: %#v\nwant: %#v\nparts: %#v", got, want, message.Parts) + } +} + func TestDoneFallbackAfterPriorTextUsesCurrentWriterOnly(t *testing.T) { run := aistream.NewRun("run", "thread", "beeper/fake", "assistant:run", "Fake", timeNow()) run.MessageID = "assistant:run" @@ -901,7 +980,7 @@ func TestAssistantModelProfileUsesCatalogDisplayName(t *testing.T) { if r.URL.Path != "/models" { t.Fatalf("unexpected path %s", r.URL.Path) } - _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT 5.5 Catalog","runtime":{"provider":"openai","model":"gpt-5.5","api":"openai-responses","baseUrl":"/proxy/openai/v1"}}]}`)) + _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.5","name":"GPT 5.5 Catalog","runtime":{"provider":"openai","model":"gpt-5.5","api":"openai-responses","base_url":"/proxy/openai/v1"}}]}`)) })) defer server.Close() @@ -1238,6 +1317,17 @@ func textContentMessageIDs(events []agui.Event) []string { return ids } +func toolParentMessageIDs(events []agui.Event) map[string]string { + parents := map[string]string{} + for _, evt := range events { + if evt.Type() != agui.EventToolCallStart { + continue + } + parents[firstNonEmptyString(evt.Get("toolCallId"))] = firstNonEmptyString(evt.Get("parentMessageId")) + } + return parents +} + func uiPartSummary(parts []aistream.MessagePart) []string { out := make([]string, 0, len(parts)) for _, part := range parts { From 542dbd2c03cd3ee5c01ae32b1c7e4741938f6abd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 22:04:11 +0200 Subject: [PATCH 18/22] tool defaults --- pkg/connector/contacts_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/connector/contacts_test.go b/pkg/connector/contacts_test.go index e854d21c..ff96709a 100644 --- a/pkg/connector/contacts_test.go +++ b/pkg/connector/contacts_test.go @@ -221,6 +221,9 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { if supported, ok := models[0].Compat["tools_supported"].(bool); !ok || !supported { t.Fatalf("expected AI Services tool support metadata, got %#v", models[0].Compat) } + if supported, ok := models[1].Compat["tools_supported"].(bool); !ok || supported { + t.Fatalf("expected missing AI Services tools capability to disable agent tools, got %#v", models[1].Compat) + } if models[1].DefaultThinkingLevel != ai.ModelThinkingLevelLow || roomThinkingLevelSupported(models[1], ai.ModelThinkingLevelOff) { t.Fatalf("expected MiniMax reasoning to default to low and reject off, got %#v", models[1]) } From f9db847d0c7372517d6a20873937edea76ffc8c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 22:04:16 +0200 Subject: [PATCH 19/22] Update chat_tools.go --- pkg/connector/chat_tools.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/connector/chat_tools.go b/pkg/connector/chat_tools.go index b4e3a9b3..96df7e2c 100644 --- a/pkg/connector/chat_tools.go +++ b/pkg/connector/chat_tools.go @@ -85,10 +85,10 @@ func modelSupportsAgentTools(model ai.Model) bool { return false } if model.Compat == nil { - return true + return false } supported, ok := model.Compat["tools_supported"].(bool) - return !ok || supported + return ok && supported } func modelHasOutputModality(model ai.Model, modality string) bool { From 9e7f8ed9defb8203b260d177a78e085a975adf16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 22:04:19 +0200 Subject: [PATCH 20/22] Update chat_tools_test.go --- pkg/connector/chat_tools_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/connector/chat_tools_test.go b/pkg/connector/chat_tools_test.go index cf987fe7..79d9e06c 100644 --- a/pkg/connector/chat_tools_test.go +++ b/pkg/connector/chat_tools_test.go @@ -28,9 +28,12 @@ func TestChatToolsSkipGoogleVertexImageModels(t *testing.T) { } } -func TestModelSupportsAgentToolsDefaultsToTrue(t *testing.T) { - if !modelSupportsAgentTools(ai.Model{}) { - t.Fatal("models without catalog tool metadata should keep default tool behavior") +func TestModelSupportsAgentToolsRequiresExplicitCatalogSupport(t *testing.T) { + if modelSupportsAgentTools(ai.Model{}) { + t.Fatal("models without catalog tool metadata should not expose agent tools") + } + if !modelSupportsAgentTools(ai.Model{Compat: map[string]any{"tools_supported": true}}) { + t.Fatal("models with explicit catalog tool support should expose agent tools") } } From d601302898bad542f3ecb96458a964e6e48018dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 22:04:22 +0200 Subject: [PATCH 21/22] Update contacts_test.go --- pkg/connector/contacts_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/connector/contacts_test.go b/pkg/connector/contacts_test.go index ff96709a..4d62e180 100644 --- a/pkg/connector/contacts_test.go +++ b/pkg/connector/contacts_test.go @@ -221,7 +221,7 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { if supported, ok := models[0].Compat["tools_supported"].(bool); !ok || !supported { t.Fatalf("expected AI Services tool support metadata, got %#v", models[0].Compat) } - if supported, ok := models[1].Compat["tools_supported"].(bool); !ok || supported { + if modelSupportsAgentTools(models[1]) { t.Fatalf("expected missing AI Services tools capability to disable agent tools, got %#v", models[1].Compat) } if models[1].DefaultThinkingLevel != ai.ModelThinkingLevelLow || roomThinkingLevelSupported(models[1], ai.ModelThinkingLevelOff) { From edf0bc77006b6d664c0ea9d02cfadef719201de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 1 Jun 2026 22:18:52 +0200 Subject: [PATCH 22/22] Skip duplicate tool results; merge provider models Prevent duplicate tool-result source emission and improve provider model handling. - Avoid re-emitting sources for toolresult events by adding hasPriorToolResult and checking it in streamPublisherWithEndFrom. - Replace shallow maps.Clone usage with deep cloning (clonePayloadMap -> cloneAnyMap/cloneAny) to avoid shared mutable payloads when building built-in tools. - Add mergeProviderCatalogModels and mergeProviderCatalogModel to combine configured provider models with catalog metadata (preserve custom configured models, inherit catalog metadata for matching models). - Change provider resolution: look up provider from Connector.providersForLogin, load catalog models, default model selection from provider, and return an error when a configured model is not offered by the provider. - Improve provider routes error handling by introducing errInvalidProviderLoginID and providerRequestErrorStatus to map login lookup failures to HTTP 400 where appropriate. - Update README security note to explicitly state the direct fetch path bypasses AI-services and has no SSRF guard and recommend upstream deny-lists or network policies. - Add unit tests covering canonical model resolution, catalog merge behavior, provider validation, and duplicate toolresult source suppression, plus a helper for counting source custom events. Also includes minor import/cleanup adjustments. --- README.md | 4 +- pkg/connector/builtin_tools.go | 37 +++++++++++++-- pkg/connector/client.go | 17 ++++++- pkg/connector/contacts.go | 54 ++++++++++++++++++++- pkg/connector/contacts_test.go | 80 ++++++++++++++++++++++++++++++++ pkg/connector/provider.go | 14 +++++- pkg/connector/provider_routes.go | 19 ++++++-- pkg/connector/provider_test.go | 9 ++-- pkg/connector/room_state.go | 2 +- pkg/connector/stream_test.go | 76 ++++++++++++++++++++++++++++++ 10 files changed, 291 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 4e610037..95122582 100644 --- a/README.md +++ b/README.md @@ -386,7 +386,7 @@ Tools are gated per-room via the `com.beeper.ai.tools` state event. `search` may 4. Wire config in `pkg/connector/chat_tools.go` and honor `DisabledTools`. 5. If it produces citable sources, add canonical source observations in `pkg/connector/sources.go` so URLs surface as message sources. -> **Security note:** the direct fetch path has **no SSRF guard** — it can reach localhost/private/link-local addresses when the bridge bypasses AI-services for raw assets and local/private targets. Treat it accordingly in your threat model. +> **Security note:** the direct fetch path intentionally bypasses AI-services for localhost/private/link-local addresses, raw asset URLs, and source-like files, and has **no SSRF guard**. Deployments that cannot allow bridge-origin egress to private networks should enforce an upstream deny-list or network policy before enabling `fetch`. ## Sessions: the branching conversation tree @@ -518,7 +518,7 @@ It serves `/v1/models`, `/v1/responses`, `/v1/chat/completions`, and `/api/strea - **Reasoning is double-validated and clamped** — setting a model can silently change the effective reasoning level. - **Two parallel session-tree implementations** (`aidb` vs `session` SQLite files) with near-duplicate SQL and one subtle difference (`ON DELETE CASCADE`). - **Token counts are estimates** (≈ chars/4) — compaction thresholds are approximate. -- **The direct fetch path has no SSRF protection.** +- **The direct fetch path intentionally allows private-network/raw-asset egress and has no SSRF protection.** - **`ProviderConfig` holds secrets** (API keys, refresh tokens) in login metadata and serializes to JSON *and* YAML — don't log it. - **AG-UI `Event` is a map, not a struct** — read typed fields via `Get`/`String`; unknown fields survive round-trips. diff --git a/pkg/connector/builtin_tools.go b/pkg/connector/builtin_tools.go index 9e618660..f4819854 100644 --- a/pkg/connector/builtin_tools.go +++ b/pkg/connector/builtin_tools.go @@ -2,7 +2,6 @@ package connector import ( "context" - "maps" "slices" "strings" @@ -202,7 +201,7 @@ func appendBuiltInTool(tools []any, toolPayload map[string]any) []any { } } } - return append(tools, maps.Clone(toolPayload)) + return append(tools, clonePayloadMap(toolPayload)) } func builtInToolKey(tool map[string]any) string { @@ -245,5 +244,37 @@ func toolsAsAny(raw any) []any { } func clonePayloadMap(in map[string]any) map[string]any { - return maps.Clone(in) + return cloneAnyMap(in) +} + +func cloneAnyMap(in map[string]any) map[string]any { + if in == nil { + return nil + } + out := make(map[string]any, len(in)) + for key, value := range in { + out[key] = cloneAny(value) + } + return out +} + +func cloneAny(value any) any { + switch typed := value.(type) { + case map[string]any: + return cloneAnyMap(typed) + case []any: + out := make([]any, len(typed)) + for i, item := range typed { + out[i] = cloneAny(item) + } + return out + case []map[string]any: + out := make([]map[string]any, len(typed)) + for i, item := range typed { + out[i] = cloneAnyMap(item) + } + return out + default: + return value + } } diff --git a/pkg/connector/client.go b/pkg/connector/client.go index cffc947e..b57b2826 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -1258,7 +1258,7 @@ func (cl *Client) streamPublisherWithEndFrom(publisher bridgev2.BeeperStreamPubl writer.Custom("com.beeper.source", source) } } - if evt.Type == "toolresult" && evt.ToolCall != nil { + if evt.Type == "toolresult" && evt.ToolCall != nil && !hasPriorToolResult(run.Events, beforeEvents, evt.ToolCall.ID) { output := toolOutputEvent{ID: evt.ToolCall.ID, Name: evt.ToolCall.Name, Input: evt.ToolCall.Arguments} for _, source := range streamSources.addToolOutput(output, evt.CustomValue) { writer.Custom("com.beeper.source", source) @@ -1738,6 +1738,21 @@ type toolOutputEvent struct { IsError bool } +func hasPriorToolResult(events []agui.Event, before int, toolCallID string) bool { + if toolCallID == "" { + return false + } + if before > len(events) { + before = len(events) + } + for _, evt := range events[:before] { + if evt.Type() == agui.EventToolCallResult && evt.Get("toolCallId") == toolCallID { + return true + } + } + return false +} + func appendToolOutputs(run *aistream.Run, outputs []toolOutputEvent, messages ...ai.Message) { if run == nil { return diff --git a/pkg/connector/contacts.go b/pkg/connector/contacts.go index 5de3f294..b981a89c 100644 --- a/pkg/connector/contacts.go +++ b/pkg/connector/contacts.go @@ -322,7 +322,7 @@ func (cl *Client) providerWithCatalogModelsStrictWithRefresh(ctx context.Context return provider, err } if len(models) > 0 { - provider.Models = models + provider.Models = mergeProviderCatalogModels(provider, models) zerolog.Ctx(ctx).Debug(). Str("action", "ai_model_catalog"). Str("provider_id", provider.ID). @@ -332,6 +332,58 @@ func (cl *Client) providerWithCatalogModelsStrictWithRefresh(ctx context.Context return provider, nil } +func mergeProviderCatalogModels(provider aiid.ProviderConfig, catalog []ai.Model) []ai.Model { + if provider.ID == aiid.DefaultProvider || len(provider.Models) == 0 { + return cloneModels(catalog) + } + byID := make(map[string]ai.Model, len(catalog)) + for _, model := range catalog { + byID[model.ID] = model + } + models := make([]ai.Model, 0, len(provider.Models)) + for _, configured := range provider.Models { + model := normalizeProviderModel(configured, provider) + if catalogModel, ok := byID[model.ID]; ok { + model = mergeProviderCatalogModel(model, catalogModel) + } + models = append(models, model) + } + return models +} + +func mergeProviderCatalogModel(configured ai.Model, catalog ai.Model) ai.Model { + model := catalog + if configured.Name != "" && configured.Name != configured.ID { + model.Name = configured.Name + } + if len(configured.Input) > 0 { + model.Input = slices.Clone(configured.Input) + } + if len(configured.Output) > 0 { + model.Output = slices.Clone(configured.Output) + } + if len(configured.BuiltInTools) > 0 { + model.BuiltInTools = slices.Clone(configured.BuiltInTools) + } + if configured.ContextWindow != 0 { + model.ContextWindow = configured.ContextWindow + } + if configured.MaxTokens != 0 { + model.MaxTokens = configured.MaxTokens + } + if configured.Headers != nil { + model.Headers = maps.Clone(configured.Headers) + } + if configured.Compat != nil { + compat := maps.Clone(model.Compat) + for key, value := range configured.Compat { + compat[key] = value + } + model.Compat = compat + } + return model +} + func (cl *Client) cachedAIServicesCatalogModels(ctx context.Context, provider aiid.ProviderConfig, refresh bool) ([]ai.Model, error) { providerKey := providerCatalogCacheKey(provider) cl.catalogCacheMu.Lock() diff --git a/pkg/connector/contacts_test.go b/pkg/connector/contacts_test.go index 4d62e180..bcb5d0fa 100644 --- a/pkg/connector/contacts_test.go +++ b/pkg/connector/contacts_test.go @@ -238,6 +238,45 @@ func TestAIServicesCatalogModelsFetchesVisibleModels(t *testing.T) { } } +func TestResolveCanonicalRoomModelUsesAIServicesCatalogBeforeValidation(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/models" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + _, _ = w.Write([]byte(`{"type":"com.beeper.ai.model_list","data":[{"id":"openai/gpt-5.4","name":"GPT-5.4","runtime":{"provider":"openai","model":"gpt-5.4","api":"openai-responses","base_url":"/proxy/openai/v1"},"capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]}}}]}`)) + })) + defer server.Close() + + provider := aiid.ProviderConfig{ + ID: aiid.DefaultProvider, + Provider: ai.ProviderOpenAI, + API: ai.ApiOpenAIResponses, + BaseURL: server.URL, + DefaultModel: "beeper/default", + Models: []ai.Model{{ID: "beeper/default", Provider: ai.ProviderOpenAI, API: ai.ApiOpenAIResponses}}, + } + client := &Client{ + Main: &Connector{ + AppServiceToken: "as-token", + }, + UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ + ID: "login", + UserMXID: "@test:beeper.localtest.me", + Metadata: &aiid.UserLoginMetadata{Providers: map[string]aiid.ProviderConfig{ + provider.ID: provider, + }}, + }}, + } + + _, model, canonical, err := client.resolveCanonicalRoomModel(context.Background(), RoomConfig{ProviderID: aiid.DefaultProvider, ModelID: "openai/gpt-5.4"}) + if err != nil { + t.Fatal(err) + } + if model.ID != "openai/gpt-5.4" || model.BaseURL != server.URL+"/proxy/openai/v1" || canonical != "beeper/openai/gpt-5.4" { + t.Fatalf("unexpected resolved model canonical=%q model=%#v", canonical, model) + } +} + func TestAIServicesModelEntryReasoningDefaultsCanRequireReasoning(t *testing.T) { var entry aiServicesModelEntry if err := json.Unmarshal([]byte(`{"id":"deepseek/deepseek-r1-0528","name":"DeepSeek R1 (0528)","capabilities":{"input":{"modalities":["text"]},"output":{"modalities":["text"]},"reasoning":{"supported":true,"levels":["minimal","low","medium","high"],"level_map":{"off":null},"default_level":"minimal"}}}`), &entry); err != nil { @@ -405,6 +444,47 @@ func TestResolveModelForProviderPreservesOpenAICatalogModelID(t *testing.T) { } } +func TestMergeProviderCatalogModelsPreservesConfiguredCustomModels(t *testing.T) { + provider := aiid.ProviderConfig{ + ID: "custom", + Provider: ai.ProviderOpenAI, + API: ai.ApiOpenAIResponses, + BaseURL: "https://custom.test/v1", + Models: []ai.Model{ + {ID: "local-model", Name: "Local Model", Input: []string{"text", "image"}}, + {ID: "gpt-5.5"}, + }, + } + catalog := []ai.Model{ + { + ID: "gpt-5.5", + Name: "GPT-5.5", + Provider: ai.ProviderOpenAI, + API: ai.ApiOpenAIResponses, + BaseURL: "https://custom.test/v1", + Reasoning: true, + DefaultThinkingLevel: ai.ModelThinkingLevelLow, + ContextWindow: 128000, + }, + { + ID: "catalog-only", + Provider: ai.ProviderOpenAI, + API: ai.ApiOpenAIResponses, + }, + } + + models := mergeProviderCatalogModels(provider, catalog) + if len(models) != 2 { + t.Fatalf("expected only configured models, got %#v", models) + } + if models[0].ID != "local-model" || models[0].Name != "Local Model" || !isImageModel(models[0]) { + t.Fatalf("expected configured local model to be preserved, got %#v", models[0]) + } + if models[1].ID != "gpt-5.5" || !models[1].Reasoning || models[1].ContextWindow != 128000 { + t.Fatalf("expected matching configured model to inherit catalog metadata, got %#v", models[1]) + } +} + func TestResolveModelForProviderRejectsUnlistedCustomModelID(t *testing.T) { provider := aiid.ProviderConfig{ ID: "custom-openai", diff --git a/pkg/connector/provider.go b/pkg/connector/provider.go index 2ede84cc..9ad04a9a 100644 --- a/pkg/connector/provider.go +++ b/pkg/connector/provider.go @@ -45,11 +45,17 @@ func (cl *Client) resolveProvider(ctx context.Context, roomConfig RoomConfig) (a } log := logCtx.Logger() ctx = log.WithContext(ctx) - provider, modelID, err := cl.Main.ResolveProvider(ctx, cl.UserLogin, roomConfig) - if err != nil { + providerID := roomConfig.ProviderID + if providerID == "" { + providerID = aiid.DefaultProvider + } + provider, ok := cl.Main.providersForLogin(cl.UserLogin)[providerID] + if !ok { + err := fmt.Errorf("provider %s is not available for login %s", providerID, cl.UserLogin.ID) log.Err(err).Msg("Failed to resolve AI provider") return aiid.ProviderConfig{}, "", err } + var err error provider, err = cl.providerWithCatalogModelsStrict(ctx, provider) if err != nil { log.Err(err).Str("provider_id", provider.ID).Msg("Failed to load AI provider model catalog") @@ -60,6 +66,10 @@ func (cl *Client) resolveProvider(ctx context.Context, roomConfig RoomConfig) (a log.Err(err).Str("provider_id", provider.ID).Msg("AI provider model catalog is empty") return aiid.ProviderConfig{}, "", err } + modelID := roomConfig.ModelID + if modelID == "" { + modelID = provider.DefaultModel + } if resolvedModelID, ok := resolveProviderModelID(provider, modelID); ok { log.Debug(). Str("provider_id", provider.ID). diff --git a/pkg/connector/provider_routes.go b/pkg/connector/provider_routes.go index cc6606c3..dd117db2 100644 --- a/pkg/connector/provider_routes.go +++ b/pkg/connector/provider_routes.go @@ -14,6 +14,8 @@ import ( "maunium.net/go/mautrix/bridgev2/networkid" ) +var errInvalidProviderLoginID = errors.New("invalid provider login_id") + type providersListResponse struct { Providers []ProviderResponse `json:"providers"` } @@ -47,7 +49,7 @@ func (c *Connector) handleProvidersList(provisioning bridgev2.IProvisioningAPI) return func(w http.ResponseWriter, r *http.Request) { login, err := c.loginForProviderRequest(r.Context(), provisioning.GetUser(r), r.URL.Query().Get("login_id")) if err != nil { - writeProviderError(w, providerErrorStatus(err), err) + writeProviderError(w, providerRequestErrorStatus(err), err) return } writeProviderJSON(r.Context(), w, http.StatusOK, providersListResponse{Providers: sortedProviderResponses(c.providersForLogin(login))}) @@ -58,7 +60,7 @@ func (c *Connector) handleProvidersGet(provisioning bridgev2.IProvisioningAPI) h return func(w http.ResponseWriter, r *http.Request) { login, err := c.loginForProviderRequest(r.Context(), provisioning.GetUser(r), r.URL.Query().Get("login_id")) if err != nil { - writeProviderError(w, providerErrorStatus(err), err) + writeProviderError(w, providerRequestErrorStatus(err), err) return } providerID := strings.TrimSpace(r.PathValue("providerID")) @@ -98,7 +100,7 @@ func (c *Connector) handleProviderUpsert(w http.ResponseWriter, r *http.Request, } login, err := c.loginForProviderRequest(r.Context(), provisioning.GetUser(r), r.URL.Query().Get("login_id")) if err != nil { - writeProviderError(w, providerErrorStatus(err), err) + writeProviderError(w, providerRequestErrorStatus(err), err) return } provider, err := c.VerifyProviderConfig(r.Context(), login, input) @@ -121,7 +123,7 @@ func (c *Connector) handleProvidersDelete(provisioning bridgev2.IProvisioningAPI return func(w http.ResponseWriter, r *http.Request) { login, err := c.loginForProviderRequest(r.Context(), provisioning.GetUser(r), r.URL.Query().Get("login_id")) if err != nil { - writeProviderError(w, providerErrorStatus(err), err) + writeProviderError(w, providerRequestErrorStatus(err), err) return } err = c.DeleteProvider(r.Context(), login, r.PathValue("providerID")) @@ -143,7 +145,7 @@ func (c *Connector) loginForProviderRequest(ctx context.Context, user *bridgev2. return nil, err } if login == nil || login.UserMXID != user.MXID { - return nil, fmt.Errorf("login %s not found", loginID) + return nil, fmt.Errorf("%w: login %s not found", errInvalidProviderLoginID, loginID) } if err := c.ensureAIChatsMetadata(ctx, login); err != nil { return nil, err @@ -159,6 +161,13 @@ func writeProviderJSON(ctx context.Context, w http.ResponseWriter, status int, b } } +func providerRequestErrorStatus(err error) int { + if errors.Is(err, errInvalidProviderLoginID) { + return http.StatusBadRequest + } + return providerErrorStatus(err) +} + func providerErrorStatus(err error) int { if err == nil { return http.StatusOK diff --git a/pkg/connector/provider_test.go b/pkg/connector/provider_test.go index 5cb1cbd7..0171a044 100644 --- a/pkg/connector/provider_test.go +++ b/pkg/connector/provider_test.go @@ -712,7 +712,7 @@ func TestModelForProviderAppliesRouteBaseURLToDefaultModel(t *testing.T) { } } -func TestResolveProviderDefersModelValidationToCatalogLoad(t *testing.T) { +func TestResolveProviderRejectsUnknownConfiguredModel(t *testing.T) { conn := &Connector{} provider := aiid.ProviderConfig{ ID: "custom", @@ -726,11 +726,8 @@ func TestResolveProviderDefersModelValidationToCatalogLoad(t *testing.T) { Metadata: &aiid.UserLoginMetadata{Providers: map[string]aiid.ProviderConfig{provider.ID: provider}}, }} _, modelID, err := conn.ResolveProvider(context.Background(), login, RoomConfig{ProviderID: "custom", ModelID: "missing"}) - if err != nil { - t.Fatal(err) - } - if modelID != "missing" { - t.Fatalf("unexpected model ID %q", modelID) + if err == nil { + t.Fatalf("expected missing model to fail, got model ID %q", modelID) } _, modelID, err = conn.ResolveProvider(context.Background(), login, RoomConfig{ProviderID: "custom", ModelID: "allowed"}) if err != nil { diff --git a/pkg/connector/room_state.go b/pkg/connector/room_state.go index f9cb9076..7d4f6924 100644 --- a/pkg/connector/room_state.go +++ b/pkg/connector/room_state.go @@ -139,7 +139,7 @@ func (c *Connector) ResolveProvider(ctx context.Context, login *bridgev2.UserLog if resolvedModelID, ok := resolveProviderModelID(provider, modelID); ok { return provider, resolvedModelID, nil } - return provider, modelID, nil + return aiid.ProviderConfig{}, "", fmt.Errorf("provider %s does not offer model %s", providerID, modelID) } func providerAllowsModel(provider aiid.ProviderConfig, modelID string) bool { diff --git a/pkg/connector/stream_test.go b/pkg/connector/stream_test.go index ab9c17f5..6f59d7c1 100644 --- a/pkg/connector/stream_test.go +++ b/pkg/connector/stream_test.go @@ -1279,6 +1279,72 @@ func TestPublishToolOutputStreamsLiveResult(t *testing.T) { } } +func TestStreamPublisherSkipsDuplicateToolResultSourcesAfterLiveToolOutput(t *testing.T) { + ctx := context.Background() + testAPI := ai.Api("test-duplicate-toolresult-sources") + toolCall := &ai.ToolCall{ + ID: "call-fetch", + Name: "fetch", + Arguments: map[string]any{"url": "https://example.com"}, + } + result := map[string]any{ + "url": "https://example.com", + "title": "Example", + "description": "Example source", + } + ai.RegisterAPIProvider(testAPI, func(ctx context.Context, model ai.Model, llmContext ai.Context, options ai.SimpleStreamOptions) *ai.AssistantMessageEventStream { + stream := ai.NewAssistantMessageEventStream() + go func() { + stream.Push(ai.AssistantMessageEvent{Type: "toolresult", ToolCall: toolCall, CustomValue: result}) + stream.Push(ai.AssistantMessageEvent{Type: "done", Reason: ai.StopReasonStop, Message: &ai.Message{Role: "assistant", StopReason: ai.StopReasonStop}}) + }() + return stream + }) + defer ai.UnregisterAPIProvider(testAPI) + + publisher := &recordingStreamPublisher{} + run := aistream.NewRun("run", "thread", "beeper/gpt-5.5", "assistant:run", "GPT-5.5", timeNow()) + run.MessageID = "assistant:run" + writer := aistream.NewWriter(run, timeNow) + writer.Start() + writer.ToolStart(toolCall.ID, toolCall.Name, 0, nil) + active := &activeAIRun{streams: []*assistantStreamState{{ + eventID: "$event", + run: run, + sources: newSourceCollector(), + publish: streamPublishCursor{ + nextSeq: 1, + published: len(run.Events), + started: true, + }, + }}} + + err := active.publishToolOutput(ctx, &Client{}, publisher, "!room:example.com", toolOutputEvent{ + ID: toolCall.ID, + Name: toolCall.Name, + Input: toolCall.Arguments, + Result: agent.AgentToolResult[any]{ + Details: result, + }, + }) + if err != nil { + t.Fatal(err) + } + sourceEventsBefore := sourceCustomEventCount(run.Events) + if sourceEventsBefore != 1 { + t.Fatalf("expected live tool output to publish one source, got %d events=%#v", sourceEventsBefore, run.Events) + } + + stream := active.streams[0] + response := (&Client{}).streamPublisherWithEndFrom(publisher, "!room:example.com", "$event", run, &stream.publish, nil)(ctx, ai.Model{ID: "fake", API: testAPI}, ai.Context{}, ai.SimpleStreamOptions{}).Result() + if response.StopReason != ai.StopReasonStop { + t.Fatalf("unexpected stream response %#v", response) + } + if sourceEventsAfter := sourceCustomEventCount(run.Events); sourceEventsAfter != sourceEventsBefore { + t.Fatalf("duplicate toolresult re-emitted sources: before=%d after=%d events=%#v", sourceEventsBefore, sourceEventsAfter, run.Events) + } +} + func TestPublishNewStreamEventsSuppressesMautrixRequestBodyLogs(t *testing.T) { logger := zerolog.New(io.Discard).Level(zerolog.DebugLevel) ctx := logger.WithContext(context.Background()) @@ -1328,6 +1394,16 @@ func toolParentMessageIDs(events []agui.Event) map[string]string { return parents } +func sourceCustomEventCount(events []agui.Event) int { + count := 0 + for _, evt := range events { + if evt.Type() == agui.EventCustom && evt.Get("name") == "com.beeper.source" { + count++ + } + } + return count +} + func uiPartSummary(parts []aistream.MessagePart) []string { out := make([]string, 0, len(parts)) for _, part := range parts {