Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions docs/LEARNINGS.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,25 @@ automatically and can't silently regress; prefer that over prose where possible.

---

## Default provider switched from Anthropic to OpenRouter + GLM 5.2

**What changed:** `DEFAULT_SETTINGS.activeProvider` is now `openai_compat` pointing at
`https://openrouter.ai/api/v1` with model `z-ai/glm-5.2`. The previous default was `anthropic /
claude-sonnet-4-6`.

**Why:** GLM 5.2 (Z.ai via OpenRouter) costs ~$1.20/1M input + $4.10/1M output tokens versus
Claude Sonnet 4.6 at $3/1M + $15/1M β€” roughly 3-4Γ— cheaper for Slopwatch's prompt shape (dense
paragraph analysis with a large system prompt). This validates AD-4 ("OpenAI-compat adapter as
gateway"). The Anthropic adapter remains first-class; users who already configured Anthropic keep
it via the v1 settings round-trip.

**What to watch:** GLM 5.2 is a reasoning model with a different response style than Claude.
Calibration (score distribution, false-positive rate on clearly-human text) should be monitored
in early beta. The `overall` score range and `reasoning` quality may differ. If detection quality
regresses, switching back is a one-line change in `DEFAULT_SETTINGS`.

---

## Comment/review-triggered agent workflows must gate on collaborator permission BEFORE checkout

**Symptom:** `claude-pr-feedback.yml` fired on any `@claude` comment/review with no actor
Expand Down
63 changes: 61 additions & 2 deletions entrypoints/options/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ type TestState = { status: 'idle' | 'testing' | 'ok' | 'fail'; detail?: string }
export function App() {
const [settings, setSettings] = useState<Settings | null>(null);
const [anthropicConfigured, setConfigured] = useState(false);
const [openaiCompatConfigured, setOpenAICompatConfigured] = useState(false);
const [keyInput, setKeyInput] = useState('');
const [openaiCompatKeyInput, setOpenAICompatKeyInput] = useState('');
const [test, setTest] = useState<Record<string, TestState>>({});
const [ollamaModels, setOllamaModels] = useState<string[] | null>(null);
const [cacheEntries, setCacheEntries] = useState<number | null>(null);
Expand All @@ -45,6 +47,7 @@ export function App() {
void (async () => {
setSettings(await getSettings());
setConfigured(await hasSecret('anthropic'));
setOpenAICompatConfigured(await hasSecret('openai_compat'));
setCacheEntries((await cacheStats()).entries);
setDiag(await listDiagnostics());
})();
Expand Down Expand Up @@ -102,6 +105,19 @@ export function App() {
setConfigured(false);
};

const saveOpenAICompatKey = async () => {
if (!openaiCompatKeyInput.trim()) return;
await setSecret('openai_compat', openaiCompatKeyInput.trim(), settings.persistSecrets);
setOpenAICompatConfigured(true);
setOpenAICompatKeyInput('');
await patch({ activeProvider: 'openai_compat', onboarded: true });
};

const clearOpenAICompatKey = async () => {
await clearSecret('openai_compat');
setOpenAICompatConfigured(false);
};

const togglePersist = async (checked: boolean) => {
await patch({ persistSecrets: checked });
await applyPersistencePreference(checked);
Expand Down Expand Up @@ -197,6 +213,49 @@ export function App() {
<TestButton state={test.anthropic} onClick={() => testConnection('anthropic')} />
</section>

<section>
<h2>OpenAI-compatible</h2>
<p className="hint">
Works with OpenRouter, vanilla OpenAI, Azure OpenAI, and other OpenAI-compatible
gateways. The default is OpenRouter + GLM 5.2.
</p>
<label htmlFor="oc-base">Base URL</label>
<input
id="oc-base"
value={settings.providers.openai_compat.baseUrl ?? ''}
onChange={(e) => setProviderField('openai_compat', 'baseUrl', e.target.value)}
/>
<label htmlFor="oc-model">Model</label>
<input
id="oc-model"
value={settings.providers.openai_compat.model}
onChange={(e) => setProviderField('openai_compat', 'model', e.target.value)}
/>
<label htmlFor="oc-key">API key</label>
{openaiCompatConfigured ? (
<p className="configured">
Configured βœ“{' '}
<button className="link" onClick={clearOpenAICompatKey}>Clear key</button>
</p>
) : (
<div className="row">
<input
id="oc-key"
type="password"
placeholder="sk-or-…"
value={openaiCompatKeyInput}
onChange={(e) => setOpenAICompatKeyInput(e.target.value)}
/>
<button onClick={saveOpenAICompatKey}>Save key</button>
</div>
)}
<p className="hint">
Your key goes directly from your browser to the gateway and is never sent to any Slopwatch
server. Prefer a scoped or limited key.
</p>
<TestButton state={test.openai_compat} onClick={() => testConnection('openai_compat')} />
</section>

<section>
<h2>Ollama (local)</h2>
<p className="hint">Runs on your device. Nothing leaves your machine.</p>
Expand Down Expand Up @@ -365,5 +424,5 @@ const PROVIDER_LABELS: Record<ProviderId, string> = {

// The Mock provider is only offered in development builds β€” never in production.
const PROVIDER_OPTIONS: ProviderId[] = import.meta.env.DEV
? ['anthropic', 'ollama', 'mock']
: ['anthropic', 'ollama'];
? ['anthropic', 'openai_compat', 'ollama', 'mock']
: ['anthropic', 'openai_compat', 'ollama'];
12 changes: 6 additions & 6 deletions lib/providers/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { getSecret } from '../storage/secrets';
import { MockProvider } from './mock';
import { AnthropicProvider } from './anthropic';
import { OllamaProvider } from './ollama';
import { OpenAICompatProvider } from './openai-compat';
import { ProviderError } from '../errors';

/**
Expand Down Expand Up @@ -31,17 +32,16 @@ export async function createProvider(settings: Settings): Promise<AnalysisProvid
}
case 'ollama':
return new OllamaProvider(cfg.model, cfg.baseUrl, settings.thresholds);
case 'openai_compat':
throw new ProviderError(
'unknown',
"The OpenAI-compatible provider isn't implemented yet. Use Anthropic, Ollama, or Mock.",
{ retryable: false },
);
case 'openai_compat': {
const key = await getSecret('openai_compat');
return new OpenAICompatProvider(cfg.model, cfg.baseUrl, key, settings.thresholds);
}
}
}

export { isProviderConfigured } from './readiness';
export { MockProvider } from './mock';
export { AnthropicProvider, estimateCost } from './anthropic';
export { OllamaProvider, ollamaOriginsSnippet } from './ollama';
export { OpenAICompatProvider } from './openai-compat';
export * from './base';
183 changes: 183 additions & 0 deletions lib/providers/openai-compat.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
import { describe, it, expect, vi } from 'vitest';
import { OpenAICompatProvider, estimateCost } from './openai-compat';
import type { HttpDeps } from './http';
import { DEFAULT_SETTINGS } from '../storage/settings';
import type { ExtractedContent } from '../types';

const thresholds = DEFAULT_SETTINGS.thresholds;

function content(): ExtractedContent {
return {
url: 'https://example.com',
title: 'T',
segments: [{ index: 0, text: 'A paragraph long enough to be a real content segment here.' }],
truncated: false,
sampledFraction: 1,
contentHash: 'h',
};
}

function resp(status: number, body: string): Response {
return { status, ok: status >= 200 && status < 300, text: async () => body } as unknown as Response;
}

function httpWith(fetchImpl: typeof fetch): HttpDeps {
return { fetchImpl, sleep: async () => {}, timeoutMs: 1000, maxRetries: 2, backoff: [1, 1, 1] };
}

const analysisPayload = JSON.stringify({
overall: 0.7,
reasoning: 'looks AI-generated',
segments: [{ index: 0, aiLikelihood: 0.8, rationale: 'formulaic phrasing' }],
});

const validBody = JSON.stringify({
choices: [{ message: { content: analysisPayload } }],
usage: { prompt_tokens: 800, completion_tokens: 150 },
});

describe('OpenAICompatProvider', () => {
it('sends bearer auth plus attribution headers and maps a result with usage and cost', async () => {
let captured: RequestInit | undefined;
const fetchImpl = vi.fn(async (_url: string, init?: RequestInit) => {
captured = init;
return resp(200, validBody);
}) as unknown as typeof fetch;

const p = new OpenAICompatProvider(
'z-ai/glm-5.2',
'https://openrouter.ai/api/v1',
'sk-test',
thresholds,
httpWith(fetchImpl),
);
const r = await p.analyze(content(), new AbortController().signal);

const headers = captured!.headers as Record<string, string>;
expect(headers['Authorization']).toBe('Bearer sk-test');
expect(headers['HTTP-Referer']).toBeDefined();
expect(headers['X-Title']).toBe('Slopwatch');

expect(r.provider).toBe('openai_compat');
expect(r.overall).toBe(0.7);
expect(r.label).toBe('likely-ai');
expect(r.ranLocally).toBe(false);
expect(r.usage?.inputTokens).toBe(800);
expect(r.usage?.outputTokens).toBe(150);
expect(r.usage?.estCostUsd).toBeGreaterThan(0);
});

it('sends json_schema in response_format on the first call', async () => {
let parsedBody: unknown;
const fetchImpl = vi.fn(async (_url: string, init?: RequestInit) => {
parsedBody = JSON.parse(init!.body as string);
return resp(200, validBody);
}) as unknown as typeof fetch;

const p = new OpenAICompatProvider('z-ai/glm-5.2', 'https://openrouter.ai/api/v1', 'key', thresholds, httpWith(fetchImpl));
await p.analyze(content(), new AbortController().signal);

const body = parsedBody as { response_format?: { type?: string } };
expect(body.response_format?.type).toBe('json_schema');
});

it('falls back to json_object when the gateway rejects json_schema with a 400', async () => {
const rejectBody = JSON.stringify({ error: { message: 'json_schema response_format not supported', code: 'unsupported_value' } });
let callCount = 0;
let lastBody: unknown;

const fetchImpl = vi.fn(async (_url: string, init?: RequestInit) => {
callCount++;
lastBody = JSON.parse(init!.body as string);
if (callCount === 1) return resp(400, rejectBody);
return resp(200, validBody);
}) as unknown as typeof fetch;

const p = new OpenAICompatProvider('z-ai/glm-5.2', 'https://openrouter.ai/api/v1', 'key', thresholds, httpWith(fetchImpl));
const r = await p.analyze(content(), new AbortController().signal);

expect(callCount).toBe(2);
expect((lastBody as { response_format?: { type?: string } }).response_format?.type).toBe('json_object');
expect(r.overall).toBe(0.7);
});

it('does NOT fall back for a 400 unrelated to json_schema (e.g. bad model name)', async () => {
const unrelatedError = JSON.stringify({ error: { message: 'model not found', code: 'model_not_found' } });
const fetchImpl = vi.fn(async () => resp(400, unrelatedError)) as unknown as typeof fetch;

const p = new OpenAICompatProvider('bad-model', 'https://openrouter.ai/api/v1', 'key', thresholds, httpWith(fetchImpl));
await expect(p.analyze(content(), new AbortController().signal)).rejects.toMatchObject({
kind: 'bad_response',
});
expect((fetchImpl as unknown as ReturnType<typeof vi.fn>).mock.calls.length).toBe(1);
});

it('maps 401 to a non-retryable auth error', async () => {
const fetchImpl = vi.fn(async () => resp(401, '{"error":"bad key"}')) as unknown as typeof fetch;
const p = new OpenAICompatProvider('m', 'https://openrouter.ai/api/v1', 'k', thresholds, httpWith(fetchImpl));
await expect(p.analyze(content(), new AbortController().signal)).rejects.toMatchObject({
kind: 'auth',
retryable: false,
});
expect((fetchImpl as unknown as ReturnType<typeof vi.fn>).mock.calls.length).toBe(1);
});

it('maps 429 to a retryable rate_limit error', async () => {
const fetchImpl = vi.fn(async () => resp(429, 'slow down')) as unknown as typeof fetch;
const p = new OpenAICompatProvider('m', 'https://openrouter.ai/api/v1', 'k', thresholds, httpWith(fetchImpl));
await expect(p.analyze(content(), new AbortController().signal)).rejects.toMatchObject({
kind: 'rate_limit',
retryable: true,
});
});

it('repairs once when the first response is unparseable', async () => {
const badBody = JSON.stringify({ choices: [{ message: { content: 'not json at all' } }] });
const fetchImpl = vi
.fn()
.mockResolvedValueOnce(resp(200, badBody))
.mockResolvedValueOnce(resp(200, validBody)) as unknown as typeof fetch;

const p = new OpenAICompatProvider('m', 'https://openrouter.ai/api/v1', 'k', thresholds, httpWith(fetchImpl));
const r = await p.analyze(content(), new AbortController().signal);

expect(r.meta.schemaRepaired).toBe(true);
expect((fetchImpl as unknown as ReturnType<typeof vi.fn>).mock.calls.length).toBe(2);
});

it('without a key: validate fails and analyze throws auth', async () => {
const p = new OpenAICompatProvider('m', 'https://openrouter.ai/api/v1', undefined, thresholds, httpWith(vi.fn() as unknown as typeof fetch));
expect((await p.validate()).ok).toBe(false);
await expect(p.analyze(content(), new AbortController().signal)).rejects.toMatchObject({
kind: 'auth',
});
});

it('validate returns ok on a successful round-trip', async () => {
const fetchImpl = vi.fn(async () => resp(200, JSON.stringify({ choices: [] }))) as unknown as typeof fetch;
const p = new OpenAICompatProvider('z-ai/glm-5.2', 'https://openrouter.ai/api/v1', 'key', thresholds, httpWith(fetchImpl));
const result = await p.validate();
expect(result.ok).toBe(true);
expect(result.detail).toContain('z-ai/glm-5.2');
});

it('strips trailing slash from base URL', async () => {
let url = '';
const fetchImpl = vi.fn(async (u: string) => {
url = u;
return resp(200, validBody);
}) as unknown as typeof fetch;

const p = new OpenAICompatProvider('m', 'https://openrouter.ai/api/v1/', 'k', thresholds, httpWith(fetchImpl));
await p.analyze(content(), new AbortController().signal);
expect(url).toBe('https://openrouter.ai/api/v1/chat/completions');
});
});

describe('estimateCost (openai-compat)', () => {
it('prices known model slugs and returns undefined for unknown ones', () => {
expect(estimateCost('z-ai/glm-5.2', 1_000_000, 1_000_000)).toBeCloseTo(1.2 + 4.1);
expect(estimateCost('gpt-4o-mini', 1_000_000, 1_000_000)).toBeCloseTo(0.15 + 0.6);
expect(estimateCost('some-unknown-model', 1000, 1000)).toBeUndefined();
});
});
Loading
Loading