diff --git a/server/_core/embeddings/degradation-alert.ts b/server/_core/embeddings/degradation-alert.ts new file mode 100644 index 00000000..50a2d45f --- /dev/null +++ b/server/_core/embeddings/degradation-alert.ts @@ -0,0 +1,62 @@ +/** + * Silent-degradation detection for clinical search. + * + * Two failure modes must never reach a paramedic silently: + * 1. Semantic search degrades to keyword-only (embedding 403 / invalid key / + * circuit open) — results still return but retrieval quality silently drops. + * 2. A clinical query WITH a resolved agency returns zero results — a blank + * screen for e.g. "chest pain dose" mid-call. + * + * Both are surfaced as Sentry alerts (operator signal) in addition to logs. + * Required before TestFlight beta per the resubmission AI-council (2026-06-01). + */ +import { captureMessage } from '../sentry'; +import { logger } from '../logger'; + +export interface EmbeddingDegradedDetail { + reason: 'circuit_open' | 'embedding_error'; + query: string; + agencyId?: number | null; + err?: unknown; +} + +/** + * Fire when semantic search falls back to keyword-only because embeddings are + * unavailable. Operators must know clinical retrieval is degraded, not just see + * a buried logger.warn. + */ +export function alertEmbeddingDegraded(detail: EmbeddingDegradedDetail): void { + logger.error( + { reason: detail.reason, agencyId: detail.agencyId ?? null, err: detail.err }, + '[Degradation] Semantic search degraded to keyword-only', + ); + captureMessage( + `Clinical search degraded to keyword-only (${detail.reason}); semantic embeddings unavailable`, + 'error', + ); +} + +export interface SilentEmptyDetail { + query: string; + agencyId: number | null; + agencyName?: string | null; + isClinical: boolean; +} + +/** + * Fire when a clinical query that resolved to a real agency returns zero + * results — the "blank screen mid-call" case. Only fires when an agency is + * resolved AND the query is clinical, so unauthenticated / no-context probes + * that are empty by design do not alert. + */ +export function alertSilentEmptyResult(detail: SilentEmptyDetail): void { + if (detail.agencyId == null || !detail.isClinical) return; + logger.error( + { agencyId: detail.agencyId, agencyName: detail.agencyName ?? null, query: detail.query }, + '[Degradation] Clinical query with resolved agency returned ZERO results', + ); + captureMessage( + `Clinical search returned ZERO results for a resolved agency (agencyId=${detail.agencyId}) — possible silent retrieval failure`, + 'error', + ); +} diff --git a/server/_core/embeddings/search.ts b/server/_core/embeddings/search.ts index 86be7d41..988bfaee 100644 --- a/server/_core/embeddings/search.ts +++ b/server/_core/embeddings/search.ts @@ -16,6 +16,7 @@ import type { SupabaseRpcResult, SearchResult } from './types'; // Import and re-export keyword search utilities for backward compatibility import { extractProtocolNumber, keywordOnlySearch, mergeKeywordFallbackResults } from './keyword-search'; import { logger } from '../logger'; +import { alertEmbeddingDegraded } from './degradation-alert'; import { getCanonicalTitle } from './protocol-titles'; export { extractProtocolNumber, keywordOnlySearch }; @@ -174,7 +175,8 @@ export async function semanticSearchProtocols(params: { // Check if Gemini is available - fallback to BM25 if not if (!isGeminiAvailable()) { - logger.warn('[Search] Gemini unavailable, using BM25 fallback'); + // ALERT: embedding circuit breaker open — clinical search degraded to keyword-only. + alertEmbeddingDegraded({ reason: 'circuit_open', query, agencyId }); // Use keyword fallback const fallbackResults = await keywordOnlySearch({ @@ -194,8 +196,9 @@ export async function semanticSearchProtocols(params: { try { queryEmbedding = await generateEmbedding(expandedQuery); } catch (error) { - // If embedding fails, fallback to BM25 - logger.warn({ err: error }, '[Search] Embedding generation failed, using BM25 fallback'); + // If embedding fails, fallback to BM25 — but ALERT: clinical search just + // silently degraded to keyword-only (council pre-beta requirement). + alertEmbeddingDegraded({ reason: 'embedding_error', query, agencyId, err: error }); const fallbackResults = await keywordOnlySearch({ query, agencyId, diff --git a/server/routers/search/semantic.ts b/server/routers/search/semantic.ts index 016c3348..1a7b8de5 100644 --- a/server/routers/search/semantic.ts +++ b/server/routers/search/semantic.ts @@ -24,6 +24,7 @@ import { validateSearchLimit, getUserTierFeatures } from "../../_core/tier-valid import { toStateCode } from "../../lib/state-codes"; import { incrementAndCheckQueryLimit } from "../../db/users-usage"; import { logger } from "../../_core/logger"; +import { alertSilentEmptyResult } from "../../_core/embeddings/degradation-alert"; import { logQuery, createQueryLogEntry } from "../../_core/query-analytics"; import { getDrugByName } from "../../db/drugs"; import { @@ -273,6 +274,18 @@ export const semanticRouter = router({ latencyMs, }; + // ALERT: a clinical query that resolved to a real agency came back empty + // — the "blank screen mid-call" case. Only fires when an agency resolved + // and the query is clinical (council pre-beta requirement 2026-06-01). + if (response.totalFound === 0) { + alertSilentEmptyResult({ + query: input.query, + agencyId, + agencyName, + isClinical: isDrugRelated || isMedicationQuery, + }); + } + // Enrich medication queries with drug reference data if (isMedicationQuery && normalized.extractedMedications.length > 0) { try { diff --git a/tests/embedding-degradation-alert.test.ts b/tests/embedding-degradation-alert.test.ts new file mode 100644 index 00000000..27087ca9 --- /dev/null +++ b/tests/embedding-degradation-alert.test.ts @@ -0,0 +1,51 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import * as sentry from "../server/_core/sentry"; +import { + alertEmbeddingDegraded, + alertSilentEmptyResult, +} from "../server/_core/embeddings/degradation-alert"; + +/** + * Clinical-safety regression: silent-degradation detection must surface to the + * operator (Sentry), and must NOT false-alarm on benign empty results. + * Required before TestFlight beta per the resubmission AI-council (2026-06-01). + */ +describe("silent-degradation alerts", () => { + let spy: ReturnType; + + beforeEach(() => { + spy = vi.spyOn(sentry, "captureMessage").mockImplementation(() => {}); + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("alerts at 'error' level when semantic search degrades to keyword-only (embedding error)", () => { + alertEmbeddingDegraded({ reason: "embedding_error", query: "chest pain dose", agencyId: 12 }); + expect(spy).toHaveBeenCalledTimes(1); + expect(spy.mock.calls[0][1]).toBe("error"); + expect(String(spy.mock.calls[0][0])).toContain("keyword-only"); + }); + + it("alerts when the embedding circuit breaker is open", () => { + alertEmbeddingDegraded({ reason: "circuit_open", query: "stroke", agencyId: null }); + expect(spy).toHaveBeenCalledTimes(1); + expect(String(spy.mock.calls[0][0])).toContain("circuit_open"); + }); + + it("alerts when a clinical query with a resolved agency returns ZERO results", () => { + alertSilentEmptyResult({ query: "chest pain dose", agencyId: 7, isClinical: true }); + expect(spy).toHaveBeenCalledTimes(1); + expect(String(spy.mock.calls[0][0])).toContain("agencyId=7"); + }); + + it("does NOT alert on empty when no agency resolved (empty-by-design probe)", () => { + alertSilentEmptyResult({ query: "chest pain", agencyId: null, isClinical: true }); + expect(spy).not.toHaveBeenCalled(); + }); + + it("does NOT alert on empty for a non-clinical query", () => { + alertSilentEmptyResult({ query: "about page", agencyId: 7, isClinical: false }); + expect(spy).not.toHaveBeenCalled(); + }); +});