From df03036dcb49e857fd9b74366bdafa7e17ecf55c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 12 Jun 2026 06:15:25 +0000 Subject: [PATCH] Add /research endpoint experiments to investigate topic-relevance bug Co-authored-by: SynthLuvr <131367121+SynthLuvr@users.noreply.github.com> --- .github/workflows/ci.yml | 6 ++ package.json | 3 +- research.experiment.ts | 174 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 research.experiment.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e45d07d..603564e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,3 +30,9 @@ jobs: env: ELYOS_API_KEY: ${{ secrets.ELYOS_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + - name: Run research endpoint experiments + run: pnpm test:experiments + continue-on-error: true + env: + ELYOS_API_KEY: ${{ secrets.ELYOS_API_KEY }} diff --git a/package.json b/package.json index f56e440..50eed45 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "lint:oxlint": "pnpm oxlint .", "lint": "npm-run-all lint:biome lint:oxlint", "strip-braces": "pnpm exec sg scan --rule .ast-grep/rules/strip-braces.yml -U", - "test": "pnpm vitest run" + "test": "pnpm vitest run", + "test:experiments": "pnpm vitest run research.experiment.ts" }, "dependencies": { "@clack/prompts": "^1.5.1", diff --git a/research.experiment.ts b/research.experiment.ts new file mode 100644 index 0000000..7236fea --- /dev/null +++ b/research.experiment.ts @@ -0,0 +1,174 @@ +/** + * Experiments to assess whether the /research endpoint: + * (A) is being called incorrectly, or + * (B) simply does not return topic-relevant content. + * + * Run with: npx tsx research.experiment.ts + * Requires ELYOS_API_KEY to be set. + */ + +import ky from "ky"; +import { describe, expect, it } from "vitest"; + +const BASE_URL = "https://elyos-interview-907656039105.europe-west2.run.app"; + +const TOPIC_KEYWORDS: Record = { + "solar energy": [ + "solar", + "energy", + "sun", + "panel", + "photovoltaic", + "renewable", + "electricity", + "power", + ], + "quantum computing": [ + "quantum", + "qubit", + "computing", + "superposition", + "entanglement", + ], + "ocean ecology": ["ocean", "marine", "sea", "coral", "fish", "ecosystem"], +}; + +const apiKey = process.env["ELYOS_API_KEY"] ?? ""; + +const api = ky.create({ + baseUrl: BASE_URL, + headers: { "X-API-Key": apiKey }, + timeout: 30_000, +}); + +const research = (topic: string) => + api.get("research", { searchParams: { topic } }).text(); + +const containsKeyword = (text: string, keywords: string[]) => + keywords.some((kw) => text.toLowerCase().includes(kw.toLowerCase())); + +describe("/research endpoint experiments", () => { + it("has ELYOS_API_KEY available", () => { + expect(apiKey.length, "ELYOS_API_KEY must be set").toBeGreaterThan(0); + }); + + /** + * Experiment 1 – Consistency + * Call the same topic 5 times. Do we get the same (or similar) response + * each time, or does the content vary unpredictably? + */ + it("experiment 1: same topic returns consistent responses", async () => { + const topic = "solar energy"; + const responses: string[] = []; + + for (let i = 0; i < 5; i++) { + responses.push(await research(topic)); + } + + console.log("\n--- Experiment 1: Consistency (topic: solar energy) ---"); + for (const [i, r] of responses.entries()) { + const relevant = containsKeyword(r, TOPIC_KEYWORDS[topic]!); + console.log( + ` Run ${i + 1}: ${r.length} chars | topic-relevant: ${relevant} | preview: ${r.slice(0, 120).replace(/\n/g, " ")}`, + ); + } + + const relevantCount = responses.filter((r) => + containsKeyword(r, TOPIC_KEYWORDS[topic]!), + ).length; + + console.log( + ` Topic-relevant responses: ${relevantCount}/${responses.length}`, + ); + + // This assertion intentionally documents the observed behavior. + // If the endpoint works correctly all 5 should be relevant. + expect( + relevantCount, + "At least one response should be topic-relevant", + ).toBeGreaterThan(0); + }); + + /** + * Experiment 2 – Specificity + * Call three distinct topics. Do the responses differ per topic, + * or does the endpoint return identical (cached / generic) content? + */ + it("experiment 2: different topics return different responses", async () => { + const topics = Object.keys(TOPIC_KEYWORDS); + const responses: Record = {}; + + for (const topic of topics) { + responses[topic] = await research(topic); + } + + console.log("\n--- Experiment 2: Specificity ---"); + for (const topic of topics) { + const r = responses[topic]!; + const relevant = containsKeyword(r, TOPIC_KEYWORDS[topic]!); + console.log( + ` topic: "${topic}" | relevant: ${relevant} | preview: ${r.slice(0, 120).replace(/\n/g, " ")}`, + ); + } + + // Check whether all three responses are identical (sign of caching / topic-blindness) + const unique = new Set(Object.values(responses)).size; + console.log(` Unique responses: ${unique}/${topics.length}`); + + expect( + unique, + "Different topics should yield different responses", + ).toBeGreaterThan(1); + }); + + /** + * Experiment 3 – Topic-relevance check + * Verify that the response for "solar energy" contains at least one + * expected keyword, confirming the endpoint honours the topic param. + */ + it("experiment 3: solar energy response contains topic keywords", async () => { + const topic = "solar energy"; + const keywords = TOPIC_KEYWORDS[topic]!; + const response = await research(topic); + + const foundKeywords = keywords.filter((kw) => + response.toLowerCase().includes(kw.toLowerCase()), + ); + + console.log("\n--- Experiment 3: Keyword check (solar energy) ---"); + console.log(` Response length: ${response.length} chars`); + console.log(` Full response:\n${response}\n`); + console.log(` Expected keywords: ${keywords.join(", ")}`); + console.log( + ` Found keywords: ${foundKeywords.length > 0 ? foundKeywords.join(", ") : "(none)"}`, + ); + + expect( + foundKeywords.length, + `Response should contain at least one of: ${keywords.join(", ")}`, + ).toBeGreaterThan(0); + }); + + /** + * Experiment 4 – Empty response handling + * How often does the endpoint return an empty or trivially short response? + * Calls the endpoint 5 times and counts empty/near-empty responses. + */ + it("experiment 4: measure empty response rate", async () => { + const topic = "solar energy"; + const results: { length: number; empty: boolean }[] = []; + + for (let i = 0; i < 5; i++) { + const r = await research(topic); + results.push({ length: r.length, empty: r.length < 10 }); + } + + const emptyCount = results.filter((r) => r.empty).length; + + console.log("\n--- Experiment 4: Empty response rate ---"); + for (const [i, r] of results.entries()) { + console.log(` Run ${i + 1}: ${r.length} chars | empty: ${r.empty}`); + } + console.log(` Empty responses: ${emptyCount}/${results.length}`); + }); +});