diff --git a/.gitignore b/.gitignore index acd0379..026c488 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ _pypi_smoke/ # Recipe outputs — receipts and demo files written when running recipes/ recipes/*.json alice_purge_receipt.json +.coverage diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..21cea05 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,24 @@ +cff-version: 1.2.0 +title: "Lethe: AI memory built to forget" +message: "If you use Lethe or ForgetEval in your research, please cite the paper below." +type: software +authors: + - family-names: Yang + given-names: Dongxu + affiliation: DeepLethe +repository-code: "https://github.com/deeplethe/lethe" +license: MIT +preferred-citation: + type: article + title: "Control-Plane Placement Shapes Forgetting: An Architectural Study of Agent Memory Across Thirteen System Configurations" + authors: + - family-names: Yang + given-names: Dongxu + affiliation: DeepLethe + year: 2026 + journal: "arXiv preprint" + url: "https://arxiv.org/abs/2606.15903" + identifiers: + - type: other + value: "arXiv:2606.15903" + description: "arXiv preprint identifier" diff --git a/README.md b/README.md index 65d4c99..f245780 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@

+ arXiv PyPI tests python @@ -107,29 +108,54 @@ production. Pass / fail is exact substring matching on top-k recall, no LLM judge, deterministic. Full methodology: **[docs/forgeteval.md](docs/forgeteval.md)**. -| System | super | decay | amnesia | purge | drift | Overall | -|---------------|------:|------:|--------:|------:|------:|------------------------------:| -| **Lethe v1** | 100% | 100% | 98% | 100% | 99% | **99.3%** (993 / 1000) | -| Mem0 (2.0.2) | 100% | 100% | 70% | 75% | 100% | 88.8% | -| MemPalace | 0% | 0% | 0% | 0% | 0% | 0% (no forgetting primitives) | - -Mem0 ties on supersession / decay / drift but breaks at the precision -operations: forgetting one entity without bleeding into near-neighbors -(amnesia 70%) and deleting by identifier without over-pruning siblings -(purge 75%). MemPalace's zeros are not a benchmark failure — they -are an honest report that the library was built without `supersede`, -`release`, or `purge`. - -In production this maps to real failures. **70% amnesia** means three -in ten *"forget this user"* requests leave fragments reachable to -other queries — a GDPR liability and a stale-context bug. **75% -purge** means one in four deletions either miss the target or take a -neighbor with them — the silent delete-by-similarity failure that -bricks compliance audits. **MemPalace's 0%** is the opposite failure: -a GDPR Article 17 right-to-be-forgotten request becomes a manual -data-migration project, not a one-line API call. - -Reproduce: `py bench/forgeteval/run.py --adapter {lethe|mem0|mempalace} --scale 200` +| System | super | decay | amnesia | purge | drift | Overall | +|-------------------|------:|------:|--------:|------:|------:|------------------------------:| +| **Lethe v1** | 100% | 100% | 98% | 100% | 99% | **99.3%** (993 / 1000) | +| LangMem (LangGraph) | 100% | 100% | 98% | 100% | 99% | 99.5% (995 / 1000) | +| Mem0 (2.0.2) | 100% | 100% | 70% | 75% | 100% | 88.8% | +| MemPalace | 0% | 0% | 0% | 0% | 0% | 0% (no forgetting primitives) | + +The template suite is near-saturated for vector-store-backed +systems with adaptive eviction: Lethe and LangGraph's `InMemoryStore` +both clear 99 %. Mem0's gap is on amnesia (70 %) and purge (75 %), +the two families that probe width-control and identifier precision. +MemPalace returns 0 because the API has no deletion primitives. + +For more discriminative comparison we run **ForgetEval-Adv**, a +112-case hand-crafted layer covering 10 attack categories +(substring traps, prefix collisions, paraphrase supersession, +negation, temporal qualifiers, shared attributes, compound facts, +identifier obfuscation, cross-lingual identifiers, recursive +supersession). See [docs/forgeteval_adversarial.md](docs/forgeteval_adversarial.md). + +| System | adversarial overall | wall / case | trade-off shape | +|---------------------|---------------------:|-------------:|------------------------------------------------| +| **Lethe v1** | 70 / 112 (62.5 %) | ~48 ms | 100 % prefix_collision, 0 % cross_lingual | +| Mem0 v2.0.2 | 76 / 112 (67.9 %) | ~527 ms | 50 % prefix_collision, 50 % cross_lingual | +| LangMem (LangGraph) | 69 / 112 (61.6 %) | ~56 ms | 94 % prefix_collision, 0 % cross_lingual | +| MemPalace | 0 / 112 ( 0.0 %) | ~167 ms | no deletion primitives | +| **Lethe + LLM** | **108 / 112 (96.4 %)** | ~2.2 s (mutations only) | 100 % cross_lingual, 100 % shared_attribute; 8 / 10 categories at 100 % | + +The Lethe+LLM row uses the optional `llm: Callable[[str], str]` +hook on `LetheAdapter` wired to DeepSeek-V3 via SiliconFlow. +Cost: ~$0.05 for a full 112-case run. The recall hot path +remains LLM-free; only the three mutation operations (`supersede`, +`purge`, `release`) consult the model. + +Statistically separated per-category claims at p < 0.05 (non- +overlapping Wilson 95 % CIs at n=16): +**Lethe > Mem0 on prefix_collision** (lexical-precise purge wins); +**Mem0 > Lethe / LangMem on cross_lingual_identifier** (vector-soft +matching wins). Overall Wilson CIs of the three deterministic +systems overlap — the bench reads the trade-off, not a winner. + +For attack categories that need semantic understanding the engine +deliberately doesn't provide (compound_fact across all 3 systems, +identifier_obfuscation for Lethe / LangMem), the +`LetheAdapter(llm=...)` hook routes those decisions to an LLM at +mutation time; the recall hot path stays LLM-free. + +Reproduce: `py bench/forgeteval/run.py --adapter {lethe|mem0|langmem|mempalace} --suite {template,adversarial}` > ForgetEval is downstream of the depth model — and the depth model > is downstream of ForgetEval. A failing `purge_gdpr` case in early diff --git a/bench/forgeteval/adapter.py b/bench/forgeteval/adapter.py index f662aab..10683f0 100644 --- a/bench/forgeteval/adapter.py +++ b/bench/forgeteval/adapter.py @@ -36,16 +36,148 @@ def purge(self, query: str) -> int: # ─── Lethe adapter ──────────────────────────────────────────────────── +# LLM-hook prompts. Kept as module-level constants so they're easy to +# audit and to swap. Both prompts ask the model for a small, well- +# structured JSON response — no free-form prose, no embedded reasoning. +# This keeps the parsing tiny and the LLM call narrow. + +LLM_PROMPT_SUPERSEDE = """\ +You are deciding how to apply a memory supersession. + +EXISTING_MEMORY: {old_text} +SUPERSEDE_QUERY: {query} +NEW_FACT: {new_text} + +**Default is ATOMIC** — replace the whole memory with NEW_FACT. +Choose ATOMIC when EXISTING_MEMORY's topic matches SUPERSEDE_QUERY +(even if paraphrased, dated, or recursive). + +Choose PARTIAL when EXISTING_MEMORY combines two **distinct-topic** +facts (different attributes about the same subject, like +location-vs-employer or marital-status-vs-employer), and +SUPERSEDE_QUERY targets only one attribute. In partial mode, +preserve the unaffected attribute and splice NEW_FACT into the +addressed one. + +Examples +-------- + +EXISTING_MEMORY: User lives in Berlin and works at Stripe as a backend engineer. +SUPERSEDE_QUERY: user city of residence +NEW_FACT: User relocated to Madrid and continues working remotely. +→ {{"mode": "partial", "merged_text": "User relocated to Madrid and works at Stripe as a backend engineer."}} + +EXISTING_MEMORY: User is married to Jamie and works at Google. +SUPERSEDE_QUERY: user employer Google +NEW_FACT: User joined Anthropic as a research engineer. +→ {{"mode": "partial", "merged_text": "User is married to Jamie and joined Anthropic as a research engineer."}} + +EXISTING_MEMORY: User does NOT work at Anthropic and has never interviewed there. +SUPERSEDE_QUERY: user Anthropic employment status +NEW_FACT: User actually joined Anthropic last quarter. +→ {{"mode": "atomic"}} (both clauses are about Anthropic-employment — atomic supersedes them together) + +EXISTING_MEMORY: User joined Google in 2020. +SUPERSEDE_QUERY: user current employer +NEW_FACT: User joined Meta in 2022. +→ {{"mode": "atomic"}} (single-topic supersession) + +Format +------ +Reply with exactly one JSON object and nothing else: + {{"mode": "atomic"}} +or + {{"mode": "partial", "merged_text": ""}} + +Do not add facts not present in either source. +""" + +LLM_PROMPT_PURGE_MATCH = """\ +You are grouping memory items that describe the same underlying +identifier. Surface-form variations (case, whitespace, quoting, +leading @, optional separators in phone numbers / UUIDs / SSNs / +credit cards) all count as the same identifier. + +TARGET_IDENTIFIER: {target} + +CANDIDATES (one per line, indexed from 0): +{candidates} + +Return exactly one JSON object listing the candidate indices whose +text describes the SAME identifier as the target. Distinct +identifiers that happen to share a prefix (e.g. alice@acme.io vs +alice.smith@acme.io, 12345 vs 123456) MUST NOT be grouped. + + {{"matching_indices": [, ...]}} +""" + + +LLM_PROMPT_RELEASE_MATCH = """\ +You are deciding which memory items should be released (soft-deleted) +based on a natural-language release request. + +RELEASE_REQUEST: {request} + +CANDIDATES (one per line, indexed from 0): +{candidates} + +Return exactly one JSON object listing the candidate indices whose +content should be released according to the request. Include a +candidate if and only if it (a) describes the entity or topic the +request targets, OR (b) mentions that entity even as part of a +compound statement. Do NOT include candidates that only share an +attribute (e.g. same city, same job) with the target — those are +sibling facts, not target facts. + + {{"matching_indices": [, ...]}} +""" + + +def _parse_json_response(s: str) -> dict: + """Extract the first JSON object from a model response, tolerating + fenced code blocks or surrounding prose. Raises ValueError if no + JSON object is found.""" + import json + import re as _re + m = _re.search(r"\{[\s\S]*\}", s) + if not m: + raise ValueError(f"no JSON object in model response: {s!r}") + return json.loads(m.group(0)) + + class LetheAdapter: + """Default ForgetEval adapter for Lethe. + + With ``llm=None`` (the default), the adapter exposes only Lethe's + deterministic primitives and ships no semantic heuristics: + + - supersede → atomic: wipe the best BM25 match, inscribe new + - release → adaptive-gap (a documented numerical procedure, + not a string heuristic) + - purge → case-insensitive equality of the BM25-top-1 text, + plus exact-text duplicates + + With ``llm`` set to a ``Callable[[str], str]`` that takes a prompt + and returns a model response, the adapter routes semantic decisions + through the LLM — clause-aware supersede and identifier-equivalent + purge. The recall hot path remains LLM-free in both modes; only + the explicit mutation operations consult the model, and only once + per call. + """ + name = "lethe" - def __init__(self, embedder, vector_dim: int = 384, llm=None): + def __init__(self, embedder, vector_dim: int = 384, *, + llm=None, lethe_llm=None): from lethe import Lethe self._Lethe = Lethe self.embedder = embedder self.vector_dim = vector_dim - self.llm = llm + self.llm = llm # for supersede/purge planning + self.lethe_llm = lethe_llm # passed through to Lethe(...) self.lethe = None + if llm is not None: + self.name = "lethe+llm" def reset(self) -> None: if self.lethe is not None: @@ -56,7 +188,7 @@ def reset(self) -> None: self.lethe = self._Lethe(":memory:", vector_dim=self.vector_dim, embedder=self.embedder, - llm=self.llm) + llm=self.lethe_llm) def inscribe(self, text: str) -> int: return self.lethe.inscribe(text) @@ -65,23 +197,48 @@ def recall_texts(self, query: str, k: int = 5) -> list[str]: results = self.lethe.recall(query, k=k, hybrid=False) return [r.memory.text for r in results] + # ─── supersede ────────────────────────────────────────────────── + def supersede(self, old_query: str, new_text: str) -> None: - # Find the best-matching memory for old_query, supersede it with new_text. hits = self.lethe.recall(old_query, k=1, hybrid=False) if not hits: - # If nothing matches, just inscribe the new fact. self.lethe.inscribe(new_text) return + target = hits[0] + + if self.llm is not None: + plan = self._llm_plan_supersede(target.memory.text, + old_query, new_text) + if plan.get("mode") == "partial": + merged = plan.get("merged_text") or "" + if merged.strip(): + self.lethe.surrender(target.memory.id, mode="edit", + new_text=merged) + return + # mode "atomic" or unrecognized → fall through to atomic + self.lethe.surrender( - {"old": hits[0].memory.id, "new": new_text}, + {"old": target.memory.id, "new": new_text}, mode="supersede", ) + def _llm_plan_supersede(self, old_text: str, old_query: str, + new_text: str) -> dict: + prompt = LLM_PROMPT_SUPERSEDE.format( + old_text=old_text, query=old_query, new_text=new_text, + ) + try: + return _parse_json_response(self.llm(prompt)) + except Exception: + return {"mode": "atomic"} # any failure → safe default + + # ─── release ──────────────────────────────────────────────────── + @staticmethod def _gap_threshold(sims: list[float], min_gap: float = 0.05) -> float: """Find the natural cutoff in a sorted-descending similarity list: the midpoint of the largest gap. Falls back to top * 0.95 when - there is no significant gap (i.e. only one tight cluster of hits).""" + there is no significant gap (only one tight cluster of hits).""" if not sims: return float("inf") if len(sims) == 1: @@ -97,9 +254,22 @@ def _gap_threshold(sims: list[float], min_gap: float = 0.05) -> float: return best_mid if best_gap >= min_gap else s[0] * 0.95 def release(self, query: str) -> int: - hits = self.lethe.recall(query, k=20, hybrid=False) + # Hybrid recall (vec + BM25 via RRF) for release. For + # identifier-shaped queries the BM25 leg sharpens the + # ranking; for natural-language queries the vec leg carries + # the semantic load. RRF weights both — no detection + # heuristic needed. + hits = self.lethe.recall(query, k=20, hybrid=True) if not hits: return 0 + + if self.llm is not None: + matched = self._llm_match_for_release(query, hits) + if matched: + self.lethe.surrender(matched, mode="release") + return len(matched) + # empty LLM result → fall through to adaptive-gap below + thr = self._gap_threshold([h.similarity for h in hits]) ids = [h.memory.id for h in hits if h.similarity >= thr] if not ids: @@ -107,20 +277,65 @@ def release(self, query: str) -> int: self.lethe.surrender(ids, mode="release") return len(ids) + def _llm_match_for_release(self, query: str, hits: list) -> list[int]: + candidates = "\n".join(f"{i}: {h.memory.text}" + for i, h in enumerate(hits)) + prompt = LLM_PROMPT_RELEASE_MATCH.format( + request=query, candidates=candidates, + ) + try: + plan = _parse_json_response(self.llm(prompt)) + picks = plan.get("matching_indices") or [] + return [hits[i].memory.id for i in picks + if isinstance(i, int) and 0 <= i < len(hits)] + except Exception: + return [] + + # ─── purge ────────────────────────────────────────────────────── + + @staticmethod + def _norm_lexical(s: str) -> str: + """Minimal text normalization: NFKC + lowercase + collapsed + whitespace. This is plain Unicode hygiene — not an identifier- + aware canonicalizer. Two strings that differ only in case or + whitespace will compare equal under this norm; anything else + (separators, quoting, format variations) is left to the LLM.""" + import unicodedata + return " ".join(unicodedata.normalize("NFKC", s).lower().split()) + def purge(self, query: str) -> int: - # Purge by identifier is a *lexical* operation, not a semantic one. - # Two customers named alice@acme / bob@acme look almost identical - # to a vector embedder, and RRF blending can invert the rank — so - # we delete by pure BM25 ranking instead. Take the top-1 plus any - # exact-text duplicates (same fact inscribed twice). - hits = self.lethe.recall(query, k=5, lexical=True) + hits = self.lethe.recall(query, k=20, lexical=True) if not hits: return 0 - target_text = hits[0].memory.text - ids = [h.memory.id for h in hits if h.memory.text == target_text] + + if self.llm is not None: + matched = self._llm_match_for_purge(query, hits) + if matched: + self.lethe.surrender(matched, mode="purge") + return len(matched) + # Empty LLM result → fall through to default below. + + # Default: group by NFKC-lowercase-whitespace equivalence. + target = self._norm_lexical(hits[0].memory.text) + ids = [h.memory.id for h in hits + if self._norm_lexical(h.memory.text) == target] self.lethe.surrender(ids, mode="purge") return len(ids) + def _llm_match_for_purge(self, query: str, hits: list) -> list[int]: + candidates = "\n".join(f"{i}: {h.memory.text}" + for i, h in enumerate(hits)) + prompt = LLM_PROMPT_PURGE_MATCH.format( + target=query, candidates=candidates, + ) + try: + plan = _parse_json_response(self.llm(prompt)) + picks = plan.get("matching_indices") or [] + return [hits[i].memory.id for i in picks + if isinstance(i, int) and 0 <= i < len(hits)] + except Exception: + return [] + # ─── Mem0 adapter ───────────────────────────────────────────────────── @@ -253,6 +468,411 @@ def purge(self, query: str) -> int: return self._delete_matching(query) +# ─── LangGraph / LangMem adapter ────────────────────────────────────── + +class LangGraphAdapter: + """LangChain's default agent-memory primitive: LangGraph's + ``InMemoryStore`` with vector indexing. This is the bare BaseStore + that the LangMem package layers on top of — we benchmark the + underlying store directly so the comparison is between *storage* + primitives, not between LLM-driven memory managers. + + LangMem's higher-level manager calls an LLM at every write to + extract/dedup/delete memories, which (a) breaks the LLM-free + invariant we hold for ForgetEval and (b) makes supersession + implicit rather than programmatic. The InMemoryStore baseline + here is what an engineer gets when they use LangChain "out of the + box" without wiring an LLM into the write path. + + Primitive coverage: + + inscribe ↦ store.put((ns,), key=uuid, value={"text": text}) + recall ↦ store.search((ns,), query=q, limit=k) + supersede ↦ delete(old_key) + put(new uuid, new_text) + release ↦ delete (no soft-delete in BaseStore) + purge ↦ delete top-1 BM25-equivalent + + No external service, no API key, pure CPU. + """ + + name = "langmem" + + def __init__(self, embedder, vector_dim: int = 384): + try: + from langgraph.store.memory import InMemoryStore + except ImportError as e: # pragma: no cover + raise ImportError("pip install langmem (pulls langgraph)") from e + self._Store = InMemoryStore + self.embedder = embedder + self.vector_dim = vector_dim + self.store = None + self.ns = ("forget_eval",) + + def _embed_batch(self, texts: list[str]) -> list[list[float]]: + return [list(self.embedder(t)) for t in texts] + + def reset(self) -> None: + # InMemoryStore is reset by re-instantiating; embed callback + # is invoked by .put/.search internally on the text field. + self.store = self._Store( + index={"dims": self.vector_dim, + "embed": self._embed_batch, + "fields": ["text"]}, + ) + + def inscribe(self, text: str) -> str: + import uuid + key = uuid.uuid4().hex + self.store.put(self.ns, key, {"text": text}) + return key + + def recall_texts(self, query: str, k: int = 5) -> list[str]: + hits = self.store.search(self.ns, query=query, limit=k) + return [h.value.get("text", "") for h in hits] + + def supersede(self, old_query: str, new_text: str) -> None: + hits = self.store.search(self.ns, query=old_query, limit=1) + if hits: + self.store.delete(self.ns, hits[0].key) + import uuid + self.store.put(self.ns, uuid.uuid4().hex, {"text": new_text}) + + def release(self, query: str) -> int: + hits = self.store.search(self.ns, query=query, limit=20) + if not hits: + return 0 + # Use the same adaptive-gap policy as LetheAdapter for fairness. + scores = [h.score or 0.0 for h in hits] + thr = LetheAdapter._gap_threshold(scores) + evicted = 0 + for h in hits: + if (h.score or 0.0) >= thr: + self.store.delete(self.ns, h.key) + evicted += 1 + return evicted + + def purge(self, query: str) -> int: + hits = self.store.search(self.ns, query=query, limit=20) + if not hits: + return 0 + target_text = hits[0].value.get("text", "") + purged = 0 + for h in hits: + if h.value.get("text", "") == target_text: + self.store.delete(self.ns, h.key) + purged += 1 + return purged + + +# ─── LangGraph + LLM hook adapter ───────────────────────────────────── + +class LangGraphLLMAdapter(LangGraphAdapter): + """LangGraph InMemoryStore augmented with the same LLM-hook contract + as LetheAdapter (supersede planner, release-match, purge-match). + Used to disentangle the LLM-hook lift from the edit-primitive lift + in the ablation: LangGraph has no in-place edit, so for partial + supersede the LLM hook returns a merged text which we add as a + fresh row replacing the old (functionally equivalent to Lethe's + edit primitive under substring scoring).""" + + name = "langmem_llm" + + def __init__(self, embedder, vector_dim: int = 384, *, + llm=None): + super().__init__(embedder=embedder, vector_dim=vector_dim) + self.llm = llm + + def supersede(self, old_query: str, new_text: str) -> None: + hits = self.store.search(self.ns, query=old_query, limit=1) + if not hits: + import uuid + self.store.put(self.ns, uuid.uuid4().hex, {"text": new_text}) + return + target = hits[0] + target_text = target.value.get("text", "") + + # LLM-planned supersede: atomic (replace whole row) vs partial + # (merge one clause into a fresh row). + if self.llm is not None: + try: + prompt = LLM_PROMPT_SUPERSEDE.format( + old_text=target_text, query=old_query, new_text=new_text, + ) + plan = _parse_json_response(self.llm(prompt)) + except Exception: + plan = {"mode": "atomic"} + if plan.get("mode") == "partial": + merged = plan.get("merged_text") or "" + if merged.strip(): + # No native edit primitive -> delete old + add merged. + self.store.delete(self.ns, target.key) + import uuid + self.store.put(self.ns, uuid.uuid4().hex, + {"text": merged}) + return + # atomic / fallthrough: delete old + add new + self.store.delete(self.ns, target.key) + import uuid + self.store.put(self.ns, uuid.uuid4().hex, {"text": new_text}) + + def release(self, query: str) -> int: + hits = self.store.search(self.ns, query=query, limit=20) + if not hits: + return 0 + + if self.llm is not None: + try: + candidates = "\n".join( + f"{i}: {h.value.get('text', '')}" + for i, h in enumerate(hits) + ) + prompt = LLM_PROMPT_RELEASE_MATCH.format( + request=query, candidates=candidates, + ) + plan = _parse_json_response(self.llm(prompt)) + picks = plan.get("matching_indices") or [] + keys = [hits[i].key for i in picks + if isinstance(i, int) and 0 <= i < len(hits)] + if keys: + for k in keys: + self.store.delete(self.ns, k) + return len(keys) + except Exception: + pass # fall through to gap-threshold + + scores = [h.score or 0.0 for h in hits] + thr = LetheAdapter._gap_threshold(scores) + evicted = 0 + for h in hits: + if (h.score or 0.0) >= thr: + self.store.delete(self.ns, h.key) + evicted += 1 + return evicted + + def purge(self, query: str) -> int: + hits = self.store.search(self.ns, query=query, limit=20) + if not hits: + return 0 + + if self.llm is not None: + try: + candidates = "\n".join( + f"{i}: {h.value.get('text', '')}" + for i, h in enumerate(hits) + ) + prompt = LLM_PROMPT_PURGE_MATCH.format( + target=query, candidates=candidates, + ) + plan = _parse_json_response(self.llm(prompt)) + picks = plan.get("matching_indices") or [] + keys = [hits[i].key for i in picks + if isinstance(i, int) and 0 <= i < len(hits)] + if keys: + for k in keys: + self.store.delete(self.ns, k) + return len(keys) + except Exception: + pass # fall through to default + + # Default: NFKC-lowercase-whitespace equivalence on text. + target_text = LetheAdapter._norm_lexical(hits[0].value.get("text", "")) + purged = 0 + for h in hits: + if LetheAdapter._norm_lexical(h.value.get("text", "")) == target_text: + self.store.delete(self.ns, h.key) + purged += 1 + return purged + + +# ─── Cognee adapter (requires LLM API key) ──────────────────────────── + +class CogneeAdapter: + """Cognee v1's ``remember`` / ``recall`` / ``forget`` / ``improve`` + API. The v1 release explicitly mirrors a forgetting-aware verb set + at the top level — making it the closest API-level analog to + Lethe's surrender modes in the ecosystem. + + *Requires:* ``pip install cognee`` and an ``LLM_API_KEY`` + environment variable (OpenAI by default; local LLM via Ollama + extras possible). ``cognify`` invokes the LLM unconditionally, + so this adapter cannot run in a fully offline environment. + + Primitive coverage (per the Cognee v1 quickstart): + + inscribe ↦ await cognee.remember(text) + recall ↦ await cognee.recall(query) + supersede ↦ chain forget(by_query) + remember(new) + release ↦ not exposed (NotImplementedError) + purge ↦ await cognee.forget(by_text=query) + NOTE: per-query forget granularity is undocumented + in v1.0.9; only forget(everything=True) is fully + public. Adapter falls back to atomic supersede. + """ + + name = "cognee" + + def __init__(self, dataset: str = "forget_eval"): + try: + import cognee + except ImportError as e: # pragma: no cover + raise ImportError("pip install cognee") from e + self._cognee = cognee + self.dataset = dataset + + def _run(self, coro): + import asyncio + return asyncio.get_event_loop().run_until_complete(coro) + + def reset(self) -> None: + # Best-effort: forget all then re-prune the dataset. + try: + self._run(self._cognee.forget(dataset=self.dataset, everything=True)) + except Exception: + pass + + def inscribe(self, text: str) -> str: + self._run(self._cognee.remember(text, dataset=self.dataset)) + return "" # cognee does not expose stable per-fact ids + + def recall_texts(self, query: str, k: int = 5) -> list[str]: + results = self._run(self._cognee.recall(query, dataset=self.dataset)) + out: list[str] = [] + for r in (results or [])[:k]: + if isinstance(r, dict): + out.append(r.get("text") or r.get("content") or str(r)) + else: + out.append(str(r)) + return out + + def supersede(self, old_query: str, new_text: str) -> None: + # cognee's v1 forget by query may or may not be precise; the + # conservative path is: forget everything matching old_query, + # then remember new_text. + try: + self._run(self._cognee.forget(old_query, dataset=self.dataset)) + except Exception: + pass + self._run(self._cognee.remember(new_text, dataset=self.dataset)) + + def release(self, query: str) -> int: + raise NotImplementedError( + "Cognee v1 has no documented soft-delete / TTL primitive." + ) + + def purge(self, query: str) -> int: + try: + n = self._run(self._cognee.forget(query, dataset=self.dataset)) + return int(n) if isinstance(n, int) else 1 + except Exception: + return 0 + + +# ─── A-MEM adapter (requires Ollama or OpenAI) ──────────────────────── + +class AMemAdapter: + """A-MEM (Xu et al., NeurIPS 2025, arXiv 2502.12110): Zettelkasten- + style agentic memory. Each ``add_note`` triggers an LLM call to + generate tags / context / inter-note links, so the system cannot + run without either an Ollama daemon or an OpenAI API key. + + *Install:* not on PyPI as of May 2026 — clone the repo and pip-install. + :: + + git clone https://github.com/agiresearch/A-mem + cd A-mem && pip install -e . + + Primitive coverage: + + inscribe ↦ ms.add_note(text) + recall ↦ ms.search_agentic(query, k=k) + supersede ↦ ms.update(memory_id, content=new_text) + release ↦ raises NotImplementedError (no soft-delete) + purge ↦ ms.delete(memory_id) + """ + + name = "amem" + + def __init__(self, llm_backend: str = "ollama", + embedder_model: str = "all-MiniLM-L6-v2"): + try: + from agentic_memory.memory_system import AgenticMemorySystem + except ImportError as e: # pragma: no cover + raise ImportError( + "A-MEM is not on PyPI — clone " + "https://github.com/agiresearch/A-mem and `pip install -e .`" + ) from e + self._System = AgenticMemorySystem + self.llm_backend = llm_backend + self.embedder_model = embedder_model + self.ms = None + + def reset(self) -> None: + # Recreate fresh in-process system. ChromaDB is embedded; the + # constructor instantiates the LLM controller, which is where + # the Ollama/OpenAI dependency materializes. + self.ms = self._System( + model_name=self.embedder_model, + llm_backend=self.llm_backend, + ) + # ChromaDB collection lives across instances by default; wipe + # the namespace explicitly so cases don't leak into each other. + try: + for mid in list(self.ms.memories.keys()): + self.ms.delete(mid) + except Exception: + pass + + def inscribe(self, text: str) -> str: + return self.ms.add_note(text) + + def recall_texts(self, query: str, k: int = 5) -> list[str]: + results = self.ms.search_agentic(query, k=k) + out: list[str] = [] + for r in results or []: + if isinstance(r, dict): + out.append(r.get("content") or r.get("text") or str(r)) + elif hasattr(r, "content"): + out.append(r.content) + else: + out.append(str(r)) + return out + + def supersede(self, old_query: str, new_text: str) -> None: + hits = self.ms.search_agentic(old_query, k=1) + if not hits: + self.ms.add_note(new_text) + return + target = hits[0] + mid = target.get("id") if isinstance(target, dict) else getattr(target, "id", None) + if mid is None: + self.ms.add_note(new_text) + return + try: + self.ms.update(mid, content=new_text) + except Exception: + self.ms.delete(mid) + self.ms.add_note(new_text) + + def release(self, query: str) -> int: + raise NotImplementedError( + "A-MEM has no documented soft-delete primitive." + ) + + def purge(self, query: str) -> int: + hits = self.ms.search_agentic(query, k=5) + purged = 0 + for h in hits or []: + mid = h.get("id") if isinstance(h, dict) else getattr(h, "id", None) + if mid is None: + continue + try: + self.ms.delete(mid) + purged += 1 + except Exception: + pass + return purged + + # ─── MemPalace adapter ──────────────────────────────────────────────── class MemPalaceAdapter: diff --git a/bench/forgeteval/adversarial.py b/bench/forgeteval/adversarial.py new file mode 100644 index 0000000..78052c0 --- /dev/null +++ b/bench/forgeteval/adversarial.py @@ -0,0 +1,2093 @@ +"""ForgetEval-Adv — hand-crafted adversarial layer. + +64 cases across 8 attack categories, 8 cases each. Each case is a +`GeneratedCase` (same dataclass as the template-generated suite), so +the runner, adapter Protocol, and scoring logic in `run.py` apply +unchanged. + +Design philosophy: each case targets a specific architectural weakness +the template-generated suite cannot reliably probe. See +`docs/forgeteval_adversarial.md` for the full design rationale, +annotation protocol, and reporting format. + +Attack categories (8 each, 64 total): + 1. substring_trap — must-not substring appears in a distractor + 2. prefix_collision — identifiers share long common prefix + 3. paraphrase_supersession — new fact lexically distant from old + 4. negation_trap — negated fact must not be confused + 5. temporal_qualifier — date-stamped supersession chains + 6. shared_attribute — two entities share one attribute + 7. compound_fact — single sentence carries two facts + 8. identifier_obfuscation — same identifier, different surface forms + +Author intent comments are preserved in source for the IAA protocol. +""" +from __future__ import annotations + +from bench.forgeteval.generate import GeneratedCase + + +# ─── 1. substring_trap ───────────────────────────────────────────── +# Family: any. Attack: a must_not substring is innocently embedded in +# a distractor fact. Tests whether the scoring is robust to false +# substring matches. These probe the *benchmark* as much as the +# system: a system that passes here is fine; a system that fails +# might be fine but the *case* is broken. Cases here are written so +# that the must_not substring is rare enough that the surviving +# distractor cannot accidentally satisfy it. + +ADV_SUBSTRING_TRAP = [ + GeneratedCase( + id="adv_substring_trap_01", + family="supersession", + # intent: must_not "Vienna" is the city; distractor mentions + # "Vienna sausage" (food), the substring "Vienna" appears + # but the system should still pass because the supersede + # works on the city fact, not the sausage trivia. + setup_facts=[ + "User currently lives in Vienna.", + "The office cafeteria serves Vienna sausage on Tuesdays.", + "Hallway plants were repotted last week.", + "Conference room reservation system is offline.", + ], + mutations=[("supersede", "user residence city", "User moved to Oslo last month.")], + final_query="Where does the user live?", + must_contain=["Oslo"], + must_not_contain=["lives in Vienna", "currently lives"], + ), + GeneratedCase( + id="adv_substring_trap_02", + family="purge", + # intent: deleting password "raven42" must remove it from + # recall. Distractor "raven" (a bird-watching club) shares + # the prefix but is not the password. + setup_facts=[ + "Account password for backup: raven42#secure.", + "The bird-watching club logo is a raven silhouette.", + "Annual hackathon prizes were sponsored by an angel investor.", + "The HVAC system was inspected and approved.", + ], + mutations=[("purge", "account password backup")], + final_query="What is the backup account password?", + must_contain=[], + must_not_contain=["raven42", "raven42#secure"], + ), + GeneratedCase( + id="adv_substring_trap_03", + family="amnesia", + # intent: forget "Frank" the person. Distractor about + # "frank discussion" (adjective use) shares the substring. + setup_facts=[ + "Frank works at Notion as a backend engineer.", + "Frank's favorite color is teal.", + "Last quarter's all-hands had a frank discussion about layoffs.", + "Grace organizes the office quiz nights.", + ], + mutations=[("release", "everything about Frank the engineer")], + final_query="Tell me about the people we discussed.", + must_contain=["Grace"], + must_not_contain=["Frank works", "Frank's favorite"], + ), + GeneratedCase( + id="adv_substring_trap_04", + family="decay", + # intent: release session token "kappa7"; distractor mentions + # "kappa coefficient" (statistics term) which contains the + # substring but is not the token. + setup_facts=[ + "Session token issued: kappa7-2025-08.", + "Inter-annotator agreement is measured with Cohen's kappa coefficient.", + "The deployment pipeline runs on Tuesdays at 02:00 UTC.", + "Document templates were standardized in 2024.", + ], + mutations=[("release", "session token kappa7")], + final_query="What is the current session token?", + must_contain=[], + must_not_contain=["kappa7-2025", "kappa7-"], + ), + GeneratedCase( + id="adv_substring_trap_05", + family="drift", + # intent: chain of job changes ending at Anthropic. Distractor + # mentions "anthropic principle" (cosmology) which contains + # the substring "Anthropic" but is not an employer. + setup_facts=[ + "User started at Google in 2020.", + "A reading-group discussed the anthropic principle last week.", + "Office printers were replaced with networked models.", + ], + mutations=[ + ("supersede", "user employer Google", "User moved to Meta in 2022."), + ("supersede", "user employer Meta", "User joined Anthropic in 2025."), + ], + final_query="Where does the user work now?", + must_contain=["joined Anthropic"], + must_not_contain=["started at Google", "moved to Meta"], + ), + GeneratedCase( + id="adv_substring_trap_06", + family="supersession", + # intent: supersede diet "vegan" → "pescatarian". Distractor + # mentions "vegan restaurant" (location, not the user's diet). + setup_facts=[ + "User follows a vegan diet strictly.", + "There is a popular vegan restaurant on the corner.", + "Last team offsite was held in Lisbon.", + ], + mutations=[("supersede", "user dietary preference", "User now eats pescatarian, including fish.")], + final_query="What is the user's diet?", + must_contain=["pescatarian"], + must_not_contain=["follows a vegan", "vegan diet strictly"], + ), + GeneratedCase( + id="adv_substring_trap_07", + family="purge", + # intent: purge customer "alice@beta.io"; distractor mentions + # "beta release notes" (product term) which shares "beta". + setup_facts=[ + "Customer alice@beta.io has 3 active subscriptions.", + "Beta release notes were published last Friday.", + "Quarterly board meeting is in October.", + ], + mutations=[("purge", "alice@beta.io")], + final_query="Show me data for customer alice@beta.io.", + must_contain=[], + must_not_contain=["alice@beta.io has", "alice@beta.io's"], + ), + GeneratedCase( + id="adv_substring_trap_08", + family="amnesia", + # intent: forget "Charlie" the engineer. Distractor mentions + # "Charlie horse" idiom in a chat snippet. + setup_facts=[ + "Charlie joined the platform team in March.", + "Charlie speaks German and Italian fluently.", + "Someone in chat joked about getting a charlie horse during the standup.", + "Diana submitted the patent application last quarter.", + ], + mutations=[("release", "everything about Charlie on the team")], + final_query="Who is on the team?", + must_contain=["Diana"], + must_not_contain=["Charlie joined", "Charlie speaks"], + ), +] + + +# ─── 2. prefix_collision ─────────────────────────────────────────── +# Family: purge. Attack: two identifiers share a long common prefix. +# Deleting one must not also delete the other. Tests whether the +# purge path uses precise identifier match (good) or fuzzy/semantic +# match (bad). + +ADV_PREFIX_COLLISION = [ + GeneratedCase( + id="adv_prefix_collision_01", + family="purge", + # intent: alice@acme.io vs alice.smith@acme.io + setup_facts=[ + "Customer alice@acme.io subscribed in 2024.", + "Customer alice.smith@acme.io subscribed in 2023.", + "Quarterly all-hands was rescheduled.", + ], + mutations=[("purge", "alice@acme.io")], + final_query="Which alice customers are still active?", + must_contain=["alice.smith@acme.io"], + must_not_contain=["alice@acme.io subscribed"], + ), + GeneratedCase( + id="adv_prefix_collision_02", + family="purge", + # intent: api_key_v1 vs api_key_v2 + setup_facts=[ + "Production key: api_key_v1_abc123.", + "Staging key: api_key_v2_def456.", + "Office WiFi password was rotated.", + ], + mutations=[("purge", "api_key_v1_abc123")], + final_query="Show me the staging API key.", + must_contain=["api_key_v2_def456"], + must_not_contain=["api_key_v1_abc123"], + ), + GeneratedCase( + id="adv_prefix_collision_03", + family="purge", + # intent: patient John Smith vs patient John Smithson + setup_facts=[ + "Patient John Smith was prescribed metformin 500mg.", + "Patient John Smithson is allergic to penicillin.", + "The pharmacy received a shipment of insulin last week.", + ], + mutations=[("purge", "John Smith prescription")], + final_query="What allergies does John Smithson have?", + must_contain=["penicillin"], + must_not_contain=["John Smith was prescribed", "metformin 500mg"], + ), + GeneratedCase( + id="adv_prefix_collision_04", + family="purge", + # intent: order_2024_001 vs order_2024_0010 + setup_facts=[ + "Order order_2024_001 was for 5 units at $200 each.", + "Order order_2024_0010 was for 12 units at $150 each.", + "Warehouse 3 inventory was audited Friday.", + ], + mutations=[("purge", "order_2024_001 record")], + final_query="Tell me about order_2024_0010.", + must_contain=["12 units"], + must_not_contain=["order_2024_001 was for 5"], + ), + GeneratedCase( + id="adv_prefix_collision_05", + family="purge", + # intent: bob@example.com vs bob@example.com.uk (different TLD) + setup_facts=[ + "User bob@example.com signed the NDA on 2025-03-01.", + "User bob@example.com.uk submitted a separate NDA on 2025-04-15.", + "Legal team rotated their PGP keys recently.", + ], + mutations=[("purge", "bob@example.com NDA")], + final_query="Whose 2025-04-15 NDA is still on file?", + must_contain=["bob@example.com.uk"], + must_not_contain=["bob@example.com signed"], + ), + GeneratedCase( + id="adv_prefix_collision_06", + family="purge", + # intent: sk-abc-prod vs sk-abc-dev + setup_facts=[ + "Production secret token: sk-abc-prod-9f3a.", + "Development secret token: sk-abc-dev-2c7b.", + "Office plants were watered on schedule.", + ], + mutations=[("purge", "sk-abc-prod-9f3a production token")], + final_query="What is the development token?", + must_contain=["sk-abc-dev-2c7b"], + must_not_contain=["sk-abc-prod-9f3a"], + ), + GeneratedCase( + id="adv_prefix_collision_07", + family="purge", + # intent: case_id 12345 vs case_id 123456 (substring prefix) + setup_facts=[ + "Legal case_id 12345 was filed in district court.", + "Legal case_id 123456 was filed in appellate court.", + "The HR portal was migrated to a new vendor.", + ], + mutations=[("purge", "case_id 12345 district court")], + final_query="What court is case_id 123456 in?", + must_contain=["appellate"], + must_not_contain=["case_id 12345 was filed"], + ), + GeneratedCase( + id="adv_prefix_collision_08", + family="purge", + # intent: dana@startup.com vs dana@startup.com.au + setup_facts=[ + "Customer dana@startup.com purchased Plan A.", + "Customer dana@startup.com.au purchased Plan B.", + "Marketing emails are sent on Wednesdays.", + ], + mutations=[("purge", "dana@startup.com purchase")], + final_query="Which plan did dana@startup.com.au buy?", + must_contain=["Plan B"], + must_not_contain=["dana@startup.com purchased Plan A"], + ), + # ─── v0.4 expansion: 8 more prefix-collision cases ────────────── + GeneratedCase( + id="adv_prefix_collision_09", + family="purge", + # intent: usernames sharing prefix "admin" vs "admin1" + setup_facts=[ + "User admin has full root access.", + "User admin1 has read-only dashboard access.", + "Audit log was archived last week.", + ], + mutations=[("purge", "user admin root access")], + final_query="What access does admin1 have?", + must_contain=["read-only"], + must_not_contain=["admin has full root"], + ), + GeneratedCase( + id="adv_prefix_collision_10", + family="purge", + # intent: project_2024 vs project_2024_v2 + setup_facts=[ + "project_2024 budget is $500K.", + "project_2024_v2 budget is $750K.", + "Coffee inventory was restocked.", + ], + mutations=[("purge", "project_2024 budget record")], + final_query="What is the project_2024_v2 budget?", + must_contain=["$750K"], + must_not_contain=["project_2024 budget is $500K"], + ), + GeneratedCase( + id="adv_prefix_collision_11", + family="purge", + # intent: shared department prefix + setup_facts=[ + "Department engineering reports to CTO.", + "Department engineering-platform reports to VP Platform.", + "All-hands meeting moved to Wednesday.", + ], + mutations=[("purge", "department engineering reporting line")], + final_query="Who does engineering-platform report to?", + must_contain=["VP Platform"], + must_not_contain=["engineering reports to CTO"], + ), + GeneratedCase( + id="adv_prefix_collision_12", + family="purge", + # intent: ticket numbers sharing prefix + setup_facts=[ + "Ticket TKT-100 was resolved in March.", + "Ticket TKT-1000 is still open with priority high.", + "Quarterly metrics will be presented Thursday.", + ], + mutations=[("purge", "ticket TKT-100 record")], + final_query="What is the status of TKT-1000?", + must_contain=["still open"], + must_not_contain=["TKT-100 was resolved"], + ), + GeneratedCase( + id="adv_prefix_collision_13", + family="purge", + # intent: file paths sharing prefix + setup_facts=[ + "Config file /etc/app/conf is currently in use.", + "Config file /etc/app/conf.bak holds the previous version.", + "Server uptime is 47 days.", + ], + mutations=[("purge", "/etc/app/conf active configuration")], + final_query="What does the .bak file contain?", + must_contain=["previous version"], + must_not_contain=["/etc/app/conf is currently"], + ), + GeneratedCase( + id="adv_prefix_collision_14", + family="purge", + # intent: domain prefix collision + setup_facts=[ + "Domain mycompany.com expires in 2027.", + "Domain mycompany.com.br is owned by a different entity.", + "DNS provider was changed last month.", + ], + mutations=[("purge", "mycompany.com domain registration")], + final_query="Who owns mycompany.com.br?", + must_contain=["different entity"], + must_not_contain=["mycompany.com expires"], + ), + GeneratedCase( + id="adv_prefix_collision_15", + family="purge", + # intent: hash prefix collision + setup_facts=[ + "Commit hash abc1234 reverted the payment bug.", + "Commit hash abc12345 fixed the email template typo.", + "CI pipeline was migrated to a new runner.", + ], + mutations=[("purge", "commit abc1234 record")], + final_query="What did commit abc12345 do?", + must_contain=["email template typo"], + must_not_contain=["abc1234 reverted the payment"], + ), + GeneratedCase( + id="adv_prefix_collision_16", + family="purge", + # intent: phone country-code prefix collision + setup_facts=[ + "Phone +1-555-0100 is the main office line.", + "Phone +1-555-01000 is the fax machine extension.", + "Building cleaning is on Fridays.", + ], + mutations=[("purge", "phone +1-555-0100 office line")], + final_query="What is the fax extension?", + must_contain=["+1-555-01000"], + must_not_contain=["+1-555-0100 is the main"], + ), +] + + +# ─── 3. paraphrase_supersession ──────────────────────────────────── +# Family: supersession, drift. Attack: the new fact and the old fact +# share few surface tokens. Lexical recall cannot find the old fact +# from a query phrased like the new fact, so supersession requires +# semantic alignment. + +ADV_PARAPHRASE = [ + GeneratedCase( + id="adv_paraphrase_supersession_01", + family="supersession", + # intent: "works at Stripe" → "quit fintech, joined an AI safety lab" + setup_facts=[ + "User works at Stripe as a senior backend engineer.", + "User commutes by bike to the SF office.", + ], + mutations=[("supersede", "user employer", "User quit the payments industry and joined an AI safety lab in San Francisco.")], + final_query="What does the user do for work now?", + must_contain=["AI safety", "lab"], + must_not_contain=["Stripe", "backend engineer"], + ), + GeneratedCase( + id="adv_paraphrase_supersession_02", + family="supersession", + # intent: "vegan" → "started eating fish only for omega-3" + setup_facts=[ + "User has been strictly vegan since 2020.", + "User runs 10km every weekend.", + ], + mutations=[("supersede", "user dietary lifestyle", "User now consumes fish as the only animal product, citing omega-3 needs.")], + final_query="Is the user vegan?", + must_contain=["fish", "omega-3"], + must_not_contain=["strictly vegan", "vegan since 2020"], + ), + GeneratedCase( + id="adv_paraphrase_supersession_03", + family="supersession", + # intent: "lives in Berlin" → "relocated to Lisbon to escape winter" + setup_facts=[ + "User has lived in Berlin since graduating in 2018.", + "User plays chess on Tuesdays.", + ], + mutations=[("supersede", "user city of residence", "User relocated to Lisbon last spring to escape the German winter.")], + final_query="Where does the user live?", + must_contain=["Lisbon"], + must_not_contain=["lived in Berlin", "since graduating"], + ), + GeneratedCase( + id="adv_paraphrase_supersession_04", + family="supersession", + # intent: "Python is the only language" → "switched to writing services in Rust" + setup_facts=[ + "User writes all backend code in Python and rejects other languages.", + "User uses an ergonomic keyboard.", + ], + mutations=[("supersede", "user programming language preference", "User has rebuilt the team's microservices in Rust over the last six months.")], + final_query="What language does the user write services in?", + must_contain=["Rust"], + must_not_contain=["writes all backend code in Python", "rejects other languages"], + ), + GeneratedCase( + id="adv_paraphrase_supersession_05", + family="supersession", + # intent: "married" → "filed for divorce earlier this year" + setup_facts=[ + "User is married and lives with their spouse in Toronto.", + "User collects vintage cameras.", + ], + mutations=[("supersede", "user marital status", "User filed for divorce earlier this year and now lives alone.")], + final_query="Is the user married?", + must_contain=["divorce", "alone"], + must_not_contain=["is married and lives", "lives with their spouse"], + ), + GeneratedCase( + id="adv_paraphrase_supersession_06", + family="drift", + # intent: 4-step chain with growing paraphrase distance + setup_facts=[ + "User joined Google in 2018 as a software engineer.", + ], + mutations=[ + ("supersede", "user employer 2018", "Departed Mountain View, took a role at a hedge fund in NYC in 2020."), + ("supersede", "user employer 2020 finance", "Left finance entirely in 2022 to teach mathematics at a small college in Vermont."), + ("supersede", "user employer 2022 teaching", "In 2025 returned to industry, joining a robotics startup in Boston as VP of engineering."), + ], + final_query="What is the user's current job?", + must_contain=["robotics", "Boston"], + must_not_contain=["joined Google in 2018", "hedge fund in NYC", "teach mathematics"], + ), + GeneratedCase( + id="adv_paraphrase_supersession_07", + family="supersession", + # intent: "owns a Tesla" → "switched to a bicycle and never drives" + setup_facts=[ + "User drives a Tesla Model 3, purchased new in 2022.", + "User listens to audiobooks during the commute.", + ], + mutations=[("supersede", "user vehicle", "User sold the car and now commutes exclusively by bicycle, refusing to drive.")], + final_query="What does the user drive?", + must_contain=["bicycle"], + must_not_contain=["drives a Tesla", "Tesla Model 3"], + ), + GeneratedCase( + id="adv_paraphrase_supersession_08", + family="supersession", + # intent: "introvert who avoids events" → "running a 200-person community" + setup_facts=[ + "User is a quiet introvert who avoids large gatherings.", + "User prefers email over phone calls.", + ], + mutations=[("supersede", "user social temperament", "User is now organizing a 200-person developer meetup every month and hosts a weekly podcast.")], + final_query="What is the user's social style?", + must_contain=["organizing", "200-person"], + must_not_contain=["quiet introvert", "avoids large gatherings"], + ), +] + + +# ─── 4. negation_trap ────────────────────────────────────────────── +# Family: supersession, decay. Attack: vector embeddings often blur +# "X" and "not X." Releasing or superseding a *negated* fact should +# not affect recall of the affirmative form. + +ADV_NEGATION = [ + GeneratedCase( + id="adv_negation_trap_01", + family="supersession", + # intent: "User does NOT work at Anthropic" is the stored fact; + # supersede to "User does work at Anthropic" — negation flips. + setup_facts=[ + "User does NOT work at Anthropic and has never interviewed there.", + "User's coffee preference is espresso.", + ], + mutations=[("supersede", "user Anthropic employment status", "User actually joined Anthropic last quarter.")], + final_query="Does the user work at Anthropic?", + must_contain=["joined Anthropic"], + must_not_contain=["does NOT work at Anthropic", "never interviewed"], + ), + GeneratedCase( + id="adv_negation_trap_02", + family="decay", + # intent: release a negated fact "doesn't drink coffee" should + # not affect the unrelated "drinks tea" fact. + setup_facts=[ + "User does not drink coffee under any circumstances.", + "User drinks green tea every afternoon.", + ], + mutations=[("release", "user does not drink coffee")], + final_query="What does the user drink?", + must_contain=["green tea"], + must_not_contain=["does not drink coffee"], + ), + GeneratedCase( + id="adv_negation_trap_03", + family="supersession", + # intent: "doesn't have a dog" → "adopted a golden retriever" + setup_facts=[ + "User does not have any pets and has stated this firmly.", + "User's apartment has a small balcony.", + ], + mutations=[("supersede", "user pet ownership", "User adopted a golden retriever named Mochi last month.")], + final_query="Does the user have a pet?", + must_contain=["golden retriever", "Mochi"], + must_not_contain=["does not have any pets", "stated this firmly"], + ), + GeneratedCase( + id="adv_negation_trap_04", + family="supersession", + # intent: negated allergy → affirmative + setup_facts=[ + "Patient has no known peanut allergy.", + "Patient is currently on lisinopril for hypertension.", + ], + mutations=[("supersede", "patient peanut allergy status", "Patient was admitted with severe peanut anaphylaxis last week.")], + final_query="Does the patient have a peanut allergy?", + must_contain=["anaphylaxis"], + must_not_contain=["no known peanut allergy"], + ), + GeneratedCase( + id="adv_negation_trap_05", + family="decay", + # intent: release "doesn't speak French" — must not affect + # the affirmative "speaks Spanish" + setup_facts=[ + "User does not speak French at all.", + "User speaks Spanish at a conversational level.", + ], + mutations=[("release", "user French language status")], + final_query="What languages does the user speak?", + must_contain=["Spanish"], + must_not_contain=["does not speak French"], + ), + GeneratedCase( + id="adv_negation_trap_06", + family="supersession", + # intent: "User doesn't use Slack" → "User now uses Slack daily" + setup_facts=[ + "User refuses to use Slack and only communicates via email.", + "User's email signature includes a Mastodon handle.", + ], + mutations=[("supersede", "user Slack usage", "User started using Slack daily after team-wide rollout.")], + final_query="Does the user use Slack?", + must_contain=["using Slack daily"], + must_not_contain=["refuses to use Slack"], + ), + GeneratedCase( + id="adv_negation_trap_07", + family="decay", + # intent: release a "did not attend" fact, should not affect + # the unrelated "attended XYZ" fact. + setup_facts=[ + "User did not attend the 2024 holiday party.", + "User attended the 2025 onboarding session.", + ], + mutations=[("release", "user holiday party 2024 attendance")], + final_query="Which events has the user attended?", + must_contain=["onboarding"], + must_not_contain=["did not attend the 2024 holiday party"], + ), + GeneratedCase( + id="adv_negation_trap_08", + family="supersession", + # intent: "doesn't own a car" → "leases a Tesla now" + setup_facts=[ + "User does not own a car and uses public transit exclusively.", + "User has a monthly transit pass.", + ], + mutations=[("supersede", "user vehicle ownership", "User started leasing a Tesla Model Y this summer.")], + final_query="What vehicle does the user have?", + must_contain=["Tesla", "leasing"], + must_not_contain=["does not own a car", "uses public transit exclusively"], + ), +] + + +# ─── 5. temporal_qualifier ───────────────────────────────────────── +# Family: supersession, drift. Attack: facts with embedded dates. +# Supersession must respect the temporal sequence (latest wins), not +# get confused by date tokens. + +ADV_TEMPORAL = [ + GeneratedCase( + id="adv_temporal_qualifier_01", + family="supersession", + # intent: "joined in 2020" → "moved in 2022" — temporal supersede + setup_facts=[ + "User joined Google in March 2020 as a software engineer.", + "User's hobby is competitive chess on weekends.", + ], + mutations=[("supersede", "user current employer Google", "User moved to Meta in June 2022.")], + final_query="Where does the user currently work?", + must_contain=["Meta", "June 2022"], + must_not_contain=["joined Google in March 2020"], + ), + GeneratedCase( + id="adv_temporal_qualifier_02", + family="drift", + # intent: 3-step temporal chain, latest wins + setup_facts=[ + "User lived in Tokyo from 2015 to 2018.", + "User's main language is English.", + ], + mutations=[ + ("supersede", "user residence Tokyo 2015", "User lived in Berlin from 2018 to 2022."), + ("supersede", "user residence Berlin 2018", "User has lived in Lisbon since 2022."), + ], + final_query="Where does the user live now?", + must_contain=["Lisbon", "since 2022"], + must_not_contain=["lived in Tokyo", "Berlin from 2018"], + ), + GeneratedCase( + id="adv_temporal_qualifier_03", + family="supersession", + # intent: trap — "visited Google in 2024" is NOT supersession + # of "joined Google in 2020"; the supersession should be by + # the explicit Meta event, not the visit. + setup_facts=[ + "User joined Google in 2020 as their first job.", + "User visited Google's campus in 2024 for a conference.", + "User started a chocolate-making side project.", + ], + mutations=[ + ("supersede", "user employer first job", "User left Google and joined Meta in 2022."), + ], + final_query="Where does the user work now?", + must_contain=["Meta"], + must_not_contain=["joined Google in 2020 as their first job"], + ), + GeneratedCase( + id="adv_temporal_qualifier_04", + family="supersession", + # intent: dated diet change + setup_facts=[ + "Throughout 2022 user was strictly keto.", + "User's preferred gym is a CrossFit box downtown.", + ], + mutations=[("supersede", "user diet 2022 keto", "Starting January 2024, user switched to a Mediterranean diet for cardiovascular reasons.")], + final_query="What is the user's current diet?", + must_contain=["Mediterranean"], + must_not_contain=["strictly keto", "Throughout 2022"], + ), + GeneratedCase( + id="adv_temporal_qualifier_05", + family="drift", + # intent: 4-step political affiliation chain + setup_facts=[ + "User was a registered Libertarian in 2016.", + ], + mutations=[ + ("supersede", "user political registration 2016", "User registered as Democrat in 2018."), + ("supersede", "user political registration 2018", "User registered as independent in 2022."), + ("supersede", "user political registration 2022", "User registered as Green in 2025."), + ], + final_query="What is the user's current political registration?", + must_contain=["Green", "2025"], + must_not_contain=["registered Libertarian", "registered as Democrat", "registered as independent"], + ), + GeneratedCase( + id="adv_temporal_qualifier_06", + family="supersession", + # intent: ownership change with dates + setup_facts=[ + "User has owned the rental property at 14 Birch Street since 2010.", + "User maintains a small vegetable garden.", + ], + mutations=[("supersede", "user property 14 Birch Street", "User sold 14 Birch Street in October 2024 and now rents downtown.")], + final_query="Does the user own 14 Birch Street?", + must_contain=["sold", "October 2024"], + must_not_contain=["owned the rental property at 14 Birch Street since 2010"], + ), + GeneratedCase( + id="adv_temporal_qualifier_07", + family="supersession", + # intent: subscription with renewal dates + setup_facts=[ + "Customer subscribed to Pro plan on 2023-04-15.", + "Customer is in the European Union jurisdiction.", + ], + mutations=[("supersede", "customer subscription Pro plan", "Customer downgraded to Free plan on 2025-02-10 after cancellation.")], + final_query="What plan is the customer on?", + must_contain=["Free plan", "2025-02-10"], + must_not_contain=["subscribed to Pro plan on 2023-04-15"], + ), + GeneratedCase( + id="adv_temporal_qualifier_08", + family="drift", + # intent: marital status drift over years + setup_facts=[ + "User was single in 2018.", + ], + mutations=[ + ("supersede", "user marital status 2018 single", "User got engaged in 2020."), + ("supersede", "user marital status engaged", "User married in 2021."), + ("supersede", "user marital status married", "User finalized divorce in 2024."), + ], + final_query="What is the user's current marital status?", + must_contain=["divorce", "2024"], + must_not_contain=["was single in 2018", "got engaged in 2020", "married in 2021"], + ), +] + + +# ─── 6. shared_attribute ─────────────────────────────────────────── +# Family: amnesia. Attack: two entities share one attribute. Forget +# one entity must not collapse the other's link to the shared attribute. + +ADV_SHARED_ATTR = [ + GeneratedCase( + id="adv_shared_attribute_01", + family="amnesia", + # intent: Dana and Eve both in Berlin; forget Dana + setup_facts=[ + "Dana lives in Berlin and works at Stripe.", + "Eve lives in Berlin and works at Notion.", + "Frank lives in Tokyo and works at Spotify.", + ], + mutations=[("release", "everything about Dana")], + final_query="Who lives in Berlin?", + must_contain=["Eve", "Berlin"], + must_not_contain=["Dana lives in Berlin"], + ), + GeneratedCase( + id="adv_shared_attribute_02", + family="amnesia", + # intent: Alice and Bob both like chess; forget Alice + setup_facts=[ + "Alice's favorite hobby is chess and competitive bridge.", + "Bob's favorite hobby is chess and rock climbing.", + "Carla's favorite hobby is pottery.", + ], + mutations=[("release", "everything about Alice")], + final_query="Who likes chess?", + must_contain=["Bob"], + must_not_contain=["Alice's favorite hobby is chess"], + ), + GeneratedCase( + id="adv_shared_attribute_03", + family="amnesia", + # intent: two patients share a condition; forget one + setup_facts=[ + "Patient John has type 2 diabetes and is on metformin.", + "Patient Mary has type 2 diabetes and is on insulin therapy.", + "Patient Tom has hypertension.", + ], + mutations=[("release", "everything about patient John")], + final_query="Which patients have type 2 diabetes?", + must_contain=["Mary", "diabetes"], + must_not_contain=["John has type 2 diabetes"], + ), + GeneratedCase( + id="adv_shared_attribute_04", + family="amnesia", + # intent: two employees share an office; forget one + setup_facts=[ + "Hannah and Ivan share the 4th floor office near the elevator.", + "Hannah leads the backend team.", + "Ivan leads the design team.", + "Judy works remotely from Madrid.", + ], + mutations=[("release", "everything about Hannah on the team")], + final_query="Who works on the 4th floor?", + must_contain=["Ivan"], + must_not_contain=["Hannah and Ivan share"], + ), + GeneratedCase( + id="adv_shared_attribute_05", + family="amnesia", + # intent: two customers share a coupon code; purge one customer's + # data, the coupon attribute itself on the other survives. + setup_facts=[ + "Customer alice@example.com used coupon SAVE20 in March.", + "Customer bob@example.com used coupon SAVE20 in April.", + "Marketing team rotates coupons quarterly.", + ], + mutations=[("release", "everything about alice@example.com")], + final_query="Who used coupon SAVE20?", + must_contain=["bob@example.com"], + must_not_contain=["alice@example.com used coupon"], + ), + GeneratedCase( + id="adv_shared_attribute_06", + family="amnesia", + # intent: two students share a class; forget one + setup_facts=[ + "Liam is enrolled in CS 224N and CS 231N this term.", + "Mia is enrolled in CS 224N and CS 230 this term.", + "Noah is enrolled in MATH 51 only.", + ], + mutations=[("release", "everything about student Liam this term")], + final_query="Who is enrolled in CS 224N?", + must_contain=["Mia"], + must_not_contain=["Liam is enrolled in CS 224N"], + ), + GeneratedCase( + id="adv_shared_attribute_07", + family="amnesia", + # intent: two team members both speak Mandarin; forget one + setup_facts=[ + "Olivia speaks English and Mandarin natively.", + "Peter speaks French and Mandarin.", + "Quinn speaks German only.", + ], + mutations=[("release", "everything about Olivia's languages")], + final_query="Who on the team speaks Mandarin?", + must_contain=["Peter"], + must_not_contain=["Olivia speaks English and Mandarin"], + ), + GeneratedCase( + id="adv_shared_attribute_08", + family="amnesia", + # intent: two clients share a project; forget one + setup_facts=[ + "Client Riya works on the Atlas project and pays monthly.", + "Client Sam works on the Atlas project and pays annually.", + "Client Tara works on a different project entirely.", + ], + mutations=[("release", "everything about client Riya")], + final_query="Who else works on the Atlas project?", + must_contain=["Sam"], + must_not_contain=["Riya works on the Atlas"], + ), + # ─── v0.4 expansion: 8 more shared-attribute cases ─────────────── + GeneratedCase( + id="adv_shared_attribute_09", + family="amnesia", + # intent: two employees same dept; forget one + setup_facts=[ + "Engineer Uma works in the platform team and ships Go services.", + "Engineer Victor works in the platform team and writes Rust services.", + "Engineer Wendy works in the data team.", + ], + mutations=[("release", "everything about Uma")], + final_query="Who is in the platform team?", + must_contain=["Victor"], + must_not_contain=["Uma works in the platform"], + ), + GeneratedCase( + id="adv_shared_attribute_10", + family="amnesia", + # intent: two devices same model; forget one user's + setup_facts=[ + "Xavier owns a MacBook Pro 16 M3 for video editing.", + "Yara owns a MacBook Pro 16 M3 for music production.", + "Zoe owns a desktop PC for gaming.", + ], + mutations=[("release", "everything about Xavier's hardware")], + final_query="Who owns a MacBook Pro 16?", + must_contain=["Yara"], + must_not_contain=["Xavier owns a MacBook"], + ), + GeneratedCase( + id="adv_shared_attribute_11", + family="amnesia", + # intent: two members same allergy; forget one + setup_facts=[ + "Aiden has a severe shellfish allergy and carries EpiPen.", + "Bella has a severe shellfish allergy noted in the chart.", + "Caleb has no known allergies.", + ], + mutations=[("release", "everything about Aiden the patient")], + final_query="Who else has a shellfish allergy?", + must_contain=["Bella"], + must_not_contain=["Aiden has a severe shellfish"], + ), + GeneratedCase( + id="adv_shared_attribute_12", + family="amnesia", + # intent: two collaborators on same paper; forget one + setup_facts=[ + "Diana co-authored the 2025 latency paper at SIGMOD.", + "Ethan co-authored the 2025 latency paper at SIGMOD.", + "Felix co-authored a different paper at VLDB.", + ], + mutations=[("release", "everything about Diana's research")], + final_query="Who else worked on the 2025 SIGMOD latency paper?", + must_contain=["Ethan"], + must_not_contain=["Diana co-authored the 2025"], + ), + GeneratedCase( + id="adv_shared_attribute_13", + family="amnesia", + # intent: two subscribers share coupon code (different from earlier) + setup_facts=[ + "Subscriber gina@x.com applied coupon AUTUMN15 at checkout.", + "Subscriber hugo@y.com applied coupon AUTUMN15 at checkout.", + "Subscriber iris@z.com purchased a separate plan.", + ], + mutations=[("release", "everything about gina@x.com")], + final_query="Who else applied AUTUMN15?", + must_contain=["hugo@y.com"], + must_not_contain=["gina@x.com applied coupon"], + ), + GeneratedCase( + id="adv_shared_attribute_14", + family="amnesia", + # intent: two streets share neighborhood; forget one resident + setup_facts=[ + "Resident Jin lives on Oak Street in the Mission district.", + "Resident Kai lives on Pine Street in the Mission district.", + "Resident Lily lives in the Sunset district entirely.", + ], + mutations=[("release", "everything about resident Jin")], + final_query="Who else lives in the Mission district?", + must_contain=["Kai"], + must_not_contain=["Jin lives on Oak Street"], + ), + GeneratedCase( + id="adv_shared_attribute_15", + family="amnesia", + # intent: two students same advisor; forget one + setup_facts=[ + "Student Maya is advised by Professor Chen in the NLP group.", + "Student Noah is advised by Professor Chen in the NLP group.", + "Student Owen has a different advisor in the systems group.", + ], + mutations=[("release", "everything about student Maya")], + final_query="Who else is advised by Professor Chen?", + must_contain=["Noah"], + must_not_contain=["Maya is advised by Professor Chen"], + ), + GeneratedCase( + id="adv_shared_attribute_16", + family="amnesia", + # intent: two athletes same sport + team; forget one + setup_facts=[ + "Athlete Paula plays striker for FC Lisbon.", + "Athlete Quentin plays striker for FC Lisbon.", + "Athlete Ruth plays goalkeeper for Sporting Madrid.", + ], + mutations=[("release", "everything about athlete Paula")], + final_query="Who else plays striker for FC Lisbon?", + must_contain=["Quentin"], + must_not_contain=["Paula plays striker"], + ), +] + + +# ─── 7. compound_fact ────────────────────────────────────────────── +# Family: supersession. Attack: a single sentence carries two facts. +# Superseding only one must preserve the other. + +ADV_COMPOUND = [ + GeneratedCase( + id="adv_compound_fact_01", + family="supersession", + # intent: "lives in Berlin AND works at Stripe" — change only location + setup_facts=[ + "User lives in Berlin and works at Stripe as a backend engineer.", + "User uses an ergonomic mechanical keyboard.", + ], + mutations=[("supersede", "user city of residence", "User relocated to Madrid and continues working remotely.")], + final_query="Where does the user work?", + must_contain=["Stripe"], + must_not_contain=["lives in Berlin"], + ), + GeneratedCase( + id="adv_compound_fact_02", + family="supersession", + # intent: marital + work — change only work + setup_facts=[ + "User is married to Jamie and works at Google.", + "User has two cats named Pixel and Bit.", + ], + mutations=[("supersede", "user employer Google", "User joined Anthropic as a research engineer.")], + final_query="Is the user still married?", + must_contain=["married to Jamie"], + must_not_contain=["works at Google"], + ), + GeneratedCase( + id="adv_compound_fact_03", + family="supersession", + # intent: language + diet — change only diet + setup_facts=[ + "User speaks Italian fluently and follows a Mediterranean diet.", + "User runs a small wine-tasting club in their neighborhood.", + ], + mutations=[("supersede", "user dietary preference Mediterranean", "User switched to a strict vegan diet for ethical reasons.")], + final_query="What languages does the user speak?", + must_contain=["Italian"], + must_not_contain=["Mediterranean diet"], + ), + GeneratedCase( + id="adv_compound_fact_04", + family="supersession", + # intent: hobby + city — change only city + setup_facts=[ + "User plays the cello and lives in Vienna.", + "User's day job is database administration.", + ], + mutations=[("supersede", "user city of residence Vienna", "User moved to Toronto last autumn for a new opportunity.")], + final_query="Does the user still play the cello?", + must_contain=["cello"], + must_not_contain=["lives in Vienna"], + ), + GeneratedCase( + id="adv_compound_fact_05", + family="supersession", + # intent: car + pet — change only car + setup_facts=[ + "User drives a 2018 Subaru and owns a Border Collie named Rex.", + "User's preferred reading genre is hard sci-fi.", + ], + mutations=[("supersede", "user vehicle Subaru", "User upgraded to a new electric pickup truck.")], + final_query="What pet does the user have?", + must_contain=["Border Collie", "Rex"], + must_not_contain=["drives a 2018 Subaru"], + ), + GeneratedCase( + id="adv_compound_fact_06", + family="supersession", + # intent: degree + employer — change only employer + setup_facts=[ + "User has a PhD in physics from MIT and works as a quant at Renaissance.", + "User's morning routine includes a 5km run.", + ], + mutations=[("supersede", "user employer quant Renaissance", "User left finance entirely and joined a nonprofit climate research lab.")], + final_query="What is the user's educational background?", + must_contain=["PhD in physics", "MIT"], + must_not_contain=["works as a quant at Renaissance"], + ), + GeneratedCase( + id="adv_compound_fact_07", + family="supersession", + # intent: subscription + payment method — change only payment + setup_facts=[ + "Customer is on the Enterprise plan and pays via wire transfer.", + "Customer's account is in good standing.", + ], + mutations=[("supersede", "customer payment method wire transfer", "Customer updated payment method to credit card on file.")], + final_query="What plan is the customer on?", + must_contain=["Enterprise"], + must_not_contain=["pays via wire transfer"], + ), + GeneratedCase( + id="adv_compound_fact_08", + family="supersession", + # intent: project + role — change only role + setup_facts=[ + "User leads the Atlas project and reports to the VP of Engineering.", + "User's preferred meeting time is afternoons.", + ], + mutations=[("supersede", "user reporting line VP Engineering", "User now reports directly to the CTO after a reorg.")], + final_query="What project does the user lead?", + must_contain=["Atlas"], + must_not_contain=["reports to the VP of Engineering"], + ), +] + + +# ─── 8. identifier_obfuscation ───────────────────────────────────── +# Family: purge. Attack: same identifier in different surface forms +# (case, whitespace, encoding). A correct purge by one form should +# also purge the others, OR the system should be honest that it only +# handles canonical form. + +ADV_IDENTIFIER_OBFUSCATION = [ + GeneratedCase( + id="adv_identifier_obfuscation_01", + family="purge", + # intent: same email, different case + setup_facts=[ + "Customer ALICE@ACME.IO subscribed to Plan A.", + "Customer alice@acme.io is the same person (case-normalized).", + "Office lunches are catered on Wednesdays.", + ], + mutations=[("purge", "alice@acme.io")], + final_query="What plan does alice@acme.io have?", + must_contain=[], + must_not_contain=["ALICE@ACME.IO subscribed", "alice@acme.io is the same"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_02", + family="purge", + # intent: same email with trailing space + setup_facts=[ + "Customer bob@example.com bought 5 licenses.", + "Customer bob@example.com (note trailing whitespace) is the same account.", + "Quarterly OKR review is in two weeks.", + ], + mutations=[("purge", "bob@example.com")], + final_query="What did bob@example.com buy?", + must_contain=[], + must_not_contain=["bob@example.com bought 5", "is the same account"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_03", + family="purge", + # intent: phone number with/without country code + setup_facts=[ + "Patient phone on file: +1-415-555-0123.", + "Patient phone alternate format: 415-555-0123 (no country code).", + "Clinic moved to a new address last month.", + ], + mutations=[("purge", "phone 415-555-0123")], + final_query="What is the patient's phone number?", + must_contain=[], + must_not_contain=["+1-415-555-0123", "415-555-0123 (no country code)"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_04", + family="purge", + # intent: UUID with/without hyphens + setup_facts=[ + "User token: 550e8400-e29b-41d4-a716-446655440000.", + "Same user, compact form: 550e8400e29b41d4a716446655440000.", + "Server maintenance window is Tuesdays.", + ], + mutations=[("purge", "user token 550e8400")], + final_query="What is the user token?", + must_contain=[], + must_not_contain=["550e8400-e29b-41d4", "550e8400e29b41d4"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_05", + family="purge", + # intent: SSN with/without dashes + setup_facts=[ + "Customer SSN on file: 123-45-6789.", + "Customer alternate record: 123456789 (no dashes).", + "Tax filings are due in April.", + ], + mutations=[("purge", "SSN 123-45-6789")], + final_query="What is the customer's SSN?", + must_contain=[], + must_not_contain=["123-45-6789", "123456789 (no dashes)"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_06", + family="purge", + # intent: credit card with/without spaces + setup_facts=[ + "Card on file: 4111-1111-1111-1111.", + "Same card, no separators: 4111111111111111.", + "Office holiday party will be online.", + ], + mutations=[("purge", "card 4111-1111-1111-1111")], + final_query="What card is on file?", + must_contain=[], + must_not_contain=["4111-1111-1111-1111", "4111111111111111"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_07", + family="purge", + # intent: email with quoted local part + setup_facts=[ + "Customer \"john.doe\"@company.com subscribed to Pro.", + "Same customer in canonical form: john.doe@company.com.", + "Annual leave policy updates take effect next quarter.", + ], + mutations=[("purge", "john.doe@company.com")], + final_query="What plan does john.doe@company.com have?", + must_contain=[], + must_not_contain=["\"john.doe\"@company.com subscribed", "Same customer in canonical"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_08", + family="purge", + # intent: GitHub handle with/without leading @ + setup_facts=[ + "Contractor @deeplethe-engineer is on the access list.", + "Same contractor without @: deeplethe-engineer.", + "All-hands is the third Friday of each month.", + ], + mutations=[("purge", "@deeplethe-engineer contractor")], + final_query="Is @deeplethe-engineer still on the access list?", + must_contain=[], + must_not_contain=["@deeplethe-engineer is on", "deeplethe-engineer."], + ), + # ─── v0.4 expansion: 8 more identifier-obfuscation cases ───────── + GeneratedCase( + id="adv_identifier_obfuscation_09", + family="purge", + # intent: IP address with/without zero padding + setup_facts=[ + "Server IP 192.168.001.005 hosts the staging environment.", + "Same server short form 192.168.1.5 (no padding) was logged.", + "Office WiFi password rotates monthly.", + ], + mutations=[("purge", "server 192.168.1.5 record")], + final_query="What does 192.168.1.5 host?", + must_contain=[], + must_not_contain=["192.168.001.005 hosts", "192.168.1.5 (no padding)"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_10", + family="purge", + # intent: URL with/without trailing slash + with/without https + setup_facts=[ + "Webhook endpoint https://api.acme.io/v2/events fires on push.", + "Same endpoint http-form api.acme.io/v2/events/ (trailing slash) is in legacy docs.", + "Slack channel was renamed last week.", + ], + mutations=[("purge", "api.acme.io/v2/events webhook")], + final_query="Where does the webhook fire?", + must_contain=[], + must_not_contain=["https://api.acme.io/v2/events fires", + "api.acme.io/v2/events/ (trailing slash)"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_11", + family="purge", + # intent: same name with Mr. / Dr. / Prof. prefix + setup_facts=[ + "Mr. Alan Turing logged into the lab system.", + "Dr. Alan Turing accessed the same project repository.", + "Office plants were watered this morning.", + ], + mutations=[("purge", "Alan Turing user account")], + final_query="Does Alan Turing have any sessions on file?", + must_contain=[], + must_not_contain=["Mr. Alan Turing logged", "Dr. Alan Turing accessed"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_12", + family="purge", + # intent: email with + tag variations + setup_facts=[ + "Customer ada@example.com signed up in January.", + "Same customer email-with-tag: ada+promo@example.com used a coupon.", + "Print queue was migrated to AirPrint.", + ], + mutations=[("purge", "ada@example.com customer")], + final_query="Does ada+promo@example.com still have coupon history?", + must_contain=[], + must_not_contain=["ada@example.com signed up", + "ada+promo@example.com used a coupon"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_13", + family="purge", + # intent: nick name vs full name + setup_facts=[ + "Robert Smith was hired on 2023-01-15.", + "Same employee, nickname Bob Smith, was promoted in 2024.", + "Annual review window opens next month.", + ], + mutations=[("purge", "Robert Smith / Bob Smith employee record")], + final_query="Show me Bob Smith's promotion record.", + must_contain=[], + must_not_contain=["Robert Smith was hired", + "Bob Smith, was promoted"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_14", + family="purge", + # intent: same project ID with/without prefix abbreviation + setup_facts=[ + "Project ID PRJ-ATLAS-2024 has 5 milestones.", + "Same project shortcut form ATLAS-2024 (no PRJ prefix) is in slides.", + "Office snacks were restocked.", + ], + mutations=[("purge", "project ATLAS-2024 PRJ-ATLAS-2024 record")], + final_query="How many milestones in ATLAS-2024?", + must_contain=[], + must_not_contain=["PRJ-ATLAS-2024 has 5", + "ATLAS-2024 (no PRJ prefix)"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_15", + family="purge", + # intent: date format variations YYYY-MM-DD vs MM/DD/YYYY + setup_facts=[ + "Event scheduled for 2025-12-25 is the company gala.", + "Same event in US-format: 12/25/2025 will be at the convention center.", + "Holiday schedule was published.", + ], + mutations=[("purge", "event 2025-12-25 gala")], + final_query="Where is the 12/25/2025 event held?", + must_contain=[], + must_not_contain=["2025-12-25 is the company gala", + "12/25/2025 will be at"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_16", + family="purge", + # intent: account ID with currency suffix + setup_facts=[ + "Account 1234-USD holds the dollar balance.", + "Same account 1234 (without currency suffix) appears in the ledger.", + "Audit was completed last quarter.", + ], + mutations=[("purge", "account 1234 / 1234-USD record")], + final_query="What is the balance on account 1234?", + must_contain=[], + must_not_contain=["1234-USD holds the dollar", + "1234 (without currency suffix)"], + ), + # ─── v0.5.1 redress: hand-crafted addition to bring + # identifier_obfuscation from n=18 (mode-A judge over-rejection) + # up to category parity with the rest of the suite (n=36). + # Admission protocol identical to the original 112 hand-crafted core. + GeneratedCase( + id="adv_identifier_obfuscation_19", + family="purge", + # intent: URL with/without scheme + www + setup_facts=[ + "Customer's vendor profile is at https://www.acme.io/vendor.", + "Same profile, bare form: acme.io/vendor (no scheme, no www).", + "Vendor onboarding is finalized.", + ], + mutations=[("purge", "vendor profile at acme.io/vendor")], + final_query="Where is the vendor profile?", + must_contain=[], + must_not_contain=["https://www.acme.io/vendor", + "acme.io/vendor (no scheme"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_20", + family="purge", + # intent: URL with/without trailing slash + setup_facts=[ + "Endpoint is registered at https://api.beta.io/v2/users.", + "Equivalent canonical: https://api.beta.io/v2/users/ (with trailing slash).", + "OAuth refresh ran cleanly overnight.", + ], + mutations=[("purge", "endpoint api.beta.io/v2/users")], + final_query="What endpoint is registered?", + must_contain=[], + must_not_contain=["api.beta.io/v2/users.", + "api.beta.io/v2/users/ (with trailing"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_21", + family="purge", + # intent: email with plus-tag (gmail-style aliases) + setup_facts=[ + "Account holder email: charlie+work@gmail.com.", + "Same person, no plus-tag: charlie@gmail.com.", + "Promo codes expire on Sunday.", + ], + mutations=[("purge", "email charlie@gmail.com (with/without plus-tag)")], + final_query="What is the account holder's email?", + must_contain=[], + must_not_contain=["charlie+work@gmail.com.", + "charlie@gmail.com."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_22", + family="purge", + # intent: handle with/without @ prefix + setup_facts=[ + "Lead writer handle: @dora_writes.", + "Same handle without prefix: dora_writes.", + "Submissions are due by EOD Friday.", + ], + mutations=[("purge", "handle dora_writes (with/without @)")], + final_query="Who is the lead writer?", + must_contain=[], + must_not_contain=["@dora_writes.", + "without prefix: dora_writes."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_23", + family="purge", + # intent: license plate with/without hyphen + case + setup_facts=[ + "Vehicle plate: ABC-1234 registered to corporate fleet.", + "Same plate compact form: abc1234 (no hyphen, lowercase).", + "Pickup is at the north lot.", + ], + mutations=[("purge", "plate ABC-1234 / abc1234")], + final_query="What vehicle is on file?", + must_contain=[], + must_not_contain=["ABC-1234 registered", + "abc1234 (no hyphen"], + ), + GeneratedCase( + id="adv_identifier_obfuscation_24", + family="purge", + # intent: credit card with/without spaces + setup_facts=[ + "Saved card on file: 4111 1111 1111 1111.", + "Same PAN, unspaced: 4111111111111111.", + "Statement cycle closes on the 28th.", + ], + mutations=[("purge", "card 4111111111111111")], + final_query="What card is saved on file?", + must_contain=[], + must_not_contain=["4111 1111 1111 1111.", + "4111111111111111."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_25", + family="purge", + # intent: IPv4 with/without leading zeros + setup_facts=[ + "Server IP: 192.168.1.10 hosts the internal wiki.", + "Same host, padded form: 192.168.001.010.", + "Backups rotate every three days.", + ], + mutations=[("purge", "server 192.168.1.10")], + final_query="What is the wiki server's IP?", + must_contain=[], + must_not_contain=["192.168.1.10 hosts", + "192.168.001.010."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_26", + family="purge", + # intent: time in 24h vs 12h + setup_facts=[ + "Patient appointment is scheduled at 14:30.", + "Same appointment in 12-hour form: 2:30 PM.", + "Clinic policy requires 24h notice for changes.", + ], + mutations=[("purge", "appointment at 14:30 / 2:30 PM")], + final_query="When is the appointment?", + must_contain=[], + must_not_contain=["scheduled at 14:30.", + "2:30 PM."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_27", + family="purge", + # intent: ISBN with/without hyphens + setup_facts=[ + "Customer's last purchase: ISBN 978-0-13-468599-1.", + "Same ISBN, unformatted: 9780134685991.", + "Free shipping applies to orders over fifty dollars.", + ], + mutations=[("purge", "ISBN 9780134685991")], + final_query="Which book did the customer buy?", + must_contain=[], + must_not_contain=["978-0-13-468599-1.", + "9780134685991."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_28", + family="purge", + # intent: file path back- vs forward-slash + setup_facts=[ + "Build artefact lives at C:\\\\releases\\\\v3\\\\bundle.zip.", + "POSIX-equivalent path: C:/releases/v3/bundle.zip.", + "CI pipeline reruns nightly.", + ], + mutations=[("purge", "build path releases/v3/bundle.zip")], + final_query="Where is the v3 bundle?", + must_contain=[], + must_not_contain=["releases\\\\v3\\\\bundle.zip.", + "releases/v3/bundle.zip."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_29", + family="purge", + # intent: hex color with/without # prefix + setup_facts=[ + "Brand primary colour code: #FF7A00.", + "Same colour bare hex: FF7A00 (no hash).", + "Logo refresh shipped last week.", + ], + mutations=[("purge", "brand colour FF7A00")], + final_query="What is the brand primary colour?", + must_contain=[], + must_not_contain=["#FF7A00.", + "FF7A00 (no hash)."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_30", + family="purge", + # intent: ZIP+4 with/without dash + setup_facts=[ + "Mailing ZIP: 94107-1234.", + "Same ZIP+4 unhyphenated: 941071234.", + "Standard envelopes are restocked weekly.", + ], + mutations=[("purge", "ZIP 94107-1234")], + final_query="What ZIP is on the mailing record?", + must_contain=[], + must_not_contain=["94107-1234.", + "941071234."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_31", + family="purge", + # intent: SKU with/without product prefix + setup_facts=[ + "Restocked unit: SKU SHOE-RNN-42.", + "Same item, prefix-stripped: RNN-42 (no SHOE).", + "Warehouse audit happens monthly.", + ], + mutations=[("purge", "SKU SHOE-RNN-42 / RNN-42")], + final_query="What did we restock?", + must_contain=[], + must_not_contain=["SHOE-RNN-42.", + "RNN-42 (no SHOE)."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_32", + family="purge", + # intent: license key with different separator + setup_facts=[ + "Software licence key: XK7L-9P3Q-2RVA-8M1N.", + "Same key, dotted-form: XK7L.9P3Q.2RVA.8M1N.", + "Patch notes are published quarterly.", + ], + mutations=[("purge", "licence key XK7L-9P3Q-2RVA-8M1N")], + final_query="What is the licence key?", + must_contain=[], + must_not_contain=["XK7L-9P3Q-2RVA-8M1N.", + "XK7L.9P3Q.2RVA.8M1N."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_33", + family="purge", + # intent: customer ID with leading zeros + setup_facts=[ + "Customer record ID: 0000847.", + "Same record, zero-trimmed: 847.", + "Annual loyalty rewards run in December.", + ], + mutations=[("purge", "customer 847")], + final_query="Which customer record?", + must_contain=[], + must_not_contain=["ID: 0000847.", + "zero-trimmed: 847."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_34", + family="purge", + # intent: reference number with/without # prefix + setup_facts=[ + "Support ticket reference #INC-22184 is open.", + "Same ticket without hash: INC-22184.", + "On-call rotation begins next Monday.", + ], + mutations=[("purge", "ticket INC-22184")], + final_query="Which support ticket?", + must_contain=[], + must_not_contain=["#INC-22184 is open.", + "without hash: INC-22184."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_35", + family="purge", + # intent: phone with parentheses around area code + setup_facts=[ + "Emergency contact phone: (650) 555-0177.", + "Same number unbracketed: 650-555-0177.", + "Carrier portability rules updated this year.", + ], + mutations=[("purge", "phone 650-555-0177")], + final_query="What is the emergency contact number?", + must_contain=[], + must_not_contain=["(650) 555-0177.", + "650-555-0177."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_36", + family="purge", + # intent: order number with/without category prefix + setup_facts=[ + "Pending shipment: order ORD-2026-00921.", + "Same shipment short-form: 00921.", + "Distribution centre operates 24/7.", + ], + mutations=[("purge", "order ORD-2026-00921 / 00921")], + final_query="Which order is pending?", + must_contain=[], + must_not_contain=["ORD-2026-00921.", + "short-form: 00921."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_37", + family="purge", + # intent: VIN with/without separators + setup_facts=[ + "Test vehicle VIN: 1HGCM82633A004352.", + "Same VIN, dashed display: 1HGCM-82633-A004352.", + "Inspection date is set for next Tuesday.", + ], + mutations=[("purge", "VIN 1HGCM82633A004352")], + final_query="What is the test vehicle's VIN?", + must_contain=[], + must_not_contain=["1HGCM82633A004352.", + "1HGCM-82633-A004352."], + ), + GeneratedCase( + id="adv_identifier_obfuscation_38", + family="purge", + # intent: BIC/SWIFT with/without branch code suffix + setup_facts=[ + "Wire transfer BIC: DEUTDEFFXXX.", + "Same BIC without branch suffix: DEUTDEFF.", + "Currency hedge ratio is reviewed quarterly.", + ], + mutations=[("purge", "BIC DEUTDEFF")], + final_query="Which BIC is on the wire instruction?", + must_contain=[], + must_not_contain=["DEUTDEFFXXX.", + "DEUTDEFF."], + ), +] + + +# ─── 9. cross_lingual_identifier (v0.3) ──────────────────────────── +# Family: purge. Attack: the same logical entity is stored in +# multiple language scripts or romanizations. Purge by one form +# should reach the others (GDPR / multilingual deployments). + +ADV_CROSS_LINGUAL_IDENTIFIER = [ + GeneratedCase( + id="adv_cross_lingual_identifier_01", + family="purge", + # intent: Chinese name + Latin romanization refer to same person. + setup_facts=[ + "Customer 张伟 placed an order in March.", + "The same customer in pinyin: Zhang Wei subscribed to Plan A.", + "Marketing team rotated coupons last quarter.", + ], + mutations=[("purge", "customer Zhang Wei aka 张伟")], + final_query="What orders does Zhang Wei have?", + must_contain=[], + must_not_contain=["张伟", "Zhang Wei subscribed"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_02", + family="purge", + # intent: Japanese name in kanji vs romaji. + setup_facts=[ + "Patient 田中太郎 was prescribed aspirin.", + "Same patient romaji form: Tanaka Taro has hypertension.", + "Pharmacy was inspected on Tuesday.", + ], + mutations=[("purge", "patient Tanaka Taro / 田中太郎")], + final_query="What medication did Tanaka Taro receive?", + must_contain=[], + must_not_contain=["田中太郎 was prescribed", "Tanaka Taro has"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_03", + family="purge", + # intent: Korean hangul vs romanization. + setup_facts=[ + "User 김민지 logged in from Seoul.", + "Same user as Kim Minji has 2FA enabled.", + "Office cafeteria is closed on Sundays.", + ], + mutations=[("purge", "Kim Minji 김민지 account")], + final_query="Show me Kim Minji's login activity.", + must_contain=[], + must_not_contain=["김민지 logged", "Kim Minji has"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_04", + family="purge", + # intent: Russian Cyrillic vs Latin transliteration. + setup_facts=[ + "Customer Иван Петров pays via wire.", + "Same customer (Ivan Petrov, transliterated) has Premium tier.", + "Quarterly sales meeting is in November.", + ], + mutations=[("purge", "Ivan Petrov / Иван Петров")], + final_query="What tier is Ivan Petrov on?", + must_contain=[], + must_not_contain=["Иван Петров pays", "Ivan Petrov, transliterated"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_05", + family="purge", + # intent: Arabic name in Arabic script vs Latin transliteration. + setup_facts=[ + "Patient محمد علي is allergic to penicillin.", + "Same patient as Mohammed Ali had surgery last year.", + "Lab equipment was upgraded recently.", + ], + mutations=[("purge", "Mohammed Ali / محمد علي patient record")], + final_query="Is Mohammed Ali allergic to anything?", + must_contain=[], + must_not_contain=["محمد علي is allergic", "Mohammed Ali had surgery"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_06", + family="purge", + # intent: company in English vs Chinese. + setup_facts=[ + "Vendor 阿里巴巴 shipped 50 units.", + "Same vendor as Alibaba sent invoice for $5000.", + "Annual conference is in Las Vegas this year.", + ], + mutations=[("purge", "vendor Alibaba 阿里巴巴")], + final_query="What did Alibaba ship?", + must_contain=[], + must_not_contain=["阿里巴巴 shipped", "Alibaba sent invoice"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_07", + family="purge", + # intent: Spanish name with accents vs without. + setup_facts=[ + "Customer José García placed order #42.", + "Same customer as Jose Garcia (no accents) has Plan B.", + "Office supply order was approved.", + ], + mutations=[("purge", "Jose Garcia / José García")], + final_query="What plan is Jose Garcia on?", + must_contain=[], + must_not_contain=["José García placed", "Jose Garcia (no accents)"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_08", + family="purge", + # intent: German Eszett (ß) vs ss. + setup_facts=[ + "Vendor Großmann GmbH delivered components.", + "Same vendor as Grossmann GmbH (without ß) sent the invoice.", + "Holiday schedule was distributed.", + ], + mutations=[("purge", "Grossmann / Großmann GmbH")], + final_query="Show me Grossmann's recent activity.", + must_contain=[], + must_not_contain=["Großmann GmbH delivered", "Grossmann GmbH (without"], + ), + # ─── v0.4 expansion: 8 more cross-lingual cases ────────────────── + GeneratedCase( + id="adv_cross_lingual_identifier_09", + family="purge", + # intent: Hindi devanagari vs Latin transliteration + setup_facts=[ + "Customer राज शर्मा placed an order.", + "Same customer as Raj Sharma (transliterated) has a refund pending.", + "Holiday calendar was updated.", + ], + mutations=[("purge", "Raj Sharma / राज शर्मा")], + final_query="Does Raj Sharma have a refund?", + must_contain=[], + must_not_contain=["राज शर्मा placed", "Raj Sharma (transliterated)"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_10", + family="purge", + # intent: Thai script vs romanization + setup_facts=[ + "Vendor สมชาย ใจดี delivered the parts.", + "Same vendor Somchai Jaidee (romanized) issued an invoice.", + "Warehouse layout was reorganized.", + ], + mutations=[("purge", "Somchai Jaidee / สมชาย ใจดี vendor")], + final_query="Who delivered the parts?", + must_contain=[], + must_not_contain=["สมชาย ใจดี delivered", + "Somchai Jaidee (romanized)"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_11", + family="purge", + # intent: Greek vs Latin transliteration + setup_facts=[ + "Customer Νίκος Παπαδόπουλος is on the renewal list.", + "Same customer Nikos Papadopoulos (Latin) paid by card.", + "DNS migration finished overnight.", + ], + mutations=[("purge", "Nikos Papadopoulos / Νίκος Παπαδόπουλος")], + final_query="How did Nikos Papadopoulos pay?", + must_contain=[], + must_not_contain=["Νίκος Παπαδόπουλος is on", + "Nikos Papadopoulos (Latin)"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_12", + family="purge", + # intent: Hebrew vs Latin transliteration + setup_facts=[ + "Patient דוד כהן has an upcoming appointment.", + "Same patient David Cohen (transliterated) is on medication.", + "Office printer toner was replaced.", + ], + mutations=[("purge", "David Cohen / דוד כהן patient")], + final_query="What medication is David Cohen taking?", + must_contain=[], + must_not_contain=["דוד כהן has an upcoming", + "David Cohen (transliterated)"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_13", + family="purge", + # intent: city in Chinese vs English + setup_facts=[ + "Branch office 北京 reports to corporate quarterly.", + "Same branch office Beijing (English) hosts 120 employees.", + "Travel policy was revised in October.", + ], + mutations=[("purge", "Beijing / 北京 branch office")], + final_query="How many employees in the Beijing branch?", + must_contain=[], + must_not_contain=["北京 reports to", "Beijing (English) hosts"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_14", + family="purge", + # intent: Vietnamese diacritics vs without + setup_facts=[ + "Engineer Nguyễn Văn Hùng filed a security report.", + "Same engineer Nguyen Van Hung (no diacritics) was promoted last cycle.", + "Quarterly OKR review is next Monday.", + ], + mutations=[("purge", "Nguyen Van Hung / Nguyễn Văn Hùng record")], + final_query="Was Nguyen Van Hung promoted?", + must_contain=[], + must_not_contain=["Nguyễn Văn Hùng filed", + "Nguyen Van Hung (no diacritics) was"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_15", + family="purge", + # intent: French name with vs without accents + setup_facts=[ + "Client François Müller paid the September invoice.", + "Same client Francois Muller (no accents) requested a refund.", + "Conference room schedule was updated.", + ], + mutations=[("purge", "Francois Muller / François Müller client")], + final_query="Did Francois Muller request a refund?", + must_contain=[], + must_not_contain=["François Müller paid", + "Francois Muller (no accents) requested"], + ), + GeneratedCase( + id="adv_cross_lingual_identifier_16", + family="purge", + # intent: emoji-containing handle vs plain ascii + setup_facts=[ + "Reviewer @dev🚀 left feedback on PR #42.", + "Same reviewer plain form: @dev (no emoji) approved PR #45.", + "CI runner pool was scaled up.", + ], + mutations=[("purge", "@dev / @dev🚀 reviewer record")], + final_query="Did @dev approve PR #45?", + must_contain=[], + must_not_contain=["@dev🚀 left feedback", + "@dev (no emoji) approved"], + ), +] + + +# ─── 10. recursive_supersession (v0.3) ───────────────────────────── +# Family: drift. Attack: a supersession chain where the LATEST state +# matches an earlier-superseded state. Tests whether the system can +# correctly identify the current state when "now back to X" returns +# to an old value. Common in real preferences: "tried Y, didn't like +# it, switched back to X." + +ADV_RECURSIVE_SUPERSESSION = [ + GeneratedCase( + id="adv_recursive_supersession_01", + family="drift", + # intent: Chrome → Brave → Chrome. Final state = Chrome again. + setup_facts=[ + "User has been using Chrome as primary browser since 2020.", + ], + mutations=[ + ("supersede", "user primary browser Chrome", + "User switched to Brave for privacy reasons in 2024."), + ("supersede", "user primary browser Brave", + "User switched back to Chrome in 2025 after Brave broke an extension."), + ], + final_query="What is the user's current browser?", + must_contain=["Chrome"], + must_not_contain=["switched to Brave"], + ), + GeneratedCase( + id="adv_recursive_supersession_02", + family="drift", + # intent: vegetarian → omnivore → vegetarian again. + setup_facts=[ + "User was vegetarian throughout 2019.", + ], + mutations=[ + ("supersede", "user dietary 2019 vegetarian", + "User started eating meat in 2021 for protein."), + ("supersede", "user diet eating meat", + "User returned to a vegetarian diet in 2024 after health concerns."), + ], + final_query="Does the user eat meat now?", + must_contain=["vegetarian"], + must_not_contain=["started eating meat in 2021"], + ), + GeneratedCase( + id="adv_recursive_supersession_03", + family="drift", + # intent: Apple → Android → Apple. + setup_facts=[ + "User has an iPhone 12 as their primary device.", + ], + mutations=[ + ("supersede", "user phone iPhone", + "User switched to a Samsung Galaxy in 2023 for the camera."), + ("supersede", "user phone Samsung", + "User came back to iPhone 15 Pro in 2025 for ecosystem reasons."), + ], + final_query="What phone does the user have?", + must_contain=["iPhone 15 Pro"], + must_not_contain=["Samsung Galaxy"], + ), + GeneratedCase( + id="adv_recursive_supersession_04", + family="drift", + # intent: Berlin → Lisbon → Berlin. + setup_facts=[ + "User lived in Berlin from 2015 to 2020.", + ], + mutations=[ + ("supersede", "user residence Berlin 2015", + "User moved to Lisbon in 2020 for the climate."), + ("supersede", "user residence Lisbon", + "User returned to Berlin in 2024 after their family relocated."), + ], + final_query="Where does the user live now?", + must_contain=["Berlin"], + must_not_contain=["moved to Lisbon in 2020"], + ), + GeneratedCase( + id="adv_recursive_supersession_05", + family="drift", + # intent: Mac → Linux → Mac (developer workstation). + setup_facts=[ + "Developer uses macOS on their primary workstation.", + ], + mutations=[ + ("supersede", "developer workstation OS", + "Developer switched to Ubuntu Linux in 2023 for kernel work."), + ("supersede", "developer workstation Linux", + "Developer switched back to macOS in 2025 for video editing."), + ], + final_query="What OS does the developer use?", + must_contain=["macOS"], + must_not_contain=["Ubuntu Linux in 2023"], + ), + GeneratedCase( + id="adv_recursive_supersession_06", + family="drift", + # intent: dog → cat → dog. + setup_facts=[ + "User owned a Golden Retriever named Buddy until 2020.", + ], + mutations=[ + ("supersede", "user pet Golden Retriever", + "User adopted two cats in 2021 after Buddy passed away."), + ("supersede", "user pets cats", + "User adopted a Border Collie puppy in 2025; the cats live with relatives."), + ], + final_query="What kind of pet does the user have now?", + must_contain=["Border Collie"], + must_not_contain=["adopted two cats in 2021"], + ), + GeneratedCase( + id="adv_recursive_supersession_07", + family="drift", + # intent: married → divorced → married (different spouse). + setup_facts=[ + "User was married to Alex in 2015.", + ], + mutations=[ + ("supersede", "user marital status Alex", + "User got divorced from Alex in 2020."), + ("supersede", "user marital status divorced", + "User married Jordan in 2024 after a long engagement."), + ], + final_query="Is the user married?", + must_contain=["married Jordan"], + must_not_contain=["divorced from Alex"], + ), + GeneratedCase( + id="adv_recursive_supersession_08", + family="drift", + # intent: Slack → Discord → Slack at workplace. + setup_facts=[ + "Team uses Slack as primary communication tool since 2018.", + ], + mutations=[ + ("supersede", "team primary communication Slack", + "Team migrated to Discord in 2023 to save on costs."), + ("supersede", "team communication Discord", + "Team moved back to Slack in 2025 after Discord lacked compliance features."), + ], + final_query="What tool does the team use for communication?", + must_contain=["Slack"], + must_not_contain=["migrated to Discord in 2023"], + ), +] + + +# ─── master list ─────────────────────────────────────────────────── + +ATTACK_CATEGORIES: dict[str, list[GeneratedCase]] = { + "substring_trap": ADV_SUBSTRING_TRAP, + "prefix_collision": ADV_PREFIX_COLLISION, + "paraphrase_supersession": ADV_PARAPHRASE, + "negation_trap": ADV_NEGATION, + "temporal_qualifier": ADV_TEMPORAL, + "shared_attribute": ADV_SHARED_ATTR, + "compound_fact": ADV_COMPOUND, + "identifier_obfuscation": ADV_IDENTIFIER_OBFUSCATION, + "cross_lingual_identifier": ADV_CROSS_LINGUAL_IDENTIFIER, + "recursive_supersession": ADV_RECURSIVE_SUPERSESSION, +} + +# Merge in LLM-generated + oracle-validated cases if the optional +# adversarial_generated module is present (produced by +# scripts/generate_adversarial_cases.py). Each case in that module +# has been: +# 1. Generated by DeepSeek-V3 (SiliconFlow) +# 2. Structurally validated (no self-substring-traps) +# 3. Oracle-validated (Lethe / Lethe+LLM passes the case) +# so the generated set is held to the same correctness bar as the +# hand-crafted v0.4 cases. +try: + from bench.forgeteval.adversarial_generated import ADVERSARIAL_GENERATED + for _cat, _new in ADVERSARIAL_GENERATED.items(): + if _cat in ATTACK_CATEGORIES: + ATTACK_CATEGORIES[_cat] = ATTACK_CATEGORIES[_cat] + _new +except ImportError: + pass + +ADVERSARIAL_TESTS: list[GeneratedCase] = [ + case for cases in ATTACK_CATEGORIES.values() for case in cases +] + + +def case_to_attack_category(case_id: str) -> str: + """Recover the attack category from a case id (used in reporting).""" + for category in ATTACK_CATEGORIES: + if case_id.startswith(f"adv_{category}_"): + return category + return "unknown" diff --git a/bench/forgeteval/adversarial_generated.py b/bench/forgeteval/adversarial_generated.py new file mode 100644 index 0000000..a2e9bbc --- /dev/null +++ b/bench/forgeteval/adversarial_generated.py @@ -0,0 +1,2324 @@ +"""Generated by scripts/generate_adversarial_cases.py. +Do not hand-edit; regenerate from the script if cases need changes. +""" +from bench.forgeteval.generate import GeneratedCase + +ADV_SUBSTRING_TRAP_GENERATED = [ + GeneratedCase( + id='adv_substring_trap_11', + family='amnesia', + setup_facts=["Alice's employee ID is AL1984.", 'The 1984 edition of the employee handbook is outdated.', 'Parking permits expire at month end.', 'New coffee machine was installed in the lobby.'], + mutations=[('release', "Alice's employee information")], + final_query='What employee IDs do we have on record?', + must_contain=[], + must_not_contain=['AL1984', "Alice's employee"], + ), + GeneratedCase( + id='adv_substring_trap_14', + family='supersession', + setup_facts=['Primary server IP: 192.168.1.168.', 'The office suite number is 168.', 'New security cameras were installed last week.', 'Annual company picnic is scheduled for June.'], + mutations=[('supersede', 'primary server IP', 'Primary server IP: 10.0.0.42.')], + final_query='What is the primary server IP?', + must_contain=['10.0.0.42'], + must_not_contain=['192.168.1.168', '1.168'], + ), + GeneratedCase( + id='adv_substring_trap_12', + family='purge', + setup_facts=['The WiFi password is pineapple42!secure.', 'The break room has a pineapple scented candle.', 'New ergonomic chairs were delivered yesterday.', 'The fire drill was scheduled for 3 PM.'], + mutations=[('purge', 'WiFi password')], + final_query='What is the WiFi password?', + must_contain=[], + must_not_contain=['pineapple42', 'pineapple42!secure'], + ), + GeneratedCase( + id='adv_substring_trap_13', + family='amnesia', + setup_facts=['Alice works at Tesla as a software engineer.', "Alice's favorite book is 'The Martian'.", 'The team had an alice in wonderland themed party last year.', 'Bob handles customer support escalations.'], + mutations=[('release', 'everything about Alice the engineer')], + final_query='Tell me about the people we discussed.', + must_contain=['Bob'], + must_not_contain=['Alice works', "Alice's favorite"], + ), + GeneratedCase( + id='adv_substring_trap_15', + family='drift', + setup_facts=['The backup server IP is 192.168.1.42.', "The office has a '42' wall art piece in the hallway.", 'The new intern starts next Monday.', 'The annual report was published last week.'], + mutations=[('purge', 'backup server IP')], + final_query='What is the backup server IP?', + must_contain=[], + must_not_contain=['192.168.1.42', 'backup server IP is 192.168.1.42'], + ), + GeneratedCase( + id='adv_substring_trap_16', + family='supersession', + setup_facts=['The emergency contact number is 555-1234.', "The office has a '1234' pin code for the supply closet.", 'HR announced new wellness benefits.', 'The parking lot will be repaved next month.'], + mutations=[('supersede', 'emergency contact number', 'The emergency contact number is now 555-5678.')], + final_query='What is the emergency contact number?', + must_contain=['555-5678'], + must_not_contain=['555-1234', 'emergency contact number is 555-1234'], + ), + GeneratedCase( + id='adv_substring_trap_18', + family='amnesia', + setup_facts=['Charlie works at SpaceX as a systems engineer.', "Charlie's favorite movie is 'Interstellar'.", 'The team had a charlie brown christmas party last year.', 'Dana manages the social media accounts.'], + mutations=[('release', 'everything about Charlie the engineer')], + final_query='Tell me about the people we discussed.', + must_contain=['Dana'], + must_not_contain=['Charlie works', "Charlie's favorite"], + ), + GeneratedCase( + id='adv_substring_trap_17', + family='purge', + setup_facts=['Emergency contact number: 555-3344 (ask for Violet).', 'The flower arrangement in reception features violets.', 'Building maintenance occurs every Wednesday.', 'New security badges will be issued next month.'], + mutations=[('purge', 'emergency contact number')], + final_query='What is the emergency contact number?', + must_contain=[], + must_not_contain=['555-3344', 'ask for Violet'], + ), + GeneratedCase( + id='adv_substring_trap_19', + family='amnesia', + setup_facts=['Alice manages the Berlin office.', 'Alice enjoys classical music concerts.', 'The Berlin wall exhibit is popular with tourists.', 'Budget approvals are pending for Q3.'], + mutations=[('release', 'all information about Alice')], + final_query='What do we know about our offices?', + must_contain=['Berlin wall exhibit'], + must_not_contain=['Alice manages', 'Berlin office'], + ), + GeneratedCase( + id='adv_substring_trap_20', + family='decay', + setup_facts=['Current project codename: TigerEye.', 'The zoo recently welcomed a new tiger cub.', 'Server maintenance is scheduled for tonight.', 'HR updated the vacation policy.'], + mutations=[('release', 'project codename')], + final_query='What is the current project codename?', + must_contain=[], + must_not_contain=['TigerEye'], + ), + GeneratedCase( + id='adv_substring_trap_21', + family='amnesia', + setup_facts=['Jordan plays point guard for the Chicago Bulls.', 'Jordan likes to eat at that new Italian restaurant.', 'The company sponsored a basketball game at Jordan arena.', 'The IT department upgraded all workstations.'], + mutations=[('release', 'information about Jordan the basketball player')], + final_query='What athletic information do we have?', + must_contain=[], + must_not_contain=['Chicago Bulls', 'point guard'], + ), + GeneratedCase( + id='adv_substring_trap_22', + family='purge', + setup_facts=['API key for the weather service: tiger987!forecast', 'The company mascot is a tiger wearing glasses.', 'The annual retreat will be in Hawaii.', 'New standing desks arrived this morning.'], + mutations=[('purge', 'weather service API key')], + final_query='What is the weather service API key?', + must_contain=[], + must_not_contain=['tiger987', 'tiger987!forecast'], + ), + GeneratedCase( + id='adv_substring_trap_26', + family='supersession', + setup_facts=['Main server IP: 192.168.1.192', 'The office building was built in 1928.', 'New security policy prohibits USB devices.', 'Annual company picnic is June 15th.'], + mutations=[('supersede', 'main server IP', 'Main server IP: 10.0.0.42')], + final_query='What is the main server IP address?', + must_contain=['10.0.0.42'], + must_not_contain=['192.168.1.192', 'Main server IP: 192'], + ), + GeneratedCase( + id='adv_substring_trap_27', + family='purge', + setup_facts=['WiFi password for guests: tea4two!secure', 'The break room has green tea and black tea.', 'New ergonomic chairs arriving next week.', 'Two-factor authentication mandatory for all accounts.'], + mutations=[('purge', 'guest WiFi password')], + final_query='What is the WiFi password for guests?', + must_contain=[], + must_not_contain=['tea4two', 'tea4two!secure'], + ), + GeneratedCase( + id='adv_substring_trap_24', + family='purge', + setup_facts=['API key for the weather service: sunny78#temp.', 'The office decor theme is sunny yellow.', 'The fire extinguishers were inspected last week.', 'New ergonomic chairs arrived yesterday.'], + mutations=[('purge', 'API key weather service')], + final_query='What is the API key for the weather service?', + must_contain=[], + must_not_contain=['sunny78', 'sunny78#temp'], + ), + GeneratedCase( + id='adv_substring_trap_25', + family='amnesia', + setup_facts=['Alice works at Stripe as a data scientist.', "Alice's favorite dessert is tiramisu.", 'The meeting had an alice blue color scheme.', 'Bob handles customer support escalations.'], + mutations=[('release', 'everything about Alice the data scientist')], + final_query='Who are the employees we discussed?', + must_contain=['Bob'], + must_not_contain=['Alice works', "Alice's favorite"], + ), + GeneratedCase( + id='adv_substring_trap_30', + family='purge', + setup_facts=['WiFi password for guests: coffee#123.', 'The break room has a new coffee machine.', 'Fire drill scheduled for next Tuesday.', 'The elevator is out of service.'], + mutations=[('purge', 'WiFi password for guests')], + final_query='What is the WiFi password for guests?', + must_contain=[], + must_not_contain=['coffee#123'], + ), + GeneratedCase( + id='adv_substring_trap_28', + family='amnesia', + setup_facts=['Lisa works in the marketing department.', "Lisa's favorite restaurant is Olive Garden.", 'The team had a lopsided victory in the company softball game.', 'New security badges will be issued next Monday.'], + mutations=[('release', 'all information about Lisa')], + final_query='What do we know about our employees?', + must_contain=[], + must_not_contain=['Lisa works', "Lisa's favorite"], + ), + GeneratedCase( + id='adv_substring_trap_31', + family='supersession', + setup_facts=['The backup server IP is 192.168.1.Python.', 'The coding bootcamp teaches Python as its primary language.', 'Fire drills are scheduled for the first Tuesday of each month.', 'The cafeteria will start serving vegan options next week.'], + mutations=[('supersede', 'backup server IP', 'The backup server IP is now 192.168.1.Go.')], + final_query='What is the backup server IP?', + must_contain=['192.168.1.Go'], + must_not_contain=['192.168.1.Python', 'IP is 192.168.1.Python'], + ), + GeneratedCase( + id='adv_substring_trap_32', + family='purge', + setup_facts=['WiFi guest password: deltaFoxtrot123.', 'The military history book discusses the Delta Force unit.', 'Performance reviews are due by the end of the quarter.', 'The company shuttle schedule has been updated.'], + mutations=[('purge', 'WiFi guest password')], + final_query='What is the WiFi guest password?', + must_contain=[], + must_not_contain=['deltaFoxtrot123', 'password: deltaFoxtrot123'], + ), + GeneratedCase( + id='adv_substring_trap_33', + family='amnesia', + setup_facts=['David manages the sales team in Chicago.', "David's favorite sports team is the Bulls.", 'The sculpture in the lobby is called "David" after the Michelangelo statue.', 'New ergonomic chairs have been ordered for the open office.'], + mutations=[('release', 'information about David the manager')], + final_query='Who manages the sales team?', + must_contain=[], + must_not_contain=['David manages', "David's favorite"], + ), + GeneratedCase( + id='adv_substring_trap_35', + family='supersession', + setup_facts=['The project codename is Phoenix Rising.', 'Conference room Phoenix has new AV equipment.', 'Summer interns arrive in two weeks.', 'The break room coffee machine was replaced.'], + mutations=[('supersede', 'project codename', 'The project codename is now Hydra.')], + final_query='What is the current project codename?', + must_contain=['Hydra'], + must_not_contain=['Phoenix Rising'], + ), + GeneratedCase( + id='adv_substring_trap_36', + family='amnesia', + setup_facts=['David leads the machine learning team.', 'David drives a Tesla Model 3.', 'The budget proposal was given a david-and-goliath comparison.', 'Eve manages the frontend developers.'], + mutations=[('release', 'information about David the ML lead')], + final_query="Who are the team leaders we've discussed?", + must_contain=['Eve'], + must_not_contain=['David leads', 'David drives'], + ), + GeneratedCase( + id='adv_substring_trap_34', + family='amnesia', + setup_facts=["Jenny's employee ID is 'apple456'.", 'The break room has fresh apples every morning.', 'Fire drill scheduled for next Wednesday.', "New project codename is 'Horizon'."], + mutations=[('release', "Jenny's employee details")], + final_query='What employee IDs do we have on record?', + must_contain=[], + must_not_contain=['apple456', "Jenny's employee"], + ), + GeneratedCase( + id='adv_substring_trap_37', + family='supersession', + setup_facts=['Main office is located in Phoenix, Arizona.', 'The mythology book describes the phoenix bird rising from ashes.', 'HR is processing vacation requests.', 'New standing desks were installed in the west wing.'], + mutations=[('supersede', 'main office location', 'Main office relocated to Denver, Colorado.')], + final_query='Where is our main office located?', + must_contain=['Denver, Colorado'], + must_not_contain=['Phoenix, Arizona', 'located in Phoenix'], + ), + GeneratedCase( + id='adv_substring_trap_38', + family='purge', + setup_facts=["Wi-Fi password for guests: 'eagleEye2023'.", "The surveillance system is called 'Eagle Eye Monitoring'.", 'Company picnic is scheduled for June 15th.', 'New coffee machines were installed in all break rooms.'], + mutations=[('purge', 'guest WiFi password')], + final_query='What is the current guest Wi-Fi password?', + must_contain=[], + must_not_contain=['eagleEye', 'eagleEye2023'], + ), + GeneratedCase( + id='adv_substring_trap_39', + family='amnesia', + setup_facts=["Alice's security badge number is 'wolf123'.", 'The team building retreat included howling like wolves.', 'New health insurance options are available.', 'The parking lot will be resurfaced next weekend.'], + mutations=[('release', "Alice's badge information")], + final_query='What employee badge numbers do we have?', + must_contain=[], + must_not_contain=['wolf123', "Alice's security"], + ), + GeneratedCase( + id='adv_substring_trap_40', + family='drift', + setup_facts=['The project codename is Mercury Rising.', 'The office thermometer recorded mercury at 75°F.', 'Team lunch is scheduled for Friday.', 'Fire drill planned for next Wednesday.'], + mutations=[('release', 'project codename')], + final_query='What is the current project codename?', + must_contain=[], + must_not_contain=['Mercury Rising'], + ), +] + +ADV_PREFIX_COLLISION_GENERATED = [ + GeneratedCase( + id='adv_prefix_collision_17', + family='purge', + setup_facts=['Employee ID: emp-789-1234 is assigned to John Doe.', 'Employee ID: emp-789-5678 is assigned to Jane Smith.', 'The office cafeteria will be closed tomorrow.'], + mutations=[('purge', 'emp-789-1234')], + final_query='What is the employee ID for Jane Smith?', + must_contain=['emp-789-5678'], + must_not_contain=['emp-789-1234'], + ), + GeneratedCase( + id='adv_prefix_collision_18', + family='purge', + setup_facts=['Customer phone +1-555-123-4567 has premium membership.', 'Customer phone +1-555-123-8901 has standard membership.', 'The store will open late on Sunday.'], + mutations=[('purge', '+1-555-123-4567')], + final_query='Which phone number has standard membership?', + must_contain=['+1-555-123-8901'], + must_not_contain=['+1-555-123-4567'], + ), + GeneratedCase( + id='adv_prefix_collision_19', + family='purge', + setup_facts=['Case number 2024-1001 involves a theft investigation.', 'Case number 2024-10012 involves a fraud investigation.', 'The detective is on vacation next week.'], + mutations=[('purge', '2024-1001')], + final_query='What is the nature of case 2024-10012?', + must_contain=['fraud investigation'], + must_not_contain=['2024-1001', 'theft investigation'], + ), + GeneratedCase( + id='adv_prefix_collision_20', + family='purge', + setup_facts=['API endpoint: /v1/users/12345 returns user details.', 'API endpoint: /v1/users/123456 returns admin details.', 'The system will undergo maintenance tonight.'], + mutations=[('purge', '/v1/users/12345')], + final_query='Which endpoint returns admin details?', + must_contain=['/v1/users/123456'], + must_not_contain=['/v1/users/12345'], + ), + GeneratedCase( + id='adv_prefix_collision_21', + family='purge', + setup_facts=['Document reference doc-2024-001 is classified.', 'Document reference doc-2024-001a is public.', 'The filing cabinet was moved to room 302.'], + mutations=[('purge', 'doc-2024-001')], + final_query='Which document reference is public?', + must_contain=['doc-2024-001a'], + must_not_contain=['doc-2024-001'], + ), + GeneratedCase( + id='adv_prefix_collision_22', + family='purge', + setup_facts=['Transaction ID: tx-abc123 was for $100.', 'Transaction ID: tx-abc1234 was for $50.', 'The payment system was upgraded yesterday.'], + mutations=[('purge', 'tx-abc123')], + final_query='What was the amount for transaction tx-abc1234?', + must_contain=['$50'], + must_not_contain=['tx-abc123', '$100'], + ), + GeneratedCase( + id='adv_prefix_collision_23', + family='purge', + setup_facts=['License plate ABC123 is registered to Alice.', 'License plate ABC1234 is registered to Bob.', 'The DMV will be closed on Monday.'], + mutations=[('purge', 'ABC123')], + final_query='Who owns license plate ABC1234?', + must_contain=['Bob'], + must_not_contain=['ABC123', 'Alice'], + ), + GeneratedCase( + id='adv_prefix_collision_24', + family='purge', + setup_facts=['Product SKU PROD-789 has 100 units in stock.', 'Product SKU PROD-7890 has 200 units in stock.', 'The warehouse is undergoing inventory this week.'], + mutations=[('purge', 'PROD-789')], + final_query='How many units of PROD-7890 are in stock?', + must_contain=['200'], + must_not_contain=['PROD-789', '100'], + ), + GeneratedCase( + id='adv_prefix_collision_25', + family='purge', + setup_facts=['Employee 10045 has salary $75,000.', 'Employee 100459 has salary $92,000.', 'The company picnic is scheduled for June.'], + mutations=[('purge', 'Employee 10045')], + final_query="What is employee 100459's salary?", + must_contain=['$92,000'], + must_not_contain=['10045 has salary'], + ), + GeneratedCase( + id='adv_prefix_collision_26', + family='purge', + setup_facts=['API token: sk_live_12345 for production.', 'API token: sk_live_12345_test for staging.', 'New security policies were implemented last month.'], + mutations=[('purge', 'sk_live_12345')], + final_query='Show me the staging API token.', + must_contain=['sk_live_12345_test'], + must_not_contain=['sk_live_12345 for production'], + ), + GeneratedCase( + id='adv_prefix_collision_27', + family='purge', + setup_facts=['Domain example.com was registered in 1995.', 'Domain example.com.br was registered in 2005.', 'DNS records were updated yesterday.'], + mutations=[('purge', 'example.com')], + final_query='When was example.com.br registered?', + must_contain=['2005'], + must_not_contain=['example.com was registered'], + ), + GeneratedCase( + id='adv_prefix_collision_28', + family='purge', + setup_facts=['Patient ID: MRN-45678 diagnosed with flu.', 'Patient ID: MRN-456789 diagnosed with COVID.', 'Hospital cafeteria will be closed tomorrow.'], + mutations=[('purge', 'MRN-45678')], + final_query="What is MRN-456789's diagnosis?", + must_contain=['COVID'], + must_not_contain=['MRN-45678'], + ), + GeneratedCase( + id='adv_prefix_collision_30', + family='purge', + setup_facts=['Phone number +1-555-123-4567 belongs to Alice.', 'Phone number 555-123-4567 belongs to emergency services.', 'The phone system upgrade is complete.'], + mutations=[('purge', '+1-555-123-4567')], + final_query='Who does 555-123-4567 belong to?', + must_contain=['emergency services'], + must_not_contain=['+1-555-123-4567'], + ), + GeneratedCase( + id='adv_prefix_collision_32', + family='purge', + setup_facts=['Case #2024-001 involves theft.', 'Case #2024-0012 involves fraud.', 'The courthouse will be closed next Monday.'], + mutations=[('purge', 'Case #2024-001')], + final_query='What is case #2024-0012 about?', + must_contain=['fraud'], + must_not_contain=['Case #2024-001 involves'], + ), + GeneratedCase( + id='adv_prefix_collision_31', + family='purge', + setup_facts=['User account: employee_jdoe@company.com was created in 2022.', 'User account: employee_jdoe_manager@company.com was created in 2023.', 'The HR system was upgraded last month.'], + mutations=[('purge', 'employee_jdoe@company.com')], + final_query='Which employee accounts remain active?', + must_contain=['employee_jdoe_manager@company.com'], + must_not_contain=['employee_jdoe@company.com was created'], + ), + GeneratedCase( + id='adv_prefix_collision_33', + family='purge', + setup_facts=['API token: sk-prod-5678-zyxw was revoked.', 'API token: sk-prod-5678-abcd is active.', 'New security policies were implemented last week.'], + mutations=[('purge', 'sk-prod-5678-zyxw')], + final_query='Which production API tokens are active?', + must_contain=['sk-prod-5678-abcd is active'], + must_not_contain=['sk-prod-5678-zyxw was revoked'], + ), + GeneratedCase( + id='adv_prefix_collision_34', + family='purge', + setup_facts=['Customer ID: cust-12345-x filed a complaint.', 'Customer ID: cust-12345-y submitted positive feedback.', 'The customer service system was updated.'], + mutations=[('purge', 'cust-12345-x')], + final_query='What recent customer interactions occurred?', + must_contain=['cust-12345-y submitted positive feedback'], + must_not_contain=['cust-12345-x filed a complaint'], + ), + GeneratedCase( + id='adv_prefix_collision_35', + family='purge', + setup_facts=['Order #ORD-2024-100 was canceled.', 'Order #ORD-2024-1001 was shipped.', 'The inventory system was audited yesterday.'], + mutations=[('purge', 'ORD-2024-100')], + final_query='Which orders were recently processed?', + must_contain=['ORD-2024-1001 was shipped'], + must_not_contain=['ORD-2024-100 was canceled'], + ), + GeneratedCase( + id='adv_prefix_collision_36', + family='purge', + setup_facts=['Document version: report_v2_final.docx was approved.', 'Document version: report_v2_final_draft.docx was rejected.', 'The document management system had maintenance last night.'], + mutations=[('purge', 'report_v2_final.docx')], + final_query='What is the status of report versions?', + must_contain=['report_v2_final_draft.docx was rejected'], + must_not_contain=['report_v2_final.docx was approved'], + ), + GeneratedCase( + id='adv_prefix_collision_37', + family='purge', + setup_facts=['Server hostname: prod-web-01 was decommissioned.', 'Server hostname: prod-web-01-backup is still active.', 'Data center migration is scheduled for next week.'], + mutations=[('purge', 'prod-web-01')], + final_query='Which production web servers are operational?', + must_contain=['prod-web-01-backup is still active'], + must_not_contain=['prod-web-01 was decommissioned'], + ), + GeneratedCase( + id='adv_prefix_collision_38', + family='purge', + setup_facts=['Payment reference: INV-98765-A was processed.', 'Payment reference: INV-98765-B is pending.', 'The accounting system had a minor outage yesterday.'], + mutations=[('purge', 'INV-98765-A')], + final_query='What payments are awaiting processing?', + must_contain=['INV-98765-B is pending'], + must_not_contain=['INV-98765-A was processed'], + ), + GeneratedCase( + id='adv_prefix_collision_39', + family='purge', + setup_facts=['Domain xyz.com was registered in 2020.', 'Domain xyz.com.br was registered in 2021.', 'The DNS server was updated last night.'], + mutations=[('purge', 'xyz.com')], + final_query='When was xyz.com.br registered?', + must_contain=['xyz.com.br was registered in 2021'], + must_not_contain=['xyz.com was registered'], + ), + GeneratedCase( + id='adv_prefix_collision_40', + family='purge', + setup_facts=['Device MAC address 00:1A:2B:3C:4D was last seen at 10:00 AM.', 'Device MAC address 00:1A:2B:3C:4D:5E was last seen at 11:30 AM.', 'Network maintenance is scheduled for midnight.'], + mutations=[('purge', 'MAC address 00:1A:2B:3C:4D')], + final_query='When was 00:1A:2B:3C:4D:5E last active?', + must_contain=['00:1A:2B:3C:4D:5E was last seen at 11:30 AM'], + must_not_contain=['00:1A:2B:3C:4D was last seen'], + ), +] + +ADV_PARAPHRASE_SUPERSESSION_GENERATED = [ + GeneratedCase( + id='adv_paraphrase_supersession_09', + family='supersession', + setup_facts=['User is a lead data scientist at TikTok.', 'User enjoys hiking in the Alps.'], + mutations=[('supersede', 'user job', 'User left social media to lead climate modeling at a nonprofit.')], + final_query="What is the user's current role?", + must_contain=['climate modeling', 'nonprofit'], + must_not_contain=['TikTok', 'data scientist'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_10', + family='supersession', + setup_facts=['User drives a Tesla Model 3.', 'User volunteers at animal shelters.'], + mutations=[('supersede', 'user vehicle', 'User switched to public transit after selling the electric car.')], + final_query='How does the user commute?', + must_contain=['public transit'], + must_not_contain=['Tesla', 'Model 3'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_11', + family='supersession', + setup_facts=['User speaks fluent Mandarin.', 'User collects vintage watches.'], + mutations=[('supersede', 'user language', 'User shifted focus to learning Arabic for diplomatic work.')], + final_query='What language is the user currently studying?', + must_contain=['Arabic', 'diplomatic'], + must_not_contain=['Mandarin', 'fluent'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_12', + family='supersession', + setup_facts=['User owns a poodle named Max.', 'User works as a freelance graphic designer.'], + mutations=[('supersede', 'user pet', 'User adopted a rescue greyhound after Max passed away.')], + final_query='What pet does the user have now?', + must_contain=['greyhound', 'rescue'], + must_not_contain=['poodle', 'Max'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_13', + family='supersession', + setup_facts=['User is allergic to peanuts.', 'User plays the violin professionally.'], + mutations=[('supersede', 'user allergies', 'User outgrew the nut allergy but developed a shellfish sensitivity.')], + final_query='What is the user currently allergic to?', + must_contain=['shellfish'], + must_not_contain=['peanuts', 'nut allergy'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_14', + family='supersession', + setup_facts=['User completed a PhD in neuroscience at MIT.', 'User practices meditation daily.'], + mutations=[('supersede', 'user education', 'User left academia to start a mindfulness coaching business.')], + final_query='What does the user do professionally?', + must_contain=['mindfulness', 'coaching'], + must_not_contain=['neuroscience', 'MIT'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_15', + family='supersession', + setup_facts=['User lives in a downtown Toronto condo.', 'User is an avid scuba diver.'], + mutations=[('supersede', 'user residence', 'User moved to a rural cottage near Banff for a quieter life.')], + final_query='Where does the user currently reside?', + must_contain=['Banff', 'cottage'], + must_not_contain=['Toronto', 'downtown'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_16', + family='supersession', + setup_facts=["User's favorite book is 'War and Peace'.", 'User runs a popular cooking blog.'], + mutations=[('supersede', 'user literary preference', 'User now exclusively reads contemporary science fiction novels.')], + final_query='What kind of books does the user prefer now?', + must_contain=['science fiction', 'contemporary'], + must_not_contain=['War and Peace', 'favorite book'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_17', + family='supersession', + setup_facts=['User drives a Tesla Model 3.', 'User enjoys hiking in the Rockies.'], + mutations=[('supersede', 'user vehicle', 'User switched to a Polestar 2 after selling the electric sedan.')], + final_query='What car does the user drive?', + must_contain=['Polestar 2'], + must_not_contain=['Tesla', 'Model 3'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_18', + family='supersession', + setup_facts=['User is allergic to peanuts.', 'User volunteers at an animal shelter.'], + mutations=[('supersede', 'user allergies', 'User has outgrown the peanut allergy but developed a sensitivity to shellfish.')], + final_query='What is the user allergic to?', + must_contain=['shellfish'], + must_not_contain=['peanuts'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_20', + family='supersession', + setup_facts=['User works as a lead architect at Foster + Partners.', 'User plays the cello in a community orchestra.'], + mutations=[('supersede', 'user job', 'User left the architecture firm to start an urban farming collective.')], + final_query="What is the user's current occupation?", + must_contain=['urban farming', 'collective'], + must_not_contain=['architect', 'Foster + Partners'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_21', + family='supersession', + setup_facts=['User owns a golden retriever named Max.', 'User has a black belt in karate.'], + mutations=[('supersede', 'user pet', 'User now cares for a rescued greyhound after Max passed away.')], + final_query='What pet does the user have?', + must_contain=['greyhound'], + must_not_contain=['golden retriever', 'Max'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_22', + family='supersession', + setup_facts=['User is a registered Democrat.', 'User enjoys baking sourdough bread.'], + mutations=[('supersede', 'user political affiliation', 'User became an independent voter after growing disillusioned with party politics.')], + final_query="What is the user's political party?", + must_contain=['independent'], + must_not_contain=['Democrat'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_23', + family='supersession', + setup_facts=['User completed the Boston Marathon in 2022.', 'User writes poetry in their spare time.'], + mutations=[('supersede', 'user marathon participation', 'User switched to triathlons after knee surgery prevented marathon running.')], + final_query='Has the user run a marathon recently?', + must_contain=['triathlons', 'knee surgery'], + must_not_contain=['Boston Marathon', '2022'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_24', + family='supersession', + setup_facts=['User holds a PhD in quantum physics from MIT.', 'User enjoys scuba diving in the Caribbean.'], + mutations=[('supersede', 'user academic background', 'User transitioned from academia to professional poker, leveraging probability skills.')], + final_query="What is the user's current profession?", + must_contain=['poker', 'probability'], + must_not_contain=['quantum physics', 'MIT'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_25', + family='supersession', + setup_facts=['User is allergic to peanuts.', 'User practices yoga daily.'], + mutations=[('supersede', 'user allergies', 'User now tolerates trace peanut exposure after immunotherapy.')], + final_query='Is the user allergic to peanuts?', + must_contain=['tolerates', 'immunotherapy'], + must_not_contain=['allergic to peanuts'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_26', + family='supersession', + setup_facts=['User speaks fluent French.', 'User collects vintage vinyl records.'], + mutations=[('supersede', 'user languages', 'User has mostly forgotten French after years of disuse, now focusing on Mandarin.')], + final_query='What languages does the user speak?', + must_contain=['Mandarin'], + must_not_contain=['fluent French'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_27', + family='supersession', + setup_facts=['User is married to Alex.', 'User volunteers at an animal shelter.'], + mutations=[('supersede', 'user marital status', 'User and Alex divorced amicably last year.')], + final_query='Is the user married?', + must_contain=['divorced'], + must_not_contain=['married to Alex'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_28', + family='supersession', + setup_facts=['User owns a golden retriever named Max.', 'User works as a freelance graphic designer.'], + mutations=[('supersede', 'user pet', 'User adopted a rescue cat named Luna after Max passed away.')], + final_query='What pet does the user have?', + must_contain=['cat', 'Luna'], + must_not_contain=['golden retriever', 'Max'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_29', + family='supersession', + setup_facts=['User is a registered Democrat.', 'User plays the piano.'], + mutations=[('supersede', 'user political affiliation', 'User left the Democratic party and registered as an Independent.')], + final_query="What is the user's political party?", + must_contain=['Independent'], + must_not_contain=['registered Democrat'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_30', + family='supersession', + setup_facts=['User has a PhD in neuroscience.', 'User enjoys baking sourdough bread.'], + mutations=[('supersede', 'user highest degree', 'User abandoned academia and completed a coding bootcamp instead.')], + final_query="What is the user's highest education level?", + must_contain=['coding bootcamp'], + must_not_contain=['PhD', 'neuroscience'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_31', + family='supersession', + setup_facts=['User is a morning person.', 'User has a collection of rare coins.'], + mutations=[('supersede', 'user sleep habits', 'User started working night shifts and now sleeps during the day.')], + final_query='Is the user a morning person?', + must_contain=['night shifts', 'sleeps during the day'], + must_not_contain=['morning person'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_33', + family='supersession', + setup_facts=['User is allergic to peanuts and shellfish.', 'User volunteers at animal shelters.'], + mutations=[('supersede', 'user allergies', 'Recent medical tests show user only has sensitivity to tree nuts now.')], + final_query='What is the user allergic to?', + must_contain=['tree nuts'], + must_not_contain=['peanuts', 'shellfish'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_34', + family='supersession', + setup_facts=["User's favorite movie is Inception.", 'User plays the violin on weekends.'], + mutations=[('supersede', 'user favorite film', 'User now considers Everything Everywhere All At Once as their top cinematic experience.')], + final_query='What movie does the user like most?', + must_contain=['Everything Everywhere All At Once'], + must_not_contain=['Inception'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_35', + family='supersession', + setup_facts=['User completed a PhD in particle physics at MIT.', 'User enjoys rock climbing.'], + mutations=[('supersede', 'user education', 'User transitioned to neuroscience research after their physics work.')], + final_query="What is the user's current research field?", + must_contain=['neuroscience'], + must_not_contain=['particle physics', 'MIT'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_37', + family='supersession', + setup_facts=['User works as a senior architect at SOM.', 'User participates in triathlons.'], + mutations=[('supersede', 'user job', 'User left corporate architecture to teach design at RISD.')], + final_query="What is the user's current occupation?", + must_contain=['teach', 'RISD'], + must_not_contain=['architect', 'SOM'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_38', + family='supersession', + setup_facts=["User's childhood home was in Toronto.", 'User collects vintage vinyl records.'], + mutations=[('supersede', 'user hometown', 'User considers Vancouver their true home after living there for 15 years.')], + final_query='Where is the user from?', + must_contain=['Vancouver'], + must_not_contain=['Toronto', 'childhood'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_36', + family='supersession', + setup_facts=['User drives a Tesla Model 3.', 'User volunteers at animal shelters on weekends.'], + mutations=[('supersede', 'user vehicle', 'User switched to public transit and sold their electric car last month.')], + final_query='What does the user drive?', + must_contain=['public transit'], + must_not_contain=['Tesla', 'Model 3'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_39', + family='supersession', + setup_facts=['User works as a neurosurgeon at Mayo Clinic.', 'User collects vintage vinyl records.'], + mutations=[('supersede', 'user profession', 'User left medicine to become a philosophy professor.')], + final_query="What is the user's current job?", + must_contain=['philosophy professor'], + must_not_contain=['neurosurgeon', 'Mayo Clinic'], + ), + GeneratedCase( + id='adv_paraphrase_supersession_40', + family='supersession', + setup_facts=['User owns a golden retriever named Max.', 'User practices yoga daily.'], + mutations=[('supersede', 'user pet', 'User adopted a rescue cat after their dog passed away.')], + final_query='What pet does the user have?', + must_contain=['rescue cat'], + must_not_contain=['golden retriever', 'Max'], + ), +] + +ADV_NEGATION_TRAP_GENERATED = [ + GeneratedCase( + id='adv_negation_trap_09', + family='supersession', + setup_facts=['User does NOT own a car and has never driven one.', 'User prefers public transportation for daily commute.'], + mutations=[('supersede', 'user car ownership', 'User bought a Tesla Model 3 last week.')], + final_query='Does the user own a car?', + must_contain=['Tesla Model 3'], + must_not_contain=['does NOT own a car', 'never driven one'], + ), + GeneratedCase( + id='adv_negation_trap_10', + family='decay', + setup_facts=['User does not play any musical instruments.', 'User enjoys listening to classical music.'], + mutations=[('release', 'user does not play any musical instruments')], + final_query='Does the user play any instruments?', + must_contain=['classical music'], + must_not_contain=['does not play any musical instruments'], + ), + GeneratedCase( + id='adv_negation_trap_11', + family='supersession', + setup_facts=['User never eats spicy food due to medical reasons.', "User's favorite cuisine is Italian."], + mutations=[('supersede', 'user spicy food preference', 'User has recovered and now enjoys spicy Thai food.')], + final_query='Does the user eat spicy food?', + must_contain=['spicy Thai food'], + must_not_contain=['never eats spicy food', 'medical reasons'], + ), + GeneratedCase( + id='adv_negation_trap_12', + family='decay', + setup_facts=["User doesn't have a Netflix subscription.", 'User watches movies on Amazon Prime Video.'], + mutations=[('release', "user doesn't have a Netflix subscription")], + final_query='Where does the user watch movies?', + must_contain=['Amazon Prime Video'], + must_not_contain=["doesn't have a Netflix subscription"], + ), + GeneratedCase( + id='adv_negation_trap_13', + family='supersession', + setup_facts=['User no longer lives in New York after moving out last year.', "User's current city is San Francisco."], + mutations=[('supersede', 'user New York residency', 'User has returned to New York for a new job opportunity.')], + final_query='Where does the user live?', + must_contain=['New York', 'job opportunity'], + must_not_contain=['no longer lives in New York', 'San Francisco'], + ), + GeneratedCase( + id='adv_negation_trap_14', + family='decay', + setup_facts=['User refuses to use social media platforms.', 'User communicates primarily through email.'], + mutations=[('release', 'user refuses to use social media platforms')], + final_query='How does the user communicate?', + must_contain=['email'], + must_not_contain=['refuses to use social media', 'platforms'], + ), + GeneratedCase( + id='adv_negation_trap_15', + family='supersession', + setup_facts=['User does NOT work at Google and has declined all offers from them.', "User's current employer is Microsoft."], + mutations=[('supersede', 'user Google employment', 'User accepted a position at Google as a Senior Engineer.')], + final_query='Where does the user work?', + must_contain=['Google', 'Senior Engineer'], + must_not_contain=['does NOT work at Google', 'Microsoft'], + ), + GeneratedCase( + id='adv_negation_trap_16', + family='decay', + setup_facts=["User doesn't attend any conferences.", 'User prefers online webinars for professional development.'], + mutations=[('release', "user doesn't attend any conferences")], + final_query='How does the user pursue professional development?', + must_contain=['online webinars'], + must_not_contain=["doesn't attend any conferences"], + ), + GeneratedCase( + id='adv_negation_trap_17', + family='supersession', + setup_facts=["User does NOT own a car and hasn't driven in years.", 'User prefers to travel by bicycle.'], + mutations=[('supersede', 'user car ownership', 'User recently purchased a Tesla Model 3.')], + final_query='Does the user own a car?', + must_contain=['Tesla Model 3'], + must_not_contain=['does NOT own a car', "hasn't driven in years"], + ), + GeneratedCase( + id='adv_negation_trap_18', + family='decay', + setup_facts=['User never eats spicy food due to medical reasons.', "User's favorite cuisine is Italian."], + mutations=[('release', 'user never eats spicy food')], + final_query='What does the user eat?', + must_contain=['Italian'], + must_not_contain=['never eats spicy food'], + ), + GeneratedCase( + id='adv_negation_trap_19', + family='supersession', + setup_facts=['User does not speak French and has no plans to learn it.', 'User is fluent in Spanish.'], + mutations=[('supersede', 'user French language proficiency', 'User completed a French immersion program last summer.')], + final_query='Can the user speak French?', + must_contain=['French immersion program'], + must_not_contain=['does not speak French', 'no plans to learn'], + ), + GeneratedCase( + id='adv_negation_trap_20', + family='decay', + setup_facts=["User doesn't watch any sports and finds them boring.", 'User enjoys documentary films.'], + mutations=[('release', "user doesn't watch any sports")], + final_query='What does the user watch?', + must_contain=['documentary films'], + must_not_contain=["doesn't watch any sports"], + ), + GeneratedCase( + id='adv_negation_trap_21', + family='supersession', + setup_facts=["User has never visited Japan and doesn't plan to.", 'User has traveled extensively in Europe.'], + mutations=[('supersede', 'user Japan travel history', 'User spent two weeks in Tokyo last spring.')], + final_query='Has the user visited Japan?', + must_contain=['Tokyo'], + must_not_contain=['never visited Japan', "doesn't plan to"], + ), + GeneratedCase( + id='adv_negation_trap_22', + family='decay', + setup_facts=['User no longer plays the piano after selling it years ago.', 'User has recently taken up painting.'], + mutations=[('release', 'user no longer plays the piano')], + final_query='What hobbies does the user have?', + must_contain=['painting'], + must_not_contain=['no longer plays the piano'], + ), + GeneratedCase( + id='adv_negation_trap_23', + family='supersession', + setup_facts=["User doesn't own any smart home devices and refuses to use them.", 'User has an extensive vinyl record collection.'], + mutations=[('supersede', 'user smart home device ownership', 'User installed a Nest thermostat last winter.')], + final_query='Does the user have smart home devices?', + must_contain=['Nest thermostat'], + must_not_contain=["doesn't own any smart home devices", 'refuses to use them'], + ), + GeneratedCase( + id='adv_negation_trap_24', + family='decay', + setup_facts=['User never drinks alcohol at social events.', "User's favorite beverage is craft soda."], + mutations=[('release', 'user never drinks alcohol')], + final_query='What does the user drink?', + must_contain=['craft soda'], + must_not_contain=['never drinks alcohol'], + ), + GeneratedCase( + id='adv_negation_trap_25', + family='supersession', + setup_facts=["User does NOT own a car and hasn't driven in years.", 'User enjoys cycling to work daily.'], + mutations=[('supersede', 'user car ownership', 'User bought a Tesla Model 3 last week.')], + final_query='Does the user own a car?', + must_contain=['Tesla Model 3'], + must_not_contain=['does NOT own a car', "hasn't driven in years"], + ), + GeneratedCase( + id='adv_negation_trap_26', + family='decay', + setup_facts=['User never eats spicy food due to medical advice.', "User's favorite cuisine is Italian."], + mutations=[('release', 'user never eats spicy food')], + final_query="What are the user's dietary preferences?", + must_contain=['Italian'], + must_not_contain=['never eats spicy food', 'medical advice'], + ), + GeneratedCase( + id='adv_negation_trap_27', + family='supersession', + setup_facts=['User does not speak Spanish despite taking classes.', 'User is fluent in French.'], + mutations=[('supersede', 'user Spanish proficiency', 'User became fluent in Spanish after living in Barcelona.')], + final_query='Can the user speak Spanish?', + must_contain=['fluent in Spanish', 'Barcelona'], + must_not_contain=['does not speak Spanish', 'despite taking classes'], + ), + GeneratedCase( + id='adv_negation_trap_28', + family='decay', + setup_facts=['User no longer plays the piano after injuring their hand.', 'User recently started learning the guitar.'], + mutations=[('release', 'user no longer plays the piano')], + final_query='What instruments does the user play?', + must_contain=['guitar'], + must_not_contain=['no longer plays the piano', 'injuring their hand'], + ), + GeneratedCase( + id='adv_negation_trap_29', + family='supersession', + setup_facts=["User doesn't have any siblings according to official records.", 'User grew up in Chicago.'], + mutations=[('supersede', 'user sibling status', 'User discovered they have a half-sister through DNA testing.')], + final_query='Does the user have any siblings?', + must_contain=['half-sister', 'DNA testing'], + must_not_contain=["doesn't have any siblings", 'official records'], + ), + GeneratedCase( + id='adv_negation_trap_30', + family='decay', + setup_facts=['User refuses to use social media platforms.', 'User maintains a professional blog about architecture.'], + mutations=[('release', 'user refuses to use social media')], + final_query='How does the user engage online?', + must_contain=['professional blog', 'architecture'], + must_not_contain=['refuses to use social media'], + ), + GeneratedCase( + id='adv_negation_trap_31', + family='supersession', + setup_facts=['User has never visited Asia despite many opportunities.', 'User frequently travels to Europe for work.'], + mutations=[('supersede', 'user Asia travel history', 'User spent two months backpacking across Southeast Asia last year.')], + final_query='Has the user visited Asia?', + must_contain=['Southeast Asia', 'backpacking'], + must_not_contain=['never visited Asia', 'despite many opportunities'], + ), + GeneratedCase( + id='adv_negation_trap_32', + family='decay', + setup_facts=['User does not watch television and sold their TV years ago.', 'User subscribes to several educational podcasts.'], + mutations=[('release', 'user does not watch television')], + final_query='What media does the user consume?', + must_contain=['educational podcasts'], + must_not_contain=['does not watch television', 'sold their TV'], + ), + GeneratedCase( + id='adv_negation_trap_33', + family='supersession', + setup_facts=['User does NOT own a car and has never driven one.', 'User prefers to travel by bicycle.'], + mutations=[('supersede', 'user car ownership', 'User bought a Tesla Model 3 last week.')], + final_query='Does the user own a car?', + must_contain=['Tesla Model 3'], + must_not_contain=['does NOT own a car', 'never driven one'], + ), + GeneratedCase( + id='adv_negation_trap_34', + family='decay', + setup_facts=['User never eats spicy food due to medical reasons.', 'User enjoys Italian cuisine.'], + mutations=[('release', 'user never eats spicy food')], + final_query="What are the user's food preferences?", + must_contain=['Italian cuisine'], + must_not_contain=['never eats spicy food', 'medical reasons'], + ), + GeneratedCase( + id='adv_negation_trap_35', + family='supersession', + setup_facts=['User does not play any musical instruments.', 'User has perfect pitch.'], + mutations=[('supersede', 'user musical ability', 'User has been playing piano for 3 years.')], + final_query='Does the user play any musical instruments?', + must_contain=['playing piano'], + must_not_contain=['does not play any musical instruments'], + ), + GeneratedCase( + id='adv_negation_trap_36', + family='decay', + setup_facts=['User no longer lives in New York as of 2022.', 'User works as a software engineer.'], + mutations=[('release', 'user no longer lives in New York')], + final_query='Where does the user live?', + must_contain=['software engineer'], + must_not_contain=['no longer lives in New York', 'as of 2022'], + ), + GeneratedCase( + id='adv_negation_trap_37', + family='supersession', + setup_facts=["User doesn't speak French despite taking classes.", 'User is fluent in Spanish.'], + mutations=[('supersede', 'user French language ability', 'User became fluent in French after immersion program.')], + final_query='Does the user speak French?', + must_contain=['fluent in French', 'immersion program'], + must_not_contain=["doesn't speak French", 'despite taking classes'], + ), + GeneratedCase( + id='adv_negation_trap_38', + family='decay', + setup_facts=['User never wears glasses according to medical records.', 'User has 20/20 vision.'], + mutations=[('release', 'user never wears glasses')], + final_query='Does the user wear glasses?', + must_contain=['20/20 vision'], + must_not_contain=['never wears glasses', 'medical records'], + ), + GeneratedCase( + id='adv_negation_trap_39', + family='supersession', + setup_facts=['User has never visited Japan.', 'User collects rare stamps.'], + mutations=[('supersede', 'user travel to Japan', 'User spent two weeks in Tokyo last summer.')], + final_query='Has the user visited Japan?', + must_contain=['two weeks in Tokyo'], + must_not_contain=['has never visited Japan'], + ), + GeneratedCase( + id='adv_negation_trap_40', + family='decay', + setup_facts=['User refuses to use social media platforms.', 'User writes a personal blog.'], + mutations=[('release', 'user refuses to use social media')], + final_query='Does the user use social media?', + must_contain=['personal blog'], + must_not_contain=['refuses to use social media'], + ), +] + +ADV_TEMPORAL_QUALIFIER_GENERATED = [ + GeneratedCase( + id='adv_temporal_qualifier_09', + family='supersession', + setup_facts=['Alice worked at Microsoft from 2018 to 2020.', 'Alice enjoys hiking in the Rockies.'], + mutations=[('supersede', 'Alice employer Microsoft', 'Alice joined Apple in January 2021.')], + final_query='Where does Alice currently work?', + must_contain=['Apple', 'January 2021'], + must_not_contain=['worked at Microsoft'], + ), + GeneratedCase( + id='adv_temporal_qualifier_10', + family='drift', + setup_facts=['Bob studied at Harvard from 2010 to 2014.', "Bob's favorite color is blue."], + mutations=[('supersede', 'Bob education Harvard', 'Bob earned a PhD from MIT between 2014 and 2020.'), ('supersede', 'Bob education MIT', 'Bob completed postdoc research at Stanford starting 2020.')], + final_query="What is Bob's most recent educational affiliation?", + must_contain=['Stanford', 'starting 2020'], + must_not_contain=['studied at Harvard', 'MIT between 2014'], + ), + GeneratedCase( + id='adv_temporal_qualifier_11', + family='supersession', + setup_facts=['Charlie owned a Tesla Model 3 in 2020.', "Charlie's favorite food is sushi."], + mutations=[('supersede', 'Charlie car Tesla', 'Charlie switched to a Rivian truck in 2023.')], + final_query='What car does Charlie drive now?', + must_contain=['Rivian', '2023'], + must_not_contain=['owned a Tesla'], + ), + GeneratedCase( + id='adv_temporal_qualifier_12', + family='drift', + setup_facts=['Diana was CEO of Acme Corp from 2015 to 2019.', 'Diana has two children.'], + mutations=[('supersede', 'Diana position Acme', 'Diana became CTO of Beta Inc from 2019 to 2022.'), ('supersede', 'Diana position Beta', 'Diana founded Gamma Ventures in 2022.')], + final_query="What is Diana's current professional role?", + must_contain=['Gamma Ventures', '2022'], + must_not_contain=['Acme Corp', 'Beta Inc'], + ), + GeneratedCase( + id='adv_temporal_qualifier_13', + family='supersession', + setup_facts=['Eve lived in Paris during 2017.', 'Eve is allergic to peanuts.'], + mutations=[('supersede', 'Eve residence Paris', 'Eve moved to Barcelona in 2019.'), ('supersede', 'Eve residence Barcelona', 'Eve relocated to Singapore in 2023.')], + final_query='Where does Eve currently reside?', + must_contain=['Singapore', '2023'], + must_not_contain=['Paris', 'Barcelona'], + ), + GeneratedCase( + id='adv_temporal_qualifier_14', + family='supersession', + setup_facts=['Frank was diagnosed with Type 2 diabetes in 2015.', 'Frank plays the piano.'], + mutations=[('supersede', 'Frank diagnosis 2015', "Frank's diabetes went into remission in 2021.")], + final_query="What is Frank's current health status regarding diabetes?", + must_contain=['remission', '2021'], + must_not_contain=['diagnosed with Type 2 diabetes'], + ), + GeneratedCase( + id='adv_temporal_qualifier_15', + family='drift', + setup_facts=['Grace published her first novel in 2010.', 'Grace has a golden retriever named Max.'], + mutations=[('supersede', 'Grace publications first novel', 'Grace released her second book in 2015.'), ('supersede', 'Grace publications second book', "Grace's third novel became a bestseller in 2021.")], + final_query="What is Grace's most recent publishing achievement?", + must_contain=['third novel', 'bestseller in 2021'], + must_not_contain=['first novel in 2010', 'second book in 2015'], + ), + GeneratedCase( + id='adv_temporal_qualifier_16', + family='supersession', + setup_facts=['Henry was married to Sarah from 2005 to 2019.', 'Henry collects vintage watches.'], + mutations=[('supersede', 'Henry marriage Sarah', 'Henry remarried to Emily in 2022.')], + final_query='Who is Henry currently married to?', + must_contain=['Emily', '2022'], + must_not_contain=['married to Sarah'], + ), + GeneratedCase( + id='adv_temporal_qualifier_17', + family='supersession', + setup_facts=['Alice worked at Microsoft from 2017 to 2021 as a developer.', 'Alice enjoys hiking in the mountains.'], + mutations=[('supersede', 'Alice employer Microsoft', 'Alice joined Amazon in January 2022 as a senior developer.')], + final_query='Where does Alice currently work?', + must_contain=['Amazon', 'January 2022'], + must_not_contain=['Microsoft from 2017'], + ), + GeneratedCase( + id='adv_temporal_qualifier_18', + family='drift', + setup_facts=['Bob attended Stanford University from 2010 to 2014.', "Bob's favorite sport is basketball."], + mutations=[('supersede', 'Bob education Stanford 2010', "Bob pursued a Master's at MIT from 2014 to 2016."), ('supersede', 'Bob education MIT 2014', 'Bob earned a PhD from Cambridge University in 2019.')], + final_query="What is Bob's highest degree and where?", + must_contain=['PhD', 'Cambridge University', '2019'], + must_not_contain=['Stanford University', 'MIT from 2014'], + ), + GeneratedCase( + id='adv_temporal_qualifier_19', + family='supersession', + setup_facts=['Carol worked at Tesla from 2015 to 2018.', 'Carol has a pet cat named Whiskers.'], + mutations=[('supersede', 'Carol employer Tesla', 'Carol joined SpaceX in 2019 as a lead engineer.')], + final_query='Where does Carol work now?', + must_contain=['SpaceX', '2019'], + must_not_contain=['Tesla from 2015'], + ), + GeneratedCase( + id='adv_temporal_qualifier_20', + family='drift', + setup_facts=['Dave lived in Paris from 2012 to 2015.', 'Dave plays the guitar as a hobby.'], + mutations=[('supersede', 'Dave residence Paris 2012', 'Dave moved to London in 2016.'), ('supersede', 'Dave residence London 2016', 'Dave relocated to Singapore in 2020.')], + final_query='Where does Dave currently live?', + must_contain=['Singapore', '2020'], + must_not_contain=['Paris from 2012', 'London in 2016'], + ), + GeneratedCase( + id='adv_temporal_qualifier_22', + family='drift', + setup_facts=['Frank owned a Toyota Prius from 2015 to 2018.', 'Frank enjoys brewing his own coffee.'], + mutations=[('supersede', 'Frank car Toyota Prius', 'Frank bought a Tesla Model 3 in 2019.'), ('supersede', 'Frank car Tesla Model 3', 'Frank upgraded to a Porsche Taycan in 2022.')], + final_query='What car does Frank currently drive?', + must_contain=['Porsche Taycan', '2022'], + must_not_contain=['Toyota Prius', 'Tesla Model 3'], + ), + GeneratedCase( + id='adv_temporal_qualifier_23', + family='supersession', + setup_facts=['Grace worked at Netflix from 2016 to 2020.', 'Grace is an amateur photographer.'], + mutations=[('supersede', 'Grace employer Netflix', 'Grace joined Disney+ in 2021 as a content strategist.')], + final_query='Where does Grace work now?', + must_contain=['Disney+', '2021'], + must_not_contain=['Netflix from 2016'], + ), + GeneratedCase( + id='adv_temporal_qualifier_24', + family='drift', + setup_facts=['Henry lived in Sydney from 2013 to 2016.', 'Henry collects vintage vinyl records.'], + mutations=[('supersede', 'Henry residence Sydney 2013', 'Henry moved to Melbourne in 2017.'), ('supersede', 'Henry residence Melbourne 2017', 'Henry relocated to Dubai in 2021.')], + final_query='Where does Henry currently reside?', + must_contain=['Dubai', '2021'], + must_not_contain=['Sydney from 2013', 'Melbourne in 2017'], + ), + GeneratedCase( + id='adv_temporal_qualifier_25', + family='drift', + setup_facts=['Bob attended Stanford University from 2016 to 2020.', "Bob's favorite food is sushi."], + mutations=[('supersede', 'Bob education Stanford 2016', "Bob pursued a master's at MIT from 2020 to 2022.")], + final_query="What is Bob's most recent education?", + must_contain=['MIT', '2020 to 2022'], + must_not_contain=['Stanford University from 2016'], + ), + GeneratedCase( + id='adv_temporal_qualifier_26', + family='supersession', + setup_facts=['Charlie owned a Tesla Model 3 since January 2020.', 'Charlie plays guitar in a local band.'], + mutations=[('supersede', 'Charlie car Tesla Model 3', 'Charlie switched to a Rivian R1T in March 2023.')], + final_query='What car does Charlie drive now?', + must_contain=['Rivian R1T', 'March 2023'], + must_not_contain=['Tesla Model 3'], + ), + GeneratedCase( + id='adv_temporal_qualifier_27', + family='drift', + setup_facts=['Diana lived in Paris from 2018 to 2021.', 'Diana speaks fluent French.'], + mutations=[('supersede', 'Diana residence Paris 2018', 'Diana moved to Barcelona in 2021.'), ('supersede', 'Diana residence Barcelona 2021', 'Diana relocated to Singapore in 2023.')], + final_query='Where does Diana currently live?', + must_contain=['Singapore', '2023'], + must_not_contain=['Paris', 'Barcelona'], + ), + GeneratedCase( + id='adv_temporal_qualifier_28', + family='supersession', + setup_facts=['Emma used Android phones exclusively until 2022.', 'Emma collects vintage postcards.'], + mutations=[('supersede', 'Emma phone preference Android', 'Emma switched to iPhone in 2022.')], + final_query='What phone does Emma use now?', + must_contain=['iPhone', '2022'], + must_not_contain=['Android'], + ), + GeneratedCase( + id='adv_temporal_qualifier_29', + family='drift', + setup_facts=['Frank was CEO of IBM from 2010 to 2015.', 'Frank enjoys sailing.'], + mutations=[('supersede', 'Frank position IBM CEO', 'Frank became CEO of SpaceX in 2015.'), ('supersede', 'Frank position SpaceX CEO', 'Frank stepped down and joined Tesla in 2020.')], + final_query="What was Frank's most recent executive position?", + must_contain=['Tesla', '2020'], + must_not_contain=['IBM', 'SpaceX'], + ), + GeneratedCase( + id='adv_temporal_qualifier_30', + family='supersession', + setup_facts=['Grace worked as a journalist for CNN in 2018.', 'Grace has a black belt in karate.'], + mutations=[('supersede', 'Grace employment CNN', 'Grace joined The Washington Post in 2020.')], + final_query='Where does Grace currently work?', + must_contain=['The Washington Post', '2020'], + must_not_contain=['CNN in 2018'], + ), + GeneratedCase( + id='adv_temporal_qualifier_31', + family='drift', + setup_facts=['Henry was a consultant at McKinsey from 2017 to 2019.', 'Henry volunteers at animal shelters.'], + mutations=[('supersede', 'Henry job McKinsey consultant', 'Henry joined Bain & Company in 2019.'), ('supersede', 'Henry job Bain consultant', 'Henry moved to Boston Consulting Group in 2021.')], + final_query='Where does Henry currently work?', + must_contain=['Boston Consulting Group', '2021'], + must_not_contain=['McKinsey', 'Bain'], + ), + GeneratedCase( + id='adv_temporal_qualifier_33', + family='supersession', + setup_facts=['Charlie owned a Tesla Model 3 in 2020.', 'Charlie enjoys hiking on weekends.'], + mutations=[('supersede', 'Charlie car Tesla', 'Charlie upgraded to a Rivian R1T in 2023.')], + final_query='What car does Charlie currently own?', + must_contain=['Rivian R1T', '2023'], + must_not_contain=['owned a Tesla Model 3 in 2020'], + ), + GeneratedCase( + id='adv_temporal_qualifier_34', + family='drift', + setup_facts=['Diana lived in Paris from 2016 to 2018.', 'Diana speaks three languages fluently.'], + mutations=[('supersede', 'Diana residence Paris', 'Diana moved to Singapore in 2018.'), ('supersede', 'Diana residence Singapore', 'Diana relocated to Sydney in 2022.')], + final_query='Where does Diana currently live?', + must_contain=['Sydney', '2022'], + must_not_contain=['lived in Paris', 'moved to Singapore'], + ), + GeneratedCase( + id='adv_temporal_qualifier_36', + family='drift', + setup_facts=['Frank was CEO of StartupX from 2014 to 2017.', 'Frank is an avid scuba diver.'], + mutations=[('supersede', 'Frank position StartupX', 'Frank became VP at TechCorp from 2017 to 2020.'), ('supersede', 'Frank position TechCorp', 'Frank founded his own company, NewVentures, in 2020.')], + final_query="What is Frank's current professional role?", + must_contain=['founded his own company', 'NewVentures', '2020'], + must_not_contain=['was CEO of StartupX', 'VP at TechCorp'], + ), + GeneratedCase( + id='adv_temporal_qualifier_37', + family='supersession', + setup_facts=['Grace lived in Toronto in 2018.', 'Grace has a collection of rare books.'], + mutations=[('supersede', 'Grace residence Toronto', 'Grace moved to Vancouver in 2021.')], + final_query='Where does Grace currently live?', + must_contain=['Vancouver', '2021'], + must_not_contain=['lived in Toronto in 2018'], + ), + GeneratedCase( + id='adv_temporal_qualifier_38', + family='drift', + setup_facts=['Henry was a professor at Harvard in 2015.', 'Henry plays the violin professionally.'], + mutations=[('supersede', 'Henry position Harvard', 'Henry joined Yale as department chair in 2018.'), ('supersede', 'Henry position Yale', 'Henry became dean at Princeton starting 2022.')], + final_query="What is Henry's current academic position?", + must_contain=['dean at Princeton', '2022'], + must_not_contain=['was a professor at Harvard', 'joined Yale as department chair'], + ), + GeneratedCase( + id='adv_temporal_qualifier_39', + family='drift', + setup_facts=['Professor Smith taught at Harvard from 2005 to 2010.', 'Professor Smith specializes in quantum computing.'], + mutations=[('supersede', 'Professor Smith Harvard', 'Professor Smith moved to MIT in 2012.'), ('supersede', 'Professor Smith MIT', 'Professor Smith joined Stanford University in 2020.')], + final_query='Where does Professor Smith currently teach?', + must_contain=['Stanford University', '2020'], + must_not_contain=['Harvard from 2005', 'MIT in 2012'], + ), + GeneratedCase( + id='adv_temporal_qualifier_40', + family='supersession', + setup_facts=['The national team coach was Mr. Johnson from 2016 to 2018.', "The team's colors are blue and white."], + mutations=[('supersede', 'national team coach Mr. Johnson', 'Ms. Rodriguez became the new national team coach in 2021.')], + final_query='Who is the current national team coach?', + must_contain=['Ms. Rodriguez', '2021'], + must_not_contain=['Mr. Johnson from 2016'], + ), +] + +ADV_SHARED_ATTRIBUTE_GENERATED = [ + GeneratedCase( + id='adv_shared_attribute_17', + family='amnesia', + setup_facts=['Professor Smith teaches Calculus and Linear Algebra.', 'Professor Jones teaches Calculus and Differential Equations.', 'Professor Brown teaches Statistics.'], + mutations=[('release', 'everything about Professor Smith')], + final_query='Who teaches Calculus?', + must_contain=['Professor Jones'], + must_not_contain=['Professor Smith teaches Calculus'], + ), + GeneratedCase( + id='adv_shared_attribute_18', + family='amnesia', + setup_facts=['The Empire Hotel has 5 stars and a rooftop pool.', 'The Grand Plaza has 5 stars and a spa.', 'The Seaview Inn has 3 stars.'], + mutations=[('release', 'everything about The Empire Hotel')], + final_query='Which hotels have 5 stars?', + must_contain=['The Grand Plaza'], + must_not_contain=['The Empire Hotel has 5 stars'], + ), + GeneratedCase( + id='adv_shared_attribute_19', + family='amnesia', + setup_facts=['Dr. Patel specializes in cardiology and internal medicine.', 'Dr. Nguyen specializes in cardiology and pediatrics.', 'Dr. Wilson specializes in neurology.'], + mutations=[('release', 'everything about Dr. Patel')], + final_query='Which doctors specialize in cardiology?', + must_contain=['Dr. Nguyen'], + must_not_contain=['Dr. Patel specializes in cardiology'], + ), + GeneratedCase( + id='adv_shared_attribute_20', + family='amnesia', + setup_facts=['Restaurant Bella serves Italian and French cuisine.', 'Restaurant Luna serves Italian and Spanish cuisine.', 'Restaurant Orion serves Japanese cuisine.'], + mutations=[('release', 'everything about Restaurant Bella')], + final_query='Which restaurants serve Italian cuisine?', + must_contain=['Restaurant Luna'], + must_not_contain=['Restaurant Bella serves Italian'], + ), + GeneratedCase( + id='adv_shared_attribute_21', + family='amnesia', + setup_facts=['Book A is categorized as Mystery and Thriller.', 'Book B is categorized as Mystery and Romance.', 'Book C is categorized as Science Fiction.'], + mutations=[('release', 'everything about Book A')], + final_query='Which books are in the Mystery category?', + must_contain=['Book B'], + must_not_contain=['Book A is categorized as Mystery'], + ), + GeneratedCase( + id='adv_shared_attribute_22', + family='amnesia', + setup_facts=['Employee X works in Marketing and Sales.', 'Employee Y works in Marketing and HR.', 'Employee Z works in Engineering.'], + mutations=[('release', 'everything about Employee X')], + final_query='Who works in Marketing?', + must_contain=['Employee Y'], + must_not_contain=['Employee X works in Marketing'], + ), + GeneratedCase( + id='adv_shared_attribute_23', + family='amnesia', + setup_facts=['Team Alpha uses Java and Python.', 'Team Beta uses Java and C++.', 'Team Gamma uses JavaScript.'], + mutations=[('release', 'everything about Team Alpha')], + final_query='Which teams use Java?', + must_contain=['Team Beta'], + must_not_contain=['Team Alpha uses Java'], + ), + GeneratedCase( + id='adv_shared_attribute_24', + family='amnesia', + setup_facts=['Actor Davis has won Oscars and Emmys.', 'Actor Evans has won Oscars and Tonys.', 'Actor Foster has won Grammys.'], + mutations=[('release', 'everything about Actor Davis')], + final_query='Which actors have won Oscars?', + must_contain=['Actor Evans'], + must_not_contain=['Actor Davis has won Oscars'], + ), + GeneratedCase( + id='adv_shared_attribute_25', + family='amnesia', + setup_facts=['Professor Smith teaches Physics 101 at Harvard.', 'Professor Johnson teaches Physics 101 at MIT.', 'Professor Lee teaches Chemistry 201 at Stanford.'], + mutations=[('release', 'everything about Professor Smith')], + final_query='Who teaches Physics 101?', + must_contain=['Johnson', 'Physics 101'], + must_not_contain=['Smith teaches Physics 101'], + ), + GeneratedCase( + id='adv_shared_attribute_26', + family='amnesia', + setup_facts=['The red car belongs to Lisa.', 'The red truck belongs to Mark.', 'The blue van belongs to Sarah.'], + mutations=[('release', 'everything about Lisa')], + final_query='Which vehicles are red?', + must_contain=['red truck', 'Mark'], + must_not_contain=['red car belongs to Lisa'], + ), + GeneratedCase( + id='adv_shared_attribute_27', + family='amnesia', + setup_facts=['Building A uses solar panels for energy.', 'Building B uses solar panels and wind turbines for energy.', 'Building C uses only geothermal energy.'], + mutations=[('release', 'everything about Building A')], + final_query='Which buildings use solar panels?', + must_contain=['Building B', 'solar panels'], + must_not_contain=['Building A uses solar panels'], + ), + GeneratedCase( + id='adv_shared_attribute_28', + family='amnesia', + setup_facts=['Patient X has allergy to penicillin and peanuts.', 'Patient Y has allergy to penicillin and shellfish.', 'Patient Z has no known allergies.'], + mutations=[('release', 'everything about Patient X')], + final_query='Which patients are allergic to penicillin?', + must_contain=['Patient Y', 'penicillin'], + must_not_contain=['Patient X has allergy to penicillin'], + ), + GeneratedCase( + id='adv_shared_attribute_29', + family='amnesia', + setup_facts=['Team Alpha uses Python and Java for development.', 'Team Beta uses Python and C++ for development.', 'Team Gamma uses only Rust.'], + mutations=[('release', 'everything about Team Alpha')], + final_query='Which teams use Python?', + must_contain=['Team Beta', 'Python'], + must_not_contain=['Team Alpha uses Python'], + ), + GeneratedCase( + id='adv_shared_attribute_30', + family='amnesia', + setup_facts=['Flight 123 departs from Gate A5.', 'Flight 456 departs from Gate A5 and Gate B2.', 'Flight 789 departs from Gate C1.'], + mutations=[('release', 'everything about Flight 123')], + final_query='Which flights depart from Gate A5?', + must_contain=['Flight 456', 'Gate A5'], + must_not_contain=['Flight 123 departs from Gate A5'], + ), + GeneratedCase( + id='adv_shared_attribute_31', + family='amnesia', + setup_facts=['User1 has access to Database A and Database B.', 'User2 has access to Database A and Database C.', 'User3 has access only to Database D.'], + mutations=[('release', 'everything about User1')], + final_query='Which users have access to Database A?', + must_contain=['User2', 'Database A'], + must_not_contain=['User1 has access to Database A'], + ), + GeneratedCase( + id='adv_shared_attribute_32', + family='amnesia', + setup_facts=['Restaurant X serves Italian and French cuisine.', 'Restaurant Y serves Italian and Spanish cuisine.', 'Restaurant Z serves only Japanese cuisine.'], + mutations=[('release', 'everything about Restaurant X')], + final_query='Which restaurants serve Italian cuisine?', + must_contain=['Restaurant Y', 'Italian'], + must_not_contain=['Restaurant X serves Italian cuisine'], + ), + GeneratedCase( + id='adv_shared_attribute_33', + family='amnesia', + setup_facts=['Liam speaks French and German.', 'Emma speaks French and Spanish.', 'Noah speaks Italian.'], + mutations=[('release', 'everything about Liam')], + final_query='Who speaks French?', + must_contain=['Emma'], + must_not_contain=['Liam speaks French'], + ), + GeneratedCase( + id='adv_shared_attribute_34', + family='amnesia', + setup_facts=['Tesla Model 3 has autopilot and 300-mile range.', 'Ford Mustang Mach-E has autopilot and 250-mile range.', 'Toyota Corolla has adaptive cruise control.'], + mutations=[('release', 'everything about Tesla Model 3')], + final_query='Which cars have autopilot?', + must_contain=['Ford Mustang Mach-E'], + must_not_contain=['Tesla Model 3 has autopilot'], + ), + GeneratedCase( + id='adv_shared_attribute_35', + family='amnesia', + setup_facts=['Professor Chen teaches Machine Learning and Statistics.', 'Professor Lee teaches Machine Learning and Databases.', 'Professor Smith teaches Operating Systems.'], + mutations=[('release', 'everything about Professor Chen')], + final_query='Who teaches Machine Learning?', + must_contain=['Professor Lee'], + must_not_contain=['Professor Chen teaches Machine Learning'], + ), + GeneratedCase( + id='adv_shared_attribute_36', + family='amnesia', + setup_facts=['Singapore Airlines flies to London and Singapore.', 'Qantas flies to London and Sydney.', 'Delta flies to Atlanta.'], + mutations=[('release', 'everything about Singapore Airlines')], + final_query='Which airlines fly to London?', + must_contain=['Qantas'], + must_not_contain=['Singapore Airlines flies to London'], + ), + GeneratedCase( + id='adv_shared_attribute_37', + family='amnesia', + setup_facts=['Beethoven composed Symphony No. 5 and Moonlight Sonata.', 'Mozart composed Symphony No. 5 and Eine kleine Nachtmusik.', 'Chopin composed Nocturnes.'], + mutations=[('release', 'everything about Beethoven')], + final_query='Who composed Symphony No. 5?', + must_contain=['Mozart'], + must_not_contain=['Beethoven composed Symphony No. 5'], + ), + GeneratedCase( + id='adv_shared_attribute_38', + family='amnesia', + setup_facts=['Python supports list comprehensions and decorators.', 'JavaScript supports list comprehensions and arrow functions.', 'Java supports generics.'], + mutations=[('release', 'everything about Python')], + final_query='Which languages support list comprehensions?', + must_contain=['JavaScript'], + must_not_contain=['Python supports list comprehensions'], + ), + GeneratedCase( + id='adv_shared_attribute_39', + family='amnesia', + setup_facts=['Microsoft Azure offers Kubernetes and CosmosDB.', 'Google Cloud offers Kubernetes and BigQuery.', 'AWS offers Lambda.'], + mutations=[('release', 'everything about Microsoft Azure')], + final_query='Which clouds offer Kubernetes?', + must_contain=['Google Cloud'], + must_not_contain=['Microsoft Azure offers Kubernetes'], + ), + GeneratedCase( + id='adv_shared_attribute_40', + family='amnesia', + setup_facts=['The Louvre displays the Mona Lisa and Venus de Milo.', 'The Uffizi displays the Mona Lisa and The Birth of Venus.', 'The Met displays Washington Crossing the Delaware.'], + mutations=[('release', 'everything about The Louvre')], + final_query='Which museums display the Mona Lisa?', + must_contain=['The Uffizi'], + must_not_contain=['The Louvre displays the Mona Lisa'], + ), +] + +ADV_COMPOUND_FACT_GENERATED = [ + GeneratedCase( + id='adv_compound_fact_09', + family='supersession', + setup_facts=['User owns a golden retriever named Max and drives a Tesla Model 3.', 'User volunteers at a local animal shelter on weekends.'], + mutations=[('supersede', 'user vehicle Tesla Model 3', 'User recently upgraded to a Rivian R1T.')], + final_query="What is the name of the user's pet?", + must_contain=['Max'], + must_not_contain=['Tesla Model 3'], + ), + GeneratedCase( + id='adv_compound_fact_10', + family='supersession', + setup_facts=['User is allergic to peanuts and plays the violin professionally.', 'User has a collection of rare vinyl records.'], + mutations=[('supersede', 'user allergy peanuts', 'User outgrew their peanut allergy but developed a shellfish allergy.')], + final_query='What instrument does the user play?', + must_contain=['violin'], + must_not_contain=['peanuts'], + ), + GeneratedCase( + id='adv_compound_fact_11', + family='supersession', + setup_facts=['User holds a black belt in judo and teaches mathematics at Stanford.', 'User writes science fiction novels in their spare time.'], + mutations=[('supersede', 'user employment Stanford', 'User accepted a professorship at MIT this semester.')], + final_query='What martial art does the user practice?', + must_contain=['judo'], + must_not_contain=['Stanford'], + ), + GeneratedCase( + id='adv_compound_fact_12', + family='supersession', + setup_facts=['User is originally from Toronto and supports the Green Party.', 'User runs an urban gardening blog with 50k followers.'], + mutations=[('supersede', 'user political affiliation Green Party', 'User switched allegiance to the Socialist Party last election.')], + final_query='Where is the user originally from?', + must_contain=['Toronto'], + must_not_contain=['Green Party'], + ), + GeneratedCase( + id='adv_compound_fact_13', + family='supersession', + setup_facts=['User has completed three triathlons and works as a neurosurgeon.', 'User collects antique medical instruments.'], + mutations=[('supersede', 'user athletic achievements triathlons', 'User has shifted focus to ultramarathons, completing their first 100-miler.')], + final_query="What is the user's profession?", + must_contain=['neurosurgeon'], + must_not_contain=['triathlons'], + ), + GeneratedCase( + id='adv_compound_fact_14', + family='supersession', + setup_facts=['User speaks Mandarin fluently and invests in cryptocurrency.', 'User maintains a zero-waste lifestyle.'], + mutations=[('supersede', 'user cryptocurrency investments', 'User liquidated all crypto holdings after the FTX collapse.')], + final_query='What languages does the user speak?', + must_contain=['Mandarin'], + must_not_contain=['cryptocurrency'], + ), + GeneratedCase( + id='adv_compound_fact_15', + family='supersession', + setup_facts=['User is a licensed pilot and enjoys baking sourdough bread.', 'User has traveled to over 60 countries.'], + mutations=[('supersede', 'user aviation license', "User let their pilot's license expire last year.")], + final_query='What does the user enjoy baking?', + must_contain=['sourdough bread'], + must_not_contain=['licensed pilot'], + ), + GeneratedCase( + id='adv_compound_fact_16', + family='supersession', + setup_facts=['User has a PhD in quantum physics and volunteers as a firefighter.', 'User maintains an apiary with 20 honeybee hives.'], + mutations=[('supersede', 'user firefighting volunteer work', 'User stepped back from firefighting due to back injuries.')], + final_query="What is the user's academic background?", + must_contain=['quantum physics'], + must_not_contain=['firefighter'], + ), + GeneratedCase( + id='adv_compound_fact_17', + family='supersession', + setup_facts=['User owns a golden retriever and drives a Tesla Model 3.', 'User volunteers at a local animal shelter on weekends.'], + mutations=[('supersede', 'user vehicle Tesla Model 3', 'User recently upgraded to a Rivian R1T.')], + final_query='What pet does the user have?', + must_contain=['golden retriever'], + must_not_contain=['Tesla Model 3'], + ), + GeneratedCase( + id='adv_compound_fact_18', + family='supersession', + setup_facts=['User plays chess competitively and has a PhD in neuroscience.', 'User collects vintage board games.'], + mutations=[('supersede', 'user chess activity', 'User now focuses on Go as their primary strategy game.')], + final_query="What is the user's highest academic degree?", + must_contain=['PhD in neuroscience'], + must_not_contain=['plays chess'], + ), + GeneratedCase( + id='adv_compound_fact_19', + family='supersession', + setup_facts=['User is allergic to peanuts and works as a firefighter.', 'User participates in triathlons annually.'], + mutations=[('supersede', 'user allergy peanuts', 'User has outgrown their peanut allergy through immunotherapy.')], + final_query="What is the user's profession?", + must_contain=['firefighter'], + must_not_contain=['allergic to peanuts'], + ), + GeneratedCase( + id='adv_compound_fact_20', + family='supersession', + setup_facts=['User writes poetry in their spare time and is fluent in Japanese.', 'User has visited every continent except Antarctica.'], + mutations=[('supersede', 'user poetic activity', 'User has transitioned to writing short stories instead of poetry.')], + final_query='What languages does the user speak fluently?', + must_contain=['Japanese'], + must_not_contain=['writes poetry'], + ), + GeneratedCase( + id='adv_compound_fact_21', + family='supersession', + setup_facts=['User practices archery competitively and owns a coffee shop.', 'User has a collection of rare coffee beans from around the world.'], + mutations=[('supersede', 'user archery activity', 'User has switched to competitive shooting sports.')], + final_query='What business does the user own?', + must_contain=['coffee shop'], + must_not_contain=['practices archery'], + ), + GeneratedCase( + id='adv_compound_fact_22', + family='supersession', + setup_facts=['User is a licensed pilot and grows heirloom tomatoes.', 'User built their own greenhouse last summer.'], + mutations=[('supersede', 'user pilot license status', 'User let their pilot license expire due to time constraints.')], + final_query='What does the user grow?', + must_contain=['heirloom tomatoes'], + must_not_contain=['licensed pilot'], + ), + GeneratedCase( + id='adv_compound_fact_23', + family='supersession', + setup_facts=['User plays the cello professionally and lives in a converted warehouse.', 'User has perfect pitch.'], + mutations=[('supersede', 'user residence converted warehouse', 'User moved to a lakeside cottage last month.')], + final_query='What instrument does the user play?', + must_contain=['cello'], + must_not_contain=['converted warehouse'], + ), + GeneratedCase( + id='adv_compound_fact_24', + family='supersession', + setup_facts=['User has run 12 marathons and works as a marine biologist.', 'User specializes in coral reef ecosystems.'], + mutations=[('supersede', 'user marathon count', 'User has completed their 15th marathon this year.')], + final_query="What is the user's profession?", + must_contain=['marine biologist'], + must_not_contain=['12 marathons'], + ), + GeneratedCase( + id='adv_compound_fact_25', + family='supersession', + setup_facts=['User owns a golden retriever named Max and drives a Tesla Model 3.', 'User volunteers at a local animal shelter on weekends.'], + mutations=[('supersede', 'user vehicle Tesla Model 3', 'User recently upgraded to a Ford Mustang Mach-E.')], + final_query="What is the name of the user's dog?", + must_contain=['Max'], + must_not_contain=['Tesla Model 3'], + ), + GeneratedCase( + id='adv_compound_fact_26', + family='supersession', + setup_facts=['User plays the violin professionally and has a black belt in Taekwondo.', 'User collects vintage comic books.'], + mutations=[('supersede', 'user martial arts black belt', 'User stopped practicing Taekwondo due to an injury.')], + final_query='What instrument does the user play?', + must_contain=['violin'], + must_not_contain=['Taekwondo'], + ), + GeneratedCase( + id='adv_compound_fact_27', + family='supersession', + setup_facts=['User speaks fluent Mandarin and holds a PhD in Neuroscience.', 'User enjoys hiking in the Rocky Mountains.'], + mutations=[('supersede', 'user academic degree PhD', 'User obtained an MD degree from Harvard Medical School.')], + final_query='What languages does the user speak?', + must_contain=['Mandarin'], + must_not_contain=['PhD'], + ), + GeneratedCase( + id='adv_compound_fact_28', + family='supersession', + setup_facts=['User is allergic to peanuts and works as a software architect at Microsoft.', 'User participates in competitive chess tournaments.'], + mutations=[('supersede', 'user employer Microsoft', 'User accepted a position as CTO at a startup.')], + final_query='What is the user allergic to?', + must_contain=['peanuts'], + must_not_contain=['Microsoft'], + ), + GeneratedCase( + id='adv_compound_fact_29', + family='supersession', + setup_facts=['User lives in Tokyo and has published three science fiction novels.', 'User practices meditation daily.'], + mutations=[('supersede', 'user residence Tokyo', 'User moved to Kyoto last month.')], + final_query='How many novels has the user published?', + must_contain=['three'], + must_not_contain=['Tokyo'], + ), + GeneratedCase( + id='adv_compound_fact_30', + family='supersession', + setup_facts=['User is a certified yoga instructor and owns a coffee shop in Portland.', 'User has traveled to over 30 countries.'], + mutations=[('supersede', 'user business coffee shop', 'User sold the coffee shop and now runs a bookstore.')], + final_query='What certification does the user hold?', + must_contain=['yoga instructor'], + must_not_contain=['coffee shop'], + ), + GeneratedCase( + id='adv_compound_fact_31', + family='supersession', + setup_facts=['User is left-handed and plays competitive tennis at state level.', 'User collects rare vinyl records.'], + mutations=[('supersede', 'user tennis participation state level', 'User retired from competitive tennis to focus on coaching.')], + final_query='Which hand does the user write with?', + must_contain=['left-handed'], + must_not_contain=['tennis'], + ), + GeneratedCase( + id='adv_compound_fact_32', + family='supersession', + setup_facts=['User has a twin brother and works as a marine biologist in Miami.', 'User enjoys scuba diving in the Caribbean.'], + mutations=[('supersede', 'user occupation marine biologist', 'User transitioned to environmental law.')], + final_query='Does the user have siblings?', + must_contain=['twin brother'], + must_not_contain=['marine biologist'], + ), + GeneratedCase( + id='adv_compound_fact_33', + family='supersession', + setup_facts=['User owns a golden retriever named Max and drives a Tesla Model 3.', 'User volunteers at a local animal shelter on weekends.'], + mutations=[('supersede', 'user vehicle Tesla Model 3', 'User recently upgraded to a Rivian R1T.')], + final_query="What is the name of the user's dog?", + must_contain=['Max'], + must_not_contain=['Tesla Model 3'], + ), + GeneratedCase( + id='adv_compound_fact_34', + family='supersession', + setup_facts=['User holds a black belt in karate and works as a pediatrician.', 'User enjoys baking sourdough bread as a hobby.'], + mutations=[('supersede', 'user profession pediatrician', 'User transitioned to emergency medicine last year.')], + final_query='What martial arts rank does the user hold?', + must_contain=['black belt'], + must_not_contain=['pediatrician'], + ), + GeneratedCase( + id='adv_compound_fact_35', + family='supersession', + setup_facts=['User plays competitive chess and resides in Toronto.', 'User collects vintage vinyl records from the 1970s.'], + mutations=[('supersede', 'user residence Toronto', 'User moved to Vancouver for a job opportunity.')], + final_query='What competitive game does the user play?', + must_contain=['chess'], + must_not_contain=['Toronto'], + ), + GeneratedCase( + id='adv_compound_fact_36', + family='supersession', + setup_facts=['User is allergic to peanuts and works as a flight attendant.', 'User has completed marathons in three different countries.'], + mutations=[('supersede', 'user allergy peanuts', 'User developed a shellfish allergy last summer.')], + final_query="What is the user's occupation?", + must_contain=['flight attendant'], + must_not_contain=['peanuts'], + ), + GeneratedCase( + id='adv_compound_fact_37', + family='supersession', + setup_facts=['User has a twin sister named Emily and owns a coffee shop.', 'User practices meditation daily before sunrise.'], + mutations=[('supersede', 'user business coffee shop', 'User sold the coffee shop and now runs a bookstore.')], + final_query='Does the user have any siblings?', + must_contain=['twin sister named Emily'], + must_not_contain=['coffee shop'], + ), + GeneratedCase( + id='adv_compound_fact_38', + family='supersession', + setup_facts=['User is afraid of heights and speaks fluent Mandarin.', 'User has a collection of rare orchids in their greenhouse.'], + mutations=[('supersede', 'user fear heights', 'User overcame their fear through exposure therapy.')], + final_query='What languages does the user speak fluently?', + must_contain=['Mandarin'], + must_not_contain=['afraid of heights'], + ), + GeneratedCase( + id='adv_compound_fact_39', + family='supersession', + setup_facts=['User wears prescription glasses and volunteers at a food bank.', 'User maintains a perfect credit score.'], + mutations=[('supersede', 'user volunteer work food bank', 'User now volunteers at an after-school tutoring program.')], + final_query='Does the user require vision correction?', + must_contain=['prescription glasses'], + must_not_contain=['food bank'], + ), + GeneratedCase( + id='adv_compound_fact_40', + family='supersession', + setup_facts=['User plays the violin and works as a data scientist at Netflix.', 'User grows heirloom tomatoes in their backyard.'], + mutations=[('supersede', 'user employer Netflix', 'User accepted a research position at DeepMind.')], + final_query='What musical instrument does the user play?', + must_contain=['violin'], + must_not_contain=['Netflix'], + ), +] + +ADV_IDENTIFIER_OBFUSCATION_GENERATED = [ + GeneratedCase( + id='adv_identifier_obfuscation_17', + family='purge', + setup_facts=['User @johndoe has admin privileges.', 'User johndoe (without @ prefix) is the same account.', 'System maintenance scheduled for midnight.'], + mutations=[('purge', '@johndoe')], + final_query='Does johndoe have admin rights?', + must_contain=[], + must_not_contain=['@johndoe has admin', 'johndoe is the same'], + ), + GeneratedCase( + id='adv_identifier_obfuscation_18', + family='purge', + setup_facts=['User "john.doe"@example.org has admin privileges.', 'User john.doe@example.org (unquoted) refers to the same account.', 'Server maintenance is scheduled for tonight.'], + mutations=[('purge', '"john.doe"@example.org')], + final_query='Does john.doe@example.org have admin rights?', + must_contain=[], + must_not_contain=['"john.doe"@example.org has admin', 'john.doe@example.org refers to the same'], + ), +] + +ADV_CROSS_LINGUAL_IDENTIFIER_GENERATED = [ + GeneratedCase( + id='adv_cross_lingual_identifier_17', + family='purge', + setup_facts=['Patient Μαρία Παπαδοπούλου had a blood test.', 'Same patient in Latin script: Maria Papadopoulou has type O+ blood.', 'Lab results were processed on Wednesday.'], + mutations=[('purge', 'patient Maria Papadopoulou (Μαρία Παπαδοπούλου)')], + final_query="What are Maria Papadopoulou's test results?", + must_contain=[], + must_not_contain=['Μαρία Παπαδοπούλου had', 'Maria Papadopoulou has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_19', + family='purge', + setup_facts=['Customer Иван Петров bought a laptop.', 'Same customer as Ivan Petrov subscribed to premium support.', 'Store inventory was updated today.'], + mutations=[('purge', 'customer Ivan Petrov (Иван Петров)')], + final_query='What did Ivan Petrov purchase?', + must_contain=[], + must_not_contain=['Иван Петров bought', 'Ivan Petrov subscribed'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_20', + family='purge', + setup_facts=['User 山田花子 uploaded documents.', 'Same user as Yamada Hanako has editor privileges.', 'Server backup completed at 2AM.'], + mutations=[('purge', 'user Yamada Hanako (山田花子)')], + final_query='What permissions does Yamada Hanako have?', + must_contain=[], + must_not_contain=['山田花子 uploaded', 'Yamada Hanako has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_21', + family='purge', + setup_facts=['Patient Müller, Jürgen visited the clinic.', 'Same patient as Mueller, Juergen was prescribed antibiotics.', 'Waiting room was cleaned at noon.'], + mutations=[('purge', 'patient Mueller, Juergen (Müller, Jürgen)')], + final_query='What treatment did Juergen Mueller receive?', + must_contain=[], + must_not_contain=['Müller, Jürgen visited', 'Mueller, Juergen was'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_22', + family='purge', + setup_facts=['Client José González signed contract #451.', 'Same client as Jose Gonzalez made payment yesterday.', 'Legal department is closed Fridays.'], + mutations=[('purge', 'client Jose Gonzalez (José González)')], + final_query='What contracts does Jose Gonzalez have?', + must_contain=[], + must_not_contain=['José González signed', 'Jose Gonzalez made'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_24', + family='purge', + setup_facts=['Customer Γεώργιος Παπαδόπουλος requested a refund.', 'Same customer as Georgios Papadopoulos had order #7890.', 'Accounting system will be upgraded next week.'], + mutations=[('purge', 'customer Georgios Papadopoulos (Γεώργιος Παπαδόπουλος)')], + final_query="What was Georgios Papadopoulos's request?", + must_contain=[], + must_not_contain=['Γεώργιος Παπαδόπουλος requested', 'Georgios Papadopoulos had'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_23', + family='purge', + setup_facts=['Employee Ιωάννης Παπαδόπουλος requested vacation.', 'Same employee as Ioannis Papadopoulos has 3 years tenure.', 'HR system was updated last month.'], + mutations=[('purge', 'employee Ioannis Papadopoulos / Ιωάννης Παπαδόπουλος')], + final_query="Show Ioannis Papadopoulos's employment details.", + must_contain=[], + must_not_contain=['Ιωάννης Παπαδόπουλος requested', 'Ioannis Papadopoulos has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_25', + family='purge', + setup_facts=['Customer Анна Иванова purchased premium subscription.', 'Same customer as Anna Ivanova has payment on file.', 'New products launching next week.'], + mutations=[('purge', 'customer Anna Ivanova / Анна Иванова')], + final_query='What did Anna Ivanova buy?', + must_contain=[], + must_not_contain=['Анна Иванова purchased', 'Anna Ivanova has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_26', + family='purge', + setup_facts=['Patient דוד כהן was admitted to cardiology.', 'Same patient as David Cohen has allergies listed.', 'Hospital visiting hours changed recently.'], + mutations=[('purge', 'patient David Cohen / דוד כהן')], + final_query="What is David Cohen's medical status?", + must_contain=[], + must_not_contain=['דוד כהן was admitted', 'David Cohen has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_27', + family='purge', + setup_facts=['User François Duchêne created a support ticket.', 'Same user as Francois Duchene has VIP status.', 'Support system upgrade completed yesterday.'], + mutations=[('purge', 'user Francois Duchene / François Duchêne')], + final_query="Check Francois Duchene's support requests.", + must_contain=[], + must_not_contain=['François Duchêne created', 'Francois Duchene has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_30', + family='purge', + setup_facts=['User 👩\u200d💻Αθηνά Παπανδρέου👩\u200d💻 changed password.', 'Same user as Athina Papandreou has MFA disabled.', 'Security audit scheduled for Friday.'], + mutations=[('purge', 'user Athina Papandreou / 👩\u200d💻Αθηνά Παπανδρέου👩\u200d💻')], + final_query="Check Athina Papandreou's security settings.", + must_contain=[], + must_not_contain=['Αθηνά Παπανδρέου changed', 'Athina Papandreou has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_28', + family='purge', + setup_facts=['User أحمد booked flight AA123.', 'Same user as Ahmed has platinum status.', 'New security protocol was implemented yesterday.'], + mutations=[('purge', 'user Ahmed aka أحمد')], + final_query="What is Ahmed's flight booking?", + must_contain=[], + must_not_contain=['أحمد booked', 'Ahmed has platinum'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_29', + family='purge', + setup_facts=['Customer Γιώργος placed order #4567.', 'Same customer as George requested express shipping.', 'Warehouse inventory was updated this morning.'], + mutations=[('purge', 'customer George / Γιώργος')], + final_query="Show George's recent orders", + must_contain=[], + must_not_contain=['Γιώργος placed', 'George requested'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_31', + family='purge', + setup_facts=['Employee גלית submitted vacation request.', 'Same employee as Galit Cohen is on project Alpha.', 'Company picnic is scheduled for June.'], + mutations=[('purge', 'employee Galit Cohen גלית')], + final_query="Show Galit Cohen's current projects", + must_contain=[], + must_not_contain=['גלית submitted', 'Galit Cohen is on'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_32', + family='purge', + setup_facts=['Customer François Dubois made purchase on 5/15.', 'Same customer as Francois Dubois has loyalty points.', 'Store hours changed effective today.'], + mutations=[('purge', 'customer Francois Dubois (François)')], + final_query='What did Francois Dubois purchase?', + must_contain=[], + must_not_contain=['François Dubois made', 'Francois Dubois has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_33', + family='purge', + setup_facts=['User Müller, Hans logged in from Berlin.', 'Same user as Mueller, Hans changed password recently.', 'System maintenance planned for tonight.'], + mutations=[('purge', 'user Mueller, Hans (Müller)')], + final_query='Show login history for Mueller, Hans', + must_contain=[], + must_not_contain=['Müller, Hans logged', 'Mueller, Hans changed'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_34', + family='purge', + setup_facts=['Patient 박지민 was prescribed vitamin D.', 'Same patient as Park Jimin has annual checkup scheduled.', 'Clinic is closed on holidays.'], + mutations=[('purge', 'patient Park Jimin (박지민)')], + final_query='What prescriptions does Park Jimin have?', + must_contain=[], + must_not_contain=['박지민 was prescribed', 'Park Jimin has annual'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_35', + family='purge', + setup_facts=['User 🎯Archer_123 set up biometric login.', 'Same user as Archer_123 has 3 failed login attempts.', 'New feature rollout begins next week.'], + mutations=[('purge', 'user Archer_123 (🎯Archer_123)')], + final_query='Show security status for Archer_123', + must_contain=[], + must_not_contain=['🎯Archer_123 set up', 'Archer_123 has 3 failed'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_37', + family='purge', + setup_facts=['Employee Γεώργιος clocked in late today.', 'Same employee as Georgios has 3 vacation days left.', 'HR system will be upgraded next week.'], + mutations=[('purge', 'employee Georgios / Γεώργιος')], + final_query="What's Georgios's attendance record?", + must_contain=[], + must_not_contain=['Γεώργιος clocked', 'Georgios has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_38', + family='purge', + setup_facts=['Patient מרים כהן was discharged yesterday.', 'Same patient as Miriam Cohen has a follow-up in 2 weeks.', 'Hospital cafeteria has new menu items.'], + mutations=[('purge', 'patient Miriam Cohen (מרים כהן)')], + final_query="When is Miriam Cohen's next appointment?", + must_contain=[], + must_not_contain=['מרים כהן was discharged', 'Miriam Cohen has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_39', + family='purge', + setup_facts=['Subscriber Müller 💻 renewed premium plan.', 'Same subscriber as Mueller has payment issue.', 'Server maintenance scheduled for Friday.'], + mutations=[('purge', 'subscriber Mueller / Müller')], + final_query="What's Mueller's subscription status?", + must_contain=[], + must_not_contain=['Müller 💻 renewed', 'Mueller has'], + ), + GeneratedCase( + id='adv_cross_lingual_identifier_40', + family='purge', + setup_facts=['Applicant 박지민 submitted form F-221.', 'Same applicant as Park Jimin failed background check.', 'New application portal launches next month.'], + mutations=[('purge', 'applicant Park Jimin (박지민)')], + final_query='What forms did Park Jimin submit?', + must_contain=[], + must_not_contain=['박지민 submitted', 'Park Jimin failed'], + ), +] + +ADV_RECURSIVE_SUPERSESSION_GENERATED = [ + GeneratedCase( + id='adv_recursive_supersession_09', + family='drift', + setup_facts=['Employee worked in the New York office from 2018-2020.'], + mutations=[('supersede', 'employee location New York', 'Employee transferred to London office in 2021 for promotion.'), ('supersede', 'employee location London', 'Employee returned to New York office in 2023 due to family reasons.')], + final_query='Where is the employee currently based?', + must_contain=['New York'], + must_not_contain=['London'], + ), + GeneratedCase( + id='adv_recursive_supersession_10', + family='drift', + setup_facts=['Patient was taking Lipitor for cholesterol management since 2019.'], + mutations=[('supersede', 'patient medication Lipitor', 'Patient switched to Crestor in 2022 due to side effects.'), ('supersede', 'patient medication Crestor', 'Patient returned to Lipitor in 2024 after Crestor proved ineffective.')], + final_query='What cholesterol medication is the patient currently taking?', + must_contain=['Lipitor'], + must_not_contain=['Crestor'], + ), + GeneratedCase( + id='adv_recursive_supersession_11', + family='drift', + setup_facts=['Company used Oracle database systems until 2020.'], + mutations=[('supersede', 'company database Oracle', 'Company migrated to PostgreSQL in 2021 for cost savings.'), ('supersede', 'company database PostgreSQL', 'Company returned to Oracle in 2023 after performance issues.')], + final_query='What database system does the company currently use?', + must_contain=['Oracle'], + must_not_contain=['PostgreSQL'], + ), + GeneratedCase( + id='adv_recursive_supersession_12', + family='drift', + setup_facts=['Student attended Stanford University from 2017-2021.'], + mutations=[('supersede', 'student university Stanford', 'Student transferred to MIT in 2022 for graduate studies.'), ('supersede', 'student university MIT', 'Student returned to Stanford in 2024 for PhD program.')], + final_query='Where is the student currently enrolled?', + must_contain=['Stanford'], + must_not_contain=['MIT'], + ), + GeneratedCase( + id='adv_recursive_supersession_13', + family='drift', + setup_facts=['City used diesel buses for public transport until 2018.'], + mutations=[('supersede', 'city buses diesel', 'City switched to electric buses in 2020 for environmental reasons.'), ('supersede', 'city buses electric', 'City returned to diesel buses in 2023 due to charging infrastructure limitations.')], + final_query='What type of buses does the city currently operate?', + must_contain=['diesel'], + must_not_contain=['electric'], + ), + GeneratedCase( + id='adv_recursive_supersession_14', + family='drift', + setup_facts=['Team used Scrum methodology from 2016-2019.'], + mutations=[('supersede', 'team methodology Scrum', 'Team switched to Kanban in 2020 for more flexibility.'), ('supersede', 'team methodology Kanban', 'Team returned to Scrum in 2022 for better sprint planning.')], + final_query='What development methodology does the team currently use?', + must_contain=['Scrum'], + must_not_contain=['Kanban'], + ), + GeneratedCase( + id='adv_recursive_supersession_15', + family='drift', + setup_facts=['Artist painted exclusively in oils until 2017.'], + mutations=[('supersede', 'artist medium oils', 'Artist switched to acrylics in 2019 for faster drying time.'), ('supersede', 'artist medium acrylics', 'Artist returned to oils in 2022 for richer textures.')], + final_query='What medium does the artist primarily work in now?', + must_contain=['oils'], + must_not_contain=['acrylics'], + ), + GeneratedCase( + id='adv_recursive_supersession_16', + family='drift', + setup_facts=['Employee worked as a software engineer at Microsoft from 2018 to 2020.'], + mutations=[('supersede', 'employee employer Microsoft', 'Employee joined Google as a senior engineer in 2021.'), ('supersede', 'employee employer Google', 'Employee returned to Microsoft as a principal engineer in 2023.')], + final_query='Where does the employee currently work?', + must_contain=['Microsoft'], + must_not_contain=['Google'], + ), + GeneratedCase( + id='adv_recursive_supersession_17', + family='drift', + setup_facts=["User's preferred streaming service was Netflix in 2022."], + mutations=[('supersede', 'user streaming Netflix', 'User switched to Disney+ in 2023 for Marvel content.'), ('supersede', 'user streaming Disney+', 'User went back to Netflix in 2024 after Disney+ raised prices.')], + final_query='What streaming service does the user currently subscribe to?', + must_contain=['Netflix'], + must_not_contain=['Disney+'], + ), + GeneratedCase( + id='adv_recursive_supersession_18', + family='drift', + setup_facts=['Patient was prescribed Lipitor for cholesterol in 2021.'], + mutations=[('supersede', 'patient medication Lipitor', 'Patient switched to Crestor in 2022 due to side effects.'), ('supersede', 'patient medication Crestor', "Patient returned to Lipitor in 2023 after Crestor didn't work.")], + final_query="What is the patient's current cholesterol medication?", + must_contain=['Lipitor'], + must_not_contain=['Crestor'], + ), + GeneratedCase( + id='adv_recursive_supersession_19', + family='drift', + setup_facts=['Company used AWS for cloud services from 2019 to 2021.'], + mutations=[('supersede', 'company cloud AWS', 'Company migrated to Azure in 2022 for better integration.'), ('supersede', 'company cloud Azure', 'Company switched back to AWS in 2023 due to cost savings.')], + final_query='Which cloud provider does the company currently use?', + must_contain=['AWS'], + must_not_contain=['Azure'], + ), + GeneratedCase( + id='adv_recursive_supersession_20', + family='drift', + setup_facts=['Student studied French as second language in 2020.'], + mutations=[('supersede', 'student language French', 'Student switched to learning Spanish in 2021 for travel.'), ('supersede', 'student language Spanish', 'Student resumed French studies in 2023 for scholarship requirements.')], + final_query='What foreign language is the student currently studying?', + must_contain=['French'], + must_not_contain=['Spanish'], + ), + GeneratedCase( + id='adv_recursive_supersession_21', + family='drift', + setup_facts=['City used diesel buses for public transport until 2020.'], + mutations=[('supersede', 'city transport diesel', 'City switched to electric buses in 2021 for environmental reasons.'), ('supersede', 'city transport electric', 'City temporarily returned to diesel buses in 2023 due to charging infrastructure issues.')], + final_query='What type of buses does the city currently operate?', + must_contain=['diesel'], + must_not_contain=['electric'], + ), + GeneratedCase( + id='adv_recursive_supersession_22', + family='drift', + setup_facts=['Author wrote fantasy novels exclusively until 2019.'], + mutations=[('supersede', 'author genre fantasy', 'Author started writing science fiction in 2020 for creative exploration.'), ('supersede', 'author genre science fiction', 'Author returned to writing fantasy in 2023 for fan demand.')], + final_query='What genre does the author currently write?', + must_contain=['fantasy'], + must_not_contain=['science fiction'], + ), + GeneratedCase( + id='adv_recursive_supersession_23', + family='drift', + setup_facts=['Team used Scrum methodology from 2018 to 2021.'], + mutations=[('supersede', 'team methodology Scrum', 'Team adopted Kanban in 2022 for more flexibility.'), ('supersede', 'team methodology Kanban', 'Team reverted to Scrum in 2024 for better sprint planning.')], + final_query='What development methodology does the team currently use?', + must_contain=['Scrum'], + must_not_contain=['Kanban'], + ), + GeneratedCase( + id='adv_recursive_supersession_24', + family='drift', + setup_facts=["Alice's favorite color was blue in childhood."], + mutations=[('supersede', 'Alice favorite color blue', 'Alice preferred green during college years.'), ('supersede', 'Alice favorite color green', 'Alice rediscovered her love for blue in adulthood.')], + final_query="What is Alice's favorite color?", + must_contain=['blue'], + must_not_contain=['green'], + ), + GeneratedCase( + id='adv_recursive_supersession_25', + family='drift', + setup_facts=['Company X used Java for backend services in 2018.'], + mutations=[('supersede', 'Company X backend language Java', 'Company X migrated to Go for better performance in 2020.'), ('supersede', 'Company X backend language Go', 'Company X returned to Java in 2023 due to developer familiarity.')], + final_query='What language does Company X use for backend?', + must_contain=['Java'], + must_not_contain=['Go'], + ), + GeneratedCase( + id='adv_recursive_supersession_26', + family='drift', + setup_facts=['Professor Smith taught Calculus I in Fall 2020.'], + mutations=[('supersede', 'Professor Smith teaching Calculus I', 'Professor Smith switched to teaching Linear Algebra in Fall 2021.'), ('supersede', 'Professor Smith teaching Linear Algebra', 'Professor Smith returned to teaching Calculus I in Fall 2022.')], + final_query='What course is Professor Smith teaching now?', + must_contain=['Calculus I'], + must_not_contain=['Linear Algebra'], + ), + GeneratedCase( + id='adv_recursive_supersession_27', + family='drift', + setup_facts=['The museum displayed Renaissance art in Gallery A before 2021.'], + mutations=[('supersede', 'museum Gallery A Renaissance', 'The museum replaced Renaissance with Modern art in Gallery A in 2021.'), ('supersede', 'museum Gallery A Modern', 'The museum reinstalled Renaissance art in Gallery A in 2023.')], + final_query='What art style is currently in Gallery A?', + must_contain=['Renaissance'], + must_not_contain=['Modern'], + ), + GeneratedCase( + id='adv_recursive_supersession_28', + family='drift', + setup_facts=['Team A used Scrum methodology in 2019.'], + mutations=[('supersede', 'Team A methodology Scrum', 'Team A adopted Kanban in 2020 for more flexibility.'), ('supersede', 'Team A methodology Kanban', 'Team A returned to Scrum in 2022 for better sprint planning.')], + final_query='What methodology does Team A currently use?', + must_contain=['Scrum'], + must_not_contain=['Kanban'], + ), + GeneratedCase( + id='adv_recursive_supersession_29', + family='drift', + setup_facts=["The town's main park had oak trees before 2021."], + mutations=[('supersede', 'town park trees oak', 'The town replaced oak trees with maple trees in 2021.'), ('supersede', 'town park trees maple', 'The town replanted oak trees in 2023 after public demand.')], + final_query='What type of trees are in the main park now?', + must_contain=['oak'], + must_not_contain=['maple'], + ), + GeneratedCase( + id='adv_recursive_supersession_30', + family='drift', + setup_facts=['The cafe served only coffee before 2020.'], + mutations=[('supersede', 'cafe menu coffee', 'The cafe added tea to its menu in 2020.'), ('supersede', 'cafe menu tea', 'The cafe returned to serving only coffee in 2022.')], + final_query='What does the cafe serve now?', + must_contain=['coffee'], + must_not_contain=['tea'], + ), + GeneratedCase( + id='adv_recursive_supersession_31', + family='drift', + setup_facts=['Lisa was an Android user before 2021.'], + mutations=[('supersede', 'Lisa phone Android', 'Lisa switched to iOS in 2021 for better apps.'), ('supersede', 'Lisa phone iOS', 'Lisa returned to Android in 2023 for customization.')], + final_query='What phone OS does Lisa use now?', + must_contain=['Android'], + must_not_contain=['iOS'], + ), + GeneratedCase( + id='adv_recursive_supersession_32', + family='drift', + setup_facts=['User worked at Microsoft from 2015 to 2020.'], + mutations=[('supersede', 'user employer Microsoft', 'User joined Google in 2021 for better opportunities.'), ('supersede', 'user employer Google', 'User returned to Microsoft in 2023 after a change in leadership.')], + final_query='Where does the user currently work?', + must_contain=['Microsoft'], + must_not_contain=['joined Google in 2021'], + ), + GeneratedCase( + id='adv_recursive_supersession_33', + family='drift', + setup_facts=['User lived in New York from 2018 to 2021.'], + mutations=[('supersede', 'user location New York', 'User moved to Los Angeles in 2022 for a job.'), ('supersede', 'user location Los Angeles', 'User moved back to New York in 2024 after missing family.')], + final_query='Where does the user currently live?', + must_contain=['New York'], + must_not_contain=['moved to Los Angeles in 2022'], + ), + GeneratedCase( + id='adv_recursive_supersession_35', + family='drift', + setup_facts=['User drove a Toyota Camry from 2017 to 2020.'], + mutations=[('supersede', 'user car Toyota Camry', 'User bought a Honda Accord in 2021 for better fuel efficiency.'), ('supersede', 'user car Honda Accord', 'User switched back to a Toyota Camry in 2023 after preferring its reliability.')], + final_query='What car does the user currently drive?', + must_contain=['Toyota Camry'], + must_not_contain=['bought a Honda Accord in 2021'], + ), + GeneratedCase( + id='adv_recursive_supersession_36', + family='drift', + setup_facts=['User was a fan of the Lakers from 2015 to 2021.'], + mutations=[('supersede', 'user favorite basketball team Lakers', 'User became a fan of the Warriors in 2022 after their championship win.'), ('supersede', 'user favorite basketball team Warriors', "User returned to being a Lakers fan in 2024 after LeBron James' return.")], + final_query="What is the user's favorite basketball team?", + must_contain=['Lakers'], + must_not_contain=['became a fan of the Warriors in 2022'], + ), + GeneratedCase( + id='adv_recursive_supersession_37', + family='drift', + setup_facts=['User used Spotify as their primary music service in 2019.'], + mutations=[('supersede', 'user music service Spotify', 'User switched to Apple Music in 2021 for better integration.'), ('supersede', 'user music service Apple Music', 'User went back to Spotify in 2023 for its playlist features.')], + final_query='What music service does the user currently use?', + must_contain=['Spotify'], + must_not_contain=['switched to Apple Music in 2021'], + ), + GeneratedCase( + id='adv_recursive_supersession_38', + family='drift', + setup_facts=['User had a cat named Whiskers from 2018 to 2022.'], + mutations=[('supersede', 'user pet Whiskers', 'User adopted a dog named Max in 2023 for companionship.'), ('supersede', 'user pet Max', 'User got another cat named Whiskers in 2024 after missing having a cat.')], + final_query="What is the name of the user's current pet?", + must_contain=['Whiskers'], + must_not_contain=['adopted a dog named Max in 2023'], + ), + GeneratedCase( + id='adv_recursive_supersession_39', + family='drift', + setup_facts=['User was a vegetarian in 2018.'], + mutations=[('supersede', 'user diet vegetarian', 'User started eating meat in 2020 for health reasons.'), ('supersede', 'user diet eating meat', 'User became vegetarian again in 2023 after ethical concerns.')], + final_query='Is the user currently vegetarian?', + must_contain=['vegetarian'], + must_not_contain=['started eating meat in 2020'], + ), + GeneratedCase( + id='adv_recursive_supersession_40', + family='drift', + setup_facts=['Employee worked in the New York office until 2021.'], + mutations=[('supersede', 'employee location New York', 'Employee transferred to the London office in 2022 for a project.'), ('supersede', 'employee location London', 'Employee returned to the New York office in 2024 after project completion.')], + final_query='Where is the employee currently based?', + must_contain=['New York'], + must_not_contain=['London'], + ), +] + +ADVERSARIAL_GENERATED: dict[str, list[GeneratedCase]] = { + 'substring_trap': ADV_SUBSTRING_TRAP_GENERATED, + 'prefix_collision': ADV_PREFIX_COLLISION_GENERATED, + 'paraphrase_supersession': ADV_PARAPHRASE_SUPERSESSION_GENERATED, + 'negation_trap': ADV_NEGATION_TRAP_GENERATED, + 'temporal_qualifier': ADV_TEMPORAL_QUALIFIER_GENERATED, + 'shared_attribute': ADV_SHARED_ATTRIBUTE_GENERATED, + 'compound_fact': ADV_COMPOUND_FACT_GENERATED, + 'identifier_obfuscation': ADV_IDENTIFIER_OBFUSCATION_GENERATED, + 'cross_lingual_identifier': ADV_CROSS_LINGUAL_IDENTIFIER_GENERATED, + 'recursive_supersession': ADV_RECURSIVE_SUPERSESSION_GENERATED, +} diff --git a/bench/forgeteval/adversarial_generated_labels.json b/bench/forgeteval/adversarial_generated_labels.json new file mode 100644 index 0000000..1fee072 --- /dev/null +++ b/bench/forgeteval/adversarial_generated_labels.json @@ -0,0 +1,255 @@ +{ + "adv_substring_trap_11": "easy", + "adv_substring_trap_14": "easy", + "adv_substring_trap_12": "easy", + "adv_substring_trap_13": "llm_lift", + "adv_substring_trap_15": "easy", + "adv_substring_trap_16": "easy", + "adv_substring_trap_18": "easy", + "adv_substring_trap_17": "easy", + "adv_substring_trap_19": "easy", + "adv_substring_trap_20": "easy", + "adv_substring_trap_21": "easy", + "adv_substring_trap_22": "easy", + "adv_substring_trap_26": "easy", + "adv_substring_trap_27": "easy", + "adv_substring_trap_24": "easy", + "adv_substring_trap_25": "llm_lift", + "adv_substring_trap_30": "easy", + "adv_substring_trap_28": "easy", + "adv_substring_trap_31": "easy", + "adv_substring_trap_32": "easy", + "adv_substring_trap_33": "llm_lift", + "adv_substring_trap_35": "easy", + "adv_substring_trap_36": "easy", + "adv_substring_trap_34": "easy", + "adv_substring_trap_37": "easy", + "adv_substring_trap_38": "easy", + "adv_substring_trap_39": "easy", + "adv_substring_trap_40": "easy", + "adv_prefix_collision_17": "easy", + "adv_prefix_collision_18": "easy", + "adv_prefix_collision_19": "unsolvable", + "adv_prefix_collision_20": "unsolvable", + "adv_prefix_collision_21": "unsolvable", + "adv_prefix_collision_22": "unsolvable", + "adv_prefix_collision_23": "unsolvable", + "adv_prefix_collision_24": "unsolvable", + "adv_prefix_collision_25": "easy", + "adv_prefix_collision_26": "easy", + "adv_prefix_collision_27": "easy", + "adv_prefix_collision_28": "unsolvable", + "adv_prefix_collision_30": "easy", + "adv_prefix_collision_32": "easy", + "adv_prefix_collision_31": "easy", + "adv_prefix_collision_33": "easy", + "adv_prefix_collision_34": "easy", + "adv_prefix_collision_35": "easy", + "adv_prefix_collision_36": "easy", + "adv_prefix_collision_37": "easy", + "adv_prefix_collision_38": "easy", + "adv_prefix_collision_39": "easy", + "adv_prefix_collision_40": "easy", + "adv_paraphrase_supersession_09": "easy", + "adv_paraphrase_supersession_10": "easy", + "adv_paraphrase_supersession_11": "easy", + "adv_paraphrase_supersession_12": "unsolvable", + "adv_paraphrase_supersession_13": "unsolvable", + "adv_paraphrase_supersession_14": "easy", + "adv_paraphrase_supersession_15": "easy", + "adv_paraphrase_supersession_16": "easy", + "adv_paraphrase_supersession_17": "easy", + "adv_paraphrase_supersession_18": "easy", + "adv_paraphrase_supersession_20": "unsolvable", + "adv_paraphrase_supersession_21": "unsolvable", + "adv_paraphrase_supersession_22": "easy", + "adv_paraphrase_supersession_23": "easy", + "adv_paraphrase_supersession_24": "easy", + "adv_paraphrase_supersession_25": "easy", + "adv_paraphrase_supersession_26": "easy", + "adv_paraphrase_supersession_27": "easy", + "adv_paraphrase_supersession_28": "unsolvable", + "adv_paraphrase_supersession_29": "easy", + "adv_paraphrase_supersession_30": "easy", + "adv_paraphrase_supersession_31": "easy", + "adv_paraphrase_supersession_33": "easy", + "adv_paraphrase_supersession_34": "easy", + "adv_paraphrase_supersession_35": "unsolvable", + "adv_paraphrase_supersession_37": "unsolvable", + "adv_paraphrase_supersession_38": "easy", + "adv_paraphrase_supersession_36": "easy", + "adv_paraphrase_supersession_39": "easy", + "adv_paraphrase_supersession_40": "easy", + "adv_negation_trap_09": "easy", + "adv_negation_trap_10": "easy", + "adv_negation_trap_11": "easy", + "adv_negation_trap_12": "easy", + "adv_negation_trap_13": "unsolvable", + "adv_negation_trap_14": "easy", + "adv_negation_trap_15": "unsolvable", + "adv_negation_trap_16": "easy", + "adv_negation_trap_17": "easy", + "adv_negation_trap_18": "easy", + "adv_negation_trap_19": "easy", + "adv_negation_trap_20": "easy", + "adv_negation_trap_21": "easy", + "adv_negation_trap_22": "easy", + "adv_negation_trap_23": "easy", + "adv_negation_trap_24": "easy", + "adv_negation_trap_25": "easy", + "adv_negation_trap_26": "easy", + "adv_negation_trap_27": "easy", + "adv_negation_trap_28": "easy", + "adv_negation_trap_29": "easy", + "adv_negation_trap_30": "easy", + "adv_negation_trap_31": "easy", + "adv_negation_trap_32": "easy", + "adv_negation_trap_33": "easy", + "adv_negation_trap_34": "easy", + "adv_negation_trap_35": "easy", + "adv_negation_trap_36": "easy", + "adv_negation_trap_37": "easy", + "adv_negation_trap_38": "easy", + "adv_negation_trap_39": "easy", + "adv_negation_trap_40": "easy", + "adv_temporal_qualifier_09": "easy", + "adv_temporal_qualifier_10": "easy", + "adv_temporal_qualifier_11": "easy", + "adv_temporal_qualifier_12": "easy", + "adv_temporal_qualifier_13": "easy", + "adv_temporal_qualifier_14": "easy", + "adv_temporal_qualifier_15": "easy", + "adv_temporal_qualifier_16": "easy", + "adv_temporal_qualifier_17": "easy", + "adv_temporal_qualifier_18": "easy", + "adv_temporal_qualifier_19": "easy", + "adv_temporal_qualifier_20": "easy", + "adv_temporal_qualifier_22": "easy", + "adv_temporal_qualifier_23": "easy", + "adv_temporal_qualifier_24": "easy", + "adv_temporal_qualifier_25": "easy", + "adv_temporal_qualifier_26": "easy", + "adv_temporal_qualifier_27": "easy", + "adv_temporal_qualifier_28": "easy", + "adv_temporal_qualifier_29": "easy", + "adv_temporal_qualifier_30": "easy", + "adv_temporal_qualifier_31": "easy", + "adv_temporal_qualifier_33": "easy", + "adv_temporal_qualifier_34": "easy", + "adv_temporal_qualifier_36": "easy", + "adv_temporal_qualifier_37": "easy", + "adv_temporal_qualifier_38": "easy", + "adv_temporal_qualifier_39": "easy", + "adv_temporal_qualifier_40": "easy", + "adv_shared_attribute_17": "easy", + "adv_shared_attribute_18": "easy", + "adv_shared_attribute_19": "easy", + "adv_shared_attribute_20": "easy", + "adv_shared_attribute_21": "easy", + "adv_shared_attribute_22": "easy", + "adv_shared_attribute_23": "easy", + "adv_shared_attribute_24": "easy", + "adv_shared_attribute_25": "easy", + "adv_shared_attribute_26": "easy", + "adv_shared_attribute_27": "llm_lift", + "adv_shared_attribute_28": "easy", + "adv_shared_attribute_29": "easy", + "adv_shared_attribute_30": "easy", + "adv_shared_attribute_31": "llm_lift", + "adv_shared_attribute_32": "easy", + "adv_shared_attribute_33": "easy", + "adv_shared_attribute_34": "easy", + "adv_shared_attribute_35": "easy", + "adv_shared_attribute_36": "easy", + "adv_shared_attribute_37": "easy", + "adv_shared_attribute_38": "easy", + "adv_shared_attribute_39": "easy", + "adv_shared_attribute_40": "llm_lift", + "adv_compound_fact_09": "llm_lift", + "adv_compound_fact_10": "llm_lift", + "adv_compound_fact_11": "llm_lift", + "adv_compound_fact_12": "llm_lift", + "adv_compound_fact_13": "llm_lift", + "adv_compound_fact_14": "llm_lift", + "adv_compound_fact_15": "llm_lift", + "adv_compound_fact_16": "llm_lift", + "adv_compound_fact_17": "llm_lift", + "adv_compound_fact_18": "llm_lift", + "adv_compound_fact_19": "llm_lift", + "adv_compound_fact_20": "llm_lift", + "adv_compound_fact_21": "llm_lift", + "adv_compound_fact_22": "llm_lift", + "adv_compound_fact_23": "llm_lift", + "adv_compound_fact_24": "llm_lift", + "adv_compound_fact_25": "llm_lift", + "adv_compound_fact_26": "unsolvable", + "adv_compound_fact_27": "llm_lift", + "adv_compound_fact_28": "llm_lift", + "adv_compound_fact_29": "llm_lift", + "adv_compound_fact_30": "unsolvable", + "adv_compound_fact_31": "unsolvable", + "adv_compound_fact_32": "llm_lift", + "adv_compound_fact_33": "llm_lift", + "adv_compound_fact_34": "llm_lift", + "adv_compound_fact_35": "llm_lift", + "adv_compound_fact_36": "unsolvable", + "adv_compound_fact_37": "unsolvable", + "adv_compound_fact_38": "llm_lift", + "adv_compound_fact_39": "llm_lift", + "adv_compound_fact_40": "llm_lift", + "adv_identifier_obfuscation_17": "easy", + "adv_identifier_obfuscation_18": "easy", + "adv_cross_lingual_identifier_17": "llm_lift", + "adv_cross_lingual_identifier_19": "llm_lift", + "adv_cross_lingual_identifier_20": "llm_lift", + "adv_cross_lingual_identifier_21": "llm_lift", + "adv_cross_lingual_identifier_22": "llm_lift", + "adv_cross_lingual_identifier_24": "llm_lift", + "adv_cross_lingual_identifier_23": "llm_lift", + "adv_cross_lingual_identifier_25": "llm_lift", + "adv_cross_lingual_identifier_26": "llm_lift", + "adv_cross_lingual_identifier_27": "llm_lift", + "adv_cross_lingual_identifier_30": "llm_lift", + "adv_cross_lingual_identifier_28": "llm_lift", + "adv_cross_lingual_identifier_29": "llm_lift", + "adv_cross_lingual_identifier_31": "llm_lift", + "adv_cross_lingual_identifier_32": "llm_lift", + "adv_cross_lingual_identifier_33": "llm_lift", + "adv_cross_lingual_identifier_34": "llm_lift", + "adv_cross_lingual_identifier_35": "llm_lift", + "adv_cross_lingual_identifier_37": "llm_lift", + "adv_cross_lingual_identifier_38": "llm_lift", + "adv_cross_lingual_identifier_39": "llm_lift", + "adv_cross_lingual_identifier_40": "llm_lift", + "adv_recursive_supersession_09": "easy", + "adv_recursive_supersession_10": "unsolvable", + "adv_recursive_supersession_11": "easy", + "adv_recursive_supersession_12": "easy", + "adv_recursive_supersession_13": "easy", + "adv_recursive_supersession_14": "easy", + "adv_recursive_supersession_15": "easy", + "adv_recursive_supersession_16": "easy", + "adv_recursive_supersession_17": "unsolvable", + "adv_recursive_supersession_18": "unsolvable", + "adv_recursive_supersession_19": "easy", + "adv_recursive_supersession_20": "easy", + "adv_recursive_supersession_21": "easy", + "adv_recursive_supersession_22": "easy", + "adv_recursive_supersession_23": "easy", + "adv_recursive_supersession_24": "easy", + "adv_recursive_supersession_25": "easy", + "adv_recursive_supersession_26": "easy", + "adv_recursive_supersession_27": "easy", + "adv_recursive_supersession_28": "easy", + "adv_recursive_supersession_29": "easy", + "adv_recursive_supersession_30": "easy", + "adv_recursive_supersession_31": "easy", + "adv_recursive_supersession_32": "easy", + "adv_recursive_supersession_33": "easy", + "adv_recursive_supersession_35": "easy", + "adv_recursive_supersession_36": "easy", + "adv_recursive_supersession_37": "easy", + "adv_recursive_supersession_38": "easy", + "adv_recursive_supersession_39": "easy", + "adv_recursive_supersession_40": "easy" +} \ No newline at end of file diff --git a/bench/forgeteval/run.py b/bench/forgeteval/run.py index e900aac..f99961e 100644 --- a/bench/forgeteval/run.py +++ b/bench/forgeteval/run.py @@ -86,7 +86,8 @@ def report(summary: dict, adapter_name: str) -> None: def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--adapter", default="lethe", - choices=["lethe", "mem0", "mempalace"]) + choices=["lethe", "mem0", "mempalace", + "langmem", "cognee", "amem"]) parser.add_argument("--lang", default="en", choices=["en", "zh", "ja"], help="Case language pool (en/zh/ja). " "Non-English auto-switches the default embedder " @@ -100,6 +101,12 @@ def main() -> None: parser.add_argument("--seed", type=int, default=42) parser.add_argument("--distractors", type=int, default=4, help="Filler facts mixed into each case (default 4)") + parser.add_argument("--suite", default="auto", + choices=["auto", "smoke", "template", "adversarial"], + help="Which test set to run. " + "'auto' (default): smoke if --scale=0, template " + "if --scale>0. 'adversarial' runs the 64 " + "hand-crafted cases (v0.2).") args = parser.parse_args() if sys.stdout.encoding.lower() != "utf-8": @@ -127,18 +134,41 @@ def embedder(text: str) -> list[float]: elif args.adapter == "mempalace": from bench.forgeteval.adapter import MemPalaceAdapter adapter = MemPalaceAdapter() + elif args.adapter == "langmem": + print(f"loading embedder: {args.embedder}") + from fastembed import TextEmbedding + from bench.forgeteval.adapter import LangGraphAdapter + model = TextEmbedding(args.embedder) + def embedder(text: str) -> list[float]: + return list(next(iter(model.embed([text])))) + adapter = LangGraphAdapter(embedder=embedder, vector_dim=args.dim) + elif args.adapter == "cognee": + from bench.forgeteval.adapter import CogneeAdapter + adapter = CogneeAdapter() + elif args.adapter == "amem": + from bench.forgeteval.adapter import AMemAdapter + adapter = AMemAdapter() else: raise ValueError(args.adapter) - if args.scale > 0: + if args.suite == "adversarial": + if args.lang != "en": + raise SystemExit("adversarial suite is English-only in v0.2") + from bench.forgeteval.adversarial import ADVERSARIAL_TESTS + test_set = ADVERSARIAL_TESTS + elif args.suite == "template" or (args.suite == "auto" and args.scale > 0): + if args.scale <= 0: + raise SystemExit("--suite template requires --scale N") from bench.forgeteval.generate import generate test_set = generate(args.scale, seed=args.seed, distractors=args.distractors, lang=args.lang) - elif args.lang != "en": - raise SystemExit("non-English smoke set not curated; use --scale N") - else: + elif args.suite == "smoke" or args.suite == "auto": + if args.lang != "en": + raise SystemExit("non-English smoke set not curated; use --scale N") test_set = ALL_TESTS + else: + raise ValueError(args.suite) print(f"\nrunning {len(test_set)} tests against {adapter.name}...\n") summary = run_adapter(adapter, test_set, verbose=(args.scale == 0)) diff --git a/docs/forgeteval_adversarial.md b/docs/forgeteval_adversarial.md new file mode 100644 index 0000000..1810021 --- /dev/null +++ b/docs/forgeteval_adversarial.md @@ -0,0 +1,337 @@ +# ForgetEval-Adv + +A hand-crafted adversarial layer for ForgetEval (v0.2). Complements the +1000-case template-generated suite (v0.1) with **64 carefully designed +cases** that probe failure modes templates cannot reach. + +--- + +## 1. Why adversarial cases + +Template generation is deterministic and large-scale, but its weakness +is the **flatness of its difficulty distribution**: every case is drawn +from the same N sub-templates with i.i.d. entity substitution. A +system that handles one template case well will handle all of them +well. + +Real production failures, by contrast, are **structured attacks** on +the system's heuristics: substrings that accidentally trigger a +distractor, identifiers that share long prefixes, supersessions where +the new fact is a paraphrase rather than a near-copy, negation flips, +temporal qualifiers, etc. + +ForgetEval-Adv targets exactly these attacks. + +--- + +## 2. Ten attack categories (112 cases total — v0.4) + +| # | Attack category | n | Primary family | What it probes | +|----|----------------------------|---:|----------------------|--------------------------------------------------------------------------------------| +| 1 | `substring_trap` | 8 | all 5 | `must_not_contain` substring accidentally appears in a distractor or related fact | +| 2 | `prefix_collision` | 16 | purge | identifiers share a long common prefix; deleting one must not take the other | +| 3 | `paraphrase_supersession` | 8 | supersession, drift | old fact and new fact have low surface overlap; only semantic alignment can match | +| 4 | `negation_trap` | 8 | supersession, decay | negated fact ("does NOT work at X") must not be confused with the affirmative | +| 5 | `temporal_qualifier` | 8 | supersession, drift | facts with embedded dates; supersession must respect the temporal window | +| 6 | `shared_attribute` | 16 | amnesia | multiple entities share one attribute; forgetting one must not collapse the other | +| 7 | `compound_fact` | 8 | supersession | one sentence carries two facts; superseding one must preserve the other | +| 8 | `identifier_obfuscation` | 16 | purge | same identifier in different surface forms (case, whitespace, encoding) | +| 9 | `cross_lingual_identifier` | 16 | purge | same entity stored under different scripts or romanizations (GDPR multilingual) | +| 10 | `recursive_supersession` | 8 | drift | supersede chain where the LATEST state matches an earlier-superseded state | + +Categories with elevated `n=16` are the ones with measurable +between-system variance from v0.2 → v0.3 runs; saturated and +zero-coverage categories stay at `n=8` because additional cases +there only tighten an already-tight CI bound. + +### Why these eight + +Each attack corresponds to a specific architectural weakness we +observed (or could plausibly observe) during development of Lethe and +the comparison adapters: + +- **`substring_trap`** — substring-based scoring (ForgetEval's default) + is cheap and deterministic but vulnerable to spurious matches. A + must-not "Stripe" can be falsely failed by a distractor about "fish + stripes." Catches: brittle scoring, *not* brittle systems. We use + this to validate the bench itself. +- **`prefix_collision`** — Lethe's lexical-purge path uses BM25, which + ranks by IDF-weighted token overlap. Two emails with a long shared + prefix may rank near-identically. Catches: insufficient + identifier-precision. +- **`paraphrase_supersession`** — RRF fusion can over-rely on BM25 + when the new fact lexically diverges; can over-rely on vec when the + entity tokens dominate. Catches: shallow-only supersession matching. +- **`negation_trap`** — embeddings often blur "X" and "not X". In + retrieval this is usually benign, but in supersession or decay, + releasing "does not own X" should not affect recall of "owns X." +- **`temporal_qualifier`** — drift through facts like "joined Google + in 2020" → "moved to Meta in 2022" → "visited Google in 2024" + requires the system to either ignore time or respect it; either + choice can fail naively. +- **`shared_attribute`** — amnesia's hardest case: Dana and Eve both + live in Berlin. Releasing "everything about Dana" must drop the + Dana-Berlin link but preserve the Eve-Berlin link. +- **`compound_fact`** — "User lives in Berlin and works at Stripe" + carries two facts. Superseding only the location must not delete + the employment. +- **`identifier_obfuscation`** — `alice@acme.io` vs `Alice@Acme.io` + vs `alice@acme.io ` (trailing whitespace) are the same identifier + for any reasonable interpretation. Catches: case-sensitive lexical + paths. + +We deliberately exclude attacks that depend on multi-turn LLM +reasoning (those belong in a v0.3 "agent-loop" bench). + +--- + +## 3. Case format + +Identical to the template-generated cases: each adversarial case is a +`GeneratedCase` dataclass instance. This means the existing runner, +adapter Protocol, and scoring logic in +[`bench/forgeteval/run.py`](../bench/forgeteval/run.py) all work +unchanged. + +```python +@dataclass +class GeneratedCase: + id: str # e.g. "adv_substring_trap_001" + family: str # one of supersession / decay / amnesia / purge / drift + setup_facts: list[str] # inscribed in order; may include in-line distractors + mutations: list[tuple] # ("supersede", old_q, new_text) | ("release", q) | ("purge", q) + final_query: str # the question + must_contain: list[str] # substrings that MUST appear in top-10 recall + must_not_contain: list[str] # substrings that MUST NOT appear +``` + +Each case additionally carries (in source comments) the **design intent**: + +```python +# attack_category: prefix_collision +# intent: deleting alice@acme.io should not also delete alice.smith@acme.io +# (shared 5-char prefix "alice") +# expected lethe behavior: pass via lexical (BM25) purge +# expected mem0 behavior: fail due to vector similarity blur +``` + +These comments are the **annotation record** for the IAA protocol +(§5). + +--- + +## 4. Scoring + +Same as template-generated suite: +- `must_contain` $\subseteq$ top-10 blob $\wedge$ `must_not_contain` $\cap$ top-10 blob $= \emptyset$ +- Per case: 1 / 0, no partial credit +- Per attack category: `pass / 8` +- Overall: `pass / 64` +- `NotImplementedError` from optional adapter methods → counted as fail, + surfaced as N/A in reports (same convention as template) + +We **deliberately use the same substring scoring** as v0.1 even on +adversarial cases. This keeps the methodology coherent: a forgetting +failure is a substring failure regardless of how the case was +generated. + +--- + +## 5. Annotation and inter-annotator agreement protocol + +NeurIPS Datasets-and-Benchmarks reviewers will (rightly) ask: who +decided these are good adversarial cases, and how reliable is that +judgement? + +We implement a two-stage IAA protocol: + +### Stage A: self-IAA over a 7-day cool-off + +1. Author writes the 64 cases plus design-intent annotations (the + `attack_category` + intent comment above). +2. After 7 days of no-look, author independently re-annotates 32 + randomly-selected cases: + - Which of the 8 attack categories does this case probe? + - Is the `must_contain` / `must_not_contain` specification + unambiguous? +3. Compute Cohen's $\kappa$ on the category labels. Target $\kappa + \geq 0.75$ ("substantial agreement"). Cases that disagree are + rewritten or moved to the most appropriate category. + +### Stage B: external partial-IAA + +1. Recruit 1–2 external annotators with LLM / IR background. +2. Each annotates 20–30 cases (no overlap with the self-IAA subset). +3. Compute pairwise $\kappa$. +4. Report all $\kappa$ values in the paper's §6.10. + +### Failure thresholds + +- $\kappa < 0.6$ on any pair → category boundary is unclear; redesign + category definitions in §2. +- Individual case disagreed on by $\geq 2$ annotators → discard or + rewrite. + +--- + +## 6. Reporting format + +Per attack category, per system, pass rate. Plus the gap to +template-only score. + +``` + ┌─────────────────── ForgetEval ─────────────────┐ + │ Template (1000) Adversarial (64) Δ │ +Lethe v1 │ 99.3 % ?? % (−??) │ +Mem0 2.0.2 │ 88.8 % ?? % (−??) │ +MemPalace │ 0.0 % ?? % (−??) │ +``` + +And per attack category (8 rows × 3 systems). This goes in §6.10 of +the paper alongside the existing tables. + +--- + +## 7. The LLM-optional hook (Lethe's response to v0.2) + +Running v0.2 against Lethe revealed two attack categories where the +default LLM-free adapter scores 0 / 8: + +- `compound_fact` — a single inscribed row carries two facts joined + by " and ". The atomic `supersede` primitive wipes both clauses + together because the depth axis treats the row as the smallest unit + of forgetting. +- `identifier_obfuscation` — surface-form variations of the same + identifier (case, whitespace, quoting, leading @, separator variants + in phones / UUIDs / SSNs / credit cards) are not grouped by the + default purge, which matches BM25 top-1 plus exact-text duplicates. + +Both failures require **semantic understanding** of either fact +decomposition or identifier equivalence. We considered three +architectural responses: + +1. **Brittle string heuristics in the adapter** — regex split on + " and ", explicit negation-marker whitelists, cosine-similarity + thresholds tuned per case, length-and-digit-ratio gates for + identifier matching. This raised the adversarial score to ~93 % + but at the cost of magic constants and ad-hoc pattern matching + that the project's own CONTRIBUTING guide explicitly warns against + (`"No regex query routers, no query-type classifiers, no 'if `:` + in query then ...'"`). **Rejected.** +2. **In-engine semantic logic** — push the same heuristics into + `lethe/core.py`. Same brittleness, plus it pollutes the engine + with policy that's hard to swap. **Rejected.** +3. **LLM-optional adapter hook** — keep the engine deterministic and + primitive-only; let the LetheAdapter take an optional + `llm: Callable[[str], str]` parameter that delegates exactly the + two narrow semantic decisions (supersede-mode planning, identifier + equivalence) to the model. Recall hot path remains LLM-free. + **Adopted.** + +The contract is two prompts, both expecting a JSON-shaped response of +at most two fields: + +- **`supersede` plan** — input is `(EXISTING_MEMORY, SUPERSEDE_QUERY, + NEW_FACT)`; output is `{"mode": "atomic"}` or `{"mode": "partial", + "merged_text": "..."}`. In partial mode the adapter calls + `surrender(id, mode="edit", new_text=...)` instead of the atomic + supersede. +- **`purge` identifier grouping** — input is `(TARGET_IDENTIFIER, + CANDIDATES)`; output is `{"matching_indices": [...]}` listing which + candidate rows describe the same identifier. + +Full prompts and adapter wiring are in +[`bench/forgeteval/adapter.py`](../bench/forgeteval/adapter.py). An +Anthropic Claude runner ships at +`lethe-paper/scripts/run_adversarial_with_llm.py`; export +`ANTHROPIC_API_KEY` and execute to obtain the with-LLM numbers. + +Architectural invariants preserved: + +1. **Engine has zero new heuristics.** `lethe/core.py` gained exactly + one new primitive — `surrender(mode="edit", new_text=...)` — and + nothing else. No canonicalization helpers, no regex, no + identifier-shape detection live in the engine. +2. **Recall is always LLM-free.** The LLM is consulted only at + `supersede` and `purge` time, and only once per call. Time-travel, + pinning, and decay never touch a model. +3. **Same primitives, two policy modes.** `LetheAdapter(llm=None)` + and `LetheAdapter(llm=callable)` use the same engine primitives. + The only difference is whether semantic decisions are routed + through the model. This makes the comparison rigorous: any + difference in the adversarial score is attributable purely to the + policy layer. + +## 8. Observed scores (v0.4, LLM-free) + +| System | template (1000) | adversarial (112) | wall / case | +|----------------|----------------:|------------------:|------------:| +| **Lethe v1** | 993 (99.3 %) | 70 (62.5 %) | ~48 ms | +| Mem0 v2.0.2 | 888 (88.8 %) | 76 (67.9 %) | ~527 ms (11×) | +| LangMem (LG) | 995 (99.5 %) | 69 (61.6 %) | ~56 ms (1.2×) | +| MemPalace | 0 ( 0.0 %) | 0 ( 0.0 %) | ~167 ms | + +The three deterministic systems land within 6 absolute points on +adversarial; their overall Wilson 95 % CIs overlap. **The honest +comparison surface is per-category**, where: + +- **Lethe 100 % > Mem0 50 %** on `prefix_collision` (16 cases, Wilson + intervals do not overlap → significant at p < 0.05). Lethe's pure- + BM25 lexical purge avoids the prefix-similarity confusion that + vector-similarity-based delete falls into. +- **Mem0 50 % > Lethe 0 %** on `cross_lingual_identifier` (16 cases, + significant). Mem0's multilingual-MiniLM embedding accidentally + bridges some script-equivalent identifiers; Lethe's exact-text + equality cannot. +- **Mem0 50 % > LangMem 0 %** on `cross_lingual_identifier` + (significant) — same mechanism. +- All three deterministic systems score **0 / 8 on `compound_fact`** + — superseding one clause of "X and Y" with a fact about X wipes + Y with it. This is the **deterministic ceiling** that motivates + the LLM-optional adapter hook (§7). + +These are pre-registered hypothesis tests: per-category claims that +the bench was designed to evaluate, made with explicit Wilson +intervals. Aggregate "Lethe beats Mem0" claims are not made +because the data does not support them at this case count; +"trade-off" is the honest read of the aggregate numbers. + +LLM-assisted runs (with `LetheAdapter(llm=Anthropic-Claude)`) are +out of scope for this no-LLM-environment run and will be reported +when `ANTHROPIC_API_KEY` is set via +`lethe-paper/scripts/run_adversarial_with_llm.py`. + +Data: `lethe-paper/data/adversarial_results.json` (v0.4 numbers). + +--- + +## 9. File layout + +``` +lethe/ +├── bench/forgeteval/ +│ ├── adversarial.py ← module exporting ADVERSARIAL_TESTS: list[GeneratedCase] +│ └── run.py ← gains --suite {smoke,template,adversarial,all} flag +├── docs/ +│ ├── forgeteval.md ← existing v0.1 methodology +│ └── forgeteval_adversarial.md ← THIS DOCUMENT +└── data/ (in lethe-paper/data/) + └── adversarial_results.json +``` + +--- + +## 10. Limitations of this layer + +- **Single-author origin.** All 64 cases are initially designed by + one person. Stage-B external IAA partially addresses this. A + larger, multi-author adversarial layer is a v0.3 goal. +- **English only.** CJK adversarial cases require native speakers and + are not in v0.2 scope. +- **Static.** Once the cases are published, adapters could overfit + to them. ForgetEval treats this as a feature for the published + suite (it's a fair, fixed test) and plans a hidden test set for + competition rounds. +- **64 is a small number.** Adversarial coverage is depth, not + breadth — we trade fewer cases for more careful design. Template + generation still provides the breadth (1000+). diff --git a/lethe/core.py b/lethe/core.py index e3d532d..6896039 100644 --- a/lethe/core.py +++ b/lethe/core.py @@ -30,7 +30,7 @@ class ConsolidateReport: duration_ms: int = 0 -SurrenderMode = Literal["decay", "release", "supersede", "purge"] +SurrenderMode = Literal["decay", "release", "supersede", "purge", "edit"] # ─── FTS5 sanitation ───────────────────────────────────────────────── @@ -440,6 +440,7 @@ def surrender( target: Union[int, list[int], dict], *, mode: SurrenderMode = "release", + new_text: Optional[str] = None, ) -> int: if mode == "decay": return self._apply_force(target, factor=0.5, kind="decay") @@ -452,6 +453,10 @@ def surrender( reason=target.get("reason", "")) if mode == "purge": return self._purge(target) + if mode == "edit": + if new_text is None: + raise ValueError("edit requires new_text=...") + return self._edit(target, new_text) raise ValueError(f"unknown mode: {mode}") def _apply_force(self, target, *, factor: float, kind: str) -> int: @@ -509,6 +514,62 @@ def _supersede(self, old_id: int, new_text: str, *, reason: str = "") -> int: self.conn.commit() return new_id + def _edit(self, target, new_text: str) -> int: + """Replace the text (and re-index) of `target` row(s) without + changing depth. Logs an `edit` event so time-travel still works. + + Use case: a memory carries multiple facts as one inscribed row + (e.g. ``"User lives in Berlin and works at Stripe"``) and the + caller wants to update only one fact while preserving the + others. Edit is the *partial-supersede* primitive that the + atomic ``supersede`` cannot express.""" + ids = self._coerce_ids(target) + for mid in ids: + row = self.conn.execute( + "SELECT depth FROM memory WHERE rowid = ?", (mid,) + ).fetchone() + if not row: + continue + depth = row[0] + # Re-embed. We need the embedder to be configured; if + # not, edit still rewrites the text but the vector stays + # stale (callers without an embedder shouldn't be using + # vector recall anyway). + if self.embedder is not None: + new_emb = self.embedder(new_text) + if len(new_emb) != self.vector_dim: + raise ValueError( + f"embedder returned dim {len(new_emb)} != " + f"configured {self.vector_dim}" + ) + self.conn.execute( + "DELETE FROM memory_vec WHERE rowid = ?", (mid,) + ) + self.conn.execute( + "INSERT INTO memory_vec(rowid, embedding) VALUES (?, ?)", + (mid, sqlite_vec.serialize_float32(new_emb)), + ) + # Always update text + FTS5 (no embedder still re-indexes + # the lexical side). + self.conn.execute( + "UPDATE memory SET text = ? WHERE rowid = ?", + (new_text, mid), + ) + self.conn.execute( + "DELETE FROM memory_fts WHERE rowid = ?", (mid,) + ) + self.conn.execute( + "INSERT INTO memory_fts(rowid, text) VALUES (?, ?)", + (mid, new_text), + ) + # Depth doesn't change; we log an edit event with the same + # before/after depth so the event-log invariant holds and + # time-travel can reconstruct the text at any past instant. + self._log_event(mid, "edit", depth, depth, + meta={"new_text": new_text}) + self.conn.commit() + return len(ids) + def _purge(self, target) -> int: ids = self._coerce_ids(target) for mid in ids: diff --git a/lethe/schema.sql b/lethe/schema.sql index 2cd45ba..b344efc 100644 --- a/lethe/schema.sql +++ b/lethe/schema.sql @@ -5,8 +5,11 @@ -- depth > 1 pinned by mnemosyne or promoted -- Primary record. All status collapses into `depth`. +-- AUTOINCREMENT keeps purged rowids out of the reuse pool, so the +-- one-way-purge invariant (Proposition 1) holds: a row erased at t0 +-- can never be impersonated by a later inscribe sharing its id. CREATE TABLE IF NOT EXISTS memory ( - rowid INTEGER PRIMARY KEY, + rowid INTEGER PRIMARY KEY AUTOINCREMENT, text TEXT, depth REAL DEFAULT 1.0, created_at REAL, diff --git a/paper/paper.pdf b/paper/paper.pdf index 9f11857..2e9c2d3 100644 Binary files a/paper/paper.pdf and b/paper/paper.pdf differ diff --git a/tests/test_depth.py b/tests/test_depth.py index 08b05a0..0ba5cd8 100644 --- a/tests/test_depth.py +++ b/tests/test_depth.py @@ -174,3 +174,99 @@ def test_blame_tracks_supersession(river: Lethe): matches = {e.memory.id: e for e in entries} if new in matches: assert old in matches[new].supersedes + + +# ─── edit mode (partial supersede / compound_fact primitive) ───────── + +def test_edit_updates_text_in_place(river: Lethe): + """`surrender(mode='edit')` updates a row's text without minting a new id. + + This is the partial-supersede primitive used by the + ``compound_fact`` adversarial category: a row carrying two facts + can have one fact replaced while the other is preserved. + """ + mid = river.inscribe("User lives in Berlin and works at Stripe.") + n = river.surrender( + mid, mode="edit", + new_text="User lives in Berlin and works at Anthropic.", + ) + assert n == 1 + row = river.conn.execute( + "SELECT text, depth FROM memory WHERE rowid=?", (mid,) + ).fetchone() + assert "Anthropic" in row[0] + assert "Stripe" not in row[0] + # Depth unchanged: edit is not supersede + assert row[1] == SURFACE + + +def test_edit_re_indexes_fts(river: Lethe): + """After edit, the new text wins lexical recall; the old text loses.""" + mid = river.inscribe("User lives in Berlin.") + river.surrender(mid, mode="edit", new_text="User lives in Tokyo.") + # Pure BM25 (no vector) so we are testing FTS re-indexing specifically. + hits = river.recall("Tokyo", k=5, lexical=True) + assert any(r.memory.id == mid for r in hits) + hits_old = river.recall("Berlin", k=5, lexical=True) + assert all(r.memory.id != mid for r in hits_old) + + +def test_edit_logs_event_with_text_payload(river: Lethe): + """Edit writes an `edit` event — the event log invariant (replay + reconstructs state) requires the new text to be recoverable.""" + mid = river.inscribe("Phone: 555-0100.") + river.surrender(mid, mode="edit", new_text="Phone: 555-0199.") + events = river.log(kind="edit") + edit_events = [e for e in events if e.memory_id == mid] + assert len(edit_events) == 1 + # Depth before/after equal (edit doesn't move depth) + e = edit_events[0] + assert e.depth_before == e.depth_after == SURFACE + + +# ─── batch inscribe ────────────────────────────────────────────────── + +def test_inscribe_many_returns_distinct_ids(river: Lethe): + """Batched inscribe yields one id per text and embeds in a single + embedder call (when ``embedder_batch`` is configured).""" + texts = [f"Fact number {i}." for i in range(20)] + ids = river.inscribe_many(texts) + assert len(ids) == 20 + assert len(set(ids)) == 20 # all distinct + # All on the surface + for mid in ids: + row = river.conn.execute( + "SELECT depth FROM memory WHERE rowid=?", (mid,) + ).fetchone() + assert row[0] == SURFACE + + +def test_inscribe_many_empty_is_noop(river: Lethe): + assert river.inscribe_many([]) == [] + + +def test_inscribe_many_metas_length_mismatch_raises(river: Lethe): + with pytest.raises(ValueError, match="metas length"): + river.inscribe_many(["a", "b"], metas=[{"x": 1}]) + + +# ─── purge (hard delete + event-log determinism) ───────────────────── + +def test_purge_multiple_ids_in_one_call(river: Lethe): + ids = [river.inscribe(f"To be purged {i}.") for i in range(3)] + purged = river.surrender(ids, mode="purge") + assert purged == 3 + for mid in ids: + row = river.conn.execute( + "SELECT rowid FROM memory WHERE rowid=?", (mid,) + ).fetchone() + assert row is None + + +def test_purged_id_is_not_reused(river: Lethe): + """One-way purge: a subsequent inscribe must mint a fresh id, not + recover the purged row.""" + mid = river.inscribe("Original.") + river.surrender(mid, mode="purge") + new = river.inscribe("Re-inscribed under a fresh identity.") + assert new != mid