From b49853058c88586cd800f7520c2b980b37312b60 Mon Sep 17 00:00:00 2001 From: PRATHAMESH75 Date: Sat, 20 Jun 2026 19:43:40 +0530 Subject: [PATCH] feat: disk cache for profile/config LLM generation (closes #20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-running a campaign on the same entities re-issues identical, expensive profile- and config-generation LLM calls. Add a content-addressed disk cache so identical requests skip the model. - app/utils/llm_cache.py: LLMResponseCache — namespaced, keyed by a SHA-256 of the request material (model + system + prompt), one JSON file per entry under //. Best-effort I/O (any error degrades to a miss), optional TTL, and an enable flag. - Config: LLM_CACHE_ENABLED (default on), LLM_CACHE_DIR, LLM_CACHE_TTL. - Wired into the two generation choke points: oasis _generate_profile_with_llm and simulation_config _call_llm_with_retry check the cache before calling and store on success (genuine LLM results only — never rule-based fallbacks). Keying on model+prompt means a prompt or model change invalidates naturally. - Documented env vars in .env.example. - tests/test_llm_cache.py: hit/miss, order-independent keys, disabled no-op, TTL expiry, namespace isolation, corrupt-entry-as-miss. Stacked on the #15 branch (shared config.py + generator changes). Co-Authored-By: PRATHAMESH75 --- .env.example | 6 ++ backend/app/config.py | 8 ++ .../app/services/oasis_profile_generator.py | 17 +++- .../services/simulation_config_generator.py | 15 ++- backend/app/utils/llm_cache.py | 81 ++++++++++++++++ backend/tests/test_llm_cache.py | 95 +++++++++++++++++++ 6 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 backend/app/utils/llm_cache.py create mode 100644 backend/tests/test_llm_cache.py diff --git a/.env.example b/.env.example index b06bcec..3bd7bf1 100644 --- a/.env.example +++ b/.env.example @@ -41,6 +41,12 @@ ZEP_API_KEY=your_zep_api_key_here # LLM_FALLBACK_BASE_URL= # LLM_FALLBACK_API_KEY= +# Disk cache for profile/config LLM generation (keyed by model + prompt). +# Enabled by default; set to false to always regenerate. +# LLM_CACHE_ENABLED=true +# LLM_CACHE_DIR= # default: backend/uploads/llm_cache +# LLM_CACHE_TTL=0 # seconds before an entry expires; 0 = never + # API key that callers must supply in X-Api-Key or Authorization: Bearer headers. # Leave empty (default) for open access — suitable for localhost development. # For public / self-hosted deployments set a strong random value: diff --git a/backend/app/config.py b/backend/app/config.py index de30f13..cb63ea0 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -56,6 +56,14 @@ class Config: LLM_FALLBACK_BASE_URL = os.environ.get('LLM_FALLBACK_BASE_URL', '').strip() LLM_FALLBACK_API_KEY = os.environ.get('LLM_FALLBACK_API_KEY', '').strip() + # Disk cache for deterministic LLM generation (profile/config) — issue #20. + # Keyed by a hash of model + prompt, so config changes invalidate naturally. + LLM_CACHE_ENABLED = _get_bool('LLM_CACHE_ENABLED', True) + LLM_CACHE_DIR = os.environ.get( + 'LLM_CACHE_DIR', os.path.join(os.path.dirname(__file__), '../uploads/llm_cache') + ) + LLM_CACHE_TTL = int(os.environ.get('LLM_CACHE_TTL', '0')) # seconds; 0 = no expiry + ZEP_API_KEY = os.environ.get('ZEP_API_KEY') APP_API_KEY = os.environ.get('APP_API_KEY', '').strip() diff --git a/backend/app/services/oasis_profile_generator.py b/backend/app/services/oasis_profile_generator.py index 713254e..e2a0f5b 100644 --- a/backend/app/services/oasis_profile_generator.py +++ b/backend/app/services/oasis_profile_generator.py @@ -22,6 +22,7 @@ from ..config import Config from ..utils.logger import get_logger from ..utils.llm_sanitizer import sanitize_content, close_truncated_json, repair_json +from ..utils.llm_cache import LLMResponseCache from .zep_entity_reader import EntityNode, ZepEntityReader logger = get_logger('mirofish.oasis_profile') @@ -201,6 +202,9 @@ def __init__( base_url=self.base_url ) + # Cache deterministic persona generations across re-runs (issue #20). + self._cache = LLMResponseCache('profile') + # Zep client for retrieving enriched context self.zep_api_key = zep_api_key or Config.ZEP_API_KEY self.zep_client = None @@ -524,6 +528,15 @@ def _generate_profile_with_llm( entity_name, entity_type, entity_summary, entity_attributes, context ) + system_prompt = self._get_system_prompt(is_individual) + + # Return a cached persona for an identical request (issue #20). Keyed by + # model + prompts, so any prompt/model change is a natural cache miss. + cache_key = {"model": self.model_name, "system": system_prompt, "prompt": prompt} + cached = self._cache.get(cache_key) + if cached is not None: + return cached + # Attempt generation multiple times until success or max retries reached max_attempts = 3 last_error = None @@ -533,7 +546,7 @@ def _generate_profile_with_llm( response = self.client.chat.completions.create( model=self.model_name, messages=[ - {"role": "system", "content": self._get_system_prompt(is_individual)}, + {"role": "system", "content": system_prompt}, {"role": "user", "content": prompt} ], response_format={"type": "json_object"}, @@ -561,6 +574,7 @@ def _generate_profile_with_llm( if "persona" not in result or not result["persona"]: result["persona"] = entity_summary or f"{entity_name}是一个{entity_type}。" + self._cache.set(cache_key, result) return result except json.JSONDecodeError as je: @@ -570,6 +584,7 @@ def _generate_profile_with_llm( result = self._try_fix_json(content, entity_name, entity_type, entity_summary) if result.get("_fixed"): del result["_fixed"] + self._cache.set(cache_key, result) return result last_error = je diff --git a/backend/app/services/simulation_config_generator.py b/backend/app/services/simulation_config_generator.py index 2ff40a0..8ba6053 100644 --- a/backend/app/services/simulation_config_generator.py +++ b/backend/app/services/simulation_config_generator.py @@ -21,6 +21,7 @@ from ..config import Config from ..utils.logger import get_logger from ..utils.llm_sanitizer import sanitize_content, close_truncated_json, repair_json +from ..utils.llm_cache import LLMResponseCache from .zep_entity_reader import EntityNode, ZepEntityReader logger = get_logger('mirofish.simulation_config') @@ -240,6 +241,9 @@ def __init__( base_url=self.base_url ) + # Cache deterministic config sub-generations across re-runs (issue #20). + self._cache = LLMResponseCache('config') + def generate_config( self, simulation_id: str, @@ -435,6 +439,12 @@ def _call_llm_with_retry(self, prompt: str, system_prompt: str) -> Dict[str, Any """LLM call with retry, including JSON repair logic""" import re + # Serve an identical config sub-generation from cache (issue #20). + cache_key = {"model": self.model_name, "system": system_prompt, "prompt": prompt} + cached = self._cache.get(cache_key) + if cached is not None: + return cached + max_attempts = 3 last_error = None @@ -463,13 +473,16 @@ def _call_llm_with_retry(self, prompt: str, system_prompt: str) -> Dict[str, Any # Attempt to parse JSON try: - return json.loads(content) + result = json.loads(content) + self._cache.set(cache_key, result) + return result except json.JSONDecodeError as e: logger.warning(f"JSON parsing failed (attempt {attempt+1}): {str(e)[:80]}") # Attempt to repair JSON fixed = self._try_fix_config_json(content) if fixed: + self._cache.set(cache_key, fixed) return fixed last_error = e diff --git a/backend/app/utils/llm_cache.py b/backend/app/utils/llm_cache.py new file mode 100644 index 0000000..7831afb --- /dev/null +++ b/backend/app/utils/llm_cache.py @@ -0,0 +1,81 @@ +""" +Disk-backed cache for deterministic LLM generation results (issue #20). + +Profile and simulation-config generation re-issue the same expensive LLM calls +whenever a campaign is re-run on the same entities. Keying a small on-disk cache +by a hash of the request (model + system + prompt) lets identical requests skip +the model entirely. + +One JSON file per entry under ``//.json`` so +concurrent generation never rewrites a single shared store. Reads and writes are +best-effort: any failure degrades to a cache miss rather than raising. +""" + +import hashlib +import json +import os +import time +from typing import Any, Optional + +from ..config import Config +from .logger import get_logger + +logger = get_logger('mirofish.llm_cache') + + +def hash_key(material: Any) -> str: + """Stable SHA-256 over the canonical JSON of the request material.""" + canonical = json.dumps(material, sort_keys=True, ensure_ascii=False, default=str) + return hashlib.sha256(canonical.encode('utf-8')).hexdigest() + + +class LLMResponseCache: + """Namespaced, content-addressed cache for LLM responses.""" + + def __init__( + self, + namespace: str, + *, + enabled: Optional[bool] = None, + cache_dir: Optional[str] = None, + ttl: Optional[int] = None, + ): + self.namespace = namespace + self.enabled = Config.LLM_CACHE_ENABLED if enabled is None else enabled + base_dir = cache_dir if cache_dir is not None else Config.LLM_CACHE_DIR + self.dir = os.path.join(base_dir, namespace) + self.ttl = Config.LLM_CACHE_TTL if ttl is None else ttl + + def _path(self, key: str) -> str: + return os.path.join(self.dir, f"{key}.json") + + def get(self, material: Any) -> Optional[Any]: + """Return the cached value for ``material``, or None on miss/expiry/error.""" + if not self.enabled: + return None + path = self._path(hash_key(material)) + if not os.path.exists(path): + return None + try: + with open(path, 'r', encoding='utf-8') as handle: + entry = json.load(handle) + except (OSError, json.JSONDecodeError): + return None + if self.ttl and self.ttl > 0: + if time.time() - entry.get('stored_at', 0) > self.ttl: + return None + return entry.get('value') + + def set(self, material: Any, value: Any) -> None: + """Store ``value`` for ``material``. Best-effort; never raises.""" + if not self.enabled: + return + path = self._path(hash_key(material)) + try: + os.makedirs(self.dir, exist_ok=True) + tmp_path = f"{path}.tmp" + with open(tmp_path, 'w', encoding='utf-8') as handle: + json.dump({'stored_at': time.time(), 'value': value}, handle, ensure_ascii=False) + os.replace(tmp_path, path) # atomic publish + except OSError as exc: + logger.warning("Failed to write LLM cache entry (%s): %s", self.namespace, exc) diff --git a/backend/tests/test_llm_cache.py b/backend/tests/test_llm_cache.py new file mode 100644 index 0000000..b7cfccc --- /dev/null +++ b/backend/tests/test_llm_cache.py @@ -0,0 +1,95 @@ +"""Tests for the disk-backed LLM response cache (issue #20).""" + +import time + +from app.utils.llm_cache import LLMResponseCache, hash_key + + +def _cache(tmp_path, namespace="profile", **kwargs): + kwargs.setdefault("enabled", True) + kwargs.setdefault("cache_dir", str(tmp_path)) + return LLMResponseCache(namespace, **kwargs) + + +def test_set_then_get_round_trips(tmp_path): + cache = _cache(tmp_path) + material = {"model": "m", "prompt": "hello"} + assert cache.get(material) is None # miss + + value = {"bio": "x", "persona": "y"} + cache.set(material, value) + assert cache.get(material) == value + + +def test_distinct_material_does_not_collide(tmp_path): + cache = _cache(tmp_path) + cache.set({"prompt": "a"}, "result-a") + cache.set({"prompt": "b"}, "result-b") + assert cache.get({"prompt": "a"}) == "result-a" + assert cache.get({"prompt": "b"}) == "result-b" + + +def test_key_is_order_independent(tmp_path): + cache = _cache(tmp_path) + cache.set({"a": 1, "b": 2}, "v") + # Same logical material, different dict ordering -> same cache entry. + assert cache.get({"b": 2, "a": 1}) == "v" + + +def test_disabled_cache_is_a_noop(tmp_path): + cache = _cache(tmp_path, enabled=False) + cache.set({"prompt": "x"}, "v") + assert cache.get({"prompt": "x"}) is None + + +def test_ttl_zero_never_expires(tmp_path): + cache = _cache(tmp_path, ttl=0) + material = {"prompt": "x"} + cache.set(material, "v") + # Backdate the entry; with ttl=0 it must still be served. + _backdate(cache, material, seconds_ago=10_000) + assert cache.get(material) == "v" + + +def test_ttl_expires_old_entries(tmp_path): + cache = _cache(tmp_path, ttl=1) + material = {"prompt": "x"} + cache.set(material, "v") + assert cache.get(material) == "v" # fresh -> hit + _backdate(cache, material, seconds_ago=10) + assert cache.get(material) is None # older than ttl -> miss + + +def _backdate(cache, material, seconds_ago): + import json + path = cache._path(hash_key(material)) + with open(path, "r", encoding="utf-8") as fh: + entry = json.load(fh) + entry["stored_at"] = time.time() - seconds_ago + with open(path, "w", encoding="utf-8") as fh: + json.dump(entry, fh) + + +def test_namespaces_are_isolated(tmp_path): + profile = _cache(tmp_path, namespace="profile") + config = _cache(tmp_path, namespace="config") + profile.set({"prompt": "x"}, "profile-value") + assert config.get({"prompt": "x"}) is None + config.set({"prompt": "x"}, "config-value") + assert profile.get({"prompt": "x"}) == "profile-value" + assert config.get({"prompt": "x"}) == "config-value" + + +def test_corrupt_entry_is_a_miss(tmp_path): + cache = _cache(tmp_path) + material = {"prompt": "x"} + cache.set(material, "v") + with open(cache._path(hash_key(material)), "w", encoding="utf-8") as fh: + fh.write("{ not valid json") + assert cache.get(material) is None + + +def test_hash_key_is_stable_and_hex(tmp_path): + key = hash_key({"a": 1}) + assert key == hash_key({"a": 1}) + assert len(key) == 64 and all(c in "0123456789abcdef" for c in key)