feat: manifest-first entity recall for all platforms (#224)

platform-integrations/bob/evolve-lite/lib/entity_io.py

@@ -78,12 +78,13 @@ def find_entities_dir():
 def find_recall_entity_dirs():
     """Locate all directories that should be searched during recall.
 
-    Returns the existing recall roots. Only ``entities/`` is canonical —
-    private entities live in ``entities/guideline/`` and shared entities
-    live in ``entities/subscribed/{repo}/guideline/``.
+    Returns the existing recall roots. Two trees contribute to recall:
+    ``entities/`` (private entities in ``entities/guideline/`` and
+    subscribed entities in ``entities/subscribed/{repo}/guideline/``) and
+    ``public/`` (entities published by the local project).
     """
     evolve_dir = get_evolve_dir()
-    candidates = [evolve_dir / "entities"]
+    candidates = [evolve_dir / "entities", evolve_dir / "public"]
     return [path for path in candidates if path.is_dir()]
 
 
@@ -224,6 +225,97 @@ def markdown_to_entity(path):
     return entity
 
 
+def _parse_frontmatter_lines(lines):
+    """Parse simple YAML-style frontmatter lines into a dict."""
+    entity = {}
+    for raw_line in lines:
+        line = raw_line.strip()
+        if not line:
+            continue
+        key, _, value = line.partition(":")
+        key = key.strip()
+        value = value.strip()
+        if key and value:
+            entity[key] = value
+    return entity
+
+
+def _parse_frontmatter_only(path):
+    """Parse only the frontmatter section from a markdown entity file."""
+    path = Path(path)
+    try:
+        with path.open(encoding="utf-8") as handle:
+            if handle.readline().strip() != "---":
+                return {}
+
+            frontmatter_lines = []
+            found_closing = False
+            for line in handle:
+                if line.strip() == "---":
+                    found_closing = True
+                    break
+                frontmatter_lines.append(line)
+    except (OSError, UnicodeDecodeError):
+        return {}
+
+    if not found_closing:
+        return {}
+
+    return _parse_frontmatter_lines(frontmatter_lines)
+
+
+def _manifest_path(path):
+    """Return a manifest path relative to the project root (parent of the evolve dir).
+
+    This keeps manifest paths stable regardless of the caller's working directory,
+    so hooks invoked from a subdirectory still emit ``.evolve/entities/...`` paths.
+    """
+    path = Path(path)
+    try:
+        project_root = get_evolve_dir().resolve().parent
+        return str(path.resolve().relative_to(project_root))
+    except ValueError:
+        return str(path)
+
+
+def dedupe_manifest_entries(entries):
+    """Return deterministically ordered manifest entries with exact dedupe."""
+    normalized = []
+    seen = set()
+    for entry in sorted(entries, key=lambda item: (item["path"], item["type"], item["trigger"])):
+        key = (entry["path"], entry["type"], entry["trigger"])
+        if key in seen:
+            continue
+        seen.add(key)
+        normalized.append(entry)
+    return normalized
+
+
+def load_manifest(root_dir):
+    """Load a frontmatter-only manifest from a recall root."""
+    root_dir = Path(root_dir)
+    entries = []
+    for md in sorted(root_dir.glob("**/*.md")):
+        if md.is_symlink() or ".git" in md.parts:
+            continue
+
+        entity = _parse_frontmatter_only(md)
+        entity_type = entity.get("type")
+        trigger = entity.get("trigger")
+        if not entity_type or not trigger:
+            continue
+
+        entries.append(
+            {
+                "path": _manifest_path(md),
+                "type": entity_type,
+                "trigger": trigger,
+            }
+        )
+
+    return dedupe_manifest_entries(entries)
+
+
 # ---------------------------------------------------------------------------
 # Bulk load / write
 # ---------------------------------------------------------------------------

platform-integrations/bob/evolve-lite/skills/evolve-lite-recall/SKILL.md

@@ -27,7 +27,7 @@ Before any non-trivial local work, you must complete the recall workflow below.
 
 Do not proceed to other analysis or tool use until all steps below are complete.
 
-1. Inspect `${EVOLVE_DIR:-.evolve}/entities/` for guidance relevant to the current task.
+1. If a manifest has already been injected for this turn, use it to pick which entity files to open. Otherwise inspect `${EVOLVE_DIR:-.evolve}/entities/` and `${EVOLVE_DIR:-.evolve}/public/` for guidance relevant to the current task.
 2. Read each matching entity file that appears relevant.
 3. Summarize the applicable guidance in your own words before proceeding.
 4. If no relevant entities exist, state that explicitly before proceeding.
@@ -41,7 +41,7 @@ Before moving on, produce an explicit completion note in your reasoning or user
 
 ### Minimum Acceptable Procedure
 
-1. List or search files under `${EVOLVE_DIR:-.evolve}/entities/`.
+1. List or search files under `${EVOLVE_DIR:-.evolve}/entities/` and `${EVOLVE_DIR:-.evolve}/public/` (or read the injected manifest if one is present).
 2. Identify candidate entities relevant to the task.
 3. Open and read those entity files.
 4. Summarize what applies, or state that nothing applies.
@@ -80,7 +80,14 @@ Entities can come from multiple sources:
         alice-guideline.md                        <- annotated [from: alice]
 ```
 
-Each file uses markdown with YAML frontmatter:
+The manifest output is human-readable:
+
+```text
+- `.evolve/entities/guideline/use-context-managers-for-file-operations.md` [guideline] — When processing files or managing resources
+- `.evolve/entities/subscribed/alice/guideline/error-handling.md` [guideline] — When writing error handlers
+```
+
+Each file still uses markdown with YAML frontmatter:
 
 ```markdown
 ---
@@ -94,3 +101,7 @@ Use context managers for file operations
 
 Ensures proper resource cleanup
 ```
+
+## On-Demand Expansion
+
+When a manifest entry's trigger matches the current task, use `read_file` to load the full entity. The file body contains the guideline content and an optional `## Rationale` section.

platform-integrations/bob/evolve-lite/skills/evolve-lite-recall/scripts/retrieve_entities.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Retrieve and output entities for the agent to use as extra context."""
+"""Retrieve and output an entity manifest for bob to expand on demand."""
 
 import json
 import os
@@ -21,7 +21,7 @@
 if _lib is None:
     raise ImportError(f"Cannot find plugin lib directory above {_script}")
 sys.path.insert(0, str(_lib))
-from entity_io import find_entities_dir, get_evolve_dir, markdown_to_entity, log as _log  # noqa: E402
+from entity_io import dedupe_manifest_entries, find_recall_entity_dirs, get_evolve_dir, load_manifest, log as _log  # noqa: E402
 import audit  # noqa: E402
 
 
@@ -33,63 +33,29 @@ def log(message):
 
 
 def format_entities(entities):
-    """Format all entities for the agent to review.
+    """Format a manifest of entities for bob to expand on demand."""
+    header = """## Evolve entity manifest for this task
 
-    Entities that came from a subscribed source have their path recorded in
-    the private ``_source`` key (set by load_entities_with_source). These are
-    annotated with ``[from: {name}]`` so the agent knows their provenance.
-    """
-    header = """## Evolve entities for this task
-
-Review these stored entities and apply any that are relevant to the user's request:
+These stored entities are available for this repo. Read only the files whose trigger looks relevant to the user's request:
 
 """
-    items = []
-    for entity in entities:
-        content = entity.get("content")
-        if not content:
-            continue
-        source = entity.get("_source")
-        if source:
-            content = f"[from: {source}] {content}"
-        item = f"- **[{entity.get('type', 'general')}]** {content}"
-        if entity.get("rationale"):
-            item += f"\n  Rationale: {entity['rationale']}"
-        if entity.get("trigger"):
-            item += f"\n  When: {entity['trigger']}"
-        items.append(item)
-
-    return header + "\n".join(items)
-
-
-def load_entities_with_source(entities_dir):
-    """Load markdown entities from one recall root and annotate subscribed content.
-
-    Symlinks and any files inside a ``.git`` directory are skipped so we don't
-    surface git's own bookkeeping or sneak past path validation when a write
-    -scope clone lives under entities/subscribed/{name}/.
-    """
-    entities_dir = Path(entities_dir)
-    entities = []
-    for md in sorted(p for p in entities_dir.glob("**/*.md") if ".git" not in p.parts):
-        if md.is_symlink():
-            continue
-        try:
-            entity = markdown_to_entity(md)
-        except (OSError, UnicodeError):
-            continue
-        if not entity.get("content"):
-            continue
+    lines = [f"- `{e['path']}` [{e['type']}] — {e['trigger']}" for e in entities]
+    return header + "\n".join(lines)
 
-        entity.pop("_source", None)
-        entity["_id"] = str(md.relative_to(entities_dir).with_suffix(""))
-        parts = md.relative_to(entities_dir).parts
-        if parts and parts[0] == "subscribed" and len(parts) > 1:
-            entity["_source"] = parts[1]
 
-        entities.append(entity)
+def _audit_id(path_str):
+    """Derive the audit entity id from a manifest path.
 
-    return entities
+    Matches upstream's convention for entities/: id is the path relative to
+    ``entities/`` with ``.md`` stripped (e.g. ``guideline/foo``,
+    ``subscribed/alice/guideline/bar``). Public entities are prefixed with
+    ``public/`` to keep the id space distinct from private entities.
+    """
+    if "/entities/" in path_str:
+        return path_str.split("/entities/", 1)[1].removesuffix(".md")
+    if "/public/" in path_str:
+        return "public/" + path_str.split("/public/", 1)[1].removesuffix(".md")
+    return path_str.removesuffix(".md")
 
 
 def main():
@@ -124,12 +90,13 @@ def main():
             log(f"  {key}={value}")
     log("=== End Environment Variables ===")
 
-    entities_dir = find_entities_dir()
-    log(f"Entities dir: {entities_dir}")
-
     entities = []
-    if entities_dir:
-        entities = load_entities_with_source(entities_dir)
+    recall_dirs = find_recall_entity_dirs()
+    log(f"Recall dirs: {recall_dirs}")
+    for root_dir in recall_dirs:
+        entities.extend(load_manifest(root_dir))
+
+    entities = dedupe_manifest_entries(entities)
 
     if not entities:
         log("No entities found")
@@ -156,7 +123,7 @@ def main():
                 session_id = stem.removeprefix("claude-transcript_")
         if not session_id and isinstance(input_data, dict) and isinstance(input_data.get("session_id"), str):
             session_id = input_data["session_id"]
-        entity_ids = sorted({entity["_id"] for entity in entities if entity.get("_id")})
+        entity_ids = sorted({_audit_id(entity["path"]) for entity in entities if entity.get("path")})
         if session_id and entity_ids:
             audit.append(
                 evolve_dir=str(get_evolve_dir().resolve()),

platform-integrations/claude/plugins/evolve-lite/lib/entity_io.py

@@ -78,12 +78,13 @@ def find_entities_dir():
 def find_recall_entity_dirs():
     """Locate all directories that should be searched during recall.
 
-    Returns the existing recall roots. Only ``entities/`` is canonical —
-    private entities live in ``entities/guideline/`` and shared entities
-    live in ``entities/subscribed/{repo}/guideline/``.
+    Returns the existing recall roots. Two trees contribute to recall:
+    ``entities/`` (private entities in ``entities/guideline/`` and
+    subscribed entities in ``entities/subscribed/{repo}/guideline/``) and
+    ``public/`` (entities published by the local project).
     """
     evolve_dir = get_evolve_dir()
-    candidates = [evolve_dir / "entities"]
+    candidates = [evolve_dir / "entities", evolve_dir / "public"]
     return [path for path in candidates if path.is_dir()]
 
 
@@ -224,6 +225,97 @@ def markdown_to_entity(path):
     return entity
 
 
+def _parse_frontmatter_lines(lines):
+    """Parse simple YAML-style frontmatter lines into a dict."""
+    entity = {}
+    for raw_line in lines:
+        line = raw_line.strip()
+        if not line:
+            continue
+        key, _, value = line.partition(":")
+        key = key.strip()
+        value = value.strip()
+        if key and value:
+            entity[key] = value
+    return entity
+
+
+def _parse_frontmatter_only(path):
+    """Parse only the frontmatter section from a markdown entity file."""
+    path = Path(path)
+    try:
+        with path.open(encoding="utf-8") as handle:
+            if handle.readline().strip() != "---":
+                return {}
+
+            frontmatter_lines = []
+            found_closing = False
+            for line in handle:
+                if line.strip() == "---":
+                    found_closing = True
+                    break
+                frontmatter_lines.append(line)
+    except (OSError, UnicodeDecodeError):
+        return {}
+
+    if not found_closing:
+        return {}
+
+    return _parse_frontmatter_lines(frontmatter_lines)
+
+
+def _manifest_path(path):
+    """Return a manifest path relative to the project root (parent of the evolve dir).
+
+    This keeps manifest paths stable regardless of the caller's working directory,
+    so hooks invoked from a subdirectory still emit ``.evolve/entities/...`` paths.
+    """
+    path = Path(path)
+    try:
+        project_root = get_evolve_dir().resolve().parent
+        return str(path.resolve().relative_to(project_root))
+    except ValueError:
+        return str(path)
+
+
+def dedupe_manifest_entries(entries):
+    """Return deterministically ordered manifest entries with exact dedupe."""
+    normalized = []
+    seen = set()
+    for entry in sorted(entries, key=lambda item: (item["path"], item["type"], item["trigger"])):
+        key = (entry["path"], entry["type"], entry["trigger"])
+        if key in seen:
+            continue
+        seen.add(key)
+        normalized.append(entry)
+    return normalized
+
+
+def load_manifest(root_dir):
+    """Load a frontmatter-only manifest from a recall root."""
+    root_dir = Path(root_dir)
+    entries = []
+    for md in sorted(root_dir.glob("**/*.md")):
+        if md.is_symlink() or ".git" in md.parts:
+            continue
+
+        entity = _parse_frontmatter_only(md)
+        entity_type = entity.get("type")
+        trigger = entity.get("trigger")
+        if not entity_type or not trigger:
+            continue
+
+        entries.append(
+            {
+                "path": _manifest_path(md),
+                "type": entity_type,
+                "trigger": trigger,
+            }
+        )
+
+    return dedupe_manifest_entries(entries)
+
+
 # ---------------------------------------------------------------------------
 # Bulk load / write
 # ---------------------------------------------------------------------------