From 88ac21fece9c1e844898a718089b244f740b388c Mon Sep 17 00:00:00 2001 From: Yan Xiao Date: Mon, 6 Jul 2026 22:36:17 +0800 Subject: [PATCH 1/4] refactor(agent): drop tool_describe, fold schema into tool_search hits Progressive tool disclosure exposed three meta-tools; tool_describe was a separate round-trip to fetch a tool's parameter schema after tool_search. Collapse to two: tool_search now returns each hit's parameter schema alongside name and description, so the model can go straight to tool_call. Also fold parameter names, descriptions and enum values into the BM25 index body so discriminating keywords that live in the schema (a repo owner, a channel id, an image size) lift recall. Lower the default search_result_limit from 10 to 5 to offset the larger per-hit payload. Co-authored-by: Claude (claude-opus-4-8) --- raven/agent/loop/main.py | 2 - raven/agent/tools/tool_index.py | 102 +++++++--- raven/agent/tools/tool_search.py | 85 +++------ raven/config/schema.py | 11 +- .../test_tool_search_disclosure_real_llm.py | 176 ++++++++++++++++++ tests/test_agent_loop_tool_search.py | 6 +- tests/test_tool_search.py | 146 +++++++++++---- 7 files changed, 394 insertions(+), 134 deletions(-) create mode 100644 tests/integration/test_tool_search_disclosure_real_llm.py diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index 0bd8b9a..3a37731 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -648,7 +648,6 @@ def _register_default_tools(self) -> None: from raven.agent.tools.tool_search import ( DEFAULT_ALWAYS_VISIBLE, ToolCallTool, - ToolDescribeTool, ToolSearchController, ToolSearchStrategy, ToolSearchTool, @@ -661,7 +660,6 @@ def _register_default_tools(self) -> None: search_result_limit=cfg.search_result_limit, ) self.tools.register(ToolSearchTool(self.tool_search_controller)) - self.tools.register(ToolDescribeTool(self.tool_search_controller)) self.tools.register(ToolCallTool(self.tool_search_controller)) # ``first=True``: filter the tool list before CacheOptimizer marks # the final tool with ``cache_control`` (else the marked tool may be diff --git a/raven/agent/tools/tool_index.py b/raven/agent/tools/tool_index.py index c7d37c7..3570a04 100644 --- a/raven/agent/tools/tool_index.py +++ b/raven/agent/tools/tool_index.py @@ -1,36 +1,48 @@ """BM25 keyword index over the agent's tool catalog. -Backs ``tool_search``: ranks tools by a query over their name and description, -reusing the shared Okapi BM25 in :mod:`raven.utils.bm25` (CJK-aware, so -Chinese queries match). The name is repeated in the indexed text so a hit on -the tool name outranks an incidental description hit — the same field-weighting -idea as the skill ``LocalPool``. - -The index rebuilds only when the catalog's ``(name, description)`` set changes -(startup + each MCP connect is one burst); steady-state ``search`` is one -query-side tokenize plus a BM25 dot product over precomputed ``doc_freqs``. +Backs ``tool_search``: ranks tools by a query over each tool's name, +description and parameter schema, reusing the shared Okapi BM25 in +:mod:`raven.utils.bm25` (CJK-aware, so Chinese queries match). The name is +repeated in the indexed text so a hit on the tool name outranks an incidental +body hit — the same field-weighting idea as the skill ``LocalPool``. Parameter +names, their descriptions and enum values are folded into the indexed body: +many tools carry the discriminating keywords (a repo owner, a channel id, an +image size) in the schema rather than the one-line description, so indexing the +schema lifts recall without touching the schema the model sees. + +The index rebuilds only when the catalog's ``(name, description, parameters)`` +set changes (startup + each MCP connect is one burst); steady-state ``search`` +is one query-side tokenize plus a BM25 dot product over precomputed +``doc_freqs``. Each ``ensure`` flattens every tool's text to compute the +signature (cheap string work); the expensive BM25 rebuild is skipped on a hit. A single process-level slot caches the most recently built index keyed by that same signature, so the many short-lived agent loops a resident process spins up over one fixed tool set reuse one prebuilt BM25 rather than each rebuilding an identical one. A single slot suffices because only the main loop feeds this index (subagents run a separate minimal tool set), so one signature dominates; -keying on description (not just name) also forces a rebuild when a tool's -description changes under hot-reload. +keying on the full indexed text (not just the name) also forces a rebuild when +a tool's description or parameters change under hot-reload. """ from __future__ import annotations import threading -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from raven.utils.bm25 import BM25Okapi, tokenize if TYPE_CHECKING: from raven.agent.tools.base import Tool -# Catalog signature: the exact inputs to the BM25 build (name + description). +# Catalog signature: one (name, indexed-text) pair per tool. The indexed text +# embeds description + parameter schema, so the signature changes whenever any +# indexed field does — the exact trigger for a rebuild. _Signature = frozenset[tuple[str, str]] + +# Cap schema-walk recursion so a pathologically nested MCP schema can't blow the +# stack or balloon the indexed text; tool schemas are shallow in practice. +_MAX_SCHEMA_DEPTH = 6 # A built index: the names list paired with the BM25 built from it (same # ordering). Read-only for searchers, so safe to share across loops. _BuiltIndex = tuple[list[str], BM25Okapi] @@ -44,22 +56,61 @@ _cached_index: _BuiltIndex | None = None +def _schema_text(parameters: "dict[str, Any] | None") -> str: + """Natural-language keywords drawn from a JSON-Schema parameter block. + + Collects property names, their ``description`` strings and enum values, + recursing through nested objects and array items. Types and structural + keywords are skipped — they carry no query signal. + """ + if not isinstance(parameters, dict): + return "" + parts: list[str] = [] + + def walk(node: Any, depth: int) -> None: + if depth > _MAX_SCHEMA_DEPTH or not isinstance(node, dict): + return + props = node.get("properties") + if isinstance(props, dict): + for key, sub in props.items(): + parts.append(str(key)) + walk(sub, depth + 1) + desc = node.get("description") + if isinstance(desc, str): + parts.append(desc) + enum = node.get("enum") + if isinstance(enum, list): + parts.extend(str(v) for v in enum if isinstance(v, (str, int, float))) + items = node.get("items") + if isinstance(items, dict): + walk(items, depth + 1) + + walk(parameters, 0) + return " ".join(parts) + + def _format_tool_text(tool: "Tool") -> str: - """Indexed text for one tool. name ×3, description ×1 — TF-based field - weighting so a query term on the name dominates a description hit.""" + """Indexed text for one tool: name ×3, then description + parameter-schema + keywords ×1 — TF-based field weighting so a query term on the name + dominates a hit in the body.""" name = tool.name - desc = tool.description or "" - return f"{name} {name} {name} {desc}" + body = f"{tool.description or ''} {_schema_text(tool.parameters)}".strip() + return f"{name} {name} {name} {body}".rstrip() -def _get_or_build(sig: _Signature, tools: "list[Tool]") -> _BuiltIndex: - """Return the shared prebuilt index for ``sig``, building it once on miss.""" +def _get_or_build(sig: _Signature, items: list[tuple[str, str]]) -> _BuiltIndex: + """Return the shared prebuilt index for ``sig``, building it once on miss. + + ``items`` are ``(name, indexed-text)`` pairs already flattened by the + caller, so the text is computed once per ``ensure`` (for the signature) and + reused here rather than recomputed. + """ global _cached_sig, _cached_index with _cache_lock: if sig == _cached_sig and _cached_index is not None: return _cached_index - names = [t.name for t in tools] - corpus = [tokenize(_format_tool_text(t)) for t in tools] + names = [name for name, _ in items] + corpus = [tokenize(text) for _, text in items] built: _BuiltIndex = (names, BM25Okapi(corpus)) with _cache_lock: _cached_sig, _cached_index = sig, built @@ -67,7 +118,7 @@ def _get_or_build(sig: _Signature, tools: "list[Tool]") -> _BuiltIndex: class ToolIndex: - """Prebuilt BM25 over a tool catalog, rebuilt on (name, description) change. + """Prebuilt BM25 over a tool catalog, rebuilt on (name, description, parameters) change. Thread-safety: a lock guards the ``(names, BM25Okapi)`` pair. The expensive build is delegated to the shared ``_get_or_build`` slot; the lock here is @@ -81,12 +132,13 @@ def __init__(self) -> None: self._bm25: BM25Okapi | None = None def ensure(self, tools: "list[Tool]") -> None: - """Adopt the shared index iff the (name, description) set changed.""" - sig: _Signature = frozenset((t.name, t.description or "") for t in tools) + """Adopt the shared index iff the (name, description, parameters) set changed.""" + items = [(t.name, _format_tool_text(t)) for t in tools] + sig: _Signature = frozenset(items) with self._lock: if sig == self._sig and self._bm25 is not None: return - names, bm25 = _get_or_build(sig, tools) + names, bm25 = _get_or_build(sig, items) with self._lock: self._names, self._bm25, self._sig = names, bm25, sig diff --git a/raven/agent/tools/tool_search.py b/raven/agent/tools/tool_search.py index ee5e08a..13b8c71 100644 --- a/raven/agent/tools/tool_search.py +++ b/raven/agent/tools/tool_search.py @@ -2,17 +2,17 @@ When built-ins + plugins + MCP push the tool count past a threshold, injecting every schema into each request burns context that scales with tool count. This -module withholds most schemas and exposes three meta-tools instead: +module withholds most schemas and exposes two meta-tools instead: - - ``tool_search`` — BM25 keyword search over the hidden catalog; compact - hits (name + description, no parameters). - - ``tool_describe`` — one tool's full parameter schema (so the model knows - the argument shape before calling it). - - ``tool_call`` — invoke a cataloged tool by name; forwards through the - registry, which still validates arguments. + - ``tool_search`` — BM25 keyword search over the hidden catalog; each hit + carries name + description + parameter schema, enough to + call the tool without a second lookup. + - ``tool_call`` — invoke a cataloged tool by name; forwards through the + registry, which validates arguments and returns a + correctable error when they don't fit the schema. The tool list sent to the model never changes turn-to-turn (always the core -set + these three meta-tools), so the prompt cache stays stable across the +set + these two meta-tools), so the prompt cache stays stable across the whole session — tools sit ahead of system+messages in the cached prefix, so a changing tool list would invalidate everything after it. The cost is that cataloged tools are invoked through ``tool_call`` rather than native @@ -56,7 +56,7 @@ TOOL_CALL_NAME: str = "tool_call" # The meta-tools: always registered when the feature is on, never cataloged. -META_TOOL_NAMES: frozenset[str] = frozenset({"tool_search", "tool_describe", TOOL_CALL_NAME}) +META_TOOL_NAMES: frozenset[str] = frozenset({"tool_search", TOOL_CALL_NAME}) class ToolSearchController: @@ -72,7 +72,7 @@ def __init__( registry: "ToolRegistry", *, always_visible: set[str], - search_result_limit: int = 10, + search_result_limit: int = 5, ) -> None: self._registry = registry self.always_visible = set(always_visible) | META_TOOL_NAMES @@ -98,23 +98,23 @@ def visible_names(self) -> set[str]: return self.always_visible def search(self, query: str, limit: int | None = None) -> list[dict[str, Any]]: - """Compact hits: name + description, never parameters.""" + """Hits carry name + description + parameter schema, so the model can go + straight to tool_call without a separate describe round-trip.""" names = self._index.search(query, limit or self.search_result_limit) hits = [] for name in names: tool = self._registry.get(name) if tool is None: continue - hits.append({"name": name, "description": tool.description}) + hits.append( + { + "name": name, + "description": tool.description, + "parameters": tool.parameters, + } + ) return hits - def describe(self, name: str) -> dict[str, Any] | None: - """Return the tool's full function schema (for argument shape).""" - tool = self._registry.get(name) - if tool is None or name in META_TOOL_NAMES: - return None - return tool.to_schema()["function"] - async def call(self, name: str, arguments: dict[str, Any] | None) -> str: """Invoke a cataloged tool: forward to the registry (validates args). @@ -147,10 +147,9 @@ def name(self) -> str: def description(self) -> str: return ( "Search the catalog of additional tools that are available but not " - "currently loaded. Returns matching tool names with a short " - "description (no parameters). Use tool_describe to see a tool's " - "arguments, then tool_call to invoke it. Query with task keywords, " - "e.g. 'create github issue' or '生成图片'." + "currently loaded. Returns matching tools with their description and " + "parameter schema, ready to invoke with tool_call. Query with task " + "keywords, e.g. 'create github issue' or '生成图片'." ) @property @@ -179,43 +178,6 @@ async def execute(self, query: str, limit: int | None = None) -> str: return json.dumps(hits, ensure_ascii=False) -class ToolDescribeTool(Tool): - """Return a cataloged tool's full parameter schema.""" - - def __init__(self, controller: ToolSearchController) -> None: - self._ctrl = controller - - @property - def name(self) -> str: - return "tool_describe" - - @property - def description(self) -> str: - return ( - "Show the full parameter schema of a tool found via tool_search, so " - "you know its arguments. Invoke the tool afterwards with tool_call." - ) - - @property - def parameters(self) -> dict[str, Any]: - return { - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "Exact tool name from a tool_search result.", - }, - }, - "required": ["name"], - } - - async def execute(self, name: str) -> str: - schema = self._ctrl.describe(name) - if schema is None: - return f"Tool '{name}' not found. Use tool_search to find available tools." - return json.dumps(schema, ensure_ascii=False) - - class ToolCallTool(Tool): """Invoke a cataloged tool by name. Arguments are validated by the registry.""" @@ -230,7 +192,8 @@ def name(self) -> str: def description(self) -> str: return ( "Invoke a tool found via tool_search by name, passing its arguments. " - "Use tool_describe first to learn the argument shape." + "If the arguments don't fit the tool's schema the registry returns a " + "validation error describing the fix; adjust and call again." ) @property diff --git a/raven/config/schema.py b/raven/config/schema.py index 790fee1..97a289f 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -504,18 +504,17 @@ class ToolSearchConfig(Base): When the live tool catalog (built-ins + plugins + MCP) grows past ``compaction_threshold``, most tool schemas are withheld from each request and reached - on demand through the ``tool_search`` / ``tool_describe`` / ``tool_call`` - meta-tools, so context cost stops scaling with tool count and the per-turn - tool list (and thus the prompt cache) stays stable. At or below the - threshold every tool is exposed directly (unchanged behavior) and the - meta-tools are omitted. + on demand through the ``tool_search`` / ``tool_call`` meta-tools, so context + cost stops scaling with tool count and the per-turn tool list (and thus the + prompt cache) stays stable. At or below the threshold every tool is exposed + directly (unchanged behavior) and the meta-tools are omitted. """ enabled: bool = False compaction_threshold: int = 50 """Tool-catalog size that triggers compaction: at or below this many tools everything is exposed directly; above it, schemas are withheld.""" - search_result_limit: int = 10 + search_result_limit: int = 5 """Default number of hits ``tool_search`` returns per query.""" always_visible: list[str] = Field(default_factory=list) """Extra tool names kept exposed every turn, on top of the core set.""" diff --git a/tests/integration/test_tool_search_disclosure_real_llm.py b/tests/integration/test_tool_search_disclosure_real_llm.py new file mode 100644 index 0000000..add1ea0 --- /dev/null +++ b/tests/integration/test_tool_search_disclosure_real_llm.py @@ -0,0 +1,176 @@ +"""Real-LLM e2e for progressive tool disclosure (search -> call, no describe). + +Drives a real ``AgentLoop`` against OpenRouter (deepseek-v3.2). With the +catalog compacted, a sentinel domain tool (``convert_currency``) is hidden; +the model must ``tool_search`` to find it and ``tool_call`` to invoke it. Since +``tool_search`` now returns each hit's parameter schema and ``tool_describe`` +no longer exists, a correct call proves the model used the schema carried in +the search result directly. + +Skips unless an OpenRouter key is configured in ``~/.raven/config.json`` or +``~/.everclaw/config.json`` (providers.openrouter.apiKey). + + uv run pytest tests/integration/test_tool_search_disclosure_real_llm.py -m real_llm -s +""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path +from typing import Any + +import pytest + +from raven.agent.loop import AgentLoop +from raven.agent.tools.base import Tool +from raven.config.schema import ToolSearchConfig +from raven.providers.litellm_provider import LiteLLMProvider + +MODEL = "openrouter/deepseek/deepseek-v3.2-exp" + + +def _load_openrouter_key() -> str | None: + for name in ("~/.raven/config.json", "~/.everclaw/config.json"): + p = Path(name).expanduser() + if not p.exists(): + continue + try: + cfg = json.loads(p.read_text(encoding="utf-8")) + except (ValueError, OSError): + continue + key = (cfg.get("providers", {}).get("openrouter", {}) or {}).get("apiKey") + if key and str(key).startswith("sk-or-"): + return key + return None + + +class _FakeDomainTool(Tool): + """A no-op cataloged tool used to pad the catalog past the threshold.""" + + def __init__(self, name: str, description: str) -> None: + self._name = name + self._description = description + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str: + return self._description + + @property + def parameters(self) -> dict[str, Any]: + return {"type": "object", "properties": {}} + + async def execute(self, **kwargs: Any) -> str: + return f"{self._name} ok" + + +class _ConvertCurrencyTool(Tool): + """Sentinel: records the args it was invoked with.""" + + def __init__(self, calls: list[dict[str, Any]]) -> None: + self._calls = calls + + @property + def name(self) -> str: + return "convert_currency" + + @property + def description(self) -> str: + return "Convert a monetary amount from one currency to another." + + @property + def parameters(self) -> dict[str, Any]: + return { + "type": "object", + "properties": { + "amount": {"type": "number", "description": "Amount to convert."}, + "from_currency": {"type": "string", "description": "ISO code to convert from, e.g. USD."}, + "to_currency": {"type": "string", "description": "ISO code to convert to, e.g. JPY."}, + }, + "required": ["amount", "from_currency", "to_currency"], + } + + async def execute(self, **kwargs: Any) -> str: + self._calls.append(kwargs) + return "15000 JPY" + + +DISTRACTORS = [ + ("send_slack_message", "Post a message to a Slack channel."), + ("create_github_issue", "Open a new issue in a GitHub repository."), + ("generate_image", "Generate an image from a text prompt."), + ("query_database", "Run a read-only SQL query against the warehouse."), + ("book_flight", "Book a flight given origin, destination and date."), + ("translate_text", "Translate text between languages."), + ("summarize_document", "Summarize a long document."), + ("fetch_weather", "Get the weather forecast for a city."), + ("schedule_meeting", "Schedule a calendar meeting."), + ("transcribe_audio", "Transcribe an audio file to text."), + ("resize_photo", "Resize a photo to given dimensions."), + ("send_email", "Send an email to a recipient."), + ("crawl_webpage", "Fetch and extract text from a web page."), + ("plot_chart", "Render a chart from a data series."), + ("detect_language", "Detect the language of a text."), +] + + +@pytest.mark.real_llm +@pytest.mark.asyncio +async def test_search_then_call_with_returned_schema_real_llm() -> None: + key = _load_openrouter_key() + if not key: + pytest.skip("OpenRouter key not configured (providers.openrouter.apiKey)") + + provider = LiteLLMProvider( + api_key=key, + api_base="https://openrouter.ai/api/v1", + default_model=MODEL, + provider_name="openrouter", + ) + + calls: list[dict[str, Any]] = [] + with tempfile.TemporaryDirectory() as td: + loop = AgentLoop( + provider=provider, + workspace=Path(td), + model=MODEL, + max_iterations=6, + restrict_to_workspace=True, + tool_search_config=ToolSearchConfig(enabled=True, compaction_threshold=10), + ) + for nm, desc in DISTRACTORS: + loop.tools.register(_FakeDomainTool(nm, desc)) + loop.tools.register(_ConvertCurrencyTool(calls)) + + events: list[tuple[str, str]] = [] + + async def on_tool_event(phase: str, payload: dict) -> None: + events.append((phase, payload.get("name", ""))) + + task = ( + "I need to convert 100 USD to Japanese yen. You do not have a currency " + "tool loaded — search the tool catalog for one, then use it. Amount is 100, " + "from USD, to JPY." + ) + final, tools_used, _messages, _outcome = await loop._run_agent_loop( + [{"role": "user", "content": task}], + on_tool_event=on_tool_event, + ) + + print("\ntools_used:", tools_used) + print("tool events:", events) + print("convert_currency calls:", calls) + print("final:", final) + + assert "tool_search" in tools_used, "model should search the hidden catalog" + assert "tool_call" in tools_used, "model should invoke the found tool via tool_call" + assert "tool_describe" not in tools_used, "tool_describe was removed" + assert calls, "the sentinel convert_currency tool must have been executed" + args = calls[0] + assert str(args.get("from_currency", "")).upper() == "USD" + assert str(args.get("to_currency", "")).upper() == "JPY" + assert float(args.get("amount")) == 100 diff --git a/tests/test_agent_loop_tool_search.py b/tests/test_agent_loop_tool_search.py index ee0fff3..10529df 100644 --- a/tests/test_agent_loop_tool_search.py +++ b/tests/test_agent_loop_tool_search.py @@ -66,7 +66,7 @@ def _make_loop(workspace: Path, cfg, strategies=None) -> AgentLoop: def test_enabled_registers_meta_tools(workspace) -> None: loop = _make_loop(workspace, ToolSearchConfig(enabled=True)) - for name in ("tool_search", "tool_describe", "tool_call"): + for name in ("tool_search", "tool_call"): assert loop.tools.has(name), f"{name} should be registered" assert loop.strategies.get("tool_search") is not None @@ -83,7 +83,7 @@ def test_strategy_registered_first(workspace) -> None: def test_disabled_registers_nothing(workspace) -> None: loop = _make_loop(workspace, ToolSearchConfig(enabled=False)) - for name in ("tool_search", "tool_describe", "tool_call"): + for name in ("tool_search", "tool_call"): assert not loop.tools.has(name) assert loop.strategies.get("tool_search") is None @@ -106,5 +106,5 @@ async def test_enabled_loop_keeps_interaction_primitives_visible(workspace) -> N _, out, _ = await loop.strategies.before_llm_call([], tools, "stub") names = {t["function"]["name"] for t in out} assert {"read_file", "message", "ask_user", "spawn"} <= names, "primitives must stay visible" - assert {"tool_search", "tool_describe", "tool_call"} <= names, "meta-tools must stay visible" + assert {"tool_search", "tool_call"} <= names, "meta-tools must stay visible" assert "web_search" not in names, "cataloged domain tools are withheld above threshold" diff --git a/tests/test_tool_search.py b/tests/test_tool_search.py index 58547c9..18003bf 100644 --- a/tests/test_tool_search.py +++ b/tests/test_tool_search.py @@ -7,23 +7,29 @@ from raven.agent.tools.base import Tool from raven.agent.tools.registry import ToolRegistry -from raven.agent.tools.tool_index import ToolIndex +from raven.agent.tools.tool_index import ToolIndex, _schema_text from raven.agent.tools.tool_search import ( DEFAULT_ALWAYS_VISIBLE, META_TOOL_NAMES, TOOL_CALL_NAME, ToolCallTool, - ToolDescribeTool, ToolSearchController, ToolSearchStrategy, ToolSearchTool, ) +from raven.config.schema import ToolSearchConfig class _FakeTool(Tool): - def __init__(self, name: str, description: str) -> None: + def __init__( + self, + name: str, + description: str, + parameters: dict[str, Any] | None = None, + ) -> None: self._name = name self._description = description + self._parameters = parameters or {"type": "object", "properties": {}} @property def name(self) -> str: @@ -35,7 +41,7 @@ def description(self) -> str: @property def parameters(self) -> dict[str, Any]: - return {"type": "object", "properties": {}} + return self._parameters async def execute(self, **kwargs: Any) -> str: return f"ran {self._name}" @@ -80,6 +86,82 @@ def test_index_rebuilds_on_name_or_description_change() -> None: assert idx._bm25 is not first, "should rebuild when the name set changes" +def test_index_matches_parameter_schema_keywords() -> None: + # A discriminating keyword living only in the parameter schema (not the + # one-line description) should still make the tool findable. + idx = ToolIndex() + tools = [ + _FakeTool( + "create_issue", + "open a ticket", + parameters={ + "type": "object", + "properties": { + "repository": {"type": "string", "description": "target github repository"}, + }, + }, + ), + _FakeTool("send_message", "post to a channel"), + ] + idx.ensure(tools) + assert idx.search("github repository", limit=5)[0] == "create_issue" + + +def test_schema_text_extracts_names_descriptions_enums_and_nesting() -> None: + schema = { + "type": "object", + "properties": { + "channel": {"type": "string", "description": "slack channel id"}, + "mode": {"type": "string", "enum": ["fast", "thorough"]}, + "opts": { + "type": "object", + "properties": {"retries": {"type": "integer", "description": "retry count"}}, + }, + "tags": {"type": "array", "items": {"type": "string", "description": "a label"}}, + }, + } + text = _schema_text(schema) + for token in ( + "channel", + "slack channel id", + "mode", + "fast", + "thorough", + "opts", + "retries", + "retry count", + "tags", + "a label", + ): + assert token in text, f"{token!r} missing from schema text" + + +def test_schema_text_handles_non_dict_and_empty() -> None: + assert _schema_text(None) == "" + assert _schema_text([1, 2]) == "" # type: ignore[arg-type] + assert _schema_text({"type": "object"}) == "" # no properties/desc/enum + + +def test_schema_text_respects_depth_cap() -> None: + # Build nesting deeper than the cap; the deepest description must be dropped. + node: dict[str, Any] = {"type": "string", "description": "TOODEEP"} + for _ in range(10): + node = {"type": "object", "properties": {"n": node}} + assert "TOODEEP" not in _schema_text(node) + + +def test_index_rebuilds_on_parameters_change() -> None: + base = {"type": "object", "properties": {"a": {"type": "string", "description": "alpha"}}} + idx = ToolIndex() + idx.ensure([_FakeTool("t", "desc", parameters=base)]) + first = idx._bm25 + idx.ensure([_FakeTool("t", "desc", parameters=dict(base))]) # same schema content + assert idx._bm25 is first, "should not rebuild when parameters are unchanged" + changed = {"type": "object", "properties": {"a": {"type": "string", "description": "beta"}}} + idx.ensure([_FakeTool("t", "desc", parameters=changed)]) # param description changed + assert idx._bm25 is not first, "should rebuild when a parameter description changes" + + def test_index_reused_across_instances_for_same_catalog() -> None: tools = [_FakeTool("x", "xray"), _FakeTool("y", "yankee")] first = ToolIndex() @@ -100,35 +182,34 @@ def _controller(registry: ToolRegistry) -> ToolSearchController: ) -def test_controller_search_compact_has_no_parameters() -> None: +def test_controller_search_includes_parameters() -> None: + schema = { + "type": "object", + "properties": {"repo": {"type": "string", "description": "target repository"}}, + "required": ["repo"], + } reg = ToolRegistry() - reg.register(_FakeTool("create_issue", "open a github issue")) + reg.register(_FakeTool("create_issue", "open a github issue", parameters=schema)) ctrl = _controller(reg) ctrl.refresh() hits = ctrl.search("github issue") assert hits[0]["name"] == "create_issue" - assert "parameters" not in hits[0] - - -def test_controller_describe_returns_full_schema() -> None: - reg = ToolRegistry() - reg.register(_FakeTool("create_issue", "open a github issue")) - ctrl = _controller(reg) - fn = ctrl.describe("create_issue") - assert fn is not None and fn["name"] == "create_issue" and "parameters" in fn + assert hits[0]["parameters"] == schema -def test_controller_describe_unknown_returns_none() -> None: - ctrl = _controller(ToolRegistry()) - assert ctrl.describe("nope") is None +def test_meta_includes_tool_call() -> None: + assert TOOL_CALL_NAME in META_TOOL_NAMES -def test_controller_describe_rejects_meta() -> None: - assert _controller(ToolRegistry()).describe("tool_search") is None +def test_meta_no_longer_includes_describe() -> None: + assert "tool_describe" not in META_TOOL_NAMES + assert META_TOOL_NAMES == {"tool_search", TOOL_CALL_NAME} -def test_meta_includes_tool_call() -> None: - assert TOOL_CALL_NAME in META_TOOL_NAMES +def test_default_search_result_limit_is_five() -> None: + assert ToolSearchConfig().search_result_limit == 5 + ctrl = ToolSearchController(ToolRegistry(), always_visible=set()) + assert ctrl.search_result_limit == 5 def test_default_always_visible_covers_core_and_interaction_primitives() -> None: @@ -159,7 +240,6 @@ def test_meta_tools_not_self_searchable() -> None: reg = ToolRegistry() ctrl = _controller(reg) reg.register(ToolSearchTool(ctrl)) - reg.register(ToolDescribeTool(ctrl)) reg.register(ToolCallTool(ctrl)) reg.register(_FakeTool("create_issue", "open a github issue")) ctrl.refresh() @@ -171,13 +251,16 @@ def test_meta_tools_not_self_searchable() -> None: @pytest.mark.asyncio -async def test_tool_search_tool_returns_json_hits() -> None: +async def test_tool_search_tool_returns_json_hits_with_parameters() -> None: + schema = {"type": "object", "properties": {"repo": {"type": "string"}}} reg = ToolRegistry() - reg.register(_FakeTool("create_issue", "open a github issue")) + reg.register(_FakeTool("create_issue", "open a github issue", parameters=schema)) ctrl = _controller(reg) ctrl.refresh() out = await ToolSearchTool(ctrl).execute(query="github issue") - assert json.loads(out)[0]["name"] == "create_issue" + hit = json.loads(out)[0] + assert hit["name"] == "create_issue" + assert hit["parameters"] == schema @pytest.mark.asyncio @@ -190,15 +273,6 @@ async def test_tool_search_tool_no_match_message() -> None: assert "No tools matched" in out -@pytest.mark.asyncio -async def test_tool_describe_tool_returns_schema() -> None: - reg = ToolRegistry() - reg.register(_FakeTool("create_issue", "open a github issue")) - ctrl = _controller(reg) - out = await ToolDescribeTool(ctrl).execute(name="create_issue") - assert json.loads(out)["name"] == "create_issue" - - @pytest.mark.asyncio async def test_tool_call_forwards_to_registry() -> None: reg = ToolRegistry() @@ -244,7 +318,6 @@ def _registry_with_n(n: int) -> tuple[ToolRegistry, ToolSearchController]: search_result_limit=10, ) reg.register(ToolSearchTool(ctrl)) - reg.register(ToolDescribeTool(ctrl)) reg.register(ToolCallTool(ctrl)) return reg, ctrl @@ -311,7 +384,6 @@ async def test_strategy_passthrough_when_meta_tools_absent() -> None: # expose everything rather than strand cataloged tools. reg, ctrl = _registry_with_n(40) reg.unregister("tool_search") - reg.unregister("tool_describe") reg.unregister(TOOL_CALL_NAME) strat = ToolSearchStrategy(ctrl, compaction_threshold=25) tools = reg.get_definitions() From a131c2b5f5f8b74c9c61c71f5d801168f9907575 Mon Sep 17 00:00:00 2001 From: Yan Xiao Date: Mon, 6 Jul 2026 22:44:41 +0800 Subject: [PATCH 2/4] refactor(agent): restore default search_result_limit to 10 The prior commit lowered tool_search's default result count to 5 to offset the larger per-hit payload from folding parameter schemas into hits. Restore it to 10: recall from more candidates outweighs the marginal payload, and a caller can still cap results per query via the limit argument. Co-authored-by: Claude (claude-opus-4-8) --- raven/agent/tools/tool_search.py | 2 +- raven/config/schema.py | 2 +- tests/test_tool_search.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/raven/agent/tools/tool_search.py b/raven/agent/tools/tool_search.py index 13b8c71..9f09300 100644 --- a/raven/agent/tools/tool_search.py +++ b/raven/agent/tools/tool_search.py @@ -72,7 +72,7 @@ def __init__( registry: "ToolRegistry", *, always_visible: set[str], - search_result_limit: int = 5, + search_result_limit: int = 10, ) -> None: self._registry = registry self.always_visible = set(always_visible) | META_TOOL_NAMES diff --git a/raven/config/schema.py b/raven/config/schema.py index 97a289f..fb2fd65 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -514,7 +514,7 @@ class ToolSearchConfig(Base): compaction_threshold: int = 50 """Tool-catalog size that triggers compaction: at or below this many tools everything is exposed directly; above it, schemas are withheld.""" - search_result_limit: int = 5 + search_result_limit: int = 10 """Default number of hits ``tool_search`` returns per query.""" always_visible: list[str] = Field(default_factory=list) """Extra tool names kept exposed every turn, on top of the core set.""" diff --git a/tests/test_tool_search.py b/tests/test_tool_search.py index 18003bf..dfd81b0 100644 --- a/tests/test_tool_search.py +++ b/tests/test_tool_search.py @@ -206,10 +206,10 @@ def test_meta_no_longer_includes_describe() -> None: assert META_TOOL_NAMES == {"tool_search", TOOL_CALL_NAME} -def test_default_search_result_limit_is_five() -> None: - assert ToolSearchConfig().search_result_limit == 5 +def test_default_search_result_limit_is_ten() -> None: + assert ToolSearchConfig().search_result_limit == 10 ctrl = ToolSearchController(ToolRegistry(), always_visible=set()) - assert ctrl.search_result_limit == 5 + assert ctrl.search_result_limit == 10 def test_default_always_visible_covers_core_and_interaction_primitives() -> None: From c8ace5270900156b764b39fa7972361ce279cefd Mon Sep 17 00:00:00 2001 From: Yan Xiao Date: Mon, 6 Jul 2026 22:44:55 +0800 Subject: [PATCH 3/4] test(agent): drop non-deterministic real-llm tool-search e2e The OpenRouter-backed disclosure e2e needs a live LLM and network, so it cannot run as a deterministic unit test. The search -> call flow (hits carry parameter schemas, tool_describe removed) stays covered by the deterministic cases in tests/test_tool_search.py. Co-authored-by: Claude (claude-opus-4-8) --- .../test_tool_search_disclosure_real_llm.py | 176 ------------------ 1 file changed, 176 deletions(-) delete mode 100644 tests/integration/test_tool_search_disclosure_real_llm.py diff --git a/tests/integration/test_tool_search_disclosure_real_llm.py b/tests/integration/test_tool_search_disclosure_real_llm.py deleted file mode 100644 index add1ea0..0000000 --- a/tests/integration/test_tool_search_disclosure_real_llm.py +++ /dev/null @@ -1,176 +0,0 @@ -"""Real-LLM e2e for progressive tool disclosure (search -> call, no describe). - -Drives a real ``AgentLoop`` against OpenRouter (deepseek-v3.2). With the -catalog compacted, a sentinel domain tool (``convert_currency``) is hidden; -the model must ``tool_search`` to find it and ``tool_call`` to invoke it. Since -``tool_search`` now returns each hit's parameter schema and ``tool_describe`` -no longer exists, a correct call proves the model used the schema carried in -the search result directly. - -Skips unless an OpenRouter key is configured in ``~/.raven/config.json`` or -``~/.everclaw/config.json`` (providers.openrouter.apiKey). - - uv run pytest tests/integration/test_tool_search_disclosure_real_llm.py -m real_llm -s -""" - -from __future__ import annotations - -import json -import tempfile -from pathlib import Path -from typing import Any - -import pytest - -from raven.agent.loop import AgentLoop -from raven.agent.tools.base import Tool -from raven.config.schema import ToolSearchConfig -from raven.providers.litellm_provider import LiteLLMProvider - -MODEL = "openrouter/deepseek/deepseek-v3.2-exp" - - -def _load_openrouter_key() -> str | None: - for name in ("~/.raven/config.json", "~/.everclaw/config.json"): - p = Path(name).expanduser() - if not p.exists(): - continue - try: - cfg = json.loads(p.read_text(encoding="utf-8")) - except (ValueError, OSError): - continue - key = (cfg.get("providers", {}).get("openrouter", {}) or {}).get("apiKey") - if key and str(key).startswith("sk-or-"): - return key - return None - - -class _FakeDomainTool(Tool): - """A no-op cataloged tool used to pad the catalog past the threshold.""" - - def __init__(self, name: str, description: str) -> None: - self._name = name - self._description = description - - @property - def name(self) -> str: - return self._name - - @property - def description(self) -> str: - return self._description - - @property - def parameters(self) -> dict[str, Any]: - return {"type": "object", "properties": {}} - - async def execute(self, **kwargs: Any) -> str: - return f"{self._name} ok" - - -class _ConvertCurrencyTool(Tool): - """Sentinel: records the args it was invoked with.""" - - def __init__(self, calls: list[dict[str, Any]]) -> None: - self._calls = calls - - @property - def name(self) -> str: - return "convert_currency" - - @property - def description(self) -> str: - return "Convert a monetary amount from one currency to another." - - @property - def parameters(self) -> dict[str, Any]: - return { - "type": "object", - "properties": { - "amount": {"type": "number", "description": "Amount to convert."}, - "from_currency": {"type": "string", "description": "ISO code to convert from, e.g. USD."}, - "to_currency": {"type": "string", "description": "ISO code to convert to, e.g. JPY."}, - }, - "required": ["amount", "from_currency", "to_currency"], - } - - async def execute(self, **kwargs: Any) -> str: - self._calls.append(kwargs) - return "15000 JPY" - - -DISTRACTORS = [ - ("send_slack_message", "Post a message to a Slack channel."), - ("create_github_issue", "Open a new issue in a GitHub repository."), - ("generate_image", "Generate an image from a text prompt."), - ("query_database", "Run a read-only SQL query against the warehouse."), - ("book_flight", "Book a flight given origin, destination and date."), - ("translate_text", "Translate text between languages."), - ("summarize_document", "Summarize a long document."), - ("fetch_weather", "Get the weather forecast for a city."), - ("schedule_meeting", "Schedule a calendar meeting."), - ("transcribe_audio", "Transcribe an audio file to text."), - ("resize_photo", "Resize a photo to given dimensions."), - ("send_email", "Send an email to a recipient."), - ("crawl_webpage", "Fetch and extract text from a web page."), - ("plot_chart", "Render a chart from a data series."), - ("detect_language", "Detect the language of a text."), -] - - -@pytest.mark.real_llm -@pytest.mark.asyncio -async def test_search_then_call_with_returned_schema_real_llm() -> None: - key = _load_openrouter_key() - if not key: - pytest.skip("OpenRouter key not configured (providers.openrouter.apiKey)") - - provider = LiteLLMProvider( - api_key=key, - api_base="https://openrouter.ai/api/v1", - default_model=MODEL, - provider_name="openrouter", - ) - - calls: list[dict[str, Any]] = [] - with tempfile.TemporaryDirectory() as td: - loop = AgentLoop( - provider=provider, - workspace=Path(td), - model=MODEL, - max_iterations=6, - restrict_to_workspace=True, - tool_search_config=ToolSearchConfig(enabled=True, compaction_threshold=10), - ) - for nm, desc in DISTRACTORS: - loop.tools.register(_FakeDomainTool(nm, desc)) - loop.tools.register(_ConvertCurrencyTool(calls)) - - events: list[tuple[str, str]] = [] - - async def on_tool_event(phase: str, payload: dict) -> None: - events.append((phase, payload.get("name", ""))) - - task = ( - "I need to convert 100 USD to Japanese yen. You do not have a currency " - "tool loaded — search the tool catalog for one, then use it. Amount is 100, " - "from USD, to JPY." - ) - final, tools_used, _messages, _outcome = await loop._run_agent_loop( - [{"role": "user", "content": task}], - on_tool_event=on_tool_event, - ) - - print("\ntools_used:", tools_used) - print("tool events:", events) - print("convert_currency calls:", calls) - print("final:", final) - - assert "tool_search" in tools_used, "model should search the hidden catalog" - assert "tool_call" in tools_used, "model should invoke the found tool via tool_call" - assert "tool_describe" not in tools_used, "tool_describe was removed" - assert calls, "the sentinel convert_currency tool must have been executed" - args = calls[0] - assert str(args.get("from_currency", "")).upper() == "USD" - assert str(args.get("to_currency", "")).upper() == "JPY" - assert float(args.get("amount")) == 100 From cbd67d1bb75fed3c889eae1c4d134cb491f6c84b Mon Sep 17 00:00:00 2001 From: Yan Xiao Date: Mon, 6 Jul 2026 23:32:55 +0800 Subject: [PATCH 4/4] refactor(agent): cap tool_search limit at 10 to match default Align the JSON schema maximum for the tool_search limit param with the default search_result_limit of 10, so the model cannot request more results than the configured default. Co-authored-by: Claude (claude-opus-4-8) --- raven/agent/tools/tool_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raven/agent/tools/tool_search.py b/raven/agent/tools/tool_search.py index 9f09300..7e50deb 100644 --- a/raven/agent/tools/tool_search.py +++ b/raven/agent/tools/tool_search.py @@ -165,7 +165,7 @@ def parameters(self) -> dict[str, Any]: "type": "integer", "description": "Max number of results.", "minimum": 1, - "maximum": 25, + "maximum": 10, }, }, "required": ["query"],