feat: add a tool that allows the agent to query the location of the text on the screen.

CLAUDE.md

@@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 
 ## Project Overview
 
-Windows-MCP is a Python MCP (Model Context Protocol) server that bridges AI LLM agents with the Windows OS, enabling direct desktop automation. It exposes 15 tools (App, Shell, Snapshot, Click, Type, Scroll, Move, Shortcut, Wait, Scrape, MultiSelect, MultiEdit, Clipboard, Process, Notification) via FastMCP.
+Windows-MCP is a Python MCP (Model Context Protocol) server that bridges AI LLM agents with the Windows OS, enabling direct desktop automation. It exposes 16 tools (App, Shell, Snapshot, Click, Type, Scroll, Move, Shortcut, Wait, Scrape, MultiSelect, MultiEdit, Clipboard, Process, Notification, LocateText) via FastMCP.
 
 ## Build & Development Commands
 

manifest.json

@@ -131,6 +131,10 @@
     {
       "name": "Registry",
       "description": "Accesses the Windows Registry. Use mode=\"get\" to read a value, mode=\"set\" to create/update a value, mode=\"delete\" to remove a value or key, mode=\"list\" to list values and sub-keys under a path."
+    },
+    {
+      "name": "LocateText",
+      "description": "Finds center [x, y], bounding box [left, top, width, height], and id for text. Set use_vision=True to return an annotated screenshot for verification. Use region_hint (\"top\", \"bottom\", \"left\", \"right\", \"center\") to narrow search; defaults to 'all'. Returns a list of match objects. Use occurrence_index with a specific id to select a targeted result."
     }
   ],
   "compatibility": {

pyproject.toml

@@ -36,6 +36,15 @@ dependencies = [
     "tabulate>=0.9.0",
     "thefuzz>=0.22.1",
     "uuid7>=0.1.0",
+    "winrt-runtime>=3.2.1",
+    "winrt-windows-foundation>=3.2.1",
+    "winrt-windows-foundation-collections>=3.2.1",
+    "winrt-windows-globalization>=3.2.1",
+    "winrt-windows-graphics-imaging>=3.2.1",
+    "winrt-windows-media-ocr>=3.2.1",
+    "winrt-windows-security-cryptography>=3.2.1",
+    "winrt-windows-storage>=3.2.1",
+    "winrt-windows-storage-streams>=3.2.1",
 ]
 
 [project.optional-dependencies]
@@ -74,3 +83,9 @@ ignore = ["E501"]
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 asyncio_mode = "auto"
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.2",
+    "pytest-asyncio>=1.3.0",
+]

src/windows_mcp/__main__.py

@@ -187,6 +187,7 @@ def file_system_tool(
         openWorldHint=False,
     ),
 )
+
 @with_analytics(analytics, "State-Tool")
 def state_tool(
     use_vision: bool | str = False,
@@ -745,6 +746,40 @@ def registry_tool(mode: Literal['get', 'set', 'delete', 'list'], path: str, name
     except Exception as e:
         return f'Error accessing registry: {str(e)}'
 
+
+@mcp.tool(
+    name="LocateText",
+    description="Finds center bounding box for text. Keywords: text_query, use_vision, region_hint, occurrence_index; Set use_vision=True to return an annotated screenshot for verification. Use region_hint (\"top\", \"bottom\", \"left\", \"right\", \"center\") to narrow search; defaults to 'all'. Returns a list of match objects. Use occurrence_index with a specific id to select a targeted result.",
+    annotations=ToolAnnotations(
+        title="Locate Text",
+        readOnlyHint=True,
+        destructiveHint=False,
+        idempotentHint=True,
+        openWorldHint=False,
+    ),
+)
+@with_analytics(analytics, "Locate-Text-Tool")
+async def locate_text_tool(
+    text_query: str,
+    use_vision: bool | str = False,
+    region_hint: Literal["all", "top", "bottom", "left", "right", "center"] = "all",
+    occurrence_index: int | None = None,
+    ctx: Context = None,
+):
+    try:
+        from windows_mcp.desktop.locate_text import locate_text_tool
+        return await locate_text_tool(
+            desktop=desktop,
+            use_vision=use_vision,
+            text_query=text_query,
+            region_hint=region_hint,
+            occurrence_index=occurrence_index
+        )
+    except Exception as e:
+        import traceback
+        logger.error(f"locate_text error: {str(e)}\n{traceback.format_exc()}")
+        return [f"Error in locate_text: {str(e)}"]
+
 class Transport(Enum):
     STDIO = "stdio"
     SSE = "sse"

src/windows_mcp/analytics.py

@@ -104,7 +104,11 @@ async def track_error(self, error: Exception, context: Dict[str, Any]) -> None:
                 event="exception",
                 properties={
                     "exception": str(error),
-                    "traceback": str(error) if not hasattr(error, "__traceback__") else str(error),
+                    "traceback": (
+                        str(error)
+                        if not hasattr(error, "__traceback__")
+                        else str(error)
+                    ),
                     "session_id": self.mcp_interaction_id,
                     "mode": self.mode,
                     "process_person_profile": True,