Cl 1581 lib python get cursor position

pyproject.toml

@@ -56,11 +56,14 @@ typecheck = "mypy"
 "typecheck:all" = "mypy ."
 "chat:api" = "uvicorn askui.chat.api.app:app --reload --port 9261"
 "mcp:dev" = "mcp dev src/askui/mcp/__init__.py"
+<<<<<<< HEAD
+=======
 "qa:fix" = { composite = [
     "typecheck:all",
     "format",
     "lint:fix",
 ] }
+>>>>>>> feat/update-controller-grpc
 "grpc:gen" = "bash scripts/grpc-gen.sh"
 "json:gen" = "datamodel-codegen --output-model-type pydantic_v2.BaseModel --input src/askui/tools/askui/askui_ui_controller_grpc/json_schema/ --input-file-type jsonschema --output src/askui/tools/askui/askui_ui_controller_grpc/generated/"
 
@@ -222,3 +225,7 @@ pynput = [
 web = [
     "playwright>=1.41.0",
 ]
+dev = [
+    "datamodel-code-generator>=0.31.2",
+    "grpcio-tools>=1.73.1",
+]

src/askui/agent.py

@@ -10,14 +10,12 @@
 from askui.agent_base import AgentBase
 from askui.container import telemetry
 from askui.locators.locators import Locator
-from askui.models.shared.settings import (
-    COMPUTER_USE_20241022_BETA_FLAG,
-    COMPUTER_USE_20250124_BETA_FLAG,
-    ActSettings,
-    MessageSettings,
-)
+from askui.models.shared.settings import (COMPUTER_USE_20241022_BETA_FLAG,
+                                          COMPUTER_USE_20250124_BETA_FLAG,
+                                          ActSettings, MessageSettings)
 from askui.models.shared.tools import Tool
 from askui.tools.computer import Computer20241022Tool, Computer20250124Tool
+from askui.tools.cursor_position_tool import CursorPositionTool
 from askui.tools.exception_tool import ExceptionTool
 
 from .logger import logger
@@ -115,6 +113,7 @@ def __init__(
             models=models,
             tools=[
                 ExceptionTool(),
+                CursorPositionTool(agent_os=self.tools.os),
             ]
             + (act_tools or []),
             agent_os=self.tools.os,

src/askui/tools/askui/askui_controller.py

@@ -847,6 +847,55 @@ def get_automation_target_list(
         return response
 
     @telemetry.record_call()
+<<<<<<< HEAD
+    def set_test_configuration(
+        self,
+        default_capture_parameters: controller_v1_pbs.CaptureParameters | None = None,
+        mouse_delay_ms: int = 0,
+        keyboard_delay_ms: int = 0,
+    ) -> None:
+        """
+        Configure test settings including default capture parameters and delays.
+
+        Args:
+            default_capture_parameters
+                (controller_v1_pbs.CaptureParameters | None, optional):
+                Default capture parameters with displayID and captureArea.
+                Defaults to `None`.
+            mouse_delay_ms (int, optional): Mouse delay in milliseconds.
+                Defaults to `0`.
+            keyboard_delay_ms (int, optional): Keyboard delay in milliseconds.
+                Defaults to `0`.
+        """
+        assert isinstance(self._stub, controller_v1.ControllerAPIStub), (
+            "Stub is not initialized"
+        )
+
+        self._reporter.add_message(
+            "AgentOS",
+            f"set_test_configuration("
+            f"{default_capture_parameters}, "
+            f"{mouse_delay_ms}, "
+            f"{keyboard_delay_ms})",
+        )
+
+        # Use provided capture parameters or create empty one
+        capture_params = (
+            default_capture_parameters or controller_v1_pbs.CaptureParameters()
+        )
+
+        self._stub.SetTestConfiguration(
+            controller_v1_pbs.Reuqest_SetTestConfiguration(
+                sessionInfo=self._session_info,
+                defaultCaptureParameters=capture_params,
+                mouseDelayInMilliseconds=mouse_delay_ms,
+                keyboardDelayInMilliseconds=keyboard_delay_ms,
+            )
+        )
+
+    @telemetry.record_call()
+=======
+>>>>>>> feat/update-controller-grpc
     def set_mouse_delay(self, delay_ms: int) -> None:
         """
         Configure mouse action delay.
@@ -1083,7 +1132,12 @@ def remove_all_actions(self) -> None:
             controller_v1_pbs.Request_RemoveAllActions(sessionInfo=self._session_info)
         )
 
+<<<<<<< HEAD
+    @telemetry.record_call(exclude={"message"})
+    def send_message(self, message: str) -> controller_v1_pbs.Response_Send:
+=======
     def _send_message(self, message: str) -> controller_v1_pbs.Response_Send:
+>>>>>>> feat/update-controller-grpc
         """
         Send a general message to the controller.
 
@@ -1121,7 +1175,11 @@ def get_mouse_position(self) -> Coordinate:
             self._session_guid
         ).model_dump_json(exclude_unset=True)
         self._reporter.add_message("AgentOS", "get_mouse_position()")
+<<<<<<< HEAD
+        res = self.send_message(req_json)
+=======
         res = self._send_message(req_json)
+>>>>>>> feat/update-controller-grpc
         parsed_res = AskuiAgentosSendResponseSchema.model_validate_json(res.message)
         return Coordinate(
             x=parsed_res.message.command.response.position.x.root,  # type: ignore[union-attr]
@@ -1144,7 +1202,11 @@ def set_mouse_position(self, x: int, y: int) -> None:
             x, y, self._session_guid
         ).model_dump_json(exclude_unset=True)
         self._reporter.add_message("AgentOS", f"set_mouse_position({x},{y})")
+<<<<<<< HEAD
+        self.send_message(req_json)
+=======
         self._send_message(req_json)
+>>>>>>> feat/update-controller-grpc
 
     @telemetry.record_call()
     def render_quad(self, style: RenderObjectStyle) -> int:
@@ -1164,7 +1226,11 @@ def render_quad(self, style: RenderObjectStyle) -> int:
         req_json = create_quad_command(style, self._session_guid).model_dump_json(
             exclude_unset=True, by_alias=True
         )
+<<<<<<< HEAD
+        res = self.send_message(req_json)
+=======
         res = self._send_message(req_json)
+>>>>>>> feat/update-controller-grpc
         parsed_response = AskuiAgentosSendResponseSchema.model_validate_json(
             res.message
         )
@@ -1189,7 +1255,11 @@ def render_line(self, style: RenderObjectStyle, points: list[Coordinate]) -> int
         req = create_line_command(style, points, self._session_guid).model_dump_json(
             exclude_unset=True, by_alias=True
         )
+<<<<<<< HEAD
+        res = self.send_message(req)
+=======
         res = self._send_message(req)
+>>>>>>> feat/update-controller-grpc
         parsed_response = AskuiAgentosSendResponseSchema.model_validate_json(
             res.message
         )
@@ -1214,7 +1284,11 @@ def render_image(self, style: RenderObjectStyle, image_data: str) -> int:
         req = create_image_command(
             style, image_data, self._session_guid
         ).model_dump_json(exclude_unset=True, by_alias=True)
+<<<<<<< HEAD
+        res = self.send_message(req)
+=======
         res = self._send_message(req)
+>>>>>>> feat/update-controller-grpc
 
         parsed_response = AskuiAgentosSendResponseSchema.model_validate_json(
             res.message
@@ -1241,7 +1315,11 @@ def render_text(self, style: RenderObjectStyle, content: str) -> int:
         req = create_text_command(style, content, self._session_guid).model_dump_json(
             exclude_unset=True, by_alias=True
         )
+<<<<<<< HEAD
+        res = self.send_message(req)
+=======
         res = self._send_message(req)
+>>>>>>> feat/update-controller-grpc
         parsed_response = AskuiAgentosSendResponseSchema.model_validate_json(
             res.message
         )
@@ -1268,7 +1346,11 @@ def update_render_object(self, object_id: int, style: RenderObjectStyle) -> None
         req = create_update_render_object_command(
             object_id, style, self._session_guid
         ).model_dump_json(exclude_unset=True, by_alias=True)
+<<<<<<< HEAD
+        self.send_message(req)
+=======
         self._send_message(req)
+>>>>>>> feat/update-controller-grpc
 
     @telemetry.record_call()
     def delete_render_object(self, object_id: int) -> None:
@@ -1285,7 +1367,11 @@ def delete_render_object(self, object_id: int) -> None:
         req = create_delete_render_object_command(
             object_id, self._session_guid
         ).model_dump_json(exclude_unset=True, by_alias=True)
+<<<<<<< HEAD
+        self.send_message(req)
+=======
         self._send_message(req)
+>>>>>>> feat/update-controller-grpc
 
     @telemetry.record_call()
     def clear_render_objects(self) -> None:
@@ -1299,4 +1385,8 @@ def clear_render_objects(self) -> None:
         req = create_clear_render_objects_command(self._session_guid).model_dump_json(
             exclude_unset=True, by_alias=True
         )
+<<<<<<< HEAD
+        self.send_message(req)
+=======
         self._send_message(req)
+>>>>>>> feat/update-controller-grpc

src/askui/tools/cursor_position_tool.py

@@ -0,0 +1,29 @@
+from askui.models.shared.tools import Tool
+from askui.tools.agent_os import AgentOs, Coordinate
+
+
+class CursorPositionTool(Tool):
+    """
+    Tool to get the current mouse cursor position.
+    """
+
+    def __init__(self, agent_os: AgentOs):
+        super().__init__(
+            name="get_cursor_position",
+            description=
+                """
+                Gets the current position of the mouse cursor.
+                Returns a JSON string with 'x' and 'y' coordinates, e.g. {"x": 100, "y": 200}.
+                """
+        )
+        self._agent_os: AgentOs = agent_os
+
+    def __call__(self) -> str:
+        """
+        Gets the current mouse cursor position.
+
+        Returns:
+            str: A JSON string with 'x' and 'y' coordinates of the cursor, e.g. {"x": 100, "y": 200}.
+        """
+        point: Coordinate = self._agent_os.get_mouse_position()
+        return point.model_dump_json()