diff --git a/pdm.lock b/pdm.lock index 90e89af4..bece99d1 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,11 @@ groups = ["default", "all", "android", "chat", "dev", "mcp", "pynput", "test", "web"] strategy = ["inherit_metadata"] lock_version = "4.5.0" +<<<<<<< HEAD +content_hash = "sha256:c4c815df35c7f817156b1db01cf65a70559e024220543b4e15b8b64f1669c070" +======= content_hash = "sha256:ecb4a5b9f12fc5d3cf4c6e98a65c9bb90df5c523ba9c8f335e5e40b8a4e9b9d1" +>>>>>>> feat/update-controller-grpc [[metadata.targets]] requires_python = ">=3.10" @@ -1082,6 +1086,8 @@ files = [ ] [[package]] +<<<<<<< HEAD +======= name = "jsonref" version = "1.1.0" requires_python = ">=3.7" @@ -1093,6 +1099,7 @@ files = [ ] [[package]] +>>>>>>> feat/update-controller-grpc name = "jsonschema" version = "4.25.0" requires_python = ">=3.9" @@ -1199,7 +1206,11 @@ files = [ [[package]] name = "mcp" +<<<<<<< HEAD +version = "1.12.0" +======= version = "1.12.1" +>>>>>>> feat/update-controller-grpc requires_python = ">=3.10" summary = "Model Context Protocol SDK" groups = ["all", "mcp"] @@ -1217,8 +1228,13 @@ dependencies = [ "uvicorn>=0.23.1; sys_platform != \"emscripten\"", ] files = [ +<<<<<<< HEAD + {file = "mcp-1.12.0-py3-none-any.whl", hash = "sha256:19a498b2bf273283e463b4dd1ed83f791fbba5c25bfa16b8b34cfd5571673e7f"}, + {file = "mcp-1.12.0.tar.gz", hash = "sha256:853f6b17a3f31ea6e2f278c2ec7d3b38457bc80c7c2c675260dd7f04a6fd0e70"}, +======= {file = "mcp-1.12.1-py3-none-any.whl", hash = "sha256:34147f62891417f8b000c39718add844182ba424c8eb2cea250b4267bda4b08b"}, {file = "mcp-1.12.1.tar.gz", hash = "sha256:d1d0bdeb09e4b17c1a72b356248bf3baf75ab10db7008ef865c4afbeb0eb810e"}, +>>>>>>> feat/update-controller-grpc ] [[package]] @@ -2258,13 +2274,13 @@ files = [ [[package]] name = "setuptools" -version = "78.1.0" +version = "80.9.0" requires_python = ">=3.9" summary = "Easily download, build, install, upgrade, and uninstall Python packages" groups = ["dev"] files = [ - {file = "setuptools-78.1.0-py3-none-any.whl", hash = "sha256:3e386e96793c8702ae83d17b853fb93d3e09ef82ec62722e61da5cd22376dcd8"}, - {file = "setuptools-78.1.0.tar.gz", hash = "sha256:18fd474d4a82a5f83dac888df697af65afa82dec7323d09c3e37d1f14288da54"}, + {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, + {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 75d8baa4..be6bb4be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,11 +56,14 @@ typecheck = "mypy" "typecheck:all" = "mypy ." "chat:api" = "uvicorn askui.chat.api.app:app --reload --port 9261" "mcp:dev" = "mcp dev src/askui/mcp/__init__.py" +<<<<<<< HEAD +======= "qa:fix" = { composite = [ "typecheck:all", "format", "lint:fix", ] } +>>>>>>> feat/update-controller-grpc "grpc:gen" = "bash scripts/grpc-gen.sh" "json:gen" = "datamodel-codegen --output-model-type pydantic_v2.BaseModel --input src/askui/tools/askui/askui_ui_controller_grpc/json_schema/ --input-file-type jsonschema --output src/askui/tools/askui/askui_ui_controller_grpc/generated/" @@ -222,3 +225,7 @@ pynput = [ web = [ "playwright>=1.41.0", ] +dev = [ + "datamodel-code-generator>=0.31.2", + "grpcio-tools>=1.73.1", +] diff --git a/src/askui/agent.py b/src/askui/agent.py index 30dbce32..30c8a47d 100644 --- a/src/askui/agent.py +++ b/src/askui/agent.py @@ -10,14 +10,12 @@ from askui.agent_base import AgentBase from askui.container import telemetry from askui.locators.locators import Locator -from askui.models.shared.settings import ( - COMPUTER_USE_20241022_BETA_FLAG, - COMPUTER_USE_20250124_BETA_FLAG, - ActSettings, - MessageSettings, -) +from askui.models.shared.settings import (COMPUTER_USE_20241022_BETA_FLAG, + COMPUTER_USE_20250124_BETA_FLAG, + ActSettings, MessageSettings) from askui.models.shared.tools import Tool from askui.tools.computer import Computer20241022Tool, Computer20250124Tool +from askui.tools.cursor_position_tool import CursorPositionTool from askui.tools.exception_tool import ExceptionTool from .logger import logger @@ -115,6 +113,7 @@ def __init__( models=models, tools=[ ExceptionTool(), + CursorPositionTool(agent_os=self.tools.os), ] + (act_tools or []), agent_os=self.tools.os, diff --git a/src/askui/tools/askui/askui_controller.py b/src/askui/tools/askui/askui_controller.py index 2ff1f2d8..a48ecab6 100644 --- a/src/askui/tools/askui/askui_controller.py +++ b/src/askui/tools/askui/askui_controller.py @@ -847,6 +847,55 @@ def get_automation_target_list( return response @telemetry.record_call() +<<<<<<< HEAD + def set_test_configuration( + self, + default_capture_parameters: controller_v1_pbs.CaptureParameters | None = None, + mouse_delay_ms: int = 0, + keyboard_delay_ms: int = 0, + ) -> None: + """ + Configure test settings including default capture parameters and delays. + + Args: + default_capture_parameters + (controller_v1_pbs.CaptureParameters | None, optional): + Default capture parameters with displayID and captureArea. + Defaults to `None`. + mouse_delay_ms (int, optional): Mouse delay in milliseconds. + Defaults to `0`. + keyboard_delay_ms (int, optional): Keyboard delay in milliseconds. + Defaults to `0`. + """ + assert isinstance(self._stub, controller_v1.ControllerAPIStub), ( + "Stub is not initialized" + ) + + self._reporter.add_message( + "AgentOS", + f"set_test_configuration(" + f"{default_capture_parameters}, " + f"{mouse_delay_ms}, " + f"{keyboard_delay_ms})", + ) + + # Use provided capture parameters or create empty one + capture_params = ( + default_capture_parameters or controller_v1_pbs.CaptureParameters() + ) + + self._stub.SetTestConfiguration( + controller_v1_pbs.Reuqest_SetTestConfiguration( + sessionInfo=self._session_info, + defaultCaptureParameters=capture_params, + mouseDelayInMilliseconds=mouse_delay_ms, + keyboardDelayInMilliseconds=keyboard_delay_ms, + ) + ) + + @telemetry.record_call() +======= +>>>>>>> feat/update-controller-grpc def set_mouse_delay(self, delay_ms: int) -> None: """ Configure mouse action delay. @@ -1083,7 +1132,12 @@ def remove_all_actions(self) -> None: controller_v1_pbs.Request_RemoveAllActions(sessionInfo=self._session_info) ) +<<<<<<< HEAD + @telemetry.record_call(exclude={"message"}) + def send_message(self, message: str) -> controller_v1_pbs.Response_Send: +======= def _send_message(self, message: str) -> controller_v1_pbs.Response_Send: +>>>>>>> feat/update-controller-grpc """ Send a general message to the controller. @@ -1121,7 +1175,11 @@ def get_mouse_position(self) -> Coordinate: self._session_guid ).model_dump_json(exclude_unset=True) self._reporter.add_message("AgentOS", "get_mouse_position()") +<<<<<<< HEAD + res = self.send_message(req_json) +======= res = self._send_message(req_json) +>>>>>>> feat/update-controller-grpc parsed_res = AskuiAgentosSendResponseSchema.model_validate_json(res.message) return Coordinate( x=parsed_res.message.command.response.position.x.root, # type: ignore[union-attr] @@ -1144,7 +1202,11 @@ def set_mouse_position(self, x: int, y: int) -> None: x, y, self._session_guid ).model_dump_json(exclude_unset=True) self._reporter.add_message("AgentOS", f"set_mouse_position({x},{y})") +<<<<<<< HEAD + self.send_message(req_json) +======= self._send_message(req_json) +>>>>>>> feat/update-controller-grpc @telemetry.record_call() def render_quad(self, style: RenderObjectStyle) -> int: @@ -1164,7 +1226,11 @@ def render_quad(self, style: RenderObjectStyle) -> int: req_json = create_quad_command(style, self._session_guid).model_dump_json( exclude_unset=True, by_alias=True ) +<<<<<<< HEAD + res = self.send_message(req_json) +======= res = self._send_message(req_json) +>>>>>>> feat/update-controller-grpc parsed_response = AskuiAgentosSendResponseSchema.model_validate_json( res.message ) @@ -1189,7 +1255,11 @@ def render_line(self, style: RenderObjectStyle, points: list[Coordinate]) -> int req = create_line_command(style, points, self._session_guid).model_dump_json( exclude_unset=True, by_alias=True ) +<<<<<<< HEAD + res = self.send_message(req) +======= res = self._send_message(req) +>>>>>>> feat/update-controller-grpc parsed_response = AskuiAgentosSendResponseSchema.model_validate_json( res.message ) @@ -1214,7 +1284,11 @@ def render_image(self, style: RenderObjectStyle, image_data: str) -> int: req = create_image_command( style, image_data, self._session_guid ).model_dump_json(exclude_unset=True, by_alias=True) +<<<<<<< HEAD + res = self.send_message(req) +======= res = self._send_message(req) +>>>>>>> feat/update-controller-grpc parsed_response = AskuiAgentosSendResponseSchema.model_validate_json( res.message @@ -1241,7 +1315,11 @@ def render_text(self, style: RenderObjectStyle, content: str) -> int: req = create_text_command(style, content, self._session_guid).model_dump_json( exclude_unset=True, by_alias=True ) +<<<<<<< HEAD + res = self.send_message(req) +======= res = self._send_message(req) +>>>>>>> feat/update-controller-grpc parsed_response = AskuiAgentosSendResponseSchema.model_validate_json( res.message ) @@ -1268,7 +1346,11 @@ def update_render_object(self, object_id: int, style: RenderObjectStyle) -> None req = create_update_render_object_command( object_id, style, self._session_guid ).model_dump_json(exclude_unset=True, by_alias=True) +<<<<<<< HEAD + self.send_message(req) +======= self._send_message(req) +>>>>>>> feat/update-controller-grpc @telemetry.record_call() def delete_render_object(self, object_id: int) -> None: @@ -1285,7 +1367,11 @@ def delete_render_object(self, object_id: int) -> None: req = create_delete_render_object_command( object_id, self._session_guid ).model_dump_json(exclude_unset=True, by_alias=True) +<<<<<<< HEAD + self.send_message(req) +======= self._send_message(req) +>>>>>>> feat/update-controller-grpc @telemetry.record_call() def clear_render_objects(self) -> None: @@ -1299,4 +1385,8 @@ def clear_render_objects(self) -> None: req = create_clear_render_objects_command(self._session_guid).model_dump_json( exclude_unset=True, by_alias=True ) +<<<<<<< HEAD + self.send_message(req) +======= self._send_message(req) +>>>>>>> feat/update-controller-grpc diff --git a/src/askui/tools/cursor_position_tool.py b/src/askui/tools/cursor_position_tool.py new file mode 100644 index 00000000..5ad11616 --- /dev/null +++ b/src/askui/tools/cursor_position_tool.py @@ -0,0 +1,29 @@ +from askui.models.shared.tools import Tool +from askui.tools.agent_os import AgentOs, Coordinate + + +class CursorPositionTool(Tool): + """ + Tool to get the current mouse cursor position. + """ + + def __init__(self, agent_os: AgentOs): + super().__init__( + name="get_cursor_position", + description= + """ + Gets the current position of the mouse cursor. + Returns a JSON string with 'x' and 'y' coordinates, e.g. {"x": 100, "y": 200}. + """ + ) + self._agent_os: AgentOs = agent_os + + def __call__(self) -> str: + """ + Gets the current mouse cursor position. + + Returns: + str: A JSON string with 'x' and 'y' coordinates of the cursor, e.g. {"x": 100, "y": 200}. + """ + point: Coordinate = self._agent_os.get_mouse_position() + return point.model_dump_json()