From a379289dcbb1a379914b0df0bba195676c250143 Mon Sep 17 00:00:00 2001 From: Dominik Klotz Date: Thu, 30 Oct 2025 22:38:00 +0100 Subject: [PATCH 1/6] feat(agent): add offset and absolute position to click, type and move_mouse --- src/askui/agent.py | 51 +++-- tests/e2e/agent/test_agent_offset.py | 270 +++++++++++++++++++++++++++ 2 files changed, 306 insertions(+), 15 deletions(-) create mode 100644 tests/e2e/agent/test_agent_offset.py diff --git a/src/askui/agent.py b/src/askui/agent.py index c91d76b8..a9242a30 100644 --- a/src/askui/agent.py +++ b/src/askui/agent.py @@ -21,7 +21,7 @@ from askui.tools.set_active_display_tool import SetActiveDisplayTool from .models import ModelComposition -from .models.models import ModelChoice, ModelRegistry +from .models.models import ModelChoice, ModelRegistry, Point from .reporting import CompositeReporter, Reporter from .retry import Retry from .tools import AgentToolbox, ModifierKey, PcKey @@ -96,18 +96,20 @@ def __init__( @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def click( self, - locator: Optional[str | Locator] = None, + locator: Optional[str | Locator | Point] = None, button: Literal["left", "middle", "right"] = "left", repeat: Annotated[int, Field(gt=0)] = 1, + offset: Optional[Point] = None, model: ModelComposition | str | None = None, ) -> None: """ Simulates a mouse click on the user interface element identified by the provided locator. Args: - locator (str | Locator | None, optional): The identifier or description of the element to click. If `None`, clicks at current position. + locator (str | Locator | Point | None, optional): UI element description, structured locator, or absolute coordinates (x, y). If `None`, clicks at current position. button ('left' | 'middle' | 'right', optional): Specifies which mouse button to click. Defaults to `'left'`. repeat (int, optional): The number of times to click. Must be greater than `0`. Defaults to `1`. + offset (Point | None, optional): Pixel offset (x, y) from the target location. Positive x=right, negative x=left, positive y=down, negative y=up. model (ModelComposition | str | None, optional): The composition or name of the model(s) to be used for locating the element to click on using the `locator`. Example: @@ -117,9 +119,11 @@ def click( with VisionAgent() as agent: agent.click() # Left click on current position agent.click("Edit") # Left click on text "Edit" + agent.click((100, 200)) # Left click at absolute coordinates (100, 200) agent.click("Edit", button="right") # Right click on text "Edit" agent.click(repeat=2) # Double left click on current position agent.click("Edit", button="middle", repeat=4) # 4x middle click on text "Edit" + agent.click("Submit", offset=(10, -5)) # Click 10 pixels right and 5 pixels up from "Submit" ``` """ msg = "click" @@ -129,39 +133,50 @@ def click( msg += f" {repeat}x times" if locator is not None: msg += f" on {locator}" + if offset is not None: + msg += f" with offset {offset}" logger.debug("VisionAgent received instruction to %s", msg) self._reporter.add_message("User", msg) - self._click(locator, button, repeat, model) + self._click(locator, button, repeat, offset, model) def _click( self, - locator: Optional[str | Locator], + locator: Optional[str | Locator | Point], button: Literal["left", "middle", "right"], repeat: int, + offset: Optional[Point], model: ModelComposition | str | None, ) -> None: if locator is not None: - self._mouse_move(locator, model) + self._mouse_move(locator, offset, model) self.tools.os.click(button, repeat) def _mouse_move( - self, locator: str | Locator, model: ModelComposition | str | None = None + self, locator: str | Locator | Point, offset: Optional[Point], model: ModelComposition | str | None = None ) -> None: - point = self._locate(locator=locator, model=model)[0] + point: Point + if isinstance(locator, tuple): + point: Point = locator + else: + point: Point = self._locate(locator=locator, model=model)[0] + if offset is not None: + point = (point[0] + offset[0], point[1] + offset[1]) self.tools.os.mouse_move(point[0], point[1]) @telemetry.record_call(exclude={"locator"}) @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def mouse_move( self, - locator: str | Locator, + locator: str | Locator | Point, + offset: Optional[Point] = None, model: ModelComposition | str | None = None, ) -> None: """ Moves the mouse cursor to the UI element identified by the provided locator. Args: - locator (str | Locator): The identifier or description of the element to move to. + locator (str | Locator | Point): UI element description, structured locator, or absolute coordinates (x, y). + offset (Point | None, optional): Pixel offset (x, y) from the target location. Positive x=right, negative x=left, positive y=down, negative y=up. model (ModelComposition | str | None, optional): The composition or name of the model(s) to be used for locating the element to move the mouse to using the `locator`. Example: @@ -170,13 +185,15 @@ def mouse_move( with VisionAgent() as agent: agent.mouse_move("Submit button") # Moves cursor to submit button - agent.mouse_move("Close") # Moves cursor to close element + agent.mouse_move((300, 150)) # Moves cursor to absolute coordinates (300, 150) + agent.mouse_move("Close") # Moves cursor to close element agent.mouse_move("Profile picture", model="custom_model") # Uses specific model + agent.mouse_move("Menu", offset=(5, 10)) # Move 5 pixels right and 10 pixels down from "Menu" ``` """ self._reporter.add_message("User", f"mouse_move: {locator}") logger.debug("VisionAgent received instruction to mouse_move to %s", locator) - self._mouse_move(locator, model) + self._mouse_move(locator, offset, model) @telemetry.record_call() @validate_call @@ -217,7 +234,8 @@ def mouse_scroll( def type( self, text: Annotated[str, Field(min_length=1)], - locator: str | Locator | None = None, + locator: str | Locator | Point | None = None, + offset: Optional[Point] = None, model: ModelComposition | str | None = None, clear: bool = True, ) -> None: @@ -231,7 +249,8 @@ def type( Args: text (str): The text to be typed. Must be at least `1` character long. - locator (str | Locator | None, optional): The identifier or description of the element (e.g., input field) to type into. If `None`, types at the current focus. + locator (str | Locator | Point | None, optional): UI element description, structured locator, or absolute coordinates (x, y). If `None`, types at current focus. + offset (Point | None, optional): Pixel offset (x, y) from the target location. Positive x=right, negative x=left, positive y=down, negative y=up. model (ModelComposition | str | None, optional): The composition or name of the model(s) to be used for locating the element, i.e., input field, to type into using the `locator`. clear (bool, optional): Whether to triple click on the element to give it focus and select the current text before typing. Defaults to `True`. @@ -242,8 +261,10 @@ def type( with VisionAgent() as agent: agent.type("Hello, world!") # Types "Hello, world!" at current focus agent.type("user@example.com", locator="Email") # Clicks on "Email" input, then types + agent.type("username", locator=(200, 100)) # Clicks at coordinates (200, 100), then types agent.type("password123", locator="Password field", model="custom_model") # Uses specific model agent.type("Hello, world!", locator="Textarea", clear=False) # Types "Hello, world!" into textarea without clearing + agent.type("text", locator="Input field", offset=(5, 0)) # Click 5 pixels right of "Input field", then type ``` """ msg = f'type "{text}"' @@ -254,7 +275,7 @@ def type( msg += " clearing the current content (line/paragraph) of input field" else: repeat = 1 - self._click(locator=locator, button="left", repeat=repeat, model=model) + self._click(locator=locator, button="left", repeat=repeat, offset=offset, model=model) logger.debug("VisionAgent received instruction to %s", msg) self._reporter.add_message("User", msg) self.tools.os.type(text) diff --git a/tests/e2e/agent/test_agent_offset.py b/tests/e2e/agent/test_agent_offset.py new file mode 100644 index 00000000..d2c872c3 --- /dev/null +++ b/tests/e2e/agent/test_agent_offset.py @@ -0,0 +1,270 @@ +"""Tests for VisionAgent offset functionality with different locator types and models""" + +import pathlib +from unittest.mock import Mock + +import pytest +from PIL import Image as PILImage + +from askui.agent import VisionAgent +from askui.locators import AiElement, Element, Prompt, Text +from askui.locators.locators import Image +from askui.models import ModelName +from askui.models.models import Point + + +@pytest.mark.parametrize( + "model", + [ + ModelName.ASKUI, + ], +) +class TestVisionAgentOffset: + """Test class for VisionAgent offset functionality.""" + + def _setup_mocks(self, vision_agent: VisionAgent, github_login_screenshot: PILImage.Image): + """Helper method to setup common mocks.""" + mock_mouse_move = Mock() + mock_click = Mock() + mock_type = Mock() + mock_screenshot = Mock(return_value=github_login_screenshot) + + vision_agent.tools.os.mouse_move = mock_mouse_move + vision_agent.tools.os.click = mock_click + vision_agent.tools.os.type = mock_type + vision_agent.tools.os.screenshot = mock_screenshot + + return mock_mouse_move, mock_click, mock_type, mock_screenshot + + def test_click_with_positive_offset( + self, + vision_agent: VisionAgent, + github_login_screenshot: PILImage.Image, + model: str, + ) -> None: + """Test clicking with positive offset (right and down).""" + locator = "Forgot password?" + offset: Point = (10, 5) + + mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + + # Get original position + x, y = vision_agent.locate(locator, github_login_screenshot, model=model) + + # Click with offset should work without error + vision_agent.click(locator, offset=offset, model=model) + + # Verify calls + mock_screenshot.assert_called() + expected_x, expected_y = x + offset[0], y + offset[1] + mock_mouse_move.assert_called_once_with(expected_x, expected_y) + mock_click.assert_called_once_with("left", 1) + + def test_click_with_negative_offset( + self, + vision_agent: VisionAgent, + github_login_screenshot: PILImage.Image, + model: str, + ) -> None: + """Test clicking with negative offset (left and up).""" + locator = "Forgot password?" + offset: Point = (-10, -5) + + mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + + # Get original position + x, y = vision_agent.locate(locator, github_login_screenshot, model=model) + + # Click with negative offset should work without error + vision_agent.click(locator, offset=offset, model=model) + + # Verify calls + mock_screenshot.assert_called() + expected_x, expected_y = x + offset[0], y + offset[1] + mock_mouse_move.assert_called_once_with(expected_x, expected_y) + mock_click.assert_called_once_with("left", 1) + + def test_click_with_zero_offset( + self, + vision_agent: VisionAgent, + github_login_screenshot: PILImage.Image, + model: str, + ) -> None: + """Test clicking with zero offset (same as no offset).""" + locator = "Forgot password?" + offset: Point = (0, 0) + + mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + + # Get original position + x, y = vision_agent.locate(locator, github_login_screenshot, model=model) + + # Click with zero offset should work without error + vision_agent.click(locator, offset=offset, model=model) + + # Verify calls + mock_screenshot.assert_called() + mock_mouse_move.assert_called_once_with(x, y) + mock_click.assert_called_once_with("left", 1) + + def test_click_with_point_locator_and_offset( + self, + vision_agent: VisionAgent, + github_login_screenshot: PILImage.Image, + model: str, + ) -> None: + """Test clicking with Point locator and offset.""" + point_locator: Point = (100, 100) + offset: Point = (20, 15) + + mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + + # Click with Point locator and offset should work + vision_agent.click(point_locator, offset=offset, model=model) + + # Verify calls + mock_screenshot.assert_not_called() + expected_x, expected_y = point_locator[0] + offset[0], point_locator[1] + offset[1] + mock_mouse_move.assert_called_once_with(expected_x, expected_y) + mock_click.assert_called_once_with("left", 1) + + def test_mouse_move_with_positive_offset( + self, + vision_agent: VisionAgent, + github_login_screenshot: PILImage.Image, + model: str, + ) -> None: + """Test mouse movement with positive offset.""" + locator = "Forgot password?" + offset: Point = (15, 10) + + mock_mouse_move, _, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + + # Get original position + x, y = vision_agent.locate(locator, github_login_screenshot, model=model) + + # Mouse move with offset should work without error + vision_agent.mouse_move(locator, offset=offset, model=model) + + # Verify calls + mock_screenshot.assert_called() + expected_x, expected_y = x + offset[0], y + offset[1] + mock_mouse_move.assert_called_once_with(expected_x, expected_y) + + def test_mouse_move_with_negative_offset( + self, + vision_agent: VisionAgent, + github_login_screenshot: PILImage.Image, + model: str, + ) -> None: + """Test mouse movement with negative offset.""" + locator = "Forgot password?" + offset: Point = (-15, -10) + + mock_mouse_move, _, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + + # Get original position + x, y = vision_agent.locate(locator, github_login_screenshot, model=model) + + # Mouse move with negative offset should work without error + vision_agent.mouse_move(locator, offset=offset, model=model) + + # Verify calls + mock_screenshot.assert_called() + expected_x, expected_y = x + offset[0], y + offset[1] + mock_mouse_move.assert_called_once_with(expected_x, expected_y) + + def test_mouse_move_with_point_locator_and_offset( + self, + vision_agent: VisionAgent, + github_login_screenshot: PILImage.Image, + model: str, + ) -> None: + """Test mouse movement with Point locator and offset.""" + point_locator: Point = (200, 150) + offset: Point = (25, -10) + + mock_mouse_move, _, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + + # Mouse move with Point locator and offset should work + vision_agent.mouse_move(point_locator, offset=offset, model=model) + + # Verify calls + mock_screenshot.assert_not_called() + expected_x, expected_y = point_locator[0] + offset[0], point_locator[1] + offset[1] + mock_mouse_move.assert_called_once_with(expected_x, expected_y) + + def test_type_with_positive_offset( + self, + vision_agent: VisionAgent, + github_login_screenshot: PILImage.Image, + model: str, + ) -> None: + """Test typing with positive offset.""" + locator = Element("textfield") + offset: Point = (5, 3) + text = "test@example.com" + + mock_mouse_move, mock_click, mock_type, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + + # Get original position + x, y = vision_agent.locate(locator, github_login_screenshot, model=model) + + # Type with offset should work without error + vision_agent.type(text, locator=locator, offset=offset, model=model) + + # Verify calls + mock_screenshot.assert_called() + expected_x, expected_y = x + offset[0], y + offset[1] + mock_mouse_move.assert_called_once_with(expected_x, expected_y) + mock_click.assert_called_once_with("left", 3) + mock_type.assert_called_once_with(text) + + def test_type_with_negative_offset( + self, + vision_agent: VisionAgent, + github_login_screenshot: PILImage.Image, + model: str, + ) -> None: + """Test typing with negative offset.""" + locator = Element("textfield") + offset: Point = (-5, -3) + text = "test@example.com" + + mock_mouse_move, mock_click, mock_type, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + + # Get original position + x, y = vision_agent.locate(locator, github_login_screenshot, model=model) + + # Type with negative offset should work without error + vision_agent.type(text, locator=locator, offset=offset, model=model) + + # Verify calls + mock_screenshot.assert_called() + expected_x, expected_y = x + offset[0], y + offset[1] + mock_mouse_move.assert_called_once_with(expected_x, expected_y) + mock_click.assert_called_once_with("left", 3) + mock_type.assert_called_once_with(text) + + def test_type_with_point_locator_and_offset( + self, + vision_agent: VisionAgent, + github_login_screenshot: PILImage.Image, + model: str, + ) -> None: + """Test typing with Point locator and offset.""" + point_locator: Point = (460, 195) # Approximate textfield location + offset: Point = (10, 5) + text = "username" + + mock_mouse_move, mock_click, mock_type, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + + # Type with Point locator and offset should work + vision_agent.type(text, locator=point_locator, offset=offset, model=model) + + # Verify calls + mock_screenshot.assert_not_called() + expected_x, expected_y = point_locator[0] + offset[0], point_locator[1] + offset[1] + mock_mouse_move.assert_called_once_with(expected_x, expected_y) + mock_click.assert_called_once_with("left", 3) + mock_type.assert_called_once_with(text) From 285e7454da42cc19b300a527d0b30f1ce4b2c2f7 Mon Sep 17 00:00:00 2001 From: Dominik Klotz Date: Thu, 30 Oct 2025 22:47:09 +0100 Subject: [PATCH 2/6] style: fix formatting --- src/askui/agent.py | 13 +++++- tests/e2e/agent/test_agent_offset.py | 68 ++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/src/askui/agent.py b/src/askui/agent.py index a9242a30..e0bc92b5 100644 --- a/src/askui/agent.py +++ b/src/askui/agent.py @@ -152,7 +152,10 @@ def _click( self.tools.os.click(button, repeat) def _mouse_move( - self, locator: str | Locator | Point, offset: Optional[Point], model: ModelComposition | str | None = None + self, + locator: str | Locator | Point, + offset: Optional[Point], + model: ModelComposition | str | None = None, ) -> None: point: Point if isinstance(locator, tuple): @@ -275,7 +278,13 @@ def type( msg += " clearing the current content (line/paragraph) of input field" else: repeat = 1 - self._click(locator=locator, button="left", repeat=repeat, offset=offset, model=model) + self._click( + locator=locator, + button="left", + repeat=repeat, + offset=offset, + model=model, + ) logger.debug("VisionAgent received instruction to %s", msg) self._reporter.add_message("User", msg) self.tools.os.type(text) diff --git a/tests/e2e/agent/test_agent_offset.py b/tests/e2e/agent/test_agent_offset.py index d2c872c3..4568dfed 100644 --- a/tests/e2e/agent/test_agent_offset.py +++ b/tests/e2e/agent/test_agent_offset.py @@ -1,16 +1,17 @@ """Tests for VisionAgent offset functionality with different locator types and models""" -import pathlib +from typing import TYPE_CHECKING from unittest.mock import Mock import pytest from PIL import Image as PILImage from askui.agent import VisionAgent -from askui.locators import AiElement, Element, Prompt, Text -from askui.locators.locators import Image +from askui.locators import Element from askui.models import ModelName -from askui.models.models import Point + +if TYPE_CHECKING: + from askui.models.models import Point @pytest.mark.parametrize( @@ -22,7 +23,9 @@ class TestVisionAgentOffset: """Test class for VisionAgent offset functionality.""" - def _setup_mocks(self, vision_agent: VisionAgent, github_login_screenshot: PILImage.Image): + def _setup_mocks( + self, vision_agent: VisionAgent, github_login_screenshot: PILImage.Image + ): """Helper method to setup common mocks.""" mock_mouse_move = Mock() mock_click = Mock() @@ -46,7 +49,9 @@ def test_click_with_positive_offset( locator = "Forgot password?" offset: Point = (10, 5) - mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks( + vision_agent, github_login_screenshot + ) # Get original position x, y = vision_agent.locate(locator, github_login_screenshot, model=model) @@ -70,7 +75,9 @@ def test_click_with_negative_offset( locator = "Forgot password?" offset: Point = (-10, -5) - mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks( + vision_agent, github_login_screenshot + ) # Get original position x, y = vision_agent.locate(locator, github_login_screenshot, model=model) @@ -94,7 +101,9 @@ def test_click_with_zero_offset( locator = "Forgot password?" offset: Point = (0, 0) - mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks( + vision_agent, github_login_screenshot + ) # Get original position x, y = vision_agent.locate(locator, github_login_screenshot, model=model) @@ -117,14 +126,19 @@ def test_click_with_point_locator_and_offset( point_locator: Point = (100, 100) offset: Point = (20, 15) - mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + mock_mouse_move, mock_click, _, mock_screenshot = self._setup_mocks( + vision_agent, github_login_screenshot + ) # Click with Point locator and offset should work vision_agent.click(point_locator, offset=offset, model=model) # Verify calls mock_screenshot.assert_not_called() - expected_x, expected_y = point_locator[0] + offset[0], point_locator[1] + offset[1] + expected_x, expected_y = ( + point_locator[0] + offset[0], + point_locator[1] + offset[1], + ) mock_mouse_move.assert_called_once_with(expected_x, expected_y) mock_click.assert_called_once_with("left", 1) @@ -138,7 +152,9 @@ def test_mouse_move_with_positive_offset( locator = "Forgot password?" offset: Point = (15, 10) - mock_mouse_move, _, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + mock_mouse_move, _, _, mock_screenshot = self._setup_mocks( + vision_agent, github_login_screenshot + ) # Get original position x, y = vision_agent.locate(locator, github_login_screenshot, model=model) @@ -161,7 +177,9 @@ def test_mouse_move_with_negative_offset( locator = "Forgot password?" offset: Point = (-15, -10) - mock_mouse_move, _, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + mock_mouse_move, _, _, mock_screenshot = self._setup_mocks( + vision_agent, github_login_screenshot + ) # Get original position x, y = vision_agent.locate(locator, github_login_screenshot, model=model) @@ -184,14 +202,19 @@ def test_mouse_move_with_point_locator_and_offset( point_locator: Point = (200, 150) offset: Point = (25, -10) - mock_mouse_move, _, _, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + mock_mouse_move, _, _, mock_screenshot = self._setup_mocks( + vision_agent, github_login_screenshot + ) # Mouse move with Point locator and offset should work vision_agent.mouse_move(point_locator, offset=offset, model=model) # Verify calls mock_screenshot.assert_not_called() - expected_x, expected_y = point_locator[0] + offset[0], point_locator[1] + offset[1] + expected_x, expected_y = ( + point_locator[0] + offset[0], + point_locator[1] + offset[1], + ) mock_mouse_move.assert_called_once_with(expected_x, expected_y) def test_type_with_positive_offset( @@ -205,7 +228,9 @@ def test_type_with_positive_offset( offset: Point = (5, 3) text = "test@example.com" - mock_mouse_move, mock_click, mock_type, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + mock_mouse_move, mock_click, mock_type, mock_screenshot = self._setup_mocks( + vision_agent, github_login_screenshot + ) # Get original position x, y = vision_agent.locate(locator, github_login_screenshot, model=model) @@ -231,7 +256,9 @@ def test_type_with_negative_offset( offset: Point = (-5, -3) text = "test@example.com" - mock_mouse_move, mock_click, mock_type, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + mock_mouse_move, mock_click, mock_type, mock_screenshot = self._setup_mocks( + vision_agent, github_login_screenshot + ) # Get original position x, y = vision_agent.locate(locator, github_login_screenshot, model=model) @@ -257,14 +284,19 @@ def test_type_with_point_locator_and_offset( offset: Point = (10, 5) text = "username" - mock_mouse_move, mock_click, mock_type, mock_screenshot = self._setup_mocks(vision_agent, github_login_screenshot) + mock_mouse_move, mock_click, mock_type, mock_screenshot = self._setup_mocks( + vision_agent, github_login_screenshot + ) # Type with Point locator and offset should work vision_agent.type(text, locator=point_locator, offset=offset, model=model) # Verify calls mock_screenshot.assert_not_called() - expected_x, expected_y = point_locator[0] + offset[0], point_locator[1] + offset[1] + expected_x, expected_y = ( + point_locator[0] + offset[0], + point_locator[1] + offset[1], + ) mock_mouse_move.assert_called_once_with(expected_x, expected_y) mock_click.assert_called_once_with("left", 3) mock_type.assert_called_once_with(text) From f945e6213783001c9be5c06f5995683635e704e5 Mon Sep 17 00:00:00 2001 From: Dominik Klotz Date: Thu, 30 Oct 2025 22:54:00 +0100 Subject: [PATCH 3/6] style: fix typecheck errors --- src/askui/agent.py | 10 +++++----- tests/e2e/agent/test_agent_offset.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/askui/agent.py b/src/askui/agent.py index e0bc92b5..2cc18b01 100644 --- a/src/askui/agent.py +++ b/src/askui/agent.py @@ -157,11 +157,11 @@ def _mouse_move( offset: Optional[Point], model: ModelComposition | str | None = None, ) -> None: - point: Point - if isinstance(locator, tuple): - point: Point = locator - else: - point: Point = self._locate(locator=locator, model=model)[0] + point: Point = ( + locator + if isinstance(locator, tuple) + else self._locate(locator=locator, model=model)[0] + ) if offset is not None: point = (point[0] + offset[0], point[1] + offset[1]) self.tools.os.mouse_move(point[0], point[1]) diff --git a/tests/e2e/agent/test_agent_offset.py b/tests/e2e/agent/test_agent_offset.py index 4568dfed..da29ea48 100644 --- a/tests/e2e/agent/test_agent_offset.py +++ b/tests/e2e/agent/test_agent_offset.py @@ -25,7 +25,7 @@ class TestVisionAgentOffset: def _setup_mocks( self, vision_agent: VisionAgent, github_login_screenshot: PILImage.Image - ): + ) -> tuple[Mock, Mock, Mock, Mock]: """Helper method to setup common mocks.""" mock_mouse_move = Mock() mock_click = Mock() From c6b8bef672a416d09a4b6f627bb5844aaccace83 Mon Sep 17 00:00:00 2001 From: Dominik Klotz Date: Thu, 30 Oct 2025 22:57:15 +0100 Subject: [PATCH 4/6] style: fix typecheck errors --- tests/e2e/agent/test_agent_offset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/agent/test_agent_offset.py b/tests/e2e/agent/test_agent_offset.py index da29ea48..684d580b 100644 --- a/tests/e2e/agent/test_agent_offset.py +++ b/tests/e2e/agent/test_agent_offset.py @@ -25,7 +25,7 @@ class TestVisionAgentOffset: def _setup_mocks( self, vision_agent: VisionAgent, github_login_screenshot: PILImage.Image - ) -> tuple[Mock, Mock, Mock, Mock]: + ) -> tuple[Mock, Mock, Mock, Mock]: # type: ignore[method-assign] """Helper method to setup common mocks.""" mock_mouse_move = Mock() mock_click = Mock() From ee5b406bbd90cf901fba5e15fd4fbb39836fd7f9 Mon Sep 17 00:00:00 2001 From: Dominik Klotz Date: Thu, 30 Oct 2025 22:58:23 +0100 Subject: [PATCH 5/6] style: fix typecheck errors --- tests/e2e/agent/test_agent_offset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/agent/test_agent_offset.py b/tests/e2e/agent/test_agent_offset.py index 684d580b..ee38d816 100644 --- a/tests/e2e/agent/test_agent_offset.py +++ b/tests/e2e/agent/test_agent_offset.py @@ -25,7 +25,7 @@ class TestVisionAgentOffset: def _setup_mocks( self, vision_agent: VisionAgent, github_login_screenshot: PILImage.Image - ) -> tuple[Mock, Mock, Mock, Mock]: # type: ignore[method-assign] + ) -> tuple[Mock, Mock, Mock, Mock]: # type: ignore[method-assign] """Helper method to setup common mocks.""" mock_mouse_move = Mock() mock_click = Mock() From a560fc813f96dc02bbfc3825890d8046f421ee2e Mon Sep 17 00:00:00 2001 From: Dominik Klotz Date: Thu, 30 Oct 2025 23:00:37 +0100 Subject: [PATCH 6/6] style: fix typecheck errors --- tests/e2e/agent/test_agent_offset.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/agent/test_agent_offset.py b/tests/e2e/agent/test_agent_offset.py index ee38d816..59c1f65c 100644 --- a/tests/e2e/agent/test_agent_offset.py +++ b/tests/e2e/agent/test_agent_offset.py @@ -25,17 +25,17 @@ class TestVisionAgentOffset: def _setup_mocks( self, vision_agent: VisionAgent, github_login_screenshot: PILImage.Image - ) -> tuple[Mock, Mock, Mock, Mock]: # type: ignore[method-assign] + ) -> tuple[Mock, Mock, Mock, Mock]: """Helper method to setup common mocks.""" mock_mouse_move = Mock() mock_click = Mock() mock_type = Mock() mock_screenshot = Mock(return_value=github_login_screenshot) - vision_agent.tools.os.mouse_move = mock_mouse_move - vision_agent.tools.os.click = mock_click - vision_agent.tools.os.type = mock_type - vision_agent.tools.os.screenshot = mock_screenshot + vision_agent.tools.os.mouse_move = mock_mouse_move # type: ignore[method-assign] + vision_agent.tools.os.click = mock_click # type: ignore[method-assign] + vision_agent.tools.os.type = mock_type # type: ignore[method-assign] + vision_agent.tools.os.screenshot = mock_screenshot # type: ignore[method-assign] return mock_mouse_move, mock_click, mock_type, mock_screenshot