Open up underlying messages of VisionAgent.act() and extract api from chat (experimental)

.cursorrules

@@ -0,0 +1,74 @@
+# Code Style
+- Use `_` prefix for all private variables, constants, functions, methods, properties, etc. that don't need to be accessible from outside the module
+- Mark everything as private that does not absolutely need to be accessible from outside the module
+- Use `@override` (imported from `typing_extensions`) decorator for all methods that override a parent class method
+- Use type hints for all variables, functions, methods, properties, return values, parameters etc.
+- Omit `Any` within type hints unless absolutely necessary
+- Use built-in types (e.g., `list`, `dict`, `tuple`, `set`, `str | None`) instead of types from `typing` module (e.g. `List`, `Dict`, `Tuple`, `Set`, `Optional`, `Union`) wherever possible
+- Instead of `Optional` use `| None`
+- Create a `__init__.py` file in each folder
+- Never pass literals, e.g., `error_msg`, directly to `Exceptions`, but instead assign them to variables and pass them to the exception, e.g., `raise FileNotFoundError(error_msg)` instead of `raise FileNotFoundError(f"Thread {thread_id} not found")`
+
+## FastAPI
+- Instead of defining `response_model` within route annotation, use the model as the response type in the function signature
+- Do not assign `None` to dependencies but instead move it before arguments with default values
+
+# Testing
+- Use `pytest-mock` for mocking in tests wherever you need to mock something and pytest-mock can do the job.
+
+# Documentation
+
+## Docstrings
+- All public functions, constants, classes, types etc. should have docstrings
+- Document the constructor (`__init__`) args as part of the class docstring
+- Omit the `__init__` docstring
+- All function parameter should be documented with their type (followed by `, optional` if there is a default value) in parenthesis and description
+- In descriptions, use backticks for all code references (variables, types, etc.), including types, e.g., `str`
+- When referencing a function, use the function name in backticks plus parentheses, e.g., `click()`
+- When referencing a class, use the class name in backticks, e.g., `VisionAgent`
+- When referencing a method, use the class name in backticks plus the method name in parentheses, e.g., `VisionAgent.click()`
+- When referencing a class attribute, use the class name in backticks plus the attribute name, e.g., `VisionAgent.display`
+- Use `Example` section for code examples
+- Use `Returns` section for return values
+- Use `Raises` section for exceptions listing all possible exceptions that can be raised by the function
+- Use `Notes` section for additional notes
+- Use `See Also` section for related functions
+- Use `References` section for references
+- Use `Examples` section for code examples
+- Example of a good docstring:
+  ```python
+  def locate(
+      self,
+      locator: str | Locator,
+      screenshot: Img | None = None,
+      model: ModelComposition | str | None = None,
+  ) -> Point:
+      """
+      Find the position of the UI element identified by the `locator` using the `model`.
+
+      Args:
+          locator (str | Locator): The identifier or description of the element to locate.
+          screenshot (Img | None, optional): The screenshot to use for locating the
+              element. Can be a path to an image file, a PIL Image object or a data URL.
+              If `None`, takes a screenshot of the currently selected screen.
+          model (ModelComposition | str | None, optional): The composition or name of
+              the model(s) to be used for locating the element using the `locator`.
+
+      Returns:
+          Point: The coordinates of a point on the element, usually the center of the element, as a tuple (x, y).
+
+      Raises:
+          ValueError: If the arguments are not of the correct type.
+          ElementNotFoundError: If no element can be found.
+
+      Example:
+          ```python
+          from askui import VisionAgent
+
+          with VisionAgent() as agent:
+              point = agent.locate("Submit button")
+              print(f"Element found at coordinates: {point}")
+          ```
+      """
+      ...
+  ```

.nvmrc

@@ -0,0 +1 @@
+22

.vscode/extensions.json

@@ -15,7 +15,6 @@
         "tamasfe.even-better-toml",
         "visualstudioexptteam.vscodeintellicode",
         "dongli.python-preview",
-        "mintlify.document",
         "kaih2o.python-resource-monitor",
         "littlefoxteam.vscode-python-test-adapter",
         "almenon.arepl"

README.md

@@ -284,32 +284,48 @@ You can create and use your own models by subclassing the `ActModel` (used for `
 Here's how to create and use custom models:
 
 ```python
+import functools
 from askui import (
     ActModel,
     GetModel,
     LocateModel,
     Locator,
     ImageSource,
+    MessageParam,
     ModelComposition,
     ModelRegistry,
+    OnMessageCb,
     Point,
     ResponseSchema,
     VisionAgent,
 )
 from typing import Type
+from typing_extensions import override
 
 # Define custom models
 class MyActModel(ActModel):
-    def act(self, goal: str, model_choice: str) -> None:
+    @override
+    def act(
+        self,
+        messages: list[MessageParam],
+        model_choice: str,
+        on_message: OnMessageCb | None = None,
+    ) -> None:
         # Implement custom act logic, e.g.:
         # - Use a different AI model
         # - Implement custom business logic
         # - Call external services
-        print(f"Custom act model executing goal: {goal}")
+        if len(messages) > 0:
+            goal = messages[0].content
+            print(f"Custom act model executing goal: {goal}")
+        else:
+            error_msg = "No messages provided"
+            raise ValueError(error_msg)
 
 # Because Python supports multiple inheritance, we can subclass both `GetModel` and `LocateModel` (and even `ActModel`)
 # to create a model that can both get and locate elements.
 class MyGetAndLocateModel(GetModel, LocateModel):
+    @override
     def get(
         self,
         query: str,
@@ -324,6 +340,7 @@ class MyGetAndLocateModel(GetModel, LocateModel):
         return f"Custom response to query: {query}"
 
 
+    @override
     def locate(
         self,
         locator: str | Locator,
@@ -366,11 +383,15 @@ You can also use model factories if you need to create models dynamically:
 
 ```python
 class DynamicActModel(ActModel):
-    def act(self, goal: str, model_choice: str) -> None:
-        # Use api_key in implementation
+    @override
+    def act(
+        self,
+        messages: list[MessageParam],
+        model_choice: str,
+        on_message: OnMessageCb | None = None,
+    ) -> None:
         pass
 
-
 # going to be called each time model is chosen using `model` parameter
 def create_custom_model(api_key: str) -> ActModel:
     return DynamicActModel()
@@ -410,7 +431,7 @@ The controller for the operating system.
 
 ```python
 agent.tools.os.click("left", 2) # clicking
-agent.tools.os.mouse(100, 100) # mouse movement
+agent.tools.os.mouse_move(100, 100) # mouse movement
 agent.tools.os.keyboard_tap("v", modifier_keys=["control"]) # Paste
 # and many more
 ```