feat/web-support: add web agent

Author: adi-wan-askui

pdm.lock

@@ -24,6 +24,7 @@ dependencies = [
     "httpx>=0.28.1",
     "fastmcp>=2.3.4",
     "pure-python-adb>=0.3.0.dev0",
+    "playwright>=1.0.0",
 ]
 requires-python = ">=3.10"
 readme = "README.md"
@@ -85,6 +86,7 @@ test = [
     "types-pyperclip>=1.8.2.20240311",
     "pytest-timeout>=2.4.0",
     "types-pynput>=1.8.1.20250318",
+    "playwright>=1.41.0",
 ]
 
 

pyproject.toml

@@ -28,136 +28,12 @@
 from .tools import AgentToolbox, ModifierKey, PcKey
 from .tools.askui import AskUiControllerClient
 
-_PC_KEY = [
-    "backspace",
-    "delete",
-    "enter",
-    "tab",
-    "escape",
-    "up",
-    "down",
-    "right",
-    "left",
-    "home",
-    "end",
-    "pageup",
-    "pagedown",
-    "f1",
-    "f2",
-    "f3",
-    "f4",
-    "f5",
-    "f6",
-    "f7",
-    "f8",
-    "f9",
-    "f10",
-    "f11",
-    "f12",
-    "space",
-    "0",
-    "1",
-    "2",
-    "3",
-    "4",
-    "5",
-    "6",
-    "7",
-    "8",
-    "9",
-    "a",
-    "b",
-    "c",
-    "d",
-    "e",
-    "f",
-    "g",
-    "h",
-    "i",
-    "j",
-    "k",
-    "l",
-    "m",
-    "n",
-    "o",
-    "p",
-    "q",
-    "r",
-    "s",
-    "t",
-    "u",
-    "v",
-    "w",
-    "x",
-    "y",
-    "z",
-    "A",
-    "B",
-    "C",
-    "D",
-    "E",
-    "F",
-    "G",
-    "H",
-    "I",
-    "J",
-    "K",
-    "L",
-    "M",
-    "N",
-    "O",
-    "P",
-    "Q",
-    "R",
-    "S",
-    "T",
-    "U",
-    "V",
-    "W",
-    "X",
-    "Y",
-    "Z",
-    "!",
-    '"',
-    "#",
-    "$",
-    "%",
-    "&",
-    "'",
-    "(",
-    ")",
-    "*",
-    "+",
-    ",",
-    "-",
-    ".",
-    "/",
-    ":",
-    ";",
-    "<",
-    "=",
-    ">",
-    "?",
-    "@",
-    "[",
-    "\\",
-    "]",
-    "^",
-    "_",
-    "`",
-    "{",
-    "|",
-    "}",
-    "~",
-]
-
 _SYSTEM_PROMPT = f"""<SYSTEM_CAPABILITY>
 * You are utilising a {sys.platform} machine using {platform.machine()} architecture with internet access.
 * When asked to perform web tasks try to open the browser (firefox, chrome, safari, ...) if not already open. Often you can find the browser icons in the toolbars of the operating systems.
 * When viewing a page it can be helpful to zoom out so that you can see everything on the page.  Either that, or make sure you scroll down to see everything before deciding something isn't available.
 * When using your computer function calls, they take a while to run and send back to you.  Where possible/feasible, try to chain multiple of these calls all into one function calls request.
-* Valid keyboard keys available are {", ".join(_PC_KEY)}
-* The current date is {datetime.now(timezone.utc).strftime("%A, %B %d, %Y").replace(" 0", " ")}.
+* The current date and time is {datetime.now(timezone.utc).strftime("%A, %B %d, %Y %H:%M:%S %z")}.
 </SYSTEM_CAPABILITY>
 
 <IMPORTANT>
@@ -201,7 +77,7 @@ class VisionAgent(AgentBase):
         ```
     """
 
-    @telemetry.record_call(exclude={"model_router", "reporters", "tools"})
+    @telemetry.record_call(exclude={"model_router", "reporters", "tools", "act_tools"})
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def __init__(
         self,
@@ -212,6 +88,7 @@ def __init__(
         model: ModelChoice | ModelComposition | str | None = None,
         retry: Retry | None = None,
         models: ModelRegistry | None = None,
+        act_tools: list[Tool] | None = None,
     ) -> None:
         reporter = CompositeReporter(reporters=reporters)
         self.tools = tools or AgentToolbox(
@@ -228,7 +105,8 @@ def __init__(
             models=models,
             tools=[
                 ExceptionTool(),
-            ],
+            ]
+            + (act_tools or []),
             settings=_SETTINGS,
             agent_os=self.tools.os,
         )

src/askui/agent.py

@@ -206,7 +206,7 @@ def create_message(
         }
         response = self._post(
             "/act/inference",
-            json={k: v for k, v in json.items() if v is not NOT_GIVEN},
+            json={k: v for k, v in json.items() if not isinstance(v, NotGiven)},
             timeout=300.0,
         )
         return MessageParam.model_validate_json(response.text)

src/askui/models/askui/inference_api.py

@@ -4,11 +4,15 @@
 from PIL import Image
 from pydantic import BaseModel
 
-ModifierKey = Literal["command", "alt", "control", "shift", "right_shift"]
+ModifierKey = Literal[
+    "command",
+    "alt",
+    "control",
+    "shift",
+    "right_shift",
+]
 """Modifier keys for keyboard actions."""
 
-ModifierKeys: list[ModifierKey] = ["command", "alt", "control", "shift", "right_shift"]
-
 PcKey = Literal[
     "backspace",
     "delete",
@@ -308,7 +312,6 @@ def keyboard_tap(
         """
         raise NotImplementedError
 
-    @abstractmethod
     def set_display(self, display: int = 1) -> None:
         """
         Sets the active display for screen interactions.
@@ -319,7 +322,6 @@ def set_display(self, display: int = 1) -> None:
         """
         raise NotImplementedError
 
-    @abstractmethod
     def run_command(self, command: str, timeout_ms: int = 30000) -> None:
         """
         Executes a shell command.