Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions src/askui/models/shared/tools.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from typing import Any, cast
from typing import Any

from anthropic.types.beta import BetaToolParam, BetaToolUnionParam
from anthropic.types.beta.beta_tool_param import InputSchema
Expand Down Expand Up @@ -147,10 +147,7 @@ def _run_tool(
tool_use_id=tool_use_block_param.id,
)
try:
tool_result: ToolCallResult = cast(
"ToolCallResult",
tool(**tool_use_block_param.input), # type: ignore
)
tool_result: ToolCallResult = tool(**tool_use_block_param.input) # type: ignore
return ToolResultBlockParam(
content=_convert_to_content(tool_result),
tool_use_id=tool_use_block_param.id,
Expand Down
137 changes: 9 additions & 128 deletions src/askui/tools/askui/askui_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,13 @@

import grpc
from PIL import Image
from pydantic import BaseModel, Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
from typing_extensions import Self, override

from askui.container import telemetry
from askui.logger import logger
from askui.reporting import Reporter
from askui.tools.agent_os import AgentOs, Coordinate, ModifierKey, PcKey
from askui.tools.askui.askui_controller_settings import AskUiControllerSettings
from askui.tools.askui.askui_ui_controller_grpc.generated import (
Controller_V1_pb2 as controller_v1_pbs,
)
Expand Down Expand Up @@ -48,137 +47,19 @@
)


class RemoteDeviceController(BaseModel):
askui_remote_device_controller: pathlib.Path = Field(
alias="AskUIRemoteDeviceController"
)


class Executables(BaseModel):
executables: RemoteDeviceController = Field(alias="Executables")


class InstalledPackages(BaseModel):
remote_device_controller_uuid: Executables = Field(
alias="{aed1b543-e856-43ad-b1bc-19365d35c33e}"
)


class AskUiComponentRegistry(BaseModel):
definition_version: int = Field(alias="DefinitionVersion")
installed_packages: InstalledPackages = Field(alias="InstalledPackages")


class AskUiControllerSettings(BaseSettings):
model_config = SettingsConfigDict(
env_prefix="ASKUI_",
)

component_registry_file: pathlib.Path | None = None
installation_directory: pathlib.Path | None = None

@model_validator(mode="after")
def validate_either_component_registry_or_installation_directory_is_set(
self,
) -> "AskUiControllerSettings":
if self.component_registry_file is None and self.installation_directory is None:
error_msg = (
"Either ASKUI_COMPONENT_REGISTRY_FILE or "
"ASKUI_INSTALLATION_DIRECTORY environment variable must be set"
)
raise ValueError(error_msg)
return self


class AskUiControllerServer:
"""
Concrete implementation of `ControllerServer` for managing the AskUI Remote Device
Controller process.
Handles process discovery, startup, and shutdown for the native controller binary.

Args:
settings (AskUiControllerSettings | None, optional): Settings for the AskUI.
"""

def __init__(self) -> None:
def __init__(self, settings: AskUiControllerSettings | None = None) -> None:
self._process: subprocess.Popen[bytes] | None = None
self._settings = AskUiControllerSettings()

def _find_remote_device_controller(self) -> pathlib.Path:
if (
self._settings.installation_directory is not None
and self._settings.component_registry_file is None
):
logger.warning(
"Outdated AskUI Suite detected. Please update to the latest version."
)
askui_remote_device_controller_path = (
self._find_remote_device_controller_by_legacy_path()
)
if not askui_remote_device_controller_path.is_file():
error_msg = (
"AskUIRemoteDeviceController executable does not exist under "
f"'{askui_remote_device_controller_path}'"
)
raise FileNotFoundError(error_msg)
return askui_remote_device_controller_path
return self._find_remote_device_controller_by_component_registry()

def _find_remote_device_controller_by_component_registry(self) -> pathlib.Path:
assert self._settings.component_registry_file is not None, (
"Component registry file is not set"
)
component_registry = AskUiComponentRegistry.model_validate_json(
self._settings.component_registry_file.read_text()
)
askui_remote_device_controller_path = (
component_registry.installed_packages.remote_device_controller_uuid.executables.askui_remote_device_controller # noqa: E501
)
if not askui_remote_device_controller_path.is_file():
error_msg = (
"AskUIRemoteDeviceController executable does not exist under "
f"'{askui_remote_device_controller_path}'"
)
raise FileNotFoundError(error_msg)
return askui_remote_device_controller_path

def _find_remote_device_controller_by_legacy_path(self) -> pathlib.Path:
assert self._settings.installation_directory is not None, (
"Installation directory is not set"
)
match sys.platform:
case "win32":
return (
self._settings.installation_directory
/ "Binaries"
/ "resources"
/ "assets"
/ "binaries"
/ "AskuiRemoteDeviceController.exe"
)
case "darwin":
return (
self._settings.installation_directory
/ "Binaries"
/ "askui-ui-controller.app"
/ "Contents"
/ "Resources"
/ "assets"
/ "binaries"
/ "AskuiRemoteDeviceController"
)
case "linux":
return (
self._settings.installation_directory
/ "Binaries"
/ "resources"
/ "assets"
/ "binaries"
/ "AskuiRemoteDeviceController"
)
case _:
error_msg = (
f"Platform {sys.platform} not supported by "
"AskUI Remote Device Controller"
)
raise NotImplementedError(error_msg)
self._settings = settings or AskUiControllerSettings()

def _start_process(self, path: pathlib.Path) -> None:
self._process = subprocess.Popen(path)
Expand All @@ -198,11 +79,11 @@ def start(self, clean_up: bool = False) -> None:
and process_exists("AskuiRemoteDeviceController.exe")
):
self.clean_up()
remote_device_controller_path = self._find_remote_device_controller()
logger.debug(
"Starting AskUI Remote Device Controller: %s", remote_device_controller_path
"Starting AskUI Remote Device Controller: %s",
self._settings.controller_path,
)
self._start_process(remote_device_controller_path)
self._start_process(self._settings.controller_path)
time.sleep(0.5)

def clean_up(self) -> None:
Expand Down
150 changes: 150 additions & 0 deletions src/askui/tools/askui/askui_controller_settings.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
import pathlib
import sys
from functools import cached_property

from pydantic import BaseModel, Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
from typing_extensions import Self


class RemoteDeviceController(BaseModel):
askui_remote_device_controller: pathlib.Path = Field(
alias="AskUIRemoteDeviceController"
)


class Executables(BaseModel):
executables: RemoteDeviceController = Field(alias="Executables")


class InstalledPackages(BaseModel):
remote_device_controller_uuid: Executables = Field(
alias="{aed1b543-e856-43ad-b1bc-19365d35c33e}"
)


class AskUiComponentRegistry(BaseModel):
definition_version: int = Field(alias="DefinitionVersion")
installed_packages: InstalledPackages = Field(alias="InstalledPackages")


class AskUiControllerSettings(BaseSettings):
model_config = SettingsConfigDict(
env_prefix="ASKUI_",
)

component_registry_file: pathlib.Path | None = None
installation_directory: pathlib.Path | None = Field(
None,
deprecated="ASKUI_INSTALLATION_DIRECTORY has been deprecated in favor of "
"ASKUI_COMPONENT_REGISTRY_FILE and ASKUI_CONTROLLER_PATH. You may be using an "
"outdated AskUI Suite. If you think so, reinstall to upgrade the AskUI Suite "
"(see https://docs.askui.com/01-tutorials/00-installation).",
)
controller_path_setting: pathlib.Path | None = Field(
None,
validation_alias="ASKUI_CONTROLLER_PATH",
description="Path to the AskUI Remote Device Controller executable. Takes "
"precedence over ASKUI_COMPONENT_REGISTRY_FILE and ASKUI_INSTALLATION_DIRECTORY"
".",
)

@model_validator(mode="after")
def validate_either_component_registry_or_installation_directory_is_set(
self,
) -> "Self":
if (
self.component_registry_file is None
and self.installation_directory is None
and self.controller_path_setting is None
):
error_msg = (
"Either ASKUI_COMPONENT_REGISTRY_FILE, ASKUI_INSTALLATION_DIRECTORY, "
"or ASKUI_CONTROLLER_PATH environment variable must be set"
)
raise ValueError(error_msg)
return self

def _find_remote_device_controller_by_installation_directory(
self,
) -> pathlib.Path | None:
if self.installation_directory is None:
return None

return self._build_controller_path(self.installation_directory)

def _build_controller_path(
self, installation_directory: pathlib.Path
) -> pathlib.Path:
match sys.platform:
case "win32":
return (
installation_directory
/ "Binaries"
/ "resources"
/ "assets"
/ "binaries"
/ "AskuiRemoteDeviceController.exe"
)
case "darwin":
return (
installation_directory
/ "Binaries"
/ "askui-ui-controller.app"
/ "Contents"
/ "Resources"
/ "assets"
/ "binaries"
/ "AskuiRemoteDeviceController"
)
case "linux":
return (
installation_directory
/ "Binaries"
/ "resources"
/ "assets"
/ "binaries"
/ "AskuiRemoteDeviceController"
)
case _:
error_msg = (
f'Platform "{sys.platform}" not supported by '
"AskUI Remote Device Controller"
)
raise NotImplementedError(error_msg)

def _find_remote_device_controller_by_component_registry_file(
self,
) -> pathlib.Path | None:
if self.component_registry_file is None:
return None

component_registry = AskUiComponentRegistry.model_validate_json(
self.component_registry_file.read_text()
)
return (
component_registry.installed_packages.remote_device_controller_uuid.executables.askui_remote_device_controller # noqa: E501
)

@cached_property
def controller_path(self) -> pathlib.Path:
result = (
self.controller_path_setting
or self._find_remote_device_controller_by_component_registry_file()
or self._find_remote_device_controller_by_installation_directory()
)
assert result is not None, (
"No AskUI Remote Device Controller found. Please set the "
"ASKUI_COMPONENT_REGISTRY_FILE, ASKUI_INSTALLATION_DIRECTORY, or "
"ASKUI_CONTROLLER_PATH environment variable."
)
if not result.is_file():
error_msg = (
"AskUIRemoteDeviceController executable does not exist under "
f"`{result}`"
)
raise FileNotFoundError(error_msg)
return result


__all__ = ["AskUiControllerSettings"]
10 changes: 0 additions & 10 deletions tests/e2e/tools/askui/test_askui_controller.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import base64
import io
from pathlib import Path
from typing import Literal

import pytest
Expand Down Expand Up @@ -31,15 +30,6 @@ def controller_client(
)


def test_find_remote_device_controller_by_component_registry(
controller_server: AskUiControllerServer,
) -> None:
remote_device_controller_path = Path(
controller_server._find_remote_device_controller_by_component_registry()
)
assert "AskuiRemoteDeviceController" == remote_device_controller_path.stem


def test_actions(controller_client: AskUiControllerClient) -> None:
with controller_client:
controller_client.screenshot()
Expand Down
Empty file added tests/unit/tools/__init__.py
Empty file.
Empty file.
Loading