feat!: introduce Claude 4 Sonnet

- Updated model references in the documentation and codebase from `anthropic-claude-3-5-sonnet-20241022` to `claude-sonnet-4-20250514`.
- Added support for `stop_reason` to the `BaseAgent` class to manage stop reasons and raise appropriate exceptions.
- Added new exceptions `MaxTokensExceededError` and `ModelRefusalError` to handle specific model response scenarios via `stop_reason` on `act()`.
- Added support for thinking and tool choice via `AskUiComputerAgentSettings` and `ClaudeComputerAgentSettings`.
- Updated tests to reflect the new model references.

Author: onur-askui

README.md

@@ -128,8 +128,8 @@ with VisionAgent() as agent:
 
     agent.click("search field")
 
-    # Use Anthropic (Claude 3.5 Sonnet V2) as model
-    agent.click("search field", model="anthropic-claude-3-5-sonnet-20241022")
+    # Use Anthropic (Claude 4 Sonnet) as model
+    agent.click("search field", model="claude-sonnet-4-20250514")
 ```
 
 
@@ -201,7 +201,7 @@ with VisionAgent(model="askui-combo") as agent:
 
 # Use different models for different tasks
 with VisionAgent(model={
-    "act": "anthropic-claude-3-5-sonnet-20241022",  # Use Claude for act()
+    "act": "claude-sonnet-4-20250514",  # Use Claude for act()
     "get": "askui",  # Use AskUI for get()
     "locate": "askui-combo",  # Use AskUI combo for locate() (and click(), mouse_move())
 }) as agent:
@@ -240,7 +240,7 @@ Supported commands are: `act()`, `click()`, `get()`, `locate()`, `mouse_move()`
 Supported commands are: `act()`, `get()`, `click()`, `locate()`, `mouse_move()`
 | Model Name  | Info | Execution Speed | Security | Cost | Reliability |
 |-------------|--------------------|--------------|--------------|--------------|--------------|
-| `anthropic-claude-3-5-sonnet-20241022` | The [Computer Use](https://docs.anthropic.com/en/docs/agents-and-tools/computer-use) model from Antrophic is a Large Action Model (LAM), which can autonomously achieve goals. e.g. `"Book me a flight from Berlin to Rom"` | slow, >1s per step | Model hosting by Anthropic | High, up to 1,5$ per act | Not recommended for production usage |
+| `claude-sonnet-4-20250514` | The [Computer Use](https://docs.anthropic.com/en/docs/agents-and-tools/computer-use) model from Antrophic is a Large Action Model (LAM), which can autonomously achieve goals. e.g. `"Book me a flight from Berlin to Rom"` | slow, >1s per step | Model hosting by Anthropic | High, up to 1,5$ per act | Not recommended for production usage |
 > **Note:** Configure your Antrophic Model Provider [here](#3a-authenticate-with-an-ai-model-provider)
 
 
@@ -409,7 +409,7 @@ custom_models: ModelRegistry = {
     "dynamic-model": lambda: create_custom_model("your-api-key"),
     "dynamic-model-cached": lambda: create_custom_model_cached("your-api-key"),
     "askui": lambda: create_custom_model_cached("your-api-key"), # overrides default model
-    "anthropic-claude-3-5-sonnet-20241022": lambda: create_custom_model_cached("your-api-key"), # overrides model
+    "claude-sonnet-4-20250514": lambda: create_custom_model_cached("your-api-key"), # overrides model
 }
 
 
@@ -735,7 +735,7 @@ with VisionAgent() as agent:
 
 **AskUI Vision Agent** is a versatile AI powered framework that enables you to automate computer tasks in Python.
 
-It connects Agent OS with powerful computer use models like Anthropic's Claude Sonnet 3.5 v2 and the AskUI Prompt-to-Action series. It is your entry point for building complex automation scenarios with detailed instructions or let the agent explore new challenges on its own.
+It connects Agent OS with powerful computer use models like Anthropic's Claude Sonnet 4 and the AskUI Prompt-to-Action series. It is your entry point for building complex automation scenarios with detailed instructions or let the agent explore new challenges on its own.
 
 
 ![image](docs/assets/Architecture.svg)

src/askui/agent.py

@@ -11,7 +11,7 @@
 from askui.models.shared.computer_agent_cb_param import OnMessageCb
 from askui.models.shared.computer_agent_message_param import MessageParam
 from askui.models.shared.tools import ToolCollection
-from askui.tools.computer import Computer20241022Tool
+from askui.tools.computer import Computer20250124Tool
 from askui.tools.exception_tool import ExceptionTool
 from askui.utils.image_utils import ImageSource, Img
 
@@ -83,7 +83,7 @@ def __init__(
         )
         _models = initialize_default_model_registry(
             tool_collection=ToolCollection(
-                tools=[Computer20241022Tool(self.tools.os), ExceptionTool()]
+                tools=[Computer20250124Tool(self.tools.os), ExceptionTool()]
             ),
             reporter=self._reporter,
         )
@@ -598,6 +598,11 @@ def act(
         Returns:
             None
 
+        Raises:
+            MaxTokensExceededError: If the model reaches the maximum token limit
+                defined in the agent settings.
+            ModelRefusalError: If the model refuses to process the request.
+
         Example:
             ```python
             from askui import VisionAgent

src/askui/models/anthropic/computer_agent.py

@@ -1,6 +1,11 @@
 from typing import TYPE_CHECKING, cast
 
 from anthropic import Anthropic
+from anthropic.types import (
+    ThinkingConfigDisabledParam,
+    ThinkingConfigEnabledParam,
+    ThinkingConfigParam,
+)
 from typing_extensions import override
 
 from askui.models.anthropic.settings import ClaudeComputerAgentSettings
@@ -30,16 +35,23 @@ def __init__(
     def _create_message(
         self, messages: list[MessageParam], model_choice: str
     ) -> MessageParam:
+        _thinking: ThinkingConfigParam = ThinkingConfigDisabledParam(type="disabled")
+
+        if self._settings.thinking:
+            _thinking = ThinkingConfigEnabledParam(
+                type="enabled",
+                budget_tokens=self._settings.thinking.budget_tokens,
+            )
+
         response = self._client.beta.messages.with_raw_response.create(
             max_tokens=self._settings.max_tokens,
-            messages=[
-                cast("BetaMessageParam", message.model_dump(mode="json"))
-                for message in messages
-            ],
+            messages=cast("list[BetaMessageParam]", messages),
             model=ANTHROPIC_MODEL_NAME_MAPPING[ModelName(model_choice)],
             system=[self._system],
             tools=self._tool_collection.to_params(),
             betas=self._settings.betas,
+            thinking=_thinking,
+            tool_choice=self._settings.tool_choice,
         )
         parsed_response = response.parse()
         return MessageParam.model_validate(parsed_response.model_dump())

src/askui/models/anthropic/settings.py

@@ -1,10 +1,12 @@
+from anthropic import NotGiven
+from anthropic.types.beta import BetaToolChoiceAutoParam, BetaToolChoiceParam
 from pydantic import BaseModel, Field, SecretStr
 from pydantic_settings import BaseSettings
 
 from askui.models.shared.computer_agent import ComputerAgentSettingsBase
-from askui.models.shared.settings import ChatCompletionsCreateSettings
+from askui.models.shared.settings import ChatCompletionsCreateSettings, ThinkingSettings
 
-COMPUTER_USE_BETA_FLAG = "computer-use-2024-10-22"
+NOT_GIVEN = NotGiven()
 
 
 class AnthropicSettings(BaseSettings):
@@ -28,4 +30,7 @@ class ClaudeSettings(ClaudeSettingsBase):
 
 
 class ClaudeComputerAgentSettings(ComputerAgentSettingsBase, ClaudeSettingsBase):
-    pass
+    thinking: ThinkingSettings | None = None
+    tool_choice: BetaToolChoiceParam | BetaToolChoiceAutoParam = (
+        BetaToolChoiceAutoParam(type="auto")
+    )

src/askui/models/askui/computer_agent.py

@@ -1,6 +1,15 @@
 import httpx
+from anthropic import NotGiven
+from anthropic.types import ThinkingConfigEnabledParam
+from anthropic.types.beta import (
+    BetaTextBlockParam,
+    BetaThinkingConfigParam,
+    BetaToolChoiceParam,
+    BetaToolUnionParam,
+)
+from pydantic import BaseModel, ConfigDict
 from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
-from typing_extensions import override
+from typing_extensions import Literal, override
 
 from askui.models.askui.settings import AskUiComputerAgentSettings
 from askui.models.shared.computer_agent import ComputerAgent
@@ -10,6 +19,21 @@
 
 from ...logger import logger
 
+NOT_GIVEN = NotGiven()
+
+
+class RequestBody(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    max_tokens: int
+    messages: list[MessageParam]
+    provider: Literal["gcp_vertex"] = "gcp_vertex"
+    model: str
+    tools: list[BetaToolUnionParam]
+    betas: list[str]
+    system: list[BetaTextBlockParam]
+    thinking: BetaThinkingConfigParam | NotGiven = NOT_GIVEN
+    tool_choice: BetaToolChoiceParam | NotGiven = NOT_GIVEN
+
 
 def is_retryable_error(exception: BaseException) -> bool:
     """Check if the exception is a retryable error (status codes 429 or 529)."""
@@ -47,20 +71,26 @@ def _create_message(
         model_choice: str,  # noqa: ARG002
     ) -> MessageParam:
         try:
-            request_body = {
-                "max_tokens": self._settings.max_tokens,
-                "messages": [msg.model_dump(mode="json") for msg in messages],
-                "model": self._settings.model,
-                "tools": self._tool_collection.to_params(),
-                "betas": self._settings.betas,
-                "system": [self._system],
-            }
+            request_body = RequestBody(
+                max_tokens=self._settings.max_tokens,
+                messages=messages,
+                model=self._settings.model,
+                tools=self._tool_collection.to_params(),
+                betas=self._settings.betas,
+                system=[self._system],
+                tool_choice=self._settings.tool_choice,
+            )
+            if self._settings.thinking:
+                request_body.thinking = ThinkingConfigEnabledParam(
+                    budget_tokens=self._settings.thinking.budget_tokens,
+                    type="enabled",
+                )
+
             response = self._client.post(
-                "/act/inference", json=request_body, timeout=300.0
+                "/act/inference", json=request_body.model_dump(), timeout=300.0
             )
             response.raise_for_status()
-            response_data = response.json()
-            return MessageParam.model_validate(response_data)
+            return MessageParam.model_validate(response.json())
         except Exception as e:  # noqa: BLE001
             if is_retryable_error(e):
                 logger.debug(e)

src/askui/models/askui/settings.py

@@ -1,12 +1,20 @@
 import base64
 from functools import cached_property
 
+from anthropic import NotGiven
+from anthropic.types.beta import (
+    BetaToolChoiceAutoParam,
+    BetaToolChoiceParam,
+)
 from pydantic import UUID4, Field, HttpUrl, SecretStr
 from pydantic_settings import BaseSettings
 
 from askui.models.models import ModelName
 from askui.models.shared.base_agent import AgentSettingsBase
 from askui.models.shared.computer_agent import ComputerAgentSettingsBase
+from askui.models.shared.settings import ThinkingSettings
+
+NOT_GIVEN = NotGiven()
 
 
 class AskUiSettings(BaseSettings):
@@ -40,12 +48,16 @@ def base_url(self) -> str:
 
 
 class AskUiComputerAgentSettings(ComputerAgentSettingsBase):
-    model: str = ModelName.ANTHROPIC__CLAUDE__3_5__SONNET__20241022
+    model: str = ModelName.ANTHROPIC__CLAUDE__SONNET__4__20250514
     askui: AskUiSettings = Field(default_factory=AskUiSettings)
+    thinking: ThinkingSettings | None = None
+    tool_choice: BetaToolChoiceParam | BetaToolChoiceAutoParam = (
+        BetaToolChoiceAutoParam(type="auto")
+    )
 
 
 class AskUiAndroidAgentSettings(AgentSettingsBase):
     """Settings for AskUI Android agent."""
 
-    model: str = ModelName.ANTHROPIC__CLAUDE__3_5__SONNET__20241022
+    model: str = ModelName.ANTHROPIC__CLAUDE__SONNET__4__20250514
     askui: AskUiSettings = Field(default_factory=AskUiSettings)

src/askui/models/exceptions.py

@@ -99,3 +99,41 @@ def __init__(
             message=f'Model "{model_choice}" is an instance of {actual_type.mro()}, '
             f"expected it to be an instance of {expected_type.mro()}",
         )
+
+
+class MaxTokensExceededError(AutomationError):
+    """Exception raised when the model stops due to reaching the maximum token limit.
+
+    Args:
+        max_tokens (int): The maximum token limit that was exceeded.
+        message (str, optional): Custom error message. If not provided, a default
+            message will be generated.
+    """
+
+    def __init__(self, max_tokens: int, message: str | None = None):
+        self.max_tokens = max_tokens
+        error_msg = (
+            f"Model stopped due to reaching maximum token limit of {max_tokens} tokens"
+            if message is None
+            else message
+        )
+        super().__init__(error_msg)
+
+
+class ModelRefusalError(AutomationError):
+    """Exception raised when the model refuses to process the request.
+
+    Args:
+        reason (str, optional): The reason for the refusal if provided by the model.
+        message (str, optional): Custom error message. If not provided, a default
+            message will be generated.
+    """
+
+    def __init__(self, reason: str | None = None, message: str | None = None):
+        self.reason = reason
+        error_msg = (
+            f"Model refused to process the request{f': {reason}' if reason else ''}"
+            if message is None
+            else message
+        )
+        super().__init__(error_msg)

src/askui/models/model_router.py

@@ -1,6 +1,7 @@
 import functools
 from typing import Type, overload
 
+from anthropic.types.beta import BetaToolChoiceAutoParam
 from typing_extensions import Literal
 
 from askui.locators.locators import Locator
@@ -34,6 +35,7 @@
 from askui.models.shared.computer_agent_cb_param import OnMessageCb
 from askui.models.shared.computer_agent_message_param import MessageParam
 from askui.models.shared.facade import ModelFacade
+from askui.models.shared.settings import ThinkingSettings
 from askui.models.shared.tools import ToolCollection
 from askui.models.types.response_schemas import ResponseSchema
 from askui.reporting import CompositeReporter, Reporter
@@ -102,6 +104,12 @@ def askui_facade() -> ModelFacade:
             reporter=reporter,
             settings=AskUiComputerAgentSettings(
                 askui=askui_settings(),
+                tool_choice=BetaToolChoiceAutoParam(
+                    type="auto", disable_parallel_tool_use=False
+                ),
+                thinking=ThinkingSettings(
+                    budget_tokens=2000,
+                ),
             ),
         )
         return ModelFacade(
@@ -122,7 +130,7 @@ def hf_spaces_handler() -> HFSpacesHandler:
         ModelName.ASKUI__COMBO: askui_model_router,
         ModelName.ASKUI__OCR: askui_model_router,
         ModelName.ASKUI__PTA: askui_model_router,
-        ModelName.ANTHROPIC__CLAUDE__3_5__SONNET__20241022: anthropic_facade,
+        ModelName.ANTHROPIC__CLAUDE__SONNET__4__20250514: anthropic_facade,
         ModelName.HF__SPACES__ASKUI__PTA_1: hf_spaces_handler,
         ModelName.HF__SPACES__QWEN__QWEN2_VL_2B_INSTRUCT: hf_spaces_handler,
         ModelName.HF__SPACES__QWEN__QWEN2_VL_7B_INSTRUCT: hf_spaces_handler,

src/askui/models/models.py

@@ -23,7 +23,7 @@ class ModelName(str, Enum):
     elements.
     """
 
-    ANTHROPIC__CLAUDE__3_5__SONNET__20241022 = "anthropic-claude-3-5-sonnet-20241022"
+    ANTHROPIC__CLAUDE__SONNET__4__20250514 = "claude-sonnet-4-20250514"
     ASKUI = "askui"
     ASKUI__AI_ELEMENT = "askui-ai-element"
     ASKUI__COMBO = "askui-combo"
@@ -38,7 +38,7 @@ class ModelName(str, Enum):
 
 
 ANTHROPIC_MODEL_NAME_MAPPING = {
-    ModelName.ANTHROPIC__CLAUDE__3_5__SONNET__20241022: "claude-3-5-sonnet-20241022",
+    ModelName.ANTHROPIC__CLAUDE__SONNET__4__20250514: "claude-sonnet-4-20250514",
 }
 
 

src/askui/models/shared/base_agent.py

@@ -5,6 +5,7 @@
 from pydantic import BaseModel
 from typing_extensions import TypeVar, override
 
+from askui.models.exceptions import MaxTokensExceededError, ModelRefusalError
 from askui.models.models import ActModel
 from askui.models.shared.computer_agent_cb_param import OnMessageCb, OnMessageCbParam
 from askui.models.shared.computer_agent_message_param import (
@@ -117,6 +118,8 @@ def _step(
             )
         else:
             message_by_assistant = messages[-1]
+
+        self._handle_stop_reason(message_by_assistant)
         if tool_result_message := self._use_tools(message_by_assistant):
             if tool_result_message := self._call_on_message(
                 on_message, tool_result_message, messages
@@ -238,3 +241,11 @@ def _maybe_filter_to_n_most_recent_images(
                     new_content.append(content)
                 tool_result.content = new_content
         return messages
+
+    def _handle_stop_reason(self, message: MessageParam) -> None:
+        if message.stop_reason == "max_tokens":
+            error_msg = f"Model stopped due to reaching maximum token limit of {self._settings.max_tokens} tokens"  # noqa: E501
+            raise MaxTokensExceededError(self._settings.max_tokens, error_msg)
+        if message.stop_reason == "refusal":
+            error_msg = "Model refused to process the request"
+            raise ModelRefusalError(message=error_msg)