diff --git a/packages/narada-pyodide/pyproject.toml b/packages/narada-pyodide/pyproject.toml index 62f8333..10bfc3b 100644 --- a/packages/narada-pyodide/pyproject.toml +++ b/packages/narada-pyodide/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "narada-pyodide" -version = "0.0.54" +version = "0.0.55" description = "Pyodide-compatible Python client SDK for Narada" license = "Apache-2.0" readme = "README.md" diff --git a/packages/narada-pyodide/src/narada/retry.py b/packages/narada-pyodide/src/narada/retry.py new file mode 100644 index 0000000..d7f6abd --- /dev/null +++ b/packages/narada-pyodide/src/narada/retry.py @@ -0,0 +1,103 @@ +import asyncio +import time +from http import HTTPStatus +from typing import Any + +from pyodide.http import pyfetch + +__all__ = ["pyfetch_with_retries"] + +_PYFETCH_RETRY_ATTEMPTS = 3 +_PYFETCH_INITIAL_BACKOFF_SECONDS = 0.5 +_PYFETCH_BACKOFF_MULTIPLIER = 2.0 +_PYFETCH_RETRYABLE_STATUSES = frozenset( + status.value + for status in ( + HTTPStatus.REQUEST_TIMEOUT, + HTTPStatus.TOO_MANY_REQUESTS, + HTTPStatus.INTERNAL_SERVER_ERROR, + HTTPStatus.BAD_GATEWAY, + HTTPStatus.SERVICE_UNAVAILABLE, + HTTPStatus.GATEWAY_TIMEOUT, + ) +) + + +def _abort_signal_aborted(signal: Any) -> bool: + if signal is None: + return False + + try: + return bool(getattr(signal, "aborted", False)) + except Exception: + return False + + +async def _sleep_before_pyfetch_retry( + backoff_seconds: float, retry_deadline: float | None +) -> bool: + if retry_deadline is None: + await asyncio.sleep(backoff_seconds) + return True + + remaining_seconds = retry_deadline - time.monotonic() + if remaining_seconds <= backoff_seconds: + return False + + await asyncio.sleep(backoff_seconds) + return time.monotonic() < retry_deadline + + +async def pyfetch_with_retries( + url: str, + *, + max_attempts: int = _PYFETCH_RETRY_ATTEMPTS, + initial_backoff_seconds: float = _PYFETCH_INITIAL_BACKOFF_SECONDS, + backoff_multiplier: float = _PYFETCH_BACKOFF_MULTIPLIER, + retry_statuses: frozenset[int] | None = _PYFETCH_RETRYABLE_STATUSES, + retry_deadline: float | None = None, + **kwargs: Any, +) -> Any: + """Retries transient pyfetch exceptions with exponential backoff. + + HTTP responses are returned as-is once attempts are exhausted so each caller + can preserve its existing status-specific handling. + """ + if max_attempts < 1: + raise ValueError("max_attempts must be at least 1") + + backoff_seconds = initial_backoff_seconds + signal = kwargs.get("signal") + for attempt in range(max_attempts): + if retry_deadline is not None and time.monotonic() >= retry_deadline: + raise asyncio.TimeoutError + + try: + response = await pyfetch(url, **kwargs) + except Exception: + if ( + attempt == max_attempts - 1 + or _abort_signal_aborted(signal) + or (retry_deadline is not None and time.monotonic() >= retry_deadline) + ): + raise + + if not await _sleep_before_pyfetch_retry(backoff_seconds, retry_deadline): + raise + backoff_seconds *= backoff_multiplier + continue + + if ( + retry_statuses + and response.status in retry_statuses + and attempt < max_attempts - 1 + and not _abort_signal_aborted(signal) + and (retry_deadline is None or time.monotonic() < retry_deadline) + ): + if await _sleep_before_pyfetch_retry(backoff_seconds, retry_deadline): + backoff_seconds *= backoff_multiplier + continue + + return response + + raise AssertionError("unreachable") diff --git a/packages/narada-pyodide/src/narada/window.py b/packages/narada-pyodide/src/narada/window.py index 0bf4d3b..a9c1716 100644 --- a/packages/narada-pyodide/src/narada/window.py +++ b/packages/narada-pyodide/src/narada/window.py @@ -81,6 +81,7 @@ from pyodide.http import pyfetch from . import _trace +from .retry import pyfetch_with_retries # Magic variable injected by the JavaScript harness that stores the IDs of the current runnables # in the stack on the frontend. @@ -450,10 +451,11 @@ async def dispatch_request( create_once_callable(abort_controller.abort), (deadline - now) * 1000, ) - fetch_response = await pyfetch( + fetch_response = await pyfetch_with_retries( f"{self._base_url}/remote-dispatch/responses/{request_id}", headers=headers, signal=signal, + retry_deadline=deadline, ) if not fetch_response.ok: @@ -1182,7 +1184,7 @@ async def _fetch_presigned_download_url( session_id: str, key: str, ) -> str: - fetch_response = await pyfetch( + fetch_response = await pyfetch_with_retries( _build_cloud_browser_url( base_url, "/cloud-browser/replay/download-url", @@ -1205,7 +1207,7 @@ async def _get_cloud_browser_downloads( auth_headers: dict[str, str], session_id: str, ) -> list[SessionDownloadItem]: - fetch_response = await pyfetch( + fetch_response = await pyfetch_with_retries( _build_cloud_browser_url( base_url, "/cloud-browser/replay/downloads", diff --git a/packages/narada-pyodide/tests/test_cloud_browser.py b/packages/narada-pyodide/tests/test_cloud_browser.py index eecedc8..3a058a1 100644 --- a/packages/narada-pyodide/tests/test_cloud_browser.py +++ b/packages/narada-pyodide/tests/test_cloud_browser.py @@ -277,6 +277,63 @@ async def test_cloud_browser_window_dispatch_request_omits_parent_run_ids( assert "parentRunIds" not in payload +@pytest.mark.asyncio +async def test_cloud_browser_window_dispatch_request_retries_poll_fetch_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyfetch = AsyncMock( + side_effect=[ + _FakeResponse(json_data={"requestId": "req-123"}), + RuntimeError("temporary fetch failure"), + _FakeResponse(ok=False, status=502, text_data="bad gateway"), + _FakeResponse(json_data={"status": "success", "response": None}), + ] + ) + _, _, window_module = _import_pyodide_narada(monkeypatch, pyfetch=pyfetch) + sleep_delays: list[float] = [] + + async def fake_sleep(delay: float) -> None: + sleep_delays.append(delay) + + retry_module = sys.modules["narada.retry"] + monkeypatch.setattr(retry_module.asyncio, "sleep", fake_sleep) + + window = window_module.CloudBrowserWindow( + browser_window_id="browser-window-123", + session_id="session-123", + api_key="test-api-key", + ) + response = await window.dispatch_request(prompt="hello from cloud browser") + + assert response["status"] == "success" + assert pyfetch.await_count == 4 + assert sleep_delays == [0.5, 1.0] + + +@pytest.mark.asyncio +async def test_pyfetch_with_retries_does_not_start_retry_at_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyfetch = AsyncMock( + return_value=_FakeResponse(ok=False, status=502, text_data="bad gateway") + ) + _import_pyodide_narada(monkeypatch, pyfetch=pyfetch) + retry_module = sys.modules["narada.retry"] + sleep = AsyncMock() + + monkeypatch.setattr(retry_module.asyncio, "sleep", sleep) + monkeypatch.setattr(retry_module.time, "monotonic", lambda: 10.0) + + response = await retry_module.pyfetch_with_retries( + "https://example.test/retry", + retry_deadline=10.5, + ) + + assert response.status == 502 + assert pyfetch.await_count == 1 + sleep.assert_not_awaited() + + @pytest.mark.asyncio async def test_dispatch_request_emits_string_trace_agent_type_for_sdk_enum( monkeypatch: pytest.MonkeyPatch, diff --git a/uv.lock b/uv.lock index 2f2b9a2..4d9f5a7 100644 --- a/uv.lock +++ b/uv.lock @@ -356,7 +356,7 @@ requires-dist = [{ name = "pydantic", specifier = "==2.12.5" }] [[package]] name = "narada-pyodide" -version = "0.0.54" +version = "0.0.55" source = { editable = "packages/narada-pyodide" } dependencies = [ { name = "narada-core" },