From 27be14267dca763d1ed491d3ab61eeb979f4b495 Mon Sep 17 00:00:00 2001 From: Zizheng Tai Date: Wed, 13 May 2026 20:17:09 -0700 Subject: [PATCH 1/2] Implement pyfetch retry --- packages/narada-pyodide/pyproject.toml | 2 +- packages/narada-pyodide/src/narada/retry.py | 100 ++++++++++++++++++ packages/narada-pyodide/src/narada/window.py | 16 +-- .../tests/test_cloud_browser.py | 33 ++++++ uv.lock | 2 +- 5 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 packages/narada-pyodide/src/narada/retry.py diff --git a/packages/narada-pyodide/pyproject.toml b/packages/narada-pyodide/pyproject.toml index 62f8333..10bfc3b 100644 --- a/packages/narada-pyodide/pyproject.toml +++ b/packages/narada-pyodide/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "narada-pyodide" -version = "0.0.54" +version = "0.0.55" description = "Pyodide-compatible Python client SDK for Narada" license = "Apache-2.0" readme = "README.md" diff --git a/packages/narada-pyodide/src/narada/retry.py b/packages/narada-pyodide/src/narada/retry.py new file mode 100644 index 0000000..22389a5 --- /dev/null +++ b/packages/narada-pyodide/src/narada/retry.py @@ -0,0 +1,100 @@ +import asyncio +import time +from http import HTTPStatus +from typing import Any + +from pyodide.http import pyfetch + +__all__ = ["pyfetch_with_retries"] + +_PYFETCH_RETRY_ATTEMPTS = 3 +_PYFETCH_INITIAL_BACKOFF_SECONDS = 0.5 +_PYFETCH_BACKOFF_MULTIPLIER = 2.0 +_PYFETCH_RETRYABLE_STATUSES = frozenset( + status.value + for status in ( + HTTPStatus.REQUEST_TIMEOUT, + HTTPStatus.TOO_MANY_REQUESTS, + HTTPStatus.INTERNAL_SERVER_ERROR, + HTTPStatus.BAD_GATEWAY, + HTTPStatus.SERVICE_UNAVAILABLE, + HTTPStatus.GATEWAY_TIMEOUT, + ) +) + + +def _abort_signal_aborted(signal: Any) -> bool: + if signal is None: + return False + + try: + return bool(getattr(signal, "aborted", False)) + except Exception: + return False + + +async def _sleep_before_pyfetch_retry( + backoff_seconds: float, retry_deadline: float | None +) -> bool: + if retry_deadline is None: + await asyncio.sleep(backoff_seconds) + return True + + remaining_seconds = retry_deadline - time.monotonic() + if remaining_seconds <= 0: + return False + + await asyncio.sleep(min(backoff_seconds, remaining_seconds)) + return True + + +async def pyfetch_with_retries( + url: str, + *, + max_attempts: int = _PYFETCH_RETRY_ATTEMPTS, + initial_backoff_seconds: float = _PYFETCH_INITIAL_BACKOFF_SECONDS, + backoff_multiplier: float = _PYFETCH_BACKOFF_MULTIPLIER, + retry_statuses: frozenset[int] | None = _PYFETCH_RETRYABLE_STATUSES, + retry_deadline: float | None = None, + **kwargs: Any, +) -> Any: + """Retries transient pyfetch exceptions with exponential backoff. + + HTTP responses are returned as-is once attempts are exhausted so each caller + can preserve its existing status-specific handling. + """ + if max_attempts < 1: + raise ValueError("max_attempts must be at least 1") + + backoff_seconds = initial_backoff_seconds + signal = kwargs.get("signal") + for attempt in range(max_attempts): + try: + response = await pyfetch(url, **kwargs) + except Exception: + if ( + attempt == max_attempts - 1 + or _abort_signal_aborted(signal) + or (retry_deadline is not None and time.monotonic() >= retry_deadline) + ): + raise + + if not await _sleep_before_pyfetch_retry(backoff_seconds, retry_deadline): + raise + backoff_seconds *= backoff_multiplier + continue + + if ( + retry_statuses + and response.status in retry_statuses + and attempt < max_attempts - 1 + and not _abort_signal_aborted(signal) + and (retry_deadline is None or time.monotonic() < retry_deadline) + ): + if await _sleep_before_pyfetch_retry(backoff_seconds, retry_deadline): + backoff_seconds *= backoff_multiplier + continue + + return response + + raise AssertionError("unreachable") diff --git a/packages/narada-pyodide/src/narada/window.py b/packages/narada-pyodide/src/narada/window.py index 0bf4d3b..e05cef5 100644 --- a/packages/narada-pyodide/src/narada/window.py +++ b/packages/narada-pyodide/src/narada/window.py @@ -78,9 +78,9 @@ from narada_core.tracing.model import parse_action_trace from pydantic import BaseModel from pyodide.ffi import JsProxy, create_once_callable -from pyodide.http import pyfetch from . import _trace +from .retry import pyfetch_with_retries # Magic variable injected by the JavaScript harness that stores the IDs of the current runnables # in the stack on the frontend. @@ -427,12 +427,13 @@ async def dispatch_request( signal = controller.signal setTimeout(create_once_callable(controller.abort), timeout * 1000) - fetch_response = await pyfetch( + fetch_response = await pyfetch_with_retries( f"{self._base_url}/remote-dispatch", method="POST", headers=headers, body=json.dumps(body), signal=signal, + retry_deadline=deadline, ) if not fetch_response.ok: @@ -450,10 +451,11 @@ async def dispatch_request( create_once_callable(abort_controller.abort), (deadline - now) * 1000, ) - fetch_response = await pyfetch( + fetch_response = await pyfetch_with_retries( f"{self._base_url}/remote-dispatch/responses/{request_id}", headers=headers, signal=signal, + retry_deadline=deadline, ) if not fetch_response.ok: @@ -989,7 +991,7 @@ async def _run_extension_action( if timeout is not None: body["timeout"] = timeout - fetch_response = await pyfetch( + fetch_response = await pyfetch_with_retries( f"{self._base_url}/extension-actions", method="POST", headers=headers, @@ -1182,7 +1184,7 @@ async def _fetch_presigned_download_url( session_id: str, key: str, ) -> str: - fetch_response = await pyfetch( + fetch_response = await pyfetch_with_retries( _build_cloud_browser_url( base_url, "/cloud-browser/replay/download-url", @@ -1205,7 +1207,7 @@ async def _get_cloud_browser_downloads( auth_headers: dict[str, str], session_id: str, ) -> list[SessionDownloadItem]: - fetch_response = await pyfetch( + fetch_response = await pyfetch_with_retries( _build_cloud_browser_url( base_url, "/cloud-browser/replay/downloads", @@ -1253,7 +1255,7 @@ async def _stop_cloud_browser_session( timeout: int | None = None, ) -> None: try: - fetch_response = await pyfetch( + fetch_response = await pyfetch_with_retries( f"{base_url}/cloud-browser/stop-cloud-browser-session", method="POST", headers={**auth_headers, "Content-Type": "application/json"}, diff --git a/packages/narada-pyodide/tests/test_cloud_browser.py b/packages/narada-pyodide/tests/test_cloud_browser.py index eecedc8..ace0e2a 100644 --- a/packages/narada-pyodide/tests/test_cloud_browser.py +++ b/packages/narada-pyodide/tests/test_cloud_browser.py @@ -277,6 +277,39 @@ async def test_cloud_browser_window_dispatch_request_omits_parent_run_ids( assert "parentRunIds" not in payload +@pytest.mark.asyncio +async def test_cloud_browser_window_dispatch_request_retries_poll_fetch_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyfetch = AsyncMock( + side_effect=[ + _FakeResponse(json_data={"requestId": "req-123"}), + RuntimeError("temporary fetch failure"), + _FakeResponse(ok=False, status=502, text_data="bad gateway"), + _FakeResponse(json_data={"status": "success", "response": None}), + ] + ) + _, _, window_module = _import_pyodide_narada(monkeypatch, pyfetch=pyfetch) + sleep_delays: list[float] = [] + + async def fake_sleep(delay: float) -> None: + sleep_delays.append(delay) + + retry_module = sys.modules["narada.retry"] + monkeypatch.setattr(retry_module.asyncio, "sleep", fake_sleep) + + window = window_module.CloudBrowserWindow( + browser_window_id="browser-window-123", + session_id="session-123", + api_key="test-api-key", + ) + response = await window.dispatch_request(prompt="hello from cloud browser") + + assert response["status"] == "success" + assert pyfetch.await_count == 4 + assert sleep_delays == [0.5, 1.0] + + @pytest.mark.asyncio async def test_dispatch_request_emits_string_trace_agent_type_for_sdk_enum( monkeypatch: pytest.MonkeyPatch, diff --git a/uv.lock b/uv.lock index 2f2b9a2..4d9f5a7 100644 --- a/uv.lock +++ b/uv.lock @@ -356,7 +356,7 @@ requires-dist = [{ name = "pydantic", specifier = "==2.12.5" }] [[package]] name = "narada-pyodide" -version = "0.0.54" +version = "0.0.55" source = { editable = "packages/narada-pyodide" } dependencies = [ { name = "narada-core" }, From 317991e96f74337f1e11f0db39d5b242a3c998d9 Mon Sep 17 00:00:00 2001 From: Zizheng Tai Date: Wed, 13 May 2026 20:25:09 -0700 Subject: [PATCH 2/2] Fixes --- packages/narada-pyodide/src/narada/retry.py | 9 ++++--- packages/narada-pyodide/src/narada/window.py | 8 +++---- .../tests/test_cloud_browser.py | 24 +++++++++++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/packages/narada-pyodide/src/narada/retry.py b/packages/narada-pyodide/src/narada/retry.py index 22389a5..d7f6abd 100644 --- a/packages/narada-pyodide/src/narada/retry.py +++ b/packages/narada-pyodide/src/narada/retry.py @@ -41,11 +41,11 @@ async def _sleep_before_pyfetch_retry( return True remaining_seconds = retry_deadline - time.monotonic() - if remaining_seconds <= 0: + if remaining_seconds <= backoff_seconds: return False - await asyncio.sleep(min(backoff_seconds, remaining_seconds)) - return True + await asyncio.sleep(backoff_seconds) + return time.monotonic() < retry_deadline async def pyfetch_with_retries( @@ -69,6 +69,9 @@ async def pyfetch_with_retries( backoff_seconds = initial_backoff_seconds signal = kwargs.get("signal") for attempt in range(max_attempts): + if retry_deadline is not None and time.monotonic() >= retry_deadline: + raise asyncio.TimeoutError + try: response = await pyfetch(url, **kwargs) except Exception: diff --git a/packages/narada-pyodide/src/narada/window.py b/packages/narada-pyodide/src/narada/window.py index e05cef5..a9c1716 100644 --- a/packages/narada-pyodide/src/narada/window.py +++ b/packages/narada-pyodide/src/narada/window.py @@ -78,6 +78,7 @@ from narada_core.tracing.model import parse_action_trace from pydantic import BaseModel from pyodide.ffi import JsProxy, create_once_callable +from pyodide.http import pyfetch from . import _trace from .retry import pyfetch_with_retries @@ -427,13 +428,12 @@ async def dispatch_request( signal = controller.signal setTimeout(create_once_callable(controller.abort), timeout * 1000) - fetch_response = await pyfetch_with_retries( + fetch_response = await pyfetch( f"{self._base_url}/remote-dispatch", method="POST", headers=headers, body=json.dumps(body), signal=signal, - retry_deadline=deadline, ) if not fetch_response.ok: @@ -991,7 +991,7 @@ async def _run_extension_action( if timeout is not None: body["timeout"] = timeout - fetch_response = await pyfetch_with_retries( + fetch_response = await pyfetch( f"{self._base_url}/extension-actions", method="POST", headers=headers, @@ -1255,7 +1255,7 @@ async def _stop_cloud_browser_session( timeout: int | None = None, ) -> None: try: - fetch_response = await pyfetch_with_retries( + fetch_response = await pyfetch( f"{base_url}/cloud-browser/stop-cloud-browser-session", method="POST", headers={**auth_headers, "Content-Type": "application/json"}, diff --git a/packages/narada-pyodide/tests/test_cloud_browser.py b/packages/narada-pyodide/tests/test_cloud_browser.py index ace0e2a..3a058a1 100644 --- a/packages/narada-pyodide/tests/test_cloud_browser.py +++ b/packages/narada-pyodide/tests/test_cloud_browser.py @@ -310,6 +310,30 @@ async def fake_sleep(delay: float) -> None: assert sleep_delays == [0.5, 1.0] +@pytest.mark.asyncio +async def test_pyfetch_with_retries_does_not_start_retry_at_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyfetch = AsyncMock( + return_value=_FakeResponse(ok=False, status=502, text_data="bad gateway") + ) + _import_pyodide_narada(monkeypatch, pyfetch=pyfetch) + retry_module = sys.modules["narada.retry"] + sleep = AsyncMock() + + monkeypatch.setattr(retry_module.asyncio, "sleep", sleep) + monkeypatch.setattr(retry_module.time, "monotonic", lambda: 10.0) + + response = await retry_module.pyfetch_with_retries( + "https://example.test/retry", + retry_deadline=10.5, + ) + + assert response.status == 502 + assert pyfetch.await_count == 1 + sleep.assert_not_awaited() + + @pytest.mark.asyncio async def test_dispatch_request_emits_string_trace_agent_type_for_sdk_enum( monkeypatch: pytest.MonkeyPatch,