Implement pyfetch retry

packages/narada-pyodide/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "narada-pyodide"
-version = "0.0.54"
+version = "0.0.55"
 description = "Pyodide-compatible Python client SDK for Narada"
 license = "Apache-2.0"
 readme = "README.md"

packages/narada-pyodide/src/narada/retry.py

@@ -0,0 +1,103 @@
+import asyncio
+import time
+from http import HTTPStatus
+from typing import Any
+
+from pyodide.http import pyfetch
+
+__all__ = ["pyfetch_with_retries"]
+
+_PYFETCH_RETRY_ATTEMPTS = 3
+_PYFETCH_INITIAL_BACKOFF_SECONDS = 0.5
+_PYFETCH_BACKOFF_MULTIPLIER = 2.0
+_PYFETCH_RETRYABLE_STATUSES = frozenset(
+    status.value
+    for status in (
+        HTTPStatus.REQUEST_TIMEOUT,
+        HTTPStatus.TOO_MANY_REQUESTS,
+        HTTPStatus.INTERNAL_SERVER_ERROR,
+        HTTPStatus.BAD_GATEWAY,
+        HTTPStatus.SERVICE_UNAVAILABLE,
+        HTTPStatus.GATEWAY_TIMEOUT,
+    )
+)
+
+
+def _abort_signal_aborted(signal: Any) -> bool:
+    if signal is None:
+        return False
+
+    try:
+        return bool(getattr(signal, "aborted", False))
+    except Exception:
+        return False
+
+
+async def _sleep_before_pyfetch_retry(
+    backoff_seconds: float, retry_deadline: float | None
+) -> bool:
+    if retry_deadline is None:
+        await asyncio.sleep(backoff_seconds)
+        return True
+
+    remaining_seconds = retry_deadline - time.monotonic()
+    if remaining_seconds <= backoff_seconds:
+        return False
+
+    await asyncio.sleep(backoff_seconds)
+    return time.monotonic() < retry_deadline
+
+
+async def pyfetch_with_retries(
+    url: str,
+    *,
+    max_attempts: int = _PYFETCH_RETRY_ATTEMPTS,
+    initial_backoff_seconds: float = _PYFETCH_INITIAL_BACKOFF_SECONDS,
+    backoff_multiplier: float = _PYFETCH_BACKOFF_MULTIPLIER,
+    retry_statuses: frozenset[int] | None = _PYFETCH_RETRYABLE_STATUSES,
+    retry_deadline: float | None = None,
+    **kwargs: Any,
+) -> Any:
+    """Retries transient pyfetch exceptions with exponential backoff.
+
+    HTTP responses are returned as-is once attempts are exhausted so each caller
+    can preserve its existing status-specific handling.
+    """
+    if max_attempts < 1:
+        raise ValueError("max_attempts must be at least 1")
+
+    backoff_seconds = initial_backoff_seconds
+    signal = kwargs.get("signal")
+    for attempt in range(max_attempts):
+        if retry_deadline is not None and time.monotonic() >= retry_deadline:
+            raise asyncio.TimeoutError
+
+        try:
+            response = await pyfetch(url, **kwargs)
+        except Exception:
+            if (
+                attempt == max_attempts - 1
+                or _abort_signal_aborted(signal)
+                or (retry_deadline is not None and time.monotonic() >= retry_deadline)
+            ):
+                raise
+
+            if not await _sleep_before_pyfetch_retry(backoff_seconds, retry_deadline):
+                raise
+            backoff_seconds *= backoff_multiplier
+            continue
+
+        if (
+            retry_statuses
+            and response.status in retry_statuses
+            and attempt < max_attempts - 1
+            and not _abort_signal_aborted(signal)
+            and (retry_deadline is None or time.monotonic() < retry_deadline)
+        ):
+            if await _sleep_before_pyfetch_retry(backoff_seconds, retry_deadline):
+                backoff_seconds *= backoff_multiplier
+                continue
+
+        return response
+
+    raise AssertionError("unreachable")

packages/narada-pyodide/src/narada/window.py

@@ -81,6 +81,7 @@
 from pyodide.http import pyfetch
 
 from . import _trace
+from .retry import pyfetch_with_retries
 
 # Magic variable injected by the JavaScript harness that stores the IDs of the current runnables
 # in the stack on the frontend.
@@ -450,10 +451,11 @@ async def dispatch_request(
                     create_once_callable(abort_controller.abort),
                     (deadline - now) * 1000,
                 )
-                fetch_response = await pyfetch(
+                fetch_response = await pyfetch_with_retries(
                     f"{self._base_url}/remote-dispatch/responses/{request_id}",
                     headers=headers,
                     signal=signal,
+                    retry_deadline=deadline,
                 )
 
                 if not fetch_response.ok:
@@ -1182,7 +1184,7 @@ async def _fetch_presigned_download_url(
     session_id: str,
     key: str,
 ) -> str:
-    fetch_response = await pyfetch(
+    fetch_response = await pyfetch_with_retries(
         _build_cloud_browser_url(
             base_url,
             "/cloud-browser/replay/download-url",
@@ -1205,7 +1207,7 @@ async def _get_cloud_browser_downloads(
     auth_headers: dict[str, str],
     session_id: str,
 ) -> list[SessionDownloadItem]:
-    fetch_response = await pyfetch(
+    fetch_response = await pyfetch_with_retries(
         _build_cloud_browser_url(
             base_url,
             "/cloud-browser/replay/downloads",

packages/narada-pyodide/tests/test_cloud_browser.py

@@ -277,6 +277,63 @@ async def test_cloud_browser_window_dispatch_request_omits_parent_run_ids(
     assert "parentRunIds" not in payload
 
 
+@pytest.mark.asyncio
+async def test_cloud_browser_window_dispatch_request_retries_poll_fetch_failures(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    pyfetch = AsyncMock(
+        side_effect=[
+            _FakeResponse(json_data={"requestId": "req-123"}),
+            RuntimeError("temporary fetch failure"),
+            _FakeResponse(ok=False, status=502, text_data="bad gateway"),
+            _FakeResponse(json_data={"status": "success", "response": None}),
+        ]
+    )
+    _, _, window_module = _import_pyodide_narada(monkeypatch, pyfetch=pyfetch)
+    sleep_delays: list[float] = []
+
+    async def fake_sleep(delay: float) -> None:
+        sleep_delays.append(delay)
+
+    retry_module = sys.modules["narada.retry"]
+    monkeypatch.setattr(retry_module.asyncio, "sleep", fake_sleep)
+
+    window = window_module.CloudBrowserWindow(
+        browser_window_id="browser-window-123",
+        session_id="session-123",
+        api_key="test-api-key",
+    )
+    response = await window.dispatch_request(prompt="hello from cloud browser")
+
+    assert response["status"] == "success"
+    assert pyfetch.await_count == 4
+    assert sleep_delays == [0.5, 1.0]
+
+
+@pytest.mark.asyncio
+async def test_pyfetch_with_retries_does_not_start_retry_at_deadline(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    pyfetch = AsyncMock(
+        return_value=_FakeResponse(ok=False, status=502, text_data="bad gateway")
+    )
+    _import_pyodide_narada(monkeypatch, pyfetch=pyfetch)
+    retry_module = sys.modules["narada.retry"]
+    sleep = AsyncMock()
+
+    monkeypatch.setattr(retry_module.asyncio, "sleep", sleep)
+    monkeypatch.setattr(retry_module.time, "monotonic", lambda: 10.0)
+
+    response = await retry_module.pyfetch_with_retries(
+        "https://example.test/retry",
+        retry_deadline=10.5,
+    )
+
+    assert response.status == 502
+    assert pyfetch.await_count == 1
+    sleep.assert_not_awaited()
+
+
 @pytest.mark.asyncio
 async def test_dispatch_request_emits_string_trace_agent_type_for_sdk_enum(
     monkeypatch: pytest.MonkeyPatch,