diff --git a/packages/js-sdk/src/api/http2.ts b/packages/js-sdk/src/api/http2.ts index 49bf1f6beb..1a4ec5a9a7 100644 --- a/packages/js-sdk/src/api/http2.ts +++ b/packages/js-sdk/src/api/http2.ts @@ -1,5 +1,5 @@ import { runtime } from '../utils' -import { parseInflightLimitEnv, parsePositiveIntEnv } from './metadata' +import { parseInflightLimitEnv, parseIntEnv, parsePositiveIntEnv } from './metadata' import { limitConcurrency } from './inflight' import { loadUndici, @@ -9,6 +9,8 @@ import { } from '../undici' const DEFAULT_API_CONNECTION_LIMIT = 100 +const DEFAULT_REQUEST_RETRIES = 3 +const RETRYABLE_STATUS_CODES = new Set([502, 503, 504]) // 1000 = ~10 streams per connection (with the 100-conn default). // Override via env if your workload needs different. const DEFAULT_API_INFLIGHT_LIMIT = 1000 @@ -64,21 +66,21 @@ async function buildApiFetcher(options: { const inflightLimit = options.inflightLimit ?? getApiInflightLimit() if (!undici) { - return limitConcurrency(fetch, inflightLimit) + return limitConcurrency(withRetry(fetch), inflightLimit) } const { Agent, ProxyAgent, fetch: undiciFetch } = undici const connections = options.connectionLimit ?? getApiConnectionLimit() const dispatcher = options.proxy ? new ProxyAgent({ - uri: options.proxy, - allowH2: true, - connections, - }) + uri: options.proxy, + allowH2: true, + connections, + }) : new Agent({ - allowH2: true, - connections, - }) + allowH2: true, + connections, + }) const fetchWithDispatcher = undiciFetch as unknown as ( input: RequestInfo | URL, init?: UndiciRequestInit @@ -93,7 +95,56 @@ async function buildApiFetcher(options: { }) }) as typeof fetch - return limitConcurrency(wrapped, inflightLimit) + return limitConcurrency(withRetry(wrapped), inflightLimit) +} + +function getRequestRetries(): number { + return parseIntEnv('E2B_REQUEST_RETRIES', DEFAULT_REQUEST_RETRIES) +} + +export function withRetry(baseFetch: typeof fetch): typeof fetch { + const maxRetries = getRequestRetries() + if (maxRetries <= 0) return baseFetch + + return (async (input: RequestInfo | URL, init?: RequestInit) => { + let lastError: unknown + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + const response = await baseFetch(input, init) + if (RETRYABLE_STATUS_CODES.has(response.status) && attempt < maxRetries) { + // Consume the body before retrying + await response.text().catch(() => { }) + const delay = Math.min(2 ** attempt * 1000, 8000) + const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url + console.warn( + `[e2b] Retrying ${init?.method ?? 'GET'} ${url} (attempt ${attempt + 1}/${maxRetries}, backoff ${delay}ms): server returned ${response.status}` + ) + await sleep(delay) + continue + } + return response + } catch (error: unknown) { + lastError = error + // Don't retry abort/timeout errors + if (error instanceof DOMException && error.name === 'AbortError') throw error + if (attempt < maxRetries) { + const delay = Math.min(2 ** attempt * 1000, 8000) + const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url + console.warn( + `[e2b] Retrying ${init?.method ?? 'GET'} ${url} (attempt ${attempt + 1}/${maxRetries}, backoff ${delay}ms): ${error}` + ) + await sleep(delay) + continue + } + throw error + } + } + throw lastError + }) as typeof fetch +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) } export function getApiConnectionLimit(): number { diff --git a/packages/js-sdk/tests/api/retry.test.ts b/packages/js-sdk/tests/api/retry.test.ts new file mode 100644 index 0000000000..c91af2aaed --- /dev/null +++ b/packages/js-sdk/tests/api/retry.test.ts @@ -0,0 +1,167 @@ +import { afterEach, describe, expect, test, vi } from 'vitest' + +afterEach(() => { + vi.restoreAllMocks() + vi.resetModules() + delete process.env.E2B_REQUEST_RETRIES +}) + +describe('withRetry', () => { + test('returns response on success without retrying', async () => { + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi.fn(() => Promise.resolve(new Response('ok', { status: 200 }))) + const retryFetch = withRetry(baseFetch as typeof fetch) + + const response = await retryFetch('https://api.e2b.dev/test') + + expect(response.status).toBe(200) + expect(baseFetch).toHaveBeenCalledOnce() + }) + + test('retries on 502 and succeeds', async () => { + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi + .fn() + .mockResolvedValueOnce(new Response('bad gateway', { status: 502 })) + .mockResolvedValueOnce(new Response('ok', { status: 200 })) + const retryFetch = withRetry(baseFetch as typeof fetch) + + const response = await retryFetch('https://api.e2b.dev/test') + + expect(response.status).toBe(200) + expect(baseFetch).toHaveBeenCalledTimes(2) + }) + + test('retries on 503 and succeeds', async () => { + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi + .fn() + .mockResolvedValueOnce(new Response('unavailable', { status: 503 })) + .mockResolvedValueOnce(new Response('ok', { status: 200 })) + const retryFetch = withRetry(baseFetch as typeof fetch) + + const response = await retryFetch('https://api.e2b.dev/test') + + expect(response.status).toBe(200) + expect(baseFetch).toHaveBeenCalledTimes(2) + }) + + test('retries on 504 and succeeds', async () => { + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi + .fn() + .mockResolvedValueOnce(new Response('timeout', { status: 504 })) + .mockResolvedValueOnce(new Response('ok', { status: 200 })) + const retryFetch = withRetry(baseFetch as typeof fetch) + + const response = await retryFetch('https://api.e2b.dev/test') + + expect(response.status).toBe(200) + expect(baseFetch).toHaveBeenCalledTimes(2) + }) + + test('does not retry on 400', async () => { + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi.fn(() => + Promise.resolve(new Response('bad request', { status: 400 })) + ) + const retryFetch = withRetry(baseFetch as typeof fetch) + + const response = await retryFetch('https://api.e2b.dev/test') + + expect(response.status).toBe(400) + expect(baseFetch).toHaveBeenCalledOnce() + }) + + test('does not retry on 401', async () => { + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi.fn(() => + Promise.resolve(new Response('unauthorized', { status: 401 })) + ) + const retryFetch = withRetry(baseFetch as typeof fetch) + + const response = await retryFetch('https://api.e2b.dev/test') + + expect(response.status).toBe(401) + expect(baseFetch).toHaveBeenCalledOnce() + }) + + test('does not retry on 429', async () => { + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi.fn(() => + Promise.resolve(new Response('rate limited', { status: 429 })) + ) + const retryFetch = withRetry(baseFetch as typeof fetch) + + const response = await retryFetch('https://api.e2b.dev/test') + + expect(response.status).toBe(429) + expect(baseFetch).toHaveBeenCalledOnce() + }) + + test('retries on network error and succeeds', async () => { + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi + .fn() + .mockRejectedValueOnce(new TypeError('fetch failed')) + .mockResolvedValueOnce(new Response('ok', { status: 200 })) + const retryFetch = withRetry(baseFetch as typeof fetch) + + const response = await retryFetch('https://api.e2b.dev/test') + + expect(response.status).toBe(200) + expect(baseFetch).toHaveBeenCalledTimes(2) + }) + + test('does not retry on AbortError', async () => { + const { withRetry } = await import('../../src/api/http2') + const abortError = new DOMException('The operation was aborted', 'AbortError') + const baseFetch = vi.fn().mockRejectedValue(abortError) + const retryFetch = withRetry(baseFetch as typeof fetch) + + await expect(retryFetch('https://api.e2b.dev/test')).rejects.toThrow( + 'The operation was aborted' + ) + expect(baseFetch).toHaveBeenCalledOnce() + }) + + test('exhausts retries on persistent network error', async () => { + process.env.E2B_REQUEST_RETRIES = '2' + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi.fn().mockRejectedValue(new TypeError('fetch failed')) + const retryFetch = withRetry(baseFetch as typeof fetch) + + await expect(retryFetch('https://api.e2b.dev/test')).rejects.toThrow('fetch failed') + // 1 initial + 2 retries = 3 total + expect(baseFetch).toHaveBeenCalledTimes(3) + }) + + test('returns last retryable response when retries exhausted', async () => { + process.env.E2B_REQUEST_RETRIES = '2' + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi.fn(() => + Promise.resolve(new Response('bad gateway', { status: 502 })) + ) + const retryFetch = withRetry(baseFetch as typeof fetch) + + const response = await retryFetch('https://api.e2b.dev/test') + + expect(response.status).toBe(502) + // 1 initial + 2 retries = 3 total + expect(baseFetch).toHaveBeenCalledTimes(3) + }) + + test('disables retry when E2B_REQUEST_RETRIES=0', async () => { + process.env.E2B_REQUEST_RETRIES = '0' + const { withRetry } = await import('../../src/api/http2') + const baseFetch = vi.fn(() => + Promise.resolve(new Response('bad gateway', { status: 502 })) + ) + const retryFetch = withRetry(baseFetch as typeof fetch) + + const response = await retryFetch('https://api.e2b.dev/test') + + expect(response.status).toBe(502) + expect(baseFetch).toHaveBeenCalledOnce() + }) +}) diff --git a/packages/python-sdk/e2b/api/__init__.py b/packages/python-sdk/e2b/api/__init__.py index 2a1e42d209..44cf039b5c 100644 --- a/packages/python-sdk/e2b/api/__init__.py +++ b/packages/python-sdk/e2b/api/__init__.py @@ -70,6 +70,15 @@ async def on_response(response: Response) -> None: connection_retries = int(os.getenv("E2B_CONNECTION_RETRIES", "3")) +# Number of times to retry a request when a transient error occurs +# (5xx responses, network/protocol errors like h2 ConnectionTerminated, etc.) +request_retries = int(os.getenv("E2B_REQUEST_RETRIES", "3")) + +# Status codes that are safe to retry (server-side transient errors) +RETRYABLE_STATUS_CODES = frozenset({502, 503, 504}) + +_retry_logger = logging.getLogger("e2b.api.retry") + @dataclass class SandboxCreateResponse: diff --git a/packages/python-sdk/e2b/api/client_async/__init__.py b/packages/python-sdk/e2b/api/client_async/__init__.py index dd001a3a41..e0b8be2481 100644 --- a/packages/python-sdk/e2b/api/client_async/__init__.py +++ b/packages/python-sdk/e2b/api/client_async/__init__.py @@ -6,7 +6,7 @@ from httpx._types import ProxyTypes -from e2b.api import AsyncApiClient, connection_retries, limits +from e2b.api import AsyncApiClient, connection_retries, limits, request_retries, RETRYABLE_STATUS_CODES, _retry_logger from e2b.connection_config import ConnectionConfig TransportKey = Tuple[bool, Optional[ProxyTypes]] @@ -33,6 +33,39 @@ class AsyncTransportWithLogger(httpx.AsyncHTTPTransport): def pool(self): return self._pool + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + last_exc: Optional[Exception] = None + for attempt in range(1 + request_retries): + try: + response = await super().handle_async_request(request) + if response.status_code in RETRYABLE_STATUS_CODES and attempt < request_retries: + await response.aread() + await response.aclose() + delay = min(2 ** attempt, 8) + _retry_logger.warning( + "Retrying %s %s (attempt %d/%d, backoff %ds): server returned %d", + request.method, request.url, attempt + 1, request_retries, delay, + response.status_code, + ) + await asyncio.sleep(delay) + continue + return response + except httpx.TimeoutException: + raise + except Exception as exc: + last_exc = exc + if attempt < request_retries: + delay = min(2 ** attempt, 8) + _retry_logger.warning( + "Retrying %s %s (attempt %d/%d, backoff %ds): %s", + request.method, request.url, attempt + 1, request_retries, delay, + exc, + ) + await asyncio.sleep(delay) + continue + raise + raise last_exc # type: ignore[misc] + def _get_cached_transport(cls, config: ConnectionConfig, http2: bool): loop = asyncio.get_running_loop() diff --git a/packages/python-sdk/e2b/api/client_sync/__init__.py b/packages/python-sdk/e2b/api/client_sync/__init__.py index 41fb4f023d..3ab2241a08 100644 --- a/packages/python-sdk/e2b/api/client_sync/__init__.py +++ b/packages/python-sdk/e2b/api/client_sync/__init__.py @@ -1,11 +1,12 @@ from typing import Dict, Optional, Tuple +import time import httpx import threading from httpx._types import ProxyTypes -from e2b.api import ApiClient, connection_retries, limits +from e2b.api import ApiClient, connection_retries, limits, request_retries, RETRYABLE_STATUS_CODES, _retry_logger from e2b.connection_config import ConnectionConfig TransportKey = Tuple[bool, Optional[ProxyTypes]] @@ -26,6 +27,40 @@ class TransportWithLogger(httpx.HTTPTransport): def pool(self): return self._pool + def handle_request(self, request: httpx.Request) -> httpx.Response: + last_exc: Optional[Exception] = None + for attempt in range(1 + request_retries): + try: + response = super().handle_request(request) + if response.status_code in RETRYABLE_STATUS_CODES and attempt < request_retries: + # Read and close the response before retrying + response.read() + response.close() + delay = min(2 ** attempt, 8) + _retry_logger.warning( + "Retrying %s %s (attempt %d/%d, backoff %ds): server returned %d", + request.method, request.url, attempt + 1, request_retries, delay, + response.status_code, + ) + time.sleep(delay) + continue + return response + except httpx.TimeoutException: + raise + except Exception as exc: + last_exc = exc + if attempt < request_retries: + delay = min(2 ** attempt, 8) + _retry_logger.warning( + "Retrying %s %s (attempt %d/%d, backoff %ds): %s", + request.method, request.url, attempt + 1, request_retries, delay, + exc, + ) + time.sleep(delay) + continue + raise + raise last_exc # type: ignore[misc] + def get_transport(config: ConnectionConfig, http2: bool = True) -> TransportWithLogger: instances: Dict[TransportKey, TransportWithLogger] = getattr( diff --git a/packages/python-sdk/tests/test_transport_retry.py b/packages/python-sdk/tests/test_transport_retry.py new file mode 100644 index 0000000000..d2e755ed40 --- /dev/null +++ b/packages/python-sdk/tests/test_transport_retry.py @@ -0,0 +1,375 @@ +"""Tests for HTTP transport-level retry logic. + +Verifies that TransportWithLogger and AsyncTransportWithLogger correctly +retry on transient errors (5xx status codes, network/protocol errors) +and do NOT retry on timeouts or non-retryable responses. +""" + +import asyncio +from unittest.mock import MagicMock, AsyncMock, patch + +import httpx +import pytest + +from e2b.api import RETRYABLE_STATUS_CODES +from e2b.api.client_sync import TransportWithLogger +from e2b.api.client_async import AsyncTransportWithLogger + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_request() -> httpx.Request: + return httpx.Request("GET", "https://api.e2b.dev/test") + + +def _make_response(status_code: int = 200) -> httpx.Response: + return httpx.Response(status_code, request=_make_request()) + + +# --------------------------------------------------------------------------- +# Sync Transport Tests +# --------------------------------------------------------------------------- + + +class TestSyncTransportRetry: + """Tests for TransportWithLogger.handle_request retry logic.""" + + @patch("e2b.api.client_sync.time.sleep") + def test_successful_request_no_retry(self, mock_sleep): + transport = TransportWithLogger() + response = _make_response(200) + + with patch.object( + httpx.HTTPTransport, "handle_request", return_value=response + ) as mock_handle: + result = transport.handle_request(_make_request()) + + assert result.status_code == 200 + mock_handle.assert_called_once() + mock_sleep.assert_not_called() + + @patch("e2b.api.client_sync.time.sleep") + def test_retries_on_502(self, mock_sleep): + transport = TransportWithLogger() + bad_response = _make_response(502) + good_response = _make_response(200) + + with patch.object( + httpx.HTTPTransport, + "handle_request", + side_effect=[bad_response, good_response], + ) as mock_handle: + result = transport.handle_request(_make_request()) + + assert result.status_code == 200 + assert mock_handle.call_count == 2 + mock_sleep.assert_called_once() + + @patch("e2b.api.client_sync.time.sleep") + def test_retries_on_503(self, mock_sleep): + transport = TransportWithLogger() + bad_response = _make_response(503) + good_response = _make_response(200) + + with patch.object( + httpx.HTTPTransport, + "handle_request", + side_effect=[bad_response, good_response], + ) as mock_handle: + result = transport.handle_request(_make_request()) + + assert result.status_code == 200 + assert mock_handle.call_count == 2 + + @patch("e2b.api.client_sync.time.sleep") + def test_retries_on_504(self, mock_sleep): + transport = TransportWithLogger() + bad_response = _make_response(504) + good_response = _make_response(200) + + with patch.object( + httpx.HTTPTransport, + "handle_request", + side_effect=[bad_response, good_response], + ) as mock_handle: + result = transport.handle_request(_make_request()) + + assert result.status_code == 200 + assert mock_handle.call_count == 2 + + @patch("e2b.api.client_sync.time.sleep") + def test_does_not_retry_on_400(self, mock_sleep): + transport = TransportWithLogger() + response = _make_response(400) + + with patch.object( + httpx.HTTPTransport, "handle_request", return_value=response + ) as mock_handle: + result = transport.handle_request(_make_request()) + + assert result.status_code == 400 + mock_handle.assert_called_once() + mock_sleep.assert_not_called() + + @patch("e2b.api.client_sync.time.sleep") + def test_does_not_retry_on_401(self, mock_sleep): + transport = TransportWithLogger() + response = _make_response(401) + + with patch.object( + httpx.HTTPTransport, "handle_request", return_value=response + ) as mock_handle: + result = transport.handle_request(_make_request()) + + assert result.status_code == 401 + mock_handle.assert_called_once() + + @patch("e2b.api.client_sync.time.sleep") + def test_does_not_retry_on_429(self, mock_sleep): + transport = TransportWithLogger() + response = _make_response(429) + + with patch.object( + httpx.HTTPTransport, "handle_request", return_value=response + ) as mock_handle: + result = transport.handle_request(_make_request()) + + assert result.status_code == 429 + mock_handle.assert_called_once() + + @patch("e2b.api.client_sync.time.sleep") + def test_retries_on_network_error(self, mock_sleep): + transport = TransportWithLogger() + good_response = _make_response(200) + + with patch.object( + httpx.HTTPTransport, + "handle_request", + side_effect=[ConnectionError("connection reset"), good_response], + ) as mock_handle: + result = transport.handle_request(_make_request()) + + assert result.status_code == 200 + assert mock_handle.call_count == 2 + mock_sleep.assert_called_once() + + @patch("e2b.api.client_sync.time.sleep") + def test_does_not_retry_on_timeout(self, mock_sleep): + transport = TransportWithLogger() + + with patch.object( + httpx.HTTPTransport, + "handle_request", + side_effect=httpx.ReadTimeout("read timed out"), + ): + with pytest.raises(httpx.ReadTimeout): + transport.handle_request(_make_request()) + + mock_sleep.assert_not_called() + + @patch("e2b.api.client_sync.request_retries", 3) + @patch("e2b.api.client_sync.time.sleep") + def test_exhausts_retries_on_persistent_error(self, mock_sleep): + transport = TransportWithLogger() + + with patch.object( + httpx.HTTPTransport, + "handle_request", + side_effect=ConnectionError("connection reset"), + ) as mock_handle: + with pytest.raises(ConnectionError, match="connection reset"): + transport.handle_request(_make_request()) + + # 1 initial + 3 retries = 4 total attempts + assert mock_handle.call_count == 4 + assert mock_sleep.call_count == 3 + + @patch("e2b.api.client_sync.request_retries", 3) + @patch("e2b.api.client_sync.time.sleep") + def test_returns_last_retryable_response_when_exhausted(self, mock_sleep): + transport = TransportWithLogger() + bad_response = _make_response(503) + + with patch.object( + httpx.HTTPTransport, + "handle_request", + return_value=bad_response, + ) as mock_handle: + result = transport.handle_request(_make_request()) + + # After exhausting retries, returns the last 503 response + assert result.status_code == 503 + assert mock_handle.call_count == 4 + + @patch("e2b.api.client_sync.request_retries", 3) + @patch("e2b.api.client_sync.time.sleep") + def test_exponential_backoff_timing(self, mock_sleep): + transport = TransportWithLogger() + + with patch.object( + httpx.HTTPTransport, + "handle_request", + side_effect=ConnectionError("fail"), + ): + with pytest.raises(ConnectionError): + transport.handle_request(_make_request()) + + # Backoff: min(2^0, 8)=1, min(2^1, 8)=2, min(2^2, 8)=4 + assert mock_sleep.call_args_list[0][0][0] == 1 + assert mock_sleep.call_args_list[1][0][0] == 2 + assert mock_sleep.call_args_list[2][0][0] == 4 + + @patch("e2b.api.client_sync.request_retries", 0) + @patch("e2b.api.client_sync.time.sleep") + def test_no_retry_when_retries_disabled(self, mock_sleep): + transport = TransportWithLogger() + + with patch.object( + httpx.HTTPTransport, + "handle_request", + side_effect=ConnectionError("fail"), + ) as mock_handle: + with pytest.raises(ConnectionError): + transport.handle_request(_make_request()) + + mock_handle.assert_called_once() + mock_sleep.assert_not_called() + + +# --------------------------------------------------------------------------- +# Async Transport Tests +# --------------------------------------------------------------------------- + + +class TestAsyncTransportRetry: + """Tests for AsyncTransportWithLogger.handle_async_request retry logic.""" + + @patch("e2b.api.client_async.asyncio.sleep", new_callable=AsyncMock) + async def test_successful_request_no_retry(self, mock_sleep): + transport = AsyncTransportWithLogger() + response = _make_response(200) + + with patch.object( + httpx.AsyncHTTPTransport, + "handle_async_request", + return_value=response, + ) as mock_handle: + result = await transport.handle_async_request(_make_request()) + + assert result.status_code == 200 + mock_handle.assert_called_once() + mock_sleep.assert_not_called() + + @patch("e2b.api.client_async.asyncio.sleep", new_callable=AsyncMock) + async def test_retries_on_502(self, mock_sleep): + transport = AsyncTransportWithLogger() + bad_response = _make_response(502) + good_response = _make_response(200) + + with patch.object( + httpx.AsyncHTTPTransport, + "handle_async_request", + side_effect=[bad_response, good_response], + ) as mock_handle: + result = await transport.handle_async_request(_make_request()) + + assert result.status_code == 200 + assert mock_handle.call_count == 2 + mock_sleep.assert_called_once() + + @patch("e2b.api.client_async.asyncio.sleep", new_callable=AsyncMock) + async def test_retries_on_network_error(self, mock_sleep): + transport = AsyncTransportWithLogger() + good_response = _make_response(200) + + with patch.object( + httpx.AsyncHTTPTransport, + "handle_async_request", + side_effect=[ConnectionError("h2 connection terminated"), good_response], + ) as mock_handle: + result = await transport.handle_async_request(_make_request()) + + assert result.status_code == 200 + assert mock_handle.call_count == 2 + + @patch("e2b.api.client_async.asyncio.sleep", new_callable=AsyncMock) + async def test_does_not_retry_on_timeout(self, mock_sleep): + transport = AsyncTransportWithLogger() + + with patch.object( + httpx.AsyncHTTPTransport, + "handle_async_request", + side_effect=httpx.ReadTimeout("read timed out"), + ): + with pytest.raises(httpx.ReadTimeout): + await transport.handle_async_request(_make_request()) + + mock_sleep.assert_not_called() + + @patch("e2b.api.client_async.request_retries", 3) + @patch("e2b.api.client_async.asyncio.sleep", new_callable=AsyncMock) + async def test_exhausts_retries_on_persistent_error(self, mock_sleep): + transport = AsyncTransportWithLogger() + + with patch.object( + httpx.AsyncHTTPTransport, + "handle_async_request", + side_effect=ConnectionError("connection terminated"), + ) as mock_handle: + with pytest.raises(ConnectionError, match="connection terminated"): + await transport.handle_async_request(_make_request()) + + assert mock_handle.call_count == 4 + assert mock_sleep.call_count == 3 + + @patch("e2b.api.client_async.request_retries", 3) + @patch("e2b.api.client_async.asyncio.sleep", new_callable=AsyncMock) + async def test_returns_last_retryable_response_when_exhausted(self, mock_sleep): + transport = AsyncTransportWithLogger() + bad_response = _make_response(503) + + with patch.object( + httpx.AsyncHTTPTransport, + "handle_async_request", + return_value=bad_response, + ) as mock_handle: + result = await transport.handle_async_request(_make_request()) + + assert result.status_code == 503 + assert mock_handle.call_count == 4 + + @patch("e2b.api.client_async.request_retries", 0) + @patch("e2b.api.client_async.asyncio.sleep", new_callable=AsyncMock) + async def test_no_retry_when_retries_disabled(self, mock_sleep): + transport = AsyncTransportWithLogger() + + with patch.object( + httpx.AsyncHTTPTransport, + "handle_async_request", + side_effect=ConnectionError("fail"), + ) as mock_handle: + with pytest.raises(ConnectionError): + await transport.handle_async_request(_make_request()) + + mock_handle.assert_called_once() + mock_sleep.assert_not_called() + + @patch("e2b.api.client_async.request_retries", 3) + @patch("e2b.api.client_async.asyncio.sleep", new_callable=AsyncMock) + async def test_exponential_backoff_timing(self, mock_sleep): + transport = AsyncTransportWithLogger() + + with patch.object( + httpx.AsyncHTTPTransport, + "handle_async_request", + side_effect=ConnectionError("fail"), + ): + with pytest.raises(ConnectionError): + await transport.handle_async_request(_make_request()) + + assert mock_sleep.call_args_list[0][0][0] == 1 + assert mock_sleep.call_args_list[1][0][0] == 2 + assert mock_sleep.call_args_list[2][0][0] == 4