fix(llm): add retry with exponential backoff for transient API errors

Requests that fail with 429, 502, 503, 504, or network errors (EOF,
connection refused, timeout) are now retried up to 3 times with
exponential backoff (1s → 2s → 4s). Permanent errors (400, 401, etc.)
still fail immediately.

This prevents single transient API hiccups from killing the entire
agent turn — previously, one 429 from DeepSeek would abort the run
with no recovery. Now the agent loop continues transparently.

5 new tests: RetryOn429, RetryOn503, NoRetryOn400, RetryExhausted,
RetryOnNetworkError.

Author: molty3000

internal/llm/client.go

@@ -277,40 +277,89 @@ func (c *Client) Call(ctx context.Context, messages []Message, systemBlocks []Sy
 	}
 
 	url := c.BaseURL + "/chat/completions"
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(reqBytes))
-	if err != nil {
-		return nil, fmt.Errorf("llm: create request: %w", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Authorization", "Bearer "+c.APIKey)
 
-	// Add Anthropic-specific API version header (required for prompt caching)
-	// Safe to always send — other providers ignore unknown headers.
-	req.Header.Set("anthropic-version", "2023-06-01")
+	const maxRetries = 3
+	var lastErr error
+
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		if attempt > 0 {
+			// Exponential backoff: 1s, 2s, 4s
+			backoff := time.Duration(1<<(attempt-1)) * time.Second
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			case <-time.After(backoff):
+			}
+		}
 
-	resp, err := c.http.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("llm: %w", err)
-	}
-	defer resp.Body.Close()
+		req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(reqBytes))
+		if err != nil {
+			return nil, fmt.Errorf("llm: create request: %w", err)
+		}
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Authorization", "Bearer "+c.APIKey)
+		req.Header.Set("anthropic-version", "2023-06-01")
+
+		resp, err := c.http.Do(req)
+		if err != nil {
+			lastErr = fmt.Errorf("llm: %w", err)
+			if isRetryableNetworkError(err) {
+				continue
+			}
+			return nil, lastErr
+		}
 
-	respBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseSize+1))
-	if err != nil {
-		return nil, fmt.Errorf("llm: read response: %w", err)
-	}
-	if len(respBytes) > maxResponseSize {
-		return nil, fmt.Errorf("llm: response exceeds maximum size (%d bytes)", maxResponseSize)
-	}
+		respBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseSize+1))
+		resp.Body.Close()
+		if err != nil {
+			lastErr = fmt.Errorf("llm: read response: %w", err)
+			continue
+		}
+		if len(respBytes) > maxResponseSize {
+			return nil, fmt.Errorf("llm: response exceeds maximum size (%d bytes)", maxResponseSize)
+		}
 
-	if resp.StatusCode != http.StatusOK {
-		body := strings.TrimSpace(string(respBytes))
-		if body != "" {
-			return nil, fmt.Errorf("llm: %s (status %d): %s", resp.Status, resp.StatusCode, body)
+		if resp.StatusCode != http.StatusOK {
+			errBody := strings.TrimSpace(string(respBytes))
+			if errBody != "" {
+				lastErr = fmt.Errorf("llm: %s (status %d): %s", resp.Status, resp.StatusCode, errBody)
+			} else {
+				lastErr = fmt.Errorf("llm: %s (status %d)", resp.Status, resp.StatusCode)
+			}
+			if isRetryableHTTPStatus(resp.StatusCode) {
+				continue
+			}
+			return nil, lastErr
 		}
-		return nil, fmt.Errorf("llm: %s (status %d)", resp.Status, resp.StatusCode)
+
+		return parseResponse(respBytes)
 	}
 
-	return parseResponse(respBytes)
+	return nil, fmt.Errorf("llm: retry exhausted (%d attempts): %w", maxRetries+1, lastErr)
+}
+
+// isRetryableHTTPStatus returns true for HTTP status codes that indicate
+// a transient error safe to retry after a backoff.
+func isRetryableHTTPStatus(code int) bool {
+	return code == http.StatusTooManyRequests ||
+		code == http.StatusBadGateway ||
+		code == http.StatusServiceUnavailable ||
+		code == http.StatusGatewayTimeout
+}
+
+// isRetryableNetworkError returns true for network errors that are likely
+// transient (connection refused, timeout, EOF before headers).
+func isRetryableNetworkError(err error) bool {
+	if err == nil {
+		return false
+	}
+	s := err.Error()
+	// Common transient network error patterns
+	return strings.Contains(s, "connection refused") ||
+		strings.Contains(s, "connection reset") ||
+		strings.Contains(s, "EOF") ||
+		strings.Contains(s, "timeout") ||
+		strings.Contains(s, "TLS handshake timeout")
 }
 
 func parseResponse(data []byte) (*CallResult, error) {

internal/llm/retry_test.go

@@ -0,0 +1,148 @@
+package llm
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestClient_Call_RetryOn429(t *testing.T) {
+	var callCount atomic.Int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		count := int(callCount.Add(1))
+		if count <= 2 {
+			// First two calls return 429
+			w.WriteHeader(http.StatusTooManyRequests)
+			w.Header().Set("Content-Type", "application/json")
+			w.Write([]byte(`{"error":{"message":"Rate limited"}}`))
+			return
+		}
+		// Third call succeeds
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"choices":[{"message":{"content":"hello"}}]}`))
+	}))
+	defer ts.Close()
+
+	c := New(ts.URL, "key", "model", "", 10*time.Second)
+	result, err := c.Call(context.Background(), []Message{
+		{Role: "user", Content: "hi"},
+	}, nil, nil)
+
+	if err != nil {
+		t.Fatalf("unexpected error after retries: %v", err)
+	}
+	if result.Content != "hello" {
+		t.Errorf("content = %q, want %q", result.Content, "hello")
+	}
+	if callCount.Load() != 3 {
+		t.Errorf("call count = %d, want 3", callCount.Load())
+	}
+}
+
+func TestClient_Call_RetryOn503(t *testing.T) {
+	var callCount atomic.Int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		count := int(callCount.Add(1))
+		if count <= 1 {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
+	}))
+	defer ts.Close()
+
+	c := New(ts.URL, "key", "model", "", 10*time.Second)
+	result, err := c.Call(context.Background(), []Message{
+		{Role: "user", Content: "hi"},
+	}, nil, nil)
+
+	if err != nil {
+		t.Fatalf("unexpected error after retry: %v", err)
+	}
+	if result.Content != "ok" {
+		t.Errorf("content = %q, want %q", result.Content, "ok")
+	}
+}
+
+func TestClient_Call_NoRetryOn400(t *testing.T) {
+	var callCount atomic.Int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		callCount.Add(1)
+		w.WriteHeader(http.StatusBadRequest)
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"error":{"message":"bad request"}}`))
+	}))
+	defer ts.Close()
+
+	c := New(ts.URL, "key", "model", "", 10*time.Second)
+	_, err := c.Call(context.Background(), []Message{
+		{Role: "user", Content: "hi"},
+	}, nil, nil)
+
+	if err == nil {
+		t.Fatal("expected error for 400, got nil")
+	}
+	if callCount.Load() != 1 {
+		t.Errorf("call count = %d, want 1 (no retry on 400)", callCount.Load())
+	}
+}
+
+func TestClient_Call_RetryExhausted(t *testing.T) {
+	var callCount atomic.Int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		callCount.Add(1)
+		w.WriteHeader(http.StatusTooManyRequests)
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"error":{"message":"always rate limited"}}`))
+	}))
+	defer ts.Close()
+
+	c := New(ts.URL, "key", "model", "", 10*time.Second)
+	_, err := c.Call(context.Background(), []Message{
+		{Role: "user", Content: "hi"},
+	}, nil, nil)
+
+	if err == nil {
+		t.Fatal("expected error after exhausting retries, got nil")
+	}
+	errStr := err.Error()
+	if errStr == "" || errStr == "expected error after exhausting retries, got nil" {
+		t.Error("expected non-empty error message")
+	}
+	// Should have tried: initial + 3 retries = 4 total
+	if callCount.Load() != 4 {
+		t.Errorf("call count = %d, want 4 (1 initial + 3 retries)", callCount.Load())
+	}
+}
+
+func TestClient_Call_RetryOnNetworkError(t *testing.T) {
+	var callCount atomic.Int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		count := int(callCount.Add(1))
+		if count <= 2 {
+			// Simulate network error by closing the connection
+			conn, _, _ := w.(http.Hijacker).Hijack()
+			conn.Close()
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"choices":[{"message":{"content":"recovered"}}]}`))
+	}))
+	defer ts.Close()
+
+	c := New(ts.URL, "key", "model", "", 10*time.Second)
+	result, err := c.Call(context.Background(), []Message{
+		{Role: "user", Content: "hi"},
+	}, nil, nil)
+
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if result.Content != "recovered" {
+		t.Errorf("content = %q, want %q", result.Content, "recovered")
+	}
+}