Skip to content

fix(uffd): add retry with exponential backoff on source.Slice() error #2389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jakubno merged 4 commits into from
Apr 14, 2026
+55 −3
Merged

fix(uffd): add retry with exponential backoff on source.Slice() error #2389

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
893cfac
fix(uffd): add retry with exponential backoff on source.Slice() error…
jakubno Apr 14, 2026
df89aa2
refactor(uffd): rename sliceMaxAttempts to sliceMaxRetries for clarity
jakubno Apr 14, 2026
a7ed495
fix(uffd): skip retry logging when context is already cancelled
jakubno Apr 14, 2026
5fa2cb0
fix(uffd): stop backoff timer explicitly on context cancellation
jakubno Apr 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 55 additions & 3 deletions packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ import (
"context"
"errors"
"fmt"
"math/rand"
"sync"
"syscall"
"time"
"unsafe"

"go.opentelemetry.io/otel"
Expand All @@ -25,6 +27,17 @@ var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/pkg/san

const maxRequestsInProgress = 4096

const (
// sliceMaxRetries is the number of times to retry source.Slice() after the initial attempt.
// Total attempts = sliceMaxRetries + 1.
sliceMaxRetries = 3
// sliceRetryBaseDelay is the initial backoff delay before the first retry.
// Subsequent retries double the delay (exponential backoff), capped at sliceRetryMaxDelay.
sliceRetryBaseDelay = 50 * time.Millisecond
// sliceRetryMaxDelay is the maximum backoff delay between retries.
sliceRetryMaxDelay = 500 * time.Millisecond
)

var ErrUnexpectedEventType = errors.New("unexpected event type")

// hasEvent checks if a specific poll event flag is set in revents.
Expand Down Expand Up @@ -331,7 +344,43 @@ func (u *Userfaultfd) faultPage(
}
}()

b, dataErr := source.Slice(ctx, offset, int64(pagesize))
var b []byte
var dataErr error
var attempt int

retryLoop:
for attempt = range sliceMaxRetries + 1 {
b, dataErr = source.Slice(ctx, offset, int64(pagesize))
if dataErr == nil {
break
}

if attempt >= sliceMaxRetries || ctx.Err() != nil {
break
}

u.logger.Warn(ctx, "UFFD serve slice error, retrying",
zap.Int("attempt", attempt+1),
zap.Int("max_attempts", sliceMaxRetries+1),
zap.Error(dataErr),
)

delay := min(sliceRetryBaseDelay<<attempt, sliceRetryMaxDelay)
jitter := time.Duration(rand.Int63n(int64(delay) / 2))

Comment thread
jakubno marked this conversation as resolved.
backoff := time.NewTimer(delay + jitter)

select {
case <-ctx.Done():
backoff.Stop()

dataErr = errors.Join(dataErr, ctx.Err())

break retryLoop
case <-backoff.C:
}
}

if dataErr != nil {
var signalErr error
if onFailure != nil {
Expand All @@ -341,9 +390,12 @@ func (u *Userfaultfd) faultPage(
joinedErr := errors.Join(dataErr, signalErr)

span.RecordError(joinedErr)
u.logger.Error(ctx, "UFFD serve data fetch error", zap.Error(joinedErr))
u.logger.Error(ctx, "UFFD serve data fetch error after retries",
zap.Int("attempts", attempt+1),
zap.Error(joinedErr),
)

return fmt.Errorf("failed to read from source: %w", joinedErr)
return fmt.Errorf("failed to read from source after %d attempts: %w", attempt+1, joinedErr)
}

var copyMode CULong
Expand Down
Loading