fix(docker): retry devcontainer builds on transient failures

pkg/driver/docker/build.go

@@ -17,6 +17,7 @@ import (
 	"github.com/devsy-org/devsy/pkg/driver"
 	"github.com/devsy-org/devsy/pkg/log"
 	"github.com/devsy-org/devsy/pkg/provider"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 func (d *dockerDriver) BuildDevContainer(
@@ -405,13 +406,48 @@ func (d *dockerDriver) executeBuild(
 	buildOptions *build.BuildOptions,
 ) error {
 	log.Infof("build with %s", strategy.name())
+
+	var lastErr error
+	err := wait.ExponentialBackoffWithContext(
+		ctx,
+		buildBackoff,
+		func(ctx context.Context) (bool, error) {
+			output, buildErr := d.runBuildAttempt(ctx, strategy, req, buildOptions)
+			lastErr = classifyBuildError(buildErr, output)
+			if lastErr == nil {
+				return true, nil
+			}
+			if !errors.Is(lastErr, errTransientBuild) {
+				return false, lastErr
+			}
+			log.Warnf("transient build failure, retrying: %v", lastErr)
+			return false, nil
+		},
+	)
+	if wait.Interrupted(err) {
+		return lastErr
+	}
+	return err
+}
+
+// runBuildAttempt runs the build once and returns the tail of the streamed
+// output alongside the error for transient-failure classification.
+func (d *dockerDriver) runBuildAttempt(
+	ctx context.Context,
+	strategy buildStrategy,
+	req driver.BuildRequest,
+	buildOptions *build.BuildOptions,
+) (string, error) {
 	writer := log.Writer(log.LevelInfo)
 	defer func() { _ = writer.Close() }()
 
-	if err := strategy.build(ctx, writer, req.Options.Platform, buildOptions); err != nil {
-		return fmt.Errorf("%s build: %w", strategy.name(), err)
+	outputTail := &tailBuffer{limit: maxStderrDiagnostics}
+	multiWriter := io.MultiWriter(writer, outputTail)
+
+	if err := strategy.build(ctx, multiWriter, req.Options.Platform, buildOptions); err != nil {
+		return outputTail.String(), fmt.Errorf("%s build: %w", strategy.name(), err)
 	}
-	return nil
+	return "", nil
 }
 
 func (d *dockerDriver) createBuildInfo(

pkg/driver/docker/build_retry.go

@@ -0,0 +1,45 @@
+package docker
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+var buildBackoff = wait.Backoff{
+	Duration: 2 * time.Second,
+	Factor:   2.0,
+	Steps:    3,
+}
+
+// errTransientBuild marks a build failure that is safe to retry.
+var errTransientBuild = errors.New("transient build failure")
+
+// transientBuildPatterns are lowercase substrings matched against the build
+// error and output. Add a pattern when a new transient failure is observed.
+var transientBuildPatterns = []string{
+	"incompleteread",
+}
+
+// classifyBuildError wraps err with errTransientBuild when it matches a
+// known-transient signature. Context cancellation is never transient.
+func classifyBuildError(err error, output string) error {
+	if err == nil {
+		return nil
+	}
+	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+		return err
+	}
+
+	haystack := strings.ToLower(err.Error() + "\n" + output)
+	for _, pattern := range transientBuildPatterns {
+		if strings.Contains(haystack, pattern) {
+			return fmt.Errorf("%w: %w", errTransientBuild, err)
+		}
+	}
+	return err
+}

pkg/driver/docker/build_retry_test.go

@@ -0,0 +1,64 @@
+package docker
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+)
+
+func TestClassifyBuildError(t *testing.T) {
+	t.Parallel()
+
+	// The opaque error the internal buildkit strategy returns; the transient
+	// signal lives only in the streamed output.
+	solveErr := errors.New(
+		"internal buildkit build: build: failed to solve: process " +
+			`"/bin/sh -c ./devcontainer-features-install.sh" did not complete successfully: exit code: 1`,
+	)
+	incompleteReadOutput := "http.client.IncompleteRead: IncompleteRead(13384521 bytes read, 1002976 more expected)"
+
+	tests := []struct {
+		name   string
+		err    error
+		output string
+		want   bool
+	}{
+		{name: "nil error", err: nil, want: false},
+		{
+			name:   "incomplete read in output",
+			err:    solveErr,
+			output: incompleteReadOutput,
+			want:   true,
+		},
+		{
+			name:   "genuine build failure",
+			err:    solveErr,
+			output: "npm ERR! missing script: build",
+			want:   false,
+		},
+		{
+			name: "context canceled is not transient",
+			err:  fmt.Errorf("build: %w", context.Canceled),
+			want: false,
+		},
+		{
+			name: "deadline exceeded is not transient",
+			err:  fmt.Errorf("build: %w", context.DeadlineExceeded),
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			classified := classifyBuildError(tt.err, tt.output)
+			if got := errors.Is(classified, errTransientBuild); got != tt.want {
+				t.Errorf("transient = %v, want %v", got, tt.want)
+			}
+			if tt.err != nil && !errors.Is(classified, tt.err) {
+				t.Errorf("classifyBuildError() dropped the original error")
+			}
+		})
+	}
+}