fix: SLSA cache signing retry and Rekor conflicts

cmd/root.go

@@ -60,6 +60,9 @@ const (
 
 	// EnvvarMaxSigningConcurrency configures maximum concurrent signing operations
 	EnvvarMaxSigningConcurrency = "LEEWAY_MAX_SIGNING_CONCURRENCY"
+
+	// EnvvarSigningRetryAttempts configures retry attempts for transient signing failures
+	EnvvarSigningRetryAttempts = "LEEWAY_SIGNING_RETRY_ATTEMPTS"
 )
 
 const (
@@ -134,6 +137,7 @@ variables have an effect on leeway:
         <light_blue>LEEWAY_SLSA_SOURCE_URI</>  Expected source URI for SLSA verification (github.com/owner/repo).
 <light_blue>LEEWAY_SLSA_REQUIRE_ATTESTATION</>  Require valid attestations; missing/invalid → build locally (true/false).
 <light_blue>LEEWAY_ENABLE_IN_FLIGHT_CHECKSUMS</>  Enable checksumming of cache artifacts (true/false).
+<light_blue>LEEWAY_SIGNING_RETRY_ATTEMPTS</>  Retry attempts for transient Sigstore/Rekor signing failures.
            <light_blue>LEEWAY_EXPERIMENTAL</>  Enables experimental leeway features and commands.
 `),
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {

cmd/sign-cache.go

@@ -31,11 +31,17 @@ Concurrency:
   Configure via --max-signing-concurrency flag or LEEWAY_MAX_SIGNING_CONCURRENCY env var
   Valid range: 1-100 (automatically capped)
 
+Retries:
+  Default: 3 attempts for transient Sigstore/Rekor signing failures
+  Configure via --signing-retry-attempts flag or LEEWAY_SIGNING_RETRY_ATTEMPTS env var
+  Valid range: 1-10 (automatically capped)
+
 Example:
   leeway plumbing sign-cache --from-manifest artifacts-to-sign.txt
   leeway plumbing sign-cache --from-manifest artifacts.txt --dry-run
   leeway plumbing sign-cache --from-manifest artifacts.txt --max-signing-concurrency 30
-  LEEWAY_MAX_SIGNING_CONCURRENCY=30 leeway plumbing sign-cache --from-manifest artifacts.txt`,
+  leeway plumbing sign-cache --from-manifest artifacts.txt --signing-retry-attempts 5
+  LEEWAY_MAX_SIGNING_CONCURRENCY=30 LEEWAY_SIGNING_RETRY_ATTEMPTS=5 leeway plumbing sign-cache --from-manifest artifacts.txt`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		manifestPath, _ := cmd.Flags().GetString("from-manifest")
 		dryRun, _ := cmd.Flags().GetBool("dry-run")
@@ -50,6 +56,15 @@ Example:
 			}
 		}
 
+		signingRetryAttempts, _ := cmd.Flags().GetInt("signing-retry-attempts")
+		if !cmd.Flags().Changed("signing-retry-attempts") {
+			if envVal := os.Getenv(EnvvarSigningRetryAttempts); envVal != "" {
+				if parsed, err := strconv.Atoi(envVal); err == nil && parsed > 0 {
+					signingRetryAttempts = parsed
+				}
+			}
+		}
+
 		if manifestPath == "" {
 			return fmt.Errorf("--from-manifest flag is required")
 		}
@@ -59,7 +74,9 @@ Example:
 			return fmt.Errorf("manifest file does not exist: %s", manifestPath)
 		}
 
-		return runSignCache(cmd.Context(), manifestPath, dryRun, maxConcurrency)
+		signingOptions := signing.DefaultSLSASigningOptions()
+		signingOptions.RetryOptions.MaxAttempts = signingRetryAttempts
+		return runSignCacheWithOptions(cmd.Context(), manifestPath, dryRun, maxConcurrency, signingOptions)
 	},
 }
 
@@ -68,11 +85,16 @@ func init() {
 	signCacheCmd.Flags().String("from-manifest", "", "Path to newline-separated artifact paths file")
 	signCacheCmd.Flags().Bool("dry-run", false, "Log actions without signing or uploading")
 	signCacheCmd.Flags().Int("max-signing-concurrency", 20, "Maximum concurrent signing operations (env: LEEWAY_MAX_SIGNING_CONCURRENCY)")
+	signCacheCmd.Flags().Int("signing-retry-attempts", 3, "Retry attempts for transient Sigstore/Rekor signing failures (env: LEEWAY_SIGNING_RETRY_ATTEMPTS)")
 	_ = signCacheCmd.MarkFlagRequired("from-manifest")
 }
 
 // runSignCache implements the main signing logic
 func runSignCache(ctx context.Context, manifestPath string, dryRun bool, maxConcurrency int) error {
+	return runSignCacheWithOptions(ctx, manifestPath, dryRun, maxConcurrency, signing.DefaultSLSASigningOptions())
+}
+
+func runSignCacheWithOptions(ctx context.Context, manifestPath string, dryRun bool, maxConcurrency int, signingOptions signing.SLSASigningOptions) error {
 	log.WithFields(log.Fields{
 		"manifest": manifestPath,
 		"dry_run":  dryRun,
@@ -141,6 +163,15 @@ func runSignCache(ctx context.Context, manifestPath string, dryRun bool, maxConc
 
 	log.WithField("maxConcurrency", maxConcurrency).Info("Configured signing concurrency")
 
+	if signingOptions.RetryOptions.MaxAttempts < 1 {
+		log.WithField("provided", signingOptions.RetryOptions.MaxAttempts).Warn("signing retry attempts must be at least 1, using 1")
+		signingOptions.RetryOptions.MaxAttempts = 1
+	} else if signingOptions.RetryOptions.MaxAttempts > 10 {
+		log.WithField("provided", signingOptions.RetryOptions.MaxAttempts).Warn("signing retry attempts exceeds maximum, capping at 10")
+		signingOptions.RetryOptions.MaxAttempts = 10
+	}
+	log.WithField("signingRetryAttempts", signingOptions.RetryOptions.MaxAttempts).Info("Configured signing retry attempts")
+
 	const maxAcceptableFailureRate = 0.5 // Fail command if more than 50% of artifacts fail
 	semaphore := make(chan struct{}, maxConcurrency)
 
@@ -171,7 +202,7 @@ func runSignCache(ctx context.Context, manifestPath string, dryRun bool, maxConc
 
 			log.WithField("artifact", artifactPath).Debug("Starting artifact processing")
 
-			if err := processArtifact(ctx, artifactPath, githubCtx, remoteCache, dryRun); err != nil {
+			if err := processArtifactWithOptions(ctx, artifactPath, githubCtx, remoteCache, dryRun, signingOptions); err != nil {
 				signingErr := signing.CategorizeError(artifactPath, err)
 
 				mu.Lock()
@@ -227,8 +258,10 @@ func runSignCache(ctx context.Context, manifestPath string, dryRun bool, maxConc
 	return nil
 }
 
+var generateSignedSLSAAttestation = signing.GenerateSignedSLSAAttestationWithOptions
+
 // processArtifact handles signing and uploading of a single artifact using integrated SLSA signing
-func processArtifact(ctx context.Context, artifactPath string, githubCtx *signing.GitHubContext, remoteCache cache.RemoteCache, dryRun bool) error {
+func processArtifactWithOptions(ctx context.Context, artifactPath string, githubCtx *signing.GitHubContext, remoteCache cache.RemoteCache, dryRun bool, signingOptions signing.SLSASigningOptions) error {
 	log.WithFields(log.Fields{
 		"artifact": artifactPath,
 		"dry_run":  dryRun,
@@ -240,7 +273,7 @@ func processArtifact(ctx context.Context, artifactPath string, githubCtx *signin
 	}
 
 	// Single step: generate and sign SLSA attestation using integrated approach
-	signedAttestation, err := signing.GenerateSignedSLSAAttestation(ctx, artifactPath, githubCtx)
+	signedAttestation, err := generateSignedSLSAAttestation(ctx, artifactPath, githubCtx, signingOptions)
 	if err != nil {
 		return fmt.Errorf("failed to generate signed attestation: %w", err)
 	}

cmd/sign-cache_test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -13,6 +14,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/gitpod-io/leeway/pkg/leeway/cache"
 	"github.com/gitpod-io/leeway/pkg/leeway/signing"
 )
 
@@ -41,6 +43,31 @@ func createMockArtifact(t *testing.T, dir string, name string) string {
 	return artifactPath
 }
 
+type signCacheMockRemoteCache struct {
+	uploadFileCalls int
+}
+
+func (m *signCacheMockRemoteCache) ExistingPackages(ctx context.Context, pkgs []cache.Package) (map[cache.Package]struct{}, error) {
+	return map[cache.Package]struct{}{}, nil
+}
+
+func (m *signCacheMockRemoteCache) Download(ctx context.Context, dst cache.LocalCache, pkgs []cache.Package) map[string]cache.DownloadResult {
+	return map[string]cache.DownloadResult{}
+}
+
+func (m *signCacheMockRemoteCache) Upload(ctx context.Context, src cache.LocalCache, pkgs []cache.Package) error {
+	return nil
+}
+
+func (m *signCacheMockRemoteCache) UploadFile(ctx context.Context, filePath string, key string) error {
+	m.uploadFileCalls++
+	return nil
+}
+
+func (m *signCacheMockRemoteCache) HasFile(ctx context.Context, key string) (bool, error) {
+	return false, nil
+}
+
 // TestSignCacheCommand_Exists verifies the command is properly registered
 func TestSignCacheCommand_Exists(t *testing.T) {
 	// Verify sign-cache command exists under plumbing
@@ -69,6 +96,10 @@ func TestSignCacheCommand_FlagDefinitions(t *testing.T) {
 	require.NotNil(t, dryRunFlag, "dry-run flag should exist")
 	assert.Equal(t, "bool", dryRunFlag.Value.Type())
 
+	retryFlag := cmd.Flags().Lookup("signing-retry-attempts")
+	require.NotNil(t, retryFlag, "signing-retry-attempts flag should exist")
+	assert.Equal(t, "int", retryFlag.Value.Type())
+
 	// Verify from-manifest is required
 	annotations := cmd.Flags().Lookup("from-manifest").Annotations
 	assert.NotNil(t, annotations, "from-manifest should have required annotation")
@@ -479,6 +510,32 @@ func TestSignCache_DryRunMode(t *testing.T) {
 	assert.False(t, operationsPerformed, "No real operations should occur in dry-run")
 }
 
+func TestProcessArtifact_RekorConflictWithoutProofFailsBeforeUpload(t *testing.T) {
+	tmpDir := t.TempDir()
+	artifact := createMockArtifact(t, tmpDir, "conflict.tar.gz")
+	remoteCache := &signCacheMockRemoteCache{}
+
+	originalGenerator := generateSignedSLSAAttestation
+	generateSignedSLSAAttestation = func(ctx context.Context, artifactPath string, githubCtx *signing.GitHubContext, options signing.SLSASigningOptions) (*signing.SignedAttestationResult, error) {
+		return nil, fmt.Errorf("[POST /api/v1/log/entries][409] createLogEntryConflict an equivalent entry already exists without verifiable proof")
+	}
+	t.Cleanup(func() {
+		generateSignedSLSAAttestation = originalGenerator
+	})
+
+	err := processArtifactWithOptions(context.Background(), artifact, &signing.GitHubContext{
+		RunID:       "123456",
+		Repository:  "gitpod-io/leeway",
+		SHA:         "abc123",
+		ServerURL:   "https://github.com",
+		WorkflowRef: ".github/workflows/build.yml@main",
+	}, remoteCache, false, signing.DefaultSLSASigningOptions())
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "createLogEntryConflict")
+	assert.Equal(t, 0, remoteCache.uploadFileCalls, "unverified Rekor conflicts must not upload unsigned artifacts or attestations")
+}
+
 // Helper: Set up minimal GitHub environment for testing
 func setupGitHubEnv(t *testing.T) {
 	t.Setenv("GITHUB_RUN_ID", "123456")
@@ -514,7 +571,7 @@ func TestSignCache_ErrorScenarios(t *testing.T) {
 				manifestPath := createTestManifest(t, tmpDir, []string{artifact})
 
 				// Ensure no cache env vars set
-				os.Unsetenv("LEEWAY_REMOTE_CACHE_BUCKET")
+				_ = os.Unsetenv("LEEWAY_REMOTE_CACHE_BUCKET")
 
 				return manifestPath, func() {}
 			},

go.mod

@@ -15,6 +15,8 @@ require (
 	github.com/disiqueira/gotree v1.0.0
 	github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd
 	github.com/fsnotify/fsnotify v1.9.0
+	github.com/go-openapi/runtime v0.28.0
+	github.com/go-openapi/strfmt v0.23.0
 	github.com/google/go-cmp v0.7.0
 	github.com/google/go-containerregistry v0.20.6
 	github.com/google/uuid v1.6.0
@@ -28,6 +30,7 @@ require (
 	github.com/segmentio/analytics-go/v3 v3.3.0
 	github.com/segmentio/textio v1.2.0
 	github.com/sigstore/protobuf-specs v0.5.0
+	github.com/sigstore/rekor v1.4.2
 	github.com/sigstore/sigstore-go v1.1.3
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.10.1
@@ -185,9 +188,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/loads v0.22.0 // indirect
-	github.com/go-openapi/runtime v0.28.0 // indirect
 	github.com/go-openapi/spec v0.21.0 // indirect
-	github.com/go-openapi/strfmt v0.23.0 // indirect
 	github.com/go-openapi/swag v0.24.1 // indirect
 	github.com/go-openapi/swag/cmdutils v0.24.0 // indirect
 	github.com/go-openapi/swag/conv v0.24.0 // indirect
@@ -307,7 +308,6 @@ require (
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/shibumi/go-pathspec v1.3.0 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
-	github.com/sigstore/rekor v1.4.2 // indirect
 	github.com/sigstore/rekor-tiles v0.1.11 // indirect
 	github.com/sigstore/sigstore v1.9.6-0.20250729224751-181c5d3339b3 // indirect
 	github.com/sigstore/timestamp-authority v1.2.9 // indirect

pkg/leeway/signing/attestation.go

@@ -77,8 +77,25 @@ type SignedAttestationResult struct {
 	ArtifactName     string `json:"artifact_name"`     // Name of the artifact
 }
 
+// SLSASigningOptions configures SLSA attestation signing.
+type SLSASigningOptions struct {
+	RetryOptions RetryOptions
+}
+
+// DefaultSLSASigningOptions returns conservative defaults for CI signing.
+func DefaultSLSASigningOptions() SLSASigningOptions {
+	return SLSASigningOptions{
+		RetryOptions: DefaultRetryOptions(),
+	}
+}
+
 // GenerateSignedSLSAAttestation generates and signs SLSA provenance in one integrated step
 func GenerateSignedSLSAAttestation(ctx context.Context, artifactPath string, githubCtx *GitHubContext) (*SignedAttestationResult, error) {
+	return GenerateSignedSLSAAttestationWithOptions(ctx, artifactPath, githubCtx, DefaultSLSASigningOptions())
+}
+
+// GenerateSignedSLSAAttestationWithOptions generates and signs SLSA provenance with explicit signing options.
+func GenerateSignedSLSAAttestationWithOptions(ctx context.Context, artifactPath string, githubCtx *GitHubContext, signingOptions SLSASigningOptions) (*SignedAttestationResult, error) {
 	// Calculate artifact checksum
 	checksum, err := computeSHA256(artifactPath)
 	if err != nil {
@@ -94,8 +111,20 @@ func GenerateSignedSLSAAttestation(ctx context.Context, artifactPath string, git
 
 	// Extract builder ID from OIDC token to match certificate identity
 	// This is critical for compatibility with reusable workflows
-	builderID, err := extractBuilderIDFromOIDC(ctx, githubCtx)
-	if err != nil {
+	var builderID string
+	if err := WithRetryOptions(ctx, signingOptions.RetryOptions, func() error {
+		var retryErr error
+		builderID, retryErr = extractBuilderIDFromOIDC(ctx, githubCtx)
+		if retryErr != nil {
+			return &SigningError{
+				Type:     ErrorTypeSigstore,
+				Artifact: filepath.Base(artifactPath),
+				Message:  fmt.Sprintf("failed to extract builder ID from OIDC token: %v", retryErr),
+				Cause:    retryErr,
+			}
+		}
+		return nil
+	}); err != nil {
 		return nil, fmt.Errorf("failed to extract builder ID from OIDC token: %w", err)
 	}
 
@@ -163,8 +192,12 @@ func GenerateSignedSLSAAttestation(ctx context.Context, artifactPath string, git
 	}).Debug("Generated SLSA provenance, proceeding with integrated signing")
 
 	// Generate and sign the SLSA provenance using Sigstore
-	signedAttestation, err := signProvenanceWithSigstore(ctx, stmt)
-	if err != nil {
+	var signedAttestation []byte
+	if err := WithRetryOptions(ctx, signingOptions.RetryOptions, func() error {
+		var retryErr error
+		signedAttestation, retryErr = signProvenanceWithSigstore(ctx, stmt, signingOptions)
+		return retryErr
+	}); err != nil {
 		return nil, fmt.Errorf("failed to sign SLSA provenance: %w", err)
 	}
 
@@ -192,7 +225,7 @@ func computeSHA256(filePath string) (string, error) {
 }
 
 // signProvenanceWithSigstore signs SLSA provenance using Sigstore keyless signing
-func signProvenanceWithSigstore(ctx context.Context, statement *in_toto.Statement) ([]byte, error) {
+func signProvenanceWithSigstore(ctx context.Context, statement *in_toto.Statement, signingOptions SLSASigningOptions) ([]byte, error) {
 	// Validate GitHub OIDC environment
 	if err := validateSigstoreEnvironment(); err != nil {
 		return nil, fmt.Errorf("sigstore environment validation failed: %w", err)
@@ -307,7 +340,16 @@ func signProvenanceWithSigstore(ctx context.Context, statement *in_toto.Statemen
 				Retries: 1,
 				Version: rekorService.MajorAPIVersion,
 			}
-			bundleOpts.TransparencyLogs = append(bundleOpts.TransparencyLogs, sign.NewRekor(rekorOpts))
+			if rekorService.MajorAPIVersion == 1 {
+				bundleOpts.TransparencyLogs = append(bundleOpts.TransparencyLogs, newRecoveringRekorV1(recoveringRekorOptions{
+					BaseURL:      rekorOpts.BaseURL,
+					Timeout:      rekorOpts.Timeout,
+					Retries:      rekorOpts.Retries,
+					RetryOptions: signingOptions.RetryOptions,
+				}))
+			} else {
+				bundleOpts.TransparencyLogs = append(bundleOpts.TransparencyLogs, sign.NewRekor(rekorOpts))
+			}
 		}
 	}
 

pkg/leeway/signing/attestation_test.go

@@ -926,7 +926,7 @@ func TestSigningError_IsRetryable_AllTypes(t *testing.T) {
 		retryable bool
 	}{
 		{ErrorTypeNetwork, true},
-		{ErrorTypeSigstore, true},
+		{ErrorTypeSigstore, false},
 		{ErrorTypePermission, false},
 		{ErrorTypeValidation, false},
 		{ErrorTypeFileSystem, false},