fix(controlplane): bound CAS checker lock-hold and release durations

Cap how long a single tick can hold the distributed lock (25 min) so a
hung validation doesn't pin the lock past one tick; the next tick retries.
Also bound the pg_advisory_unlock call (5 s) so a stuck session can't
hang the release path — Postgres will release the lock on session
disconnect anyway.

Assisted-by: Claude Code
Signed-off-by: Miguel Martinez Trivino <miguel@chainloop.dev>

Chainloop-Trace-Sessions: 052e8b56-72b5-4c6c-8d82-ab2d00728889

Author: migmartri

app/controlplane/pkg/biz/casbackend_checker.go

@@ -30,6 +30,10 @@ var casBackendCheckerTracer = otelx.Tracer("chainloop-controlplane", "biz/casbac
 const (
 	defaultInterval          = 30 * time.Minute
 	defaultValidationTimeout = 10 * time.Second
+	// Upper bound on how long a single tick is allowed to hold the
+	// distributed lock. Defends against a hung validation pinning the lock
+	// past one tick; the next tick will retry.
+	defaultMaxTickDuration = 25 * time.Minute
 
 	// Separate keys per scope so the two checker goroutines (defaults vs all backends)
 	// don't block each other.
@@ -152,6 +156,11 @@ func (c *CASBackendChecker) checkBackends(ctx context.Context, defaultsOrFallbac
 	}
 	defer release()
 
+	// Cap how long we can hold the lock. If validations hang, the next tick
+	// retries instead of one stuck pod pinning the lock indefinitely.
+	ctx, cancel := context.WithTimeout(ctx, defaultMaxTickDuration)
+	defer cancel()
+
 	ctx, span := otelx.Start(ctx, casBackendCheckerTracer, "CASBackendChecker.checkBackends")
 	defer span.End()
 

app/controlplane/pkg/data/lock.go

@@ -20,11 +20,17 @@ import (
 	"database/sql"
 	"fmt"
 	"hash/fnv"
+	"time"
 
 	"github.com/chainloop-dev/chainloop/app/controlplane/pkg/biz"
 	"github.com/go-kratos/kratos/v2/log"
 )
 
+// Cap on how long the release path may block. Defends against a stuck
+// session: if pg_advisory_unlock can't return, we drop the connection
+// and let Postgres release the lock on session disconnect.
+const advisoryUnlockTimeout = 5 * time.Second
+
 // PostgresLock implements biz.DistributedLock using Postgres session-level
 // advisory locks (pg_try_advisory_lock / pg_advisory_unlock).
 //
@@ -65,7 +71,10 @@ func (l *PostgresLock) TryAcquire(ctx context.Context, key string) (bool, func()
 	release := func() {
 		// pg_advisory_unlock must run on the same session that took the lock,
 		// and must run even if the caller's context was cancelled (e.g. shutdown).
-		if _, err := conn.ExecContext(context.Background(), "SELECT pg_advisory_unlock($1)", intKey); err != nil {
+		// Bounded so a stuck session can't hang the release path.
+		releaseCtx, cancel := context.WithTimeout(context.Background(), advisoryUnlockTimeout)
+		defer cancel()
+		if _, err := conn.ExecContext(releaseCtx, "SELECT pg_advisory_unlock($1)", intKey); err != nil {
 			l.log.Warnw("msg", "failed to release advisory lock", "key", key, "error", err)
 		}
 		if err := conn.Close(); err != nil {