retry and error parsing

common/persistence/schema/version.go

@@ -15,7 +15,7 @@ func VerifyCompatibleVersion(
 
 	version, err := versionReader.ReadSchemaVersion(dbName)
 	if err != nil {
-		return fmt.Errorf("unable to read DB schema version keyspace/database: %s error: %v", dbName, err.Error())
+		return fmt.Errorf("unable to read DB schema version keyspace/database: %s error: %w", dbName, err)
 	}
 	// In most cases, the versions should match. However if after a schema upgrade there is a code
 	// rollback, the code version (expected version) would fall lower than the actual version in

common/persistence/sql/version_checker.go

@@ -1,8 +1,14 @@
 package sql
 
 import (
+	"errors"
+	"time"
+
+	"go.temporal.io/api/serviceerror"
+	"go.temporal.io/server/common/backoff"
 	"go.temporal.io/server/common/config"
 	"go.temporal.io/server/common/log"
+	"go.temporal.io/server/common/log/tag"
 	"go.temporal.io/server/common/metrics"
 	"go.temporal.io/server/common/persistence/sql/sqlplugin"
 	"go.temporal.io/server/common/resolver"
@@ -55,11 +61,26 @@ func checkCompatibleVersion(
 	dbKind sqlplugin.DbKind,
 	logger log.Logger,
 ) error {
-	db, err := NewSQLAdminDB(dbKind, cfg, r, logger, metrics.NoopMetricsHandler)
-	if err != nil {
-		return err
-	}
-	defer func() { _ = db.Close() }()
-
-	return db.VerifyVersion()
+	policy := backoff.NewExponentialRetryPolicy(1 * time.Second).
+		WithMaximumInterval(10 * time.Second).
+		WithExpirationInterval(backoff.NoInterval)
+	return backoff.ThrottleRetry(
+		func() error {
+			db, err := NewSQLAdminDB(dbKind, cfg, r, logger, metrics.NoopMetricsHandler)
+			if err != nil {
+				return err
+			}
+			defer func() { _ = db.Close() }()
+			if err := db.VerifyVersion(); err != nil {
+				logger.Warn("sql schema version check failed, retrying", tag.Error(err))
+				return err
+			}
+			return nil
+		},
+		policy,
+		func(err error) bool {
+			var unavailableErr *serviceerror.Unavailable
+			return errors.As(err, &unavailableErr)
+		},
+	)
 }