From 83c92b7086891181422d9c41bbe137a68570ff5b Mon Sep 17 00:00:00 2001 From: Gurudatta Patil Date: Wed, 8 Apr 2026 17:02:04 -0500 Subject: [PATCH 1/2] retry and error parsing --- common/persistence/schema/version.go | 2 +- common/persistence/sql/version_checker.go | 30 +++++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/common/persistence/schema/version.go b/common/persistence/schema/version.go index a25b35f4bc..e4bd54eb54 100644 --- a/common/persistence/schema/version.go +++ b/common/persistence/schema/version.go @@ -15,7 +15,7 @@ func VerifyCompatibleVersion( version, err := versionReader.ReadSchemaVersion(dbName) if err != nil { - return fmt.Errorf("unable to read DB schema version keyspace/database: %s error: %v", dbName, err.Error()) + return fmt.Errorf("unable to read DB schema version keyspace/database: %s error: %w", dbName, err) } // In most cases, the versions should match. However if after a schema upgrade there is a code // rollback, the code version (expected version) would fall lower than the actual version in diff --git a/common/persistence/sql/version_checker.go b/common/persistence/sql/version_checker.go index 229dea832c..81ce8c6bb8 100644 --- a/common/persistence/sql/version_checker.go +++ b/common/persistence/sql/version_checker.go @@ -1,6 +1,11 @@ package sql import ( + "errors" + "time" + + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/config" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" @@ -55,11 +60,22 @@ func checkCompatibleVersion( dbKind sqlplugin.DbKind, logger log.Logger, ) error { - db, err := NewSQLAdminDB(dbKind, cfg, r, logger, metrics.NoopMetricsHandler) - if err != nil { - return err - } - defer func() { _ = db.Close() }() - - return db.VerifyVersion() + policy := backoff.NewExponentialRetryPolicy(1 * time.Second). + WithMaximumInterval(10 * time.Second). + WithExpirationInterval(backoff.NoInterval) + return backoff.ThrottleRetry( + func() error { + db, err := NewSQLAdminDB(dbKind, cfg, r, logger, metrics.NoopMetricsHandler) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + return db.VerifyVersion() + }, + policy, + func(err error) bool { + var unavailableErr *serviceerror.Unavailable + return errors.As(err, &unavailableErr) + }, + ) } From 83cd8d50d2df84873d20a17b319db9d6637f7566 Mon Sep 17 00:00:00 2001 From: Gurudatta Patil Date: Wed, 8 Apr 2026 17:10:55 -0500 Subject: [PATCH 2/2] log db not connecting --- common/persistence/sql/version_checker.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/persistence/sql/version_checker.go b/common/persistence/sql/version_checker.go index 81ce8c6bb8..cf91e9701a 100644 --- a/common/persistence/sql/version_checker.go +++ b/common/persistence/sql/version_checker.go @@ -8,6 +8,7 @@ import ( "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/config" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/persistence/sql/sqlplugin" "go.temporal.io/server/common/resolver" @@ -70,7 +71,11 @@ func checkCompatibleVersion( return err } defer func() { _ = db.Close() }() - return db.VerifyVersion() + if err := db.VerifyVersion(); err != nil { + logger.Warn("sql schema version check failed, retrying", tag.Error(err)) + return err + } + return nil }, policy, func(err error) bool {