diff --git a/base/constants_syncdocs.go b/base/constants_syncdocs.go index ab6aa2fe82..7be7e981d6 100644 --- a/base/constants_syncdocs.go +++ b/base/constants_syncdocs.go @@ -449,7 +449,7 @@ func marshalSyncInfo(syncInfo *SyncInfo, clusterCompatVersion *ClusterCompatVers if err != nil { return nil, fmt.Errorf("Error marshalling syncInfo: %v", err) } - if clusterCompatVersion != nil && clusterCompatVersion.AtLeast(4, 1) { + if clusterCompatVersion.AtLeast(4, 1) { payload = append([]byte{byte(SyncInfoTypeV1)}, payload...) } return payload, nil diff --git a/base/version_cluster_compat.go b/base/version_cluster_compat.go index 6148adab37..67d63b35b5 100644 --- a/base/version_cluster_compat.go +++ b/base/version_cluster_compat.go @@ -29,7 +29,12 @@ func NewClusterCompatVersion(major, minor uint8) ClusterCompatVersion { } // AtLeast returns true if this version is greater than or equal to the given major.minor. -func (v ClusterCompatVersion) AtLeast(major, minor uint8) bool { +// A nil receiver returns false, representing an unknown cluster compat version that should not +// gate any version-dependent behavior on. +func (v *ClusterCompatVersion) AtLeast(major, minor uint8) bool { + if v == nil { + return false + } if v.Major != major { return v.Major > major } diff --git a/db/background_mgr_attachment_migration.go b/db/background_mgr_attachment_migration.go index 86dd532d4c..17958c1a28 100644 --- a/db/background_mgr_attachment_migration.go +++ b/db/background_mgr_attachment_migration.go @@ -197,11 +197,7 @@ func (a *AttachmentMigrationManager) Run(ctx context.Context, options map[string // set sync info metadata version for _, collectionID := range currCollectionIDs { dbc := db.CollectionByID[collectionID] - var ccv *base.ClusterCompatVersion - if a.databaseCtx.Options.ClusterCompatVersion != nil { - ccv = a.databaseCtx.Options.ClusterCompatVersion() - } - if err := base.SetSyncInfoMetaVersion(ctx, dbc.dataStore, MetaVersionValue, ccv); err != nil { + if err := base.SetSyncInfoMetaVersion(ctx, dbc.dataStore, MetaVersionValue, a.databaseCtx.ClusterCompatVersion()); err != nil { base.WarnfCtx(ctx, "[%s] Completed attachment migration, but unable to update syncInfo for collection %s: %v", migrationLoggingID, dbc.Name, err) return err } diff --git a/db/background_mgr_attachment_migration_test.go b/db/background_mgr_attachment_migration_test.go index 326065dabf..09841be926 100644 --- a/db/background_mgr_attachment_migration_test.go +++ b/db/background_mgr_attachment_migration_test.go @@ -313,7 +313,7 @@ func TestAttachmentMigrationCheckpointPrefix(t *testing.T) { } // TestAttachmentMigrationWritesV1SyncInfoAtCcv41 verifies the end-to-end wiring from -// DatabaseContextOptions.ClusterCompatVersion through AttachmentMigrationManager.Run into +// DatabaseContext.ClusterCompatVersion through AttachmentMigrationManager.Run into // base.SetSyncInfoMetaVersion — when ccv>=4.1 the syncInfo doc is written with the V1 // version-byte prefix. func TestAttachmentMigrationWritesV1SyncInfoAtCcv41(t *testing.T) { @@ -321,7 +321,7 @@ func TestAttachmentMigrationWritesV1SyncInfoAtCcv41(t *testing.T) { defer db.Close(ctx) ccv := base.NewClusterCompatVersion(4, 1) - db.Options.ClusterCompatVersion = func() *base.ClusterCompatVersion { return &ccv } + db.ClusterCompatVersionFunc = func() *base.ClusterCompatVersion { return &ccv } collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db) diff --git a/db/background_mgr_metadata_migration.go b/db/background_mgr_metadata_migration.go new file mode 100644 index 0000000000..3a57267742 --- /dev/null +++ b/db/background_mgr_metadata_migration.go @@ -0,0 +1,127 @@ +/* +Copyright 2026-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package db + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/couchbase/sync_gateway/base" + "github.com/google/uuid" +) + +type MetadataMigrationManager struct { + docsProcessed atomic.Int64 + docsFailed atomic.Int64 + MigrationID string + dbContext *DatabaseContext + lock sync.RWMutex +} + +const MetadataMigrationManagerName = "metadata_migration" + +var _ BackgroundManagerProcessI = &MetadataMigrationManager{} + +func NewMetadataMigrationManager(dbContext *DatabaseContext) *BackgroundManager { + return &BackgroundManager{ + name: MetadataMigrationManagerName, + Process: &MetadataMigrationManager{dbContext: dbContext}, + clusterAwareOptions: &ClusterAwareBackgroundManagerOptions{ + metadataStore: dbContext.MetadataStore, + metaKeys: dbContext.MetadataKeys, + processSuffix: MetadataMigrationManagerName, + }, + terminator: base.NewSafeTerminator(), + } +} + +type MigrationManagerResponse struct { + BackgroundManagerStatus + DocsProcessed int64 `json:"docs_processed"` + DocsFailed int64 `json:"docs_failed"` + MigrationID string `json:"migration_id"` +} + +type MigrationManagerStatusDoc struct { + MigrationManagerResponse `json:"status"` +} + +func (m *MetadataMigrationManager) Init(ctx context.Context, options map[string]any, clusterStatus []byte) error { + newRunInit := func() error { + uniqueUUID, err := uuid.NewRandom() + if err != nil { + return err + } + + m.MigrationID = uniqueUUID.String() + base.InfofCtx(ctx, base.KeyAll, "Metadata Migration: Starting new migration run with migration ID: %s", m.MigrationID) + return nil + } + + if clusterStatus != nil { + var status MigrationManagerStatusDoc + err := base.JSONUnmarshal(clusterStatus, &status) + // If the previous run completed, or there was an error during unmarshalling the status start again + if status.State == BackgroundProcessStateCompleted || err != nil { + return newRunInit() + } + m.docsProcessed.Store(status.DocsProcessed) + m.docsFailed.Store(status.DocsFailed) + m.MigrationID = status.MigrationID + base.InfofCtx(ctx, base.KeyAll, "Metadata Migration: Resuming migration run with migration ID: %s, docs processed: %d, docs failed: %d", m.MigrationID, status.DocsProcessed, status.DocsFailed) + return nil + } + return newRunInit() +} + +func (m *MetadataMigrationManager) Run(ctx context.Context, options map[string]any, persistClusterStatusCallback updateStatusCallbackFunc, terminator *base.SafeTerminator) error { + metadataMigrationLoggingID := "Metadata Migration: " + m.MigrationID + + persistClusterStatus := func() { + err := persistClusterStatusCallback(ctx) + if err != nil { + base.WarnfCtx(ctx, "[%s] Failed to persist latest cluster status for metadata migration: %v", metadataMigrationLoggingID, err) + } + } + defer persistClusterStatus() + + base.InfofCtx(ctx, base.KeyAll, "[%s] Metadata Migration TODO: CBG-5228", metadataMigrationLoggingID) + return nil +} + +func (m *MetadataMigrationManager) ResetStatus() { + m.lock.Lock() + defer m.lock.Unlock() + m.docsProcessed.Store(0) + m.docsFailed.Store(0) + m.MigrationID = "" +} + +func (m *MetadataMigrationManager) SetProcessStatus(ctx context.Context, previousStatus []byte, newStatus []byte) { + // no-op +} + +func (m *MetadataMigrationManager) GetProcessStatus(status BackgroundManagerStatus, previousStatus []byte) (statusOut []byte, meta []byte, err error) { + m.lock.RLock() + defer m.lock.RUnlock() + resp := MigrationManagerResponse{ + BackgroundManagerStatus: status, + DocsProcessed: m.docsProcessed.Load(), + DocsFailed: m.docsFailed.Load(), + MigrationID: m.MigrationID, + } + statusOut, err = base.JSONMarshal(resp) + if err != nil { + return nil, nil, err + } + return statusOut, nil, nil +} diff --git a/db/background_mgr_metadata_migration_test.go b/db/background_mgr_metadata_migration_test.go new file mode 100644 index 0000000000..f3ae05a73e --- /dev/null +++ b/db/background_mgr_metadata_migration_test.go @@ -0,0 +1,154 @@ +/* +Copyright 2026-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package db + +import ( + "context" + "testing" + "time" + + sgbucket "github.com/couchbase/sg-bucket" + "github.com/couchbase/sync_gateway/base" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestShouldRunMetadataMigration is a truth table covering the guard that decides whether +// metadata migration may be armed: it requires both the UseSystemMetadataCollection opt-in +// and a cluster compatibility version of at least 4.1. +func TestShouldRunMetadataMigration(t *testing.T) { + compatVersion := func(major, minor uint8) func() *base.ClusterCompatVersion { + return func() *base.ClusterCompatVersion { + v := base.NewClusterCompatVersion(major, minor) + return &v + } + } + nilCompatVersion := func() *base.ClusterCompatVersion { return nil } + + testCases := []struct { + name string + useSystemMeta bool + compatVersion func() *base.ClusterCompatVersion + expectedResult bool + }{ + {name: "opt-out, no compat version", useSystemMeta: false, compatVersion: nil, expectedResult: false}, + {name: "opt-out, compat 4.1", useSystemMeta: false, compatVersion: compatVersion(4, 1), expectedResult: false}, + {name: "opt-in, nil compat func", useSystemMeta: true, compatVersion: nil, expectedResult: false}, + {name: "opt-in, compat func returns nil", useSystemMeta: true, compatVersion: nilCompatVersion, expectedResult: false}, + {name: "opt-in, compat 3.0", useSystemMeta: true, compatVersion: compatVersion(3, 0), expectedResult: false}, + {name: "opt-in, compat 4.0", useSystemMeta: true, compatVersion: compatVersion(4, 0), expectedResult: false}, + {name: "opt-in, compat 4.1", useSystemMeta: true, compatVersion: compatVersion(4, 1), expectedResult: true}, + {name: "opt-in, compat 4.2", useSystemMeta: true, compatVersion: compatVersion(4, 2), expectedResult: true}, + {name: "opt-in, compat 5.0", useSystemMeta: true, compatVersion: compatVersion(5, 0), expectedResult: true}, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + dbCtx := &DatabaseContext{ + Options: DatabaseContextOptions{ + UseSystemMetadataCollection: test.useSystemMeta, + }, + ClusterCompatVersionFunc: test.compatVersion, + } + assert.Equal(t, test.expectedResult, dbCtx.shouldRunMetadataMigration()) + }) + } +} + +// TestArmMetadataMigrationTaskNoCallback verifies that armMetadataMigrationTask returns +// immediately (rather than blocking on its poll ticker) when no ConfigFullyAppliedFunc has +// been set, so a misconfigured database cannot leak a goroutine waiting forever. +func TestArmMetadataMigrationTaskNoCallback(t *testing.T) { + dbCtx := &DatabaseContext{ + Name: "test", + terminator: make(chan bool), + } + // ConfigFullyAppliedFunc is intentionally left nil. + + done := make(chan struct{}) + go func() { + dbCtx.armMetadataMigrationTask(context.Background()) + close(done) + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + assert.Fail(t, "armMetadataMigrationTask did not return when ConfigFullyAppliedFunc was nil") + } +} + +// TestTryStartMetadataMigrationAlreadyRunning verifies that when the migration is already running on +// another node (the heartbeat lock doc exists), tryStartMetadataMigration reports done=true so the +// arm loop stops rather than re-acquiring the lock once that node completes - which would re-run the +// migration once per node in the cluster. +func TestTryStartMetadataMigrationAlreadyRunning(t *testing.T) { + ctx := base.TestCtx(t) + testBucket := base.GetTestBucket(t) + defer testBucket.Close(ctx) + + metadataStore := testBucket.DefaultDataStore(ctx) + metaKeys := base.NewMetadataKeys("test-already-running") + + dbCtx := &DatabaseContext{ + Name: "test", + terminator: make(chan bool), + MetadataStore: metadataStore, + MetadataKeys: metaKeys, + } + dbCtx.MetadataMigrationManager = NewMetadataMigrationManager(dbCtx) + + // Simulate another node holding the migration heartbeat lock, so Start returns + // errBackgroundManagerProcessAlreadyRunning. + heartbeatDocID := dbCtx.MetadataMigrationManager.clusterAwareOptions.HeartbeatDocID() + _, err := metadataStore.WriteCas(ctx, heartbeatDocID, BackgroundManagerHeartbeatExpirySecs, 0, []byte("{}"), sgbucket.Raw) + require.NoError(t, err) + + // Config is fully applied so the attempt proceeds to Start. + dbCtx.ConfigFullyAppliedFunc = func(ctx context.Context) (bool, []string, error) { + return true, nil, nil + } + + // done=true tells the arm loop to stop polling rather than retry and re-run later. + require.True(t, dbCtx.tryStartMetadataMigration(ctx)) + + // The local manager must not have started: it should still be in its initial (uninitialized) + // run state, having backed off rather than acquiring the lock. + assert.NotEqual(t, BackgroundProcessStateRunning, dbCtx.MetadataMigrationManager.GetRunState()) + + // The other node's heartbeat lock must remain untouched. + _, _, err = metadataStore.GetRaw(ctx, heartbeatDocID) + assert.NoError(t, err, "heartbeat lock doc should still exist - attempt must not have taken it") +} + +// TestTryStartMetadataMigrationConfigNotApplied verifies that while the cluster has not yet +// converged on the config, tryStartMetadataMigration reports done=false so the arm loop keeps +// polling, and does not start the migration. +func TestTryStartMetadataMigrationConfigNotApplied(t *testing.T) { + ctx := base.TestCtx(t) + testBucket := base.GetTestBucket(t) + defer testBucket.Close(ctx) + + dbCtx := &DatabaseContext{ + Name: "test", + terminator: make(chan bool), + MetadataStore: testBucket.DefaultDataStore(ctx), + MetadataKeys: base.NewMetadataKeys("test-not-applied"), + } + dbCtx.MetadataMigrationManager = NewMetadataMigrationManager(dbCtx) + + dbCtx.ConfigFullyAppliedFunc = func(ctx context.Context) (bool, []string, error) { + return false, []string{"node2"}, nil + } + + assert.False(t, dbCtx.tryStartMetadataMigration(ctx)) + assert.NotEqual(t, BackgroundProcessStateRunning, dbCtx.MetadataMigrationManager.GetRunState()) +} diff --git a/db/background_mgr_resync_dcp.go b/db/background_mgr_resync_dcp.go index cbdcda424e..4a096d0744 100644 --- a/db/background_mgr_resync_dcp.go +++ b/db/background_mgr_resync_dcp.go @@ -407,11 +407,7 @@ func (r *ResyncManagerDCP) Run(ctx context.Context, options map[string]any, pers if !ok { base.WarnfCtx(ctx, "Completed resync, but unable to update syncInfo for collection %v (not found)", collectionID) } - var ccv *base.ClusterCompatVersion - if db.Options.ClusterCompatVersion != nil { - ccv = db.Options.ClusterCompatVersion() - } - if err := base.SetSyncInfoMetadataID(ctx, dbc.dataStore, db.DatabaseContext.Options.MetadataID, ccv); err != nil { + if err := base.SetSyncInfoMetadataID(ctx, dbc.dataStore, db.DatabaseContext.Options.MetadataID, db.ClusterCompatVersion()); err != nil { base.WarnfCtx(ctx, "Completed resync, but unable to update syncInfo for collection %v: %v", collectionID, err) } updatedDsNames[base.ScopeAndCollectionName{Scope: dbc.ScopeName, Collection: dbc.Name}] = struct{}{} diff --git a/db/background_mgr_resync_dcp_test.go b/db/background_mgr_resync_dcp_test.go index 25ee1d2db3..9b171ffbba 100644 --- a/db/background_mgr_resync_dcp_test.go +++ b/db/background_mgr_resync_dcp_test.go @@ -939,7 +939,7 @@ func resyncTestModes() []resyncTestCase { } // TestResyncManagerDCPWritesV1SyncInfoAtCcv41 verifies the end-to-end wiring from -// DatabaseContextOptions.ClusterCompatVersion through ResyncManagerDCP.Run into +// DatabaseContext.ClusterCompatVersion through ResyncManagerDCP.Run into // base.SetSyncInfoMetadataID — when ccv>=4.1 the syncInfo doc is written with the V1 // version-byte prefix. regenerateSequences=true is required so the syncInfo update path runs. func TestResyncManagerDCPWritesV1SyncInfoAtCcv41(t *testing.T) { @@ -952,7 +952,7 @@ func TestResyncManagerDCPWritesV1SyncInfoAtCcv41(t *testing.T) { defer db.Close(ctx) ccv := base.NewClusterCompatVersion(4, 1) - db.Options.ClusterCompatVersion = func() *base.ClusterCompatVersion { return &ccv } + db.ClusterCompatVersionFunc = func() *base.ClusterCompatVersion { return &ccv } // Resync only writes syncInfo when MetadataID is non-empty (SetSyncInfoMetadataID is a no-op otherwise). db.Options.MetadataID = "test_resync_md_id" diff --git a/db/database.go b/db/database.go index ba8fe51e78..8552e222b7 100644 --- a/db/database.go +++ b/db/database.go @@ -100,6 +100,9 @@ const ( // completion of all background tasks and background managers before the server is stopped. const BGTCompletionMaxWait = 30 * time.Second +// metadataMigrationArmRetryInterval is the amount of time to wait in between retries for checking whether the metadata migration background task should be started. +const metadataMigrationArmRetryInterval = 10 * time.Second + // Basic description of a database. Shared between all Database objects on the same database. // This object is thread-safe so it can be shared between HTTP handlers. type DatabaseContext struct { @@ -130,40 +133,43 @@ type DatabaseContext struct { AttachmentCompactionManager *BackgroundManager AttachmentMigrationManager *BackgroundManager AsyncIndexInitManager *BackgroundManager + MetadataMigrationManager *BackgroundManager OIDCProviders auth.OIDCProviderMap // OIDC clients LocalJWTProviders auth.LocalJWTProviderMap ServerUUID string // UUID of the server, if available - DbStats *base.DbStats // stats that correspond to this database context - CompactState uint32 // Status of database compaction - terminator chan bool // Signal termination of background goroutines - CancelContext context.Context // Cancelled when the database is closed - used to notify associated processes (e.g. blipContext) - cancelContextFunc context.CancelCauseFunc // Cancel function for cancelContext - backgroundTasks []BackgroundTask // List of background tasks that are initiated. - activeChannels *channels.ActiveChannels // Tracks active replications by channel - CfgSG cbgt.Cfg // Sync Gateway cluster shared config - SGReplicateMgr *sgReplicateManager // Manages interactions with sg-replicate replications - Heartbeater base.Heartbeater // Node heartbeater for SG cluster awareness - ServeInsecureAttachmentTypes bool // Attachment content type will bypass the content-disposition handling, default false - NoX509HTTPClient *http.Client // A HTTP Client from gocb to use the management endpoints - ServerContextHasStarted chan struct{} // Closed via PostStartup once the server has fully started - UserFunctionTimeout time.Duration // Default timeout for N1QL & JavaScript queries. (Applies to REST and BLIP requests.) - Scopes map[string]Scope // A map keyed by scope name containing a set of scopes/collections. Nil if running with only _default._default - CollectionByID map[uint32]*DatabaseCollection // A map keyed by collection ID to Collection - CollectionNames map[string]map[string]struct{} // Map of scope, collection names - MetadataKeys *base.MetadataKeys // Factory to generate metadata document keys - RequireResync base.ScopeAndCollectionNames // Collections requiring resync before database can go online - RequireAttachmentMigration base.ScopeAndCollectionNames // Collections that require the attachment migration background task to run against - CORS *auth.CORSConfig // CORS configuration - EnableMou bool // Write _mou xattr when performing metadata-only update. Set based on bucket capability on connect - WasInitializedSynchronously bool // true if the database was initialized synchronously - BroadcastSlowMode atomic.Bool // bool to indicate if a slower ticker value should be used to notify changes feeds of changes - DatabaseStartupError *DatabaseError // Error that occurred during database online processes startup - CachedPurgeInterval atomic.Pointer[time.Duration] // If set, the cached value of the purge interval to avoid repeated lookups - CachedVersionPruningWindow atomic.Pointer[time.Duration] // If set, the cached value of the version pruning window to avoid repeated lookups - CachedCCVStartingCas *base.VBucketCAS // If set, the cached value of the CCV starting CAS value to avoid repeated lookups - CachedCCVEnabled atomic.Bool // If set, the cached value of the CCV Enabled flag (this is not expected to transition from true->false, but could go false->true) - numVBuckets uint16 // Number of vbuckets in the bucket + DbStats *base.DbStats // stats that correspond to this database context + CompactState uint32 // Status of database compaction + terminator chan bool // Signal termination of background goroutines + CancelContext context.Context // Cancelled when the database is closed - used to notify associated processes (e.g. blipContext) + cancelContextFunc context.CancelCauseFunc // Cancel function for cancelContext + backgroundTasks []BackgroundTask // List of background tasks that are initiated. + activeChannels *channels.ActiveChannels // Tracks active replications by channel + CfgSG cbgt.Cfg // Sync Gateway cluster shared config + SGReplicateMgr *sgReplicateManager // Manages interactions with sg-replicate replications + Heartbeater base.Heartbeater // Node heartbeater for SG cluster awareness + ServeInsecureAttachmentTypes bool // Attachment content type will bypass the content-disposition handling, default false + NoX509HTTPClient *http.Client // A HTTP Client from gocb to use the management endpoints + ServerContextHasStarted chan struct{} // Closed via PostStartup once the server has fully started + ConfigFullyAppliedFunc func(ctx context.Context) (bool, []string, error) // Returns (applied, missingNodeUIDs, err) for the current db config version + ClusterCompatVersionFunc func() *base.ClusterCompatVersion // Resolves the current cluster-wide minimum SG version, or nil if not yet known. Re-resolved at call time since CCV changes during rolling upgrades. + UserFunctionTimeout time.Duration // Default timeout for N1QL & JavaScript queries. (Applies to REST and BLIP requests.) + Scopes map[string]Scope // A map keyed by scope name containing a set of scopes/collections. Nil if running with only _default._default + CollectionByID map[uint32]*DatabaseCollection // A map keyed by collection ID to Collection + CollectionNames map[string]map[string]struct{} // Map of scope, collection names + MetadataKeys *base.MetadataKeys // Factory to generate metadata document keys + RequireResync base.ScopeAndCollectionNames // Collections requiring resync before database can go online + RequireAttachmentMigration base.ScopeAndCollectionNames // Collections that require the attachment migration background task to run against + CORS *auth.CORSConfig // CORS configuration + EnableMou bool // Write _mou xattr when performing metadata-only update. Set based on bucket capability on connect + WasInitializedSynchronously bool // true if the database was initialized synchronously + BroadcastSlowMode atomic.Bool // bool to indicate if a slower ticker value should be used to notify changes feeds of changes + DatabaseStartupError *DatabaseError // Error that occurred during database online processes startup + CachedPurgeInterval atomic.Pointer[time.Duration] // If set, the cached value of the purge interval to avoid repeated lookups + CachedVersionPruningWindow atomic.Pointer[time.Duration] // If set, the cached value of the version pruning window to avoid repeated lookups + CachedCCVStartingCas *base.VBucketCAS // If set, the cached value of the CCV starting CAS value to avoid repeated lookups + CachedCCVEnabled atomic.Bool // If set, the cached value of the CCV Enabled flag (this is not expected to transition from true->false, but could go false->true) + numVBuckets uint16 // Number of vbuckets in the bucket SameSiteCookieMode http.SameSite DBStateManager *DatabaseStateMgr // Manager used to manage the state of processes across nodes @@ -210,17 +216,17 @@ type DatabaseContextOptions struct { BlipStatsReportingInterval int64 // interval to report blip stats in milliseconds ChangesRequestPlus bool // Sets the default value for request_plus, for non-continuous changes feeds ConfigPrincipals *ConfigPrincipals - TestPurgeIntervalOverride *time.Duration // If set, use this value for db.GetMetadataPurgeInterval - test seam to force specific purge interval for tests - TestVersionPruningWindowOverride *time.Duration // If set, use this value for db.GetVersionPruningWindow - test seam to force specific pruning window for tests - LoggingConfig *base.DbLogConfig // Per-database log configuration - MaxConcurrentChangesBatches *int // Maximum number of changes batches to process concurrently per replication - MaxConcurrentRevs *int // Maximum number of revs to process concurrently per replication - NumIndexReplicas uint // Number of replicas for GSI indexes - NumIndexPartitions *uint32 // Number of partitions for GSI indexes, if not set will default to 1 - ImportVersion uint64 // Version included in import DCP checkpoints, incremented when collections added to db - DisablePublicAllDocs bool // Disable public access to the _all_docs endpoint for this database - StoreLegacyRevTreeData *bool // Whether to store additional data for legacy rev tree support in delta sync and replication backup revs - ClusterCompatVersion func() *base.ClusterCompatVersion // ClusterCompatVersion returns the current cluster-wide minimum SG version, or nil if not yet known. + TestPurgeIntervalOverride *time.Duration // If set, use this value for db.GetMetadataPurgeInterval - test seam to force specific purge interval for tests + TestVersionPruningWindowOverride *time.Duration // If set, use this value for db.GetVersionPruningWindow - test seam to force specific pruning window for tests + LoggingConfig *base.DbLogConfig // Per-database log configuration + MaxConcurrentChangesBatches *int // Maximum number of changes batches to process concurrently per replication + MaxConcurrentRevs *int // Maximum number of revs to process concurrently per replication + NumIndexReplicas uint // Number of replicas for GSI indexes + NumIndexPartitions *uint32 // Number of partitions for GSI indexes, if not set will default to 1 + ImportVersion uint64 // Version included in import DCP checkpoints, incremented when collections added to db + DisablePublicAllDocs bool // Disable public access to the _all_docs endpoint for this database + StoreLegacyRevTreeData *bool // Whether to store additional data for legacy rev tree support in delta sync and replication backup revs + UseSystemMetadataCollection bool // Whether to use a system collection for SG metadata storage } type ConfigPrincipals struct { @@ -618,6 +624,86 @@ func NewDatabaseContext(ctx context.Context, dbName string, bucket base.Bucket, return dbContext, nil } +// shouldRunMetadataMigration checks whether we should start metadata migration on db startup. Will not start the +// migration unless all of the following is true: +// - The option to use a system metadata collection is enabled on db context +// - The cluster compatibility version is at least 4.1 +func (context *DatabaseContext) shouldRunMetadataMigration() bool { + if !context.Options.UseSystemMetadataCollection { + return false + } + return context.ClusterCompatVersion().AtLeast(4, 1) +} + +// ClusterCompatVersion returns the current cluster-wide minimum SG version, or nil if it is not +// yet known (no ClusterCompatVersionFunc wired, or the cluster compat manager has not computed a +// version). The result is re-resolved on each call because CCV changes as peers join/leave during +// a rolling upgrade. +func (context *DatabaseContext) ClusterCompatVersion() *base.ClusterCompatVersion { + if context.ClusterCompatVersionFunc == nil { + return nil + } + return context.ClusterCompatVersionFunc() +} + +// armMetadataMigrationTask polls until all nodes have applied the db config that enables the +// system metadata collection, then starts the metadata migration background task. This +// prevents migration from running while some nodes are still writing metadata to the old location. +// The first attempt happens immediately so an already-converged cluster starts without delay. +func (dbCtx *DatabaseContext) armMetadataMigrationTask(ctx context.Context) { + if dbCtx.ConfigFullyAppliedFunc == nil { + base.WarnfCtx(ctx, "No ConfigFullyAppliedFunc set, cannot arm metadata migration for database %s", base.MD(dbCtx.Name)) + return + } + + ticker := time.NewTicker(metadataMigrationArmRetryInterval) + defer ticker.Stop() + + for { + if dbCtx.tryStartMetadataMigration(ctx) { + return + } + select { + case <-ctx.Done(): + base.DebugfCtx(ctx, base.KeyAll, "Context cancelled, stopping metadata migration arm for database %s", base.MD(dbCtx.Name)) + return + case <-dbCtx.terminator: + base.DebugfCtx(ctx, base.KeyAll, "Database closing, stopping metadata migration arm for database %s", base.MD(dbCtx.Name)) + return + case <-ticker.C: + } + } +} + +// tryStartMetadataMigration makes a single attempt to start the metadata migration. It returns true +// when arming is complete and the caller should stop polling - either because this node started the +// migration, or because another node already owns the run. It returns false (keep polling) when the +// cluster has not yet converged on the config or a transient error occurred. +func (dbCtx *DatabaseContext) tryStartMetadataMigration(ctx context.Context) bool { + applied, missing, err := dbCtx.ConfigFullyAppliedFunc(ctx) + if err != nil { + base.WarnfCtx(ctx, "No eligible nodes for database %s: %v", base.MD(dbCtx.Name), err) + return false + } + if !applied { + base.DebugfCtx(ctx, base.KeyAll, "Config not yet fully applied across cluster for database %s, waiting on nodes: %v", base.MD(dbCtx.Name), missing) + return false + } + base.InfofCtx(ctx, base.KeyAll, "Config fully applied across cluster for database %s, starting metadata migration", base.MD(dbCtx.Name)) + if err := dbCtx.MetadataMigrationManager.Start(ctx, nil); err != nil { + // Another node already holds the migration heartbeat lock, so it owns this run. Stop arming + // to avoid re-acquiring the lock and re-running the migration once that node completes and + // releases it. + if errors.Is(err, errBackgroundManagerProcessAlreadyRunning) { + base.InfofCtx(ctx, base.KeyAll, "Metadata migration already running on another node for database %s, stopping arm", base.MD(dbCtx.Name)) + return true + } + base.WarnfCtx(ctx, "Failed to start metadata migration for database %s: %v", base.MD(dbCtx.Name), err) + return false + } + return true +} + func (context *DatabaseContext) GetOIDCProvider(providerName string) (*auth.OIDCProvider, error) { // If providerName isn't specified, check whether there's a default provider @@ -694,6 +780,7 @@ func (dbCtx *DatabaseContext) stopBackgroundManagers(ctx context.Context) (stopp dbCtx.TombstoneCompactionManager, dbCtx.AsyncIndexInitManager, dbCtx.AttachmentMigrationManager, + dbCtx.MetadataMigrationManager, } { if manager != nil && !isBackgroundManagerStopped(manager.GetRunState()) && manager.Stop(ctx) == nil { stopped = append(stopped, manager) @@ -2429,6 +2516,11 @@ func (db *DatabaseContext) StartOnlineProcesses(ctx context.Context) (returnedEr base.DebugfCtx(ctx, base.KeyAll, "Migrating attachment metadata automatically to Sync Gateway 4.0+ for collections %v", cols) } + db.MetadataMigrationManager = NewMetadataMigrationManager(db) + if db.shouldRunMetadataMigration() { + go db.armMetadataMigrationTask(ctx) + } + if err := base.RequireNoBucketTTL(ctx, db.Bucket); err != nil { return err } diff --git a/rest/admin_api.go b/rest/admin_api.go index 4b91dbe96b..3a42585cd8 100644 --- a/rest/admin_api.go +++ b/rest/admin_api.go @@ -1686,10 +1686,8 @@ func (h *handler) handleGetStatus() error { status.Version = base.LongVersionString status.Vendor.Version = base.ProductAPIVersion status.NodeUID = h.server.NodeUID - if h.server.ClusterCompat != nil { - if v := h.server.ClusterCompat.ClusterCompatVersion(); v != nil { - status.ClusterCompatVersion = v.String() - } + if v := h.server.ClusterCompatVersion(); v != nil { + status.ClusterCompatVersion = v.String() } } diff --git a/rest/api.go b/rest/api.go index bf36c09db0..88b38b379f 100644 --- a/rest/api.go +++ b/rest/api.go @@ -70,10 +70,8 @@ func (h *handler) handleRoot() error { resp.Version = base.LongVersionString resp.Vendor.Version = base.ProductAPIVersion resp.NodeUID = h.server.NodeUID - if h.server.ClusterCompat != nil { - if v := h.server.ClusterCompat.ClusterCompatVersion(); v != nil { - resp.ClusterCompatVersion = v.String() - } + if v := h.server.ClusterCompatVersion(); v != nil { + resp.ClusterCompatVersion = v.String() } } diff --git a/rest/cluster_compat.go b/rest/cluster_compat.go index 714fe55845..88bdecaf09 100644 --- a/rest/cluster_compat.go +++ b/rest/cluster_compat.go @@ -365,11 +365,7 @@ func (m *clusterCompatManager) ClusterCompatVersion() *base.ClusterCompatVersion // ClusterIsAtLeast returns true if the cluster compat version is at least the given major.minor. // Returns false if no version has been computed yet (conservative — don't advance until we know). func (m *clusterCompatManager) ClusterIsAtLeast(major, minor uint8) bool { - v := m.getCachedVersion() - if v == nil { - return false - } - return v.AtLeast(major, minor) + return m.getCachedVersion().AtLeast(major, minor) } // NodeVersions returns the cluster compat version of each node in the cluster, keyed by node UID. diff --git a/rest/cluster_compat_test.go b/rest/cluster_compat_test.go index 566196a6be..327771803e 100644 --- a/rest/cluster_compat_test.go +++ b/rest/cluster_compat_test.go @@ -1606,8 +1606,8 @@ func TestSyncInfoUpgradeGate(t *testing.T) { require.NotNil(t, got) require.Equal(t, base.NewClusterCompatVersion(4, 0), *got, "mixed-version min should be 4.0") - // Resolve the production-wired closure and feed its result to SetSyncInfoMetadataID - ccv := dbCtx.Options.ClusterCompatVersion() + // Resolve the production-wired accessor and feed its result to SetSyncInfoMetadataID + ccv := dbCtx.ClusterCompatVersion() require.NoError(t, base.SetSyncInfoMetadataID(ctx, ds, metadataID, ccv)) raw, _, err := ds.GetRaw(ctx, base.SGSyncInfo) require.NoError(t, err) @@ -1622,9 +1622,9 @@ func TestSyncInfoUpgradeGate(t *testing.T) { require.NotNil(t, got) require.True(t, got.AtLeast(4, 1), "after 4.0 peer leaves, ccv should be >= 4.1; got %v", got) - // Same closure, called again — must resolve to the new value at call time, not the value + // Same accessor, called again — must resolve to the new value at call time, not the value // captured during phase 1. - ccv = dbCtx.Options.ClusterCompatVersion() + ccv = dbCtx.ClusterCompatVersion() require.NoError(t, base.SetSyncInfoMetadataID(ctx, ds, metadataID, ccv)) raw, _, err = ds.GetRaw(ctx, base.SGSyncInfo) require.NoError(t, err) diff --git a/rest/config.go b/rest/config.go index 6d478d5d61..4933d554c7 100644 --- a/rest/config.go +++ b/rest/config.go @@ -147,60 +147,61 @@ func (bucketConfig *BucketConfig) GetCredentials() (username string, password st // DbConfig defines a database configuration used in a config file or the REST API. type DbConfig struct { BucketConfig - Scopes ScopesConfig `json:"scopes,omitempty"` // Scopes and collection specific config - Name string `json:"name,omitempty"` // Database name in REST API (stored as key in JSON) - Sync *string `json:"sync,omitempty"` // The sync function applied to write operations in the _default scope and collection - Users map[string]*auth.PrincipalConfig `json:"users,omitempty"` // Initial user accounts - Roles map[string]*auth.PrincipalConfig `json:"roles,omitempty"` // Initial roles - RevsLimit *uint32 `json:"revs_limit,omitempty"` // Max depth a document's revision tree can grow to - AutoImport any `json:"import_docs,omitempty"` // Whether to automatically import Couchbase Server docs into SG. Xattrs must be enabled. true or "continuous" both enable this. - ImportPartitions *uint16 `json:"import_partitions,omitempty"` // Number of partitions for import sharding. Impacts the total DCP concurrency for import - ImportFilter *string `json:"import_filter,omitempty"` // The import filter applied to import operations in the _default scope and collection - ImportBackupOldRev *bool `json:"import_backup_old_rev,omitempty"` // Whether import should attempt to create a temporary backup of the previous revision body, when available. - EventHandlers *EventHandlerConfig `json:"event_handlers,omitempty"` // Event handlers (webhook) - FeedType string `json:"feed_type,omitempty"` // Feed type - "DCP" only, "TAP" is ignored - AllowEmptyPassword *bool `json:"allow_empty_password,omitempty"` // Allow empty passwords? Defaults to false - CacheConfig *CacheConfig `json:"cache,omitempty"` // Cache settings - DeprecatedRevCacheSize *uint32 `json:"rev_cache_size,omitempty"` // Maximum number of revisions to store in the revision cache (deprecated, CBG-356) - StartOffline *bool `json:"offline,omitempty"` // start the DB in the offline state, defaults to false - Unsupported *db.UnsupportedOptions `json:"unsupported,omitempty"` // Config for unsupported features - OIDCConfig *auth.OIDCOptions `json:"oidc,omitempty"` // Config properties for OpenID Connect authentication - LocalJWTConfig auth.LocalJWTConfig `json:"local_jwt,omitempty"` - OldRevExpirySeconds *uint32 `json:"old_rev_expiry_seconds,omitempty"` // The number of seconds before old revs are removed from CBS bucket - ViewQueryTimeoutSecs *uint32 `json:"view_query_timeout_secs,omitempty"` // The view query timeout in seconds - LocalDocExpirySecs *uint32 `json:"local_doc_expiry_secs,omitempty"` // The _local doc expiry time in seconds - EnableXattrs *bool `json:"enable_shared_bucket_access,omitempty"` // Whether to use extended attributes to store _sync metadata - SecureCookieOverride *bool `json:"session_cookie_secure,omitempty"` // Override cookie secure flag - SessionCookieName string `json:"session_cookie_name,omitempty"` // Custom per-database session cookie name - SessionCookieHTTPOnly *bool `json:"session_cookie_http_only,omitempty"` // HTTP only cookies - AllowConflicts *bool `json:"allow_conflicts,omitempty"` // Deprecated: False forbids creating conflicts - NumIndexReplicas *uint `json:"num_index_replicas,omitempty"` // Number of GSI index replicas used for core indexes, deprecated for IndexConfig.NumReplicas - Index *IndexConfig `json:"index,omitempty"` // Index options - UseViews *bool `json:"use_views,omitempty"` // Force use of views instead of GSI - SendWWWAuthenticateHeader *bool `json:"send_www_authenticate_header,omitempty"` // If false, disables setting of 'WWW-Authenticate' header in 401 responses. Implicitly false if disable_password_auth is true. - DisablePasswordAuth *bool `json:"disable_password_auth,omitempty"` // If true, disables user/pass authentication, only permitting OIDC or guest access - BucketOpTimeoutMs *uint32 `json:"bucket_op_timeout_ms,omitempty"` // How long bucket ops should block returning "operation timed out". If nil, uses GoCB default. GoCB buckets only. - SlowQueryWarningThresholdMs *uint32 `json:"slow_query_warning_threshold,omitempty"` // Log warnings if N1QL queries take this many ms - DeltaSync *DeltaSyncConfig `json:"delta_sync,omitempty"` // Config for delta sync - StoreLegacyRevTreeData *bool `json:"store_legacy_revtree_data,omitempty"` // Whether to store legacy revision tree pointer data to support older clients using RevTree IDs - CompactIntervalDays *float32 `json:"compact_interval_days,omitempty"` // Interval between scheduled compaction runs (in days) - 0 means don't run - SGReplicateEnabled *bool `json:"sgreplicate_enabled,omitempty"` // When false, node will not be assigned replications - SGReplicateWebsocketPingInterval *int `json:"sgreplicate_websocket_heartbeat_secs,omitempty"` // If set, uses this duration as a custom heartbeat interval for websocket ping frames - Replications map[string]*db.ReplicationConfig `json:"replications,omitempty"` // sg-replicate replication definitions - ServeInsecureAttachmentTypes *bool `json:"serve_insecure_attachment_types,omitempty"` // Attachment content type will bypass the content-disposition handling, default false - QueryPaginationLimit *int `json:"query_pagination_limit,omitempty"` // Query limit to be used during pagination of large queries - UserXattrKey *string `json:"user_xattr_key,omitempty"` // Key of user xattr that will be accessible from the Sync Function. If empty or nil the feature will be disabled. - ClientPartitionWindowSecs *int `json:"client_partition_window_secs,omitempty"` // How long clients can remain offline for without losing replication metadata. Default 30 days (in seconds) - Guest *auth.PrincipalConfig `json:"guest,omitempty"` // Guest user settings - JavascriptTimeoutSecs *uint32 `json:"javascript_timeout_secs,omitempty"` // The amount of seconds a Javascript function can run for. Set to 0 for no timeout. - UserFunctions *functions.FunctionsConfig `json:"functions,omitempty"` // Named JS fns for clients to call - Suspendable *bool `json:"suspendable,omitempty"` // Allow the database to be suspended - ChangesRequestPlus *bool `json:"changes_request_plus,omitempty"` // If set, is used as the default value of request_plus for non-continuous replications - CORS *auth.CORSConfig `json:"cors,omitempty"` // Per-database CORS config - Logging *DbLoggingConfig `json:"logging,omitempty"` // Per-database Logging config - UpdatedAt *time.Time `json:"updated_at,omitempty"` // Time at which the database config was last updated - CreatedAt *time.Time `json:"created_at,omitempty"` // Time at which the database config was created - DisablePublicAllDocs *bool `json:"disable_public_all_docs,omitempty"` // Whether to disable public access to the _all_docs endpoint for this database + Scopes ScopesConfig `json:"scopes,omitempty"` // Scopes and collection specific config + Name string `json:"name,omitempty"` // Database name in REST API (stored as key in JSON) + Sync *string `json:"sync,omitempty"` // The sync function applied to write operations in the _default scope and collection + Users map[string]*auth.PrincipalConfig `json:"users,omitempty"` // Initial user accounts + Roles map[string]*auth.PrincipalConfig `json:"roles,omitempty"` // Initial roles + RevsLimit *uint32 `json:"revs_limit,omitempty"` // Max depth a document's revision tree can grow to + AutoImport any `json:"import_docs,omitempty"` // Whether to automatically import Couchbase Server docs into SG. Xattrs must be enabled. true or "continuous" both enable this. + ImportPartitions *uint16 `json:"import_partitions,omitempty"` // Number of partitions for import sharding. Impacts the total DCP concurrency for import + ImportFilter *string `json:"import_filter,omitempty"` // The import filter applied to import operations in the _default scope and collection + ImportBackupOldRev *bool `json:"import_backup_old_rev,omitempty"` // Whether import should attempt to create a temporary backup of the previous revision body, when available. + EventHandlers *EventHandlerConfig `json:"event_handlers,omitempty"` // Event handlers (webhook) + FeedType string `json:"feed_type,omitempty"` // Feed type - "DCP" only, "TAP" is ignored + AllowEmptyPassword *bool `json:"allow_empty_password,omitempty"` // Allow empty passwords? Defaults to false + CacheConfig *CacheConfig `json:"cache,omitempty"` // Cache settings + DeprecatedRevCacheSize *uint32 `json:"rev_cache_size,omitempty"` // Maximum number of revisions to store in the revision cache (deprecated, CBG-356) + StartOffline *bool `json:"offline,omitempty"` // start the DB in the offline state, defaults to false + Unsupported *db.UnsupportedOptions `json:"unsupported,omitempty"` // Config for unsupported features + OIDCConfig *auth.OIDCOptions `json:"oidc,omitempty"` // Config properties for OpenID Connect authentication + LocalJWTConfig auth.LocalJWTConfig `json:"local_jwt,omitempty"` + OldRevExpirySeconds *uint32 `json:"old_rev_expiry_seconds,omitempty"` // The number of seconds before old revs are removed from CBS bucket + ViewQueryTimeoutSecs *uint32 `json:"view_query_timeout_secs,omitempty"` // The view query timeout in seconds + LocalDocExpirySecs *uint32 `json:"local_doc_expiry_secs,omitempty"` // The _local doc expiry time in seconds + EnableXattrs *bool `json:"enable_shared_bucket_access,omitempty"` // Whether to use extended attributes to store _sync metadata + SecureCookieOverride *bool `json:"session_cookie_secure,omitempty"` // Override cookie secure flag + SessionCookieName string `json:"session_cookie_name,omitempty"` // Custom per-database session cookie name + SessionCookieHTTPOnly *bool `json:"session_cookie_http_only,omitempty"` // HTTP only cookies + AllowConflicts *bool `json:"allow_conflicts,omitempty"` // Deprecated: False forbids creating conflicts + NumIndexReplicas *uint `json:"num_index_replicas,omitempty"` // Number of GSI index replicas used for core indexes, deprecated for IndexConfig.NumReplicas + Index *IndexConfig `json:"index,omitempty"` // Index options + UseViews *bool `json:"use_views,omitempty"` // Force use of views instead of GSI + SendWWWAuthenticateHeader *bool `json:"send_www_authenticate_header,omitempty"` // If false, disables setting of 'WWW-Authenticate' header in 401 responses. Implicitly false if disable_password_auth is true. + DisablePasswordAuth *bool `json:"disable_password_auth,omitempty"` // If true, disables user/pass authentication, only permitting OIDC or guest access + BucketOpTimeoutMs *uint32 `json:"bucket_op_timeout_ms,omitempty"` // How long bucket ops should block returning "operation timed out". If nil, uses GoCB default. GoCB buckets only. + SlowQueryWarningThresholdMs *uint32 `json:"slow_query_warning_threshold,omitempty"` // Log warnings if N1QL queries take this many ms + DeltaSync *DeltaSyncConfig `json:"delta_sync,omitempty"` // Config for delta sync + StoreLegacyRevTreeData *bool `json:"store_legacy_revtree_data,omitempty"` // Whether to store legacy revision tree pointer data to support older clients using RevTree IDs + CompactIntervalDays *float32 `json:"compact_interval_days,omitempty"` // Interval between scheduled compaction runs (in days) - 0 means don't run + SGReplicateEnabled *bool `json:"sgreplicate_enabled,omitempty"` // When false, node will not be assigned replications + SGReplicateWebsocketPingInterval *int `json:"sgreplicate_websocket_heartbeat_secs,omitempty"` // If set, uses this duration as a custom heartbeat interval for websocket ping frames + Replications map[string]*db.ReplicationConfig `json:"replications,omitempty"` // sg-replicate replication definitions + ServeInsecureAttachmentTypes *bool `json:"serve_insecure_attachment_types,omitempty"` // Attachment content type will bypass the content-disposition handling, default false + QueryPaginationLimit *int `json:"query_pagination_limit,omitempty"` // Query limit to be used during pagination of large queries + UserXattrKey *string `json:"user_xattr_key,omitempty"` // Key of user xattr that will be accessible from the Sync Function. If empty or nil the feature will be disabled. + ClientPartitionWindowSecs *int `json:"client_partition_window_secs,omitempty"` // How long clients can remain offline for without losing replication metadata. Default 30 days (in seconds) + Guest *auth.PrincipalConfig `json:"guest,omitempty"` // Guest user settings + JavascriptTimeoutSecs *uint32 `json:"javascript_timeout_secs,omitempty"` // The amount of seconds a Javascript function can run for. Set to 0 for no timeout. + UserFunctions *functions.FunctionsConfig `json:"functions,omitempty"` // Named JS fns for clients to call + Suspendable *bool `json:"suspendable,omitempty"` // Allow the database to be suspended + ChangesRequestPlus *bool `json:"changes_request_plus,omitempty"` // If set, is used as the default value of request_plus for non-continuous replications + CORS *auth.CORSConfig `json:"cors,omitempty"` // Per-database CORS config + Logging *DbLoggingConfig `json:"logging,omitempty"` // Per-database Logging config + UpdatedAt *time.Time `json:"updated_at,omitempty"` // Time at which the database config was last updated + CreatedAt *time.Time `json:"created_at,omitempty"` // Time at which the database config was created + DisablePublicAllDocs *bool `json:"disable_public_all_docs,omitempty"` // Whether to disable public access to the _all_docs endpoint for this database + UseSystemMobileMetadataCollection *bool `json:"use_system_metadata_collection,omitempty"` // Whether to use the system metadata collection for storing Sync Gateway metadata documents. } type ScopesConfig map[string]ScopeConfig @@ -731,27 +732,34 @@ func (dbConfig *DbConfig) validateConfigUpdate(ctx context.Context, old DbConfig // validateChanges compares the current DbConfig with the "old" config, and returns an error if any disallowed changes // are attempted. func (dbConfig *DbConfig) validateChanges(ctx context.Context, old DbConfig) error { + var multiError *base.MultiError + // allow switching from implicit `_default` to explicit `_default` scope _, newIsDefaultScope := dbConfig.Scopes[base.DefaultScope] - if old.Scopes == nil && len(dbConfig.Scopes) == 1 && newIsDefaultScope { - return nil - } - // early exit - if len(dbConfig.Scopes) != len(old.Scopes) { - return fmt.Errorf("cannot change scopes after database creation") - } - newScopes := make(base.Set, len(dbConfig.Scopes)) - oldScopes := make(base.Set, len(old.Scopes)) - for scopeName := range dbConfig.Scopes { - newScopes.Add(scopeName) - } - for scopeName := range old.Scopes { - oldScopes.Add(scopeName) + if !(old.Scopes == nil && len(dbConfig.Scopes) == 1 && newIsDefaultScope) { + if len(dbConfig.Scopes) != len(old.Scopes) { + multiError = multiError.Append(fmt.Errorf("cannot change scopes after database creation")) + } else { + newScopes := make(base.Set, len(dbConfig.Scopes)) + oldScopes := make(base.Set, len(old.Scopes)) + for scopeName := range dbConfig.Scopes { + newScopes.Add(scopeName) + } + for scopeName := range old.Scopes { + oldScopes.Add(scopeName) + } + if !newScopes.Equals(oldScopes) { + multiError = multiError.Append(fmt.Errorf("cannot change scopes after database creation")) + } + } } - if !newScopes.Equals(oldScopes) { - return fmt.Errorf("cannot change scopes after database creation") + + if base.ValDefault(old.UseSystemMobileMetadataCollection, false) && + !base.ValDefault(dbConfig.UseSystemMobileMetadataCollection, false) { + multiError = multiError.Append(fmt.Errorf("use_system_metadata_collection cannot be disabled once enabled")) } - return nil + + return multiError.ErrorOrNil() } // validate checks the DbConfig for any invalid or unsupported values and return a http error. If validateReplications is true, return an error if any replications are not valid. Otherwise issue a warning. diff --git a/rest/config_database.go b/rest/config_database.go index 2c2995fef3..4f7bb3c3e3 100644 --- a/rest/config_database.go +++ b/rest/config_database.go @@ -176,21 +176,22 @@ func DefaultDbConfig(sc *StartupConfig, useXattrs bool) *DbConfig { Enabled: base.Ptr(db.DefaultDeltaSyncEnabled), RevMaxAgeSeconds: base.Ptr(db.DefaultDeltaSyncRevMaxAge), }, - StoreLegacyRevTreeData: base.Ptr(db.DefaultStoreLegacyRevTreeData), - CompactIntervalDays: base.Ptr(float32(db.DefaultCompactInterval.Hours() / 24)), - SGReplicateEnabled: base.Ptr(db.DefaultSGReplicateEnabled), - SGReplicateWebsocketPingInterval: base.Ptr(int(db.DefaultSGReplicateWebsocketPingInterval.Seconds())), - Replications: nil, - ServeInsecureAttachmentTypes: base.Ptr(false), - QueryPaginationLimit: base.Ptr(db.DefaultQueryPaginationLimit), - UserXattrKey: nil, - ClientPartitionWindowSecs: base.Ptr(int(base.DefaultClientPartitionWindow.Seconds())), - Guest: &auth.PrincipalConfig{Disabled: base.Ptr(true)}, - JavascriptTimeoutSecs: base.Ptr(base.DefaultJavascriptTimeoutSecs), - Suspendable: base.Ptr(sc.IsServerless()), - ChangesRequestPlus: base.Ptr(false), - Logging: DefaultPerDBLogging(sc.Logging), - DisablePublicAllDocs: base.Ptr(false), + StoreLegacyRevTreeData: base.Ptr(db.DefaultStoreLegacyRevTreeData), + CompactIntervalDays: base.Ptr(float32(db.DefaultCompactInterval.Hours() / 24)), + SGReplicateEnabled: base.Ptr(db.DefaultSGReplicateEnabled), + SGReplicateWebsocketPingInterval: base.Ptr(int(db.DefaultSGReplicateWebsocketPingInterval.Seconds())), + Replications: nil, + ServeInsecureAttachmentTypes: base.Ptr(false), + QueryPaginationLimit: base.Ptr(db.DefaultQueryPaginationLimit), + UserXattrKey: nil, + ClientPartitionWindowSecs: base.Ptr(int(base.DefaultClientPartitionWindow.Seconds())), + Guest: &auth.PrincipalConfig{Disabled: base.Ptr(true)}, + JavascriptTimeoutSecs: base.Ptr(base.DefaultJavascriptTimeoutSecs), + Suspendable: base.Ptr(sc.IsServerless()), + ChangesRequestPlus: base.Ptr(false), + Logging: DefaultPerDBLogging(sc.Logging), + DisablePublicAllDocs: base.Ptr(false), + UseSystemMobileMetadataCollection: base.Ptr(DefaultUseSystemMetadataCollection), } if useXattrs { diff --git a/rest/metadatamigrationtest/main_test.go b/rest/metadatamigrationtest/main_test.go new file mode 100644 index 0000000000..2dbc664a57 --- /dev/null +++ b/rest/metadatamigrationtest/main_test.go @@ -0,0 +1,25 @@ +/* +Copyright 2026-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package metadatamigrationtest + +import ( + "context" + "testing" + + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" +) + +func TestMain(m *testing.M) { + ctx := context.Background() + tbpOptions := base.TestBucketPoolOptions{MemWatermarkThresholdMB: 8192} + db.TestBucketPoolWithIndexes(ctx, m, tbpOptions) +} diff --git a/rest/metadatamigrationtest/metadata_migration_test.go b/rest/metadatamigrationtest/metadata_migration_test.go new file mode 100644 index 0000000000..81985381bb --- /dev/null +++ b/rest/metadatamigrationtest/metadata_migration_test.go @@ -0,0 +1,160 @@ +/* +Copyright 2026-Present Couchbase, Inc. + +Use of this software is governed by the Business Source License included in +the file licenses/BSL-Couchbase.txt. As of the Change Date specified in that +file, in accordance with the Business Source License, use of this software will +be governed by the Apache License, Version 2.0, included in the file +licenses/APL2.txt. +*/ + +package metadatamigrationtest + +import ( + "net/http" + "testing" + "time" + + "github.com/couchbase/sync_gateway/base" + "github.com/couchbase/sync_gateway/db" + "github.com/couchbase/sync_gateway/rest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMetadataMigrationNotStartedWithoutOptIn creates a database without setting +// UseSystemMobileMetadataCollection and verifies that the metadata migration +// background task is never started. +func TestMetadataMigrationNotStartedWithoutOptIn(t *testing.T) { + rt := rest.NewRestTesterPersistentConfig(t) + defer rt.Close() + + dbCtx := rt.GetDatabase() + require.NotNil(t, dbCtx.MetadataMigrationManager) + + state := dbCtx.MetadataMigrationManager.GetRunState() + assert.Equal(t, db.BackgroundProcessState(""), state, "metadata migration should not have been started without UseSystemMobileMetadataCollection opt-in") +} + +// TestMetadataMigrationNotStartedWithExplicitFalse creates a database with +// UseSystemMobileMetadataCollection explicitly set to false and verifies the +// metadata migration background task is never started. +func TestMetadataMigrationNotStartedWithExplicitFalse(t *testing.T) { + rt := rest.NewRestTesterPersistentConfigNoDB(t) + defer rt.Close() + + dbConfig := rt.NewDbConfig() + dbConfig.UseSystemMobileMetadataCollection = base.Ptr(false) + resp := rt.CreateDatabase("db", dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + + dbCtx := rt.GetDatabase() + require.NotNil(t, dbCtx.MetadataMigrationManager) + + state := dbCtx.MetadataMigrationManager.GetRunState() + assert.Equal(t, db.BackgroundProcessState(""), state, "metadata migration should not have been started with UseSystemMobileMetadataCollection=false") +} + +// TestMetadataMigrationStartsAfterAllNodesApplyConfig uses two rest testers sharing the same +// bucket to simulate a two-node cluster. It verifies that: +// 1. Migration does not start while node B has not yet picked up the opt-in config. +// 2. Once node B applies the config and the registry reflects this, the migration starts. +func TestMetadataMigrationStartsAfterAllNodesApplyConfig(t *testing.T) { + ctx := base.TestCtx(t) + tb := base.GetTestBucket(t) + defer tb.Close(ctx) + + rtConfig := &rest.RestTesterConfig{ + CustomTestBucket: tb.NoCloseClone(), + PersistentConfig: true, + GroupID: base.Ptr("metadata_migration_cluster"), + } + + rtA := rest.NewRestTester(t, rtConfig) + defer rtA.Close() + rtB := rest.NewRestTester(t, rtConfig) + defer rtB.Close() + + // Create db on node A with migration disabled. + dbConfig := rtA.NewDbConfig() + dbConfig.UseSystemMobileMetadataCollection = base.Ptr(false) + resp := rtA.CreateDatabase("db", dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + + // Node B picks up the db via config polling. + rtB.ServerContext().ForceDbConfigsReload(t, ctx) + + // Flush both nodes' heartbeats to the registry so each node is visible. + rtA.ServerContext().ForceClusterCompatRefresh(t, ctx) + rtB.ServerContext().ForceClusterCompatRefresh(t, ctx) + + // Update db config on node A to enable the system metadata collection. + dbConfig.UseSystemMobileMetadataCollection = base.Ptr(true) + resp = rtA.UpsertDbConfig("db", dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + + // Flush node A's registry entry so its new db version is visible. + rtA.ServerContext().ForceClusterCompatRefresh(t, ctx) + + // Node B has NOT yet picked up the updated config. Assert directly on the gate that + // tryStartMetadataMigration consults: ConfigFullyAppliedFunc must report not-applied (with node + // B outstanding), which is what blocks the migration from starting. This is deterministic, + // unlike observing the run state of the background goroutine. + dbCtxA := rtA.GetDatabase() + require.NotNil(t, dbCtxA.MetadataMigrationManager) + require.NotNil(t, dbCtxA.ConfigFullyAppliedFunc) + applied, missing, err := dbCtxA.ConfigFullyAppliedFunc(ctx) + require.NoError(t, err) + assert.False(t, applied, "config must not be fully applied while node B is still on the old version") + assert.Contains(t, missing, rtB.ServerContext().NodeUID, "node B must be reported as not having applied the new config version") + + // Node B picks up the updated config. + rtB.ServerContext().ForceDbConfigsReload(t, ctx) + + // Flush node B's registry entry so the new db version is visible. + rtB.ServerContext().ForceClusterCompatRefresh(t, ctx) + + // The gate now reports fully applied across the cluster - this is what unblocks the migration. + applied, missing, err = dbCtxA.ConfigFullyAppliedFunc(ctx) + require.NoError(t, err) + assert.True(t, applied, "config should be fully applied once node B has applied the new version") + assert.Empty(t, missing) + + // The armMetadataMigrationTask goroutine on node A polls every 10s. Once the gate is open it + // starts the migration; wait for that to happen. + require.EventuallyWithT(t, func(c *assert.CollectT) { + state := dbCtxA.MetadataMigrationManager.GetRunState() + assert.NotEqual(c, db.BackgroundProcessState(""), state, + "migration should have been started after all nodes applied the config") + }, 30*time.Second, 100*time.Millisecond) +} + +// TestMetadataMigrationOptInIsIrreversibleViaREST verifies that, once a database has opted in +// to the system metadata collection, a config update over the REST API that attempts to disable +// the opt-in (set to false or omit it) is rejected. +func TestMetadataMigrationOptInIsIrreversibleViaREST(t *testing.T) { + rt := rest.NewRestTesterPersistentConfigNoDB(t) + defer rt.Close() + + dbConfig := rt.NewDbConfig() + dbConfig.UseSystemMobileMetadataCollection = base.Ptr(true) + resp := rt.CreateDatabase("db", dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) + + // Attempting to set the opt-in back to false is rejected. + dbConfig.UseSystemMobileMetadataCollection = base.Ptr(false) + resp = rt.ReplaceDbConfig("db", dbConfig) + rest.RequireStatus(t, resp, http.StatusBadRequest) + assert.Contains(t, resp.Body.String(), "use_system_metadata_collection cannot be disabled once enabled") + + // Omitting the field entirely (nil) is also rejected, since PUT replaces the whole config. + dbConfig.UseSystemMobileMetadataCollection = nil + resp = rt.ReplaceDbConfig("db", dbConfig) + rest.RequireStatus(t, resp, http.StatusBadRequest) + assert.Contains(t, resp.Body.String(), "use_system_metadata_collection cannot be disabled once enabled") + + // Keeping the opt-in enabled is permitted. + dbConfig.UseSystemMobileMetadataCollection = base.Ptr(true) + resp = rt.ReplaceDbConfig("db", dbConfig) + rest.RequireStatus(t, resp, http.StatusCreated) +} diff --git a/rest/server_context.go b/rest/server_context.go index 6b3f67f353..8db872ff08 100644 --- a/rest/server_context.go +++ b/rest/server_context.go @@ -331,6 +331,15 @@ func (sc *ServerContext) Close(ctx context.Context) { sc.invalidDatabaseConfigTracking.dbNames = nil } +// ClusterCompatVersion returns the cluster-wide minimum SG version, or nil if the cluster compat +// manager is not running or has not yet computed a version. +func (sc *ServerContext) ClusterCompatVersion() *base.ClusterCompatVersion { + if sc.ClusterCompat == nil { + return nil + } + return sc.ClusterCompat.ClusterCompatVersion() +} + func (sc *ServerContext) stopHTTPServers(ctx context.Context) { sc._httpServersLock.Lock() defer sc._httpServersLock.Unlock() @@ -834,11 +843,7 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config } // Verify whether the collection is associated with a different database's metadataID - if so, add to set requiring resync - var ccv *base.ClusterCompatVersion - if sc.ClusterCompat != nil { - ccv = sc.ClusterCompat.ClusterCompatVersion() - } - resyncRequired, requiresAttachmentMigration, err := base.InitSyncInfo(ctx, dataStore, config.MetadataID, ccv) + resyncRequired, requiresAttachmentMigration, err := base.InitSyncInfo(ctx, dataStore, config.MetadataID, sc.ClusterCompatVersion()) if err != nil { if options.loadFromBucket { sc._handleInvalidDatabaseConfig(ctx, spec.BucketName, config, db.NewDatabaseError(db.DatabaseInitSyncInfoError)) @@ -862,11 +867,7 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config // No explicitly defined scopes means we'll initialize this as a usable default collection, otherwise it's for metadata only if len(config.Scopes) == 0 { scName := base.DefaultScopeAndCollectionName() - var ccv *base.ClusterCompatVersion - if sc.ClusterCompat != nil { - ccv = sc.ClusterCompat.ClusterCompatVersion() - } - resyncRequired, requiresAttachmentMigration, err := base.InitSyncInfo(ctx, ds, config.MetadataID, ccv) + resyncRequired, requiresAttachmentMigration, err := base.InitSyncInfo(ctx, ds, config.MetadataID, sc.ClusterCompatVersion()) if err != nil { if options.loadFromBucket { sc._handleInvalidDatabaseConfig(ctx, spec.BucketName, config, db.NewDatabaseError(db.DatabaseInitSyncInfoError)) @@ -977,16 +978,6 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config // NewMetadataKeys handles the "_default" -> legacy (unprefixed) key mapping, so we can pass config.MetadataID through directly. contextOptions.MetadataID = config.MetadataID - // Stored as a closure so callers re-resolve at call time: cluster compat changes as peers - // join/leave during a rolling upgrade, and long-lived consumers (e.g. background managers - // mid-run) must observe the current value rather than one captured at db construction. - contextOptions.ClusterCompatVersion = func() *base.ClusterCompatVersion { - if sc.ClusterCompat != nil { - return sc.ClusterCompat.ClusterCompatVersion() - } - return nil - } - contextOptions.BlipStatsReportingInterval = defaultBytesStatsReportingInterval.Milliseconds() contextOptions.ImportVersion = config.ImportVersion @@ -1003,6 +994,27 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config dbcontext.RequireResync = collectionsRequiringResync dbcontext.RequireAttachmentMigration = collectionsRequiringAttachmentMigration + // Resolved at call time rather than captured: cluster compat changes as peers join/leave + // during a rolling upgrade, and long-lived consumers (e.g. background managers mid-run) must + // observe the current value rather than one captured at db construction. + dbcontext.ClusterCompatVersionFunc = sc.ClusterCompatVersion + + // Closure that checks whether every node in the cluster has applied the current db config + // version. Used by armMetadataMigrationTask to delay migration until the opt-in flag is + // active everywhere. + if config.Version != "" { + configGroupID := sc.Config.Bootstrap.ConfigGroupID + bucketName := spec.BucketName + cfgVersion := config.Version + dbcontext.ConfigFullyAppliedFunc = func(ctx context.Context) (bool, []string, error) { + registry, err := sc.BootstrapContext.getGatewayRegistry(ctx, bucketName) + if err != nil { + return false, nil, err + } + return registry.IsConfigFullyApplied(ctx, configGroupID, dbName, cfgVersion) + } + } + if config.Unsupported != nil && config.Unsupported.ResyncPartitions != nil && *config.Unsupported.ResyncPartitions > dbcontext.NumVBuckets() { if options.loadFromBucket { sc._handleInvalidDatabaseConfig(ctx, spec.BucketName, config, db.NewDatabaseError(db.DatabaseInvalidResyncPartitions)) @@ -1492,6 +1504,15 @@ func dbcOptionsFromConfig(ctx context.Context, sc *ServerContext, config *DbConf base.WarnfCtx(ctx, `Deprecation notice: setting database configuration option "disable_public_all_docs" to false is deprecated. In the future, public access to the all_docs API will be disabled by default.`) } + useMobileCollection := DefaultUseSystemMetadataCollection + if config.UseSystemMobileMetadataCollection != nil && *config.UseSystemMobileMetadataCollection { + useMobileCollection = true + } else { + if sc.Config.Bootstrap.UseSystemMetadataCollection != nil { + useMobileCollection = *sc.Config.Bootstrap.UseSystemMetadataCollection + } + } + contextOptions := db.DatabaseContextOptions{ CacheOptions: &cacheOptions, RevisionCacheOptions: revCacheOptions, @@ -1529,6 +1550,7 @@ func dbcOptionsFromConfig(ctx context.Context, sc *ServerContext, config *DbConf NumIndexReplicas: config.numIndexReplicas(), DisablePublicAllDocs: disablePublicAllDocs, StoreLegacyRevTreeData: base.Ptr(base.ValDefault(config.StoreLegacyRevTreeData, db.DefaultStoreLegacyRevTreeData)), + UseSystemMetadataCollection: useMobileCollection, } if config.Index != nil && config.Index.NumPartitions != nil { diff --git a/rest/server_context_test.go b/rest/server_context_test.go index c4a9c7e26e..39c5a88088 100644 --- a/rest/server_context_test.go +++ b/rest/server_context_test.go @@ -896,6 +896,158 @@ func TestCompactIntervalFromConfig(t *testing.T) { } } +func TestUseSystemMetadataCollectionConfigResolution(t *testing.T) { + testCases := []struct { + name string + bootstrap *bool // BootstrapConfig.UseSystemMetadataCollection + dbLevel *bool // DbConfig.UseSystemMobileMetadataCollection + expected bool + }{ + { + name: "both nil, default false", + expected: false, + }, + { + name: "bootstrap true, db nil", + bootstrap: base.Ptr(true), + expected: true, + }, + { + name: "bootstrap false, db nil", + bootstrap: base.Ptr(false), + expected: false, + }, + { + name: "bootstrap nil, db true", + dbLevel: base.Ptr(true), + expected: true, + }, + { + name: "bootstrap nil, db false", + dbLevel: base.Ptr(false), + expected: false, + }, + { + name: "bootstrap true, db true", + bootstrap: base.Ptr(true), + dbLevel: base.Ptr(true), + expected: true, + }, + { + name: "bootstrap true, db false — bootstrap wins", + bootstrap: base.Ptr(true), + dbLevel: base.Ptr(false), + expected: true, + }, + { + name: "bootstrap false, db true — db opt-in wins", + bootstrap: base.Ptr(false), + dbLevel: base.Ptr(true), + expected: true, + }, + { + name: "bootstrap false, db false", + bootstrap: base.Ptr(false), + dbLevel: base.Ptr(false), + expected: false, + }, + } + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + ctx := base.TestCtx(t) + startupConfig := &StartupConfig{} + startupConfig.Bootstrap.UseSystemMetadataCollection = test.bootstrap + sc := NewServerContext(ctx, startupConfig, false) + defer sc.Close(ctx) + config := &DbConfig{ + UseSystemMobileMetadataCollection: test.dbLevel, + Unsupported: &db.UnsupportedOptions{}, + } + opts, err := dbcOptionsFromConfig(ctx, sc, config, "fakedb") + require.NoError(t, err) + require.Equal(t, test.expected, opts.UseSystemMetadataCollection) + }) + } +} + +func TestValidateChangesUseSystemMetadataCollection(t *testing.T) { + testCases := []struct { + name string + oldValue *bool + newValue *bool + expectErr bool + }{ + { + name: "nil to nil — no change", + expectErr: false, + }, + { + name: "nil to true — opt in", + newValue: base.Ptr(true), + expectErr: false, + }, + { + name: "nil to false — no-op", + newValue: base.Ptr(false), + expectErr: false, + }, + { + name: "false to true — opt in", + oldValue: base.Ptr(false), + newValue: base.Ptr(true), + expectErr: false, + }, + { + name: "true to true — no change", + oldValue: base.Ptr(true), + newValue: base.Ptr(true), + expectErr: false, + }, + { + name: "true to false — rejected", + oldValue: base.Ptr(true), + newValue: base.Ptr(false), + expectErr: true, + }, + { + name: "true to nil — rejected", + oldValue: base.Ptr(true), + newValue: nil, + expectErr: true, + }, + { + name: "false to false — no change", + oldValue: base.Ptr(false), + newValue: base.Ptr(false), + expectErr: false, + }, + { + name: "false to nil — no-op", + oldValue: base.Ptr(false), + newValue: nil, + expectErr: false, + }, + } + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + ctx := base.TestCtx(t) + oldConfig := DbConfig{ + UseSystemMobileMetadataCollection: test.oldValue, + } + newConfig := DbConfig{ + UseSystemMobileMetadataCollection: test.newValue, + } + err := newConfig.validateChanges(ctx, oldConfig) + if test.expectErr { + require.Error(t, err) + assert.Contains(t, err.Error(), "use_system_metadata_collection cannot be disabled once enabled") + } else { + require.NoError(t, err) + } + }) + } +} + func TestHeapProfileValuesPopulated(t *testing.T) { totalMemory := uint64(float64(getTotalMemory(base.TestCtx(t))) * 0.85) testCases := []struct { diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index 10b6c0e652..766fcd6479 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -2658,6 +2658,18 @@ func (sc *ServerContext) ForceDbConfigsReload(t *testing.T, ctx context.Context) require.NoError(t, err) } +// ForceClusterCompatRefresh forces a cluster compat refresh cycle, flushing pending heartbeats +// and db version records to the registry document. +func (sc *ServerContext) ForceClusterCompatRefresh(t *testing.T, ctx context.Context) { + if sc.ClusterCompat == nil { + return + } + sc.ClusterCompat.mu.Lock() + sc.ClusterCompat.lastRefreshAt = time.Time{} + sc.ClusterCompat.mu.Unlock() + sc.ClusterCompat.Refresh(ctx) +} + // AllInvalidDatabaseNames returns the names of all the databases that have invalid configs. Testing only since this locks the database context. func (sc *ServerContext) AllInvalidDatabaseNames(_ *testing.T) []string { sc.invalidDatabaseConfigTracking.m.RLock()