bundle/direct: add a feature-flag list to the direct deployment state

acceptance/acceptance_test.go

@@ -277,6 +277,13 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 		cli293Path := DownloadCLI(t, buildDir, "0.293.0")
 		t.Setenv("CLI_293", cli293Path)
 		repls.SetPath(cli293Path, "[CLI_293]")
+
+		// v1.0.0 understands state schema versions only up to 2. Used by tests
+		// asserting that an older CLI rejects newer state (e.g. the v3 state
+		// written when a bundle opts into the deployment metadata service).
+		cliV1Path := DownloadCLI(t, buildDir, "1.0.0")
+		t.Setenv("CLI_V1", cliV1Path)
+		repls.SetPath(cliV1Path, "[CLI_V1]")
 	}
 
 	paths := []string{

acceptance/bundle/deploy/record-deployment-history/state-upgrade/databricks.dms.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-rdh-state-upgrade
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: foo-job-renamed

acceptance/bundle/deploy/record-deployment-history/state-upgrade/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: test-rdh-state-upgrade
+
+resources:
+  jobs:
+    foo:
+      name: foo-job

acceptance/bundle/deploy/record-deployment-history/state-upgrade/output.txt

@@ -0,0 +1,39 @@
+
+=== without the flag: no features recorded
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-rdh-state-upgrade/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_state.py
+{
+  "state_version": 3,
+  "features": null
+}
+
+=== with experimental.record_deployment_history: the feature is recorded
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-rdh-state-upgrade/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_state.py
+{
+  "state_version": 3,
+  "features": {
+    "record_deployment_history": "1.2.0"
+  }
+}
+
+=== an older CLI (v1.0.0, max state version 2) rejects the upgraded state
+>>> errcode [CLI_V1] bundle plan
+Warning: unknown field: record_deployment_history
+  at experimental
+  in databricks.yml:5:3
+
+Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: state version 3 is newer than supported version 2; upgrade the CLI
+
+
+Exit code: 1

acceptance/bundle/deploy/record-deployment-history/state-upgrade/script

@@ -0,0 +1,19 @@
+# Feature-flag lifecycle for the direct state:
+#   1. A normal deploy writes the baseline state version with no features.
+#   2. Opting into experimental.record_deployment_history records a feature flag,
+#      stamped with the CLI version that wrote it.
+#   3. An older CLI that predates this state version refuses to operate on it.
+
+title "without the flag: no features recorded"
+trace $CLI bundle deploy
+trace print_state.py | jq '{state_version, features}'
+
+title "with experimental.record_deployment_history: the feature is recorded"
+# Also renames the job so the deploy writes state (a no-op deploy would not
+# rewrite the state file and the change would not be observable yet).
+cp databricks.dms.yml databricks.yml
+trace $CLI bundle deploy
+trace print_state.py | jq '{state_version, features}'
+
+title "an older CLI (v1.0.0, max state version 2) rejects the upgraded state"
+trace errcode $CLI_V1 bundle plan 2>&1 | contains.py "state version 3 is newer than supported version 2; upgrade the CLI"

acceptance/bundle/deploy/record-deployment-history/state-upgrade/test.toml

@@ -0,0 +1,10 @@
+Local = true
+Cloud = false
+
+# databricks.yml is rewritten in-place by the script (flag added in the second step).
+Ignore = [".databricks", "databricks.yml"]
+
+# The state-version upgrade only applies to the direct engine; terraform stores
+# state differently and would diverge.
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/deploy/wal/chain-3-jobs/output.txt

@@ -12,7 +12,7 @@ Exit code: [KILLED]
   "cli_version": "[DEV_VERSION]",
   "lineage": "[UUID]",
   "serial": 1,
-  "state_version": 2
+  "state_version": 3
 }
 {
   "k": "resources.jobs.job_01",

acceptance/bundle/deploy/wal/crash-after-create/output.txt

@@ -13,7 +13,7 @@ Exit code: [KILLED]
 
 >>> cat .databricks/bundle/default/resources.json.wal
 {
-  "state_version": 2,
+  "state_version": 3,
   "cli_version": "[DEV_VERSION]",
   "lineage": "[UUID]",
   "serial": 1

acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json

@@ -1,5 +1,5 @@
 {
- "state_version": 2,
+ "state_version": 3,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 2,

acceptance/bundle/migrate/basic/out.new_state.json

@@ -1,5 +1,5 @@
 {
- "state_version": 2,
+ "state_version": 3,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 6,

acceptance/bundle/migrate/dashboards/out.new_state.json

@@ -1,5 +1,5 @@
 {
- "state_version": 2,
+ "state_version": 3,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 3,

acceptance/bundle/migrate/default-python/out.state_after_migration.json

@@ -1,5 +1,5 @@
 {
- "state_version": 2,
+ "state_version": 3,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 5,

acceptance/bundle/migrate/grants/out.new_state.json

@@ -1,5 +1,5 @@
 {
- "state_version": 2,
+ "state_version": 3,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 9,

acceptance/bundle/migrate/permissions/out.new_state.json

@@ -1,5 +1,5 @@
 {
- "state_version": 2,
+ "state_version": 3,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 7,

acceptance/bundle/migrate/runas/out.new_state.json

@@ -1,5 +1,5 @@
 {
- "state_version": 2,
+ "state_version": 3,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 5,

acceptance/bundle/resources/jobs/big_id/out.state.direct.json

@@ -1,5 +1,5 @@
 {
- "state_version": 2,
+ "state_version": 3,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 1,

acceptance/bundle/resources/jobs/update/out.state.direct.json

@@ -1,5 +1,5 @@
 {
- "state_version": 2,
+ "state_version": 3,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 1,

acceptance/bundle/state/future_version/output.txt

@@ -1,3 +1,3 @@
-state version 999 is newer than supported version 2; upgrade the CLI
+state version 999 is newer than supported version 3; upgrade the CLI
 
 Exit code: 1

acceptance/bundle/state/permission_level_migration/output.txt

@@ -12,7 +12,7 @@ Deployment complete!
 === Print state after deploy
 >>> print_state.py
 {
- "state_version": 2,
+ "state_version": 3,
  "cli_version": "[DEV_VERSION]",
  "lineage": "test-lineage",
  "serial": 2,

acceptance/bundle/state/unknown_feature/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      name: "my job"

acceptance/bundle/state/unknown_feature/output.txt

@@ -0,0 +1,3 @@
+the deployment state requires feature "future_feature" which this CLI ([DEV_VERSION]) does not support; upgrade to 9.9.9 or newer
+
+Exit code: 1

acceptance/bundle/state/unknown_feature/resources.future.json

@@ -0,0 +1,10 @@
+{
+ "state_version": 3,
+ "features": {
+  "future_feature": "9.9.9"
+ },
+ "cli_version": "0.0.0-dev",
+ "lineage": "test-lineage",
+ "serial": 1,
+ "state": {}
+}

acceptance/bundle/state/unknown_feature/script

@@ -0,0 +1,6 @@
+mkdir -p .databricks/bundle/default
+cp resources.future.json .databricks/bundle/default/resources.json
+
+# The state records a feature this CLI does not know; it must refuse and point the
+# user at the minimum CLI version recorded in the state.
+trace $CLI bundle plan 2>&1 | grep -o 'the deployment state requires feature.*'

acceptance/bundle/state/unknown_feature/test.toml

@@ -0,0 +1,4 @@
+Ignore = [".databricks"]
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json

@@ -222,7 +222,7 @@
     "overwrite": "true"
   },
   "body": {
-    "state_version": 2,
+    "state_version": 3,
     "cli_version": "[DEV_VERSION]",
     "lineage": "[UUID]",
     "serial": 1,

bundle/direct/dstate/migrate.go

@@ -10,12 +10,15 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/iam"
 )
 
-// migrateState runs all necessary migrations on the database.
-// It is called after loading state from disk.
+// migrateState brings a freshly-loaded state up to the current schema version.
+// It is called after loading state from disk. Legacy states below the current
+// version are migrated forward via the migrations map; a state newer than the
+// current version was written by a newer CLI, so we refuse it rather than risk
+// mishandling a format we don't understand.
+//
+// Feature flags recorded in the state are validated separately; see
+// checkSupportedFeatures.
 func migrateState(db *Database) error {
-	if db.StateVersion == currentStateVersion {
-		return nil
-	}
 	if db.StateVersion > currentStateVersion {
 		return fmt.Errorf("state version %d is newer than supported version %d; upgrade the CLI", db.StateVersion, currentStateVersion)
 	}
@@ -39,6 +42,14 @@ func migrateState(db *Database) error {
 var migrations = map[int]func(*Database) error{
 	0: migrateV1ToV2,
 	1: migrateV1ToV2,
+	2: migrateV2ToV3,
+}
+
+// migrateV2ToV3 upgrades to the schema that carries the feature-flag list
+// (Header.Features). The list is optional and absent by default, so there is
+// nothing to transform.
+func migrateV2ToV3(db *Database) error {
+	return nil
 }
 
 // migrateV1ToV2 migrates permissions and grants entries from the old format

bundle/direct/dstate/migrate_test.go

@@ -0,0 +1,75 @@
+package dstate
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMigrateStateLeavesCurrentUntouched(t *testing.T) {
+	db := &Database{Header: Header{StateVersion: currentStateVersion}}
+	require.NoError(t, migrateState(db))
+	assert.Equal(t, currentStateVersion, db.StateVersion)
+}
+
+func TestMigrateStateUpgradesLegacyToCurrent(t *testing.T) {
+	// A legacy state migrates forward to the current version (the v2->v3 step adds
+	// the feature list, which is absent by default).
+	db := &Database{Header: Header{StateVersion: 2}}
+	require.NoError(t, migrateState(db))
+	assert.Equal(t, currentStateVersion, db.StateVersion)
+}
+
+func TestMigrateStateRejectsNewerThanSupported(t *testing.T) {
+	db := &Database{Header: Header{StateVersion: currentStateVersion + 1}}
+	err := migrateState(db)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "upgrade the CLI")
+}
+
+// TestStateSchemaVersion pins currentStateVersion. It is part of the on-disk
+// format and a contract with older CLIs, so changing it must be deliberate.
+//
+// Bump it only for a structural schema change that older CLIs cannot read: add a
+// migration to the migrations map (TestMigrationsCoverBaseline enforces full
+// coverage) and update the assertion below. For an additive capability that does
+// not change the structure, record a feature flag instead (see knownFeatures and
+// RecordFeature) — that lets older CLIs fail with an actionable "upgrade to X"
+// message without a version bump.
+//
+// RELATED COVERAGE
+//   - acceptance/bundle/state/permission_level_migration: golden v1->v2 migration.
+//   - acceptance/bundle/state/unknown_feature: a state requiring an unknown feature
+//     is rejected with the recorded minimum CLI version.
+//   - bundle/invariant/continue_293: the current CLI reads state written by an
+//     older released CLI.
+func TestStateSchemaVersion(t *testing.T) {
+	assert.Equal(t, 3, currentStateVersion)
+}
+
+// TestMigrationsCoverBaseline guards a baseline bump: every state version below
+// currentStateVersion must have a migration to the next version, so migrateState
+// can always climb a legacy state up to the baseline. A bump that forgets a
+// migration fails here instead of at a user's deploy.
+func TestMigrationsCoverBaseline(t *testing.T) {
+	for v := range currentStateVersion {
+		assert.Containsf(t, migrations, v, "missing migration for state version %d", v)
+	}
+}
+
+func TestCheckSupportedFeatures(t *testing.T) {
+	// Known features (and none at all) are accepted; an unknown feature is rejected
+	// with its name and the recorded minimum CLI version.
+	require.NoError(t, checkSupportedFeatures(&Database{}))
+	require.NoError(t, checkSupportedFeatures(&Database{Header: Header{
+		Features: map[string]string{FeatureRecordDeploymentHistory: "0.0.0-dev"},
+	}}))
+
+	err := checkSupportedFeatures(&Database{Header: Header{
+		Features: map[string]string{"future_feature": "9.9.9"},
+	}})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `feature "future_feature"`)
+	assert.Contains(t, err.Error(), "upgrade to 9.9.9 or newer")
+}