From 9d45d976a085a5b655387a2b4aa30616995ae3e1 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 16:20:28 +0200 Subject: [PATCH 01/14] direct: ignore UC-managed schema properties as backend defaults UC auto-populates system-managed property keys (e.g. `unity.catalog.managed.delta.defaults.delta.enableRowTracking`) on schema creation. Without a backend_defaults rule, the planner sees the remote map as drift, emits Update, and DoUpdate sends an empty payload which UC rejects with "UpdateSchema Nothing to update". The rule only applies when both saved and new are nil, so user-set properties still drive real drift detection. Also mirror the UC behavior in the fake testserver so the no-drift invariant is exercised locally; added acceptance/.../schemas/drift/ managed_properties covering the reproducer. Co-authored-by: Isaac --- .../drift/managed_properties/databricks.yml | 8 +++++++ .../drift/managed_properties/out.test.toml | 3 +++ .../drift/managed_properties/output.txt | 24 +++++++++++++++++++ .../schemas/drift/managed_properties/script | 11 +++++++++ .../drift/managed_properties/test.toml | 5 ++++ .../resources/schemas/recreate/output.txt | 3 +++ bundle/direct/dresources/resources.yml | 6 +++++ libs/testserver/schemas.go | 7 ++++++ 8 files changed, 67 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/output.txt create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/script create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/test.toml diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml b/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml new file mode 100644 index 00000000000..cd05d222c48 --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test-bundle + +resources: + schemas: + schema1: + name: myschema + catalog_name: main diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt new file mode 100644 index 00000000000..869cd97c29d --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt @@ -0,0 +1,24 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan is a no-op despite UC auto-populating managed properties +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Redeploy is a no-op (no UpdateSchema call) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity +json.method = "POST"; +json.path = "/api/2.1/unity-catalog/schemas"; +json.body.catalog_name = "main"; +json.body.name = "myschema"; diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/script b/acceptance/bundle/resources/schemas/drift/managed_properties/script new file mode 100644 index 00000000000..c3ef4ac39e4 --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/script @@ -0,0 +1,11 @@ +echo "*" > .gitignore + +title "Initial deployment" +trace $CLI bundle deploy + +title "Plan is a no-op despite UC auto-populating managed properties" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "Redeploy is a no-op (no UpdateSchema call)" +trace $CLI bundle deploy +trace print_requests.py //unity | gron.py | contains.py '!json.method = "PATCH"' diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/test.toml new file mode 100644 index 00000000000..5016e85b395 --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/test.toml @@ -0,0 +1,5 @@ +RecordRequests = true + +# Terraform issues a spurious PATCH for enable_predictive_optimization on every +# deploy, which is outside the scope of backend-default handling in resources.yml. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/recreate/output.txt b/acceptance/bundle/resources/schemas/recreate/output.txt index 7c173eb11f0..bb33023f9d2 100644 --- a/acceptance/bundle/resources/schemas/recreate/output.txt +++ b/acceptance/bundle/resources/schemas/recreate/output.txt @@ -83,6 +83,9 @@ Error: Resource catalog.SchemaInfo not found: main.myschema "metastore_id": "[UUID]", "name": "myschema", "owner": "[USERNAME]", + "properties": { + "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true" + }, "schema_id": "[UUID]", "updated_at": [UNIX_TIME_MILLIS][0], "updated_by": "[USERNAME]" diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 569fca9ee82..3375908f3ba 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -323,6 +323,12 @@ resources: reason: immutable - field: storage_root reason: immutable + backend_defaults: + # UC auto-populates system-managed keys like + # `unity.catalog.managed.delta.defaults.delta.enableRowTracking` after create. + # Without this, every subsequent plan produces an Update whose payload is empty, + # and UC rejects it with "UpdateSchema Nothing to update". + - field: properties external_locations: recreate_on_changes: diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 1d4dc79e7ac..92c01bf7fc2 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -39,6 +39,13 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { schema.MetastoreId = TestMetastore.MetastoreId schema.Owner = s.CurrentUser().UserName schema.SchemaId = nextUUID() + if schema.Properties == nil { + // Mirror UC behavior: managed system defaults are populated when the user + // doesn't specify any properties. Required to cover backend-default drift. + schema.Properties = map[string]string{ + "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", + } + } s.Schemas[schema.FullName] = schema return Response{ From 1f336d8d738bcae71bff5687d0ccb68666e5623e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 18:18:15 +0200 Subject: [PATCH 02/14] Narrow properties --- bundle/direct/bundle_plan_test.go | 61 ++++++++++++++++++++++++++ bundle/direct/dresources/resources.yml | 6 +-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index ccfb7cb517f..1579217ae52 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -3,8 +3,12 @@ package direct import ( "testing" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDynPathToStructPath(t *testing.T) { @@ -35,3 +39,60 @@ func TestDynPathToStructPath(t *testing.T) { assert.Equal(t, tc.expected, node.String()) } } + +func TestShouldSkipBackendDefault_SchemaManagedPropertiesOnly(t *testing.T) { + cfg := dresources.GetResourceConfig("schemas") + require.NotNil(t, cfg) + + tests := []struct { + name string + path string + remote any + expected bool + }{ + { + name: "managed delta row tracking property", + path: "properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking']", + remote: "true", + expected: true, + }, + { + name: "managed iceberg catalog property", + path: "properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged']", + remote: "true", + expected: true, + }, + { + name: "unmanaged remote-only property is not skipped", + path: "properties['custom.remote_only']", + remote: "true", + expected: false, + }, + { + name: "parent properties map is not skipped", + path: "properties", + remote: map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true"}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path, err := structpath.ParsePath(tt.path) + require.NoError(t, err) + + reason, ok := shouldSkipBackendDefault(cfg, path, &deployplan.ChangeDesc{ + Old: nil, + New: nil, + Remote: tt.remote, + }) + + assert.Equal(t, tt.expected, ok) + if tt.expected { + assert.Equal(t, deployplan.ReasonBackendDefault, reason) + } else { + assert.Empty(t, reason) + } + }) + } +} diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 3375908f3ba..1d07bbf4ae7 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -324,11 +324,11 @@ resources: - field: storage_root reason: immutable backend_defaults: - # UC auto-populates system-managed keys like - # `unity.catalog.managed.delta.defaults.delta.enableRowTracking` after create. + # UC auto-populates these system-managed keys after create. # Without this, every subsequent plan produces an Update whose payload is empty, # and UC rejects it with "UpdateSchema Nothing to update". - - field: properties + - field: properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking'] + - field: properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged'] external_locations: recreate_on_changes: From acc3518cbe638e75d9b701238957a10d12585bcd Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 18:26:37 +0200 Subject: [PATCH 03/14] direct: handle schema backend-default map drift --- bundle/direct/bundle_plan.go | 51 ++++++++++++++++++++++++++----- bundle/direct/bundle_plan_test.go | 28 ++++++++++++++++- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index f6bcea316cd..d53c185ed83 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -452,19 +452,56 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str return "", false } for _, rule := range cfg.BackendDefaults { - if !path.HasPatternPrefix(rule.Field) { - continue - } - if len(rule.Values) == 0 { - return deployplan.ReasonBackendDefault, true - } - if matchesAllowedValue(ch.Remote, rule.Values) { + if matchesBackendDefaultRule(path, ch.Remote, rule) { return deployplan.ReasonBackendDefault, true } } + if matchesBackendDefaultMap(cfg, path, ch.Remote) { + return deployplan.ReasonBackendDefault, true + } return "", false } +func matchesBackendDefaultRule(path *structpath.PathNode, remote any, rule dresources.BackendDefaultRule) bool { + if !path.HasPatternPrefix(rule.Field) { + return false + } + if len(rule.Values) == 0 { + return true + } + return matchesAllowedValue(remote, rule.Values) +} + +// matchesBackendDefaultMap handles the nil-vs-map case from structdiff, where a +// remote-only map change is emitted at the parent path rather than per key. +// We only skip the parent map if every remote entry matches a configured +// backend-default child rule; any unmanaged key must still surface as drift. +func matchesBackendDefaultMap(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { + rv := reflect.ValueOf(remote) + if !rv.IsValid() || rv.Kind() != reflect.Map || rv.IsNil() || rv.Type().Key().Kind() != reflect.String || rv.Len() == 0 { + return false + } + + iter := rv.MapRange() + for iter.Next() { + childPath := structpath.NewBracketString(path, iter.Key().String()) + childRemote := iter.Value().Interface() + + matched := false + for _, rule := range cfg.BackendDefaults { + if matchesBackendDefaultRule(childPath, childRemote, rule) { + matched = true + break + } + } + if !matched { + return false + } + } + + return true +} + // matchesAllowedValue checks if the remote value matches one of the allowed JSON values. // Each json.RawMessage is unmarshaled into the same type as remote for comparison. func matchesAllowedValue(remote any, values []json.RawMessage) bool { diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 1579217ae52..0b3191aca30 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -69,9 +69,15 @@ func TestShouldSkipBackendDefault_SchemaManagedPropertiesOnly(t *testing.T) { expected: false, }, { - name: "parent properties map is not skipped", + name: "managed-only parent properties map is skipped", path: "properties", remote: map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true"}, + expected: true, + }, + { + name: "mixed parent properties map is not skipped", + path: "properties", + remote: map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", "custom.remote_only": "true"}, expected: false, }, } @@ -96,3 +102,23 @@ func TestShouldSkipBackendDefault_SchemaManagedPropertiesOnly(t *testing.T) { }) } } + +// Map drift handling synthesizes child paths to match against rules. structdiff +// always emits map keys in bracket notation, so synthetic child paths must too; +// otherwise rules wouldn't match for identifier-like keys. +func TestShouldSkipBackendDefault_MapDriftUsesBracketKeys(t *testing.T) { + field, err := structpath.ParsePattern("properties['simple']") + require.NoError(t, err) + cfg := &dresources.ResourceLifecycleConfig{ + BackendDefaults: []dresources.BackendDefaultRule{{Field: field}}, + } + + path, err := structpath.ParsePath("properties") + require.NoError(t, err) + + reason, ok := shouldSkipBackendDefault(cfg, path, &deployplan.ChangeDesc{ + Remote: map[string]string{"simple": "v"}, + }) + assert.True(t, ok) + assert.Equal(t, deployplan.ReasonBackendDefault, reason) +} From 483d288eefc0d101cc9fd3206bc8c1d8b53c8129 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 9 Jun 2026 17:25:12 +0200 Subject: [PATCH 04/14] testserver: scope schema managed-defaults to the drift test The fake populated UC-managed properties on every property-less schema. The direct engine ignores them as backend defaults, but terraform treats them as drift, which broke unrelated schema acceptance tests across both engines. Gate the injection on a dedicated schema name so only the backend-default drift test opts in, and drop recreate's now-incidental property line. Co-authored-by: Isaac --- .../schemas/drift/managed_properties/databricks.yml | 2 +- .../resources/schemas/drift/managed_properties/output.txt | 2 +- acceptance/bundle/resources/schemas/recreate/output.txt | 3 --- libs/testserver/schemas.go | 8 +++++++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml b/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml index cd05d222c48..5ab0d85ca59 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml @@ -4,5 +4,5 @@ bundle: resources: schemas: schema1: - name: myschema + name: schema_managed_defaults catalog_name: main diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt index 869cd97c29d..f2ecb3506f0 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt @@ -21,4 +21,4 @@ Deployment complete! json.method = "POST"; json.path = "/api/2.1/unity-catalog/schemas"; json.body.catalog_name = "main"; -json.body.name = "myschema"; +json.body.name = "schema_managed_defaults"; diff --git a/acceptance/bundle/resources/schemas/recreate/output.txt b/acceptance/bundle/resources/schemas/recreate/output.txt index a86a556a22b..39cafa5b4c3 100644 --- a/acceptance/bundle/resources/schemas/recreate/output.txt +++ b/acceptance/bundle/resources/schemas/recreate/output.txt @@ -83,9 +83,6 @@ Error: Resource catalog.SchemaInfo not found: main.myschema "metastore_id": "[UUID]", "name": "myschema", "owner": "[USERNAME]", - "properties": { - "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true" - }, "schema_id": "[UUID]", "updated_at": [UNIX_TIME_MILLIS][0], "updated_by": "[USERNAME]" diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 92c01bf7fc2..883133b8d38 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -11,6 +11,12 @@ import ( const testMetastoreName = "deco-uc-prod-isolated-aws-us-east-1" +// schemaNameManagedDefaults is the schema name the backend-default drift test uses +// to opt into UC's managed-property simulation. Scoping the injection to this name +// keeps unrelated schema tests free of the property, which terraform would otherwise +// report as drift on redeploy. +const schemaNameManagedDefaults = "schema_managed_defaults" + func (s *FakeWorkspace) SchemasCreate(req Request) Response { defer s.LockUnlock()() @@ -39,7 +45,7 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { schema.MetastoreId = TestMetastore.MetastoreId schema.Owner = s.CurrentUser().UserName schema.SchemaId = nextUUID() - if schema.Properties == nil { + if schema.Properties == nil && schema.Name == schemaNameManagedDefaults { // Mirror UC behavior: managed system defaults are populated when the user // doesn't specify any properties. Required to cover backend-default drift. schema.Properties = map[string]string{ From 56952a761cf485469fe14ad83da489cf513ad6b4 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 10 Jun 2026 14:16:24 +0200 Subject: [PATCH 05/14] Factor out asNonEmptyStringMap inside matchesBackendDefaultMap --- bundle/direct/bundle_plan.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 0fa86359f4d..fff3de0719f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -480,8 +480,8 @@ func matchesBackendDefaultRule(path *structpath.PathNode, remote any, rule dreso // We only skip the parent map if every remote entry matches a configured // backend-default child rule; any unmanaged key must still surface as drift. func matchesBackendDefaultMap(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { - rv := reflect.ValueOf(remote) - if !rv.IsValid() || rv.Kind() != reflect.Map || rv.IsNil() || rv.Type().Key().Kind() != reflect.String || rv.Len() == 0 { + rv, ok := asNonEmptyStringMap(remote) + if !ok { return false } @@ -505,6 +505,16 @@ func matchesBackendDefaultMap(cfg *dresources.ResourceLifecycleConfig, path *str return true } +// asNonEmptyStringMap returns remote as a reflected map value when it is a +// non-nil, non-empty map with string keys; ok is false otherwise. +func asNonEmptyStringMap(remote any) (reflect.Value, bool) { + rv := reflect.ValueOf(remote) + if !rv.IsValid() || rv.Kind() != reflect.Map || rv.IsNil() || rv.Type().Key().Kind() != reflect.String || rv.Len() == 0 { + return reflect.Value{}, false + } + return rv, true +} + // matchesAllowedValue checks if the remote value matches one of the allowed JSON values. // Each json.RawMessage is unmarshaled into the same type as remote for comparison. func matchesAllowedValue(remote any, values []json.RawMessage) bool { From 66711bf5393d367282209c5dde974532dbbd7dfb Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 10 Jun 2026 14:19:31 +0200 Subject: [PATCH 06/14] Factor out matchesAnyBackendDefault and use in both places --- bundle/direct/bundle_plan.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index fff3de0719f..d389cb931c9 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -454,17 +454,23 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str if cfg == nil || ch.Old != nil || ch.New != nil || ch.Remote == nil { return "", false } - for _, rule := range cfg.BackendDefaults { - if matchesBackendDefaultRule(path, ch.Remote, rule) { - return deployplan.ReasonBackendDefault, true - } - } - if matchesBackendDefaultMap(cfg, path, ch.Remote) { + if matchesAnyBackendDefault(cfg, path, ch.Remote) || matchesBackendDefaultMap(cfg, path, ch.Remote) { return deployplan.ReasonBackendDefault, true } return "", false } +// matchesAnyBackendDefault reports whether the change at path matches any of the +// resource's configured backend-default rules. +func matchesAnyBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { + for _, rule := range cfg.BackendDefaults { + if matchesBackendDefaultRule(path, remote, rule) { + return true + } + } + return false +} + func matchesBackendDefaultRule(path *structpath.PathNode, remote any, rule dresources.BackendDefaultRule) bool { if !path.HasPatternPrefix(rule.Field) { return false @@ -488,16 +494,7 @@ func matchesBackendDefaultMap(cfg *dresources.ResourceLifecycleConfig, path *str iter := rv.MapRange() for iter.Next() { childPath := structpath.NewBracketString(path, iter.Key().String()) - childRemote := iter.Value().Interface() - - matched := false - for _, rule := range cfg.BackendDefaults { - if matchesBackendDefaultRule(childPath, childRemote, rule) { - matched = true - break - } - } - if !matched { + if !matchesAnyBackendDefault(cfg, childPath, iter.Value().Interface()) { return false } } From 3ef9f90aa628ab21be01be18bdbe86ca661a8645 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 12 Jun 2026 00:06:15 +0200 Subject: [PATCH 07/14] Inline map-drift handling into shouldSkipBackendDefault Flattens the shouldSkipBackendDefault -> matchesBackendDefaultMap -> matchesAnyBackendDefault chain: the nil-vs-map walk now sits directly in shouldSkipBackendDefault, and the per-rule prefix/values check is merged into matchesAnyBackendDefault, which remains a function because the map case calls it once per entry. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 66 +++++++++++++----------------------- 1 file changed, 23 insertions(+), 43 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index d389cb931c9..1cead2065b5 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -454,62 +454,42 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str if cfg == nil || ch.Old != nil || ch.New != nil || ch.Remote == nil { return "", false } - if matchesAnyBackendDefault(cfg, path, ch.Remote) || matchesBackendDefaultMap(cfg, path, ch.Remote) { + if matchesAnyBackendDefault(cfg, path, ch.Remote) { return deployplan.ReasonBackendDefault, true } - return "", false -} - -// matchesAnyBackendDefault reports whether the change at path matches any of the -// resource's configured backend-default rules. -func matchesAnyBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { - for _, rule := range cfg.BackendDefaults { - if matchesBackendDefaultRule(path, remote, rule) { - return true - } - } - return false -} - -func matchesBackendDefaultRule(path *structpath.PathNode, remote any, rule dresources.BackendDefaultRule) bool { - if !path.HasPatternPrefix(rule.Field) { - return false - } - if len(rule.Values) == 0 { - return true - } - return matchesAllowedValue(remote, rule.Values) -} -// matchesBackendDefaultMap handles the nil-vs-map case from structdiff, where a -// remote-only map change is emitted at the parent path rather than per key. -// We only skip the parent map if every remote entry matches a configured -// backend-default child rule; any unmanaged key must still surface as drift. -func matchesBackendDefaultMap(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { - rv, ok := asNonEmptyStringMap(remote) - if !ok { - return false + // Nil-vs-map case from structdiff: a remote-only map change is emitted at the + // parent path rather than per key. Only skip the parent map if every remote + // entry matches a configured backend-default child rule; any unmanaged key + // must still surface as drift. rv is always valid here (ch.Remote != nil + // above) and a nil map is excluded by Len() == 0. + rv := reflect.ValueOf(ch.Remote) + if rv.Kind() != reflect.Map || rv.Type().Key().Kind() != reflect.String || rv.Len() == 0 { + return "", false } - iter := rv.MapRange() for iter.Next() { childPath := structpath.NewBracketString(path, iter.Key().String()) if !matchesAnyBackendDefault(cfg, childPath, iter.Value().Interface()) { - return false + return "", false } } - - return true + return deployplan.ReasonBackendDefault, true } -// asNonEmptyStringMap returns remote as a reflected map value when it is a -// non-nil, non-empty map with string keys; ok is false otherwise. -func asNonEmptyStringMap(remote any) (reflect.Value, bool) { - rv := reflect.ValueOf(remote) - if !rv.IsValid() || rv.Kind() != reflect.Map || rv.IsNil() || rv.Type().Key().Kind() != reflect.String || rv.Len() == 0 { - return reflect.Value{}, false +// matchesAnyBackendDefault reports whether the remote value at path matches any of +// the resource's configured backend-default rules (and the rule's allowed values, +// if specified). +func matchesAnyBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { + for _, rule := range cfg.BackendDefaults { + if !path.HasPatternPrefix(rule.Field) { + continue + } + if len(rule.Values) == 0 || matchesAllowedValue(remote, rule.Values) { + return true + } } - return rv, true + return false } // matchesAllowedValue checks if the remote value matches one of the allowed JSON values. From e30e9dfad1cc52d62bd5e37f61c4a490e30ac3fc Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 12 Jun 2026 00:06:27 +0200 Subject: [PATCH 08/14] Decouple backend-default unit test from resources.yml Build the test's lifecycle config inline instead of reading the real schemas config via GetResourceConfig, so future resources.yml edits can't break the unit test. The real wiring stays covered by the schemas/drift acceptance test. Co-authored-by: Isaac --- bundle/direct/bundle_plan_test.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 0b3191aca30..c7f6e8dbf36 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -40,9 +40,20 @@ func TestDynPathToStructPath(t *testing.T) { } } -func TestShouldSkipBackendDefault_SchemaManagedPropertiesOnly(t *testing.T) { - cfg := dresources.GetResourceConfig("schemas") - require.NotNil(t, cfg) +func TestShouldSkipBackendDefault_ManagedPropertiesOnly(t *testing.T) { + // Rules mirror the schemas backend_defaults in resources.yml, but the test is + // deliberately self-contained so that edits to resources.yml don't break it. + // The real wiring is covered by acceptance/bundle/resources/schemas/drift. + rowTracking, err := structpath.ParsePattern("properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking']") + require.NoError(t, err) + catalogManaged, err := structpath.ParsePattern("properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged']") + require.NoError(t, err) + cfg := &dresources.ResourceLifecycleConfig{ + BackendDefaults: []dresources.BackendDefaultRule{ + {Field: rowTracking}, + {Field: catalogManaged}, + }, + } tests := []struct { name string From 00faf77321e0e8bf75f0bc195858b37f3943b44d Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 12 Jun 2026 01:18:38 +0200 Subject: [PATCH 09/14] Resolve conflict markers committed in the main merge The GitHub web conflict editor left the <<<<<<< / ======= / >>>>>>> markers in resources.yml, breaking YAML parsing (lint, direct tests, and validate-generated). Keep both sides: backend_defaults from this branch plus normalize_case/normalize_slash from main (#5531), ordered to match the volumes section. Co-authored-by: Isaac --- bundle/direct/dresources/resources.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index e26a55e5218..c42ce2056c9 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -311,14 +311,6 @@ resources: reason: immutable - field: storage_root reason: immutable -<<<<<<< janniklasrose/schema-backend-defaults - backend_defaults: - # UC auto-populates these system-managed keys after create. - # Without this, every subsequent plan produces an Update whose payload is empty, - # and UC rejects it with "UpdateSchema Nothing to update". - - field: properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking'] - - field: properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged'] -======= normalize_case: # UC lowercases identifier names; remote returns "myschema" for config "MySchema". - field: name @@ -328,7 +320,12 @@ resources: normalize_slash: - field: storage_root reason: uc_strips_trailing_slash ->>>>>>> main + backend_defaults: + # UC auto-populates these system-managed keys after create. + # Without this, every subsequent plan produces an Update whose payload is empty, + # and UC rejects it with "UpdateSchema Nothing to update". + - field: properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking'] + - field: properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged'] external_locations: recreate_on_changes: From 9623c6bdc4dc01a6102fc131106787c78bc5c010 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 12 Jun 2026 13:47:57 +0200 Subject: [PATCH 10/14] Changelog --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index f26e1ea1afc..f2b18744215 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,6 +11,7 @@ * Remove API enum values and types that are still in development from the `databricks-bundles` Python package; these were never accepted by the backend ([#5484](https://github.com/databricks/cli/pull/5484)). * direct: Fix resolving a resource reference that is used more than once within the same field ([#5558](https://github.com/databricks/cli/pull/5558)). * Bundle variable references now accept Unicode letters in path segments (e.g. `${var.变量}`). ([#5532](https://github.com/databricks/cli/pull/5532)) +* Ignore backend_default values for unity catalog schema properties ([#5195](https://github.com/databricks/cli/pull/5195)). ### Dependency updates From 26214503cab7a676c7d29b4d3c373d6f05b8bc8c Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 15 Jun 2026 14:37:09 +0200 Subject: [PATCH 11/14] Add unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged to testserver --- libs/testserver/schemas.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index d5e81de2f49..d0bcbe7dae7 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -53,6 +53,7 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { // doesn't specify any properties. Required to cover backend-default drift. schema.Properties = map[string]string{ "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", + "unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged": "true", } } s.Schemas[schema.FullName] = schema From d1326b9cb8cec63979bc3817fcd59f12248a4e4d Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 15 Jun 2026 14:53:18 +0200 Subject: [PATCH 12/14] Autofmt --- libs/testserver/schemas.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index d0bcbe7dae7..cac8c5dfdd2 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -52,7 +52,7 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { // Mirror UC behavior: managed system defaults are populated when the user // doesn't specify any properties. Required to cover backend-default drift. schema.Properties = map[string]string{ - "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", + "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", "unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged": "true", } } From 5b3da5a4a1b88b0c6afaee30ec8c2c2be165721b Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 16 Jun 2026 13:36:02 +0200 Subject: [PATCH 13/14] nits --- NEXT_CHANGELOG.md | 2 +- bundle/direct/bundle_plan.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index efd18611ef0..6f46cabddda 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -13,7 +13,7 @@ * Bundle variable references now accept Unicode letters in path segments (e.g. `${var.变量}`). ([#5532](https://github.com/databricks/cli/pull/5532)) * Ignore remote changes for vector search direct_access_index_spec.schema_json to prevent drift when the backend normalizes the schema ([#5481](https://github.com/databricks/cli/pull/5481)). * Remove hidden, never-functional `--existing-dashboard-id`, `--existing-dashboard-path`, `--existing-alert-id`, and `--existing-genie-space-id` alias flags from `bundle generate`; use the documented `--existing-id` / `--existing-path` flags instead ([#5591](https://github.com/databricks/cli/pull/5591)). -* Ignore backend_default values for unity catalog schema properties ([#5195](https://github.com/databricks/cli/pull/5195)). +* Ignore unity catalog managed schema property defaults to avoid unnecessary drift ([#5195](https://github.com/databricks/cli/pull/5195)). ### Dependency updates diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index ab301b5e6dc..d7da077302c 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -514,7 +514,10 @@ func matchesAnyBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str if !path.HasPatternPrefix(rule.Field) { continue } - if len(rule.Values) == 0 || matchesAllowedValue(remote, rule.Values) { + if len(rule.Values) == 0 { + return true + } + if matchesAllowedValue(remote, rule.Values) { return true } } From 4251089e29de2e9d7de6fb9e3d3eac328d703996 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 16 Jun 2026 13:40:37 +0200 Subject: [PATCH 14/14] capture plan json changes --- .../schemas/drift/managed_properties/output.txt | 13 +++++++++++++ .../schemas/drift/managed_properties/script | 3 +++ 2 files changed, 16 insertions(+) diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt index f2ecb3506f0..50274732725 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt @@ -10,6 +10,19 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +=== The remote-only properties map is skipped as a backend default (confirms the matched rule) +>>> [CLI] bundle plan --output json +{ + "properties": { + "action": "skip", + "reason": "backend_default", + "remote": { + "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", + "unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged": "true" + } + } +} + === Redeploy is a no-op (no UpdateSchema call) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/script b/acceptance/bundle/resources/schemas/drift/managed_properties/script index c3ef4ac39e4..fadd886cd48 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/script +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/script @@ -6,6 +6,9 @@ trace $CLI bundle deploy title "Plan is a no-op despite UC auto-populating managed properties" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" +title "The remote-only properties map is skipped as a backend default (confirms the matched rule)" +trace $CLI bundle plan --output json | jq '.plan[].changes' + title "Redeploy is a no-op (no UpdateSchema call)" trace $CLI bundle deploy trace print_requests.py //unity | gron.py | contains.py '!json.method = "PATCH"'