direct: ignore UC-managed schema property defaults (#5195)

## Changes
- Add `backend_defaults` for two fields `properties['...']`
- Handle the nil-vs-map case from `structdiff` where a remote-only map
change is emitted at the parent path: only skip the parent if every
remote entry matches a configured child rule. Synthesize child paths
with `structpath.NewBracketString` to match what `structdiff` produces
(rather than `NewStringKey`, which would diverge for identifier-like
keys).
- Acceptance test with test server setting the properties based on
schema name

## Why

UC auto-populates system-managed property keys out of band (e.g. setting
`unity.catalog.managed.delta.defaults.delta.enableRowTracking`). This
causes the planner to output Update when there is nothing to do

## Tests
- [x] Existing & new Acceptance tests
- [ ] ⚠️ our current cloud test accounts do not exhibit this behaviour,
only saw it in my own staging workspace

This pull request and its description were written by Isaac.

Author: janniklasrose

NEXT_CHANGELOG.md

@@ -17,6 +17,7 @@
 * Remove hidden, never-functional `--existing-dashboard-id`, `--existing-dashboard-path`, `--existing-alert-id`, and `--existing-genie-space-id` alias flags from `bundle generate`; use the documented `--existing-id` / `--existing-path` flags instead ([#5591](https://github.com/databricks/cli/pull/5591)).
 * engine/direct: Fix WAL corruption after two consecutive failed deploys ([#5606](https://github.com/databricks/cli/pull/5606)).
 * engine/direct: Don't open the deployment state WAL when a deploy's plan fails ([#5607](https://github.com/databricks/cli/pull/5607)).
+* Ignore unity catalog managed schema property defaults to avoid unnecessary drift ([#5195](https://github.com/databricks/cli/pull/5195)).
 
 ### Dependency updates
 

acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml

@@ -0,0 +1,8 @@
+bundle:
+  name: test-bundle
+
+resources:
+  schemas:
+    schema1:
+      name: schema_managed_defaults
+      catalog_name: main

acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml

@@ -0,0 +1,37 @@
+
+=== Initial deployment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Plan is a no-op despite UC auto-populating managed properties
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== The remote-only properties map is skipped as a backend default (confirms the matched rule)
+>>> [CLI] bundle plan --output json
+{
+  "properties": {
+    "action": "skip",
+    "reason": "backend_default",
+    "remote": {
+      "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true",
+      "unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged": "true"
+    }
+  }
+}
+
+=== Redeploy is a no-op (no UpdateSchema call)
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //unity
+json.method = "POST";
+json.path = "/api/2.1/unity-catalog/schemas";
+json.body.catalog_name = "main";
+json.body.name = "schema_managed_defaults";

acceptance/bundle/resources/schemas/drift/managed_properties/output.txt

@@ -0,0 +1,14 @@
+echo "*" > .gitignore
+
+title "Initial deployment"
+trace $CLI bundle deploy
+
+title "Plan is a no-op despite UC auto-populating managed properties"
+trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged"
+
+title "The remote-only properties map is skipped as a backend default (confirms the matched rule)"
+trace $CLI bundle plan --output json | jq '.plan[].changes'
+
+title "Redeploy is a no-op (no UpdateSchema call)"
+trace $CLI bundle deploy
+trace print_requests.py //unity | gron.py | contains.py '!json.method = "PATCH"'

acceptance/bundle/resources/schemas/drift/managed_properties/script

@@ -0,0 +1,5 @@
+RecordRequests = true
+
+# Terraform issues a spurious PATCH for enable_predictive_optimization on every
+# deploy, which is outside the scope of backend-default handling in resources.yml.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/resources/schemas/drift/managed_properties/test.toml

@@ -483,18 +483,45 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str
 	if cfg == nil || ch.Old != nil || ch.New != nil || ch.Remote == nil {
 		return "", false
 	}
+	if matchesAnyBackendDefault(cfg, path, ch.Remote) {
+		return deployplan.ReasonBackendDefault, true
+	}
+
+	// Nil-vs-map case from structdiff: a remote-only map change is emitted at the
+	// parent path rather than per key. Only skip the parent map if every remote
+	// entry matches a configured backend-default child rule; any unmanaged key
+	// must still surface as drift. rv is always valid here (ch.Remote != nil
+	// above) and a nil map is excluded by Len() == 0.
+	rv := reflect.ValueOf(ch.Remote)
+	if rv.Kind() != reflect.Map || rv.Type().Key().Kind() != reflect.String || rv.Len() == 0 {
+		return "", false
+	}
+	iter := rv.MapRange()
+	for iter.Next() {
+		childPath := structpath.NewBracketString(path, iter.Key().String())
+		if !matchesAnyBackendDefault(cfg, childPath, iter.Value().Interface()) {
+			return "", false
+		}
+	}
+	return deployplan.ReasonBackendDefault, true
+}
+
+// matchesAnyBackendDefault reports whether the remote value at path matches any of
+// the resource's configured backend-default rules (and the rule's allowed values,
+// if specified).
+func matchesAnyBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool {
 	for _, rule := range cfg.BackendDefaults {
 		if !path.HasPatternPrefix(rule.Field) {
 			continue
 		}
 		if len(rule.Values) == 0 {
-			return deployplan.ReasonBackendDefault, true
+			return true
 		}
-		if matchesAllowedValue(ch.Remote, rule.Values) {
-			return deployplan.ReasonBackendDefault, true
+		if matchesAllowedValue(remote, rule.Values) {
+			return true
 		}
 	}
-	return "", false
+	return false
 }
 
 // matchesAllowedValue checks if the remote value matches one of the allowed JSON values.

bundle/direct/bundle_plan.go

@@ -3,8 +3,12 @@ package direct
 import (
 	"testing"
 
+	"github.com/databricks/cli/bundle/deployplan"
+	"github.com/databricks/cli/bundle/direct/dresources"
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestDynPathToStructPath(t *testing.T) {
@@ -35,3 +39,97 @@ func TestDynPathToStructPath(t *testing.T) {
 		assert.Equal(t, tc.expected, node.String())
 	}
 }
+
+func TestShouldSkipBackendDefault_ManagedPropertiesOnly(t *testing.T) {
+	// Rules mirror the schemas backend_defaults in resources.yml, but the test is
+	// deliberately self-contained so that edits to resources.yml don't break it.
+	// The real wiring is covered by acceptance/bundle/resources/schemas/drift.
+	rowTracking, err := structpath.ParsePattern("properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking']")
+	require.NoError(t, err)
+	catalogManaged, err := structpath.ParsePattern("properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged']")
+	require.NoError(t, err)
+	cfg := &dresources.ResourceLifecycleConfig{
+		BackendDefaults: []dresources.BackendDefaultRule{
+			{Field: rowTracking},
+			{Field: catalogManaged},
+		},
+	}
+
+	tests := []struct {
+		name     string
+		path     string
+		remote   any
+		expected bool
+	}{
+		{
+			name:     "managed delta row tracking property",
+			path:     "properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking']",
+			remote:   "true",
+			expected: true,
+		},
+		{
+			name:     "managed iceberg catalog property",
+			path:     "properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged']",
+			remote:   "true",
+			expected: true,
+		},
+		{
+			name:     "unmanaged remote-only property is not skipped",
+			path:     "properties['custom.remote_only']",
+			remote:   "true",
+			expected: false,
+		},
+		{
+			name:     "managed-only parent properties map is skipped",
+			path:     "properties",
+			remote:   map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true"},
+			expected: true,
+		},
+		{
+			name:     "mixed parent properties map is not skipped",
+			path:     "properties",
+			remote:   map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", "custom.remote_only": "true"},
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			path, err := structpath.ParsePath(tt.path)
+			require.NoError(t, err)
+
+			reason, ok := shouldSkipBackendDefault(cfg, path, &deployplan.ChangeDesc{
+				Old:    nil,
+				New:    nil,
+				Remote: tt.remote,
+			})
+
+			assert.Equal(t, tt.expected, ok)
+			if tt.expected {
+				assert.Equal(t, deployplan.ReasonBackendDefault, reason)
+			} else {
+				assert.Empty(t, reason)
+			}
+		})
+	}
+}
+
+// Map drift handling synthesizes child paths to match against rules. structdiff
+// always emits map keys in bracket notation, so synthetic child paths must too;
+// otherwise rules wouldn't match for identifier-like keys.
+func TestShouldSkipBackendDefault_MapDriftUsesBracketKeys(t *testing.T) {
+	field, err := structpath.ParsePattern("properties['simple']")
+	require.NoError(t, err)
+	cfg := &dresources.ResourceLifecycleConfig{
+		BackendDefaults: []dresources.BackendDefaultRule{{Field: field}},
+	}
+
+	path, err := structpath.ParsePath("properties")
+	require.NoError(t, err)
+
+	reason, ok := shouldSkipBackendDefault(cfg, path, &deployplan.ChangeDesc{
+		Remote: map[string]string{"simple": "v"},
+	})
+	assert.True(t, ok)
+	assert.Equal(t, deployplan.ReasonBackendDefault, reason)
+}

bundle/direct/bundle_plan_test.go

@@ -320,6 +320,12 @@ resources:
     normalize_slash:
       - field: storage_root
         reason: uc_strips_trailing_slash
+    backend_defaults:
+      # UC auto-populates these system-managed keys after create.
+      # Without this, every subsequent plan produces an Update whose payload is empty,
+      # and UC rejects it with "UpdateSchema Nothing to update".
+      - field: properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking']
+      - field: properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged']
 
   external_locations:
     recreate_on_changes:

bundle/direct/dresources/resources.yml

@@ -12,6 +12,12 @@ import (
 
 const testMetastoreName = "deco-uc-prod-isolated-aws-us-east-1"
 
+// schemaNameManagedDefaults is the schema name the backend-default drift test uses
+// to opt into UC's managed-property simulation. Scoping the injection to this name
+// keeps unrelated schema tests free of the property, which terraform would otherwise
+// report as drift on redeploy.
+const schemaNameManagedDefaults = "schema_managed_defaults"
+
 func (s *FakeWorkspace) SchemasCreate(req Request) Response {
 	defer s.LockUnlock()()
 
@@ -42,6 +48,14 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response {
 	schema.MetastoreId = TestMetastore.MetastoreId
 	schema.Owner = s.CurrentUser().UserName
 	schema.SchemaId = nextUUID()
+	if schema.Properties == nil && schema.Name == schemaNameManagedDefaults {
+		// Mirror UC behavior: managed system defaults are populated when the user
+		// doesn't specify any properties. Required to cover backend-default drift.
+		schema.Properties = map[string]string{
+			"unity.catalog.managed.delta.defaults.delta.enableRowTracking":        "true",
+			"unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged": "true",
+		}
+	}
 	s.Schemas[schema.FullName] = schema
 
 	return Response{