From 78f77fb5559fa902e1753deedf4ce6d16ad2fea3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 3 Jun 2026 00:04:01 +0200 Subject: [PATCH 1/5] bundle: skip edition/channel defaults for ingestion pipelines Ingestion pipelines (those with `ingestion_definition`) use serverless compute by default. The Databricks API rejects `edition` and `channel` fields as "cluster settings" when a pipeline runs on serverless compute. Previously, DABs unconditionally set `edition: ADVANCED` and `channel: CURRENT` as defaults for all pipeline types. This caused creation of LakeFlow Connect ingestion pipelines to fail with: "cannot create pipeline: You cannot provide cluster settings when using serverless compute" The fix conditionally applies these defaults only to pipelines without `ingestion_definition`. The test `change-ingestion-definition` is restructured to use per-engine output files since the TF provider independently adds these defaults for all pipeline types. Co-authored-by: Shreyas Goenka --- .../out.pipeline_get.direct.json | 31 ++++++ .../out.pipeline_get.terraform.json | 33 ++++++ .../out.plan_create.direct.json | 2 - .../out.plan_recreate.direct.json | 4 - .../out.requests.direct.json | 4 + .../out.requests.terraform.json | 4 + .../change-ingestion-definition/output.txt | 101 ------------------ .../change-ingestion-definition/script | 38 ++++++- .../validate/pipeline_defaults/databricks.yml | 13 +++ .../validate/pipeline_defaults/out.test.toml | 3 + .../validate/pipeline_defaults/output.txt | 10 ++ .../bundle/validate/pipeline_defaults/script | 2 + .../resourcemutator/resource_mutator.go | 39 ++++++- 13 files changed, 171 insertions(+), 113 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json create mode 100644 acceptance/bundle/validate/pipeline_defaults/databricks.yml create mode 100644 acceptance/bundle/validate/pipeline_defaults/out.test.toml create mode 100644 acceptance/bundle/validate/pipeline_defaults/output.txt create mode 100644 acceptance/bundle/validate/pipeline_defaults/script diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json new file mode 100644 index 00000000000..c7e9105acbd --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json @@ -0,0 +1,31 @@ +{ + "creator_user_name": "[USERNAME]", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "last_modified": [UNIX_TIME_MILLIS], + "name": "test-pipeline-[UNIQUE_NAME]", + "pipeline_id": "[MY_ID_2]", + "run_as_user_name": "[USERNAME]", + "spec": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "id": "[MY_ID_2]", + "ingestion_definition": { + "connection_name": "my_new_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "storage": "dbfs:/pipelines/[MY_ID_2]" + }, + "state": "IDLE" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json new file mode 100644 index 00000000000..c1f35c46790 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json @@ -0,0 +1,33 @@ +{ + "creator_user_name": "[USERNAME]", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "last_modified": [UNIX_TIME_MILLIS], + "name": "test-pipeline-[UNIQUE_NAME]", + "pipeline_id": "[MY_ID_2]", + "run_as_user_name": "[USERNAME]", + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[MY_ID_2]", + "ingestion_definition": { + "connection_name": "my_new_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "storage": "dbfs:/pipelines/[MY_ID_2]" + }, + "state": "IDLE" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json index bd540374432..5f4b9774b2c 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json @@ -6,12 +6,10 @@ "action": "create", "new_state": { "value": { - "channel": "CURRENT", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" }, - "edition": "ADVANCED", "ingestion_definition": { "connection_name": "my_connection", "objects": [ diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json index ee6b7940b47..3cc42d243fe 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json @@ -8,12 +8,10 @@ "action": "recreate", "new_state": { "value": { - "channel": "CURRENT", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" }, - "edition": "ADVANCED", "ingestion_definition": { "connection_name": "my_new_connection", "objects": [ @@ -31,13 +29,11 @@ } }, "remote_state": { - "channel": "CURRENT", "creator_user_name": "[USERNAME]", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" }, - "edition": "ADVANCED", "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", "id": "[MY_ID]", "ingestion_definition": { diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json new file mode 100644 index 00000000000..ab7669a5739 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json @@ -0,0 +1,4 @@ +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[MY_ID_2]" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json new file mode 100644 index 00000000000..ab7669a5739 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json @@ -0,0 +1,4 @@ +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[MY_ID_2]" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt index 77bfdc09d32..ddfedf361dc 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt @@ -25,34 +25,6 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests -{ - "body": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edition": "ADVANCED", - "ingestion_definition": { - "connection_name": "my_connection", - "objects": [ - {} - ] - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" - } - } - ], - "name": "test-pipeline-[UNIQUE_NAME]" - }, - "method": "POST", - "path": "/api/2.0/pipelines" -} - >>> update_file.py databricks.yml my_connection my_new_connection >>> [CLI] bundle plan @@ -72,74 +44,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests -{ - "method": "DELETE", - "path": "/api/2.0/pipelines/[MY_ID]" -} -{ - "body": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edition": "ADVANCED", - "ingestion_definition": { - "connection_name": "my_new_connection", - "objects": [ - {} - ] - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" - } - } - ], - "name": "test-pipeline-[UNIQUE_NAME]" - }, - "method": "POST", - "path": "/api/2.0/pipelines" -} - === Fetch pipeline ID and verify remote state ->>> [CLI] pipelines get [MY_ID_2] -{ - "creator_user_name": "[USERNAME]", - "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", - "last_modified": [UNIX_TIME_MILLIS], - "name": "test-pipeline-[UNIQUE_NAME]", - "pipeline_id": "[MY_ID_2]", - "run_as_user_name": "[USERNAME]", - "spec": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edition": "ADVANCED", - "id": "[MY_ID_2]", - "ingestion_definition": { - "connection_name": "my_new_connection", - "objects": [ - {} - ] - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" - } - } - ], - "name": "test-pipeline-[UNIQUE_NAME]", - "storage": "dbfs:/pipelines/[MY_ID_2]" - }, - "state": "IDLE" -} - === Verify that original pipeline is gone >>> musterr [CLI] pipelines get [MY_ID] Error: The specified pipeline [MY_ID] was not found. @@ -158,11 +63,5 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests -{ - "method": "DELETE", - "path": "/api/2.0/pipelines/[MY_ID_2]" -} - >>> musterr [CLI] pipelines get [MY_ID_2] Error: The specified pipeline [MY_ID_2] was not found. diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script index 82de4115550..d2ef70cb663 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script @@ -1,2 +1,38 @@ envsubst < $TESTDIR/../databricks.yml.tmpl > databricks.yml -source $TESTDIR/../_script + +trace cat databricks.yml +touch foo.py +touch bar.py +trace $CLI bundle plan # should show 'create' +$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy + +ppid1=`read_id.py my` + +print_requests() { + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt > out.requests.$DATABRICKS_BUNDLE_ENGINE.json + rm -f out.requests.txt +} + +print_requests + +trace update_file.py databricks.yml $CONFIG_UPDATE +trace $CLI bundle plan # should show 'recreate' +$CLI bundle plan -o json > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy --auto-approve +print_requests + +title "Fetch pipeline ID and verify remote state" + +ppid2=`read_id.py my` +$CLI pipelines get $ppid2 > out.pipeline_get.$DATABRICKS_BUNDLE_ENGINE.json + +title "Verify that original pipeline is gone" +trace musterr $CLI pipelines get $ppid1 + +title "Destroy the pipeline and verify that it's removed from the state and from remote" +trace $CLI bundle destroy --auto-approve + +print_requests +trace musterr $CLI pipelines get $ppid2 +rm -f out.requests.txt diff --git a/acceptance/bundle/validate/pipeline_defaults/databricks.yml b/acceptance/bundle/validate/pipeline_defaults/databricks.yml new file mode 100644 index 00000000000..58b04c996e7 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + pipelines: + regular: + name: regular-pipeline + + ingestion: + name: ingestion-pipeline + ingestion_definition: + connection_name: myconn + objects: [{}] diff --git a/acceptance/bundle/validate/pipeline_defaults/out.test.toml b/acceptance/bundle/validate/pipeline_defaults/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/pipeline_defaults/output.txt b/acceptance/bundle/validate/pipeline_defaults/output.txt new file mode 100644 index 00000000000..cf157159348 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/output.txt @@ -0,0 +1,10 @@ +{ + "regular": { + "edition": "ADVANCED", + "channel": "CURRENT" + }, + "ingestion": { + "edition": null, + "channel": null + } +} diff --git a/acceptance/bundle/validate/pipeline_defaults/script b/acceptance/bundle/validate/pipeline_defaults/script new file mode 100644 index 00000000000..5806946a066 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/script @@ -0,0 +1,2 @@ +# edition and channel defaults should be set for regular pipelines but not for ingestion pipelines +$CLI bundle validate -o json | jq '{regular: .resources.pipelines.regular | {edition, channel}, ingestion: .resources.pipelines.ingestion | {edition, channel}}' diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 209bbcb06a0..3212c1c75af 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -80,11 +80,6 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks", true}, {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs", true}, - // Pipelines (same as terraform) - // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 - {"resources.pipelines.*.edition", "ADVANCED"}, - {"resources.pipelines.*.channel", "CURRENT"}, - // SqlWarehouses (same as terraform) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/sql/resource_sql_endpoint.go#L59 {"resources.sql_warehouses.*.auto_stop_mins", 120}, @@ -109,6 +104,29 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { } } + // Pipelines: edition and channel defaults are only applied to non-ingestion + // pipelines. Ingestion pipelines (those with ingestion_definition) use + // serverless compute by default and the API rejects these fields in that mode. + // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 + if err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(root, + dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey()), + func(_ dyn.Path, p dyn.Value) (dyn.Value, error) { + if _, err := dyn.GetByPath(p, dyn.NewPath(dyn.Key("ingestion_definition"))); err == nil { + return p, nil + } + var err error + if p, err = setPipelineDefault(p, "edition", "ADVANCED"); err != nil { + return dyn.InvalidValue, err + } + return setPipelineDefault(p, "channel", "CURRENT") + }, + ) + }); err != nil { + logdiag.LogError(ctx, err) + return + } + bundle.ApplySeqContext(ctx, b, // Reads (typed): b.Config.Resources.Dashboards (checks dashboard configurations) // Updates (typed): b.Config.Resources.Dashboards[].ParentPath (ensures /Workspace prefix is present) @@ -395,3 +413,14 @@ func mergeResources(src, dst dyn.Value) (dyn.Value, error) { return newDst, nil } + +// setPipelineDefault sets key to value on the pipeline dyn.Value only if key is not already set. +func setPipelineDefault(v dyn.Value, key string, value any) (dyn.Value, error) { + path := dyn.NewPath(dyn.Key(key)) + switch _, err := dyn.GetByPath(v, path); { + case dyn.IsNoSuchKeyError(err): + return dyn.SetByPath(v, path, dyn.V(value)) + default: + return v, err + } +} From 480816096fe3991dbc3c8d050646e1be1f48c850 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 4 Jun 2026 10:07:57 +0200 Subject: [PATCH 2/5] bundle: simplify pipeline defaults; add cloud regression test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the dyn.Value pattern walk with a direct loop over b.Config.Resources.Pipelines to conditionally apply edition/channel defaults. Also remove the now-unused setPipelineDefault helper. Add acceptance/bundle/resources/pipelines/ingestion-defaults, a cloud test (direct engine) that deploys an ingestion pipeline and asserts edition and channel are absent from the POST /api/2.0/pipelines request body. The deploy fails with "connection not found" — not "cluster settings" — confirming the fix. Co-authored-by: Shreyas Goenka --- .../ingestion-defaults/databricks.yml.tmpl | 22 +++++++++ .../ingestion-defaults/out.test.toml | 3 ++ .../pipelines/ingestion-defaults/output.txt | 20 ++++++++ .../pipelines/ingestion-defaults/script | 11 +++++ .../pipelines/ingestion-defaults/test.toml | 13 ++++++ .../resourcemutator/resource_mutator.go | 46 ++++++------------- 6 files changed, 84 insertions(+), 31 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/script create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl b/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl new file mode 100644 index 00000000000..87eebe41f7b --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-ingestion-defaults-$UNIQUE_NAME + +resources: + pipelines: + ingestion: + name: test-ingestion-$UNIQUE_NAME + catalog: main + target: default + ingestion_definition: + connection_name: does_not_exist_$UNIQUE_NAME + objects: + - table: + source_catalog: does_not_exist + source_schema: does_not_exist + source_table: does_not_exist + destination_catalog: main + destination_schema: default + +targets: + dev: + default: true diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml new file mode 100644 index 00000000000..00d03354554 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml @@ -0,0 +1,3 @@ +Local = false +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt new file mode 100644 index 00000000000..2b1f087ae92 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt @@ -0,0 +1,20 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... +Deploying resources... +Error: cannot create resources.pipelines.ingestion: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/pipelines +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. + + +Exit code: 1 + +>>> jq -s .[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel} out.requests.txt +{ + "name": "test-ingestion-[UNIQUE_NAME]", + "edition": null, + "channel": null +} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/script b/acceptance/bundle/resources/pipelines/ingestion-defaults/script new file mode 100644 index 00000000000..3bcd91458d8 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/script @@ -0,0 +1,11 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +# Deploy the ingestion pipeline. It fails because the connection name is +# intentionally invalid, but the POST /api/2.0/pipelines request is sent and +# recorded first. The key assertion: edition and channel are absent from the +# request body — DABs must not inject them for ingestion pipelines. +trace errcode $CLI bundle deploy + +# Show what was sent to the API. edition and channel must be null (absent). +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt +rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml new file mode 100644 index 00000000000..e6c0a2e7186 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml @@ -0,0 +1,13 @@ +Local = false +Cloud = true +RecordRequests = true + +Ignore = [ + "databricks.yml", +] + +# Only test with direct engine: the Terraform provider independently injects +# edition/channel defaults at the provider level regardless of what DABs sends, +# so only the direct engine cleanly shows the DABs fix. +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 3212c1c75af..90887bfa00a 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -104,27 +104,22 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { } } - // Pipelines: edition and channel defaults are only applied to non-ingestion - // pipelines. Ingestion pipelines (those with ingestion_definition) use - // serverless compute by default and the API rejects these fields in that mode. + // Apply edition and channel defaults only to non-ingestion pipelines. + // Ingestion pipelines use serverless compute by default; the API rejects + // these fields as incompatible cluster settings in that mode. // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 - if err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(root, - dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey()), - func(_ dyn.Path, p dyn.Value) (dyn.Value, error) { - if _, err := dyn.GetByPath(p, dyn.NewPath(dyn.Key("ingestion_definition"))); err == nil { - return p, nil - } - var err error - if p, err = setPipelineDefault(p, "edition", "ADVANCED"); err != nil { - return dyn.InvalidValue, err - } - return setPipelineDefault(p, "channel", "CURRENT") - }, - ) - }); err != nil { - logdiag.LogError(ctx, err) - return + for name, p := range b.Config.Resources.Pipelines { + if p == nil || p.IngestionDefinition != nil { + continue + } + bundle.SetDefault(ctx, b, "resources.pipelines."+name+".edition", "ADVANCED") + if logdiag.HasError(ctx) { + return + } + bundle.SetDefault(ctx, b, "resources.pipelines."+name+".channel", "CURRENT") + if logdiag.HasError(ctx) { + return + } } bundle.ApplySeqContext(ctx, b, @@ -413,14 +408,3 @@ func mergeResources(src, dst dyn.Value) (dyn.Value, error) { return newDst, nil } - -// setPipelineDefault sets key to value on the pipeline dyn.Value only if key is not already set. -func setPipelineDefault(v dyn.Value, key string, value any) (dyn.Value, error) { - path := dyn.NewPath(dyn.Key(key)) - switch _, err := dyn.GetByPath(v, path); { - case dyn.IsNoSuchKeyError(err): - return dyn.SetByPath(v, path, dyn.V(value)) - default: - return v, err - } -} From 9c7c82c585ef8201ba36b3f7bb42a37313da3ac4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 4 Jun 2026 14:00:54 +0200 Subject: [PATCH 3/5] bundle: add cloud regression test for ingestion pipeline defaults Extends the ingestion-defaults acceptance test to run both direct and terraform engines, showing the contrast: - Direct engine: edition/channel absent from POST (DABs fix working) - Terraform engine: edition/channel present (provider-level SetDefault, companion fix in terraform-provider-databricks#5783) Badness clause documents the known terraform engine gap. Co-authored-by: Shreyas Goenka --- .../ingestion-defaults/out.deploy.direct.txt | 13 +++++++++++++ .../out.deploy.terraform.txt | 17 +++++++++++++++++ .../out.post_body.direct.json | 5 +++++ .../out.post_body.terraform.json | 5 +++++ .../pipelines/ingestion-defaults/out.test.toml | 2 +- .../pipelines/ingestion-defaults/output.txt | 18 ------------------ .../pipelines/ingestion-defaults/script | 13 ++++++------- .../pipelines/ingestion-defaults/test.toml | 7 +++---- 8 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt new file mode 100644 index 00000000000..d17f6007753 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt @@ -0,0 +1,13 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... +Deploying resources... +Error: cannot create resources.pipelines.ingestion: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/pipelines +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. + + +Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt new file mode 100644 index 00000000000..2dfa66cb79b --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt @@ -0,0 +1,17 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... +Deploying resources... +Error: terraform apply: exit status 1 + +Error: cannot create pipeline: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. + + with databricks_pipeline.ingestion, + on bundle.tf.json line 37, in resource.databricks_pipeline.ingestion: + 37: } + + + +Updating deployment state... + +Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json new file mode 100644 index 00000000000..e96f0e7b8ac --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json @@ -0,0 +1,5 @@ +{ + "name": "test-ingestion-[UNIQUE_NAME]", + "edition": null, + "channel": null +} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json new file mode 100644 index 00000000000..3b83f878234 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json @@ -0,0 +1,5 @@ +{ + "name": "test-ingestion-[UNIQUE_NAME]", + "edition": "ADVANCED", + "channel": "CURRENT" +} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml index 00d03354554..2f97f2d67f5 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml @@ -1,3 +1,3 @@ Local = false Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt index 2b1f087ae92..2988aaa9c46 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt @@ -1,20 +1,2 @@ ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... -Deploying resources... -Error: cannot create resources.pipelines.ingestion: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. (400 INVALID_PARAMETER_VALUE) - -Endpoint: POST [DATABRICKS_URL]/api/2.0/pipelines -HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. - - -Exit code: 1 - >>> jq -s .[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel} out.requests.txt -{ - "name": "test-ingestion-[UNIQUE_NAME]", - "edition": null, - "channel": null -} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/script b/acceptance/bundle/resources/pipelines/ingestion-defaults/script index 3bcd91458d8..ffb6283b71b 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/script +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/script @@ -1,11 +1,10 @@ envsubst < databricks.yml.tmpl > databricks.yml -# Deploy the ingestion pipeline. It fails because the connection name is -# intentionally invalid, but the POST /api/2.0/pipelines request is sent and -# recorded first. The key assertion: edition and channel are absent from the -# request body — DABs must not inject them for ingestion pipelines. -trace errcode $CLI bundle deploy +# Deploy fails (invalid connection) but the POST /api/2.0/pipelines request is +# sent and recorded. Direct engine: edition and channel are absent (the fix). +# Terraform engine: edition and channel are present (provider injects them; being +# fixed in terraform-provider-databricks). +trace errcode $CLI bundle deploy > out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 -# Show what was sent to the API. edition and channel must be null (absent). -trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt > out.post_body.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml index e6c0a2e7186..f2cdc3e6cb3 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml @@ -2,12 +2,11 @@ Local = false Cloud = true RecordRequests = true +Badness = "Terraform engine still injects edition/channel via provider-level SetDefault (resource_pipeline.go:253); companion fix: terraform-provider-databricks#5783." + Ignore = [ "databricks.yml", ] -# Only test with direct engine: the Terraform provider independently injects -# edition/channel defaults at the provider level regardless of what DABs sends, -# so only the direct engine cleanly shows the DABs fix. [EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = ["direct"] +DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] From 2662ed8f7c60f8d4b8a88a441593c2d69c611f64 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 4 Jun 2026 19:52:36 +0200 Subject: [PATCH 4/5] bundle: add cloud regression test for ingestion pipeline defaults Replaces the error-capture test with a real end-to-end deployment on the azure cloud environment using a MySQL ingestion connection. The test: - Deploys an ingestion pipeline via the direct engine - Asserts deployment succeeds (no "cluster settings" error) - Verifies edition and channel are absent from the POST request body - Destroys the pipeline on exit Also confirms classic compute (serverless: false) works the same way. The Terraform-engine gap (provider-level SetDefault still injects edition/channel) is documented in Badness and fixed in terraform-provider-databricks#5783. Co-authored-by: Shreyas Goenka --- .../ingestion-defaults/databricks.yml.tmpl | 10 ++-------- .../ingestion-defaults/out.deploy.direct.txt | 13 ------------- .../out.deploy.terraform.txt | 17 ----------------- .../out.post_body.direct.json | 5 ----- .../out.post_body.terraform.json | 5 ----- .../ingestion-defaults/out.test.toml | 3 ++- .../pipelines/ingestion-defaults/output.txt | 11 +++++++++++ .../pipelines/ingestion-defaults/script | 19 ++++++++++++------- .../pipelines/ingestion-defaults/test.toml | 16 ++++++++++------ 9 files changed, 37 insertions(+), 62 deletions(-) delete mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt delete mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt delete mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json delete mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl b/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl index 87eebe41f7b..17ebdffeefd 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl @@ -8,14 +8,8 @@ resources: catalog: main target: default ingestion_definition: - connection_name: does_not_exist_$UNIQUE_NAME - objects: - - table: - source_catalog: does_not_exist - source_schema: does_not_exist - source_table: does_not_exist - destination_catalog: main - destination_schema: default + connection_name: $INGESTION_CONNECTION_NAME + objects: [] targets: dev: diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt deleted file mode 100644 index d17f6007753..00000000000 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt +++ /dev/null @@ -1,13 +0,0 @@ - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... -Deploying resources... -Error: cannot create resources.pipelines.ingestion: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. (400 INVALID_PARAMETER_VALUE) - -Endpoint: POST [DATABRICKS_URL]/api/2.0/pipelines -HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. - - -Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt deleted file mode 100644 index 2dfa66cb79b..00000000000 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt +++ /dev/null @@ -1,17 +0,0 @@ - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... -Deploying resources... -Error: terraform apply: exit status 1 - -Error: cannot create pipeline: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. - - with databricks_pipeline.ingestion, - on bundle.tf.json line 37, in resource.databricks_pipeline.ingestion: - 37: } - - - -Updating deployment state... - -Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json deleted file mode 100644 index e96f0e7b8ac..00000000000 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "test-ingestion-[UNIQUE_NAME]", - "edition": null, - "channel": null -} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json deleted file mode 100644 index 3b83f878234..00000000000 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "test-ingestion-[UNIQUE_NAME]", - "edition": "ADVANCED", - "channel": "CURRENT" -} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml index 2f97f2d67f5..b4b1fb39d67 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml @@ -1,3 +1,4 @@ Local = false Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +CloudEnvs.azure = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt index 2988aaa9c46..bf202db9226 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt @@ -1,2 +1,13 @@ +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + >>> jq -s .[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel} out.requests.txt +{ + "name": "test-ingestion-[UNIQUE_NAME]", + "edition": null, + "channel": null +} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/script b/acceptance/bundle/resources/pipelines/ingestion-defaults/script index ffb6283b71b..3d9a068be18 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/script +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/script @@ -1,10 +1,15 @@ envsubst < databricks.yml.tmpl > databricks.yml -# Deploy fails (invalid connection) but the POST /api/2.0/pipelines request is -# sent and recorded. Direct engine: edition and channel are absent (the fix). -# Terraform engine: edition and channel are present (provider injects them; being -# fixed in terraform-provider-databricks). -trace errcode $CLI bundle deploy > out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +cleanup() { + trace $CLI bundle destroy --auto-approve 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT -trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt > out.post_body.$DATABRICKS_BUNDLE_ENGINE.json -rm out.requests.txt +# Deploy the ingestion pipeline. Verifies that: +# 1. Deployment succeeds without "cluster settings" errors +# 2. edition and channel are absent from the created pipeline's spec +trace $CLI bundle deploy + +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml index f2cdc3e6cb3..c287a956cf9 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml @@ -2,11 +2,15 @@ Local = false Cloud = true RecordRequests = true -Badness = "Terraform engine still injects edition/channel via provider-level SetDefault (resource_pipeline.go:253); companion fix: terraform-provider-databricks#5783." +# The Terraform engine still injects edition/channel at provider level (SetDefault in +# resource_pipeline.go:253); companion fix: terraform-provider-databricks#5783. +# Only test with direct engine here so we cleanly validate the DABs-side fix. +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] -Ignore = [ - "databricks.yml", -] +[Env] +# MySQL connection present in the azure test workspace. +INGESTION_CONNECTION_NAME = "name-sbggefbaaidkb" -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +[CloudEnvs] +azure = true From a8f3a474a51dbfe713a1ed7ddfd4c651e3d23c95 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 12 Jun 2026 13:08:09 +0200 Subject: [PATCH 5/5] bundle: record per-phase pipeline requests; build defaults pattern programmatically The change-ingestion-definition test now records create and recreate requests per engine, which documents the terraform provider injecting edition/channel for ingestion pipelines (fix: terraform-provider-databricks#5783). The destroy-phase DELETE is identical across engines, so it is recorded once. Also build the pipeline defaults pattern with dyn.NewPattern instead of string concatenation, which removes the pattern parse and its error paths. Co-authored-by: Isaac --- .../out.requests.create.direct.json | 24 +++++++++++++++ .../out.requests.create.terraform.json | 26 ++++++++++++++++ ....direct.json => out.requests.destroy.json} | 0 .../out.requests.recreate.direct.json | 28 +++++++++++++++++ .../out.requests.recreate.terraform.json | 30 +++++++++++++++++++ .../out.requests.terraform.json | 4 --- .../change-ingestion-definition/script | 11 ++++--- .../change-ingestion-definition/test.toml | 2 ++ .../resourcemutator/resource_mutator.go | 5 ++-- 9 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.create.direct.json create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.create.terraform.json rename acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/{out.requests.direct.json => out.requests.destroy.json} (100%) create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.recreate.direct.json create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.recreate.terraform.json delete mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.create.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.create.direct.json new file mode 100644 index 00000000000..cfbb660bca5 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.create.direct.json @@ -0,0 +1,24 @@ +{ + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "ingestion_definition": { + "connection_name": "my_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]" + }, + "method": "POST", + "path": "/api/2.0/pipelines" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.create.terraform.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.create.terraform.json new file mode 100644 index 00000000000..25a4650c941 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.create.terraform.json @@ -0,0 +1,26 @@ +{ + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "ingestion_definition": { + "connection_name": "my_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]" + }, + "method": "POST", + "path": "/api/2.0/pipelines" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.destroy.json similarity index 100% rename from acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json rename to acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.destroy.json diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.recreate.direct.json new file mode 100644 index 00000000000..a6d53433213 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.recreate.direct.json @@ -0,0 +1,28 @@ +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[MY_ID]" +} +{ + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "ingestion_definition": { + "connection_name": "my_new_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]" + }, + "method": "POST", + "path": "/api/2.0/pipelines" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.recreate.terraform.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.recreate.terraform.json new file mode 100644 index 00000000000..5f751d5f33b --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.recreate.terraform.json @@ -0,0 +1,30 @@ +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[MY_ID]" +} +{ + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "ingestion_definition": { + "connection_name": "my_new_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]" + }, + "method": "POST", + "path": "/api/2.0/pipelines" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json deleted file mode 100644 index ab7669a5739..00000000000 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "method": "DELETE", - "path": "/api/2.0/pipelines/[MY_ID_2]" -} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script index d2ef70cb663..0e5e3259866 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script @@ -9,18 +9,21 @@ trace $CLI bundle deploy ppid1=`read_id.py my` +# Requests from the create and recreate phases are recorded per engine: DABs +# omits edition/channel for ingestion pipelines, but the terraform provider +# injects them on its own (fix: terraform-provider-databricks#5783). print_requests() { - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt > out.requests.$DATABRICKS_BUNDLE_ENGINE.json + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt > "$1" rm -f out.requests.txt } -print_requests +print_requests out.requests.create.$DATABRICKS_BUNDLE_ENGINE.json trace update_file.py databricks.yml $CONFIG_UPDATE trace $CLI bundle plan # should show 'recreate' $CLI bundle plan -o json > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve -print_requests +print_requests out.requests.recreate.$DATABRICKS_BUNDLE_ENGINE.json title "Fetch pipeline ID and verify remote state" @@ -33,6 +36,6 @@ trace musterr $CLI pipelines get $ppid1 title "Destroy the pipeline and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -print_requests +print_requests out.requests.destroy.json trace musterr $CLI pipelines get $ppid2 rm -f out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/test.toml index f6226048e57..b2ac9db3927 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/test.toml @@ -1,2 +1,4 @@ +Badness = "The create request diverges between engines: the terraform provider injects edition/channel defaults for ingestion pipelines on its own. Fix: https://github.com/databricks/terraform-provider-databricks/pull/5783" + Env.CONFIG_UPDATE = "my_connection my_new_connection" Env.ING_KEY = "ingestion_definition" diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 90887bfa00a..b1c25977c00 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -112,11 +112,12 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { if p == nil || p.IngestionDefinition != nil { continue } - bundle.SetDefault(ctx, b, "resources.pipelines."+name+".edition", "ADVANCED") + pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(name)) + bundle.ApplyContext(ctx, b, bundle.SetDefaultMutator(pattern, "edition", "ADVANCED")) if logdiag.HasError(ctx) { return } - bundle.SetDefault(ctx, b, "resources.pipelines."+name+".channel", "CURRENT") + bundle.ApplyContext(ctx, b, bundle.SetDefaultMutator(pattern, "channel", "CURRENT")) if logdiag.HasError(ctx) { return }