Feature/resource lookup and deployment

.github/workflows/ci.yml

@@ -62,7 +62,7 @@ jobs:
         enable-cache: true
     - uses: databricks/setup-cli@main
       with:
-        version: 0.279.0
+        version: 0.280.0
     - name: Install the project
       run: uv sync --locked --all-extras
     - name: Install Databricks Connect

.github/workflows/deploy-test.yml

@@ -32,7 +32,7 @@ jobs:
         enable-cache: true
     - uses: databricks/setup-cli@main
       with:
-        version: 0.279.0
+        version: 0.280.0
     - name: Install the project
       run: uv sync --locked --all-extras
     - name: Check Databricks CLI

.github/workflows/release.yml

@@ -30,7 +30,7 @@ jobs:
         enable-cache: true
     - uses: databricks/setup-cli@main
       with:
-        version: 0.279.0
+        version: 0.280.0
     - name: Install the project
       run: uv sync --locked --all-extras
     - name: Check Databricks CLI

.github/workflows/validate-bundle.yml

@@ -31,7 +31,7 @@ jobs:
         enable-cache: true
     - uses: databricks/setup-cli@main
       with:
-        version: 0.279.0
+        version: 0.280.0
     - name: Install the project
       run: uv sync --locked --all-extras
     - name: Check Databricks CLI

README.md

@@ -41,7 +41,7 @@ A script exists set up the (Free) Workspace as described in [scripts/setup_works
 * uv: https://docs.astral.sh/uv/getting-started/installation/
   * `uv` will default to Python version specified in [.python-version](.python-version)
 * Databricks CLI: https://docs.databricks.com/aws/en/dev-tools/cli/install
-  * ">=0.259.0" for Python based workflows with `environment_version`
+  * ">=0.269.0" due to 'lifecycle prevent_destroy'
 
 ### Setup environment
 
@@ -156,7 +156,6 @@ uv run ./scripts/setup_workspace.py
 
 ## TODO:
 
-* Resources (volume, schema, permissions)
 * Streaming example
 * Logging
   * Logging to volume

databricks.yml

@@ -2,16 +2,21 @@
 # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
 bundle:
   name: dab_project
-  databricks_cli_version: ">=0.259.0"
+  databricks_cli_version: ">=0.269.0"
 
 variables:
   catalog_name:
-    description: The name of the catalog to use for this bundle.
+    description: The name of the catalog to use for this bundle (set in targets).
+  service_principal_id:
+    description: The service principal ID to use for running jobs (auto-lookup).
   serverless_environment_version:
     description: The serverless environment to use for this bundle.
     default: "4"
   dbt_sql_warehouse_id:
-    description: The ID of the SQL warehouse to use for dbt tasks.
+    description: The ID of the SQL warehouse to use for dbt tasks (auto-lookup).
+    # Lookup the warehouse ID by name.
+    lookup:
+      warehouse: "Serverless Starter Warehouse"
 
 include:
   - resources/*.yml
@@ -67,7 +72,7 @@ targets:
       trigger_pause_status: PAUSED
     variables:
       catalog_name: "lake_dev"
-      dbt_sql_warehouse_id: "c31436beb8eee93c"
+      service_principal_id: "unused for dev target"
     workspace:
       host: https://dbc-ea4bb8e5-6935.cloud.databricks.com
       root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}
@@ -79,22 +84,28 @@ targets:
       trigger_pause_status: UNPAUSED
     variables:
       catalog_name: "lake_test"
-      dbt_sql_warehouse_id: "c31436beb8eee93c"
+      service_principal_id:
+        # lookup the dev service principal ID by name
+        lookup:
+          service_principal: "sp_etl_dev"
     workspace:
       host: https://dbc-ea4bb8e5-6935.cloud.databricks.com
       root_path: /Workspace/bundle/${bundle.name}/${bundle.target}
     run_as:
-      service_principal_name: 280a0e2e-369a-440f-8bf1-8da8c975e077
+      service_principal_name: ${var.service_principal_id}
 
   prod:
     mode: production
     presets:
       trigger_pause_status: UNPAUSED
     variables:
       catalog_name: "lake_prod"
-      dbt_sql_warehouse_id: "c31436beb8eee93c"
+      service_principal_id:
+        # lookup the prod service principal ID by name
+        lookup:
+          service_principal: "sp_etl_prod"
     workspace:
       host: https://dbc-ea4bb8e5-6935.cloud.databricks.com
       root_path: /Workspace/bundle/${bundle.name}/${bundle.target}
     run_as:
-      service_principal_name: 255b38e1-a8ec-40cf-8e27-e640276bef5d
+      service_principal_name: ${var.service_principal_id}

pyproject.toml

@@ -32,7 +32,7 @@ dbt = [
 ]
 # Development & Testing
 dev = [
-    "databricks-bundles==0.279.*", # For Python-based Workflows
+    "databricks-bundles==0.280.*", # For Python-based Workflows
     "mypy", # Type hints
     "pip", # Databricks extension needs it
     "pytest", # Unit testing

resources/schemas.yml

@@ -0,0 +1,33 @@
+resources:
+  schemas:
+    # The pipeline would create schemas automatically, so this is more of a proof of concept
+    silver_schema:
+      name: silver
+      catalog_name: ${var.catalog_name}
+      comment: "Silver schema for cleaned and processed data"
+      lifecycle:
+        prevent_destroy: true
+      grants:
+        - principal: group_etl
+          privileges:
+            - ALL_PRIVILEGES
+        - principal: group_reader
+          privileges:
+            - USE_SCHEMA
+            - SELECT
+
+    gold_schema:
+      name: gold
+      catalog_name: ${var.catalog_name}
+      comment: "Gold schema for aggregated and business-ready data"
+      lifecycle:
+        prevent_destroy: true
+      grants:
+        - principal: group_etl
+          privileges:
+            - ALL_PRIVILEGES
+        - principal: group_reader
+          privileges:
+            - USE_SCHEMA
+            - SELECT
+

resources/volumes.yml

@@ -1,13 +1,32 @@
 resources:
   volumes:
-    checkpoints_volume:
+    # These volumes are unused and exist only as proof of concept
+    silver_checkpoints_volume:
       name: checkpoints
       catalog_name: ${var.catalog_name}
-      schema_name: default
-      # grants:
-      #   - privileges:
-      #       - ALL PRIVILEGES
-      #     principal: group_etl
-      #   - privileges:
-      #       - READ VOLUME
-      #     principal: group_reader
+      schema_name: silver
+      comment: "Silver volume for Spark Streaming checkpoints"
+      lifecycle:
+        prevent_destroy: true
+      grants:
+        - principal: group_etl
+          privileges:
+            - ALL_PRIVILEGES
+        - principal: group_reader
+          privileges:
+            - READ_VOLUME
+
+    gold_checkpoints_volume:
+      name: checkpoints
+      catalog_name: ${var.catalog_name}
+      schema_name: gold
+      comment: "Gold volume for Spark Streaming checkpoints"
+      lifecycle:
+        prevent_destroy: true
+      grants:
+        - principal: group_etl
+          privileges:
+            - ALL_PRIVILEGES
+        - principal: group_reader
+          privileges:
+            - READ_VOLUME