From cb774e2b9f07fee0f86d5b6906f83d37da77e46f Mon Sep 17 00:00:00 2001 From: Anand Pant Date: Tue, 21 Apr 2026 07:07:08 -0500 Subject: [PATCH 1/5] feat: add generated databricks delta autocdc pipeline --- .depot/workflows/release-rc.yml | 2 +- .depot/workflows/release.yml | 2 +- README.md | 5 + docs/monitoring.md | 26 ++--- justfile | 9 ++ platform/databricks/delta/README.md | 5 + platform/databricks/delta/databricks.yml | 16 +++ .../databricks/delta/generated/.gitignore | 1 + .../convex_delta_autocdc.pipeline.yml | 13 +++ scripts/deploy-databricks-delta-pipeline.sh | 82 ++++++++++++++ scripts/render-databricks-delta-pipeline.sh | 102 ++++++++++++++++++ scripts/run-databricks-delta-pipeline.sh | 81 ++++++++++++++ sources/meshix-api/delta-pipeline.json | 6 ++ 13 files changed, 336 insertions(+), 14 deletions(-) create mode 100644 platform/databricks/delta/generated/.gitignore create mode 100644 platform/databricks/delta/resources/convex_delta_autocdc.pipeline.yml create mode 100755 scripts/deploy-databricks-delta-pipeline.sh create mode 100755 scripts/render-databricks-delta-pipeline.sh create mode 100755 scripts/run-databricks-delta-pipeline.sh create mode 100644 sources/meshix-api/delta-pipeline.json diff --git a/.depot/workflows/release-rc.yml b/.depot/workflows/release-rc.yml index ac5ddc0..5359e2b 100644 --- a/.depot/workflows/release-rc.yml +++ b/.depot/workflows/release-rc.yml @@ -82,7 +82,7 @@ jobs: archive="convex-sync_${tag_name}_linux_amd64" stage="dist/${archive}" mkdir -p "${stage}" - CONVEX_SYNC_VERSION="${version}" depot cargo build --locked --release -p convex-sync + CONVEX_SYNC_VERSION="${version}" depot cargo build --release -p convex-sync cp target/release/convex-sync "${stage}/convex-sync" cp LICENSE NOTICE README.md "${stage}/" tar -czf "dist/${archive}.tar.gz" -C dist "${archive}" diff --git a/.depot/workflows/release.yml b/.depot/workflows/release.yml index 7eddcd2..2548144 100644 --- a/.depot/workflows/release.yml +++ b/.depot/workflows/release.yml @@ -51,7 +51,7 @@ jobs: archive="convex-sync_${version}_linux_amd64" stage="dist/${archive}" mkdir -p "${stage}" - CONVEX_SYNC_VERSION="${version#v}" depot cargo build --locked --release -p convex-sync + CONVEX_SYNC_VERSION="${version#v}" depot cargo build --release -p convex-sync cp target/release/convex-sync "${stage}/convex-sync" cp LICENSE NOTICE README.md "${stage}/" tar -czf "dist/${archive}.tar.gz" -C dist "${archive}" diff --git a/README.md b/README.md index 300054f..b021846 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,11 @@ The Delta path creates and updates: The silver schema is expected to stay empty until you stand up a Lakeflow `AUTO CDC` pipeline for the tables you actually want to materialize there. +```bash +just databricks-delta-deploy-pipeline DEFAULT prod +just databricks-delta-run-pipeline DEFAULT prod +``` + Reference Databricks over S3 flow: ```bash diff --git a/docs/monitoring.md b/docs/monitoring.md index b10aa32..5d6ddce 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -26,10 +26,18 @@ The first dashboard focuses on: - recent checkpoint history - bronze and silver table inventory -## Why Silver Is Empty +## AUTO CDC Status -Silver is still expected to be empty until a real Lakeflow `AUTO CDC` pipeline -is deployed. +The current example source now has a real generated Lakeflow `AUTO CDC` +pipeline. Deploy and run it with: + +```bash +just databricks-delta-deploy-pipeline DEFAULT prod +just databricks-delta-run-pipeline DEFAULT prod +``` + +For a newly onboarded source, silver will stay empty until you run that same +deploy/run sequence for that source. What exists today: @@ -37,15 +45,9 @@ What exists today: - bronze CDC landing - Delta extractor job - Lakeflow SQL template for per-table `AUTO CDC` +- generated per-source pipeline scripts What is still missing: -- a source-aware pipeline generation and deploy path that turns the bronze - tables for a specific source into a real deployed Lakeflow pipeline - -The generic blocker is not Lakeflow itself. It is the missing code path that: - -1. enumerates the bronze tables for one source -2. decides the silver target names consistently -3. renders the per-table `AUTO CDC` SQL -4. deploys or updates the pipeline repeatably +- only the deploy/run step for any additional source you onboard beyond the + current example profile diff --git a/justfile b/justfile index 7ccf104..e6e7a8c 100644 --- a/justfile +++ b/justfile @@ -87,6 +87,15 @@ databricks-delta-render-dashboard output_file: databricks-delta-publish-dashboard profile warehouse_id dashboard_id="": ./scripts/publish-databricks-delta-dashboard.sh {{profile}} {{warehouse_id}} {{dashboard_id}} +databricks-delta-render-pipeline profile output_file: + ./scripts/render-databricks-delta-pipeline.sh {{profile}} {{output_file}} + +databricks-delta-deploy-pipeline profile="DEFAULT" target="prod": + ./scripts/deploy-databricks-delta-pipeline.sh {{profile}} {{target}} + +databricks-delta-run-pipeline profile="DEFAULT" target="prod": + ./scripts/run-databricks-delta-pipeline.sh {{profile}} {{target}} + databricks-delta-deploy profile="DEFAULT" target="dev": ./scripts/deploy-databricks-delta.sh {{profile}} {{target}} diff --git a/platform/databricks/delta/README.md b/platform/databricks/delta/README.md index c309de8..cbf6c6f 100644 --- a/platform/databricks/delta/README.md +++ b/platform/databricks/delta/README.md @@ -42,6 +42,9 @@ Bundle lifecycle: - `scripts/bootstrap-databricks-delta.sh ` - `scripts/render-databricks-delta-dashboard.sh ` - `scripts/publish-databricks-delta-dashboard.sh [dashboard_id]` +- `scripts/render-databricks-delta-pipeline.sh ` +- `scripts/deploy-databricks-delta-pipeline.sh ` +- `scripts/run-databricks-delta-pipeline.sh ` - `scripts/deploy-databricks-delta.sh ` - `scripts/run-databricks-delta-job.sh [job_key]` - `scripts/run-databricks-delta-smoke.sh ` @@ -109,6 +112,8 @@ Recommended operator entrypoints: just databricks-delta-sync-secret just databricks-delta-bootstrap just databricks-delta-publish-dashboard DEFAULT +just databricks-delta-deploy-pipeline DEFAULT prod +just databricks-delta-run-pipeline DEFAULT prod just databricks-delta-deploy just databricks-delta-run just databricks-delta-smoke diff --git a/platform/databricks/delta/databricks.yml b/platform/databricks/delta/databricks.yml index 05e6ea7..4e596b4 100644 --- a/platform/databricks/delta/databricks.yml +++ b/platform/databricks/delta/databricks.yml @@ -5,6 +5,10 @@ bundle: include: - resources/*.yml +sync: + include: + - generated/*.sql + variables: source_slug: description: Stable source slug used for deployment names and workspace paths. @@ -28,8 +32,14 @@ variables: description: Schema that owns the checkpoint table. bronze_schema: description: Schema that owns bronze CDC tables. + silver_schema: + description: Schema that owns silver current-state tables. checkpoint_table: description: Checkpoint table name. + autocdc_pipeline_name: + description: Databricks Lakeflow pipeline name for bronze-to-silver AUTO CDC. + autocdc_pipeline_file: + description: Generated SQL file name for the Lakeflow AUTO CDC pipeline. targets: dev: @@ -49,7 +59,10 @@ targets: catalog: workspace control_schema: convex_sync_kit_meshix_api_delta_control bronze_schema: convex_sync_kit_meshix_api_delta_bronze + silver_schema: convex_sync_kit_meshix_api_delta_silver checkpoint_table: connector_checkpoint + autocdc_pipeline_name: convex-sync-kit-meshix-api-dev-autocdc + autocdc_pipeline_file: meshix-api-dev-bronze-to-silver.sql prod: mode: production workspace: @@ -66,4 +79,7 @@ targets: catalog: workspace control_schema: convex_sync_kit_meshix_api_delta_control bronze_schema: convex_sync_kit_meshix_api_delta_bronze + silver_schema: convex_sync_kit_meshix_api_delta_silver checkpoint_table: connector_checkpoint + autocdc_pipeline_name: convex-sync-kit-meshix-api-prod-autocdc + autocdc_pipeline_file: meshix-api-prod-bronze-to-silver.sql diff --git a/platform/databricks/delta/generated/.gitignore b/platform/databricks/delta/generated/.gitignore new file mode 100644 index 0000000..d1b811b --- /dev/null +++ b/platform/databricks/delta/generated/.gitignore @@ -0,0 +1 @@ +*.sql diff --git a/platform/databricks/delta/resources/convex_delta_autocdc.pipeline.yml b/platform/databricks/delta/resources/convex_delta_autocdc.pipeline.yml new file mode 100644 index 0000000..52331db --- /dev/null +++ b/platform/databricks/delta/resources/convex_delta_autocdc.pipeline.yml @@ -0,0 +1,13 @@ +resources: + pipelines: + convex_delta_autocdc: + name: ${var.autocdc_pipeline_name} + catalog: ${var.catalog} + schema: ${var.silver_schema} + serverless: true + photon: true + continuous: false + root_path: ${workspace.file_path} + libraries: + - file: + path: ${workspace.file_path}/generated/${var.autocdc_pipeline_file} diff --git a/scripts/deploy-databricks-delta-pipeline.sh b/scripts/deploy-databricks-delta-pipeline.sh new file mode 100755 index 0000000..8ef7328 --- /dev/null +++ b/scripts/deploy-databricks-delta-pipeline.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +profile="$1" +target="$2" +bundle_engine="${DATABRICKS_BUNDLE_ENGINE:-direct}" + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +bundle_root="$repo_root/platform/databricks/delta" + +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" + +read_env_file_value() { + local key="$1" + local env_file="$repo_root/.env" + if [[ ! -f "$env_file" ]]; then + return 1 + fi + local line + line="$(grep -E "^${key}=" "$env_file" | tail -n 1 || true)" + if [[ -z "$line" ]]; then + return 1 + fi + printf '%s' "${line#*=}" +} + +deployment_url="${CONVEX_DEPLOYMENT_URL:-$(read_env_file_value CONVEX_DEPLOYMENT_URL || true)}" +if [[ -z "$deployment_url" ]]; then + echo "CONVEX_DEPLOYMENT_URL is required" >&2 + exit 1 +fi + +source_id="${CONVEX_SOURCE_ID:-$deployment_url}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" +source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" +deployment_slug="${DATABRICKS_DELTA_DEPLOYMENT_SLUG:-${source_slug}-${target}}" +pipeline_name="${DATABRICKS_DELTA_AUTODC_PIPELINE_NAME:-${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}}" +pipeline_file="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_FILE:-${deployment_slug}-bronze-to-silver.sql}" +generated_file="$bundle_root/generated/$pipeline_file" + +"$repo_root/scripts/render-databricks-delta-pipeline.sh" "$profile" "$generated_file" >/dev/null + +catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" +table_name="${CONVEX_TABLE_NAME:-}" +secret_scope="${DATABRICKS_DELTA_SECRET_SCOPE:-convex-sync-kit}" +secret_key="${DATABRICKS_DELTA_SECRET_KEY:-convex-deploy-key}" +control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" +bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" +silver_schema="${DATABRICKS_DELTA_SILVER_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_silver}" +checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" + +"$repo_root/scripts/ensure-databricks-delta-secret.sh" "$profile" "$secret_scope" "$secret_key" + +bundle_args=( + --var "convex_deployment_url=$deployment_url" + --var "source_slug=$source_slug" + --var "convex_deploy_key_secret_scope=$secret_scope" + --var "convex_deploy_key_secret_key=$secret_key" + --var "source_id=$source_id" + --var "table_name=$table_name" + --var "catalog=$catalog" + --var "control_schema=$control_schema" + --var "bronze_schema=$bronze_schema" + --var "silver_schema=$silver_schema" + --var "checkpoint_table=$checkpoint_table" + --var "autocdc_pipeline_name=$pipeline_name" + --var "autocdc_pipeline_file=$pipeline_file" + --var "deployment_slug=$deployment_slug" +) + +( + cd "$bundle_root" + DATABRICKS_BUNDLE_ENGINE="$bundle_engine" databricks bundle validate -p "$profile" -t "$target" "${bundle_args[@]}" + DATABRICKS_BUNDLE_ENGINE="$bundle_engine" databricks bundle deploy -p "$profile" -t "$target" "${bundle_args[@]}" +) diff --git a/scripts/render-databricks-delta-pipeline.sh b/scripts/render-databricks-delta-pipeline.sh new file mode 100755 index 0000000..5663e26 --- /dev/null +++ b/scripts/render-databricks-delta-pipeline.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +profile="$1" +output_file="$2" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" + +catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" +source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" +bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" +silver_schema="${DATABRICKS_DELTA_SILVER_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_silver}" +config_file="$repo_root/sources/${CONVEX_SYNC_SOURCE:-}/delta-pipeline.json" +template_file="$repo_root/platform/databricks/delta/lakeflow/bronze_to_silver_template.sql" + +tables_json="$(databricks tables list "$catalog" "$bronze_schema" -p "$profile" -o json --omit-columns --omit-properties --omit-username)" + +python - "$tables_json" "$template_file" "$output_file" "$catalog" "$bronze_schema" "$silver_schema" "$config_file" "$source_slug" <<'PY' +import json +import pathlib +import re +import sys + +tables = json.loads(sys.argv[1]) +template_path = pathlib.Path(sys.argv[2]) +output_path = pathlib.Path(sys.argv[3]) +catalog = sys.argv[4] +bronze_schema = sys.argv[5] +silver_schema = sys.argv[6] +config_path = pathlib.Path(sys.argv[7]) +source_slug = sys.argv[8] + +config = { + "include_tables": [], + "exclude_tables": [], + "target_table_overrides": {}, + "flow_name_overrides": {}, +} +if config_path.name != "delta-pipeline.json": + # no source selected; keep defaults + pass +elif config_path.exists(): + config.update(json.loads(config_path.read_text())) + +include_tables = set(config.get("include_tables") or []) +exclude_tables = set(config.get("exclude_tables") or []) +target_overrides = dict(config.get("target_table_overrides") or {}) +flow_overrides = dict(config.get("flow_name_overrides") or {}) + +def sanitize_identifier(value: str) -> str: + value = re.sub(r"[^A-Za-z0-9_]", "_", value) + if re.match(r"^[0-9]", value): + value = f"t_{value}" + return value + +def target_name_for(bronze_table: str) -> str: + if bronze_table in target_overrides: + return target_overrides[bronze_table] + if bronze_table.endswith("__cdc"): + return bronze_table[:-5] + return bronze_table + +sections = [] +for table in tables: + name = table["name"] + if table.get("table_type") == "VIEW": + continue + if not name.endswith("__cdc"): + continue + if include_tables and name not in include_tables: + continue + if name in exclude_tables: + continue + + target_name = target_name_for(name) + flow_name = flow_overrides.get(name, f"autocdc_{sanitize_identifier(target_name)}") + target_table = f"`{target_name}`" + source_table = f"`{catalog}`.`{bronze_schema}`.`{name}`" + rendered = ( + template_path.read_text() + .replace("{{TARGET_TABLE}}", target_table) + .replace("{{FLOW_NAME}}", sanitize_identifier(flow_name)) + .replace("{{SOURCE_TABLE}}", source_table) + ) + sections.append(f"-- source={source_slug} bronze_table={name} silver_table={target_name}\n{rendered.strip()}") + +if not sections: + raise SystemExit(f"no AUTO CDC tables selected from {catalog}.{bronze_schema}") + +output_path.parent.mkdir(parents=True, exist_ok=True) +output_path.write_text("\n\n".join(sections) + "\n") +print(output_path) +PY diff --git a/scripts/run-databricks-delta-pipeline.sh b/scripts/run-databricks-delta-pipeline.sh new file mode 100755 index 0000000..e2eef3c --- /dev/null +++ b/scripts/run-databricks-delta-pipeline.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi + +profile="$1" +target="$2" +bundle_engine="${DATABRICKS_BUNDLE_ENGINE:-direct}" + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +bundle_root="$repo_root/platform/databricks/delta" + +# shellcheck source=/dev/null +source "$repo_root/scripts/load-source-config.sh" +load_convex_sync_source_config "$repo_root" + +read_env_file_value() { + local key="$1" + local env_file="$repo_root/.env" + if [[ ! -f "$env_file" ]]; then + return 1 + fi + local line + line="$(grep -E "^${key}=" "$env_file" | tail -n 1 || true)" + if [[ -z "$line" ]]; then + return 1 + fi + printf '%s' "${line#*=}" +} + +deployment_url="${CONVEX_DEPLOYMENT_URL:-$(read_env_file_value CONVEX_DEPLOYMENT_URL || true)}" +if [[ -z "$deployment_url" ]]; then + echo "CONVEX_DEPLOYMENT_URL is required" >&2 + exit 1 +fi + +source_id="${CONVEX_SOURCE_ID:-$deployment_url}" +source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" +source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" +deployment_slug="${DATABRICKS_DELTA_DEPLOYMENT_SLUG:-${source_slug}-${target}}" +pipeline_name="${DATABRICKS_DELTA_AUTODC_PIPELINE_NAME:-${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}}" +pipeline_file="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_FILE:-${deployment_slug}-bronze-to-silver.sql}" +generated_file="$bundle_root/generated/$pipeline_file" + +"$repo_root/scripts/render-databricks-delta-pipeline.sh" "$profile" "$generated_file" >/dev/null + +catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" +table_name="${CONVEX_TABLE_NAME:-}" +secret_scope="${DATABRICKS_DELTA_SECRET_SCOPE:-convex-sync-kit}" +secret_key="${DATABRICKS_DELTA_SECRET_KEY:-convex-deploy-key}" +control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" +bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" +silver_schema="${DATABRICKS_DELTA_SILVER_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_silver}" +checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" + +"$repo_root/scripts/ensure-databricks-delta-secret.sh" "$profile" "$secret_scope" "$secret_key" + +bundle_args=( + --var "convex_deployment_url=$deployment_url" + --var "source_slug=$source_slug" + --var "convex_deploy_key_secret_scope=$secret_scope" + --var "convex_deploy_key_secret_key=$secret_key" + --var "source_id=$source_id" + --var "table_name=$table_name" + --var "catalog=$catalog" + --var "control_schema=$control_schema" + --var "bronze_schema=$bronze_schema" + --var "silver_schema=$silver_schema" + --var "checkpoint_table=$checkpoint_table" + --var "autocdc_pipeline_name=$pipeline_name" + --var "autocdc_pipeline_file=$pipeline_file" + --var "deployment_slug=$deployment_slug" +) + +( + cd "$bundle_root" + DATABRICKS_BUNDLE_ENGINE="$bundle_engine" databricks bundle run -p "$profile" -t "$target" convex_delta_autocdc "${bundle_args[@]}" +) diff --git a/sources/meshix-api/delta-pipeline.json b/sources/meshix-api/delta-pipeline.json new file mode 100644 index 0000000..d57d8dc --- /dev/null +++ b/sources/meshix-api/delta-pipeline.json @@ -0,0 +1,6 @@ +{ + "include_tables": [], + "exclude_tables": [], + "target_table_overrides": {}, + "flow_name_overrides": {} +} From 4e944014612f35879479eda4428f56b265401363 Mon Sep 17 00:00:00 2001 From: Anand Pant Date: Tue, 21 Apr 2026 07:23:32 -0500 Subject: [PATCH 2/5] fix: correct autocdc pipeline env override name --- scripts/deploy-databricks-delta-pipeline.sh | 2 +- scripts/run-databricks-delta-pipeline.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/deploy-databricks-delta-pipeline.sh b/scripts/deploy-databricks-delta-pipeline.sh index 8ef7328..c0b9af8 100755 --- a/scripts/deploy-databricks-delta-pipeline.sh +++ b/scripts/deploy-databricks-delta-pipeline.sh @@ -41,7 +41,7 @@ source_id="${CONVEX_SOURCE_ID:-$deployment_url}" source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" deployment_slug="${DATABRICKS_DELTA_DEPLOYMENT_SLUG:-${source_slug}-${target}}" -pipeline_name="${DATABRICKS_DELTA_AUTODC_PIPELINE_NAME:-${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}}" +pipeline_name="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}" pipeline_file="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_FILE:-${deployment_slug}-bronze-to-silver.sql}" generated_file="$bundle_root/generated/$pipeline_file" diff --git a/scripts/run-databricks-delta-pipeline.sh b/scripts/run-databricks-delta-pipeline.sh index e2eef3c..2edcf79 100755 --- a/scripts/run-databricks-delta-pipeline.sh +++ b/scripts/run-databricks-delta-pipeline.sh @@ -41,7 +41,7 @@ source_id="${CONVEX_SOURCE_ID:-$deployment_url}" source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" deployment_slug="${DATABRICKS_DELTA_DEPLOYMENT_SLUG:-${source_slug}-${target}}" -pipeline_name="${DATABRICKS_DELTA_AUTODC_PIPELINE_NAME:-${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}}" +pipeline_name="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}" pipeline_file="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_FILE:-${deployment_slug}-bronze-to-silver.sql}" generated_file="$bundle_root/generated/$pipeline_file" From 2e3428962e312efbc424dd5a8dfa1abbf96e746c Mon Sep 17 00:00:00 2001 From: Anand Pant Date: Tue, 21 Apr 2026 07:35:34 -0500 Subject: [PATCH 3/5] fix: pass shared bundle vars to delta pipeline commands --- scripts/deploy-databricks-delta-pipeline.sh | 2 ++ scripts/deploy-databricks-delta.sh | 5 +++++ scripts/run-databricks-delta-job.sh | 5 +++++ scripts/run-databricks-delta-pipeline.sh | 2 ++ 4 files changed, 14 insertions(+) diff --git a/scripts/deploy-databricks-delta-pipeline.sh b/scripts/deploy-databricks-delta-pipeline.sh index c0b9af8..e053a17 100755 --- a/scripts/deploy-databricks-delta-pipeline.sh +++ b/scripts/deploy-databricks-delta-pipeline.sh @@ -41,6 +41,7 @@ source_id="${CONVEX_SOURCE_ID:-$deployment_url}" source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" deployment_slug="${DATABRICKS_DELTA_DEPLOYMENT_SLUG:-${source_slug}-${target}}" +job_name="${DATABRICKS_DELTA_JOB_NAME:-convex-sync-kit-${deployment_slug}-delta-extract}" pipeline_name="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}" pipeline_file="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_FILE:-${deployment_slug}-bronze-to-silver.sql}" generated_file="$bundle_root/generated/$pipeline_file" @@ -61,6 +62,7 @@ checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" bundle_args=( --var "convex_deployment_url=$deployment_url" --var "source_slug=$source_slug" + --var "job_name=$job_name" --var "convex_deploy_key_secret_scope=$secret_scope" --var "convex_deploy_key_secret_key=$secret_key" --var "source_id=$source_id" diff --git a/scripts/deploy-databricks-delta.sh b/scripts/deploy-databricks-delta.sh index e3c6295..766ad12 100755 --- a/scripts/deploy-databricks-delta.sh +++ b/scripts/deploy-databricks-delta.sh @@ -49,6 +49,8 @@ catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" +pipeline_name="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}" +pipeline_file="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_FILE:-${deployment_slug}-bronze-to-silver.sql}" "$repo_root/scripts/ensure-databricks-delta-secret.sh" "$profile" "$secret_scope" "$secret_key" @@ -64,7 +66,10 @@ bundle_args=( --var "catalog=$catalog" --var "control_schema=$control_schema" --var "bronze_schema=$bronze_schema" + --var "silver_schema=$silver_schema" --var "checkpoint_table=$checkpoint_table" + --var "autocdc_pipeline_name=$pipeline_name" + --var "autocdc_pipeline_file=$pipeline_file" ) ( diff --git a/scripts/run-databricks-delta-job.sh b/scripts/run-databricks-delta-job.sh index 11d5844..1b8e2c3 100755 --- a/scripts/run-databricks-delta-job.sh +++ b/scripts/run-databricks-delta-job.sh @@ -50,6 +50,8 @@ catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" +pipeline_name="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}" +pipeline_file="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_FILE:-${deployment_slug}-bronze-to-silver.sql}" "$repo_root/scripts/ensure-databricks-delta-secret.sh" "$profile" "$secret_scope" "$secret_key" @@ -65,7 +67,10 @@ bundle_args=( --var "catalog=$catalog" --var "control_schema=$control_schema" --var "bronze_schema=$bronze_schema" + --var "silver_schema=$silver_schema" --var "checkpoint_table=$checkpoint_table" + --var "autocdc_pipeline_name=$pipeline_name" + --var "autocdc_pipeline_file=$pipeline_file" ) ( diff --git a/scripts/run-databricks-delta-pipeline.sh b/scripts/run-databricks-delta-pipeline.sh index 2edcf79..4f23e0b 100755 --- a/scripts/run-databricks-delta-pipeline.sh +++ b/scripts/run-databricks-delta-pipeline.sh @@ -41,6 +41,7 @@ source_id="${CONVEX_SOURCE_ID:-$deployment_url}" source_slug="${CONVEX_SYNC_SOURCE_SLUG:-default}" source_slug_sql="${CONVEX_SYNC_SOURCE_SQL:-${source_slug//-/_}}" deployment_slug="${DATABRICKS_DELTA_DEPLOYMENT_SLUG:-${source_slug}-${target}}" +job_name="${DATABRICKS_DELTA_JOB_NAME:-convex-sync-kit-${deployment_slug}-delta-extract}" pipeline_name="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}" pipeline_file="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_FILE:-${deployment_slug}-bronze-to-silver.sql}" generated_file="$bundle_root/generated/$pipeline_file" @@ -61,6 +62,7 @@ checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" bundle_args=( --var "convex_deployment_url=$deployment_url" --var "source_slug=$source_slug" + --var "job_name=$job_name" --var "convex_deploy_key_secret_scope=$secret_scope" --var "convex_deploy_key_secret_key=$secret_key" --var "source_id=$source_id" From b3d9228924879a46e3fc8f79e1c71861a4343937 Mon Sep 17 00:00:00 2001 From: Anand Pant Date: Tue, 21 Apr 2026 07:42:13 -0500 Subject: [PATCH 4/5] fix: define silver schema in shared delta commands --- scripts/deploy-databricks-delta.sh | 1 + scripts/run-databricks-delta-job.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/scripts/deploy-databricks-delta.sh b/scripts/deploy-databricks-delta.sh index 766ad12..ff08cbb 100755 --- a/scripts/deploy-databricks-delta.sh +++ b/scripts/deploy-databricks-delta.sh @@ -48,6 +48,7 @@ secret_key="${DATABRICKS_DELTA_SECRET_KEY:-convex-deploy-key}" catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" +silver_schema="${DATABRICKS_DELTA_SILVER_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_silver}" checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" pipeline_name="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}" pipeline_file="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_FILE:-${deployment_slug}-bronze-to-silver.sql}" diff --git a/scripts/run-databricks-delta-job.sh b/scripts/run-databricks-delta-job.sh index 1b8e2c3..97605ad 100755 --- a/scripts/run-databricks-delta-job.sh +++ b/scripts/run-databricks-delta-job.sh @@ -49,6 +49,7 @@ secret_key="${DATABRICKS_DELTA_SECRET_KEY:-convex-deploy-key}" catalog="${DATABRICKS_DELTA_CATALOG:-workspace}" control_schema="${DATABRICKS_DELTA_CONTROL_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_control}" bronze_schema="${DATABRICKS_DELTA_BRONZE_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_bronze}" +silver_schema="${DATABRICKS_DELTA_SILVER_SCHEMA:-convex_sync_kit_${source_slug_sql}_delta_silver}" checkpoint_table="${DATABRICKS_DELTA_CHECKPOINT_TABLE:-connector_checkpoint}" pipeline_name="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_NAME:-convex-sync-kit-${deployment_slug}-autocdc}" pipeline_file="${DATABRICKS_DELTA_AUTOCDC_PIPELINE_FILE:-${deployment_slug}-bronze-to-silver.sql}" From f4a42ed8daa090a4a257121b58f7d6278dcc4684 Mon Sep 17 00:00:00 2001 From: Anand Pant Date: Tue, 21 Apr 2026 07:59:30 -0500 Subject: [PATCH 5/5] fix: keep release builds locked without cargo version churn --- .depot/workflows/release-rc.yml | 2 +- .depot/workflows/release.yml | 2 +- release-please-config.json | 29 +---------------------------- version.txt | 1 + 4 files changed, 4 insertions(+), 30 deletions(-) create mode 100644 version.txt diff --git a/.depot/workflows/release-rc.yml b/.depot/workflows/release-rc.yml index 5359e2b..ac5ddc0 100644 --- a/.depot/workflows/release-rc.yml +++ b/.depot/workflows/release-rc.yml @@ -82,7 +82,7 @@ jobs: archive="convex-sync_${tag_name}_linux_amd64" stage="dist/${archive}" mkdir -p "${stage}" - CONVEX_SYNC_VERSION="${version}" depot cargo build --release -p convex-sync + CONVEX_SYNC_VERSION="${version}" depot cargo build --locked --release -p convex-sync cp target/release/convex-sync "${stage}/convex-sync" cp LICENSE NOTICE README.md "${stage}/" tar -czf "dist/${archive}.tar.gz" -C dist "${archive}" diff --git a/.depot/workflows/release.yml b/.depot/workflows/release.yml index 2548144..7eddcd2 100644 --- a/.depot/workflows/release.yml +++ b/.depot/workflows/release.yml @@ -51,7 +51,7 @@ jobs: archive="convex-sync_${version}_linux_amd64" stage="dist/${archive}" mkdir -p "${stage}" - CONVEX_SYNC_VERSION="${version#v}" depot cargo build --release -p convex-sync + CONVEX_SYNC_VERSION="${version#v}" depot cargo build --locked --release -p convex-sync cp target/release/convex-sync "${stage}/convex-sync" cp LICENSE NOTICE README.md "${stage}/" tar -czf "dist/${archive}.tar.gz" -C dist "${archive}" diff --git a/release-please-config.json b/release-please-config.json index f856990..b2911df 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -10,34 +10,7 @@ "release-type": "simple", "package-name": "convex-sync-kit", "changelog-path": "CHANGELOG.md", - "include-component-in-tag": false, - "extra-files": [ - { - "type": "toml", - "path": "Cargo.toml", - "jsonpath": "$.workspace.package.version" - }, - { - "type": "toml", - "path": "apps/convex-sync/Cargo.toml", - "jsonpath": "$.package.version" - }, - { - "type": "toml", - "path": "apps/convex-inspect/Cargo.toml", - "jsonpath": "$.package.version" - }, - { - "type": "toml", - "path": "crates/convex-sync-core/Cargo.toml", - "jsonpath": "$.package.version" - }, - { - "type": "toml", - "path": "crates/convex-export-s3/Cargo.toml", - "jsonpath": "$.package.version" - } - ] + "include-component-in-tag": false } } } diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..bcab45a --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +0.0.3