From a8de0ccab783a24c6b5fa83cbc13b8653e6b60cb Mon Sep 17 00:00:00 2001 From: Andre Date: Wed, 11 Feb 2026 16:14:33 -0300 Subject: [PATCH 01/10] improve push code --- .github/workflows/onpush.yml | 23 ++++---- README.md | 75 ++++++++++++++------------- scripts/generate_template_workflow.py | 7 +-- 3 files changed, 49 insertions(+), 56 deletions(-) diff --git a/.github/workflows/onpush.yml b/.github/workflows/onpush.yml index 98cf18d..3ce8c4e 100644 --- a/.github/workflows/onpush.yml +++ b/.github/workflows/onpush.yml @@ -20,10 +20,15 @@ jobs: env: DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + DEVELOPER: ${{ github.actor }} steps: - uses: actions/checkout@v1 + - name: Dump GitHub context + run: echo '${{ toJson(github) }}' + - name: Set up Python uses: actions/setup-python@v5 with: @@ -46,15 +51,10 @@ jobs: uses: databricks/setup-cli@main - name: Deploy on staging - run: | - BRANCH_NAME="${{ github.head_ref || github.ref_name }}" - PR_NUMBER="${{ github.event.pull_request.number }}" - DEVELOPER="${{ github.actor }}" - + run: uv run python ./scripts/generate_template_workflow.py staging --serverless \ --branch "$BRANCH_NAME" \ - --developer "$DEVELOPER" \ - $(if [ -n "$PR_NUMBER" ]; then echo "--pr-number $PR_NUMBER"; fi) + --developer "$DEVELOPER" uv run databricks bundle deploy --target staging @@ -63,14 +63,9 @@ jobs: make run env=staging - name: Deploy on prod - run: | - BRANCH_NAME="${{ github.head_ref || github.ref_name }}" - PR_NUMBER="${{ github.event.pull_request.number }}" - DEVELOPER="${{ github.actor }}" - + run: uv run python ./scripts/generate_template_workflow.py prod --serverless \ --branch "$BRANCH_NAME" \ - --developer "$DEVELOPER" \ - $(if [ -n "$PR_NUMBER" ]; then echo "--pr-number $PR_NUMBER"; fi) + --developer "$DEVELOPER" uv run databricks bundle deploy --target prod diff --git a/README.md b/README.md index 4a77e42..518764e 100644 --- a/README.md +++ b/README.md @@ -33,15 +33,17 @@ Interested in bringing these principles in your own project? Let’s [connect o This project template demonstrates how to: -- structure PySpark code inside classes/packages. -- run unit tests on transformations with [pytest package](https://pypi.org/project/pytest/) - set up VSCode to run unit tests on your local machine. -- structure integration tests to be executed on different environments / catalogs. +- structure PySpark code inside classes/packages, instead of notebooks. +- package and deploy code to different environments (dev, staging, prod). +- use a CI/CD pipeline with [Github Actions](https://docs.github.com/en/actions). +- run unit tests on transformations with [pytest package](https://pypi.org/project/pytest/). Set up VSCode to run unit tests on your local machine. +- run integration tests setting the input data and validating the output data. +- isolate "dev" environments / catalogs to avoid concurrency issues between developer tests. +- show developer name and branch as job tags to track issues. - utilize [coverage package](https://pypi.org/project/coverage/) to generate test coverage reports. -- package and deploy code to different environments (dev, staging, prod) using a CI/CD pipeline with [Github Actions](https://docs.github.com/en/actions). -- isolate "dev" environments / catalogs to avoid concurrency issues between developers testing jobs. - utilize [uv](https://docs.astral.sh/uv/) as a project/package manager. -- configure the workflow to run in different environments with different parameters with [jinja package](https://pypi.org/project/jinja2/). -- configure the workflow to run tasks selectively. +- configure job to run in different environments with different parameters with [jinja package](https://pypi.org/project/jinja2/). +- configure job to run tasks selectively. - use [medallion architecture](https://www.databricks.com/glossary/medallion-architecture) pattern. - lint and format code with [ruff](https://docs.astral.sh/ruff/) and [pre-commit](https://pre-commit.com/). - use a Make file to automate repetitive tasks. @@ -52,9 +54,9 @@ This project template demonstrates how to: - utilize [Databricks Asset Bundles](https://docs.databricks.com/en/dev-tools/bundles/index.html) to package/deploy/run a Python wheel package on Databricks. - utilize [Databricks DQX](https://databrickslabs.github.io/dqx/) to define and enforce data quality rules, such as null checks, uniqueness, thresholds, and schema validation, and filter bad data on quarantine tables. - utilize [Databricks SDK for Python](https://docs.databricks.com/en/dev-tools/sdk-python.html) to manage workspaces and accounts and analyse costs. Refer to 'scripts' folder for some examples. -- utilize [Databricks Unity Catalog](https://www.databricks.com/product/unity-catalog) and get data lineage for your tables and columns and a simplified permission model for your data. +- utilize [Databricks Unity Catalog](https://www.databricks.com/product/unity-catalog) and get data lineage for your tables and columns. - utilize [Databricks Lakeflow Jobs](https://docs.databricks.com/en/workflows/index.html) to execute a DAG and [task parameters](https://docs.databricks.com/en/workflows/jobs/parameter-value-references.html) to share context information between tasks (see [Task Parameters section](#task-parameters)). Yes, you don't need Airflow to manage your DAGs here!!! -- utilize serverless job clusters on [Databricks Free Edition](https://docs.databricks.com/aws/en/getting-started/free-edition ) to deploy your pipelines. +- utilize serverless job clusters on [Databricks Free Edition](https://docs.databricks.com/aws/en/getting-started/free-edition) to deploy your pipelines. ## 🧠 Resources @@ -78,17 +80,17 @@ Other: ``` databricks-template/ β”‚ -β”œβ”€β”€ .github/ # CI/CD automation +β”œβ”€β”€ .github/ # CI/CD automation β”‚ └── workflows/ -β”‚ └── onpush.yml # GitHub Actions pipeline +β”‚ └── onpush.yml # GitHub Actions pipeline β”‚ -β”œβ”€β”€ src/ # Main source code -β”‚ └── template/ # Python package -β”‚ β”œβ”€β”€ main.py # Entry point with CLI (argparse) -β”‚ β”œβ”€β”€ config.py # Configuration management -β”‚ β”œβ”€β”€ baseTask.py # Base class for all tasks -β”‚ β”œβ”€β”€ commonSchemas.py # Shared PySpark schemas -β”‚ └── job1/ # Job-specific tasks +β”œβ”€β”€ src/ # Main source code +β”‚ └── template/ # Python package +β”‚ β”œβ”€β”€ main.py # Entry point with CLI (argparse) +β”‚ β”œβ”€β”€ config.py # Configuration management +β”‚ β”œβ”€β”€ baseTask.py # Base class for all tasks +β”‚ β”œβ”€β”€ commonSchemas.py # Shared PySpark schemas +β”‚ └── job1/ # Job-specific tasks β”‚ β”œβ”€β”€ extract_source1.py β”‚ β”œβ”€β”€ extract_source2.py β”‚ β”œβ”€β”€ generate_orders.py @@ -96,21 +98,21 @@ databricks-template/ β”‚ β”œβ”€β”€ integration_setup.py β”‚ └── integration_validate.py β”‚ -β”œβ”€β”€ tests/ # Unit tests +β”œβ”€β”€ tests/ # Unit tests β”‚ └── job1/ -β”‚ └── unit_test.py # Pytest unit tests +β”‚ └── unit_test.py # Pytest unit tests β”‚ -β”œβ”€β”€ resources/ # Databricks workflow templates +β”œβ”€β”€ resources/ # Databricks workflow templates β”‚ β”œβ”€β”€ wf_template_serverless.yml # Jinja2 template for serverless β”‚ β”œβ”€β”€ wf_template.yml # Jinja2 template for job clusters β”‚ └── workflow.yml # Generated workflow (auto-created) β”‚ -β”œβ”€β”€ scripts/ # Helper scripts +β”œβ”€β”€ scripts/ # Helper scripts β”‚ β”œβ”€β”€ generate_template_workflow.py # Workflow generator (Jinja2) -β”‚ β”œβ”€β”€ sdk_analyze_job_costs.py # Cost analysis script -β”‚ └── sdk_workspace_and_account.py # Workspace and account management -β”‚ print("SUMMARY") -β”œβ”€β”€ docs/ # Documentation assets +β”‚ β”œβ”€β”€ sdk_analyze_job_costs.py # Cost analysis script +β”‚ └── sdk_workspace_and_account.py # Workspace and account management +β”‚ +β”œβ”€β”€ docs/ # Documentation assets β”‚ β”œβ”€β”€ dag.png β”‚ β”œβ”€β”€ task_output.png β”‚ β”œβ”€β”€ data_lineage.png @@ -127,6 +129,14 @@ databricks-template/ └── README.md # This file ``` +## CI/CD pipeline + +
+ + + +
+ ## Jobs
@@ -161,24 +171,15 @@ databricks-template/
-## CI/CD pipeline - -
- - - -
- - ## Instructions 1) Create a workspace. Use a [Databricks Free Edition](https://docs.databricks.com/aws/en/getting-started/free-edition) workspace. -2) Install and configure Databricks CLI on your local machine. Follow instructions [here](https://docs.databricks.com/en/dev-tools/cli/install.html). Check the current version on databricks.yaml. +2) Install and configure Databricks CLI on your local machine. Check the current version on databricks.yaml. Follow instructions [here](https://docs.databricks.com/en/dev-tools/cli/install.html). -3) Build Python env and execute unit tests on your local machine +3) Build Python env and execute unit tests on your local machine. make sync & make test diff --git a/scripts/generate_template_workflow.py b/scripts/generate_template_workflow.py index 6623dd8..1ea5c87 100644 --- a/scripts/generate_template_workflow.py +++ b/scripts/generate_template_workflow.py @@ -35,7 +35,7 @@ def main(): epilog=""" Examples: python generate_template_workflow.py dev --serverless - python generate_template_workflow.py staging --serverless --branch main --developer john --pr-number 123 + python generate_template_workflow.py staging --serverless --branch main --developer john """, ) @@ -43,20 +43,17 @@ def main(): parser.add_argument("--serverless", action="store_true", help="Use serverless workflow template") parser.add_argument("--branch", help="Git branch name (auto-detected if not provided)") parser.add_argument("--developer", help="Developer/deployer name (auto-detected if not provided)") - parser.add_argument("--pr-number", help="Pull request number (optional)") args = parser.parse_args() # Get or auto-detect git metadata branch = args.branch if args.branch else get_git_branch() developer = args.developer if args.developer else get_git_user() - pr_number = args.pr_number if args.pr_number else "" print(f"Environment: {args.environment}") print(f"Serverless mode: {args.serverless}") print(f"Git branch: {branch}") print(f"Developer: {developer}") - print(f"PR number: {pr_number if pr_number else 'N/A'}") # Load and render template file_loader = FileSystemLoader(".") @@ -68,7 +65,7 @@ def main(): template = env.get_template("/resources/wf_template.yml") # Render the template with all variables - output = template.render(environment=args.environment, branch=branch, developer=developer, pr_number=pr_number) + output = template.render(environment=args.environment, branch=branch, developer=developer) # Save the rendered YAML to a file output_file = "./resources/workflow.yml" From f6608c0cb42c062ef8920bec3df66dd5c814c51f Mon Sep 17 00:00:00 2001 From: Andre Date: Wed, 11 Feb 2026 16:30:42 -0300 Subject: [PATCH 02/10] improve ci/cd --- Makefile | 4 ++-- README.md | 2 +- scripts/generate_template_workflow.py | 19 ++++++------------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 490a572..3f41ba8 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,8 @@ pre-commit: pre-commit autoupdate pre-commit run --all-files -deploy-serverless: - uv run python ./scripts/generate_template_workflow.py $(env) --serverless +deploy: + uv run python ./scripts/generate_template_workflow.py $(env) uv run databricks bundle deploy --target $(env) run: diff --git a/README.md b/README.md index 518764e..1ec32d2 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ databricks-template/ 4) Deploy and execute on the dev workspace. - make deploy-serverless env=dev + make deploy env=dev 5) configure CI/CD automation. Configure [Github Actions repository secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions) (DATABRICKS_HOST and DATABRICKS_TOKEN). diff --git a/scripts/generate_template_workflow.py b/scripts/generate_template_workflow.py index 1ea5c87..a0e820e 100644 --- a/scripts/generate_template_workflow.py +++ b/scripts/generate_template_workflow.py @@ -34,24 +34,20 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - python generate_template_workflow.py dev --serverless - python generate_template_workflow.py staging --serverless --branch main --developer john + python generate_template_workflow.py dev + python generate_template_workflow.py staging """, ) parser.add_argument("environment", help="Target environment (dev, staging, prod)") - parser.add_argument("--serverless", action="store_true", help="Use serverless workflow template") - parser.add_argument("--branch", help="Git branch name (auto-detected if not provided)") - parser.add_argument("--developer", help="Developer/deployer name (auto-detected if not provided)") args = parser.parse_args() - # Get or auto-detect git metadata - branch = args.branch if args.branch else get_git_branch() - developer = args.developer if args.developer else get_git_user() + # Auto-detect git metadata + branch = get_git_branch() + developer = get_git_user() print(f"Environment: {args.environment}") - print(f"Serverless mode: {args.serverless}") print(f"Git branch: {branch}") print(f"Developer: {developer}") @@ -59,10 +55,7 @@ def main(): file_loader = FileSystemLoader(".") env = Environment(loader=file_loader) - if args.serverless: - template = env.get_template("/resources/wf_template_serverless.yml") - else: - template = env.get_template("/resources/wf_template.yml") + template = env.get_template("/resources/wf_template_serverless.yml") # Render the template with all variables output = template.render(environment=args.environment, branch=branch, developer=developer) From c37f09876a9d1ae1273d7c52e81a249f11762196 Mon Sep 17 00:00:00 2001 From: Andre Date: Wed, 11 Feb 2026 16:33:41 -0300 Subject: [PATCH 03/10] fix ci/cd --- .github/workflows/onpush.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/onpush.yml b/.github/workflows/onpush.yml index 3ce8c4e..3f4582a 100644 --- a/.github/workflows/onpush.yml +++ b/.github/workflows/onpush.yml @@ -52,10 +52,7 @@ jobs: - name: Deploy on staging run: - uv run python ./scripts/generate_template_workflow.py staging --serverless \ - --branch "$BRANCH_NAME" \ - --developer "$DEVELOPER" - + uv run python ./scripts/generate_template_workflow.py staging uv run databricks bundle deploy --target staging - name: Run on staging (integration tests) @@ -64,8 +61,5 @@ jobs: - name: Deploy on prod run: - uv run python ./scripts/generate_template_workflow.py prod --serverless \ - --branch "$BRANCH_NAME" \ - --developer "$DEVELOPER" - + uv run python ./scripts/generate_template_workflow.py prod uv run databricks bundle deploy --target prod From 9e13abf6cabefc4858f5c1a19a8bcccd52ad525e Mon Sep 17 00:00:00 2001 From: Andre Date: Wed, 11 Feb 2026 16:37:56 -0300 Subject: [PATCH 04/10] fix ci/cd --- .github/workflows/onpush.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/onpush.yml b/.github/workflows/onpush.yml index 3f4582a..4fc16a0 100644 --- a/.github/workflows/onpush.yml +++ b/.github/workflows/onpush.yml @@ -20,8 +20,6 @@ jobs: env: DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - DEVELOPER: ${{ github.actor }} steps: - uses: actions/checkout@v1 @@ -53,6 +51,7 @@ jobs: - name: Deploy on staging run: uv run python ./scripts/generate_template_workflow.py staging + uv run databricks bundle deploy --target staging - name: Run on staging (integration tests) @@ -62,4 +61,5 @@ jobs: - name: Deploy on prod run: uv run python ./scripts/generate_template_workflow.py prod + uv run databricks bundle deploy --target prod From 7adf6a17d8d6faf222e3adc163e083b5e79cf50a Mon Sep 17 00:00:00 2001 From: Andre Date: Wed, 11 Feb 2026 16:46:47 -0300 Subject: [PATCH 05/10] fix ci/cd --- .github/workflows/onpush.yml | 12 +++++-- scripts/generate_template_workflow.py | 50 ++++++++++++++------------- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/.github/workflows/onpush.yml b/.github/workflows/onpush.yml index 4fc16a0..5c5f7ad 100644 --- a/.github/workflows/onpush.yml +++ b/.github/workflows/onpush.yml @@ -20,6 +20,8 @@ jobs: env: DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + DEVELOPER: ${{ github.actor }} steps: - uses: actions/checkout@v1 @@ -50,7 +52,9 @@ jobs: - name: Deploy on staging run: - uv run python ./scripts/generate_template_workflow.py staging + uv run python ./scripts/generate_template_workflow.py staging \ + --branch "$BRANCH_NAME" \ + --developer "$DEVELOPER" uv run databricks bundle deploy --target staging @@ -60,6 +64,8 @@ jobs: - name: Deploy on prod run: - uv run python ./scripts/generate_template_workflow.py prod - + uv run python ./scripts/generate_template_workflow.py prod \ + --branch "$BRANCH_NAME" \ + --developer "$DEVELOPER" \ + uv run databricks bundle deploy --target prod diff --git a/scripts/generate_template_workflow.py b/scripts/generate_template_workflow.py index a0e820e..f096ac9 100644 --- a/scripts/generate_template_workflow.py +++ b/scripts/generate_template_workflow.py @@ -4,28 +4,28 @@ from jinja2 import Environment, FileSystemLoader -def get_git_branch(): - """Get current git branch name, fallback to 'unknown' if not available.""" - try: - branch = ( - subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], stderr=subprocess.DEVNULL) - .decode("utf-8") - .strip() - ) - return branch - except Exception: - return "unknown" - - -def get_git_user(): - """Get git user name, fallback to 'unknown' if not available.""" - try: - user = ( - subprocess.check_output(["git", "config", "user.name"], stderr=subprocess.DEVNULL).decode("utf-8").strip() - ) - return user - except Exception: - return "unknown" +# def get_git_branch(): +# """Get current git branch name, fallback to 'unknown' if not available.""" +# try: +# branch = ( +# subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], stderr=subprocess.DEVNULL) +# .decode("utf-8") +# .strip() +# ) +# return branch +# except Exception: +# return "unknown" + + +# def get_git_user(): +# """Get git user name, fallback to 'unknown' if not available.""" +# try: +# user = ( +# subprocess.check_output(["git", "config", "user.name"], stderr=subprocess.DEVNULL).decode("utf-8").strip() +# ) +# return user +# except Exception: +# return "unknown" def main(): @@ -40,12 +40,14 @@ def main(): ) parser.add_argument("environment", help="Target environment (dev, staging, prod)") + parser.add_argument("--branch", help="Git branch name (auto-detected if not provided)") + parser.add_argument("--developer", help="Developer/deployer name (auto-detected if not provided)") args = parser.parse_args() # Auto-detect git metadata - branch = get_git_branch() - developer = get_git_user() + branch = args.branch + developer = args.developer print(f"Environment: {args.environment}") print(f"Git branch: {branch}") From 8f68f21a99f2dba12831a0a68a88d1a486e11790 Mon Sep 17 00:00:00 2001 From: Andre Date: Wed, 11 Feb 2026 16:54:07 -0300 Subject: [PATCH 06/10] fix ci/cd --- .github/workflows/onpush.yml | 8 ++--- scripts/generate_template_workflow.py | 50 +++++++++++++-------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/onpush.yml b/.github/workflows/onpush.yml index 5c5f7ad..8bb02b6 100644 --- a/.github/workflows/onpush.yml +++ b/.github/workflows/onpush.yml @@ -20,7 +20,7 @@ jobs: env: DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + BRANCH: ${{ github.head_ref || github.ref_name }} DEVELOPER: ${{ github.actor }} steps: @@ -53,7 +53,7 @@ jobs: - name: Deploy on staging run: uv run python ./scripts/generate_template_workflow.py staging \ - --branch "$BRANCH_NAME" \ + --branch "$BRANCH" \ --developer "$DEVELOPER" uv run databricks bundle deploy --target staging @@ -65,7 +65,7 @@ jobs: - name: Deploy on prod run: uv run python ./scripts/generate_template_workflow.py prod \ - --branch "$BRANCH_NAME" \ - --developer "$DEVELOPER" \ + --branch "$BRANCH" \ + --developer "$DEVELOPER" uv run databricks bundle deploy --target prod diff --git a/scripts/generate_template_workflow.py b/scripts/generate_template_workflow.py index f096ac9..2ae6c0c 100644 --- a/scripts/generate_template_workflow.py +++ b/scripts/generate_template_workflow.py @@ -4,28 +4,28 @@ from jinja2 import Environment, FileSystemLoader -# def get_git_branch(): -# """Get current git branch name, fallback to 'unknown' if not available.""" -# try: -# branch = ( -# subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], stderr=subprocess.DEVNULL) -# .decode("utf-8") -# .strip() -# ) -# return branch -# except Exception: -# return "unknown" - - -# def get_git_user(): -# """Get git user name, fallback to 'unknown' if not available.""" -# try: -# user = ( -# subprocess.check_output(["git", "config", "user.name"], stderr=subprocess.DEVNULL).decode("utf-8").strip() -# ) -# return user -# except Exception: -# return "unknown" +def get_git_branch(): + """Get current git branch name, fallback to 'unknown' if not available.""" + try: + branch = ( + subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], stderr=subprocess.DEVNULL) + .decode("utf-8") + .strip() + ) + return branch + except Exception: + return "unknown" + + +def get_git_user(): + """Get git user name, fallback to 'unknown' if not available.""" + try: + user = ( + subprocess.check_output(["git", "config", "user.name"], stderr=subprocess.DEVNULL).decode("utf-8").strip() + ) + return user + except Exception: + return "unknown" def main(): @@ -45,9 +45,9 @@ def main(): args = parser.parse_args() - # Auto-detect git metadata - branch = args.branch - developer = args.developer + # Auto-detect git metadata in local environments, use provided values in CI + branch = args.branch if args.branch else get_git_branch() + developer = args.developer if args.developer else get_git_user() print(f"Environment: {args.environment}") print(f"Git branch: {branch}") From a2ca49e526700d524b7fc56b0937c3cd1b7204b3 Mon Sep 17 00:00:00 2001 From: Andre Date: Wed, 11 Feb 2026 17:02:07 -0300 Subject: [PATCH 07/10] fix ci/cd --- .github/workflows/onpush.yml | 2 +- scripts/generate_template_workflow.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/onpush.yml b/.github/workflows/onpush.yml index 8bb02b6..a322b43 100644 --- a/.github/workflows/onpush.yml +++ b/.github/workflows/onpush.yml @@ -67,5 +67,5 @@ jobs: uv run python ./scripts/generate_template_workflow.py prod \ --branch "$BRANCH" \ --developer "$DEVELOPER" - + uv run databricks bundle deploy --target prod diff --git a/scripts/generate_template_workflow.py b/scripts/generate_template_workflow.py index 2ae6c0c..a9010ed 100644 --- a/scripts/generate_template_workflow.py +++ b/scripts/generate_template_workflow.py @@ -34,8 +34,8 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - python generate_template_workflow.py dev - python generate_template_workflow.py staging + python generate_template_workflow.py dev + python generate_template_workflow.py staging --branch main --developer john """, ) From 6e58302991efd8c24603011d578b02dce72116ee Mon Sep 17 00:00:00 2001 From: Andre Date: Wed, 11 Feb 2026 17:06:32 -0300 Subject: [PATCH 08/10] fix ci/cd --- .github/workflows/onpush.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/onpush.yml b/.github/workflows/onpush.yml index a322b43..cddf43e 100644 --- a/.github/workflows/onpush.yml +++ b/.github/workflows/onpush.yml @@ -52,9 +52,10 @@ jobs: - name: Deploy on staging run: - uv run python ./scripts/generate_template_workflow.py staging \ - --branch "$BRANCH" \ - --developer "$DEVELOPER" + uv run python ./scripts/generate_template_workflow.py + staging + --branch "$BRANCH" \ + --developer "$DEVELOPER" uv run databricks bundle deploy --target staging @@ -64,8 +65,9 @@ jobs: - name: Deploy on prod run: - uv run python ./scripts/generate_template_workflow.py prod \ - --branch "$BRANCH" \ - --developer "$DEVELOPER" - + uv run python ./scripts/generate_template_workflow.py + prod + --branch "$BRANCH" \ + --developer "$DEVELOPER" + uv run databricks bundle deploy --target prod From 1e15197fcb31ff826eb662940c021081ad2652ab Mon Sep 17 00:00:00 2001 From: Andre Date: Wed, 11 Feb 2026 17:10:51 -0300 Subject: [PATCH 09/10] fix ci/cd --- .github/workflows/onpush.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/onpush.yml b/.github/workflows/onpush.yml index cddf43e..1b84cc5 100644 --- a/.github/workflows/onpush.yml +++ b/.github/workflows/onpush.yml @@ -51,10 +51,9 @@ jobs: uses: databricks/setup-cli@main - name: Deploy on staging - run: - uv run python ./scripts/generate_template_workflow.py - staging - --branch "$BRANCH" \ + run: | + uv run python ./scripts/generate_template_workflow.py staging \ + --branch "$BRANCH" --developer "$DEVELOPER" uv run databricks bundle deploy --target staging @@ -64,9 +63,8 @@ jobs: make run env=staging - name: Deploy on prod - run: - uv run python ./scripts/generate_template_workflow.py - prod + run: | + uv run python ./scripts/generate_template_workflow.py prod \ --branch "$BRANCH" \ --developer "$DEVELOPER" From 57cd39f928afe527e7cf8b35dc8ac2c51ed566d9 Mon Sep 17 00:00:00 2001 From: Andre Date: Wed, 11 Feb 2026 17:12:39 -0300 Subject: [PATCH 10/10] fix ci/cd --- .github/workflows/onpush.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/onpush.yml b/.github/workflows/onpush.yml index 1b84cc5..736d9ca 100644 --- a/.github/workflows/onpush.yml +++ b/.github/workflows/onpush.yml @@ -53,7 +53,7 @@ jobs: - name: Deploy on staging run: | uv run python ./scripts/generate_template_workflow.py staging \ - --branch "$BRANCH" + --branch "$BRANCH" \ --developer "$DEVELOPER" uv run databricks bundle deploy --target staging