diff --git a/docs/source/ai_skills.md b/docs/source/ai_skills.md new file mode 100644 index 0000000..bfec64a --- /dev/null +++ b/docs/source/ai_skills.md @@ -0,0 +1,14 @@ +# AI Skills + +```{include} ../../skills/README.md +:relative-docs: ../../skills/ +:relative-images: +:start-line: 2 +``` + +```{toctree} +:hidden: +:maxdepth: 1 + +Data Flow Spec Builder +``` diff --git a/docs/source/dataflowspec_builder/README.md b/docs/source/dataflowspec_builder/README.md new file mode 100644 index 0000000..1eeb6f3 --- /dev/null +++ b/docs/source/dataflowspec_builder/README.md @@ -0,0 +1,16 @@ +```{include} ../../../skills/dataflowspec_builder/README.md +:relative-images: +``` + +```{toctree} +:hidden: + +docs/getting-started +docs/example-prompts +docs/architecture +docs/tested-medallion-example +docs/skill-development +references/patterns-guide +references/dataflow-spec-schema-reference +references/energy-domain-mapping +``` diff --git a/docs/source/dataflowspec_builder/docs/architecture.md b/docs/source/dataflowspec_builder/docs/architecture.md new file mode 100644 index 0000000..0afb6c0 --- /dev/null +++ b/docs/source/dataflowspec_builder/docs/architecture.md @@ -0,0 +1,3 @@ +```{include} ../../../../skills/dataflowspec_builder/docs/architecture.md +:relative-images: +``` diff --git a/docs/source/dataflowspec_builder/docs/example-prompts.md b/docs/source/dataflowspec_builder/docs/example-prompts.md new file mode 100644 index 0000000..af5a162 --- /dev/null +++ b/docs/source/dataflowspec_builder/docs/example-prompts.md @@ -0,0 +1,3 @@ +```{include} ../../../../skills/dataflowspec_builder/docs/example-prompts.md +:relative-images: +``` diff --git a/docs/source/dataflowspec_builder/docs/getting-started.md b/docs/source/dataflowspec_builder/docs/getting-started.md new file mode 100644 index 0000000..6c8a564 --- /dev/null +++ b/docs/source/dataflowspec_builder/docs/getting-started.md @@ -0,0 +1,3 @@ +```{include} ../../../../skills/dataflowspec_builder/docs/getting-started.md +:relative-images: +``` diff --git a/docs/source/dataflowspec_builder/docs/skill-development.md b/docs/source/dataflowspec_builder/docs/skill-development.md new file mode 100644 index 0000000..c821e17 --- /dev/null +++ b/docs/source/dataflowspec_builder/docs/skill-development.md @@ -0,0 +1,3 @@ +```{include} ../../../../skills/dataflowspec_builder/docs/skill-development.md +:relative-images: +``` diff --git a/docs/source/dataflowspec_builder/docs/tested-medallion-example.md b/docs/source/dataflowspec_builder/docs/tested-medallion-example.md new file mode 100644 index 0000000..035c83d --- /dev/null +++ b/docs/source/dataflowspec_builder/docs/tested-medallion-example.md @@ -0,0 +1,3 @@ +```{include} ../../../../skills/dataflowspec_builder/docs/tested-medallion-example.md +:relative-images: +``` diff --git a/docs/source/dataflowspec_builder/references/dataflow-spec-schema-reference.md b/docs/source/dataflowspec_builder/references/dataflow-spec-schema-reference.md new file mode 100644 index 0000000..7799db5 --- /dev/null +++ b/docs/source/dataflowspec_builder/references/dataflow-spec-schema-reference.md @@ -0,0 +1,3 @@ +```{include} ../../../../skills/dataflowspec_builder/references/dataflow-spec-schema-reference.md +:relative-images: +``` diff --git a/docs/source/dataflowspec_builder/references/energy-domain-mapping.md b/docs/source/dataflowspec_builder/references/energy-domain-mapping.md new file mode 100644 index 0000000..f6ad5bd --- /dev/null +++ b/docs/source/dataflowspec_builder/references/energy-domain-mapping.md @@ -0,0 +1,3 @@ +```{include} ../../../../skills/dataflowspec_builder/references/energy-domain-mapping.md +:relative-images: +``` diff --git a/docs/source/dataflowspec_builder/references/patterns-guide.md b/docs/source/dataflowspec_builder/references/patterns-guide.md new file mode 100644 index 0000000..c32423d --- /dev/null +++ b/docs/source/dataflowspec_builder/references/patterns-guide.md @@ -0,0 +1,3 @@ +```{include} ../../../../skills/dataflowspec_builder/references/patterns-guide.md +:relative-images: +``` diff --git a/docs/source/index.rst b/docs/source/index.rst index dbafcbc..5aaa49f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,8 +16,18 @@ Start with :doc:`what_is_lakeflow_framework` for the full overview, then use the what_is_lakeflow_framework getting_started - Concepts + Concepts Features + +.. toctree:: + :maxdepth: 4 + :titlesonly: + + AI Skills + +.. toctree:: + :maxdepth: 4 + deploy_framework deploy_samples.md build_pipeline_bundle diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 0000000..c5128fa --- /dev/null +++ b/skills/README.md @@ -0,0 +1,22 @@ +# LFF Skills + +This directory holds [Agent Skills](https://docs.databricks.com/aws/en/genie-code/skills) intended for **end users of the Lakeflow Framework (LFF)**. Each skill lives in its own subfolder and packages the instructions, reference docs, templates, and examples an agent needs to help you work with the framework from natural language. + +## Available Skills + +| Skill | Folder | Description | +|-------|--------|-------------| +| **Data Flow Spec Builder** | [`dataflowspec_builder/`](./dataflowspec_builder/README.md) | Generates complete, production-ready Data Flow Spec pipeline bundles (specs, schemas, expectations, SQL/Python transforms, substitutions, templates, pipeline resource YAMLs, and `databricks.yml`) from natural language. Covers CDC (SCD1/2), data quality, quarantine, liquid clustering, multi-source streaming, table migration, and DABs deployment. | + +## Using a Skill + +Each skill folder contains a `SKILL.md` (the entry point the agent loads) and a `README.md` with human-facing documentation. Start with the skill's own `README.md` for prompts, parameters, and examples. + +## Adding a New Skill + +When contributing a new end-user skill: + +1. Create a new subfolder under `skills/` named after the skill (e.g. `skills/my_new_skill/`). +2. Add a `SKILL.md` with YAML frontmatter (`name`, `description`) and instructions for the agent. +3. Add a `README.md` documenting what the skill does and how to invoke it. +4. Add a new row to the **Available Skills** table above so this index stays current. diff --git a/skills/dataflowspec_builder/.gitignore b/skills/dataflowspec_builder/.gitignore new file mode 100644 index 0000000..9cd385e --- /dev/null +++ b/skills/dataflowspec_builder/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +__pycache__/ +*.pyc +.env +*.log diff --git a/skills/dataflowspec_builder/README.md b/skills/dataflowspec_builder/README.md new file mode 100644 index 0000000..777e3c7 --- /dev/null +++ b/skills/dataflowspec_builder/README.md @@ -0,0 +1,185 @@ +# Data Flow Spec Builder — Genie Code Agent Skill for the Lakeflow Framework + +A comprehensive [Databricks Genie Code](https://docs.databricks.com/en/notebooks/genie-code.html) Agent Skill that generates production-ready pipeline bundles using the [Data Flow Spec Framework](https://github.com/databricks-solutions/lakeflow_framework) (`databricks-solutions/lakeflow_framework`) from natural language prompts. + +> **Important:** This skill generates pipelines using the **Data Flow Spec Framework** — a metadata-driven wrapper around Spark Declarative Pipelines (SDP). It is **not** the same as native Lakeflow Declarative Pipelines (DLT). The Data Flow Spec Framework uses JSON/YAML configuration files to define pipelines declaratively, without writing `@dlt.table` decorators or `CREATE STREAMING TABLE` SQL. + +## What It Does + +From a prompt like: + +> "Use the dataflow-spec-builder to create bronze Data Flow Specs for ingesting customer and billing tables with SCD1 CDC and data quality checks" + +Genie Code generates a **complete, deployable pipeline bundle**: + +- Data Flow Spec JSON files (standard, flows, or materialized views) +- StructType schema JSON files +- Data quality expectation files +- SQL transform files +- Pipeline resource YAMLs +- `databricks.yml` DABs configuration +- Environment substitutions (dev/staging/prod) +- Reusable templates for similar tables +- Python extension stubs (sources, transforms, sinks) + +## Framework Features Covered + +| Category | Features | +|----------|----------| +| **Data Flow Types** | Standard (1:1), Flows (multi-source), Materialized Views | +| **Source Types** | Delta, CloudFiles (Auto Loader), Delta Join, Kafka, Python, SQL | +| **CDC** | SCD Type 1, SCD Type 2, CDC from Snapshots (file + table based) | +| **Data Quality** | Expectations (expect / expect_or_drop / expect_or_fail), Quarantine (off/flag/table) | +| **Table Features** | Liquid Clustering (manual + auto), Partition Columns, Table Properties | +| **Pipeline Patterns** | Basic 1:1, Stream-Static Join, Multi-Source Streaming, CDC Snapshots, MVs | +| **Extensibility** | Python Extensions (sources, transforms, sinks), Python Function Transforms | +| **Environment** | Substitutions (token + prefix/suffix), Logical Environments, Multi-target DABs | +| **Templates** | Parameterized specs, parameter sets, template processing | +| **Operations** | Operational Metadata, Mandatory Table Properties, Logging, Versioning | +| **Advanced** | Soft Deletes, Secrets Management, Table Migration (HMS→UC), CDF, Spark Config | +| **Deployment** | DABs validate/deploy/run/destroy, CI/CD scripts, Pipeline Filters | + +## Quick Start + +### 1. Deploy the Framework Engine + +```bash +git clone https://github.com/databricks-solutions/lakeflow_framework.git +cd lakeflow_framework +databricks bundle deploy -t dev +``` + +### 2. Install the Skill + +Upload this skill folder to your workspace's `.assistant/skills/` directory: + +```bash +databricks workspace import-dir \ + ./skills/dataflowspec_builder \ + "/Workspace/Users//.assistant/skills/dataflow-spec-builder" +``` + +### 3. Use in Genie Code + +Open a notebook, enter Genie Code Agent mode, and ask: + +``` +Use the dataflow-spec-builder to create a bronze Data Flow Spec for ingesting +raw_customers from main.my_schema with SCD Type 1 CDC. +``` + +See [docs/example-prompts.md](docs/example-prompts.md) for 30+ tested prompts. + +## Example Prompts + +| Prompt | What Gets Generated | +|--------|-------------------| +| "Create bronze Data Flow Specs for 7 energy tables using a template" | Template definition + 7 parameter sets + schemas | +| "Generate a silver Data Flow Spec merging customers and billing with SCD2" | Flows spec with multi-source streaming pattern | +| "Create gold materialized view Data Flow Specs for revenue KPIs" | MV spec with inline SQL | +| "Add data quality expectations to drop null IDs and negative amounts" | Expectations JSON with expect_or_drop rules | +| "Generate a stream-static join Data Flow Spec for meter + weather" | Flows spec with deltaJoin | +| "Set up dev/staging/prod substitutions for different catalogs" | 3 environment config files | + +> **Tip:** Always include "Data Flow Spec" or "dataflow-spec-builder" in your prompt to ensure Genie Code routes to this skill instead of native Lakeflow Declarative Pipelines. + +## Repository Structure + +``` +skills/dataflowspec_builder/ +├── README.md # This file +├── SKILL.md # Agent Skill definition (Genie Code reads this) +│ +├── docs/ +│ ├── getting-started.md # Setup and installation guide +│ ├── example-prompts.md # 30+ tested prompts with explanations +│ ├── architecture.md # How the framework and skill work +│ ├── tested-medallion-example.md # Verified end-to-end example with results +│ └── skill-development.md # How to customize the skill +│ +├── scripts/ +│ ├── scaffold_lakeflow_bundle.py # Auto-generate pipeline bundles +│ ├── validate_specs.py # Validate Data Flow Spec files +│ ├── deploy_framework.sh # Deploy framework to workspace +│ └── deploy_pipeline_bundle.sh # Deploy pipeline bundles +│ +├── assets/ +│ ├── dataflowspec-templates/ # Reusable Data Flow Spec templates +│ │ ├── standard_bronze_ingestion.json +│ │ ├── standard_cloudfiles_ingestion.json +│ │ ├── flows_multi_source_silver.json +│ │ ├── flows_stream_static_join.json +│ │ └── materialized_view_gold.json +│ ├── pipeline-resource-templates/ # Pipeline YAML templates +│ │ ├── single_pipeline.yml +│ │ └── filtered_pipeline.yml +│ ├── substitution-templates/ # Environment configs +│ │ ├── dev_substitutions.json +│ │ └── prod_substitutions.json +│ └── extension-templates/ # Python extension stubs +│ ├── sources.py +│ ├── transforms.py +│ └── sinks.py +│ +├── examples/ +│ ├── energy-bronze/ # Tested bronze ingestion example +│ │ ├── dataflowspec/energy_bronze_main.json +│ │ ├── schemas/ # 7 StructType schema files +│ │ └── expectations/ # 4 DQ expectation files +│ ├── energy-silver/ # Tested silver transform examples +│ │ ├── dataflowspec/customer_360_main.json +│ │ ├── dataflowspec/meter_weather_join_main.json +│ │ └── expectations/ +│ ├── energy-gold/ # Tested gold MV examples +│ │ ├── dataflowspec/energy_kpis_main.json +│ │ └── dml/mv_consumption_heatmap.sql +│ └── energy-templates/ # Template example +│ └── dataflowspec/energy_bronze_ingestion_template.json +│ +└── references/ + ├── patterns-guide.md # Pattern selection quick reference + ├── dataflow-spec-schema-reference.md # Complete field-by-field reference + └── energy-domain-mapping.md # Energy tables → patterns mapping +``` + +## Tested and Verified + +This skill was tested end-to-end on a live Databricks workspace with: + +- **Bronze pipeline:** 7 streaming tables ingesting 10.7M+ rows with SCD1 CDC and operational metadata +- **Gold pipeline:** 3 materialized views with SQL aggregations (revenue by state, grid reliability, equipment risk) +- **Framework:** Data Flow Spec Framework v0.4.0 deployed via DABs +- **Compute:** Serverless pipelines on Unity Catalog + +See [docs/tested-medallion-example.md](docs/tested-medallion-example.md) for full details and results. + +## Documentation + +| Document | Description | +|----------|-------------| +| [Getting Started](docs/getting-started.md) | Setup, installation, and first pipeline | +| [Example Prompts](docs/example-prompts.md) | 30+ tested prompts organized by layer and feature | +| [Architecture](docs/architecture.md) | How the framework and skill work together | +| [Tested Example](docs/tested-medallion-example.md) | Verified medallion architecture with real results | +| [Skill Development](docs/skill-development.md) | How to customize for your domain | +| [Patterns Guide](references/patterns-guide.md) | Quick reference for selecting the right pattern | +| [Schema Reference](references/dataflow-spec-schema-reference.md) | Complete field reference for all spec types | + +## Prerequisites + +- Databricks workspace with Unity Catalog enabled +- Databricks CLI installed and configured +- Genie Code enabled on the workspace +- Data Flow Spec Framework v0.4.0+ deployed ([instructions](docs/getting-started.md#step-1-deploy-the-data-flow-spec-framework)) +- Python 3.9+ (for scaffolding and validation scripts) + +## Related + +- [Data Flow Spec Framework](https://github.com/databricks-solutions/lakeflow_framework) — the underlying framework +- [Framework Documentation](https://databricks-solutions.github.io/lakeflow_framework/) — official docs +- [Databricks Asset Bundles](https://docs.databricks.com/dev-tools/bundles/index.html) — DABs reference +- [Spark Declarative Pipelines](https://docs.databricks.com/en/delta-live-tables/index.html) — SDP/DLT docs + +## License + +This project is provided as-is for educational and workshop purposes. diff --git a/skills/dataflowspec_builder/SKILL.md b/skills/dataflowspec_builder/SKILL.md new file mode 100644 index 0000000..7b016d5 --- /dev/null +++ b/skills/dataflowspec_builder/SKILL.md @@ -0,0 +1,1080 @@ +--- +name: dataflow-spec-builder +description: "Generates Data Flow Spec pipeline bundles using the metadata-driven framework from github.com/databricks-solutions/lakeflow_framework. Creates Data Flow Specs (standard, flows, materialized views), schemas, expectations, SQL transforms, Python extensions, substitutions, templates, pipeline resource YAMLs, and databricks.yml from natural language. Covers CDC (SCD1/2), data quality, quarantine, liquid clustering, multi-source streaming, table migration, secrets, soft deletes, operational metadata, and DABs deployment. This is NOT native Lakeflow Declarative Pipelines — it is the Data Flow Spec Framework that wraps SDP with configuration-over-code." +--- + +# Data Flow Spec Builder + +Generate complete, production-ready pipeline bundles using the [Data Flow Spec Framework](https://github.com/databricks-solutions/lakeflow_framework) (v0.4.0) from natural language descriptions. This is a metadata-driven data engineering framework that wraps Databricks Spark Declarative Pipelines (SDP) with a configuration-over-code approach using JSON/YAML Data Flow Specs. + +> **IMPORTANT:** This skill builds pipelines using the **Data Flow Spec Framework** (`databricks-solutions/lakeflow_framework`), NOT native Lakeflow Declarative Pipelines (DLT). The Data Flow Spec Framework is a metadata-driven wrapper that uses JSON configuration files called "Data Flow Specs" to define pipelines declaratively. + +## When to Use + +- User says "generate a Data Flow Spec", "use the Data Flow Spec framework", "scaffold a DFS pipeline bundle", or "dataflow spec builder" +- User says "use the metadata-driven framework" or "databricks-solutions/lakeflow_framework" +- User needs metadata-driven SDP pipelines with JSON/YAML Data Flow Spec configuration files +- User wants CDC (SCD1/2), data quality expectations, quarantine, or multi-source streaming via Data Flow Specs +- User asks for medallion architecture pipelines using Data Flow Specs (not native DLT/Lakeflow syntax) +- User wants to create reusable pipeline templates for multiple similar tables using the Data Flow Spec Framework +- User needs environment-portable pipelines with substitutions (dev/sit/prod) using Data Flow Spec configuration +- User references `databricks-solutions/lakeflow_framework` or "DFS framework" + +## When NOT to Use + +- User wants native Lakeflow Declarative Pipelines (DLT) with Python/SQL decorators like `@dlt.table` or `CREATE STREAMING TABLE` — that is native DLT, not this framework +- User wants to write pipeline code directly without JSON/YAML configuration files + +## Parameters + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `description` | Yes | Natural language description of the pipeline(s) to build | +| `catalog` | Yes | Unity Catalog name (e.g., `main`) | +| `schema` | Yes | Target schema (e.g., `energy_workshop`) | +| `framework_source_path` | No | Workspace path where the framework bundle is deployed (default: `/Workspace/Users/{owner}/.bundle/lakeflow_framework/dev/current/files/src`) | +| `spec_format` | No | `json` (default) or `yaml` | +| `layer` | No | Medallion layer: `bronze`, `silver`, or `gold` | +| `pattern` | No | Pipeline pattern to use (see Patterns section) | +| `environments` | No | List of target environments for substitutions (e.g., `dev,sit,prod`) | + +## Workflow + +1. **Analyze request** — Determine data flow type (standard / flows / materialized_view), pattern, layer, sources, targets, and features needed +2. **Generate Data Flow Spec** — Create the main JSON/YAML spec file(s) with correct schema +3. **Generate schemas** — Create StructType JSON schema files for sources and targets +4. **Generate expectations** — Create data quality expectation files if DQ is needed +5. **Generate SQL transforms** — Create SQL files for silver/gold transforms +6. **Generate Python extensions** — Create custom source/transform/sink modules if complex logic is needed +7. **Generate substitutions** — Create environment-specific substitution configs +8. **Generate pipeline resource YAML** — Create `resources/*.yml` with pipeline definition and filters +9. **Generate databricks.yml** — Create the DAB bundle definition +10. **Generate templates** — If the user has multiple similar tables, create reusable Lakeflow templates +11. **Deploy** — Optionally run `databricks bundle deploy` and `databricks bundle run` + +--- + +## Architecture Overview + +The Lakeflow Framework has **two bundle types**: + +### Framework Bundle (deployed once per workspace) +Contains the core engine code. Deployed to workspace files via: +```bash +cd lakeflow_framework +databricks bundle deploy -t dev +``` + +### Pipeline Bundle (one per project/domain) +Contains your Data Flow Specs, schemas, expectations, transforms, and pipeline definitions: +``` +my_pipeline_bundle/ +├── databricks.yml # DAB bundle definition +├── resources/ +│ └── my_pipeline.yml # SDP pipeline definition(s) +└── src/ + ├── dataflows/ + │ └── my_table/ + │ ├── dataflowspec/ + │ │ └── my_table_main.json # Data Flow Spec + │ ├── schemas/ + │ │ └── my_table_schema.json # StructType schema + │ ├── expectations/ + │ │ └── my_table_dqe.json # DQ expectations + │ └── dml/ + │ └── transform.sql # SQL transforms + ├── extensions/ # Python extensions (optional) + │ ├── sources.py + │ ├── transforms.py + │ └── sinks.py + ├── templates/ # Reusable templates (optional) + │ └── my_template.json + └── pipeline_configs/ + ├── dev_substitutions.json # Env substitutions + └── pipeline_config.json # Pipeline-level config +``` + +--- + +## Data Flow Types + +### 1. Standard Data Flow (Bronze / 1:1 Ingestion) + +Best for: ingestion, basic 1:1 loads, single-source-to-single-target. + +```json +{ + "dataFlowId": "", + "dataFlowGroup": "", + "dataFlowType": "standard", + "sourceType": "", + "sourceSystem": "", + "sourceViewName": "v_", + "sourceDetails": { + "database": "", + "table": "", + "cdfEnabled": false, + "schemaPath": "schemas/.json", + "selectExp": ["*"], + "whereClause": [], + "readerOptions": {} + }, + "mode": "", + "targetFormat": "delta", + "targetDetails": { + "table": "", + "schemaPath": "schemas/.json", + "tableProperties": { + "delta.autoOptimize.optimizeWrite": "true", + "delta.autoOptimize.autoCompact": "true" + }, + "partitionColumns": [], + "clusterByColumns": [], + "clusterByAuto": false + }, + "cdcSettings": { + "keys": [""], + "sequence_by": "", + "scd_type": "<1|2>", + "where": "", + "ignore_null_updates": true, + "except_column_list": [], + "apply_as_deletes": "", + "track_history_column_list": [], + "track_history_except_column_list": [] + }, + "dataQualityExpectationsEnabled": false, + "dataQualityExpectationsPath": "", + "quarantineMode": "", + "quarantineTargetDetails": { + "targetFormat": "delta", + "table": "", + "tableProperties": {} + }, + "tableMigrationDetails": { + "enabled": false, + "catalogType": "", + "autoStartingVersionsEnabled": true, + "sourceDetails": { + "sourceMigrateDelta": { + "database": "", + "table": "", + "selectExp": [], + "whereClause": [], + "exceptColumns": [] + } + } + } +} +``` + +**Source Types for Standard:** +- `cloudFiles` — Auto Loader from UC Volumes or cloud storage (S3/ADLS/GCS) +- `delta` — Existing Delta table, optionally with CDF +- `deltaJoin` — Join multiple Delta tables (stream + static) +- `kafka` — Apache Kafka topics + +**Source Details by Type:** + +For `cloudFiles`: +```json +"sourceDetails": { + "path": "/Volumes/////", + "readerOptions": { + "cloudFiles.format": "", + "header": "true", + "cloudFiles.inferColumnTypes": "true", + "cloudFiles.schemaLocation": "" + }, + "schemaPath": "schemas/.json" +} +``` + +For `delta`: +```json +"sourceDetails": { + "database": ".", + "table": "", + "cdfEnabled": true, + "schemaPath": "schemas/.json", + "selectExp": ["col1", "col2", "CAST(col3 AS DATE) AS col3_date"], + "whereClause": ["col1 IS NOT NULL"] +} +``` + +For `deltaJoin`: +```json +"sourceDetails": { + "sources": [ + { + "alias": "a", + "database": ".", + "table": "", + "mode": "stream", + "cdfEnabled": true + }, + { + "alias": "b", + "database": ".", + "table": "", + "mode": "static" + } + ], + "joinType": "", + "joinCondition": "a.key_col = b.key_col", + "selectExp": ["a.*", "b.description"] +} +``` + +For `kafka`: +```json +"sourceDetails": { + "readerOptions": { + "kafka.bootstrap.servers": "", + "subscribe": "", + "startingOffsets": "earliest" + }, + "schemaPath": "schemas/.json" +} +``` + +### 2. Flows Data Flow (Silver / Multi-Source Streaming) + +Best for: complex transformations, multiple sources merging into one target, silver/gold layers. + +```json +{ + "dataFlowId": "", + "dataFlowGroup": "", + "dataFlowType": "flow", + "targetFormat": "delta", + "targetDetails": { + "table": "", + "schemaPath": "", + "tableProperties": { + "delta.enableChangeDataFeed": "true" + }, + "partitionColumns": [], + "clusterByColumns": [] + }, + "cdcSettings": { + "keys": [""], + "sequence_by": "", + "scd_type": "<1|2>", + "where": "", + "ignore_null_updates": true, + "except_column_list": [] + }, + "dataQualityExpectationsEnabled": false, + "quarantineMode": "off", + "quarantineTargetDetails": {}, + "flowGroups": [ + { + "flowGroupId": "", + "stagingTables": { + "": { + "type": "", + "schemaPath": "", + "partitionColumns": [], + "tableProperties": {}, + "cdcSettings": {} + } + }, + "flows": { + "": { + "enabled": true, + "flowType": "", + "flowDetails": { + "targetTable": "", + "sourceView": "" + }, + "views": { + "": { + "mode": "", + "sourceType": "", + "sourceDetails": { + "database": ".", + "table": "", + "cdfEnabled": true, + "selectExp": ["*"], + "whereClause": [] + } + } + } + } + } + } + ] +} +``` + +**Flow Types:** +- `append_view` — Append data from a source view to a staging/target table +- `append_sql` — Append data using a raw SQL statement (use `sqlPath` in flowDetails instead of `sourceView`) +- `merge` — CDC merge from a source view/staging table to the target table + +**Staging Table Types:** +- `ST` — Streaming Table +- `MV` — Materialized View + +**Multi-Source Streaming Pattern:** Stream multiple sources into a single staging table via `append_view` flows, then `merge` the staging table into the final target. Flow groups can be added/removed over time without requiring a full pipeline refresh. + +### 3. Materialized View Data Flow (Gold / Aggregations) + +Best for: precomputed aggregations, complex joins, gold layer KPIs. + +```json +{ + "dataFlowId": "", + "dataFlowGroup": "", + "dataFlowType": "materialized_view", + "materializedViews": { + "": { + "sourceView": { + "sourceViewName": "v_", + "sourceType": "", + "sourceDetails": { + "database": ".", + "table": "", + "cdfEnabled": false + } + }, + "sqlPath": "", + "sqlStatement": "", + "tableDetails": { + "database": ".", + "schemaPath": "schemas/.json", + "tableProperties": { + "delta.autoOptimize.optimizeWrite": "true" + }, + "partitionColumns": [], + "clusterByColumns": ["", ""], + "clusterByAuto": false, + "comment": "" + }, + "dataQualityExpectationsEnabled": false, + "dataQualityExpectationsPath": "", + "quarantineMode": "off", + "quarantineTargetDetails": {} + } + } +} +``` + +**Materialized View Source Options (pick one per MV):** +1. `sourceView` — Define a source view over a Delta table or Python function +2. `sqlPath` — Path to a `.sql` file containing the MV query +3. `sqlStatement` — Inline SQL string + +Multiple materialized views can be defined in a single Data Flow Spec by adding more keys under `materializedViews`. + +--- + +## Feature Reference + +### CDC — Change Data Capture (SCD Type 1 & 2) + +Add `cdcSettings` to any standard or flows data flow spec: + +```json +"cdcSettings": { + "keys": ["customer_id"], + "sequence_by": "updated_timestamp", + "scd_type": "2", + "where": "", + "ignore_null_updates": true, + "except_column_list": ["_metadata", "load_timestamp"], + "apply_as_deletes": "DELETE_FLAG = true", + "track_history_column_list": [], + "track_history_except_column_list": ["_metadata"] +} +``` + +| Field | Description | +|-------|-------------| +| `keys` | Primary key column(s) to identify records | +| `sequence_by` | Timestamp column for ordering CDC events | +| `scd_type` | `"1"` (overwrite) or `"2"` (history tracking with `__START_AT` / `__END_AT`) | +| `apply_as_deletes` | SQL expression — when true, treat the event as a DELETE | +| `ignore_null_updates` | If true, null values in updates don't overwrite existing data | +| `except_column_list` | Columns to exclude from the upsert | +| `track_history_column_list` | Explicit list of columns to track history for (SCD2 only) | +| `track_history_except_column_list` | Columns to exclude from history tracking (SCD2 only) | + +### CDC from Historical Snapshots + +For snapshot-based CDC ingestion: +```json +"cdcSnapshotSettings": { + "source": { + "format": "parquet", + "path": "/Volumes/main/raw/snapshots/{version}/", + "versionType": "datetime", + "datetimeFormat": "yyyyMMdd", + "startingVersion": "20240101", + "readerOptions": {}, + "schemaPath": "schemas/snapshot_schema.json", + "selectExp": ["*"], + "filter": "active = true", + "recursiveFileLookup": false + } +} +``` + +For table-based snapshot CDC: +```json +"cdcSnapshotSettings": { + "source": { + "table": "main.raw.daily_snapshot", + "versionColumn": "snapshot_date", + "startingVersion": "2024-01-01", + "selectExp": ["*"] + } +} +``` + +### CDF — Change Data Feed + +Enable CDF reading on a source: +```json +"sourceDetails": { + "database": "main.bronze", + "table": "customer", + "cdfEnabled": true +} +``` + +Enable CDF on a target table: +```json +"targetDetails": { + "table": "silver_customer", + "tableProperties": { + "delta.enableChangeDataFeed": "true" + } +} +``` + +### Data Quality — Expectations + +Create an expectations file (`expectations/_dqe.json`): +```json +{ + "expect": [ + {"name": "valid_id", "constraint": "customer_id IS NOT NULL", "tag": "completeness", "enabled": true}, + {"name": "valid_email", "constraint": "email LIKE '%@%.%'", "tag": "validity", "enabled": true} + ], + "expect_or_drop": [ + {"name": "positive_amount", "constraint": "amount > 0", "tag": "range", "enabled": true} + ], + "expect_or_fail": [ + {"name": "unique_key", "constraint": "customer_id IS NOT NULL", "tag": "integrity", "enabled": true} + ] +} +``` + +**Expectation types:** +- `expect` — Log violations but keep the record +- `expect_or_drop` — Drop records that violate +- `expect_or_fail` — Fail the entire pipeline if any record violates + +Enable in the data flow spec: +```json +"dataQualityExpectationsEnabled": true, +"dataQualityExpectationsPath": "my_table_dqe.json" +``` + +### Data Quality — Quarantine + +Three quarantine modes: +- `"off"` — No quarantine (default) +- `"flag"` — Add a `_quarantine` boolean column to the target table +- `"table"` — Write bad records to a separate quarantine table + +```json +"quarantineMode": "table", +"quarantineTargetDetails": { + "targetFormat": "delta", + "table": "customer_quarantine", + "tableProperties": {} +} +``` + +### Liquid Clustering + +Enable automatic or manual clustering on target tables: + +```json +"targetDetails": { + "table": "silver_meter_readings", + "clusterByColumns": ["customer_id", "reading_date"], + "clusterByAuto": false +} +``` + +Or use automatic clustering (Databricks chooses optimal keys): +```json +"targetDetails": { + "table": "silver_meter_readings", + "clusterByAuto": true +} +``` + +### Schemas + +Schemas are defined in StructType JSON format: +```json +{ + "type": "struct", + "fields": [ + {"name": "customer_id", "type": "integer", "nullable": false, "metadata": {}}, + {"name": "name", "type": "string", "nullable": true, "metadata": {}}, + {"name": "email", "type": "string", "nullable": true, "metadata": {}}, + {"name": "created_at", "type": "timestamp", "nullable": true, "metadata": {}}, + {"name": "balance", "type": {"type": "decimal", "precision": 10, "scale": 2}, "nullable": true, "metadata": {}} + ] +} +``` + +Or Text DDL format: +``` +customer_id INT NOT NULL, name STRING, email STRING, created_at TIMESTAMP, balance DECIMAL(10,2) +``` + +Generate from an existing table: +```python +schema_json = spark.table("main.energy.raw_customers").schema.json() +``` + +### Substitutions + +Create environment-specific config files (`src/pipeline_configs/_substitutions.json`): + +```json +{ + "tokens": { + "bronze_schema": "main.bronze_energy", + "silver_schema": "main.silver_energy", + "gold_schema": "main.gold_energy", + "landing_path": "/Volumes/main/energy/landing" + }, + "prefix_suffix": { + "database": { + "suffix": "_{workspace_env}" + } + } +} +``` + +Use tokens in Data Flow Specs with curly braces: `{bronze_schema}`, `{landing_path}`. + +**Reserved tokens:** +- `{workspace_env}` — The DABs target environment name (dev, sit, prod) + +**Precedence:** Pipeline substitutions override framework-level substitutions. + +### Templates + +Create reusable patterns for multiple similar tables. + +**Template definition** (`src/templates/.json`): +```json +{ + "name": "bronze_ingestion_template", + "parameters": { + "dataFlowId": {"type": "string", "required": true}, + "sourceTable": {"type": "string", "required": true}, + "targetTable": {"type": "string", "required": true}, + "schemaPath": {"type": "string", "required": true}, + "keyColumns": {"type": "list", "required": true}, + "sequenceBy": {"type": "string", "required": true} + }, + "template": { + "dataFlowId": "${param.dataFlowId}", + "dataFlowGroup": "bronze_energy", + "dataFlowType": "standard", + "sourceType": "delta", + "sourceSystem": "smartgrid", + "sourceViewName": "v_${param.sourceTable}", + "sourceDetails": { + "database": "{bronze_schema}", + "table": "${param.sourceTable}", + "cdfEnabled": true, + "schemaPath": "${param.schemaPath}" + }, + "mode": "stream", + "targetFormat": "delta", + "targetDetails": { + "table": "${param.targetTable}", + "tableProperties": { + "delta.autoOptimize.optimizeWrite": "true" + } + }, + "cdcSettings": { + "keys": "${param.keyColumns}", + "sequence_by": "${param.sequenceBy}", + "scd_type": "1" + } + } +} +``` + +**Template usage** (`src/dataflows//dataflowspec/_main.json`): +```json +{ + "template": "bronze_ingestion_template", + "parameterSets": [ + { + "dataFlowId": "customers_bronze", + "sourceTable": "raw_customers", + "targetTable": "bronze_customers", + "schemaPath": "schemas/customers_schema.json", + "keyColumns": ["customer_id"], + "sequenceBy": "updated_at" + }, + { + "dataFlowId": "billing_bronze", + "sourceTable": "raw_billing", + "targetTable": "bronze_billing", + "schemaPath": "schemas/billing_schema.json", + "keyColumns": ["bill_id"], + "sequenceBy": "bill_date" + } + ] +} +``` + +**Parameter types:** `string`, `integer`, `boolean`, `list`, `object` + +### Python Extensions + +Custom Python modules in `src/extensions/`: + +**Source extension** (`sources.py`): +```python +from pyspark.sql import DataFrame, SparkSession +from typing import Dict + +def get_custom_source(spark: SparkSession, tokens: Dict) -> DataFrame: + source_table = tokens["sourceTable"] + return spark.readStream.option("readChangeFeed", "true").table(source_table) +``` + +Reference in spec: +```json +"sourceType": "python", +"sourceDetails": { + "tokens": {"sourceTable": "{bronze_schema}.customer"}, + "pythonModule": "sources.get_custom_source" +} +``` + +**Transform extension** (`transforms.py`): +```python +from pyspark.sql import DataFrame +from pyspark.sql import functions as F + +def clean_and_enrich(df: DataFrame) -> DataFrame: + return df.withColumn("processed_at", F.current_timestamp()).dropDuplicates(["id"]) +``` + +Reference in spec: +```json +"sourceDetails": { + "database": "{bronze_schema}", + "table": "customer", + "pythonTransform": { + "module": "transforms.clean_and_enrich" + } +} +``` + +**Sink extension** (`sinks.py`): +```python +from pyspark.sql import DataFrame +from typing import Dict + +def write_to_api(df: DataFrame, batch_id: int, tokens: Dict) -> None: + import requests + api_url = tokens["apiUrl"] + records = df.toJSON().collect() + for record in records: + requests.post(api_url, json=record) +``` + +Reference in spec: +```json +"targetFormat": "foreach_batch_sink", +"targetDetails": { + "name": "api_sink", + "type": "python_function", + "config": { + "module": "sinks.write_to_api", + "tokens": {"apiUrl": "https://api.example.com/data"} + } +} +``` + +### Python Function Transforms + +Inline Python transforms via file path (alternative to extensions): + +```json +"sourceDetails": { + "database": "{bronze_schema}", + "table": "meter_readings", + "pythonTransform": { + "path": "python_functions/clean_readings.py", + "functionName": "clean_readings" + } +} +``` + +### Soft Deletes + +Enable soft deletes instead of physical deletes: +```json +"cdcSettings": { + "keys": ["customer_id"], + "sequence_by": "updated_at", + "scd_type": "1", + "apply_as_deletes": "operation = 'DELETE'", + "soft_delete": true +} +``` + +### Operational Metadata + +The framework automatically adds operational metadata columns. Disable per spec: +```json +"operationalMetadata": { + "enabled": false +} +``` + +Or disable at target table level: +```json +"targetDetails": { + "table": "my_table", + "operationalMetadata": false +} +``` + +### Mandatory Table Properties + +Set in framework config (`src/config/mandatory_table_properties.json`): +```json +{ + "mandatory_properties": { + "delta.autoOptimize.optimizeWrite": "true", + "delta.autoOptimize.autoCompact": "true", + "delta.enableChangeDataFeed": "true" + } +} +``` + +### Spark Configuration + +Per-materialized-view Spark config: +```json +"tableDetails": { + "spark_conf": { + "spark.sql.shuffle.partitions": "200", + "spark.databricks.delta.optimizeWrite.enabled": "true" + } +} +``` + +### Secrets Management + +Reference Databricks secrets in Data Flow Specs: +```json +"sourceDetails": { + "readerOptions": { + "kafka.bootstrap.servers": "{{secrets/my-scope/kafka-brokers}}", + "kafka.sasl.jaas.config": "{{secrets/my-scope/kafka-jaas}}" + } +} +``` + +### Table Migration + +Migrate from HMS or another UC catalog: +```json +"tableMigrationDetails": { + "enabled": true, + "catalogType": "hms", + "autoStartingVersionsEnabled": true, + "sourceDetails": { + "sourceMigrateDelta": { + "database": "legacy_db", + "table": "old_customer", + "selectExp": ["customer_id", "name", "email"], + "whereClause": ["active = true"], + "exceptColumns": ["_metadata"] + } + } +} +``` + +### Logical Environments + +Set in `databricks.yml`: +```yaml +variables: + logical_env: + description: Logical environment suffix + default: "_dev" +``` + +Pass via CLI: +```bash +databricks bundle deploy -t dev --var="logical_env=_test" +``` + +### Builder Parallelization + +Configure in framework config to parallelize spec building: +```json +{ + "parallelization": { + "enabled": true, + "max_workers": 4 + } +} +``` + +### Versioning — Data Flow Specs + +Version your data flow specs with mapping operations: +```json +{ + "version": "2.0", + "mappings": [ + {"operation": "rename", "from": "old_field", "to": "new_field"}, + {"operation": "move", "from": "sourceDetails.db", "to": "sourceDetails.database"}, + {"operation": "delete", "path": "deprecated_field"} + ] +} +``` + +### Versioning — Framework + +Deploy specific framework versions: +```bash +databricks bundle deploy -t dev --var="version=0.4.0" +``` + +Reference in pipeline resources: +```yaml +variables: + framework_version: + default: "0.4.0" + framework_source_path: + default: /Workspace/Users/${var.owner}/.bundle/lakeflow_framework/dev/${var.framework_version}/files/src +``` + +### UI Integration + +Enable pipeline editor UI integration: +```yaml +configuration: + ui.pipeline.editorEnabled: "true" +``` + +### Logging + +Set in pipeline YAML: +```yaml +configuration: + framework.logLevel: "INFO" +``` + +Levels: `DEBUG`, `INFO`, `WARN`, `ERROR` + +--- + +## Pipeline Resource YAML + +Every Pipeline Bundle needs at least one resource YAML in `resources/`: + +```yaml +resources: + pipelines: + : + name: + catalog: ${var.catalog} + schema: ${var.schema} + channel: CURRENT + serverless: true + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: /Workspace/${workspace.file_path}/src + framework.sourcePath: /Workspace/${var.framework_source_path} + workspace.host: ${workspace.host} + bundle.target: ${bundle.target} + pipeline.layer: ${var.layer} +``` + +**Pipeline Filters** (restrict which data flows a pipeline executes): + +| Config Key | Description | +|------------|-------------| +| `pipeline.dataFlowIdFilter` | Filter by data flow ID(s) | +| `pipeline.dataFlowGroupFilter` | Filter by data flow group(s) | +| `pipeline.flowGroupIdFilter` | Filter by flow group ID(s) | +| `pipeline.fileFilter` | Filter by data flow spec file path | +| `pipeline.targetTableFilter` | Filter by target table name(s) | + +Multiple values: comma-separated. + +--- + +## databricks.yml Template + +```yaml +bundle: + name: + +include: + - resources/*.yml + +variables: + owner: + description: The owner of the bundle + default: ${workspace.current_user.userName} + catalog: + description: The target UC catalog + default: main + schema: + description: The target UC schema + default: + layer: + description: The target medallion layer + default: bronze + framework_source_path: + description: Path to the deployed Lakeflow Framework + default: /Workspace/Users/${var.owner}/.bundle/lakeflow_framework/dev/current/files/src + +targets: + dev: + mode: development + default: true + workspace: + host: + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} + variables: + framework_source_path: /Workspace/Users/${var.owner}/.bundle/lakeflow_framework/dev/current/files/src + + staging: + workspace: + host: + variables: + schema: + framework_source_path: /Workspace/Users/${var.owner}/.bundle/lakeflow_framework/staging/current/files/src + + prod: + workspace: + host: + variables: + schema: + framework_source_path: /Workspace/shared/.bundle/lakeflow_framework/prod/current/files/src +``` + +--- + +## Patterns + +| Pattern | Layer | Data Flow Type | Description | +|---------|-------|---------------|-------------| +| Basic 1:1 | Bronze | `standard` | Single source → single target, append or SCD1/2 | +| Stream-Static Basic | Silver | `flow` | Streaming fact table joined with static dimension | +| Stream-Static DWH | Silver/Gold | `flow` | Full streaming data warehouse with multiple fact/dim joins | +| Multi-Source Streaming | Silver | `flow` | Multiple sources → single staging → CDC merge to target | +| CDC from Snapshots | Bronze | `standard` | Ingest periodic snapshot files with automatic CDC | + +### Pattern: Basic 1:1 (Bronze Ingestion) +- Data Flow Type: `standard` +- Use when: Ingesting data or performing 1:1 loads +- Supports: Append-only, SCD1, SCD2 +- Basic single-row transforms only (type conversion, formatting, cleansing) + +### Pattern: Stream-Static (Silver Join) +- Data Flow Type: `flow` +- Use when: Joining a streaming source with a static dimension table +- Flow structure: `append_view` for stream source → join view (batch, deltaJoin) → `merge` to target + +### Pattern: Multi-Source Streaming (Silver/Gold Merge) +- Data Flow Type: `flow` +- Use when: Multiple sources need to be merged into a single target +- Flow structure: Multiple `append_view` flows into a staging table → `merge` flow to target +- Key benefit: Add/remove flow groups without full pipeline refresh + +--- + +## Deployment + +### Deploy Framework Bundle +```bash +git clone https://github.com/databricks-solutions/lakeflow_framework.git +cd lakeflow_framework +databricks bundle deploy -t dev +``` + +### Deploy Pipeline Bundle +```bash +cd my_pipeline_bundle +databricks bundle deploy -t dev +``` + +### Run a Pipeline +```bash +databricks bundle run -t dev +``` + +### Validate Before Deploy +```bash +databricks bundle validate -t dev +``` + +### Destroy (cleanup) +```bash +databricks bundle destroy -t dev +``` + +--- + +## Script: scaffold_lakeflow_bundle.py + +Use the scaffolding script at `scripts/scaffold_lakeflow_bundle.py` to generate a complete Data Flow Spec pipeline bundle: + +```bash +python scripts/scaffold_lakeflow_bundle.py \ + --name "energy_bronze" \ + --catalog main \ + --schema energy_workshop \ + --layer bronze \ + --pattern basic_1_1 \ + --tables "raw_customers,raw_meter_readings,raw_billing" \ + --format json \ + --workspace-host "https://my-workspace.cloud.databricks.com" +``` + +This generates the full directory structure, Data Flow Specs, schemas, expectations, pipeline YAML, and databricks.yml. + +## References + +- [Data Flow Spec Framework Documentation](https://databricks-solutions.github.io/lakeflow_framework/) +- [GitHub Repository — databricks-solutions/lakeflow_framework](https://github.com/databricks-solutions/lakeflow_framework) +- [Databricks Asset Bundles](https://docs.databricks.com/dev-tools/bundles/index.html) +- [Spark Declarative Pipelines](https://docs.databricks.com/en/delta-live-tables/index.html) +- See `references/` folder for pattern guides, energy-specific examples, and full schema reference + +## Prerequisites + +- Databricks CLI installed and configured +- Unity Catalog enabled workspace +- Data Flow Spec Framework bundle deployed to the target workspace (from `databricks-solutions/lakeflow_framework`) +- Python 3.9+ (for scaffolding script) diff --git a/skills/dataflowspec_builder/assets/dataflowspec-templates/flows_multi_source_silver.json b/skills/dataflowspec_builder/assets/dataflowspec-templates/flows_multi_source_silver.json new file mode 100644 index 0000000..cd60392 --- /dev/null +++ b/skills/dataflowspec_builder/assets/dataflowspec-templates/flows_multi_source_silver.json @@ -0,0 +1,96 @@ +{ + "name": "flows_multi_source_silver", + "parameters": { + "dataFlowId": {"type": "string", "required": true}, + "targetTable": {"type": "string", "required": true}, + "keyColumns": {"type": "list", "required": true}, + "sequenceBy": {"type": "string", "required": true}, + "scdType": {"type": "string", "required": false, "default": "1"}, + "sourceTable1": {"type": "string", "required": true}, + "sourceDb1": {"type": "string", "required": true}, + "sourceTable2": {"type": "string", "required": true}, + "sourceDb2": {"type": "string", "required": true} + }, + "template": { + "dataFlowId": "${param.dataFlowId}", + "dataFlowGroup": "silver", + "dataFlowType": "flow", + "targetFormat": "delta", + "targetDetails": { + "table": "${param.targetTable}", + "tableProperties": { + "delta.enableChangeDataFeed": "true" + } + }, + "cdcSettings": { + "keys": "${param.keyColumns}", + "sequence_by": "${param.sequenceBy}", + "scd_type": "${param.scdType}", + "ignore_null_updates": true, + "except_column_list": [] + }, + "dataQualityExpectationsEnabled": false, + "quarantineMode": "off", + "quarantineTargetDetails": {}, + "flowGroups": [ + { + "flowGroupId": "fg_${param.targetTable}", + "stagingTables": { + "staging_${param.targetTable}_apnd": { + "type": "ST", + "schemaPath": "" + } + }, + "flows": { + "f_${param.sourceTable1}": { + "flowType": "append_view", + "flowDetails": { + "targetTable": "staging_${param.targetTable}_apnd", + "sourceView": "v_${param.sourceTable1}" + }, + "views": { + "v_${param.sourceTable1}": { + "mode": "stream", + "sourceType": "delta", + "sourceDetails": { + "database": "${param.sourceDb1}", + "table": "${param.sourceTable1}", + "cdfEnabled": true, + "selectExp": ["*"], + "whereClause": [] + } + } + } + }, + "f_${param.sourceTable2}": { + "flowType": "append_view", + "flowDetails": { + "targetTable": "staging_${param.targetTable}_apnd", + "sourceView": "v_${param.sourceTable2}" + }, + "views": { + "v_${param.sourceTable2}": { + "mode": "stream", + "sourceType": "delta", + "sourceDetails": { + "database": "${param.sourceDb2}", + "table": "${param.sourceTable2}", + "cdfEnabled": true, + "selectExp": ["*"], + "whereClause": [] + } + } + } + }, + "f_merge": { + "flowType": "merge", + "flowDetails": { + "targetTable": "${param.targetTable}", + "sourceView": "staging_${param.targetTable}_apnd" + } + } + } + } + ] + } +} diff --git a/skills/dataflowspec_builder/assets/dataflowspec-templates/flows_stream_static_join.json b/skills/dataflowspec_builder/assets/dataflowspec-templates/flows_stream_static_join.json new file mode 100644 index 0000000..453fdf9 --- /dev/null +++ b/skills/dataflowspec_builder/assets/dataflowspec-templates/flows_stream_static_join.json @@ -0,0 +1,78 @@ +{ + "name": "flows_stream_static_join", + "parameters": { + "dataFlowId": {"type": "string", "required": true}, + "targetTable": {"type": "string", "required": true}, + "keyColumns": {"type": "list", "required": true}, + "sequenceBy": {"type": "string", "required": true}, + "factTable": {"type": "string", "required": true}, + "factDb": {"type": "string", "required": true}, + "dimTable": {"type": "string", "required": true}, + "dimDb": {"type": "string", "required": true}, + "joinCondition": {"type": "string", "required": true}, + "selectExp": {"type": "list", "required": false, "default": ["a.*", "b.*"]} + }, + "template": { + "dataFlowId": "${param.dataFlowId}", + "dataFlowGroup": "silver", + "dataFlowType": "flow", + "targetFormat": "delta", + "targetDetails": { + "table": "${param.targetTable}", + "tableProperties": { + "delta.enableChangeDataFeed": "true" + } + }, + "cdcSettings": { + "keys": "${param.keyColumns}", + "sequence_by": "${param.sequenceBy}", + "scd_type": "1", + "ignore_null_updates": true, + "except_column_list": [] + }, + "dataQualityExpectationsEnabled": false, + "quarantineMode": "off", + "quarantineTargetDetails": {}, + "flowGroups": [ + { + "flowGroupId": "fg_${param.targetTable}", + "stagingTables": {}, + "flows": { + "f_join": { + "flowType": "merge", + "flowDetails": { + "targetTable": "${param.targetTable}", + "sourceView": "v_joined" + }, + "views": { + "v_joined": { + "mode": "stream", + "sourceType": "deltaJoin", + "sourceDetails": { + "sources": [ + { + "alias": "a", + "database": "${param.factDb}", + "table": "${param.factTable}", + "mode": "stream", + "cdfEnabled": true + }, + { + "alias": "b", + "database": "${param.dimDb}", + "table": "${param.dimTable}", + "mode": "static" + } + ], + "joinType": "left", + "joinCondition": "${param.joinCondition}", + "selectExp": "${param.selectExp}" + } + } + } + } + } + } + ] + } +} diff --git a/skills/dataflowspec_builder/assets/dataflowspec-templates/materialized_view_gold.json b/skills/dataflowspec_builder/assets/dataflowspec-templates/materialized_view_gold.json new file mode 100644 index 0000000..20ae597 --- /dev/null +++ b/skills/dataflowspec_builder/assets/dataflowspec-templates/materialized_view_gold.json @@ -0,0 +1,33 @@ +{ + "name": "materialized_view_gold", + "parameters": { + "dataFlowId": {"type": "string", "required": true}, + "mvName": {"type": "string", "required": true}, + "sqlStatement": {"type": "string", "required": true}, + "clusterByColumns": {"type": "list", "required": false, "default": []}, + "targetDatabase": {"type": "string", "required": false, "default": ""}, + "dqEnabled": {"type": "boolean", "required": false, "default": false}, + "comment": {"type": "string", "required": false, "default": ""} + }, + "template": { + "dataFlowId": "${param.dataFlowId}", + "dataFlowGroup": "gold", + "dataFlowType": "materialized_view", + "materializedViews": { + "${param.mvName}": { + "sqlStatement": "${param.sqlStatement}", + "tableDetails": { + "database": "${param.targetDatabase}", + "clusterByColumns": "${param.clusterByColumns}", + "tableProperties": { + "delta.autoOptimize.optimizeWrite": "true" + }, + "comment": "${param.comment}" + }, + "dataQualityExpectationsEnabled": "${param.dqEnabled}", + "quarantineMode": "off", + "quarantineTargetDetails": {} + } + } + } +} diff --git a/skills/dataflowspec_builder/assets/dataflowspec-templates/standard_bronze_ingestion.json b/skills/dataflowspec_builder/assets/dataflowspec-templates/standard_bronze_ingestion.json new file mode 100644 index 0000000..b8e5b2a --- /dev/null +++ b/skills/dataflowspec_builder/assets/dataflowspec-templates/standard_bronze_ingestion.json @@ -0,0 +1,47 @@ +{ + "name": "standard_bronze_ingestion", + "parameters": { + "dataFlowId": {"type": "string", "required": true}, + "sourceSystem": {"type": "string", "required": false, "default": "source"}, + "sourceTable": {"type": "string", "required": true}, + "targetTable": {"type": "string", "required": true}, + "schemaPath": {"type": "string", "required": false, "default": ""}, + "keyColumns": {"type": "list", "required": true}, + "sequenceBy": {"type": "string", "required": true}, + "scdType": {"type": "string", "required": false, "default": "1"}, + "sourceType": {"type": "string", "required": false, "default": "delta"} + }, + "template": { + "dataFlowId": "${param.dataFlowId}", + "dataFlowGroup": "bronze", + "dataFlowType": "standard", + "sourceType": "${param.sourceType}", + "sourceSystem": "${param.sourceSystem}", + "sourceViewName": "v_${param.sourceTable}", + "sourceDetails": { + "database": "{bronze_schema}", + "table": "${param.sourceTable}", + "cdfEnabled": true, + "schemaPath": "${param.schemaPath}" + }, + "mode": "stream", + "targetFormat": "delta", + "targetDetails": { + "table": "${param.targetTable}", + "tableProperties": { + "delta.autoOptimize.optimizeWrite": "true", + "delta.autoOptimize.autoCompact": "true" + } + }, + "cdcSettings": { + "keys": "${param.keyColumns}", + "sequence_by": "${param.sequenceBy}", + "scd_type": "${param.scdType}", + "ignore_null_updates": true, + "except_column_list": [] + }, + "dataQualityExpectationsEnabled": true, + "quarantineMode": "off", + "quarantineTargetDetails": {} + } +} diff --git a/skills/dataflowspec_builder/assets/dataflowspec-templates/standard_cloudfiles_ingestion.json b/skills/dataflowspec_builder/assets/dataflowspec-templates/standard_cloudfiles_ingestion.json new file mode 100644 index 0000000..8615443 --- /dev/null +++ b/skills/dataflowspec_builder/assets/dataflowspec-templates/standard_cloudfiles_ingestion.json @@ -0,0 +1,49 @@ +{ + "name": "standard_cloudfiles_ingestion", + "parameters": { + "dataFlowId": {"type": "string", "required": true}, + "sourceSystem": {"type": "string", "required": false, "default": "files"}, + "sourcePath": {"type": "string", "required": true}, + "fileFormat": {"type": "string", "required": false, "default": "csv"}, + "targetTable": {"type": "string", "required": true}, + "schemaPath": {"type": "string", "required": false, "default": ""}, + "keyColumns": {"type": "list", "required": true}, + "sequenceBy": {"type": "string", "required": true} + }, + "template": { + "dataFlowId": "${param.dataFlowId}", + "dataFlowGroup": "bronze_files", + "dataFlowType": "standard", + "sourceType": "cloudFiles", + "sourceSystem": "${param.sourceSystem}", + "sourceViewName": "v_${param.targetTable}", + "sourceDetails": { + "path": "${param.sourcePath}", + "readerOptions": { + "cloudFiles.format": "${param.fileFormat}", + "header": "true", + "cloudFiles.inferColumnTypes": "true" + }, + "schemaPath": "${param.schemaPath}" + }, + "mode": "stream", + "targetFormat": "delta", + "targetDetails": { + "table": "${param.targetTable}", + "tableProperties": { + "delta.autoOptimize.optimizeWrite": "true", + "delta.autoOptimize.autoCompact": "true" + } + }, + "cdcSettings": { + "keys": "${param.keyColumns}", + "sequence_by": "${param.sequenceBy}", + "scd_type": "1", + "ignore_null_updates": true, + "except_column_list": [] + }, + "dataQualityExpectationsEnabled": true, + "quarantineMode": "off", + "quarantineTargetDetails": {} + } +} diff --git a/skills/dataflowspec_builder/assets/extension-templates/sinks.py b/skills/dataflowspec_builder/assets/extension-templates/sinks.py new file mode 100644 index 0000000..76dc2ec --- /dev/null +++ b/skills/dataflowspec_builder/assets/extension-templates/sinks.py @@ -0,0 +1,62 @@ +""" +Custom sink extensions for the Lakeflow Framework. + +Place this file in src/extensions/sinks.py of your pipeline bundle. +Reference in Data Flow Specs via: + "targetFormat": "foreach_batch_sink", + "targetDetails": { + "name": "my_sink", + "type": "python_function", + "config": { + "module": "sinks.function_name", + "tokens": {"apiUrl": "https://..."} + } + } + +All sink functions must accept (df, batch_id, tokens) signature. +""" +from pyspark.sql import DataFrame +from typing import Dict + + +def write_to_rest_api(df: DataFrame, batch_id: int, tokens: Dict) -> None: + """Send each micro-batch to a REST API endpoint.""" + import requests + + api_url = tokens["apiUrl"] + headers = {"Content-Type": "application/json"} + if tokens.get("apiKey"): + headers["Authorization"] = f"Bearer {tokens['apiKey']}" + + records = df.toJSON().collect() + for record in records: + response = requests.post(api_url, headers=headers, data=record) + response.raise_for_status() + + +def write_to_delta_with_merge(df: DataFrame, batch_id: int, tokens: Dict) -> None: + """Custom merge logic for complex upsert scenarios.""" + from delta.tables import DeltaTable + + target_table = tokens["targetTable"] + merge_keys = tokens.get("mergeKeys", ["id"]) + spark = df.sparkSession + + if not spark.catalog.tableExists(target_table): + df.write.format("delta").saveAsTable(target_table) + return + + dt = DeltaTable.forName(spark, target_table) + merge_condition = " AND ".join([f"target.{k} = source.{k}" for k in merge_keys]) + + (dt.alias("target") + .merge(df.alias("source"), merge_condition) + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute()) + + +def write_to_console(df: DataFrame, batch_id: int, tokens: Dict) -> None: + """Debug sink — prints the micro-batch to the driver log.""" + print(f"=== Batch {batch_id} ({df.count()} rows) ===") + df.show(truncate=False) diff --git a/skills/dataflowspec_builder/assets/extension-templates/sources.py b/skills/dataflowspec_builder/assets/extension-templates/sources.py new file mode 100644 index 0000000..d5cfd74 --- /dev/null +++ b/skills/dataflowspec_builder/assets/extension-templates/sources.py @@ -0,0 +1,54 @@ +""" +Custom source extensions for the Lakeflow Framework. + +Place this file in src/extensions/sources.py of your pipeline bundle. +Reference in Data Flow Specs via: + "sourceType": "python", + "sourceDetails": { + "tokens": {"sourceTable": "catalog.schema.table"}, + "pythonModule": "sources.function_name" + } +""" +from pyspark.sql import DataFrame, SparkSession +from typing import Dict + + +def get_cdf_source(spark: SparkSession, tokens: Dict) -> DataFrame: + """Read a Delta table with Change Data Feed enabled.""" + source_table = tokens["sourceTable"] + return ( + spark.readStream + .option("readChangeFeed", "true") + .table(source_table) + ) + + +def get_filtered_source(spark: SparkSession, tokens: Dict) -> DataFrame: + """Read a Delta table with a custom filter applied.""" + source_table = tokens["sourceTable"] + filter_expr = tokens.get("filter", "1=1") + return ( + spark.readStream + .table(source_table) + .where(filter_expr) + ) + + +def get_api_source(spark: SparkSession, tokens: Dict) -> DataFrame: + """Fetch data from an external REST API and return as a DataFrame.""" + import requests + + api_url = tokens["apiUrl"] + headers = {} + if tokens.get("apiKey"): + headers["Authorization"] = f"Bearer {tokens['apiKey']}" + + response = requests.get(api_url, headers=headers) + response.raise_for_status() + data = response.json() + + if isinstance(data, list): + return spark.createDataFrame(data) + + records = data.get("results", data.get("data", [data])) + return spark.createDataFrame(records) diff --git a/skills/dataflowspec_builder/assets/extension-templates/transforms.py b/skills/dataflowspec_builder/assets/extension-templates/transforms.py new file mode 100644 index 0000000..bea19ea --- /dev/null +++ b/skills/dataflowspec_builder/assets/extension-templates/transforms.py @@ -0,0 +1,76 @@ +""" +Custom transform extensions for the Lakeflow Framework. + +Place this file in src/extensions/transforms.py of your pipeline bundle. +Reference in Data Flow Specs via: + "sourceDetails": { + "pythonTransform": { + "module": "transforms.function_name" + } + } + +Functions can accept (df) or (df, tokens) signatures. +""" +from pyspark.sql import DataFrame +from pyspark.sql import functions as F +from typing import Dict + + +def clean_and_deduplicate(df: DataFrame) -> DataFrame: + """Remove duplicates and add processing timestamp.""" + return ( + df.dropDuplicates() + .withColumn("_processed_at", F.current_timestamp()) + ) + + +def deduplicate_by_key(df: DataFrame, tokens: Dict) -> DataFrame: + """Deduplicate using a configurable key column from tokens.""" + key_columns = tokens.get("keyColumns", ["id"]) + if isinstance(key_columns, str): + key_columns = [key_columns] + return df.dropDuplicates(key_columns) + + +def standardize_timestamps(df: DataFrame) -> DataFrame: + """Convert all timestamp columns to UTC.""" + ts_cols = [f.name for f in df.schema.fields if str(f.dataType) == "TimestampType"] + result = df + for col_name in ts_cols: + result = result.withColumn(col_name, F.to_utc_timestamp(F.col(col_name), "UTC")) + return result + + +def add_hash_key(df: DataFrame, tokens: Dict) -> DataFrame: + """Add a hash key column based on specified source columns.""" + key_columns = tokens.get("keyColumns", []) + hash_col_name = tokens.get("hashColumnName", "_hash_key") + if key_columns: + cols = [F.col(c).cast("string") for c in key_columns] + return df.withColumn(hash_col_name, F.sha2(F.concat_ws("||", *cols), 256)) + return df + + +def explode_cdc_deletes(df: DataFrame) -> DataFrame: + """ + Duplicate delete records for SCD2 processing. + For deletes: is_delete=0 gets +1ms, is_delete=1 gets +2ms on the sequence column. + """ + sequence_column = "LOAD_TIMESTAMP" + change_type_column = "meta_cdc_operation" + + is_delete = F.col(change_type_column) == "delete" + array_col = F.when(is_delete, F.array(F.lit(0), F.lit(1))).otherwise(F.array(F.lit(0))) + + return ( + df.withColumnRenamed("_change_type", change_type_column) + .withColumn("is_delete", F.explode(array_col)) + .withColumn( + sequence_column, + F.when(is_delete & (F.col("is_delete") == 0), + F.col(sequence_column) + F.expr("INTERVAL 1 millisecond")) + .when(is_delete & (F.col("is_delete") == 1), + F.col(sequence_column) + F.expr("INTERVAL 2 millisecond")) + .otherwise(F.col(sequence_column)) + ) + ) diff --git a/skills/dataflowspec_builder/assets/pipeline-resource-templates/filtered_pipeline.yml b/skills/dataflowspec_builder/assets/pipeline-resource-templates/filtered_pipeline.yml new file mode 100644 index 0000000..f4f9fc9 --- /dev/null +++ b/skills/dataflowspec_builder/assets/pipeline-resource-templates/filtered_pipeline.yml @@ -0,0 +1,24 @@ +resources: + pipelines: + PIPELINE_NAME: + name: PIPELINE_NAME + catalog: ${var.catalog} + schema: ${var.schema} + channel: CURRENT + serverless: true + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: /Workspace/${workspace.file_path}/src + framework.sourcePath: /Workspace/${var.framework_source_path} + workspace.host: ${workspace.host} + bundle.target: ${bundle.target} + pipeline.layer: ${var.layer} + # Uncomment and set the filters you need: + # pipeline.dataFlowIdFilter: flow_id_1,flow_id_2 + # pipeline.dataFlowGroupFilter: group_name + # pipeline.flowGroupIdFilter: fg_1,fg_2 + # pipeline.fileFilter: dataflows/my_table/dataflowspec/my_table_main.json + # pipeline.targetTableFilter: target_table_1,target_table_2 diff --git a/skills/dataflowspec_builder/assets/pipeline-resource-templates/single_pipeline.yml b/skills/dataflowspec_builder/assets/pipeline-resource-templates/single_pipeline.yml new file mode 100644 index 0000000..96c6fab --- /dev/null +++ b/skills/dataflowspec_builder/assets/pipeline-resource-templates/single_pipeline.yml @@ -0,0 +1,18 @@ +resources: + pipelines: + PIPELINE_NAME: + name: PIPELINE_NAME + catalog: ${var.catalog} + schema: ${var.schema} + channel: CURRENT + serverless: true + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: /Workspace/${workspace.file_path}/src + framework.sourcePath: /Workspace/${var.framework_source_path} + workspace.host: ${workspace.host} + bundle.target: ${bundle.target} + pipeline.layer: ${var.layer} diff --git a/skills/dataflowspec_builder/assets/substitution-templates/dev_substitutions.json b/skills/dataflowspec_builder/assets/substitution-templates/dev_substitutions.json new file mode 100644 index 0000000..a11060a --- /dev/null +++ b/skills/dataflowspec_builder/assets/substitution-templates/dev_substitutions.json @@ -0,0 +1,10 @@ +{ + "tokens": { + "bronze_schema": "CATALOG.bronze_SCHEMA", + "silver_schema": "CATALOG.silver_SCHEMA", + "gold_schema": "CATALOG.gold_SCHEMA", + "landing_path": "/Volumes/CATALOG/SCHEMA/landing", + "checkpoint_path": "/Volumes/CATALOG/SCHEMA/checkpoints" + }, + "prefix_suffix": {} +} diff --git a/skills/dataflowspec_builder/assets/substitution-templates/prod_substitutions.json b/skills/dataflowspec_builder/assets/substitution-templates/prod_substitutions.json new file mode 100644 index 0000000..e5e2cf4 --- /dev/null +++ b/skills/dataflowspec_builder/assets/substitution-templates/prod_substitutions.json @@ -0,0 +1,14 @@ +{ + "tokens": { + "bronze_schema": "CATALOG.bronze_SCHEMA_prod", + "silver_schema": "CATALOG.silver_SCHEMA_prod", + "gold_schema": "CATALOG.gold_SCHEMA_prod", + "landing_path": "/Volumes/CATALOG/SCHEMA_prod/landing", + "checkpoint_path": "/Volumes/CATALOG/SCHEMA_prod/checkpoints" + }, + "prefix_suffix": { + "database": { + "suffix": "_{workspace_env}" + } + } +} diff --git a/skills/dataflowspec_builder/docs/architecture.md b/skills/dataflowspec_builder/docs/architecture.md new file mode 100644 index 0000000..7f75ea0 --- /dev/null +++ b/skills/dataflowspec_builder/docs/architecture.md @@ -0,0 +1,158 @@ +# Architecture + +## How the Data Flow Spec Framework Works + +The Data Flow Spec Framework (`databricks-solutions/lakeflow_framework`) is a metadata-driven data engineering framework that sits on top of Databricks Spark Declarative Pipelines (SDP, formerly DLT). Instead of writing Python/SQL pipeline code directly, engineers define pipelines through JSON/YAML configuration files called **Data Flow Specs**. + +``` +┌─────────────────────────────────────────────────────────┐ +│ Pipeline Bundle │ +│ (Your project — Data Flow Specs, schemas, expectations) │ +│ │ +│ databricks.yml │ +│ resources/*.yml (pipeline definitions) │ +│ src/dataflows/*/dataflowspec/*.json │ +│ src/dataflows/*/schemas/*.json │ +│ src/dataflows/*/expectations/*.json │ +│ src/extensions/*.py │ +│ src/pipeline_configs/*_substitutions.json │ +└──────────────────────┬──────────────────────────────────┘ + │ references + ▼ +┌─────────────────────────────────────────────────────────┐ +│ Framework Bundle │ +│ (Deployed once per workspace — engine code) │ +│ │ +│ src/dlt_pipeline (entry point notebook) │ +│ src/dataflow_spec_builder/ (spec parser & validator) │ +│ src/dataflow/targets/ (target writers) │ +│ src/dataflow/sources/ (source readers) │ +│ src/schemas/ (JSON Schema definitions) │ +└──────────────────────┬──────────────────────────────────┘ + │ generates + ▼ +┌─────────────────────────────────────────────────────────┐ +│ Spark Declarative Pipeline (SDP) │ +│ (Managed by Databricks — serverless compute) │ +│ │ +│ Streaming Tables, Materialized Views, CDC Merges │ +│ Auto-managed checkpoints, retries, scaling │ +└─────────────────────────────────────────────────────────┘ +``` + +## Two Bundle Architecture + +### Framework Bundle (deploy once) + +Contains the core engine — the Python code that reads Data Flow Specs, validates them against JSON Schemas, and generates SDP pipeline code at runtime. + +```bash +git clone https://github.com/databricks-solutions/lakeflow_framework.git +cd lakeflow_framework +databricks bundle deploy -t dev +``` + +Deployed to: `/Workspace/Users//.bundle/lakeflow_framework/dev/current/files/src` + +### Pipeline Bundle (one per project) + +Contains your project-specific configuration — Data Flow Specs, schemas, expectations, and pipeline definitions. This is what the skill generates. + +``` +my_pipeline/ +├── databricks.yml +├── resources/ +│ └── bronze_pipeline.yml +└── src/ + └── dataflows/ + └── bronze/ + ├── dataflowspec/ + │ └── customers_main.json + ├── schemas/ + │ └── customers_schema.json + └── expectations/ + └── customers_dqe.json +``` + +## Data Flow Types + +### Standard (1:1) + +Single source → single target. Best for bronze ingestion. + +``` +Source Table ──[stream/batch]──> Target Table + (with CDC merge) +``` + +### Flows (Multi-Source) + +Multiple sources → staging → target. Best for silver/gold. + +``` +Source A ──[append_view]──┐ + ├──> Staging Table ──[merge]──> Target Table +Source B ──[append_view]──┘ (with CDC) +``` + +### Materialized Views + +SQL-defined precomputed views. Best for gold KPIs. + +``` +Source Tables ──[SQL query]──> Materialized View + (auto-refreshed) +``` + +## Pipeline Execution Flow + +``` +1. databricks bundle run -t dev my_pipeline + │ +2. SDP launches dlt_pipeline notebook (from framework bundle) + │ +3. Framework reads Data Flow Specs from pipeline bundle's src/ directory + │ +4. DataflowSpecBuilder validates specs against JSON Schemas + │ +5. Substitutions are applied ({tokens} replaced with env values) + │ +6. Templates are expanded (parameterSets → individual specs) + │ +7. Specs are filtered by pipeline config (dataFlowGroupFilter, etc.) + │ +8. For each spec: + ├── Source view is created (streaming or batch) + ├── Data quality expectations are applied (if enabled) + ├── Quarantine routing is configured (if enabled) + ├── Python transforms are applied (if configured) + └── Target table is written (CDC merge or append) + │ +9. Operational metadata columns are added automatically + │ +10. Pipeline runs continuously (streaming) or completes (batch) +``` + +## Skill Architecture + +The Genie Code Agent Skill works as follows: + +``` +User Prompt ──> Genie Code reads SKILL.md + │ + ├── Understands Data Flow Spec schema + ├── Knows available patterns + ├── Has example specs from assets/ + └── Has reference docs from references/ + │ + ▼ + Genie Code generates: + ├── Data Flow Spec JSON files + ├── Schema JSON files + ├── Expectation JSON files + ├── Pipeline resource YAMLs + ├── databricks.yml + └── (optionally) deploy commands +``` + +The `SKILL.md` file is the core — it contains the complete schema reference, all patterns, feature documentation, and examples that Genie Code needs to generate correct configuration files. diff --git a/skills/dataflowspec_builder/docs/example-prompts.md b/skills/dataflowspec_builder/docs/example-prompts.md new file mode 100644 index 0000000..93c8828 --- /dev/null +++ b/skills/dataflowspec_builder/docs/example-prompts.md @@ -0,0 +1,322 @@ +# Example Prompts + +A comprehensive list of example prompts you can use with the Data Flow Spec Builder skill in Genie Code. Each section includes the prompt, what gets generated, and when to use it. + +> **Tip:** Always include "Data Flow Spec" or "dataflow-spec-builder" in your prompt to ensure Genie Code routes to this skill rather than native Lakeflow Declarative Pipelines. + +--- + +## Bronze Layer — Ingestion + +### Basic Delta-to-Delta Ingestion + +``` +Use the dataflow-spec-builder to create a bronze Data Flow Spec that ingests +the raw_customers table from main.energy_workshop with SCD Type 1 CDC, +using account_id as the primary key and signup_date as the sequence column. +``` + +**Generates:** Standard Data Flow Spec with `sourceType: delta`, `cdcSettings` with SCD1. + +### Multiple Tables with Template + +``` +Generate a Data Flow Spec template for bronze ingestion of 7 tables: +raw_customers, raw_billing, raw_meter_readings, raw_outages, raw_weather, +raw_equipment, raw_demand_response. All from main.energy_workshop. +Use SCD1 CDC for each. Create a reusable template with parameter sets. +``` + +**Generates:** Template definition JSON + template usage JSON with 7 parameter sets. + +### CloudFiles / Auto Loader Ingestion + +``` +Create a Data Flow Spec for ingesting CSV files from /Volumes/main/landing/customers/ +using CloudFiles (Auto Loader) into bronze_customers in main.energy_workshop. +Include schema enforcement with a JSON schema file. +``` + +**Generates:** Standard Data Flow Spec with `sourceType: cloudFiles`, reader options for CSV, and schema path reference. + +### Kafka Source Ingestion + +``` +Generate a Data Flow Spec for streaming from a Kafka topic called "meter-events" +into bronze_meter_events. Use Databricks secrets for the Kafka bootstrap servers +and SASL credentials. +``` + +**Generates:** Standard Data Flow Spec with `sourceType: kafka`, secrets references in reader options. + +### SCD Type 2 with History Tracking + +``` +Create a bronze Data Flow Spec for the raw_customers table with SCD Type 2. +Track history on customer_name, rate_plan, and has_solar columns. +Use account_id as the key and signup_date as the sequence column. +``` + +**Generates:** Standard Data Flow Spec with `cdcSettings.scd_type: "2"` and `track_history_column_list`. + +### CDC from Historical Snapshots + +``` +Generate a Data Flow Spec for CDC ingestion from daily Parquet snapshot files +at /Volumes/main/raw/equipment_snapshots/{date}/ into bronze_equipment. +Use datetime-based versioning with format yyyyMMdd starting from 20240101. +``` + +**Generates:** Standard Data Flow Spec with `cdcSnapshotSettings` for file-based snapshots. + +--- + +## Silver Layer — Transforms & Joins + +### Multi-Source Streaming (Customer 360) + +``` +Create a Data Flow Spec that merges bronze_customers and bronze_billing +into a silver_customer_360 table using the multi-source streaming pattern. +Stream both sources into a shared staging table, then CDC merge into the target +with SCD2 on account_id. Use the dataflow-spec-builder. +``` + +**Generates:** Flows Data Flow Spec with two `append_view` flows into a staging table, then a `merge` flow to the target. + +### Stream-Static Join + +``` +Generate a Data Flow Spec for enriching bronze_meter_readings (streaming) +with bronze_customers (static lookup) and bronze_weather (static lookup). +Join on customer_id and date/state. Output to silver_enriched_readings. +``` + +**Generates:** Flows Data Flow Spec with deltaJoin views combining stream and static sources. + +### SQL-Based Transform + +``` +Create a silver Data Flow Spec that applies a SQL transform to aggregate +hourly meter readings into daily summaries. Group by customer_id and date, +sum kwh_consumed, and calculate peak vs off-peak ratios. +``` + +**Generates:** Flows Data Flow Spec with `flowType: append_sql` referencing a SQL file. + +### Python Extension Transform + +``` +Generate a Data Flow Spec for silver_clean_customers that reads from +bronze_customers and applies a Python transform to standardize addresses, +validate emails, and deduplicate by account_id. +``` + +**Generates:** Standard or Flows Data Flow Spec with `pythonTransform` referencing a Python extension module. + +--- + +## Gold Layer — Materialized Views + +### Revenue KPIs + +``` +Create a gold materialized view Data Flow Spec for mv_revenue_by_state that +aggregates monthly revenue, active customer counts, and average bill amounts +by Australian state. Join raw_customers and raw_billing. +``` + +**Generates:** Materialized View Data Flow Spec with inline SQL aggregation. + +### Multiple Gold MVs in One Spec + +``` +Generate a Data Flow Spec with 3 gold materialized views: +1. mv_revenue_by_state — monthly revenue by state +2. mv_grid_reliability — outage counts and SAIDI by state and cause +3. mv_equipment_risk_score — risk scores based on age, failures, and load +All reading from main.energy_workshop tables. +``` + +**Generates:** Single Materialized View Data Flow Spec with 3 MV definitions. + +### SQL File-Based MV + +``` +Create a materialized view Data Flow Spec for mv_consumption_heatmap +that references an external SQL file at dml/mv_consumption_heatmap.sql. +``` + +**Generates:** Materialized View Data Flow Spec with `sqlPath` reference instead of inline SQL. + +--- + +## Data Quality + +### Add Expectations to Bronze + +``` +Add data quality expectations to the bronze_customers Data Flow Spec: +- Expect account_id is not null (log only) +- Drop rows where state is not a valid Australian state +- Fail the pipeline if any postcode is null +``` + +**Generates:** Expectations JSON file with `expect`, `expect_or_drop`, and `expect_or_fail` rules. + +### Quarantine Bad Records + +``` +Create a Data Flow Spec for bronze_billing with quarantine mode set to "table". +Bad records (negative amounts, null bill_ids) should go to bronze_billing_quarantine. +Good records go to bronze_billing. +``` + +**Generates:** Standard Data Flow Spec with `quarantineMode: "table"` and `quarantineTargetDetails`. + +--- + +## Templates & Reuse + +### Create a Reusable Template + +``` +Create a Data Flow Spec template called "bronze_delta_ingestion" that is +parameterized on: dataFlowId, sourceTable, targetTable, keyColumns, and +sequenceBy. Use main.energy_workshop as the database. +``` + +**Generates:** Template definition JSON with parameter definitions and a template body. + +### Apply Template to Multiple Tables + +``` +Using the bronze_delta_ingestion template, generate parameter sets for: +- raw_customers → bronze_customers (key: account_id, seq: signup_date) +- raw_billing → bronze_billing (key: bill_id, seq: billing_period) +- raw_outages → bronze_outages (key: outage_id, seq: start_time) +``` + +**Generates:** Template usage JSON with `parameterSets` array. + +--- + +## Environment Configuration + +### Multi-Environment Substitutions + +``` +Create environment substitution configs for dev, staging, and prod. +Dev uses main.energy_dev, staging uses main.energy_staging, +prod uses main.energy_prod. Landing path varies by environment. +``` + +**Generates:** Three substitution JSON files with environment-specific tokens. + +### Full DABs Configuration + +``` +Generate a complete databricks.yml for a bronze pipeline bundle targeting +main.energy_workshop on the e2-demo-west workspace, with dev and prod targets. +``` + +**Generates:** `databricks.yml` with variables, targets, and workspace configuration. + +--- + +## Advanced Features + +### Soft Deletes + +``` +Create a Data Flow Spec for bronze_customers with soft deletes enabled. +When the source sends a record with operation='DELETE', mark the record +as deleted instead of physically removing it. +``` + +**Generates:** Standard Data Flow Spec with `cdcSettings.apply_as_deletes` and `soft_delete: true`. + +### Table Migration from HMS + +``` +Generate a Data Flow Spec that migrates the legacy Hive Metastore table +legacy_db.old_customers into main.energy_workshop.bronze_customers. +Select only customer_id, name, email, and state columns. Filter active=true. +``` + +**Generates:** Standard Data Flow Spec with `tableMigrationDetails` configured. + +### Liquid Clustering + +``` +Create a Data Flow Spec for silver_meter_readings with liquid clustering +on customer_id and reading_date columns. +``` + +**Generates:** Data Flow Spec with `targetDetails.clusterByColumns` set. + +### Auto Clustering + +``` +Generate a Data Flow Spec for gold_revenue with automatic liquid clustering +enabled (let Databricks choose optimal keys). +``` + +**Generates:** Data Flow Spec with `targetDetails.clusterByAuto: true`. + +### Custom Python Sink + +``` +Create a Data Flow Spec that writes processed data to an external REST API +using a custom Python sink function. The API endpoint is configurable via tokens. +``` + +**Generates:** Standard Data Flow Spec with `targetFormat: "foreach_batch_sink"` and Python function sink config. + +--- + +## Deployment & Operations + +### Scaffold and Deploy End-to-End + +``` +Use the dataflow-spec-builder to scaffold a complete bronze pipeline bundle +for the energy tables, validate it, and show me the deploy commands. +``` + +**Generates:** Full bundle structure + deploy instructions. + +### Pipeline Filters + +``` +Create a pipeline resource YAML that runs only the billing and customers +Data Flow Specs, filtering by dataFlowId. +``` + +**Generates:** Pipeline YAML with `pipeline.dataFlowIdFilter` configuration. + +--- + +## Informational Queries + +These prompts ask for guidance rather than generating files: + +``` +What Data Flow Spec pattern should I use for joining meter readings with weather data? +``` + +``` +Explain the difference between standard and flows Data Flow Spec types. +``` + +``` +What quarantine modes are available in the Data Flow Spec Framework? +``` + +``` +How do substitutions work across environments in Data Flow Specs? +``` + +``` +What's the directory structure for a Data Flow Spec pipeline bundle? +``` diff --git a/skills/dataflowspec_builder/docs/getting-started.md b/skills/dataflowspec_builder/docs/getting-started.md new file mode 100644 index 0000000..26ed2da --- /dev/null +++ b/skills/dataflowspec_builder/docs/getting-started.md @@ -0,0 +1,106 @@ +# Getting Started + +This guide walks you through setting up the Data Flow Spec Builder skill with Databricks Genie Code. + +## Prerequisites + +- A Databricks workspace with Unity Catalog enabled +- Databricks CLI installed and configured (`databricks auth login`) +- Genie Code enabled on your workspace (ask your admin) +- Python 3.9+ + +## Step 1: Deploy the Data Flow Spec Framework + +The framework engine must be deployed to your workspace before the skill can generate working pipelines. + +```bash +# Clone the framework +git clone https://github.com/databricks-solutions/lakeflow_framework.git +cd lakeflow_framework + +# Validate the bundle +databricks bundle validate -t dev + +# Deploy to your workspace +databricks bundle deploy -t dev +``` + +After deployment, the framework code will be at: +``` +/Workspace/Users//.bundle/lakeflow_framework/dev/current/files/src +``` + +## Step 2: Install the Skill + +### Option A: Copy to `.assistant/skills/` (recommended) + +Upload the skill directory to your workspace's `.assistant/skills/` folder. Genie Code automatically discovers skills in this location. + +```bash +databricks workspace import-dir \ + ./path-to-this-repo \ + "/Workspace/Users//.assistant/skills/dataflow-spec-builder" +``` + +### Option B: Manual skill path + +In your notebook, open Genie Code settings (gear icon in the Genie Code panel) and add the skill path manually: +``` +/Workspace/Users//skills/dataflow-spec-builder +``` + +## Step 3: Verify the Skill + +Open any Python notebook on your workspace and enter Genie Code Agent mode. Ask: + +> "What Data Flow Spec patterns are available?" + +If the skill is loaded correctly, Genie Code will respond with the pattern list from the skill definition. + +## Step 4: Generate Your First Pipeline + +Try this prompt: + +> "Use the dataflow-spec-builder to create a bronze Data Flow Spec that ingests the `raw_customers` table from `main.my_schema` with SCD Type 1 CDC" + +Genie Code should generate: +1. A `customers_main.json` Data Flow Spec file +2. A pipeline resource YAML +3. A `databricks.yml` configuration + +## Step 5: Deploy the Generated Pipeline + +```bash +cd +databricks bundle validate -t dev +databricks bundle deploy -t dev +databricks bundle run -t dev +``` + +## Troubleshooting + +### Skill not being picked up + +- Ensure the `SKILL.md` file is in the root of the skill directory +- Check that the directory is under `.assistant/skills/` in your workspace +- Try mentioning "dataflow-spec-builder" or "Data Flow Spec" explicitly in your prompt + +### Genie Code generates native DLT instead + +If Genie Code generates `@dlt.table` decorators or `CREATE STREAMING TABLE` SQL, it's using native Lakeflow Declarative Pipelines instead of this skill. Use these trigger phrases: +- "Use the **dataflow-spec-builder** skill..." +- "Generate a **Data Flow Spec** for..." +- "Create a pipeline using the **metadata-driven framework**..." + +### Framework bundle not found + +Ensure you've deployed the framework: +```bash +cd lakeflow_framework +databricks bundle deploy -t dev +``` + +Verify the deployment: +```bash +databricks workspace list "/Workspace/Users//.bundle/lakeflow_framework/dev/current/files/src" +``` diff --git a/skills/dataflowspec_builder/docs/skill-development.md b/skills/dataflowspec_builder/docs/skill-development.md new file mode 100644 index 0000000..0acdb48 --- /dev/null +++ b/skills/dataflowspec_builder/docs/skill-development.md @@ -0,0 +1,107 @@ +# Skill Development Guide + +How to extend or customize this skill for your own use cases. + +## Skill Structure + +Genie Code Agent Skills follow a convention: + +``` +skill-directory/ +├── SKILL.md # Required — the skill definition Genie Code reads +├── assets/ # Optional — templates, configs, reference files +├── references/ # Optional — domain docs, schema references +├── scripts/ # Optional — automation scripts +└── examples/ # Optional — working examples for Genie Code to reference +``` + +The `SKILL.md` file is the core. Genie Code reads it to understand: +- **When** to activate the skill (trigger phrases) +- **What** to generate (schemas, templates, examples) +- **How** to generate it (workflow steps, constraints) + +## Customizing for Your Domain + +### 1. Update the Examples + +Replace the energy-domain examples with your own: + +``` +examples/ +├── your-bronze/ +│ ├── dataflowspec/your_bronze_main.json +│ ├── schemas/your_table_schema.json +│ └── expectations/your_table_dqe.json +├── your-silver/ +│ └── dataflowspec/your_silver_main.json +└── your-gold/ + └── dataflowspec/your_gold_main.json +``` + +### 2. Update the References + +Edit `references/energy-domain-mapping.md` to describe your tables and their relationships: + +```markdown +## Your Domain Tables + +| Table | Description | Key | CDC Pattern | +|-------|------------|-----|-------------| +| raw_orders | Customer orders | order_id | SCD1 | +| raw_products | Product catalog | product_id | SCD2 | +``` + +### 3. Add Domain-Specific Expectations + +Create expectation templates for your data quality rules: + +```json +{ + "expect": [ + {"name": "valid_order_id", "constraint": "order_id IS NOT NULL", "tag": "completeness"}, + {"name": "positive_total", "constraint": "order_total > 0", "tag": "range"} + ], + "expect_or_drop": [ + {"name": "valid_status", "constraint": "status IN ('pending','shipped','delivered')", "tag": "validity"} + ] +} +``` + +### 4. Update SKILL.md Triggers + +Modify the "When to Use" section to include your domain terminology: + +```markdown +## When to Use + +- User says "generate a Data Flow Spec for orders" +- User needs CDC ingestion for the e-commerce data platform +- User asks for order-to-shipment pipeline using Data Flow Specs +``` + +## Adding New Patterns + +To add support for a new pattern: + +1. Create an example Data Flow Spec in `assets/dataflowspec-templates/` +2. Add the pattern to the "Patterns" section in `SKILL.md` +3. Add a corresponding example in `examples/` +4. Update `references/patterns-guide.md` + +## Testing Your Skill + +1. Upload the skill to `.assistant/skills/` on your workspace +2. Open a new notebook +3. Enter Genie Code Agent mode +4. Test with progressively complex prompts: + - Start with: "What patterns does the dataflow-spec-builder support?" + - Then: "Generate a simple bronze Data Flow Spec for my_table" + - Then: "Create a complete medallion pipeline with templates and DQ" + +## Tips for Effective Skills + +- **Be specific in SKILL.md** — Include complete JSON schemas, not just descriptions +- **Include working examples** — Genie Code learns from examples in the skill directory +- **Use unique trigger phrases** — Avoid terms that overlap with built-in Databricks features +- **Add a "When NOT to Use" section** — Helps Genie Code avoid false positive activations +- **Keep the skill focused** — One skill per framework/pattern, not a catch-all diff --git a/skills/dataflowspec_builder/docs/tested-medallion-example.md b/skills/dataflowspec_builder/docs/tested-medallion-example.md new file mode 100644 index 0000000..19b86b7 --- /dev/null +++ b/skills/dataflowspec_builder/docs/tested-medallion-example.md @@ -0,0 +1,151 @@ +# Tested Medallion Architecture Example + +This document describes a complete end-to-end medallion architecture that was tested and verified on a live Databricks workspace (`e2-demo-west.cloud.databricks.com`) using the Data Flow Spec Framework. + +## Dataset — SmartGrid Analytics Platform (Australia) + +The test uses 7 synthetic energy tables in `main.sourabh_energy_workshop`: + +| Table | Rows | Description | +|-------|------|-------------| +| `raw_customers` | 50,000 | Residential/commercial energy customers across 6 Australian states | +| `raw_billing` | 600,000 | Monthly billing records with kWh consumption and charges | +| `raw_meter_readings` | 10,685,000 | Smart meter telemetry (voltage, power factor, kWh) | +| `raw_outages` | 5,000 | Grid outage events with cause, duration, affected meters | +| `raw_equipment` | 2,000 | Grid equipment (transformers, switchgear) with condition metrics | +| `raw_weather` | 2,190 | Daily weather data by state (temp, humidity, wind) | +| `raw_demand_response` | 20,000 | Demand response events and customer participation | + +## Bronze Layer — 7 Streaming Tables + +Each raw table gets a corresponding bronze streaming table via a standard Data Flow Spec with SCD Type 1 CDC. + +### Data Flow Spec (example — customers) + +```json +{ + "dataFlowId": "customers_bronze", + "dataFlowGroup": "energy_bronze", + "dataFlowType": "standard", + "sourceSystem": "smartgrid", + "sourceType": "delta", + "sourceViewName": "v_raw_customers", + "sourceDetails": { + "database": "main.sourabh_energy_workshop", + "table": "raw_customers", + "cdfEnabled": false + }, + "mode": "stream", + "targetFormat": "delta", + "targetDetails": { + "table": "bronze_customers", + "tableProperties": { + "delta.enableChangeDataFeed": "true" + } + }, + "cdcSettings": { + "keys": ["account_id"], + "sequence_by": "signup_date", + "scd_type": "1", + "ignore_null_updates": true, + "except_column_list": [] + } +} +``` + +### Pipeline Resource YAML + +```yaml +resources: + pipelines: + energy_bronze_pipeline: + name: energy_bronze_pipeline + catalog: ${var.catalog} + schema: ${var.schema} + channel: CURRENT + serverless: true + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + configuration: + bundle.sourcePath: ${workspace.file_path}/src + framework.sourcePath: /Workspace/${var.framework_source_path} + workspace.host: ${workspace.host} + bundle.target: ${bundle.target} + pipeline.layer: bronze + pipeline.dataFlowGroupFilter: energy_bronze +``` + +### Results + +All 7 bronze tables created successfully with the framework's automatic operational metadata: + +| Bronze Table | Rows | Meta Columns | +|-------------|------|-------------| +| `bronze_customers` | 50,000 | `meta_load_details` (record_insert_timestamp, record_update_timestamp, pipeline_start_timestamp) | +| `bronze_billing` | 600,000 | ✓ | +| `bronze_meter_readings` | 10,685,000 | ✓ | +| `bronze_outages` | 5,000 | ✓ | +| `bronze_equipment` | 2,000 | ✓ | +| `bronze_weather` | 2,190 | ✓ | +| `bronze_demand_response` | 20,000 | ✓ | + +## Gold Layer — 3 Materialized Views + +### Data Flow Spec + +```json +{ + "dataFlowId": "energy_gold_kpis", + "dataFlowGroup": "energy_gold", + "dataFlowType": "materialized_view", + "materializedViews": { + "mv_revenue_by_state": { + "sqlStatement": "SELECT c.state, DATE_TRUNC('month', b.billing_period) AS month, COUNT(DISTINCT b.customer_id) AS active_customers, SUM(b.amount_charged) AS total_revenue_aud, AVG(b.amount_charged) AS avg_bill_aud, SUM(b.total_kwh) AS total_kwh FROM main.sourabh_energy_workshop.raw_customers c INNER JOIN main.sourabh_energy_workshop.raw_billing b ON c.account_id = b.customer_id GROUP BY c.state, DATE_TRUNC('month', b.billing_period)", + "tableDetails": { "comment": "Monthly revenue and consumption metrics by Australian state" } + }, + "mv_grid_reliability": { + "sqlStatement": "SELECT state, cause, COUNT(*) AS outage_count, AVG(duration_minutes) AS avg_duration_minutes, SUM(affected_meters_count) AS total_affected_meters, ROUND(SUM(duration_minutes * affected_meters_count) / 50000.0, 4) AS saidi_contribution FROM main.sourabh_energy_workshop.raw_outages WHERE start_time <= current_timestamp() GROUP BY state, cause", + "tableDetails": { "comment": "Grid reliability metrics with SAIDI contribution by state and cause" } + }, + "mv_equipment_risk_score": { + "sqlStatement": "SELECT equipment_id, equipment_type, state, install_date, last_maintenance_date, maintenance_count, failure_count, capacity_rating, current_load_pct, DATEDIFF(current_date(), install_date) AS age_days, DATEDIFF(current_date(), last_maintenance_date) AS days_since_maintenance, ROUND((failure_count * 0.3) + (current_load_pct * 0.4) + (DATEDIFF(current_date(), last_maintenance_date) / 365.0 * 0.3), 4) AS risk_score, latitude, longitude FROM main.sourabh_energy_workshop.raw_equipment", + "tableDetails": { "comment": "Equipment risk scores for predictive maintenance" } + } + } +} +``` + +### Results + +| Gold Materialized View | Rows | Sample Output | +|----------------------|------|---------------| +| `mv_revenue_by_state` | 66 | NSW, May 2024: 15,992 customers, $5.4M AUD revenue | +| `mv_grid_reliability` | 36 | QLD weather outages: 301 events, avg 136 min, SAIDI 2209 | +| `mv_equipment_risk_score` | 2,000 | EQ-00616 transformer (VIC): risk score 41.78 | + +## Lessons Learned + +### Schema Validation + +The framework enforces strict JSON Schema validation on Data Flow Specs. Key required fields: +- `dataFlowId`, `dataFlowGroup`, `dataFlowType` (always required) +- `targetFormat`, `targetDetails` (required for standard specs) +- `sourceSystem`, `sourceType`, `sourceViewName`, `mode`, `sourceDetails` (required unless using `cdcSnapshotSettings`) +- `dataQualityExpectationsPath` (required when `dataQualityExpectationsEnabled: true`) + +### Column Name Matching + +Ensure CDC keys and SQL queries match actual table column names. The framework creates source views and will fail with `UNRESOLVED_COLUMN` errors if column names don't match. + +### Additional Properties + +The framework uses `additionalProperties: false` in its JSON Schema — do not add fields that aren't in the schema definition. Keep specs clean. + +### Operational Metadata + +The framework automatically adds a `meta_load_details` struct column to all target tables containing: +- `record_insert_timestamp` +- `record_update_timestamp` +- `pipeline_start_timestamp` +- `pipeline_update_id` diff --git a/skills/dataflowspec_builder/examples/energy-bronze/dataflowspec/energy_bronze_main.json b/skills/dataflowspec_builder/examples/energy-bronze/dataflowspec/energy_bronze_main.json new file mode 100644 index 0000000..f3300a0 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/dataflowspec/energy_bronze_main.json @@ -0,0 +1,61 @@ +{ + "template": "energy_bronze_ingestion_template", + "parameterSets": [ + { + "dataFlowId": "customers_bronze", + "sourceTable": "raw_customers", + "targetTable": "bronze_customers", + "schemaPath": "schemas/raw_customers_schema.json", + "keyColumns": ["customer_id"], + "sequenceBy": "signup_date" + }, + { + "dataFlowId": "meter_readings_bronze", + "sourceTable": "raw_meter_readings", + "targetTable": "bronze_meter_readings", + "schemaPath": "schemas/raw_meter_readings_schema.json", + "keyColumns": ["meter_id", "reading_timestamp"], + "sequenceBy": "reading_timestamp" + }, + { + "dataFlowId": "billing_bronze", + "sourceTable": "raw_billing", + "targetTable": "bronze_billing", + "schemaPath": "schemas/raw_billing_schema.json", + "keyColumns": ["bill_id"], + "sequenceBy": "bill_date" + }, + { + "dataFlowId": "outages_bronze", + "sourceTable": "raw_outages", + "targetTable": "bronze_outages", + "schemaPath": "schemas/raw_outages_schema.json", + "keyColumns": ["outage_id"], + "sequenceBy": "start_time" + }, + { + "dataFlowId": "weather_bronze", + "sourceTable": "raw_weather", + "targetTable": "bronze_weather", + "schemaPath": "schemas/raw_weather_schema.json", + "keyColumns": ["state", "date"], + "sequenceBy": "date" + }, + { + "dataFlowId": "equipment_bronze", + "sourceTable": "raw_equipment", + "targetTable": "bronze_equipment", + "schemaPath": "schemas/raw_equipment_schema.json", + "keyColumns": ["equipment_id"], + "sequenceBy": "last_maintenance_date" + }, + { + "dataFlowId": "demand_response_bronze", + "sourceTable": "raw_demand_response", + "targetTable": "bronze_demand_response", + "schemaPath": "schemas/raw_demand_response_schema.json", + "keyColumns": ["event_id"], + "sequenceBy": "event_date" + } + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_billing_dqe.json b/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_billing_dqe.json new file mode 100644 index 0000000..424b317 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_billing_dqe.json @@ -0,0 +1,12 @@ +{ + "expect": [ + {"name": "bill_id_not_null", "constraint": "bill_id IS NOT NULL", "tag": "completeness", "enabled": true}, + {"name": "customer_id_not_null", "constraint": "customer_id IS NOT NULL", "tag": "completeness", "enabled": true}, + {"name": "positive_amount", "constraint": "amount_aud > 0", "tag": "range", "enabled": true}, + {"name": "valid_payment_status", "constraint": "payment_status IN ('paid', 'pending', 'overdue', 'delinquent')", "tag": "validity", "enabled": true} + ], + "expect_or_drop": [ + {"name": "not_duplicate", "constraint": "is_duplicate = false OR is_duplicate IS NULL", "tag": "dedup", "enabled": true} + ], + "expect_or_fail": [] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_customers_dqe.json b/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_customers_dqe.json new file mode 100644 index 0000000..a9cbe25 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_customers_dqe.json @@ -0,0 +1,12 @@ +{ + "expect": [ + {"name": "customer_id_not_null", "constraint": "customer_id IS NOT NULL", "tag": "completeness", "enabled": true}, + {"name": "valid_state", "constraint": "state IN ('NSW', 'VIC', 'QLD', 'SA', 'WA', 'TAS')", "tag": "validity", "enabled": true}, + {"name": "valid_email", "constraint": "email LIKE '%@%.%'", "tag": "format", "enabled": true}, + {"name": "valid_customer_type", "constraint": "customer_type IN ('residential', 'commercial', 'industrial')", "tag": "validity", "enabled": true} + ], + "expect_or_drop": [ + {"name": "valid_rate_plan", "constraint": "rate_plan IN ('basic', 'time_of_use', 'demand', 'solar_feed_in')", "tag": "validity", "enabled": true} + ], + "expect_or_fail": [] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_meter_readings_dqe.json b/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_meter_readings_dqe.json new file mode 100644 index 0000000..2ea37b1 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_meter_readings_dqe.json @@ -0,0 +1,12 @@ +{ + "expect": [ + {"name": "meter_id_not_null", "constraint": "meter_id IS NOT NULL", "tag": "completeness", "enabled": true}, + {"name": "reading_timestamp_not_null", "constraint": "reading_timestamp IS NOT NULL", "tag": "completeness", "enabled": true}, + {"name": "kwh_not_null", "constraint": "kwh_used IS NOT NULL", "tag": "completeness", "enabled": true}, + {"name": "valid_voltage", "constraint": "voltage BETWEEN 200 AND 260", "tag": "range", "enabled": true} + ], + "expect_or_drop": [ + {"name": "positive_kwh", "constraint": "kwh_used >= 0", "tag": "range", "enabled": true} + ], + "expect_or_fail": [] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_outages_dqe.json b/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_outages_dqe.json new file mode 100644 index 0000000..b36b569 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/expectations/bronze_outages_dqe.json @@ -0,0 +1,11 @@ +{ + "expect": [ + {"name": "outage_id_not_null", "constraint": "outage_id IS NOT NULL", "tag": "completeness", "enabled": true}, + {"name": "valid_cause", "constraint": "cause IN ('storm', 'equipment_failure', 'vegetation', 'animal', 'overload', 'planned')", "tag": "validity", "enabled": true}, + {"name": "positive_duration", "constraint": "duration_minutes > 0", "tag": "range", "enabled": true} + ], + "expect_or_drop": [ + {"name": "not_future_dated", "constraint": "start_time <= current_timestamp()", "tag": "timeliness", "enabled": true} + ], + "expect_or_fail": [] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_billing_schema.json b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_billing_schema.json new file mode 100644 index 0000000..c6a19ea --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_billing_schema.json @@ -0,0 +1,13 @@ +{ + "type": "struct", + "fields": [ + {"name": "bill_id", "type": "integer", "nullable": false, "metadata": {}}, + {"name": "customer_id", "type": "integer", "nullable": false, "metadata": {}}, + {"name": "bill_date", "type": "date", "nullable": true, "metadata": {}}, + {"name": "due_date", "type": "date", "nullable": true, "metadata": {}}, + {"name": "amount_aud", "type": {"type": "decimal", "precision": 10, "scale": 2}, "nullable": true, "metadata": {}}, + {"name": "kwh_consumed", "type": "double", "nullable": true, "metadata": {}}, + {"name": "payment_status", "type": "string", "nullable": true, "metadata": {}}, + {"name": "is_duplicate", "type": "boolean", "nullable": true, "metadata": {}} + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_customers_schema.json b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_customers_schema.json new file mode 100644 index 0000000..ccf065d --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_customers_schema.json @@ -0,0 +1,20 @@ +{ + "type": "struct", + "fields": [ + {"name": "customer_id", "type": "integer", "nullable": false, "metadata": {}}, + {"name": "name", "type": "string", "nullable": true, "metadata": {}}, + {"name": "email", "type": "string", "nullable": true, "metadata": {}}, + {"name": "phone", "type": "string", "nullable": true, "metadata": {}}, + {"name": "address", "type": "string", "nullable": true, "metadata": {}}, + {"name": "city", "type": "string", "nullable": true, "metadata": {}}, + {"name": "state", "type": "string", "nullable": true, "metadata": {}}, + {"name": "postcode", "type": "string", "nullable": true, "metadata": {}}, + {"name": "customer_type", "type": "string", "nullable": true, "metadata": {}}, + {"name": "rate_plan", "type": "string", "nullable": true, "metadata": {}}, + {"name": "has_solar", "type": "boolean", "nullable": true, "metadata": {}}, + {"name": "has_ev", "type": "boolean", "nullable": true, "metadata": {}}, + {"name": "signup_date", "type": "date", "nullable": true, "metadata": {}}, + {"name": "latitude", "type": "double", "nullable": true, "metadata": {}}, + {"name": "longitude", "type": "double", "nullable": true, "metadata": {}} + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_demand_response_schema.json b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_demand_response_schema.json new file mode 100644 index 0000000..adc9c2c --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_demand_response_schema.json @@ -0,0 +1,12 @@ +{ + "type": "struct", + "fields": [ + {"name": "event_id", "type": "integer", "nullable": false, "metadata": {}}, + {"name": "customer_id", "type": "integer", "nullable": false, "metadata": {}}, + {"name": "event_date", "type": "date", "nullable": true, "metadata": {}}, + {"name": "event_type", "type": "string", "nullable": true, "metadata": {}}, + {"name": "participated", "type": "boolean", "nullable": true, "metadata": {}}, + {"name": "kwh_reduced", "type": "double", "nullable": true, "metadata": {}}, + {"name": "incentive_aud", "type": {"type": "decimal", "precision": 8, "scale": 2}, "nullable": true, "metadata": {}} + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_equipment_schema.json b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_equipment_schema.json new file mode 100644 index 0000000..48aed18 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_equipment_schema.json @@ -0,0 +1,14 @@ +{ + "type": "struct", + "fields": [ + {"name": "equipment_id", "type": "integer", "nullable": false, "metadata": {}}, + {"name": "equipment_type", "type": "string", "nullable": true, "metadata": {}}, + {"name": "state", "type": "string", "nullable": true, "metadata": {}}, + {"name": "install_date", "type": "date", "nullable": true, "metadata": {}}, + {"name": "last_maintenance_date", "type": "date", "nullable": true, "metadata": {}}, + {"name": "condition_score", "type": "double", "nullable": true, "metadata": {}}, + {"name": "failure_count", "type": "integer", "nullable": true, "metadata": {}}, + {"name": "latitude", "type": "double", "nullable": true, "metadata": {}}, + {"name": "longitude", "type": "double", "nullable": true, "metadata": {}} + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_meter_readings_schema.json b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_meter_readings_schema.json new file mode 100644 index 0000000..582ea19 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_meter_readings_schema.json @@ -0,0 +1,11 @@ +{ + "type": "struct", + "fields": [ + {"name": "meter_id", "type": "integer", "nullable": false, "metadata": {}}, + {"name": "customer_id", "type": "integer", "nullable": false, "metadata": {}}, + {"name": "reading_timestamp", "type": "timestamp", "nullable": false, "metadata": {}}, + {"name": "kwh_used", "type": "double", "nullable": true, "metadata": {}}, + {"name": "voltage", "type": "double", "nullable": true, "metadata": {}}, + {"name": "power_factor", "type": "double", "nullable": true, "metadata": {}} + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_outages_schema.json b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_outages_schema.json new file mode 100644 index 0000000..ac03206 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_outages_schema.json @@ -0,0 +1,14 @@ +{ + "type": "struct", + "fields": [ + {"name": "outage_id", "type": "integer", "nullable": false, "metadata": {}}, + {"name": "state", "type": "string", "nullable": true, "metadata": {}}, + {"name": "cause", "type": "string", "nullable": true, "metadata": {}}, + {"name": "start_time", "type": "timestamp", "nullable": true, "metadata": {}}, + {"name": "end_time", "type": "timestamp", "nullable": true, "metadata": {}}, + {"name": "duration_minutes", "type": "double", "nullable": true, "metadata": {}}, + {"name": "affected_meters", "type": "integer", "nullable": true, "metadata": {}}, + {"name": "latitude", "type": "double", "nullable": true, "metadata": {}}, + {"name": "longitude", "type": "double", "nullable": true, "metadata": {}} + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_weather_schema.json b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_weather_schema.json new file mode 100644 index 0000000..9703902 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-bronze/schemas/raw_weather_schema.json @@ -0,0 +1,12 @@ +{ + "type": "struct", + "fields": [ + {"name": "state", "type": "string", "nullable": false, "metadata": {}}, + {"name": "date", "type": "date", "nullable": false, "metadata": {}}, + {"name": "temp_high_c", "type": "double", "nullable": true, "metadata": {}}, + {"name": "temp_low_c", "type": "double", "nullable": true, "metadata": {}}, + {"name": "humidity_pct", "type": "double", "nullable": true, "metadata": {}}, + {"name": "wind_speed_kmh", "type": "double", "nullable": true, "metadata": {}}, + {"name": "precipitation_mm", "type": "double", "nullable": true, "metadata": {}} + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-gold/dataflowspec/energy_kpis_main.json b/skills/dataflowspec_builder/examples/energy-gold/dataflowspec/energy_kpis_main.json new file mode 100644 index 0000000..50e4377 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-gold/dataflowspec/energy_kpis_main.json @@ -0,0 +1,65 @@ +{ + "dataFlowId": "energy_gold_kpis", + "dataFlowGroup": "energy_gold", + "dataFlowType": "materialized_view", + "materializedViews": { + "mv_revenue_by_state": { + "sqlStatement": "SELECT c.state, DATE_TRUNC('month', b.bill_date) AS month, COUNT(DISTINCT b.customer_id) AS active_customers, SUM(b.amount_aud) AS total_revenue_aud, AVG(b.amount_aud) AS avg_bill_aud, SUM(b.kwh_consumed) AS total_kwh FROM {silver_schema}.silver_customer_360 c INNER JOIN {bronze_schema}.bronze_billing b ON c.customer_id = b.customer_id WHERE b.is_duplicate = false OR b.is_duplicate IS NULL GROUP BY c.state, DATE_TRUNC('month', b.bill_date)", + "tableDetails": { + "database": "{gold_schema}", + "clusterByColumns": ["state", "month"], + "tableProperties": { + "delta.autoOptimize.optimizeWrite": "true" + }, + "comment": "Monthly revenue and consumption metrics by Australian state" + }, + "dataQualityExpectationsEnabled": false, + "quarantineMode": "off", + "quarantineTargetDetails": {} + }, + "mv_grid_reliability": { + "sqlStatement": "SELECT state, cause, COUNT(*) AS outage_count, AVG(duration_minutes) AS avg_duration_minutes, SUM(affected_meters) AS total_affected_meters, ROUND(SUM(duration_minutes * affected_meters) / 50000.0, 4) AS saidi_contribution FROM {bronze_schema}.bronze_outages WHERE start_time <= current_timestamp() GROUP BY state, cause", + "tableDetails": { + "database": "{gold_schema}", + "clusterByColumns": ["state"], + "comment": "Grid reliability metrics — outages by state and cause with SAIDI contribution" + }, + "dataQualityExpectationsEnabled": false, + "quarantineMode": "off", + "quarantineTargetDetails": {} + }, + "mv_demand_response_effectiveness": { + "sqlStatement": "SELECT dr.event_type, c.state, c.customer_type, COUNT(*) AS total_events, SUM(CASE WHEN dr.participated THEN 1 ELSE 0 END) AS participated_count, ROUND(SUM(CASE WHEN dr.participated THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 2) AS participation_rate_pct, SUM(dr.kwh_reduced) AS total_kwh_reduced, SUM(dr.incentive_aud) AS total_incentive_aud, ROUND(SUM(dr.incentive_aud) / NULLIF(SUM(dr.kwh_reduced), 0), 4) AS cost_per_kwh_aud FROM {bronze_schema}.bronze_demand_response dr INNER JOIN {silver_schema}.silver_customer_360 c ON dr.customer_id = c.customer_id GROUP BY dr.event_type, c.state, c.customer_type", + "tableDetails": { + "database": "{gold_schema}", + "clusterByColumns": ["event_type", "state"], + "comment": "Demand response program effectiveness by event type, state, and customer segment" + }, + "dataQualityExpectationsEnabled": false, + "quarantineMode": "off", + "quarantineTargetDetails": {} + }, + "mv_equipment_risk_score": { + "sqlStatement": "SELECT e.equipment_id, e.equipment_type, e.state, e.install_date, e.last_maintenance_date, e.condition_score, e.failure_count, DATEDIFF(current_date(), e.install_date) AS age_days, DATEDIFF(current_date(), e.last_maintenance_date) AS days_since_maintenance, ROUND((e.failure_count * 0.3) + ((1 - e.condition_score) * 0.4) + (DATEDIFF(current_date(), e.last_maintenance_date) / 365.0 * 0.3), 4) AS risk_score, e.latitude, e.longitude FROM {bronze_schema}.bronze_equipment e", + "tableDetails": { + "database": "{gold_schema}", + "clusterByColumns": ["state", "equipment_type"], + "comment": "Equipment risk scores based on age, condition, failure history, and maintenance recency" + }, + "dataQualityExpectationsEnabled": false, + "quarantineMode": "off", + "quarantineTargetDetails": {} + }, + "mv_solar_ev_adoption": { + "sqlStatement": "SELECT c.state, c.customer_type, COUNT(*) AS total_customers, SUM(CASE WHEN c.has_solar THEN 1 ELSE 0 END) AS solar_customers, SUM(CASE WHEN c.has_ev THEN 1 ELSE 0 END) AS ev_customers, SUM(CASE WHEN c.has_solar AND c.has_ev THEN 1 ELSE 0 END) AS solar_and_ev_customers, ROUND(SUM(CASE WHEN c.has_solar THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 2) AS solar_adoption_pct, ROUND(SUM(CASE WHEN c.has_ev THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 2) AS ev_adoption_pct FROM {silver_schema}.silver_customer_360 c GROUP BY c.state, c.customer_type", + "tableDetails": { + "database": "{gold_schema}", + "clusterByColumns": ["state"], + "comment": "Solar and EV adoption rates by state and customer type for sustainability reporting" + }, + "dataQualityExpectationsEnabled": false, + "quarantineMode": "off", + "quarantineTargetDetails": {} + } + } +} diff --git a/skills/dataflowspec_builder/examples/energy-gold/dml/mv_consumption_heatmap.sql b/skills/dataflowspec_builder/examples/energy-gold/dml/mv_consumption_heatmap.sql new file mode 100644 index 0000000..964d68c --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-gold/dml/mv_consumption_heatmap.sql @@ -0,0 +1,16 @@ +-- Hourly consumption heatmap by state and hour-of-day +-- Used as sqlPath source for a materialized view +SELECT + c.state, + HOUR(m.reading_timestamp) AS hour_of_day, + DAYOFWEEK(m.reading_timestamp) AS day_of_week, + CASE WHEN DAYOFWEEK(m.reading_timestamp) IN (1, 7) THEN 'weekend' ELSE 'weekday' END AS day_type, + COUNT(*) AS reading_count, + ROUND(AVG(m.kwh_used), 4) AS avg_kwh, + ROUND(PERCENTILE_APPROX(m.kwh_used, 0.95), 4) AS p95_kwh, + ROUND(STDDEV(m.kwh_used), 4) AS stddev_kwh +FROM {bronze_schema}.bronze_meter_readings m +INNER JOIN {silver_schema}.silver_customer_360 c + ON m.customer_id = c.customer_id +WHERE m.kwh_used IS NOT NULL AND m.kwh_used >= 0 +GROUP BY c.state, HOUR(m.reading_timestamp), DAYOFWEEK(m.reading_timestamp) diff --git a/skills/dataflowspec_builder/examples/energy-silver/dataflowspec/customer_360_main.json b/skills/dataflowspec_builder/examples/energy-silver/dataflowspec/customer_360_main.json new file mode 100644 index 0000000..8cf461f --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-silver/dataflowspec/customer_360_main.json @@ -0,0 +1,110 @@ +{ + "dataFlowId": "customer_360", + "dataFlowGroup": "energy_silver", + "dataFlowType": "flow", + "targetFormat": "delta", + "targetDetails": { + "table": "silver_customer_360", + "tableProperties": { + "delta.enableChangeDataFeed": "true" + }, + "clusterByColumns": ["customer_id"] + }, + "cdcSettings": { + "keys": ["customer_id"], + "sequence_by": "last_updated", + "scd_type": "2", + "ignore_null_updates": true, + "except_column_list": ["__START_AT", "__END_AT"] + }, + "dataQualityExpectationsEnabled": true, + "dataQualityExpectationsPath": "silver_customer_360_dqe.json", + "quarantineMode": "flag", + "quarantineTargetDetails": {}, + "flowGroups": [ + { + "flowGroupId": "fg_customer_billing", + "stagingTables": { + "staging_customer_billing": { + "type": "ST", + "schemaPath": "" + } + }, + "flows": { + "f_customers": { + "flowType": "append_view", + "flowDetails": { + "targetTable": "staging_customer_billing", + "sourceView": "v_bronze_customers" + }, + "views": { + "v_bronze_customers": { + "mode": "stream", + "sourceType": "delta", + "sourceDetails": { + "database": "{bronze_schema}", + "table": "bronze_customers", + "cdfEnabled": true, + "selectExp": [ + "customer_id", + "name", + "email", + "state", + "customer_type", + "rate_plan", + "has_solar", + "has_ev", + "signup_date", + "latitude", + "longitude", + "current_timestamp() AS last_updated" + ], + "whereClause": [] + } + } + } + }, + "f_billing_summary": { + "flowType": "append_view", + "flowDetails": { + "targetTable": "staging_customer_billing", + "sourceView": "v_billing_agg" + }, + "views": { + "v_billing_agg": { + "mode": "stream", + "sourceType": "delta", + "sourceDetails": { + "database": "{bronze_schema}", + "table": "bronze_billing", + "cdfEnabled": true, + "selectExp": [ + "customer_id", + "CAST(NULL AS STRING) AS name", + "CAST(NULL AS STRING) AS email", + "CAST(NULL AS STRING) AS state", + "CAST(NULL AS STRING) AS customer_type", + "CAST(NULL AS STRING) AS rate_plan", + "CAST(NULL AS BOOLEAN) AS has_solar", + "CAST(NULL AS BOOLEAN) AS has_ev", + "CAST(NULL AS DATE) AS signup_date", + "CAST(NULL AS DOUBLE) AS latitude", + "CAST(NULL AS DOUBLE) AS longitude", + "current_timestamp() AS last_updated" + ], + "whereClause": ["is_duplicate = false OR is_duplicate IS NULL"] + } + } + } + }, + "f_merge_customer_360": { + "flowType": "merge", + "flowDetails": { + "targetTable": "silver_customer_360", + "sourceView": "staging_customer_billing" + } + } + } + } + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-silver/dataflowspec/meter_weather_join_main.json b/skills/dataflowspec_builder/examples/energy-silver/dataflowspec/meter_weather_join_main.json new file mode 100644 index 0000000..f8be590 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-silver/dataflowspec/meter_weather_join_main.json @@ -0,0 +1,86 @@ +{ + "dataFlowId": "meter_weather_enriched", + "dataFlowGroup": "energy_silver", + "dataFlowType": "flow", + "targetFormat": "delta", + "targetDetails": { + "table": "silver_meter_weather", + "tableProperties": { + "delta.enableChangeDataFeed": "true" + }, + "clusterByColumns": ["customer_id", "reading_date"] + }, + "cdcSettings": { + "keys": ["meter_id", "reading_timestamp"], + "sequence_by": "reading_timestamp", + "scd_type": "1", + "ignore_null_updates": true, + "except_column_list": [] + }, + "dataQualityExpectationsEnabled": false, + "quarantineMode": "off", + "quarantineTargetDetails": {}, + "flowGroups": [ + { + "flowGroupId": "fg_meter_weather", + "stagingTables": {}, + "flows": { + "f_stream_static_join": { + "flowType": "merge", + "flowDetails": { + "targetTable": "silver_meter_weather", + "sourceView": "v_meter_weather_joined" + }, + "views": { + "v_meter_weather_joined": { + "mode": "stream", + "sourceType": "deltaJoin", + "sourceDetails": { + "sources": [ + { + "alias": "m", + "database": "{bronze_schema}", + "table": "bronze_meter_readings", + "mode": "stream", + "cdfEnabled": true + }, + { + "alias": "c", + "database": "{bronze_schema}", + "table": "bronze_customers", + "mode": "static" + }, + { + "alias": "w", + "database": "{bronze_schema}", + "table": "bronze_weather", + "mode": "static" + } + ], + "joinType": "left", + "joinCondition": "m.customer_id = c.customer_id AND c.state = w.state AND CAST(m.reading_timestamp AS DATE) = w.date", + "selectExp": [ + "m.meter_id", + "m.customer_id", + "m.reading_timestamp", + "CAST(m.reading_timestamp AS DATE) AS reading_date", + "m.kwh_used", + "m.voltage", + "m.power_factor", + "c.state", + "c.customer_type", + "c.rate_plan", + "c.has_solar", + "w.temp_high_c", + "w.temp_low_c", + "w.humidity_pct", + "w.wind_speed_kmh" + ] + } + } + } + } + } + } + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-silver/expectations/silver_customer_360_dqe.json b/skills/dataflowspec_builder/examples/energy-silver/expectations/silver_customer_360_dqe.json new file mode 100644 index 0000000..ef99b17 --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-silver/expectations/silver_customer_360_dqe.json @@ -0,0 +1,11 @@ +{ + "expect": [ + {"name": "customer_id_not_null", "constraint": "customer_id IS NOT NULL", "tag": "completeness", "enabled": true}, + {"name": "valid_state", "constraint": "state IN ('NSW', 'VIC', 'QLD', 'SA', 'WA', 'TAS') OR state IS NULL", "tag": "validity", "enabled": true}, + {"name": "valid_customer_type", "constraint": "customer_type IN ('residential', 'commercial', 'industrial') OR customer_type IS NULL", "tag": "validity", "enabled": true} + ], + "expect_or_drop": [], + "expect_or_fail": [ + {"name": "pk_not_null", "constraint": "customer_id IS NOT NULL", "tag": "integrity", "enabled": true} + ] +} diff --git a/skills/dataflowspec_builder/examples/energy-templates/dataflowspec/energy_bronze_ingestion_template.json b/skills/dataflowspec_builder/examples/energy-templates/dataflowspec/energy_bronze_ingestion_template.json new file mode 100644 index 0000000..3c3b9fb --- /dev/null +++ b/skills/dataflowspec_builder/examples/energy-templates/dataflowspec/energy_bronze_ingestion_template.json @@ -0,0 +1,44 @@ +{ + "name": "energy_bronze_ingestion_template", + "parameters": { + "dataFlowId": {"type": "string", "required": true}, + "sourceTable": {"type": "string", "required": true}, + "targetTable": {"type": "string", "required": true}, + "schemaPath": {"type": "string", "required": false, "default": ""}, + "keyColumns": {"type": "list", "required": true}, + "sequenceBy": {"type": "string", "required": true} + }, + "template": { + "dataFlowId": "${param.dataFlowId}", + "dataFlowGroup": "energy_bronze", + "dataFlowType": "standard", + "sourceType": "delta", + "sourceSystem": "smartgrid", + "sourceViewName": "v_${param.sourceTable}", + "sourceDetails": { + "database": "{bronze_schema}", + "table": "${param.sourceTable}", + "cdfEnabled": true, + "schemaPath": "${param.schemaPath}" + }, + "mode": "stream", + "targetFormat": "delta", + "targetDetails": { + "table": "${param.targetTable}", + "tableProperties": { + "delta.autoOptimize.optimizeWrite": "true", + "delta.autoOptimize.autoCompact": "true" + } + }, + "cdcSettings": { + "keys": "${param.keyColumns}", + "sequence_by": "${param.sequenceBy}", + "scd_type": "1", + "ignore_null_updates": true, + "except_column_list": [] + }, + "dataQualityExpectationsEnabled": true, + "quarantineMode": "off", + "quarantineTargetDetails": {} + } +} diff --git a/skills/dataflowspec_builder/references/dataflow-spec-schema-reference.md b/skills/dataflowspec_builder/references/dataflow-spec-schema-reference.md new file mode 100644 index 0000000..b528b72 --- /dev/null +++ b/skills/dataflowspec_builder/references/dataflow-spec-schema-reference.md @@ -0,0 +1,202 @@ +# Data Flow Spec — Complete Field Reference + +All fields used across Standard, Flows, and Materialized View data flow specs. + +## Common Metadata (All Types) + +| Field | Type | Required | Values | Description | +|-------|------|----------|--------|-------------| +| `dataFlowId` | string | Yes | — | Unique identifier for this data flow | +| `dataFlowGroup` | string | Yes | — | Logical group (used for pipeline filters) | +| `dataFlowType` | string | Yes | `standard`, `flow`, `materialized_view` | Determines which spec schema applies | + +## Standard-Only Fields + +| Field | Type | Required | Values | Description | +|-------|------|----------|--------|-------------| +| `sourceType` | string | Yes | `cloudFiles`, `delta`, `deltaJoin`, `kafka` | Source connector type | +| `sourceSystem` | string | Yes | — | Descriptive name of origin system | +| `sourceViewName` | string | Yes | `v_` pattern | Name for the auto-created source view | +| `sourceDetails` | object | Yes | — | Source-specific configuration (see below) | +| `mode` | string | Yes | `stream`, `batch` | Pipeline execution mode | +| `targetFormat` | string | Yes | `delta`, `foreach_batch_sink` | Target write format | +| `targetDetails` | object | Yes | — | Target table configuration (see below) | + +## Source Details — By Source Type + +### delta +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `database` | string | Yes | `catalog.schema` reference | +| `table` | string | Yes | Source table name | +| `cdfEnabled` | boolean | No | Enable Change Data Feed reading | +| `schemaPath` | string | No | Path to schema JSON file | +| `selectExp` | list | No | Column expressions (`["*"]` or `["col1", "CAST(col2 AS DATE)"]`) | +| `whereClause` | list | No | Filter expressions | +| `readerOptions` | object | No | Spark reader options | +| `pythonTransform.module` | string | No | Extension module path (`transforms.func_name`) | +| `pythonTransform.path` | string | No | File path to Python function | + +### cloudFiles +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `path` | string | Yes | Volume or cloud storage path | +| `readerOptions` | object | Yes | Must include `cloudFiles.format` | +| `schemaPath` | string | No | Path to schema JSON | +| `selectExp` | list | No | Column expressions | + +### deltaJoin +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `sources` | list | Yes | Array of source objects (alias, database, table, mode, cdfEnabled) | +| `joinType` | string | Yes | `inner` or `left` | +| `joinCondition` | string | Yes | SQL join expression | +| `selectExp` | list | No | Columns to select from joined result | + +### kafka +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `readerOptions` | object | Yes | Must include `kafka.bootstrap.servers`, `subscribe` | +| `schemaPath` | string | No | Path to value schema | + +### python +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `pythonModule` | string | Yes | Extension module path (`sources.func_name`) | +| `tokens` | object | No | Key-value pairs passed to the Python function | + +### sql +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `sqlPath` | string | Yes | Path to `.sql` file in `dml/` directory | + +## Target Details + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `table` | string | Yes | Target table name | +| `schemaPath` | string | No | Path to target schema JSON | +| `tableProperties` | object | No | Delta table properties | +| `partitionColumns` | list | No | Partition columns | +| `clusterByColumns` | list | No | Liquid clustering columns | +| `clusterByAuto` | boolean | No | Let Databricks choose clustering keys | +| `path` | string | No | External storage path | +| `operationalMetadata` | boolean | No | Disable operational metadata columns | + +## CDC Settings + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `keys` | list | Yes | Primary key column(s) | +| `sequence_by` | string | Yes | Ordering column | +| `scd_type` | string | Yes | `"1"` or `"2"` | +| `where` | string | No | Row filter | +| `ignore_null_updates` | boolean | No | Skip null values in updates | +| `except_column_list` | list | No | Columns excluded from upsert | +| `apply_as_deletes` | string | No | SQL expression for delete detection | +| `track_history_column_list` | list | No | Explicit history columns (SCD2) | +| `track_history_except_column_list` | list | No | Exclude from history (SCD2) | +| `soft_delete` | boolean | No | Flag instead of physical delete | + +## CDC Snapshot Settings + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `source.format` | string | File | `parquet`, `csv`, `json`, `table` | +| `source.path` | string | File | Path with `{version}` placeholder | +| `source.versionType` | string | File | `int` or `datetime` | +| `source.datetimeFormat` | string | Conditional | Format for datetime versions | +| `source.startingVersion` | string/int | No | First version to process | +| `source.table` | string | Table | Source table name | +| `source.versionColumn` | string | Table | Column for versioning | + +## Data Quality + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `dataQualityExpectationsEnabled` | boolean | No | Enable DQ checks | +| `dataQualityExpectationsPath` | string | No | Path to expectations JSON file | +| `quarantineMode` | string | No | `off`, `flag`, `table` | +| `quarantineTargetDetails.targetFormat` | string | Conditional | `delta` | +| `quarantineTargetDetails.table` | string | Conditional | Quarantine table name | +| `quarantineTargetDetails.tableProperties` | object | Conditional | Properties | + +## Expectations File Schema + +```json +{ + "expect": [{"name": "...", "constraint": "SQL", "tag": "...", "enabled": true}], + "expect_or_drop": [...], + "expect_or_fail": [...] +} +``` + +## Flow-Specific Fields + +### flowGroups (array) +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `flowGroupId` | string | Yes | Unique flow group identifier | +| `stagingTables` | object | No | Named staging table definitions | +| `flows` | object | Yes | Named flow definitions | + +### Staging Table +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | string | Yes | `ST` (Streaming Table) or `MV` (Materialized View) | +| `schemaPath` | string | No | Schema file path | +| `partitionColumns` | list | No | Partition columns | +| `tableProperties` | object | No | Delta properties | +| `cdcSettings` | object | No | CDC config for the staging table | + +### Flow +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `enabled` | boolean | No | Enable/disable this flow | +| `flowType` | string | Yes | `append_view`, `append_sql`, `merge` | +| `flowDetails` | object | Yes | Source and target references | +| `views` | object | No | Named view definitions | + +### Flow Details (by type) +- **append_view:** `targetTable`, `sourceView`, optional `column_prefix`/`column_prefix_exceptions` +- **append_sql:** `targetTable`, `sqlPath` +- **merge:** `targetTable`, `sourceView` + +### View +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `mode` | string | Yes | `stream` or `batch` | +| `sourceType` | string | Yes | `cloudFiles`, `delta`, `deltaJoin`, `sql`, `python` | +| `sourceDetails` | object | Yes | Same as standard source details | +| `columnsToUpdate` | list | No | Columns to update | + +## Materialized View Fields + +### materializedViews (object of named MVs) +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `sourceView` | object | Pick one | Source view definition | +| `sqlPath` | string | Pick one | Path to SQL file | +| `sqlStatement` | string | Pick one | Inline SQL | +| `tableDetails` | object | No | Target table config | +| `tableDetails.database` | string | No | Target schema override | +| `tableDetails.schemaPath` | string | No | Schema file path | +| `tableDetails.tableProperties` | object | No | Delta properties | +| `tableDetails.clusterByColumns` | list | No | Clustering columns | +| `tableDetails.clusterByAuto` | boolean | No | Auto clustering | +| `tableDetails.comment` | string | No | Table description | +| `tableDetails.spark_conf` | object | No | Spark configs for this MV | +| `tableDetails.private` | boolean | No | Don't publish to metastore | + +## Table Migration + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `tableMigrationDetails.enabled` | boolean | Yes | Enable migration | +| `tableMigrationDetails.catalogType` | string | Yes | `hms` or `uc` | +| `tableMigrationDetails.autoStartingVersionsEnabled` | boolean | No | Auto-track versions | +| `tableMigrationDetails.sourceDetails.sourceMigrateDelta.database` | string | Yes | Old database | +| `tableMigrationDetails.sourceDetails.sourceMigrateDelta.table` | string | Yes | Old table | +| `tableMigrationDetails.sourceDetails.sourceMigrateDelta.selectExp` | list | No | Columns | +| `tableMigrationDetails.sourceDetails.sourceMigrateDelta.whereClause` | list | No | Filters | +| `tableMigrationDetails.sourceDetails.sourceMigrateDelta.exceptColumns` | list | No | Exclude columns | diff --git a/skills/dataflowspec_builder/references/energy-domain-mapping.md b/skills/dataflowspec_builder/references/energy-domain-mapping.md new file mode 100644 index 0000000..6f49e5e --- /dev/null +++ b/skills/dataflowspec_builder/references/energy-domain-mapping.md @@ -0,0 +1,93 @@ +# Energy Domain → Lakeflow Framework Mapping + +Maps the SmartGrid Analytics Platform tables and use cases to Lakeflow Framework patterns. + +## Source Tables (Raw) + +| Table | Rows | Key Column(s) | Sequence Column | Recommended Pattern | +|-------|------|---------------|-----------------|---------------------| +| `raw_customers` | 50,000 | `customer_id` | `signup_date` | Basic 1:1 (SCD2) | +| `raw_meter_readings` | ~10.7M | `meter_id`, `reading_timestamp` | `reading_timestamp` | Basic 1:1 (Append) | +| `raw_billing` | ~603K | `bill_id` | `bill_date` | Basic 1:1 (SCD1) | +| `raw_outages` | 5,000 | `outage_id` | `start_time` | Basic 1:1 (SCD1) | +| `raw_weather` | 2,190 | `state`, `date` | `date` | Basic 1:1 (SCD1) | +| `raw_equipment` | 2,000 | `equipment_id` | `last_maintenance_date` | Basic 1:1 (SCD2) | +| `raw_demand_response` | 20,000 | `event_id` | `event_date` | Basic 1:1 (SCD1) | + +## Bronze Layer — Data Flow Specs + +All 7 tables use the **Basic 1:1 Standard** pattern with a **template** to reduce duplication. + +**Template:** `energy_bronze_ingestion_template` +- 7 parameter sets → 7 data flow specs generated at runtime +- CDC enabled for all (SCD1 by default, SCD2 for customers and equipment) +- Data quality expectations per table + +**Key DQ Rules:** +- Customers: valid state (6 AU states), valid email, valid customer_type +- Meter readings: non-null meter_id, positive kwh, voltage in 200–260V range +- Billing: positive amount, deduplicate (drop is_duplicate=true) +- Outages: valid cause type, positive duration, drop future-dated records + +## Silver Layer — Data Flow Specs + +### 1. Customer 360 (Multi-Source Streaming) +**Pattern:** Multi-Source Streaming → `flow` type +**Sources:** bronze_customers + bronze_billing +**Target:** silver_customer_360 (SCD2) +**Logic:** Merge customer demographics with billing aggregates + +### 2. Meter-Weather Enriched (Stream-Static Join) +**Pattern:** Stream-Static Join → `flow` type +**Sources:** bronze_meter_readings (stream) + bronze_customers (static) + bronze_weather (static) +**Target:** silver_meter_weather +**Logic:** Enrich meter readings with customer state and weather conditions + +### 3. Equipment-Outage Correlation (Multi-Source Streaming) +**Pattern:** Multi-Source Streaming → `flow` type +**Sources:** bronze_equipment + bronze_outages +**Target:** silver_equipment_outages +**Logic:** Correlate equipment conditions with outage events by location + +## Gold Layer — Materialized Views + +### KPI Materialized Views (single spec, 5 MVs) + +| MV Name | Cluster By | Description | +|---------|-----------|-------------| +| `mv_revenue_by_state` | state, month | Monthly revenue and consumption by AU state | +| `mv_grid_reliability` | state | SAIDI contribution by cause and state | +| `mv_demand_response_effectiveness` | event_type, state | DR program effectiveness metrics | +| `mv_equipment_risk_score` | state, equipment_type | Composite risk scores for predictive maintenance | +| `mv_solar_ev_adoption` | state | Solar/EV adoption rates by state and segment | + +### Additional MVs (via sqlPath) + +| MV Name | SQL File | Description | +|---------|----------|-------------| +| `mv_consumption_heatmap` | `dml/mv_consumption_heatmap.sql` | Hourly consumption patterns by state | + +## Substitution Tokens + +| Token | Dev Value | Prod Value | +|-------|-----------|------------| +| `{bronze_schema}` | `main.bronze_energy` | `main.bronze_energy_prod` | +| `{silver_schema}` | `main.silver_energy` | `main.silver_energy_prod` | +| `{gold_schema}` | `main.gold_energy` | `main.gold_energy_prod` | +| `{landing_path}` | `/Volumes/main/energy/landing` | `/Volumes/main/energy_prod/landing` | + +## Pipeline Definitions + +| Pipeline | Layer | Filter | Tables | +|----------|-------|--------|--------| +| `energy_bronze_pipeline` | bronze | `pipeline.dataFlowGroupFilter: energy_bronze` | All 7 raw tables | +| `energy_silver_pipeline` | silver | `pipeline.dataFlowGroupFilter: energy_silver` | customer_360, meter_weather, equipment_outages | +| `energy_gold_pipeline` | gold | `pipeline.dataFlowGroupFilter: energy_gold` | All materialized views | + +## Recommended Extensions + +| Extension | Type | Use Case | +|-----------|------|----------| +| `transforms.clean_and_deduplicate` | Transform | Remove duplicate meter readings | +| `transforms.add_hash_key` | Transform | Generate surrogate keys for silver tables | +| `sources.get_cdf_source` | Source | Custom CDF reading with filters | diff --git a/skills/dataflowspec_builder/references/patterns-guide.md b/skills/dataflowspec_builder/references/patterns-guide.md new file mode 100644 index 0000000..3b572ff --- /dev/null +++ b/skills/dataflowspec_builder/references/patterns-guide.md @@ -0,0 +1,219 @@ +# Lakeflow Framework — Patterns Quick Reference + +## Pattern Selection Guide + +| Your Scenario | Pattern | Data Flow Type | Layer | +|---------------|---------|---------------|-------| +| Ingesting raw data from a source table | Basic 1:1 | `standard` | Bronze | +| Ingesting files from cloud storage / Volumes | CloudFiles 1:1 | `standard` | Bronze | +| Ingesting periodic snapshot exports | CDC from Snapshots | `standard` | Bronze | +| Joining a streaming fact with a static dimension | Stream-Static | `flow` | Silver | +| Merging multiple sources into one target | Multi-Source Streaming | `flow` | Silver | +| Building a full streaming data warehouse | Stream-Static DWH | `flow` | Silver/Gold | +| Creating precomputed KPI aggregations | Materialized Views | `materialized_view` | Gold | + +--- + +## Pattern 1: Basic 1:1 (Bronze Ingestion) + +**Use when:** Ingesting data from a single source table to a single target. + +``` +Source Table ──▶ [Source View] ──▶ Target Table + (streaming) (SCD0/1/2) +``` + +**Supports:** Append-only, SCD Type 1 (overwrite), SCD Type 2 (history) +**Transforms:** Basic single-row only (type conversion, formatting, concatenation) + +**Data Flow Spec type:** `standard` +**Key fields:** `sourceType: delta`, `mode: stream`, `cdcSettings` for SCD + +**When NOT to use:** Complex joins, multiple sources, window functions, aggregations. + +--- + +## Pattern 2: CloudFiles Ingestion (Bronze from Files) + +**Use when:** Ingesting CSV/JSON/Parquet files from UC Volumes or cloud storage. + +``` +Cloud Storage ──▶ [Auto Loader] ──▶ Target Table + /Volumes/... (cloudFiles) (streaming) +``` + +**Data Flow Spec type:** `standard` +**Key fields:** `sourceType: cloudFiles`, reader options for format + +```json +"sourceDetails": { + "path": "/Volumes/main/raw/data_landing/customers/", + "readerOptions": { + "cloudFiles.format": "csv", + "header": "true", + "cloudFiles.inferColumnTypes": "true" + } +} +``` + +--- + +## Pattern 3: CDC from Historical Snapshots + +**Use when:** You receive periodic full snapshots and need to derive CDC events. + +``` +Snapshot Files ──▶ [CDC from Snapshot] ──▶ Target Table + (daily exports) (automatic diff) (SCD1/2) +``` + +**Data Flow Spec type:** `standard` with `cdcSnapshotSettings` + +File-based snapshots: +```json +"cdcSnapshotSettings": { + "source": { + "format": "parquet", + "path": "/Volumes/main/raw/snapshots/{version}/", + "versionType": "datetime", + "datetimeFormat": "yyyyMMdd", + "startingVersion": "20240101" + } +} +``` + +Table-based snapshots: +```json +"cdcSnapshotSettings": { + "source": { + "table": "main.staging.daily_customer_snapshot", + "versionColumn": "snapshot_date", + "startingVersion": "2024-01-01" + } +} +``` + +--- + +## Pattern 4: Stream-Static Join (Silver) + +**Use when:** Joining a streaming fact table with one or more static dimension tables. + +``` +Streaming Fact ──┐ + ├──▶ [deltaJoin View] ──▶ [merge] ──▶ Target Table +Static Dimension ┘ (stream + static) (SCD1/2) +``` + +**Data Flow Spec type:** `flow` +**Key fields:** View with `sourceType: deltaJoin`, sources with `mode: stream/static` + +```json +"sourceDetails": { + "sources": [ + {"alias": "fact", "database": "...", "table": "meter_readings", "mode": "stream", "cdfEnabled": true}, + {"alias": "dim", "database": "...", "table": "customers", "mode": "static"} + ], + "joinType": "left", + "joinCondition": "fact.customer_id = dim.customer_id", + "selectExp": ["fact.*", "dim.state", "dim.customer_type"] +} +``` + +--- + +## Pattern 5: Multi-Source Streaming (Silver Merge) + +**Use when:** Multiple streaming sources need to converge into a single target table. + +``` +Source A ──▶ [append_view] ──┐ + ├──▶ Staging Table ──▶ [merge] ──▶ Target Table +Source B ──▶ [append_view] ──┘ (SCD1/2) +``` + +**Data Flow Spec type:** `flow` with `flowGroups` +**Key benefit:** Add/remove sources (flow groups) without full pipeline refresh. + +Flow structure: +1. Multiple `append_view` flows write into a shared staging table (`type: ST`) +2. A single `merge` flow CDC-merges the staging table into the final target + +**Evolving over time:** Add a new `flowGroup` with new sources → pipeline picks it up without affecting existing data. + +--- + +## Pattern 6: Materialized Views (Gold KPIs) + +**Use when:** Building precomputed aggregations, KPIs, or reporting tables. + +``` +Source Table(s) ──▶ [SQL Query / View] ──▶ Materialized View + (auto-refreshed) +``` + +**Data Flow Spec type:** `materialized_view` +**Source options (pick one per MV):** +1. `sourceView` — Delta table source with optional Python transform +2. `sqlPath` — Path to a `.sql` file +3. `sqlStatement` — Inline SQL string + +Multiple MVs can be defined in a single spec: +```json +{ + "dataFlowType": "materialized_view", + "materializedViews": { + "mv_revenue": {"sqlStatement": "SELECT ..."}, + "mv_reliability": {"sqlPath": "./dml/reliability.sql"}, + "mv_adoption": {"sourceView": {"sourceViewName": "v_adoption", ...}} + } +} +``` + +--- + +## Combining Patterns in a Single Bundle + +A pipeline bundle can contain multiple data flow specs of different types. Use pipeline filters to control which data flows each pipeline executes: + +``` +my_energy_bundle/ +├── src/dataflows/ +│ ├── bronze/ ← Standard specs (Pattern 1) +│ ├── silver/ ← Flow specs (Patterns 4 & 5) +│ └── gold/ ← Materialized view specs (Pattern 6) +├── resources/ +│ ├── bronze_pipeline.yml ← pipeline.layer: bronze +│ ├── silver_pipeline.yml ← pipeline.layer: silver +│ └── gold_pipeline.yml ← pipeline.layer: gold +``` + +Or use data flow group filters: +```yaml +configuration: + pipeline.dataFlowGroupFilter: energy_bronze +``` + +--- + +## Feature Compatibility Matrix + +| Feature | Standard | Flows | Materialized View | +|---------|----------|-------|-------------------| +| CDC (SCD1/2) | ✅ | ✅ | ❌ | +| CDC from Snapshots | ✅ | ❌ | ❌ | +| Data Quality Expectations | ✅ | ✅ | ✅ | +| Quarantine (flag/table) | ✅ | ✅ | ✅ | +| Liquid Clustering | ✅ | ✅ | ✅ | +| Multi-Source Streaming | ❌ | ✅ | ❌ | +| Stream-Static Joins | ❌ | ✅ | ❌ | +| SQL Transforms | ✅ | ✅ | ✅ | +| Python Transforms | ✅ | ✅ | ✅ | +| Python Extensions | ✅ | ✅ | ✅ | +| Templates | ✅ | ✅ | ✅ | +| Substitutions | ✅ | ✅ | ✅ | +| Table Migration | ✅ | ✅ | ❌ | +| Soft Deletes | ✅ | ✅ | ❌ | +| Operational Metadata | ✅ | ✅ | ❌ | +| Secrets | ✅ | ✅ | ✅ | +| Foreach Batch Sink | ✅ | ❌ | ❌ | diff --git a/skills/dataflowspec_builder/scripts/deploy_framework.sh b/skills/dataflowspec_builder/scripts/deploy_framework.sh new file mode 100755 index 0000000..727ebad --- /dev/null +++ b/skills/dataflowspec_builder/scripts/deploy_framework.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Deploy the Lakeflow Framework bundle to a Databricks workspace. +# +# Prerequisites: +# - Databricks CLI installed and configured +# - Git installed +# +# Usage: +# ./deploy_framework.sh [options] +# +# Options: +# -t, --target DABs target environment (default: dev) +# -v, --version Framework version tag (default: current) +# -p, --profile Databricks CLI profile (default: DEFAULT) +# -d, --dir Directory to clone into (default: /tmp/lakeflow_framework) +# -h, --help Show this help message + +set -euo pipefail + +TARGET="dev" +VERSION="current" +PROFILE="DEFAULT" +CLONE_DIR="/tmp/lakeflow_framework" + +usage() { + echo "Usage: $0 [-t target] [-v version] [-p profile] [-d dir]" + echo "" + echo " -t, --target DABs target (default: dev)" + echo " -v, --version Framework version (default: current)" + echo " -p, --profile Databricks CLI profile (default: DEFAULT)" + echo " -d, --dir Clone directory (default: /tmp/lakeflow_framework)" + echo " -h, --help Show help" + exit 0 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -t|--target) TARGET="$2"; shift 2 ;; + -v|--version) VERSION="$2"; shift 2 ;; + -p|--profile) PROFILE="$2"; shift 2 ;; + -d|--dir) CLONE_DIR="$2"; shift 2 ;; + -h|--help) usage ;; + *) echo "Unknown option: $1"; usage ;; + esac +done + +echo "=== Lakeflow Framework Deployment ===" +echo " Target: $TARGET" +echo " Version: $VERSION" +echo " Profile: $PROFILE" +echo "" + +if [ -d "$CLONE_DIR" ]; then + echo "Updating existing clone at $CLONE_DIR..." + cd "$CLONE_DIR" + git pull --ff-only +else + echo "Cloning Lakeflow Framework..." + git clone https://github.com/databricks-solutions/lakeflow_framework.git "$CLONE_DIR" + cd "$CLONE_DIR" +fi + +if [ "$VERSION" != "current" ]; then + echo "Checking out version: $VERSION" + git checkout "v$VERSION" 2>/dev/null || git checkout "$VERSION" +fi + +echo "" +echo "Validating bundle..." +databricks bundle validate -t "$TARGET" -p "$PROFILE" + +echo "" +echo "Deploying framework bundle..." +databricks bundle deploy -t "$TARGET" -p "$PROFILE" --var="version=$VERSION" + +echo "" +echo "=== Framework deployed successfully ===" +echo "" +echo "Framework source path:" +CURRENT_USER=$(databricks current-user me -p "$PROFILE" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['userName'])" 2>/dev/null || echo '') +echo " /Workspace/Users/$CURRENT_USER/.bundle/lakeflow_framework/$TARGET/$VERSION/files/src" +echo "" +echo "Use this path as framework_source_path in your pipeline bundles." diff --git a/skills/dataflowspec_builder/scripts/deploy_pipeline_bundle.sh b/skills/dataflowspec_builder/scripts/deploy_pipeline_bundle.sh new file mode 100755 index 0000000..a3176b9 --- /dev/null +++ b/skills/dataflowspec_builder/scripts/deploy_pipeline_bundle.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Deploy a Lakeflow Framework pipeline bundle to a Databricks workspace. +# +# Usage: +# ./deploy_pipeline_bundle.sh -b [options] +# +# Options: +# -b, --bundle Path to the pipeline bundle directory (required) +# -t, --target DABs target environment (default: dev) +# -p, --profile Databricks CLI profile (default: DEFAULT) +# -r, --run Also run the pipeline after deployment +# --pipeline-name Pipeline name to run (required with -r, auto-detected if single) +# --destroy Destroy the bundle instead of deploying +# -h, --help Show this help message + +set -euo pipefail + +BUNDLE_PATH="" +TARGET="dev" +PROFILE="DEFAULT" +RUN=false +PIPELINE_NAME="" +DESTROY=false + +usage() { + echo "Usage: $0 -b [-t target] [-p profile] [-r] [--pipeline-name name]" + echo "" + echo " -b, --bundle Pipeline bundle directory (required)" + echo " -t, --target DABs target (default: dev)" + echo " -p, --profile Databricks CLI profile (default: DEFAULT)" + echo " -r, --run Run pipeline after deploy" + echo " --pipeline-name Pipeline to run (auto-detected if single)" + echo " --destroy Destroy bundle instead of deploy" + echo " -h, --help Show help" + exit 0 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -b|--bundle) BUNDLE_PATH="$2"; shift 2 ;; + -t|--target) TARGET="$2"; shift 2 ;; + -p|--profile) PROFILE="$2"; shift 2 ;; + -r|--run) RUN=true; shift ;; + --pipeline-name) PIPELINE_NAME="$2"; shift 2 ;; + --destroy) DESTROY=true; shift ;; + -h|--help) usage ;; + *) echo "Unknown option: $1"; usage ;; + esac +done + +if [ -z "$BUNDLE_PATH" ]; then + echo "Error: --bundle is required" + usage +fi + +if [ ! -d "$BUNDLE_PATH" ]; then + echo "Error: Bundle directory not found: $BUNDLE_PATH" + exit 1 +fi + +if [ ! -f "$BUNDLE_PATH/databricks.yml" ]; then + echo "Error: No databricks.yml found in $BUNDLE_PATH" + exit 1 +fi + +cd "$BUNDLE_PATH" + +echo "=== Lakeflow Pipeline Bundle ===" +echo " Bundle: $(basename "$BUNDLE_PATH")" +echo " Target: $TARGET" +echo " Profile: $PROFILE" +echo "" + +if $DESTROY; then + echo "Destroying bundle..." + databricks bundle destroy -t "$TARGET" -p "$PROFILE" --auto-approve + echo "Bundle destroyed." + exit 0 +fi + +echo "Validating bundle..." +if ! databricks bundle validate -t "$TARGET" -p "$PROFILE"; then + echo "" + echo "Validation failed. Fix the errors above and retry." + exit 1 +fi + +echo "" +echo "Deploying bundle..." +databricks bundle deploy -t "$TARGET" -p "$PROFILE" + +echo "" +echo "=== Bundle deployed successfully ===" + +if $RUN; then + if [ -z "$PIPELINE_NAME" ]; then + RESOURCE_FILES=$(ls resources/*.yml 2>/dev/null | head -1) + if [ -n "$RESOURCE_FILES" ]; then + PIPELINE_NAME=$(grep -m1 'name:' "$RESOURCE_FILES" | head -1 | sed 's/.*name: *//' | tr -d '"' | tr -d "'") + fi + fi + + if [ -n "$PIPELINE_NAME" ]; then + echo "" + echo "Running pipeline: $PIPELINE_NAME" + databricks bundle run -t "$TARGET" -p "$PROFILE" "$PIPELINE_NAME" + else + echo "" + echo "Could not auto-detect pipeline name. Run manually:" + echo " databricks bundle run -t $TARGET " + fi +fi diff --git a/skills/dataflowspec_builder/scripts/scaffold_lakeflow_bundle.py b/skills/dataflowspec_builder/scripts/scaffold_lakeflow_bundle.py new file mode 100644 index 0000000..4ecac52 --- /dev/null +++ b/skills/dataflowspec_builder/scripts/scaffold_lakeflow_bundle.py @@ -0,0 +1,809 @@ +#!/usr/bin/env python3 +""" +Lakeflow Framework Pipeline Bundle Scaffolder + +Generates a complete, deploy-ready Lakeflow Framework pipeline bundle with: +- databricks.yml (DAB bundle definition) +- Pipeline resource YAMLs (SDP pipeline definitions) +- Data Flow Specs (standard, flows, or materialized_view) +- Schema JSON files (StructType format) +- Data quality expectation files +- SQL transform files (for silver/gold) +- Substitution configs (per environment) +- Template definitions (for multiple similar tables) +- Python extension stubs (if requested) + +Usage: + python scaffold_lakeflow_bundle.py \ + --name "energy_bronze" \ + --catalog main \ + --schema energy_workshop \ + --layer bronze \ + --pattern basic_1_1 \ + --tables "raw_customers,raw_meter_readings,raw_billing" \ + --format json \ + --workspace-host "https://my-workspace.cloud.databricks.com" +""" + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Dict, List, Optional + + +PATTERNS = { + "basic_1_1": { + "dataFlowType": "standard", + "layer": "bronze", + "description": "Basic 1:1 streaming ingestion (Bronze)", + }, + "stream_static": { + "dataFlowType": "flow", + "layer": "silver", + "description": "Stream-static join pattern (Silver)", + }, + "multi_source_streaming": { + "dataFlowType": "flow", + "layer": "silver", + "description": "Multi-source streaming merge (Silver)", + }, + "cdc_snapshot": { + "dataFlowType": "standard", + "layer": "bronze", + "description": "CDC from historical snapshots (Bronze)", + }, + "materialized_view": { + "dataFlowType": "materialized_view", + "layer": "gold", + "description": "Materialized views for aggregations (Gold)", + }, +} + + +def create_directory_structure(base_path: Path, tables: List[str], pattern: str, use_template: bool) -> None: + dirs = [ + base_path / "resources", + base_path / "src" / "pipeline_configs", + base_path / "fixtures", + base_path / "scratch", + ] + + if use_template: + dirs.append(base_path / "src" / "templates") + dirs.append(base_path / "src" / "dataflows" / "dataflowspec") + dirs.append(base_path / "src" / "dataflows" / "schemas") + dirs.append(base_path / "src" / "dataflows" / "expectations") + else: + for table in tables: + table_dir = base_path / "src" / "dataflows" / table + dirs.extend([ + table_dir / "dataflowspec", + table_dir / "schemas", + table_dir / "expectations", + table_dir / "dml", + ]) + + pattern_info = PATTERNS.get(pattern, {}) + if pattern_info.get("dataFlowType") == "flow": + dirs.append(base_path / "src" / "extensions") + + for d in dirs: + d.mkdir(parents=True, exist_ok=True) + + +def generate_databricks_yml( + base_path: Path, + bundle_name: str, + catalog: str, + schema: str, + layer: str, + workspace_host: str, + framework_path: str, + environments: List[str], +) -> None: + content = f"""bundle: + name: {bundle_name} + +include: + - resources/*.yml + +variables: + owner: + description: The owner of the bundle + default: ${{workspace.current_user.userName}} + catalog: + description: The target UC catalog + default: {catalog} + schema: + description: The target UC schema + default: {schema} + layer: + description: The target medallion layer + default: {layer} + framework_source_path: + description: Path to the deployed Lakeflow Framework source + default: {framework_path} + +targets: +""" + for i, env in enumerate(environments): + is_default = "true" if i == 0 else "false" + mode_line = " mode: development\n" if env == "dev" else "" + content += f""" {env}: +{mode_line} default: {is_default} + workspace: + host: {workspace_host} + root_path: /Workspace/Users/${{workspace.current_user.userName}}/.bundle/${{bundle.name}}/${{bundle.target}} + variables: + framework_source_path: /Workspace/Users/${{var.owner}}/.bundle/lakeflow_framework/{env}/current/files/src +""" + + (base_path / "databricks.yml").write_text(content) + + +def generate_pipeline_resource( + base_path: Path, + pipeline_name: str, + layer: str, + table_filter: Optional[str] = None, + group_filter: Optional[str] = None, +) -> None: + filters = "" + if table_filter: + filters += f"\n pipeline.targetTableFilter: {table_filter}" + if group_filter: + filters += f"\n pipeline.dataFlowGroupFilter: {group_filter}" + + content = f"""resources: + pipelines: + {pipeline_name}: + name: {pipeline_name} + catalog: ${{var.catalog}} + schema: ${{var.schema}} + channel: CURRENT + serverless: true + libraries: + - notebook: + path: ${{var.framework_source_path}}/dlt_pipeline + + configuration: + bundle.sourcePath: /Workspace/${{workspace.file_path}}/src + framework.sourcePath: /Workspace/${{var.framework_source_path}} + workspace.host: ${{workspace.host}} + bundle.target: ${{bundle.target}} + pipeline.layer: ${{var.layer}}{filters} +""" + + (base_path / "resources" / f"{pipeline_name}.yml").write_text(content) + + +def generate_standard_dataflow_spec( + table: str, + group: str, + source_db: str, + source_table: str, + target_table: str, + mode: str = "stream", + source_type: str = "delta", + cdc_enabled: bool = True, + scd_type: str = "1", + primary_keys: Optional[List[str]] = None, + sequence_by: str = "updated_at", + dq_enabled: bool = True, + quarantine_mode: str = "off", +) -> Dict: + pk = primary_keys or [f"{table}_id"] + + spec: Dict = { + "dataFlowId": f"{table}_ingestion", + "dataFlowGroup": group, + "dataFlowType": "standard", + "sourceType": source_type, + "sourceSystem": "source", + "sourceViewName": f"v_{source_table}", + "sourceDetails": { + "database": source_db, + "table": source_table, + "cdfEnabled": cdc_enabled, + "schemaPath": f"schemas/{source_table}_schema.json", + }, + "mode": mode, + "targetFormat": "delta", + "targetDetails": { + "table": target_table, + "tableProperties": { + "delta.autoOptimize.optimizeWrite": "true", + "delta.autoOptimize.autoCompact": "true", + }, + }, + } + + if cdc_enabled: + spec["cdcSettings"] = { + "keys": pk, + "sequence_by": sequence_by, + "scd_type": scd_type, + "ignore_null_updates": True, + "except_column_list": [], + } + + spec["dataQualityExpectationsEnabled"] = dq_enabled + if dq_enabled: + spec["dataQualityExpectationsPath"] = f"{target_table}_dqe.json" + + spec["quarantineMode"] = quarantine_mode + if quarantine_mode == "table": + spec["quarantineTargetDetails"] = { + "targetFormat": "delta", + "table": f"{target_table}_quarantine", + "tableProperties": {}, + } + else: + spec["quarantineTargetDetails"] = {} + + return spec + + +def generate_flows_dataflow_spec( + flow_id: str, + group: str, + target_table: str, + sources: List[Dict], + primary_keys: List[str], + sequence_by: str = "updated_at", + scd_type: str = "1", + dq_enabled: bool = False, +) -> Dict: + staging_table = f"staging_{target_table}_apnd" + + flows = {} + for src in sources: + view_name = f"v_{src['table']}" + flows[f"f_{src['table']}"] = { + "flowType": "append_view", + "flowDetails": { + "targetTable": staging_table, + "sourceView": view_name, + }, + "views": { + view_name: { + "mode": "stream", + "sourceType": "delta", + "sourceDetails": { + "database": src["database"], + "table": src["table"], + "cdfEnabled": src.get("cdfEnabled", True), + "selectExp": src.get("selectExp", ["*"]), + "whereClause": src.get("whereClause", []), + }, + }, + }, + } + + flows["f_merge"] = { + "flowType": "merge", + "flowDetails": { + "targetTable": target_table, + "sourceView": staging_table, + }, + } + + spec: Dict = { + "dataFlowId": flow_id, + "dataFlowGroup": group, + "dataFlowType": "flow", + "targetFormat": "delta", + "targetDetails": { + "table": target_table, + "tableProperties": { + "delta.enableChangeDataFeed": "true", + }, + }, + "cdcSettings": { + "keys": primary_keys, + "sequence_by": sequence_by, + "scd_type": scd_type, + "ignore_null_updates": True, + "except_column_list": [], + }, + "dataQualityExpectationsEnabled": dq_enabled, + "quarantineMode": "off", + "quarantineTargetDetails": {}, + "flowGroups": [ + { + "flowGroupId": f"fg_{target_table}", + "stagingTables": { + staging_table: { + "type": "ST", + "schemaPath": "", + } + }, + "flows": flows, + } + ], + } + + return spec + + +def generate_materialized_view_spec( + mv_id: str, + group: str, + views: List[Dict], +) -> Dict: + mv_dict = {} + for v in views: + mv_entry: Dict = {} + if v.get("sqlStatement"): + mv_entry["sqlStatement"] = v["sqlStatement"] + elif v.get("sqlPath"): + mv_entry["sqlPath"] = v["sqlPath"] + else: + mv_entry["sourceView"] = { + "sourceViewName": f"v_{v['name']}", + "sourceType": v.get("sourceType", "delta"), + "sourceDetails": { + "database": v["database"], + "table": v["sourceTable"], + }, + } + + if v.get("clusterByColumns"): + mv_entry.setdefault("tableDetails", {})["clusterByColumns"] = v["clusterByColumns"] + if v.get("database"): + mv_entry.setdefault("tableDetails", {})["database"] = v.get("targetDatabase", v["database"]) + + mv_entry["dataQualityExpectationsEnabled"] = v.get("dq_enabled", False) + mv_entry["quarantineMode"] = "off" + mv_entry["quarantineTargetDetails"] = {} + + mv_dict[v["name"]] = mv_entry + + return { + "dataFlowId": mv_id, + "dataFlowGroup": group, + "dataFlowType": "materialized_view", + "materializedViews": mv_dict, + } + + +def generate_schema_file(table_name: str, columns: List[Dict]) -> Dict: + fields = [] + for col in columns: + field = { + "name": col["name"], + "type": col.get("type", "string"), + "nullable": col.get("nullable", True), + "metadata": col.get("metadata", {}), + } + fields.append(field) + + return {"type": "struct", "fields": fields} + + +def generate_expectations_file( + table_name: str, + expectations: Optional[Dict] = None, +) -> Dict: + if expectations: + return expectations + + return { + "expect": [ + { + "name": f"{table_name}_not_null_id", + "constraint": f"{table_name}_id IS NOT NULL", + "tag": "completeness", + "enabled": True, + } + ], + "expect_or_drop": [], + "expect_or_fail": [], + } + + +def generate_substitutions( + base_path: Path, + env: str, + catalog: str, + schema: str, + layer: str, +) -> None: + schema_token = f"{catalog}.{layer}_{schema}" + if env != "dev": + schema_token = f"{catalog}.{layer}_{schema}_{env}" + + content = { + "tokens": { + f"{layer}_schema": schema_token, + "landing_path": f"/Volumes/{catalog}/{schema}/landing", + }, + "prefix_suffix": {}, + } + + filepath = base_path / "src" / "pipeline_configs" / f"{env}_substitutions.json" + filepath.write_text(json.dumps(content, indent=4) + "\n") + + +def generate_template( + base_path: Path, + template_name: str, + pattern: str, + source_type: str = "delta", +) -> Dict: + if pattern in ("basic_1_1", "cdc_snapshot"): + template = { + "name": template_name, + "parameters": { + "dataFlowId": {"type": "string", "required": True}, + "sourceTable": {"type": "string", "required": True}, + "targetTable": {"type": "string", "required": True}, + "schemaPath": {"type": "string", "required": False, "default": ""}, + "keyColumns": {"type": "list", "required": True}, + "sequenceBy": {"type": "string", "required": True}, + "scdType": {"type": "string", "required": False, "default": "1"}, + }, + "template": { + "dataFlowId": "${param.dataFlowId}", + "dataFlowGroup": template_name.replace("_template", ""), + "dataFlowType": "standard", + "sourceType": source_type, + "sourceSystem": "source", + "sourceViewName": "v_${param.sourceTable}", + "sourceDetails": { + "database": f"{{{template_name.replace('_template', '')}_schema}}", + "table": "${param.sourceTable}", + "cdfEnabled": True, + "schemaPath": "${param.schemaPath}", + }, + "mode": "stream", + "targetFormat": "delta", + "targetDetails": { + "table": "${param.targetTable}", + "tableProperties": { + "delta.autoOptimize.optimizeWrite": "true", + "delta.autoOptimize.autoCompact": "true", + }, + }, + "cdcSettings": { + "keys": "${param.keyColumns}", + "sequence_by": "${param.sequenceBy}", + "scd_type": "${param.scdType}", + "ignore_null_updates": True, + "except_column_list": [], + }, + "dataQualityExpectationsEnabled": True, + "quarantineMode": "off", + "quarantineTargetDetails": {}, + }, + } + else: + template = { + "name": template_name, + "parameters": { + "dataFlowId": {"type": "string", "required": True}, + "mvName": {"type": "string", "required": True}, + "sqlStatement": {"type": "string", "required": True}, + "clusterByColumns": {"type": "list", "required": False, "default": []}, + }, + "template": { + "dataFlowId": "${param.dataFlowId}", + "dataFlowGroup": template_name.replace("_template", ""), + "dataFlowType": "materialized_view", + "materializedViews": { + "${param.mvName}": { + "sqlStatement": "${param.sqlStatement}", + "tableDetails": { + "clusterByColumns": "${param.clusterByColumns}", + }, + "dataQualityExpectationsEnabled": False, + "quarantineMode": "off", + "quarantineTargetDetails": {}, + } + }, + }, + } + + filepath = base_path / "src" / "templates" / f"{template_name}.json" + filepath.parent.mkdir(parents=True, exist_ok=True) + filepath.write_text(json.dumps(template, indent=4) + "\n") + return template + + +def generate_template_usage( + base_path: Path, + template_name: str, + parameter_sets: List[Dict], +) -> None: + content = { + "template": template_name, + "parameterSets": parameter_sets, + } + filepath = base_path / "src" / "dataflows" / "dataflowspec" / f"{template_name}_main.json" + filepath.parent.mkdir(parents=True, exist_ok=True) + filepath.write_text(json.dumps(content, indent=4) + "\n") + + +def generate_extension_stubs(base_path: Path) -> None: + ext_dir = base_path / "src" / "extensions" + ext_dir.mkdir(parents=True, exist_ok=True) + + (ext_dir / "__init__.py").write_text("") + + (ext_dir / "sources.py").write_text('''from pyspark.sql import DataFrame, SparkSession +from typing import Dict + + +def get_custom_source(spark: SparkSession, tokens: Dict) -> DataFrame: + """Custom source function — modify for your use case.""" + source_table = tokens["sourceTable"] + return spark.readStream.option("readChangeFeed", "true").table(source_table) +''') + + (ext_dir / "transforms.py").write_text('''from pyspark.sql import DataFrame +from pyspark.sql import functions as F +from typing import Dict + + +def clean_and_deduplicate(df: DataFrame) -> DataFrame: + """Remove duplicates and add processing metadata.""" + return ( + df.dropDuplicates() + .withColumn("_processed_at", F.current_timestamp()) + ) + + +def clean_with_tokens(df: DataFrame, tokens: Dict) -> DataFrame: + """Transform with configurable parameters from the data flow spec.""" + id_column = tokens.get("idColumn", "id") + return df.dropDuplicates([id_column]) +''') + + (ext_dir / "sinks.py").write_text('''from pyspark.sql import DataFrame +from typing import Dict + + +def write_to_external(df: DataFrame, batch_id: int, tokens: Dict) -> None: + """Foreach batch sink — modify for your external target.""" + target = tokens.get("target", "console") + if target == "console": + df.show(truncate=False) + else: + records = df.toJSON().collect() + for record in records: + print(f"Batch {batch_id}: {record}") +''') + + +def write_json(path: Path, data: Dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=4) + "\n") + + +def scaffold_bundle(args: argparse.Namespace) -> None: + base_path = Path(args.output_dir) / args.name + tables = [t.strip() for t in args.tables.split(",")] if args.tables else [] + environments = [e.strip() for e in args.environments.split(",")] if args.environments else ["dev"] + pattern = args.pattern or "basic_1_1" + layer = args.layer or PATTERNS.get(pattern, {}).get("layer", "bronze") + use_template = args.use_template and len(tables) > 1 and pattern in ("basic_1_1", "cdc_snapshot", "materialized_view") + + print(f"Scaffolding Lakeflow Framework pipeline bundle: {args.name}") + print(f" Pattern: {pattern} ({PATTERNS.get(pattern, {}).get('description', 'custom')})") + print(f" Layer: {layer}") + print(f" Tables: {tables}") + print(f" Format: {args.format}") + print(f" Template mode: {use_template}") + print(f" Environments: {environments}") + print() + + create_directory_structure(base_path, tables, pattern, use_template) + + fw_path = args.framework_path or f"/Workspace/Users/${{var.owner}}/.bundle/lakeflow_framework/dev/current/files/src" + generate_databricks_yml( + base_path, args.name, args.catalog, args.schema, layer, + args.workspace_host or "https://WORKSPACE.cloud.databricks.com", + fw_path, environments, + ) + + generate_pipeline_resource( + base_path, + pipeline_name=f"{args.name}_pipeline", + layer=layer, + group_filter=args.name if len(tables) > 3 else None, + ) + + for env in environments: + generate_substitutions(base_path, env, args.catalog, args.schema, layer) + + if use_template: + template_name = f"{args.name}_template" + generate_template(base_path, template_name, pattern) + + param_sets = [] + for table in tables: + target = table.replace("raw_", f"{layer}_") if table.startswith("raw_") else f"{layer}_{table}" + param_sets.append({ + "dataFlowId": f"{table}_ingestion", + "sourceTable": table, + "targetTable": target, + "schemaPath": f"schemas/{table}_schema.json", + "keyColumns": [f"{table.replace('raw_', '')}_id"], + "sequenceBy": "updated_at", + }) + generate_template_usage(base_path, template_name, param_sets) + + for table in tables: + default_schema = generate_schema_file(table, [ + {"name": f"{table.replace('raw_', '')}_id", "type": "integer", "nullable": False}, + {"name": "created_at", "type": "timestamp"}, + {"name": "updated_at", "type": "timestamp"}, + ]) + write_json(base_path / "src" / "dataflows" / "schemas" / f"{table}_schema.json", default_schema) + + target = table.replace("raw_", f"{layer}_") if table.startswith("raw_") else f"{layer}_{table}" + dq = generate_expectations_file(table.replace("raw_", "")) + write_json(base_path / "src" / "dataflows" / "expectations" / f"{target}_dqe.json", dq) + + else: + for table in tables: + target = table.replace("raw_", f"{layer}_") if table.startswith("raw_") else f"{layer}_{table}" + source_db = f"{{{args.name}_schema}}" + + if pattern in ("basic_1_1", "cdc_snapshot"): + spec = generate_standard_dataflow_spec( + table=table.replace("raw_", ""), + group=args.name, + source_db=source_db, + source_table=table, + target_table=target, + source_type="delta", + cdc_enabled=True, + scd_type="1", + dq_enabled=True, + quarantine_mode="off", + ) + write_json( + base_path / "src" / "dataflows" / table / "dataflowspec" / f"{table}_main.json", + spec, + ) + + default_schema = generate_schema_file(table, [ + {"name": f"{table.replace('raw_', '')}_id", "type": "integer", "nullable": False}, + {"name": "created_at", "type": "timestamp"}, + {"name": "updated_at", "type": "timestamp"}, + ]) + write_json( + base_path / "src" / "dataflows" / table / "schemas" / f"{table}_schema.json", + default_schema, + ) + + dq = generate_expectations_file(table.replace("raw_", "")) + write_json( + base_path / "src" / "dataflows" / table / "expectations" / f"{target}_dqe.json", + dq, + ) + + if PATTERNS.get(pattern, {}).get("dataFlowType") == "flow" or args.extensions: + generate_extension_stubs(base_path) + + readme = f"""# {args.name} + +Lakeflow Framework Pipeline Bundle — generated by `scaffold_lakeflow_bundle.py`. + +## Pattern +**{PATTERNS.get(pattern, {}).get('description', pattern)}** + +## Tables +{chr(10).join(f'- `{t}`' for t in tables)} + +## Deploy + +```bash +# Validate +databricks bundle validate -t dev + +# Deploy +databricks bundle deploy -t dev + +# Run the pipeline +databricks bundle run -t dev {args.name}_pipeline +``` + +## Structure +``` +{args.name}/ +├── databricks.yml +├── resources/ +│ └── {args.name}_pipeline.yml +└── src/ + ├── dataflows/ + │ └── ... + ├── pipeline_configs/ + │ └── {environments[0]}_substitutions.json + └── {'templates/' if use_template else 'extensions/'} +``` +""" + (base_path / "README.md").write_text(readme) + + print(f"Bundle scaffolded at: {base_path}") + print(f"Next steps:") + print(f" 1. Review and customize the generated Data Flow Specs") + print(f" 2. Update schema files with actual column definitions") + print(f" 3. Add data quality expectations") + print(f" 4. Run: cd {base_path} && databricks bundle validate -t dev") + print(f" 5. Deploy: databricks bundle deploy -t dev") + + +def main(): + parser = argparse.ArgumentParser( + description="Scaffold a Lakeflow Framework Pipeline Bundle", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Bronze ingestion for 3 tables + %(prog)s --name energy_bronze --catalog main --schema energy --layer bronze \\ + --pattern basic_1_1 --tables "raw_customers,raw_billing,raw_outages" + + # Silver multi-source streaming + %(prog)s --name energy_silver --catalog main --schema energy --layer silver \\ + --pattern multi_source_streaming --tables "silver_customer_360" + + # Gold materialized views with template + %(prog)s --name energy_gold --catalog main --schema energy --layer gold \\ + --pattern materialized_view --tables "kpi_revenue,kpi_reliability,kpi_demand" \\ + --use-template + + # Full multi-env setup + %(prog)s --name energy_bronze --catalog main --schema energy --layer bronze \\ + --pattern basic_1_1 --tables "raw_customers,raw_meter_readings" \\ + --environments "dev,staging,prod" --use-template +""", + ) + + parser.add_argument("--name", required=True, help="Bundle name") + parser.add_argument("--catalog", required=True, help="Unity Catalog name") + parser.add_argument("--schema", required=True, help="Target schema") + parser.add_argument("--layer", choices=["bronze", "silver", "gold"], help="Medallion layer") + parser.add_argument( + "--pattern", + choices=list(PATTERNS.keys()), + default="basic_1_1", + help="Pipeline pattern", + ) + parser.add_argument("--tables", required=True, help="Comma-separated list of table names") + parser.add_argument("--format", choices=["json", "yaml"], default="json", help="Spec format") + parser.add_argument("--workspace-host", help="Databricks workspace URL") + parser.add_argument( + "--framework-path", + help="Path to deployed framework source (default: auto-computed from owner)", + ) + parser.add_argument( + "--environments", + default="dev", + help="Comma-separated environments (default: dev)", + ) + parser.add_argument( + "--use-template", + action="store_true", + help="Generate a reusable template instead of individual specs", + ) + parser.add_argument( + "--extensions", + action="store_true", + help="Include Python extension stubs", + ) + parser.add_argument( + "--output-dir", + default=".", + help="Output directory (default: current directory)", + ) + + args = parser.parse_args() + scaffold_bundle(args) + + +if __name__ == "__main__": + main() diff --git a/skills/dataflowspec_builder/scripts/validate_specs.py b/skills/dataflowspec_builder/scripts/validate_specs.py new file mode 100644 index 0000000..6b06ae7 --- /dev/null +++ b/skills/dataflowspec_builder/scripts/validate_specs.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +""" +Validate Lakeflow Framework Data Flow Spec files for common issues. + +Checks: + - Valid JSON/YAML syntax + - Required fields present per data flow type + - Source view naming convention (v_prefix) + - CDC settings completeness + - Expectation file references resolve + - Schema file references resolve + - Template parameter completeness + +Usage: + python validate_specs.py + python validate_specs.py +""" + +import json +import sys +from pathlib import Path +from typing import Dict, List, Tuple + +REQUIRED_METADATA = ["dataFlowId", "dataFlowGroup", "dataFlowType"] + +REQUIRED_STANDARD = [ + "sourceType", "sourceViewName", "sourceDetails", + "mode", "targetFormat", "targetDetails", +] + +REQUIRED_FLOW = ["targetFormat", "targetDetails", "flowGroups"] + +REQUIRED_MV = ["materializedViews"] + +VALID_SOURCE_TYPES = {"cloudFiles", "delta", "deltaJoin", "kafka", "python", "sql", "batchFiles"} +VALID_MODES = {"stream", "batch"} +VALID_FLOW_TYPES = {"append_view", "append_sql", "merge"} +VALID_QUARANTINE_MODES = {"off", "flag", "table"} +VALID_SCD_TYPES = {"1", "2"} +VALID_STAGING_TYPES = {"ST", "MV"} + + +def validate_spec(spec: Dict, filepath: str) -> List[str]: + errors = [] + + if "template" in spec and "parameterSets" in spec: + return _validate_template_usage(spec) + if "name" in spec and "parameters" in spec and "template" in spec: + return _validate_template_definition(spec) + + flow_type = spec.get("dataFlowType", "") + + for field in REQUIRED_METADATA: + if field not in spec: + errors.append(f"Missing required field: {field}") + + if not spec.get("dataFlowId", ""): + errors.append("dataFlowId must not be empty") + + if flow_type == "standard": + errors.extend(_validate_standard(spec)) + elif flow_type == "flow": + errors.extend(_validate_flows(spec)) + elif flow_type == "materialized_view": + errors.extend(_validate_materialized_view(spec)) + else: + if flow_type: + errors.append(f"Unknown dataFlowType: {flow_type}") + + return errors + + +def _validate_standard(spec: Dict) -> List[str]: + errors = [] + for field in REQUIRED_STANDARD: + if field not in spec: + errors.append(f"Standard spec missing: {field}") + + src_type = spec.get("sourceType", "") + if src_type and src_type not in VALID_SOURCE_TYPES: + errors.append(f"Invalid sourceType: {src_type}. Valid: {VALID_SOURCE_TYPES}") + + view_name = spec.get("sourceViewName", "") + if view_name and not view_name.startswith("v_") and not view_name.startswith("${"): + errors.append(f"sourceViewName '{view_name}' must start with 'v_'") + + mode = spec.get("mode", "") + if mode and mode not in VALID_MODES: + errors.append(f"Invalid mode: {mode}. Valid: {VALID_MODES}") + + errors.extend(_validate_cdc(spec.get("cdcSettings"))) + errors.extend(_validate_quarantine(spec)) + + return errors + + +def _validate_flows(spec: Dict) -> List[str]: + errors = [] + for field in REQUIRED_FLOW: + if field not in spec: + errors.append(f"Flow spec missing: {field}") + + flow_groups = spec.get("flowGroups", []) + if not flow_groups: + errors.append("flowGroups must have at least one flow group") + + for i, fg in enumerate(flow_groups): + fg_id = fg.get("flowGroupId", f"index_{i}") + if "flowGroupId" not in fg: + errors.append(f"Flow group {i} missing flowGroupId") + + staging = fg.get("stagingTables", {}) + for st_name, st_config in staging.items(): + st_type = st_config.get("type", "") + if st_type and st_type not in VALID_STAGING_TYPES: + errors.append(f"Staging table '{st_name}' invalid type: {st_type}") + + flows = fg.get("flows", {}) + if not flows: + errors.append(f"Flow group '{fg_id}' must have at least one flow") + + for flow_name, flow_config in flows.items(): + ft = flow_config.get("flowType", "") + if ft and ft not in VALID_FLOW_TYPES: + errors.append(f"Flow '{flow_name}' invalid flowType: {ft}") + + fd = flow_config.get("flowDetails", {}) + if not fd.get("targetTable"): + errors.append(f"Flow '{flow_name}' missing flowDetails.targetTable") + + if ft in ("append_view", "merge") and not fd.get("sourceView"): + if "views" not in flow_config and ft != "merge": + errors.append(f"Flow '{flow_name}' ({ft}) missing sourceView or views") + + errors.extend(_validate_cdc(spec.get("cdcSettings"))) + errors.extend(_validate_quarantine(spec)) + + return errors + + +def _validate_materialized_view(spec: Dict) -> List[str]: + errors = [] + mvs = spec.get("materializedViews", {}) + if not mvs: + errors.append("materializedViews must have at least one entry") + + for mv_name, mv_config in mvs.items(): + has_source = bool(mv_config.get("sourceView")) + has_sql_path = bool(mv_config.get("sqlPath")) + has_sql_stmt = bool(mv_config.get("sqlStatement")) + source_count = sum([has_source, has_sql_path, has_sql_stmt]) + + if source_count == 0: + errors.append(f"MV '{mv_name}' must have sourceView, sqlPath, or sqlStatement") + if source_count > 1: + errors.append(f"MV '{mv_name}' has multiple source definitions (pick one)") + + return errors + + +def _validate_cdc(cdc: Dict = None) -> List[str]: + errors = [] + if not cdc: + return errors + + if not cdc.get("keys"): + errors.append("cdcSettings.keys must not be empty") + if not cdc.get("sequence_by"): + errors.append("cdcSettings.sequence_by is required") + + scd = str(cdc.get("scd_type", "")) + if scd and scd not in VALID_SCD_TYPES: + errors.append(f"cdcSettings.scd_type must be '1' or '2', got: {scd}") + + return errors + + +def _validate_quarantine(spec: Dict) -> List[str]: + errors = [] + mode = spec.get("quarantineMode", "off") + if mode not in VALID_QUARANTINE_MODES: + errors.append(f"Invalid quarantineMode: {mode}") + if mode == "table": + qtd = spec.get("quarantineTargetDetails", {}) + if not qtd.get("table") and not qtd.get("targetFormat"): + errors.append("quarantineMode 'table' requires quarantineTargetDetails with table name") + return errors + + +def _validate_template_usage(spec: Dict) -> List[str]: + errors = [] + if not spec.get("template"): + errors.append("Template usage must have 'template' field") + ps = spec.get("parameterSets", []) + if not ps: + errors.append("parameterSets must have at least one entry") + + ids = [p.get("dataFlowId", "") for p in ps] + dupes = [i for i in ids if ids.count(i) > 1] + if dupes: + errors.append(f"Duplicate dataFlowId in parameterSets: {set(dupes)}") + + return errors + + +def _validate_template_definition(spec: Dict) -> List[str]: + errors = [] + params = spec.get("parameters", {}) + template = spec.get("template", {}) + + if not params: + errors.append("Template definition must have parameters") + if not template: + errors.append("Template definition must have template body") + + tmpl_str = json.dumps(template) + for param_name, param_def in params.items(): + placeholder = f"${{param.{param_name}}}" + if placeholder not in tmpl_str: + if param_def.get("required", True) and "default" not in param_def: + errors.append(f"Required parameter '{param_name}' not referenced in template") + + return errors + + +def validate_file(filepath: Path) -> Tuple[str, List[str]]: + try: + content = filepath.read_text() + spec = json.loads(content) + except json.JSONDecodeError as e: + return str(filepath), [f"Invalid JSON: {e}"] + except Exception as e: + return str(filepath), [f"Error reading file: {e}"] + + errors = validate_spec(spec, str(filepath)) + return str(filepath), errors + + +def main(): + if len(sys.argv) < 2: + print("Usage: python validate_specs.py ") + sys.exit(1) + + target = Path(sys.argv[1]) + + if target.is_file(): + files = [target] + elif target.is_dir(): + files = sorted(target.rglob("*.json")) + files = [f for f in files if "dataflowspec" in str(f) or "templates" in str(f)] + else: + print(f"Not found: {target}") + sys.exit(1) + + total_errors = 0 + total_files = 0 + + for f in files: + filepath, errors = validate_file(f) + total_files += 1 + if errors: + total_errors += len(errors) + print(f"\n❌ {filepath}") + for e in errors: + print(f" • {e}") + else: + print(f"✅ {filepath}") + + print(f"\n{'='*60}") + print(f"Files checked: {total_files}") + print(f"Errors found: {total_errors}") + + if total_errors > 0: + sys.exit(1) + + +if __name__ == "__main__": + main()