From 1b56465772791931c177d22a8c637f97f9e86ffd Mon Sep 17 00:00:00 2001 From: Haille W Date: Sun, 21 Jun 2026 19:20:06 +1000 Subject: [PATCH 01/19] feat(nodester): unified node-based dataflow spec Introduce the nodester dataflow spec: a single, node-graph format (source -> transformation -> target) that replaces the separate standard, flow, and materialized-view formats. The transformer lowers a node graph into the framework's existing flow spec, so all current capabilities (CDC, snapshots, data quality, quarantine, sinks, table migration, materialized views) are preserved. Highlights: - Target nodes wire inputs via an `input` list; each item is a node name (auto flow name) or `{view, flow}` to define the SDP flow name explicitly and keep it stable across edits (renaming a flow forces a full refresh). - A single spec may contain both streaming-table and materialized-view targets, including chains where one feeds the other. - Python transformation nodes become their own view that applies apply_transform to their inferred upstream; inline python_transform on a source is still supported for backward compatibility. - Inline SQL/Python sources (and append_sql) remain supported but warn, in both the nodester and legacy formats; the recommended alternative is a dedicated transformation node. - Materialized view targets no longer accept an inline source_view (breaking); chain a source node via `input` instead. - All 38 nodester samples updated and verified end to end on Databricks. - Adds ADR-0007 and a rewritten nodester spec reference. Co-authored-by: Isaac --- .claude/commands/add_feature_sample.md | 153 ++++ .claude/commands/deploy-nodester-samples.md | 138 ++++ .github/workflows/feature-test.yml | 79 ++ .../0007-unified-nodester-dataflow-spec.md | 342 ++++++++ docs/nodester_feature_samples_changes.md | 235 ++++++ .../dataflow_spec_ref_main_nodester.rst | 434 ++++++++++ docs/source/dataflow_spec_reference.rst | 3 +- samples/deploy.sh | 3 + samples/deploy_and_test.sh | 25 + samples/deploy_nodester.sh | 52 ++ samples/destroy.sh | 10 + samples/nodester_sample/.gitignore | 8 + samples/nodester_sample/.vscode/settings.json | 5 + samples/nodester_sample/README.md | 208 +++++ samples/nodester_sample/databricks.yml | 38 + samples/nodester_sample/pytest.ini | 4 + .../nodester_base_samples_pipeline.yml | 24 + .../nodester_customer_cloudfiles_pipeline.yml | 23 + ...odester_customer_multi_target_pipeline.yml | 23 + .../nodester_customer_simple_pipeline.yml | 23 + .../nodester_customer_transform_pipeline.yml | 23 + .../jobs/nodester_samples_run_job.yml | 63 ++ .../nodester_base_samples_pipeline.yml | 23 + .../nodester_customer_cloudfiles_pipeline.yml | 22 + ...odester_customer_multi_target_pipeline.yml | 22 + .../nodester_customer_simple_pipeline.yml | 22 + .../nodester_customer_transform_pipeline.yml | 22 + ..._feature_samples_data_quality_pipeline.yml | 20 + ...ester_feature_samples_general_pipeline.yml | 20 + ...re_samples_materialized_views_pipeline.yml | 20 + ...dester_feature_samples_python_pipeline.yml | 20 + ...ter_feature_samples_snapshots_pipeline.yml | 20 + ...ature_samples_table_migration_pipeline.yml | 20 + .../customer_cloudfiles_main.json | 48 ++ .../customer_multi_target_main.json | 91 +++ .../dataflowspec/customer_simple_main.json | 35 + .../customer_st_with_mv_main.json | 44 ++ .../customer_with_transform_main.json | 143 ++++ .../expectations/customer_final_dqe.json | 21 + .../expectations/customer_staging_dqe.json | 14 + .../schemas/customer_enriched_schema.json | 35 + .../base_samples/schemas/customer_schema.json | 41 + .../schemas/customer_source_schema.json | 29 + .../dataflowspec/append_sql_flow_main.json | 35 + .../dataflowspec/append_view_flow_main.json | 34 + .../append_view_once_flow_main.json | 34 + .../dataflowspec/ddl_schema_main.json | 35 + ...storical_snapshot_files_datetime_main.json | 41 + ...napshot_files_datetime_multifile_main.json | 41 + ...tetime_recursive_and_partitioned_main.json | 42 + ...ecursive_and_partitioned_parquet_main.json | 40 + ..._recursive_and_partitioned_regex_main.json | 42 + .../historical_snapshot_files_flow_main.json | 85 ++ .../historical_snapshot_files_int_main.json | 40 + ...snapshot_files_schema_and_select_main.json | 48 ++ ...storical_snapshot_table_datetime_main.json | 37 + ...snapshot_table_select_expression_main.json | 43 + .../dataflowspec/materialized_views_main.json | 98 +++ .../periodic_snapshot_scd1_main.json | 52 ++ .../periodic_snapshot_scd2_main.json | 52 ++ .../python_source_extension_main.json | 36 + .../dataflowspec/python_source_main.json | 36 + .../python_transform_extension_main.json | 43 + .../dataflowspec/python_transform_main.json | 43 + .../dataflowspec/quarantine_flag_main.json | 44 ++ .../dataflowspec/quarantine_table_main.json | 47 ++ .../quarantine_table_with_cdc_main.json | 54 ++ .../sink_custom_python_generate_cdc_feed.json | 35 + .../sink_delta_path_table_main.json | 35 + .../sink_delta_uc_table_main.json | 35 + ...k_foreach_batch_single_basic_sql_main.json | 41 + ...ach_batch_single_python_function_main.json | 41 + .../table_migration_append_only_main.json | 50 ++ .../table_migration_scd2_main.json | 68 ++ ...rsion_mapping_flows_dataflowspec_main.json | 80 ++ ...on_mapping_standard_dataflowspec_main.json | 44 ++ .../dml/customer_purchase_transformed.sql | 9 + .../dml/feature_mv_sql_path.sql | 1 + .../expectations/customer_address_dqe.json | 15 + .../feature_foreach_batch_python.py | 46 ++ .../feature_python_function_source.py | 15 + .../feature_python_function_transform.py | 12 + ...ure_historic_snapshot_customer_schema.json | 35 + .../target/customer_address_schema.json | 29 + .../schemas/target/customer_schema.json | 41 + .../schemas/target/feature_ddl_schema.ddl | 8 + ...ure_historic_snapshot_customer_schema.json | 35 + ...ure_periodic_snapshot_customer_schema.json | 41 + ...feature_python_function_source_schema.json | 47 ++ ...ture_python_function_transform_schema.json | 17 + .../feature_version_mapping_flows_schema.json | 41 + .../nodester_sample/src/extensions/sources.py | 31 + .../src/extensions/transforms.py | 57 ++ .../pipeline_configs/dev_substitutions.json | 13 + .../src/pipeline_configs/global.json | 6 + samples/nodester_sample/tests/__init__.py | 1 + .../nodester_sample/tests/test_placeholder.py | 11 + .../src/notebooks/initialize.ipynb | 1 + .../src/notebooks/validate_run_1.ipynb | 606 ++++++++++++++ .../src/notebooks/validate_run_2.ipynb | 411 ++++++++++ .../src/notebooks/validate_run_3.ipynb | 312 ++++++++ .../src/notebooks/validate_run_4.ipynb | 340 ++++++++ .../src/notebooks/validation_utils.py | 306 +++++++ scripts/migrate_to_nodester.py | 539 +++++++++++++ src/dataflow/dataflow.py | 188 ++++- src/dataflow/targets/staging_table.py | 34 +- .../dataflow_spec_builder.py | 57 +- src/dataflow_spec_builder/spec_mapper.py | 3 +- .../transformer/__init__.py | 2 + src/dataflow_spec_builder/transformer/base.py | 58 +- .../transformer/factory.py | 6 +- .../transformer/nodester.py | 748 ++++++++++++++++++ src/schemas/flow_group.json | 4 + src/schemas/main.json | 14 +- src/schemas/spec_nodester.json | 504 ++++++++++++ 115 files changed, 8784 insertions(+), 39 deletions(-) create mode 100644 .claude/commands/add_feature_sample.md create mode 100644 .claude/commands/deploy-nodester-samples.md create mode 100644 .github/workflows/feature-test.yml create mode 100644 docs/decisions/0007-unified-nodester-dataflow-spec.md create mode 100644 docs/nodester_feature_samples_changes.md create mode 100644 docs/source/dataflow_spec_ref_main_nodester.rst create mode 100755 samples/deploy_nodester.sh create mode 100644 samples/nodester_sample/.gitignore create mode 100644 samples/nodester_sample/.vscode/settings.json create mode 100644 samples/nodester_sample/README.md create mode 100644 samples/nodester_sample/databricks.yml create mode 100644 samples/nodester_sample/pytest.ini create mode 100644 samples/nodester_sample/resources/classic/nodester_base_samples_pipeline.yml create mode 100644 samples/nodester_sample/resources/classic/nodester_customer_cloudfiles_pipeline.yml create mode 100644 samples/nodester_sample/resources/classic/nodester_customer_multi_target_pipeline.yml create mode 100644 samples/nodester_sample/resources/classic/nodester_customer_simple_pipeline.yml create mode 100644 samples/nodester_sample/resources/classic/nodester_customer_transform_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/jobs/nodester_samples_run_job.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_base_samples_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_customer_cloudfiles_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_customer_multi_target_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_customer_simple_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_customer_transform_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_feature_samples_data_quality_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_feature_samples_general_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_feature_samples_materialized_views_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_feature_samples_python_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_feature_samples_snapshots_pipeline.yml create mode 100644 samples/nodester_sample/resources/serverless/nodester_feature_samples_table_migration_pipeline.yml create mode 100644 samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json create mode 100644 samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json create mode 100644 samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json create mode 100644 samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json create mode 100644 samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json create mode 100644 samples/nodester_sample/src/dataflows/base_samples/expectations/customer_final_dqe.json create mode 100644 samples/nodester_sample/src/dataflows/base_samples/expectations/customer_staging_dqe.json create mode 100644 samples/nodester_sample/src/dataflows/base_samples/schemas/customer_enriched_schema.json create mode 100644 samples/nodester_sample/src/dataflows/base_samples/schemas/customer_schema.json create mode 100644 samples/nodester_sample/src/dataflows/base_samples/schemas/customer_source_schema.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dml/customer_purchase_transformed.sql create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/dml/feature_mv_sql_path.sql create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/expectations/customer_address_dqe.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_foreach_batch_python.py create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_source.py create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_transform.py create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/schemas/source/feature_historic_snapshot_customer_schema.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_address_schema.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_schema.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_ddl_schema.ddl create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_historic_snapshot_customer_schema.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_periodic_snapshot_customer_schema.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_source_schema.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_transform_schema.json create mode 100644 samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_version_mapping_flows_schema.json create mode 100644 samples/nodester_sample/src/extensions/sources.py create mode 100644 samples/nodester_sample/src/extensions/transforms.py create mode 100644 samples/nodester_sample/src/pipeline_configs/dev_substitutions.json create mode 100644 samples/nodester_sample/src/pipeline_configs/global.json create mode 100644 samples/nodester_sample/tests/__init__.py create mode 100644 samples/nodester_sample/tests/test_placeholder.py create mode 100644 samples/pattern-samples/src/notebooks/validate_run_1.ipynb create mode 100644 samples/pattern-samples/src/notebooks/validate_run_2.ipynb create mode 100644 samples/pattern-samples/src/notebooks/validate_run_3.ipynb create mode 100644 samples/pattern-samples/src/notebooks/validate_run_4.ipynb create mode 100644 samples/pattern-samples/src/notebooks/validation_utils.py create mode 100644 scripts/migrate_to_nodester.py create mode 100644 src/dataflow_spec_builder/transformer/nodester.py create mode 100644 src/schemas/spec_nodester.json diff --git a/.claude/commands/add_feature_sample.md b/.claude/commands/add_feature_sample.md new file mode 100644 index 0000000..b95ccd1 --- /dev/null +++ b/.claude/commands/add_feature_sample.md @@ -0,0 +1,153 @@ +Add a new feature sample and its validations to the Lakeflow Framework samples. + +## Context + +You are helping add a new feature sample to the Lakeflow Framework. A feature sample demonstrates a specific framework capability end-to-end — from a dataflowspec JSON/YAML file through to validated output tables. + +The framework root is the current working directory (the repo root). + +## Your Task + +The user will describe what feature they want to add a sample for (e.g. "add a sample for the new append_dedup flow type" or "add a sample for custom Python aggregation transforms"). Follow all steps below. + +--- + +## Step 1 — Understand the Feature + +1. Read the relevant framework source code in `src/` to understand what the feature does, its configuration keys, and what output tables it produces. +2. Read at least 2 similar existing dataflowspecs in `samples/bronze_sample/src/dataflows/feature_samples/dataflowspec/` to understand the naming convention (`{feature_name}_main.json`) and structure. +3. Decide which pipeline group the new sample belongs to: + - `feature_samples_general` → goes in the General Pipeline + - `feature_samples_data_quality` → goes in the Data Quality Pipeline + - `feature_samples_snapshots` → goes in the Snapshots Pipeline + - `feature_samples_table_migration` → goes in the Table Migration Pipeline + - `feature_samples_python` → goes in the Python Pipeline + - If it's a new group, you'll also need to create a new pipeline resource YAML (see Step 4). + +--- + +## Step 2 — Create the Dataflowspec + +Create `samples/bronze_sample/src/dataflows/feature_samples/dataflowspec/{feature_name}_main.json` (or `.yaml`). + +Follow the exact naming pattern: `{feature_name}_main.json`. + +Rules: +- Set `dataFlowGroup` to the correct pipeline group from Step 1 +- Use `{staging_schema}`, `{bronze_schema}`, `{silver_schema}` etc. as placeholders (these are substituted at runtime) +- Use a descriptive `dataFlowId` that starts with `feature_` to distinguish it as a sample +- Set `targetDetails.table` to `feature_{feature_name}` or a similarly descriptive name +- Enable `delta.enableChangeDataFeed` in tableProperties where applicable +- Validate the spec using: `python scripts/validate_dataflows.py samples/bronze_sample/src/dataflows/feature_samples/dataflowspec/{feature_name}_main.json` + +--- + +## Step 3 — Add Test Data (if needed) + +If the feature requires specific source data that doesn't already exist: +1. Check `samples/test_data_and_orchestrator/src/create_schemas_and_tables.ipynb` — add DDL if a new staging table is needed +2. Check `samples/test_data_and_orchestrator/src/run_1_staging_load.ipynb` — add initial data inserts +3. Check `run_2_staging_load.ipynb`, `run_3_staging_load.ipynb`, `run_4_staging_load.ipynb` — add incremental data to exercise SCD2 history, updates, deletes, snapshot changes, etc. + +Data loading conventions: +- Run 1: initial state (all active, no history) +- Run 2: first set of changes (updates, new records, deletions via flag) +- Run 3: second set of changes (more updates, snapshot overwrites) +- Run 4: final changes (additional updates to test multiple SCD2 versions) + +--- + +## Step 4 — Register in a Pipeline (only if new pipeline group) + +If the feature uses a new `dataFlowGroup` that doesn't map to an existing pipeline, create pipeline resource YAMLs: +- `samples/bronze_sample/resources/serverless/{pipeline_name}.yml` +- `samples/bronze_sample/resources/classic/{pipeline_name}.yml` + +Copy the structure from an existing pipeline YAML (e.g. `bronze_feature_samples_pipeline_general.yml`) and set `pipeline.dataFlowGroupFilter` to the new group. + +Then add the pipeline lookup variable to `samples/test_data_and_orchestrator/databricks.yml`: +```yaml +lakeflow_samples_{pipeline_key}_id: + lookup: + pipeline: "${var.name_prefix}Lakeflow Framework - {Pipeline Display Name} (${var.logical_env})" +``` + +--- + +## Step 5 — Add Pipeline Tasks to Orchestrator Jobs + +Add the pipeline as a task in all 4 run job YAMLs (both classic and serverless): +- `samples/test_data_and_orchestrator/resources/serverless/run_1_load_and_schema_initialization_job.yml` +- `samples/test_data_and_orchestrator/resources/serverless/run_2_load_job.yml` +- `samples/test_data_and_orchestrator/resources/serverless/run_3_load_job.yml` +- `samples/test_data_and_orchestrator/resources/serverless/run_4_load_job.yml` +- Same 4 files under `resources/classic/` + +Use `full_refresh: true` in run_1 and `full_refresh: false` in run_2/3/4. + +Also add the new pipeline task key to the `depends_on` list of the `validate_run_N` task in all 8 files. + +--- + +## Step 6 — Write Validations + +Add validation cells to all 4 validate notebooks: +`samples/test_data_and_orchestrator/src/validate_run_{1,2,3,4}.ipynb` + +For each notebook, insert new markdown + code cells **before the final `v.print_summary()` cell**. + +**Key principles for good validations:** + +1. **Exact counts for deterministic tables**: Use `v.validate_row_count(table, N, description)` when you know exactly how many rows should be present after each run (e.g. append-only, SCD1 batch-overwrite, small deterministic CDC streams). + +2. **Min counts for complex tables**: Use `v.validate_min_row_count(table, N, description)` for tables where exact counts are harder to predict (e.g. file-based historical snapshots, complex SCD2 with many interacting sources). + +3. **SCD2 active/closed counts**: Use these validators for CDC/SCD2 tables: + - `v.validate_active_scd2_count(table, N, end_at_col)` — records where `end_at_col IS NULL` + - `v.validate_closed_scd2_count(table, N, end_at_col)` — records where `end_at_col IS NOT NULL` + - `v.validate_min_closed_scd2_count(table, N, end_at_col)` — at least N closed records + +4. **Value checks**: Use `v.validate_column_value(table, where_clause, column, expected_value, description)` to verify specific field values (e.g. check that John's email updated correctly across runs). + +5. **Existence checks**: Use `v.validate_values_exist(table, column, [list_of_values], description)` to verify that specific IDs or keys exist. + +6. **Null checks**: Use `v.validate_column_not_null(table, column_expr, description)` for operational metadata or required columns. + +**Per-run validation logic:** +- **Run 1**: Initial state. All streaming tables show data from the first load. SCD2 tables show all-active records (0 closed). SCD1 tables show the first snapshot. Historical snapshot tables show SCD2 history from all pre-loaded files. +- **Run 2**: First incremental. CDC tables grow by the Run 2 inserts. SCD2 tables gain closed records for updated keys. SCD1 tables update in-place. Snapshot sources are overwritten. +- **Run 3**: Second incremental. Similar pattern; note which snapshot sources are overwritten and which tables are unchanged. +- **Run 4**: Final incremental. Verify the last expected state, including checking exact final values (e.g. latest email address). + +**Add a markdown cell** before each code cell explaining what tables are being validated and the expected state. + +--- + +## Step 7 — Verify + +1. Run the dataflowspec validator to check the spec is valid: + ```bash + python scripts/validate_dataflows.py samples/bronze_sample/src/dataflows/feature_samples/dataflowspec/{feature_name}_main.json -v + ``` + +2. If Databricks CLI is configured, you can deploy and test with a logical environment alias: + ```bash + cd samples + ./deploy_and_test.sh -l "_test_{feature_name}" -u "" -h "" -p "" --runs 4 + ``` + The `logical_env` parameter (e.g. `_test_my_feature`) namespaces all schemas so you don't affect the main deployment. + +3. After a successful test run, review the validation notebook output to confirm all assertions pass. + +--- + +## Checklist + +Before finishing, confirm: +- [ ] Dataflowspec file created and validates cleanly with `validate_dataflows.py` +- [ ] Test data added (or confirmed that existing data covers the feature) +- [ ] Pipeline registered in orchestrator databricks.yml (if new group) +- [ ] Pipeline tasks added to all 8 run_N job YAMLs (classic + serverless) with correct depends_on +- [ ] Validation cells added to all 4 validate_run_N notebooks covering: Run 1 (initial), Run 2 (first incremental), Run 3 (second incremental), Run 4 (final state) +- [ ] Each validation uses the most appropriate validator method (exact count vs min count vs SCD2 active/closed) +- [ ] All validate task `depends_on` lists in the job YAMLs include the new pipeline task key diff --git a/.claude/commands/deploy-nodester-samples.md b/.claude/commands/deploy-nodester-samples.md new file mode 100644 index 0000000..153ca54 --- /dev/null +++ b/.claude/commands/deploy-nodester-samples.md @@ -0,0 +1,138 @@ +# Deploy and Test Nodester Samples + +This skill deploys and tests the Lakeflow Framework nodester samples in a Databricks workspace. + +## Overview + +The nodester samples demonstrate a node-based graph approach to defining DLT pipelines. They live in `samples/nodester_sample/` and include 4 sample dataflows: +- **customer_simple**: Single source → single target (basic streaming) +- **customer_cloudfiles**: Cloud Files → bronze table (file ingestion) +- **customer_with_transform**: Multi-source with transform chain and SCD2 merge +- **customer_multi_target**: Staging append → merge → SCD2 with quarantine table + +## Prerequisites + +- Databricks CLI installed and authenticated (`databricks auth login`) +- Access to a Databricks workspace with Unity Catalog enabled +- The repo cloned locally — all paths below are relative to the repo root + +## Step-by-Step Deployment + +### 1. Deploy the Framework (Required First) + +The framework source code must be deployed to the workspace before any sample pipelines can run. + +```bash +# From the repo root +databricks bundle deploy +``` + +### 2. Deploy All Samples + +To deploy all samples (bronze, silver, gold, yaml, nodester, orchestrator): + +```bash +cd samples +./deploy.sh -h -u -l +``` + +To deploy only the nodester sample: + +```bash +cd samples +./deploy_nodester.sh -h -u -l +``` + +**Parameters:** + +| Flag | Description | Example | +|------|-------------|---------| +| `-h` / `--host` | Databricks workspace URL | `https://my-workspace.azuredatabricks.net/` | +| `-u` / `--user` | Your Databricks user email | `alice@example.com` | +| `-l` / `--logical_env` | Logical environment suffix (prefixed with `_`) | `_alice` | +| `--catalog` | Unity Catalog catalog name (default: `main`) | `main` | +| `--schema_namespace` | Schema namespace prefix (default: `lakeflow_samples`) | `lakeflow_samples` | +| `-p` / `--profile` | Databricks CLI profile to use | `my-profile` | + +The resulting schemas will be: +- Bronze: `._bronze` +- Silver: `._silver` +- Staging: `._staging` + +### 3. Deploy and Run Tests (Run 1 - Full Refresh) + +To deploy everything and run the Day 1 initialization job: + +```bash +cd samples +./deploy_and_test.sh -h -u -l --runs 1 +``` + +This will: +1. Deploy all sample bundles (bronze, silver, gold, yaml, nodester, orchestrator) +2. Run the "Lakeflow Framework - Run 1 - Load and Schema Initialization" job +3. The job runs all pipelines including the 4 nodester pipelines + +### 4. Run Subsequent Tests (Incremental Loads) + +```bash +./deploy_and_test.sh -h -u -l --runs 2 +./deploy_and_test.sh -h -u -l --runs 3 +./deploy_and_test.sh -h -u -l --runs 4 +``` + +## Manual Pipeline Execution via Databricks CLI + +To manually trigger a specific nodester pipeline: + +```bash +# Find the pipeline ID by name +databricks pipelines list-pipelines --profile | grep -i nodester + +# Run a full refresh +databricks pipelines start-update --full-refresh --profile + +# Run an incremental update +databricks pipelines start-update --profile + +# Check update status +databricks pipelines get-update --profile +``` + +## Key File Locations + +| File | Purpose | +|------|---------| +| `samples/nodester_sample/databricks.yml` | Bundle definition for nodester sample | +| `samples/nodester_sample/src/dataflows/base_samples/dataflowspec/` | Nodester JSON spec files | +| `samples/nodester_sample/src/dataflows/base_samples/expectations/` | DQ expectation files | +| `samples/nodester_sample/src/dataflows/base_samples/schemas/` | Table schema files | +| `samples/nodester_sample/src/pipeline_configs/dev_substitutions.json` | Token substitutions | +| `samples/deploy_nodester.sh` | Nodester-only deploy script | +| `samples/deploy.sh` | Full samples deploy script | +| `samples/deploy_and_test.sh` | Deploy + run test job | +| `src/dataflow_spec_builder/transformer/nodester.py` | Nodester spec transformer | + +## Troubleshooting + +### Pipeline fails with "view not found" or "Failed to analyze flow" +- Check that `inputs` arrays reference **node IDs** (not table/view names) +- If a source reads from an internal pipeline table, ensure its `table` field matches the producing target's `table` — the framework auto-detects internal sources by graph topology + +### Quarantine table is empty +- Verify `dataQualityExpectationsEnabled: true` and `dataQualityExpectationsPath` are set on the target node +- For staging tables with quarantine, ensure the DQ path points to a valid expectations file + +### Framework source path error +- Re-run `databricks bundle deploy` from the repo root before deploying samples + +### Schema not found +- Ensure the `create_schemas_and_tables` notebook ran successfully (it is part of the Run 1 job) + +## Architecture Notes + +The nodester spec transformer (`src/dataflow_spec_builder/transformer/nodester.py`) converts node-based specs to the standard framework flow spec format. Key rules: +- `inputs` arrays reference **node IDs**, not view or table names +- The primary (terminal) target is auto-detected from the graph — no `isPrimary` flag needed +- Internal pipeline sources are detected by matching `table` against target node tables in the spec; no `database: "live"` needed in user specs +- Views are auto-named `v_{node_id}` unless `outputName` is specified on a transformation node diff --git a/.github/workflows/feature-test.yml b/.github/workflows/feature-test.yml new file mode 100644 index 0000000..01fc43c --- /dev/null +++ b/.github/workflows/feature-test.yml @@ -0,0 +1,79 @@ +name: Feature Branch - Deploy and Test Samples + +on: + push: + branches: + - feature/sample-validations + workflow_dispatch: + inputs: + logical_env: + description: 'Logical environment suffix (e.g. _hw)' + required: true + default: '_hw' + type: string + compute: + description: 'Compute type (0=Classic, 1=Serverless)' + required: true + default: '1' + type: choice + options: + - '0' + - '1' + catalog: + description: 'Unity Catalog name' + required: true + default: 'main' + type: string + schema_namespace: + description: 'Schema namespace' + required: true + default: 'lakeflow_samples' + type: string + num_runs: + description: 'Number of test runs (1-4)' + required: true + default: '4' + type: choice + options: + - '1' + - '2' + - '3' + - '4' + +permissions: + contents: read + +jobs: + deploy-and-test: + runs-on: + group: databricks-solutions-protected-runner-group + env: + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Databricks CLI + uses: databricks/setup-cli@main + + - name: Verify Databricks authentication + run: databricks auth env + + - name: Deploy framework + run: databricks bundle deploy + + - name: Deploy and Test Samples + working-directory: samples + run: | + chmod +x deploy_and_test.sh common.sh deploy.sh + ./deploy_and_test.sh \ + -h "${{ secrets.DATABRICKS_HOST }}" \ + -u "${{ secrets.DATABRICKS_USER }}" \ + -l "${{ inputs.logical_env || '_ci_test' }}" \ + -c "${{ inputs.compute || '1' }}" \ + -p "DEFAULT" \ + --catalog "${{ inputs.catalog || 'main' }}" \ + --schema_namespace "${{ inputs.schema_namespace || 'lakeflow_samples' }}" \ + --runs "${{ inputs.num_runs || '4' }}" diff --git a/docs/decisions/0007-unified-nodester-dataflow-spec.md b/docs/decisions/0007-unified-nodester-dataflow-spec.md new file mode 100644 index 0000000..53c289a --- /dev/null +++ b/docs/decisions/0007-unified-nodester-dataflow-spec.md @@ -0,0 +1,342 @@ +# ADR-0007: A unified, node-based dataflow spec (`nodester`) + +**Date:** 2026-06-19 +**Status:** Accepted + +--- + +## Context + +The framework historically shipped several distinct dataflow spec formats, and +each one has its own shape and its own rules: + +- **Standard**: a single source flattened onto a single target, with CDC, data + quality, quarantine, and table settings spread across the top level of the spec. +- **Flow**: a hand-authored flow graph with views, staging tables, and flow + groups, where the author must already understand the framework's internal + flow-spec representation. +- **Materialized view**: a separate `materializedViews` map with its own nested + `sourceView` and `tableDetails` structure. + +This creates real friction: + +1. **Three formats to learn instead of one.** A new user has to first work out + which spec type applies to their case, and then learn a different field layout + for each. Knowledge gained writing a standard spec does not transfer to a flow + or materialized view spec. + +2. **The structure leaks framework internals.** The flow format in particular + asks authors to think in terms of flow groups, view registration, and staging + tables. Those are concepts that exist to serve the engine, not the person + describing a pipeline. To write a correct spec you must understand how the + framework assembles a pipeline, not just what you want the pipeline to do. + +3. **Deep nesting is hard to write and hard to read.** Settings live at different + depths depending on the format and the feature. Answering a simple question, + such as "what feeds this table, and what happens to the data on the way in?", + can mean tracing keys across several levels of nested objects. The nesting + obscures the one thing a reader actually cares about, which is the shape of the + pipeline. + +4. **Topology is implicit.** In the older formats the connections between steps + are inferred (from SQL, from key names, from position) rather than stated. The + graph a pipeline actually forms is not something you can see by reading the + spec. + +5. **Streaming tables and materialized views must live in separate specs.** + Because each output type has its own format, a pipeline that mixes streaming + tables and materialized views has to be split across multiple specs. When those + tables feed each other, for example a streaming table chained into a + materialized view, the split is artificial and inconvenient: one logical + pipeline ends up fragmented across files for no reason other than the spec + format. + +The cost of this is paid most by the two groups this framework exists to help. +The first is newcomers writing their first pipeline, who should not have to learn +the engine's internals to get started. The second is large organizations that +want to standardize and to easily read, understand, and edit each other's +pipelines across many people. Making both of those easier is the point of the +framework, and the current formats work against it. + +## Decision + +Introduce a single, unified dataflow spec, called **`nodester`**, and make it the +way pipelines are described going forward. + +A `nodester` spec is a **graph of nodes**. There are exactly three node types, and +they compose by **chaining**: + +``` +source -> transformation -> target +(where data (how data is (where data + comes from) transformed) lands) +``` + +- A **source** node says where data comes from (a table, files, a stream). +- A **transformation** node says how data is reshaped (SQL or Python). +- A **target** node says where data lands, and carries its own table-level + settings (CDC, data quality, quarantine, clustering, and so on). + +Nodes are wired together in two complementary ways. A **target** node declares +what feeds it through an explicit `input` list. **Source** and **transformation** +nodes are connected by explicit reference inside their own definition: a +transformation names its upstream view directly in its SQL or Python (for example +`FROM STREAM(live.v_source_customer)`), and a source names the table, path, or +stream it reads. The `input` list is therefore a target-node construct, while the +intermediate source-to-transformation wiring lives where the logic that uses it +lives. That uniform model replaces all three legacy formats. Every pipeline, +whether a one-hop ingest, a CDC merge, a multi-step transform, or a materialized +view, is expressed the same way: declare nodes, then chain them together. + +Because output type is just a property of a target node, a single `nodester` spec +can declare both streaming-table and materialized-view targets at once, including +chains where one feeds the other. A logical pipeline that mixes the two no longer +has to be split across separate specs by output type. + +### The `input` list + +`input` belongs to target nodes and lists what feeds the target. Each item is +either a plain string (the upstream node name, with the flow name auto-generated) +or an object that defines the flow name: + +```json +"input": [ + "v_source_customer", + { "view": "v_source_address", "flow": "f_append_address" } +] +``` + +The two forms can be mixed in the same list. By default the framework derives the +flow name from the graph, so simple specs stay simple; the object form defines it +when stability matters. This matters because renaming a flow forces a full refresh +in SDP, so defining the flow name lets authors keep it stable across edits and +migrations without triggering one. Source and transformation nodes do not carry an +`input` list; they reference their upstream within their own definition as +described above. + +### Why a node graph + +The node-and-edge model is the mental model most people already have for data +pipelines. It is how ETL is drawn on a whiteboard, how lineage is shown, and how +virtually every visual pipeline tool represents work. By matching that model +directly, the spec stops being something you have to learn the internals of and +becomes something you can read at a glance. Newcomers can be productive without +first absorbing the framework's flow-spec representation, and teams can read each +other's pipelines without reverse-engineering the structure first. + +### Before and after + +Consider a real pattern: two streaming sources are appended into one staging +table, and that staging table is then merged into a silver table with SCD2 +history. In the flow format the author hand-builds the flow groups, registers +each view, declares the staging table, and wires the flows. The pipeline's shape +is buried several levels deep: + +```json +{ + "dataFlowId": "multi_source_streaming_decomp_staging", + "dataFlowGroup": "multi_source_streaming_decomposed_staging", + "dataFlowType": "flow", + "targetFormat": "delta", + "targetDetails": { + "database": "{silver_schema}", + "table": "customer_ms_decomp_merge", + "schemaPath": "customer.json", + "tableProperties": { "delta.enableChangeDataFeed": "true" } + }, + "cdcSettings": { + "keys": ["CUSTOMER_ID"], + "scd_type": "2", + "sequence_by": "LOAD_TIMESTAMP", + "except_column_list": ["LOAD_TIMESTAMP"], + "ignore_null_updates": true + }, + "flowGroups": [ + { + "flowGroupId": "multi_source_streaming_decomp_staging_1", + "stagingTables": { "customer_ms_decomp_appnd": { "type": "ST" } }, + "flows": { + "f_customer": { + "flowType": "append_view", + "flowDetails": { + "targetTable": "customer_ms_decomp_appnd", + "sourceView": "v_customer" + }, + "views": { + "v_customer": { + "mode": "stream", + "sourceType": "delta", + "sourceDetails": { + "database": "{bronze_schema}", + "table": "base_customer_append", + "cdfEnabled": true + } + } + } + }, + "f_customer_address": { + "flowType": "append_view", + "flowDetails": { + "targetTable": "customer_ms_decomp_appnd", + "sourceView": "v_customer_address" + }, + "views": { + "v_customer_address": { + "mode": "stream", + "sourceType": "delta", + "sourceDetails": { + "database": "{bronze_schema}", + "table": "base_customer_address_append", + "cdfEnabled": true, + "whereClause": ["STATE is not NULL"] + } + } + } + }, + "f_merge": { + "flowType": "merge", + "flowDetails": { + "targetTable": "{silver_schema}.customer_ms_decomp_merge", + "sourceView": "customer_ms_decomp_appnd" + } + } + } + } + ] +} +``` + +The same pipeline as a `nodester` graph is a flat list of nodes that reads +top-to-bottom in the order the data flows. Each concern sits on the node it +belongs to, and the connections are stated rather than inferred: + +```json +{ + "data_flow_id": "multi_source_streaming_decomp_staging", + "data_flow_group": "multi_source_streaming_decomposed_staging", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_customer", + "node_type": "source", + "source_type": "delta", + "config": { "database": "{bronze_schema}", "table": "base_customer_append", "cdf_enabled": true, "mode": "stream" } + }, + { + "name": "v_customer_address", + "node_type": "source", + "source_type": "delta", + "config": { "database": "{bronze_schema}", "table": "base_customer_address_append", "cdf_enabled": true, "mode": "stream", "where_clause": ["STATE is not NULL"] } + }, + { + "name": "staging_customer_append", + "node_type": "target", + "config": { + "table": "customer_ms_decomp_appnd", + "input": ["v_customer", "v_customer_address"] + } + }, + { + "name": "v_staging_cdf", + "node_type": "source", + "source_type": "delta", + "config": { "table": "customer_ms_decomp_appnd", "cdf_enabled": true, "mode": "stream" } + }, + { + "name": "target_customer_merge", + "node_type": "target", + "config": { + "database": "{silver_schema}", + "table": "customer_ms_decomp_merge", + "schema_path": "customer.json", + "table_properties": { "delta.enableChangeDataFeed": "true" }, + "cdc_settings": { + "keys": ["CUSTOMER_ID"], + "sequence_by": "LOAD_TIMESTAMP", + "scd_type": 2, + "except_column_list": ["LOAD_TIMESTAMP"], + "ignore_null_updates": true + }, + "input": ["v_staging_cdf"] + } + } + ] +} +``` + +There are no flow groups to assemble, no views to register by hand, and no +separate staging-table block. Fan-in is just two names in one `input` list, CDC +lives on the target it applies to, and adding another step later is another node +in the chain rather than another layer of nesting. + +### Convergence behaviour + +`nodester` does not change the engine. The transformer lowers a node graph into +the framework's existing flow-spec representation, so all current capabilities +(CDC, snapshots, data quality, quarantine, sinks, table migration, materialized +views) are available. The legacy formats continue to transform into the same +representation, so the two can coexist during adoption. + +### Aligning toward the chaining model + +To steer everyone toward the chaining model, two behaviours change: + +- **Inline SQL/Python sources are discouraged (warning, not breaking).** Folding + transformation logic into a source definition, meaning a source whose type is + `sql` or `python`, or an `append_sql` flow, conflates "where data comes from" + with "how it is transformed". This still works, but now emits a warning, in + both the legacy and `nodester` formats. The recommended replacement is a plain + source chained into a dedicated transformation node. + +- **Inline source views on materialized view targets are removed (breaking).** A + materialized view target may no longer carry an inline `source_view`. Authors + declare a source node and chain it into the materialized view target via + `input`, exactly like every other node. This keeps the graph uniform, with one + way to feed a target rather than a special case for materialized views. Because + this pattern was not in use, it is removed outright rather than deprecated. + +## Consequences + +- **One format to learn.** Knowledge transfers across every pipeline shape. + Onboarding becomes "learn the three node types and how to chain them" instead of + "learn three different specs and how to choose between them". + +- **Specs are readable.** A reader follows the chain top-to-bottom and sees the + pipeline's shape directly. Per-target settings live on the target they + describe, which makes specs easier for teams to review and edit. + +- **Topology is explicit.** Targets state what feeds them via `input`, and + transformations name their upstream views directly in their logic, which makes + specs inspectable and a natural fit for lineage and visual tooling. + +- **Streaming tables and materialized views in one spec.** A single spec can + declare both streaming-table and materialized-view targets, including chains + where one feeds the other, so a logical pipeline no longer has to be split + across specs by output type. + +- **No loss of capability.** The node graph lowers into the existing flow-spec + representation, so every current feature remains available. + +- **Smooth adoption.** Legacy formats keep working. A migration script converts + existing specs into `nodester`. The inline-source-transformation warning gives + consumers time to align before any future removal. + +- **One breaking change.** Materialized view targets no longer accept an inline + `source_view`, so authors chain a source node instead. + +## Key design decisions + +The decisions below define how the `nodester` model behaves. They are the design +choices made while building out the format and translating the full feature set +onto it. + +| Decision | Rationale | +|----------|-----------| +| **No primary or secondary target.** The spec target the backend needs (`targetDetails`) is auto-selected from the graph as the terminal target; authors never mark one. | "Primary target" was a backend implementation detail leaking into the spec. The graph already determines which target is terminal. | +| **All target settings live on the target node.** CDC, data quality, quarantine, table migration, and sink config are declared on the target they apply to, not at the spec level. | Each setting belongs with the thing it configures, which removes the spec-level scatter of the older formats. | +| **Materialized views are target nodes** (`table_type: "mv"`), and a single spec may contain both streaming-table and materialized-view targets, including chains between them. | Keeps the node model uniform and lets one logical pipeline stay in one spec instead of being split by output type. | +| **Inline SQL/Python sources and `append_sql` are discouraged.** They still work and are warned about; the recommended alternative is a transformation node, which defines the logic as a view rather than folding it into a source. | Keeps "where data comes from" separate from "how it is transformed". | +| **Materialized view inline `source_view` is removed (breaking).** Chain a source node into the MV target via `input`. | One uniform way to feed any target, with no special case for materialized views. | +| **Historical snapshot targets need no source node.** Targets with historical snapshot CDC config describe their source within that config; periodic snapshot targets still require a source. | The historical snapshot system reads files or tables directly, so a source view would have nothing to read. | +| **`input` is a target-node construct.** Each item is a string (auto flow name) or `{ "view", "flow" }` (flow name defined). | Source-to-transformation wiring lives in the transformation's own definition; `input` only describes what feeds a target. Defining the flow name avoids a full refresh in SDP on rename. | +| **Internal sources are auto-detected by table name.** A source that reads a table produced by a target in the same spec is wired internally with no extra view. | Enables staging-to-downstream chains without manual `live.` wiring. | diff --git a/docs/nodester_feature_samples_changes.md b/docs/nodester_feature_samples_changes.md new file mode 100644 index 0000000..36e6892 --- /dev/null +++ b/docs/nodester_feature_samples_changes.md @@ -0,0 +1,235 @@ +# Nodester Feature Samples — Changes, Design & Considerations + +## Overview + +The Nodester dataflow spec type introduces a **node-based graph architecture** for defining data pipelines. Instead of the traditional flat (standard) or pre-built flow (flow) spec formats, Nodester expresses pipelines as directed graphs of **source**, **transformation**, and **target** nodes connected through `inputs` arrays. + +This document covers the changes made to support all 33 feature samples from the bronze_sample in the Nodester format, the migration tooling created, and the design decisions and assumptions behind the implementation. + +--- + +## What Changed + +### 1. Transformer Extensions (`src/dataflow_spec_builder/transformer/nodester.py`) + +The Nodester transformer was extended to support all feature types: + +| Feature | Change | +|---------|--------| +| **Removed `isPrimary`** | No primary/secondary target concept. The transformer auto-selects one target for backend `targetDetails` from graph topology (terminal node detection). This is an internal detail invisible to spec authors. | +| **`targetFormat` on targets** | Targets can specify `targetFormat: "foreach_batch_sink"`, `"delta_sink"`, or `"custom_python_sink"` alongside `sinkType` and `sinkConfig` for non-delta outputs. | +| **`tableMigrationDetails`** | Targets can include table migration config, which the transformer copies to the spec level for the backend. | +| **`configFlags`** | Targets can specify `configFlags: ["disableOperationalMetadata"]`, propagated to `targetDetails`. | +| **`once` flag** | Targets can set `once: true` to execute the flow only once (batch mode). Added to `flowDetails` in the output. | +| **`append_sql` flow type** | When a source node has `sourceType: "sql"`, the transformer creates an `append_sql` flow with the SQL directly in `flowDetails` (no intermediate view). | +| **`cdcSnapshotSettings`** | Snapshot targets (periodic and historical) now correctly trigger MERGE flow type. Historical file/table snapshots work without source nodes since the CDC snapshot system reads files/tables directly. | +| **`cdcApplyChanges` alias** | Treated identically to `cdcSettings` for backward compatibility with version mapping specs. | +| **`dataFlowVersion`** | Passed through from spec level to output. | +| **Materialized view targets** | Targets with `tableType: "mv"` produce separate MV flow specs. Source views for MVs are forced to batch mode (MVs use `spark.sql()` which can't read streaming views). Returns `List[Dict]` for multi-MV specs. | +| **Targets without inputs** | Targets with `cdcSnapshotSettings` and no `inputs` create a placeholder MERGE flow. The snapshot system handles file/table reading directly. | + +### 2. Schema Extensions (`src/schemas/spec_nodester.json`) + +- Removed `isPrimary` from the node schema +- Added to `targetNodeConfig`: `targetFormat`, `tableType`, `tableMigrationDetails`, `configFlags`, `once`, `name`, `sinkType`, `sinkConfig`, `sinkOptions`, `sqlPath`, `sqlStatement`, `sourceView`, `refreshPolicy`, `tableDetails`, `cdcApplyChanges` +- Added to `sourceNodeConfig`: `functionPath`, `pythonModule`, `tokens`, `sqlPath`, `sqlStatement`, `startingVersionFromDLTSetup`, `cdfChangeTypeOverride` + +### 3. Migration Script (`scripts/migrate_to_nodester.py`) + +CLI tool that converts any existing spec to Nodester format: + +```bash +# Single file +python scripts/migrate_to_nodester.py spec_main.json --output nodester_spec.json + +# Entire directory +python scripts/migrate_to_nodester.py ./dataflowspec/ --output-dir ./nodester_dataflowspec/ +``` + +Handles all three spec types: +- **Standard** → source node (from sourceDetails) + target node (from targetDetails + spec-level CDC/DQ/quarantine) +- **Flow** → source nodes (from views) + target nodes (from staging tables + main target) with connections from flow inputs +- **Materialized View** → target nodes with `tableType: "mv"`, optional source nodes for sourceView patterns + +### 4. Feature Samples (`samples/nodester_sample/src/dataflows/feature_samples/`) + +All 33 specs translated with supporting files (schemas, expectations, SQL, Python functions) copied from `bronze_sample`. Each spec uses `dataFlowGroup: "nodester_feature_samples_*"` prefixes. + +### 5. Pipeline Resources (`samples/nodester_sample/resources/serverless/`) + +Six new pipeline definitions: +- `nodester_feature_samples_general_pipeline.yml` — 9 specs +- `nodester_feature_samples_data_quality_pipeline.yml` — 2 specs +- `nodester_feature_samples_snapshots_pipeline.yml` — 11 specs +- `nodester_feature_samples_python_pipeline.yml` — 4 specs +- `nodester_feature_samples_table_migration_pipeline.yml` — 2 specs +- `nodester_feature_samples_materialized_views_pipeline.yml` — 6 MVs + +### 6. Framework Bug Fix (`src/dlt_pipeline_builder.py`) + +`table_migration_state_volume_path` in `global.json` was hardcoded with `_es` suffix. Changed to use `{logical_env}` token and moved the initialization to after the `SubstitutionManager` is ready so the token gets resolved. + +--- + +## How Nodester Works + +### Node Graph Model + +A Nodester spec defines a pipeline as a graph of nodes: + +``` +Source Nodes ──→ Transformation Nodes ──→ Target Nodes +(data origins) (SQL/Python logic) (output tables) +``` + +Each node has: +- `id` — unique identifier +- `type` — `source`, `transformation`, or `target` +- `inputs` — array of node IDs this node reads from (source nodes have none) +- `config` — type-specific configuration + +### Transformation Pipeline + +``` +Nodester JSON Spec + │ + ▼ +NodesterSpecTransformer._process_spec() + │ + ├─ Categorize nodes by type + ├─ Validate graph (references, cycles, required nodes) + ├─ Detect internal sources (source reading from target in same spec) + ├─ Auto-select spec target (terminal node in graph) + ├─ Build flow spec: + │ ├─ targetDetails from spec target + │ ├─ CDC/DQ/quarantine from spec target → spec level + │ ├─ Other targets → stagingTables + │ └─ Flows from node connections → flowGroups + ▼ +Lakeflow Framework Flow Spec (same format as standard/flow transformers) + │ + ▼ +Backend (DataFlow, FlowGroup, CDC, etc.) +``` + +### Spec Target Selection + +With no `isPrimary` flag, the transformer auto-selects which target becomes `targetDetails`: + +1. Build set of "consumed" targets — any target whose ID appears in another node's inputs, or whose table name is referenced by a source node +2. Terminal targets = targets that are NOT consumed +3. If multiple terminal targets exist, use the last one (with a warning) +4. All other targets become staging tables + +For single-target specs (the majority), the only target is automatically selected. + +### Internal Source Detection + +When a source node reads from a table that's also a target in the same spec, it's treated as "internal": +- The source references the staging table directly (no view created) +- Database is forced to `live` so DLT resolves it during pipeline analysis +- This enables staging patterns: source → staging_target → internal_source → final_target + +### Flow Type Determination + +| Condition | Flow Type | +|-----------|-----------| +| Target has `cdcSettings` or `cdcApplyChanges` | `merge` | +| Spec target has CDC and flow writes to spec target | `merge` | +| Target has `cdcSnapshotSettings` | `merge` | +| Source is SQL type (`sourceType: "sql"`) | `append_sql` | +| Otherwise | `append_view` | + +--- + +## Design Decisions & Assumptions + +### 1. No Primary Target Concept + +**Decision:** The spec author never needs to think about which target is "primary." All targets are equal — they each specify their own CDC, DQ, quarantine, and table migration settings directly on the target node config. + +**Why:** The "primary target" was a backend implementation detail leaking into the spec format. The backend requires `targetDetails` at the spec level, but this is now handled transparently by the transformer. + +**Trade-off:** The auto-selection heuristic (terminal node detection) could pick the wrong target in ambiguous multi-target specs. A warning is logged in this case. + +### 2. Per-Target Settings at Spec Level + +**Decision:** The spec target's CDC/DQ/quarantine settings are duplicated at the spec level for backend compatibility. + +**Assumption:** The backend reads these settings from the spec level for the main target. This duplication is an interim measure — the backend should eventually read all settings from target nodes directly. + +### 3. Materialized Views as Target Nodes + +**Decision:** MVs are target nodes with `tableType: "mv"`. Each MV target in a spec becomes a separate flow spec (the transformer returns `List[Dict]`). + +**Why:** MVs are fundamentally different from streaming tables — they use `spark.sql()` batch reads and have different creation ordering (flows first, then MV). Treating them as a target type keeps the node graph model consistent. + +**Constraint:** MV source views are forced to batch mode because DLT's `spark.sql()` cannot read from streaming views. + +### 4. SQL Sources → `append_sql` Flow Type + +**Decision:** When a source node has `sourceType: "sql"`, the transformer creates an `append_sql` flow with the SQL directly in `flowDetails`, bypassing view creation. + +**Why:** `append_sql` is a distinct flow type in the framework that handles DLT-specific SQL (like `STREAM()` functions) directly. Using `append_view` with an SQL source view would work for simple cases but wouldn't match the original spec behavior. + +### 5. Snapshot Targets Without Source Nodes + +**Decision:** Targets with `cdcSnapshotSettings` don't require source nodes. Historical file/table snapshots define their data source entirely within the snapshot settings. + +**Why:** The CDC snapshot system reads files/tables directly based on `cdcSnapshotSettings.source`. A traditional source node would create a view with nothing to read from. The transformer creates a placeholder MERGE flow with no `sourceView`. + +### 6. Migration Script Assumptions + +- The script preserves `dataFlowId` and `dataFlowGroup` from the original spec (the group is prefixed with `nodester_` separately) +- Source node IDs are derived from `sourceViewName` (stripping `v_` prefix) +- Target node IDs follow the pattern `target_{table_name}` +- For flow specs, view names from the original spec become source node IDs directly +- The script doesn't validate the output against the JSON schema — the transformer handles validation at runtime + +--- + +## Known Limitations + +### 1. CDC Merge + Quarantine "table" Mode + +**Status:** Fixed. The `_get_exclude_columns` method in `dataflow.py` no longer adds `is_quarantined` to the exclude list for TABLE quarantine mode. + +**Root Cause (was):** When `quarantineMode: "table"` and `cdcSettings` coexist, the framework added `is_quarantined` to the flow's `exclude_columns` list. However, in table quarantine mode, the `is_quarantined` column is NOT added to the source view (only a separate quarantine view gets it). This created a column resolution failure when the CDC merge flow tried to reference it via `except_column_list`. + +**Fix:** Removed the logic that added `is_quarantined` to `exclude_columns` for TABLE mode. In table quarantine mode, the column never exists in the source view — the separate quarantine view/flow handles it independently. In FLAG mode, the column is part of the target schema and written through, so no exclusion is needed there either. + +### 2. Delta Sink Specs + +**Status:** Not tested (disabled in the original bronze_sample too). + +The `sink_delta_path_table` and `sink_delta_uc_table` specs have `dataFlowGroup: "nodester_feature_samples_general_DISABLED"` — they're excluded from all pipelines, matching the original behavior. + +--- + +## Test Results + +| Pipeline | Flows | Status | +|----------|-------|--------| +| General | append_sql, append_view, append_view_once, ddl_schema, version_mapping (standard+flows), foreach_batch (sql+python) | 9/9 passed | +| Data Quality | quarantine_flag, quarantine_table | 3/3 passed | +| Snapshots | 7 historical file, 2 historical table, 1 historical flow, 2 periodic | 13/13 passed | +| Python | python_source, python_source_extension, python_transform, python_transform_extension | 4/4 passed | +| Table Migration | append_only, scd2 (with import flows) | 4/4 passed | +| Materialized Views | 6 MVs (source_view, sql_path, sql_statement, quarantine, chained, refresh_policy) | 8/8 passed | + +--- + +## Files Changed + +| File | Action | +|------|--------| +| `src/dataflow_spec_builder/transformer/nodester.py` | Extended transformer with all feature support | +| `src/schemas/spec_nodester.json` | Extended schema, removed isPrimary | +| `src/dlt_pipeline_builder.py` | Moved table_migration init after SubstitutionManager | +| `samples/bronze_sample/src/pipeline_configs/global.json` | Changed hardcoded `_es` to `{logical_env}` token | +| `scripts/migrate_to_nodester.py` | New migration script | +| `samples/nodester_sample/src/dataflows/feature_samples/` | 33 translated specs + supporting files | +| `samples/nodester_sample/src/extensions/` | Copied Python extension modules | +| `samples/nodester_sample/src/pipeline_configs/global.json` | Added table_migration_state_volume_path | +| `samples/nodester_sample/resources/serverless/` | 6 new pipeline resource YAMLs | +| `samples/nodester_sample/databricks.yml` | Added serverless resource includes | diff --git a/docs/source/dataflow_spec_ref_main_nodester.rst b/docs/source/dataflow_spec_ref_main_nodester.rst new file mode 100644 index 0000000..71a3fa0 --- /dev/null +++ b/docs/source/dataflow_spec_ref_main_nodester.rst @@ -0,0 +1,434 @@ +Creating a Nodester Data Flow Spec Reference +############################################ + +The **Nodester** Data Flow Spec describes a pipeline as a graph of nodes that +chain together: + +.. code-block:: text + + source -> transformation -> target + +Instead of the target-table-driven layout of the standard and flow specs, a +Nodester spec is a flat list of nodes that reads top-to-bottom in the order the +data flows. The framework converts a Nodester spec into its internal flow-based +format at build time, so every existing capability (CDC, data quality, +quarantine, snapshots, table migration, sinks, materialized views) is available. + +Key concepts +============ + +There are three node types: + +- **source** — where data comes from (Delta table, cloud files, Kafka, a SQL + query, or a Python function). +- **transformation** — how data is reshaped (SQL or Python). +- **target** — where data lands. A target carries its own table-level settings + (CDC, data quality, quarantine, clustering, and so on). + +**How nodes connect.** A *target* node declares what feeds it through an explicit +``input`` list. *Source* and *transformation* nodes are wired by explicit +reference inside their own definition: a SQL transformation names the view it +reads in its SQL (for example ``FROM STREAM(live.v_source_customer)``), and a +source names the table, path, or stream it reads. The ``input`` list is therefore +a target-node construct. + +**Casing.** Nodester specs use ``snake_case`` field names. + +Example: simple data flow +========================= + +The simplest spec connects a source to a target: + +.. code-block:: json + + { + "data_flow_id": "nodester_customer_simple", + "data_flow_group": "nodester_base_samples", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_source_customer", + "node_type": "source", + "source_type": "delta", + "config": { + "database": "{bronze_schema}", + "table": "customer", + "cdf_enabled": true, + "mode": "stream" + } + }, + { + "name": "target_customer", + "node_type": "target", + "config": { + "table": "customer_silver", + "cluster_by_auto": true, + "input": [ + { "view": "v_source_customer", "flow": "f_customer_ingest" } + ] + } + } + ] + } + +Example: multi-step transformation and CDC +========================================== + +Chaining a transformation into a CDC target. The SQL transformation names the +view it reads; the target lists the transformation in its ``input``: + +.. code-block:: json + + { + "data_flow_id": "nodester_customer_enriched", + "data_flow_group": "nodester_base_samples", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_source_customer", + "node_type": "source", + "source_type": "delta", + "config": { "database": "{bronze_schema}", "table": "customer", "cdf_enabled": true, "mode": "stream" } + }, + { + "name": "v_enrich", + "node_type": "transformation", + "transformation_type": "sql", + "config": { + "sql_statement": "SELECT CUSTOMER_ID, EMAIL, CITY, LOAD_TIMESTAMP FROM STREAM(live.v_source_customer)" + } + }, + { + "name": "target_customer_enriched", + "node_type": "target", + "config": { + "table": "customer_enriched", + "schema_path": "customer_enriched_schema.json", + "cdc_settings": { + "keys": ["CUSTOMER_ID"], + "sequence_by": "LOAD_TIMESTAMP", + "scd_type": 2, + "ignore_null_updates": true + }, + "input": ["v_enrich"] + } + } + ] + } + +.. _dataflow-spec-nodester-metadata-configuration: + +Dataflow metadata configuration +=============================== + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - **Field** + - **Type** + - **Description** + * - **data_flow_id** + - ``string`` + - A unique identifier for the data flow. + * - **data_flow_group** + - ``string`` + - The group to which the data flow belongs. + * - **data_flow_type** + - ``string`` + - Must be ``"nodester"``. + * - **data_flow_version** (*optional*) + - ``string`` + - Version of the dataflow spec for migration purposes. + * - **tags** (*optional*) + - ``object`` + - Custom tags for the dataflow. + * - **features** (*optional*) + - ``object`` + - Feature flags (e.g., ``operationalMetadataEnabled``). + +.. _dataflow-spec-nodester-nodes: + +Node configuration +================== + +Each entry of the ``nodes`` array has: + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - **Field** + - **Type** + - **Description** + * - **name** + - ``string`` + - Unique name for the node within the spec. Source and transformation node + names double as their view names (prefixed with ``v_`` if not already). + * - **node_type** + - ``string`` + - One of ``source``, ``transformation``, ``target``. + * - **source_type** + - ``string`` + - Required for source nodes (see below). + * - **transformation_type** + - ``string`` + - Required for transformation nodes (see below). + * - **target_type** (*optional*) + - ``string`` + - For target nodes: ``delta`` (default), ``delta_sink``, + ``foreach_batch_sink``, or ``custom_python_sink``. + * - **output_view_name** (*optional*) + - ``string`` + - Override the generated view name for a source/transformation node. + * - **enabled** (*optional*) + - ``boolean`` + - Whether the node is enabled. Default: ``true``. + * - **config** + - ``object`` + - Node-specific configuration (see below). + +.. note:: + There is no primary/secondary target flag. The framework auto-selects the + spec target (the backend's ``targetDetails``) as the terminal target — the + one not consumed by any other node. Any other targets become staging tables. + +.. _dataflow-spec-nodester-source-nodes: + +Source node configuration +========================= + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - **Field** + - **Type** + - **Description** + * - **source_type** + - ``string`` + - ``delta``, ``cloudFiles``, ``batchFiles``, ``kafka``, ``sql``, ``python``. + * - **mode** (*optional*) + - ``string`` + - ``stream`` or ``batch``. Default: ``stream``. + * - **database** / **table** + - ``string`` + - For ``delta`` sources. + * - **path** + - ``string`` + - For ``cloudFiles`` / ``batchFiles`` sources. + * - **cdf_enabled** (*optional*) + - ``boolean`` + - Enable Change Data Feed for Delta sources. + * - **reader_options** (*optional*) + - ``object`` + - Reader options (cloud files, Kafka). + * - **schema_path** (*optional*) + - ``string`` + - Path to a schema definition file. + * - **select_exp** / **where_clause** (*optional*) + - ``array`` + - Select expressions / WHERE clauses applied on read. + * - **python_transform** (*optional*) + - ``object`` + - Apply a Python ``apply_transform(df)`` to the source on read. Supported + for backward compatibility; prefer a dedicated transformation node. + +.. note:: + ``source_type: "sql"`` and ``source_type: "python"`` embed transformation + logic directly in a source definition. They remain supported but are + **discouraged** and emit a warning at build time. Model the logic as a + dedicated transformation node instead. + +.. _dataflow-spec-nodester-transformation-nodes: + +Transformation node configuration +================================= + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - **Field** + - **Type** + - **Description** + * - **transformation_type** + - ``string`` + - ``sql``, ``python``, or ``passthrough``. + * - **sql_path** / **sql_statement** + - ``string`` + - For ``sql`` transforms. The SQL names the view(s) it reads, e.g. + ``FROM STREAM(live.v_source_customer)``. + * - **function_path** / **python_module** + - ``string`` + - For ``python`` transforms. The referenced file/module must contain an + ``apply_transform(df)`` (or ``apply_transform(df, tokens)``) function. + * - **tokens** (*optional*) + - ``object`` + - Tokens passed to a Python transform. + +.. note:: + A ``python`` transformation node becomes its own view that reads its upstream + view and applies ``apply_transform``. The upstream is inferred from the graph, + so no explicit reference is required. + +.. _dataflow-spec-nodester-target-nodes: + +Target node configuration +========================= + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - **Field** + - **Type** + - **Description** + * - **input** + - ``array`` + - What feeds this target. Each item is either an upstream node name + (``string``, flow name auto-generated) or an object + ``{ "view": , "flow": }`` that sets the flow name. + * - **table** + - ``string`` + - Target table name. + * - **database** (*optional*) + - ``string`` + - Target database/schema. + * - **table_type** (*optional*) + - ``string`` + - ``st`` (streaming table, default) or ``mv`` (materialized view). + * - **schema_path** (*optional*) + - ``string`` + - Schema definition file (``.json`` or ``.ddl``). + * - **table_properties** (*optional*) + - ``object`` + - Delta table properties. + * - **partition_columns** / **cluster_by_columns** / **cluster_by_auto** (*optional*) + - ``array`` / ``array`` / ``boolean`` + - Partitioning and liquid clustering. + * - **comment** / **spark_conf** / **row_filter** / **config_flags** (*optional*) + - ``string`` / ``object`` / ``string`` / ``array`` + - Additional table settings. + * - **once** (*optional*) + - ``boolean`` + - Execute the flow to this target only once (batch). + * - **cdc_settings** / **cdc_snapshot_settings** (*optional*) + - ``object`` + - CDC / snapshot-CDC settings for this target. + * - **data_quality_expectations_enabled** / **data_quality_expectations_path** (*optional*) + - ``boolean`` / ``string`` + - Enable and locate data quality expectations. + * - **quarantine_mode** / **quarantine_target_details** (*optional*) + - ``string`` / ``object`` + - ``off`` (default), ``flag``, or ``table``; quarantine table config when + ``table``. + * - **table_migration_details** (*optional*) + - ``object`` + - Table migration configuration. + +Defining flow names +------------------- + +By default the framework derives a flow name from the graph, so simple specs stay +simple. Use the object form of ``input`` to set a flow name explicitly: + +.. code-block:: json + + "input": [ + "v_source_a", + { "view": "v_source_b", "flow": "f_append_b" } + ] + +Defining the flow name keeps it stable across edits. This matters because +renaming a flow forces a full refresh in SDP, so a defined name avoids triggering +one. The string and object forms can be mixed in the same list. + +.. _dataflow-spec-nodester-mv: + +Materialized view targets +========================= + +A materialized view is a target node with ``table_type: "mv"``. It can be defined +by inline SQL or by chaining a source node into it: + +.. code-block:: json + + { + "name": "target_customer_summary", + "node_type": "target", + "config": { + "table": "customer_summary", + "table_type": "mv", + "sql_statement": "SELECT STATE, count(*) AS n FROM live.customer_silver GROUP BY STATE" + } + } + +To feed a materialized view from a source node, chain it via ``input`` (an inline +``source_view`` on the target is **not** supported): + +.. code-block:: json + + { + "name": "v_mv_source", + "node_type": "source", + "source_type": "delta", + "config": { "database": "{staging_schema}", "table": "customer", "mode": "batch" } + }, + { + "name": "target_customer_mv", + "node_type": "target", + "config": { + "table": "customer_mv", + "table_type": "mv", + "input": ["v_mv_source"] + } + } + +A single spec may contain both streaming-table and materialized-view targets, +including chains where one feeds the other, so a pipeline that mixes the two does +not have to be split across separate specs. + +.. _dataflow-spec-nodester-conversion: + +How Nodester specs are converted +================================ + +1. The terminal target (not consumed by any other node) becomes ``targetDetails``; + its CDC, data quality, and quarantine settings move to the spec level. +2. Other targets become ``stagingTables``, each with their own settings. +3. Source nodes become views; an internal source (one that reads a table produced + by a target in the same spec) references that table directly. +4. Transformation nodes become SQL or Python views. +5. Each ``input`` entry becomes a flow into its target. The flow type is + ``merge`` when the target has CDC, otherwise ``append_view`` (or ``append_sql`` + for an inline SQL source). +6. Each ``table_type: "mv"`` target becomes its own materialized-view flow spec. + +Comparison with other spec types +================================ + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - **Aspect** + - **Standard** + - **Flow** + - **Nodester** + * - **Paradigm** + - Single source → target + - Target-table-driven with flow groups + - Node graph + * - **Multiple sources** + - No + - Yes (via views) + - Yes (via source nodes) + * - **Intermediate tables** + - No + - Yes (staging tables) + - Yes (non-terminal targets) + * - **Streaming tables + materialized views in one spec** + - No + - No + - Yes diff --git a/docs/source/dataflow_spec_reference.rst b/docs/source/dataflow_spec_reference.rst index 2b38678..a555434 100644 --- a/docs/source/dataflow_spec_reference.rst +++ b/docs/source/dataflow_spec_reference.rst @@ -16,4 +16,5 @@ A Data Flow Spec is a JSON file that defines the structure of a single data flow dataflow_spec_ref_main_standard dataflow_spec_ref_main_flows - dataflow_spec_ref_main_materialized_views \ No newline at end of file + dataflow_spec_ref_main_materialized_views + dataflow_spec_ref_main_nodester \ No newline at end of file diff --git a/samples/deploy.sh b/samples/deploy.sh index e51eec7..ea757c5 100755 --- a/samples/deploy.sh +++ b/samples/deploy.sh @@ -40,6 +40,9 @@ echo "" # Deploy Pattern Samples (end-to-end medallion patterns: bronze → silver → gold) ./deploy_pattern_samples.sh -u "$user" -h "$host" -p "$profile" -c "$compute" -l "$logical_env" --catalog "$catalog" --schema_namespace "$schema_namespace" +# Deploy Nodester Samples (node-based dataflow spec) +./deploy_nodester.sh -u "$user" -h "$host" -p "$profile" -c "$compute" -l "$logical_env" --catalog "$catalog" --schema_namespace "$schema_namespace" + # Note: Kafka samples are deployed as part of feature-samples but require external Kafka infrastructure. # Run ./deploy_feature_samples.sh separately and then trigger the kafka_samples_run_job manually. # Note: TPCH sample is deployed separately via ./deploy_tpch.sh diff --git a/samples/deploy_and_test.sh b/samples/deploy_and_test.sh index 6c76620..1b3e9e0 100755 --- a/samples/deploy_and_test.sh +++ b/samples/deploy_and_test.sh @@ -144,6 +144,31 @@ for ((i=1; i<=num_runs; i++)); do echo "" done +cd "$SCRIPT_DIR" || exit 1 + +# Step 4: Execute nodester samples run job (reads from staging/bronze created by pattern run 1) +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "Executing Nodester Samples Run Job" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" + +nodester_schema="${schema_namespace}_silver${logical_env}" +setup_bundle_env "Nodester Samples Test Run" "$nodester_schema" + +cd "$SCRIPT_DIR/nodester_sample" || { + log_error "Failed to change directory to nodester_sample" + exit 1 +} + +log_info "Running nodester_samples_run_job..." +if databricks bundle run nodester_samples_run_job -t dev --profile "$profile" 2>&1; then + log_success "Nodester samples run completed successfully" +else + log_error "Nodester samples run failed" + exit 1 +fi + cd "$SCRIPT_DIR" || exit 1 unset MSYS_NO_PATHCONV diff --git a/samples/deploy_nodester.sh b/samples/deploy_nodester.sh new file mode 100755 index 0000000..040e3b6 --- /dev/null +++ b/samples/deploy_nodester.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Nodester Sample Bundle Deployment + +# Sample-specific constants +BUNDLE_NAME="Nodester Sample Bundle" +SCHEMA="${DEFAULT_SCHEMA_NAMESPACE}_silver" + +# Source common library and configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +# Parse command-line arguments +parse_common_args "$@" + +# Prompt for missing parameters +prompt_common_params + +# Validate all required parameters +if ! validate_required_params; then + exit 1 +fi + +# Set schema - use command line if provided, otherwise use constant with logical environment +if [[ -n "$schema_namespace" ]]; then + # Use schema from command line + schema="${schema_namespace}_silver$logical_env" +else + # Use schema constant with logical environment + schema="$SCHEMA$logical_env" +fi + +# Set up bundle environment +setup_bundle_env "$BUNDLE_NAME" "$schema" + +# Update substitutions file with catalog and schema namespace +if ! update_substitutions_file "nodester_sample/src/pipeline_configs/dev_substitutions.json"; then + log_error "Failed to update substitutions file. Exiting." + exit 1 +fi + +# Change to nodester_sample directory for deployment +cd nodester_sample + +# Deploy the bundle +deploy_bundle "$BUNDLE_NAME" + +# Return to parent directory +cd .. + +# Restore original substitutions file +restore_substitutions_file "nodester_sample/src/pipeline_configs/dev_substitutions.json" + diff --git a/samples/destroy.sh b/samples/destroy.sh index 8e69d45..0419a09 100755 --- a/samples/destroy.sh +++ b/samples/destroy.sh @@ -24,6 +24,16 @@ databricks bundle destroy -t dev --profile "$profile" --auto-approve echo "" cd .. +########## +# Destroy Nodester Samples +echo "Destroying Nodester Sample Bundle" +export BUNDLE_VAR_schema="${schema_namespace}_silver${logical_env}" +echo "BUNDLE_VAR_schema: $BUNDLE_VAR_schema" +cd nodester_sample +databricks bundle destroy -t dev --profile "$profile" --auto-approve +echo "" +cd .. + ########## # Destroy Pattern Samples echo "Destroying Pattern Samples Bundle" diff --git a/samples/nodester_sample/.gitignore b/samples/nodester_sample/.gitignore new file mode 100644 index 0000000..d5416f6 --- /dev/null +++ b/samples/nodester_sample/.gitignore @@ -0,0 +1,8 @@ +.databricks/ +build/ +dist/ +__pycache__/ +*.egg-info +.venv/ +scratch/ + diff --git a/samples/nodester_sample/.vscode/settings.json b/samples/nodester_sample/.vscode/settings.json new file mode 100644 index 0000000..3c11d5a --- /dev/null +++ b/samples/nodester_sample/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------" +} + diff --git a/samples/nodester_sample/README.md b/samples/nodester_sample/README.md new file mode 100644 index 0000000..bb33ec9 --- /dev/null +++ b/samples/nodester_sample/README.md @@ -0,0 +1,208 @@ +# Nodester Sample + +This sample demonstrates how to use node-based node-based dataflow specifications with the Lakeflow Framework. + +## Overview + +The Nodester dataflow type (`dataFlowType: "nodester"`) provides compatibility with [node-based](https://github.com/Mmodarre/Lakehouse_Nodester) style specifications. This allows users familiar with Nodester's graph-based approach to define pipelines using source, transformation, and target nodes. + +## Directory Structure + +``` +nodester_sample/ +├── databricks.yml # DABs bundle configuration +├── .gitignore +├── pytest.ini +├── README.md +├── resources/ +│ ├── classic/ # Classic compute pipeline configs +│ │ ├── nodester_base_samples_pipeline.yml +│ │ ├── nodester_customer_simple_pipeline.yml +│ │ ├── nodester_customer_transform_pipeline.yml +│ │ ├── nodester_customer_multi_target_pipeline.yml +│ │ └── nodester_customer_cloudfiles_pipeline.yml +│ └── serverless/ # Serverless pipeline configs +│ ├── nodester_base_samples_pipeline.yml +│ ├── nodester_customer_simple_pipeline.yml +│ ├── nodester_customer_transform_pipeline.yml +│ ├── nodester_customer_multi_target_pipeline.yml +│ └── nodester_customer_cloudfiles_pipeline.yml +├── src/ +│ ├── dataflows/ +│ │ └── base_samples/ +│ │ ├── dataflowspec/ # Nodester-style dataflow specs +│ │ └── schemas/ # Table schemas +│ └── pipeline_configs/ +│ ├── dev_substitutions.json +│ └── global.json +├── scratch/ # For local resource overrides +├── fixtures/ # Test fixtures +└── tests/ # Unit tests +``` + +## Key Concepts + +### Node Types + +1. **Source Nodes**: Define where data comes from (Delta tables, cloud files, Kafka, etc.) +2. **Transformation Nodes**: Define SQL or Python transformations +3. **Target Nodes**: Define where data is written + +### Target-Level Configuration + +All target-specific settings are configured directly on target nodes: + +| Setting | Description | Location | +|---------|-------------|----------| +| `cdcSettings` | CDC/SCD2 merge settings | Target node `config` | +| `cdcSnapshotSettings` | CDC snapshot settings | Target node `config` | +| `dataQualityExpectationsEnabled` | Enable DQ expectations | Target node `config` | +| `dataQualityExpectationsPath` | Path to DQ file | Target node `config` | +| `quarantineMode` | Quarantine mode (off/flag/table) | Target node `config` | +| `quarantineTargetDetails` | Quarantine table config | Target node `config` | + +**Example with CDC settings on target:** + +```json +{ + "id": "target_customer", + "type": "target", + "inputs": ["transform_final"], + "config": { + "table": "customer_final", + "cdcSettings": { + "keys": ["CUSTOMER_ID"], + "scd_type": "2", + "sequence_by": "LOAD_TIMESTAMP" + } + } +} +``` + +### Main Target Selection + +When multiple targets exist, the transformer needs to identify the "main" target: + +- Use `isPrimary: true` on a target to explicitly mark it as main (optional) +- If no `isPrimary` flag is specified, the first target node in the array becomes the main target +- Other targets become staging tables with their own independent settings + +**All target-level settings are now applied per-target** - including CDC, data quality, and quarantine. + +## Sample Files + +### `customer_simple_main.json` +Basic example: single source → single target. No `isPrimary` flag needed for single-target specs. + +### `customer_with_transform_main.json` +Multiple sources with append-merge pattern. Demonstrates: +- Multiple sources appending to a shared staging table +- CDC merge on staging table (configured at target level) +- Final transformation and CDC merge to enriched target + +### `customer_multi_target_main.json` +Demonstrates the staging pattern with: +- Source → Transform → Staging (append) → Transform → Final (CDC merge) +- CDC settings specified on the final target node directly + +### `customer_cloudfiles_main.json` +Example of ingesting from cloud files with transformations. + +## View Naming Convention + +When writing SQL transformations, reference input nodes using the pattern `v_`: + +```sql +SELECT * FROM v_source_customer WHERE status = 'active' +``` + +## Deployment + +### Prerequisites + +1. **Databricks CLI** installed and configured +2. **Framework deployed** to your workspace +3. **Source tables** created (for Delta source examples) + +### Deploy using DABs + +1. Copy the desired pipeline configuration from `resources/classic/` or `resources/serverless/` to `scratch/resources/`: + +```bash +# For classic compute +cp resources/classic/nodester_base_samples_pipeline.yml scratch/resources/ + +# For serverless +cp resources/serverless/nodester_base_samples_pipeline.yml scratch/resources/ +``` + +2. Deploy the bundle: + +```bash +databricks bundle deploy \ + --var catalog= \ + --var schema= \ + --var framework_source_path= \ + --var workspace_host= +``` + +3. Run the pipeline: + +```bash +databricks bundle run lakeflow_samples_nodester_base_pipeline +``` + +### Environment Variables + +| Variable | Description | Example | +|----------|-------------|---------| +| `catalog` | Unity Catalog name | `main` | +| `schema` | Target schema | `silver_dev` | +| `framework_source_path` | Workspace path to framework src | `/Workspace/Repos/user/lakeflow_framework/src` | +| `workspace_host` | Databricks workspace URL | `https://your-workspace.cloud.databricks.com/` | +| `layer` | Pipeline layer (optional) | `silver` | +| `logical_env` | Logical environment (optional) | `dev` | + +## Customization + +### Adding Substitutions + +Edit `src/pipeline_configs/dev_substitutions.json` to add your own substitution values: + +```json +{ + "substitutions": { + "bronze_schema": "bronze_dev", + "silver_schema": "silver_dev", + "catalog": "main" + } +} +``` + +### Creating New Nodester Specs + +1. Create a new `*_main.json` file in `src/dataflows/base_samples/dataflowspec/` +2. Use `dataFlowType: "nodester"` and define your nodes +3. Add corresponding schema files if needed +4. Create a pipeline resource file in `resources/classic/` and `resources/serverless/` + +The Nodester specs are automatically converted to flow specs at runtime, so all framework features (CDC, data quality, secrets, etc.) work seamlessly. + +## Backend Behavior Notes + +### Per-Target Settings +All target-level settings are now applied independently per target: + +| Setting | Main Target | Staging Tables | +|---------|-------------|----------------| +| `cdcSettings` | ✅ Applied | ✅ Applied | +| `cdcSnapshotSettings` | ✅ Applied | ✅ Applied | +| `dataQualityExpectationsEnabled/Path` | ✅ Applied | ✅ Applied | +| `quarantineMode` | ✅ Applied | ✅ Applied | +| `quarantineTargetDetails` | ✅ Applied | ✅ Applied | + +This means you can have different CDC, data quality, and quarantine configurations for different targets in the same dataflow. + +### Table Migration +Table migration (`tableMigrationDetails`) is only supported on the main target table, as it's designed for migrating external tables to the primary output. + diff --git a/samples/nodester_sample/databricks.yml b/samples/nodester_sample/databricks.yml new file mode 100644 index 0000000..d026347 --- /dev/null +++ b/samples/nodester_sample/databricks.yml @@ -0,0 +1,38 @@ +# This is a Databricks asset bundle definition for nodester_sample. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: nodester_sample + +include: + - scratch/resources/*.yml + +variables: + catalog: + description: The target UC catalog + framework_source_path: + description: The full workspace path to the framwework src folder + schema: + description: The target UC schema + workspace_host: + description: workspace url used for API calls from Framework (usually same as deployment URL) e.g. https://e2-demo-field-eng.cloud.databricks.com/ + layer: + description: The target layer + default: silver + logical_env: + description: The logical environment + default: "" + pipeline_cluster_config: + description: Basic cluster config, add node types as necessary + default: + label: default + autoscale: + min_workers: 1 + max_workers: 5 + mode: ENHANCED + +targets: + # The 'dev' target, for development purposes. This target is the default. + dev: + mode: development + default: true + diff --git a/samples/nodester_sample/pytest.ini b/samples/nodester_sample/pytest.ini new file mode 100644 index 0000000..30c5e5d --- /dev/null +++ b/samples/nodester_sample/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +testpaths = tests +pythonpath = src + diff --git a/samples/nodester_sample/resources/classic/nodester_base_samples_pipeline.yml b/samples/nodester_sample/resources/classic/nodester_base_samples_pipeline.yml new file mode 100644 index 0000000..1674b23 --- /dev/null +++ b/samples/nodester_sample/resources/classic/nodester_base_samples_pipeline.yml @@ -0,0 +1,24 @@ +resources: + pipelines: + lakeflow_samples_nodester_base_pipeline: + name: Lakeflow Framework - Nodester - Base Pipeline (${var.logical_env}) + channel: CURRENT + clusters: + - ${var.pipeline_cluster_config} + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + #pipeline.dataFlowIdFilter: nodester_customer_simple,nodester_customer_transform + pipeline.dataFlowGroupFilter: nodester_base_samples + root_path: ${workspace.file_path}/src/dataflows/base_samples + diff --git a/samples/nodester_sample/resources/classic/nodester_customer_cloudfiles_pipeline.yml b/samples/nodester_sample/resources/classic/nodester_customer_cloudfiles_pipeline.yml new file mode 100644 index 0000000..e6d6dc2 --- /dev/null +++ b/samples/nodester_sample/resources/classic/nodester_customer_cloudfiles_pipeline.yml @@ -0,0 +1,23 @@ +resources: + pipelines: + lakeflow_samples_nodester_customer_cloudfiles: + name: Lakeflow Framework - Nodester - Customer CloudFiles (${var.logical_env}) + channel: CURRENT + clusters: + - ${var.pipeline_cluster_config} + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowIdFilter: nodester_customer_cloudfiles + root_path: ${workspace.file_path}/src/dataflows/base_samples + diff --git a/samples/nodester_sample/resources/classic/nodester_customer_multi_target_pipeline.yml b/samples/nodester_sample/resources/classic/nodester_customer_multi_target_pipeline.yml new file mode 100644 index 0000000..5c77ca4 --- /dev/null +++ b/samples/nodester_sample/resources/classic/nodester_customer_multi_target_pipeline.yml @@ -0,0 +1,23 @@ +resources: + pipelines: + lakeflow_samples_nodester_customer_multi_target: + name: Lakeflow Framework - Nodester - Customer Multi Target (${var.logical_env}) + channel: CURRENT + clusters: + - ${var.pipeline_cluster_config} + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowIdFilter: nodester_customer_multi_target + root_path: ${workspace.file_path}/src/dataflows/base_samples + diff --git a/samples/nodester_sample/resources/classic/nodester_customer_simple_pipeline.yml b/samples/nodester_sample/resources/classic/nodester_customer_simple_pipeline.yml new file mode 100644 index 0000000..e5a7c23 --- /dev/null +++ b/samples/nodester_sample/resources/classic/nodester_customer_simple_pipeline.yml @@ -0,0 +1,23 @@ +resources: + pipelines: + lakeflow_samples_nodester_customer_simple: + name: Lakeflow Framework - Nodester - Customer Simple (${var.logical_env}) + channel: CURRENT + clusters: + - ${var.pipeline_cluster_config} + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowIdFilter: nodester_customer_simple + root_path: ${workspace.file_path}/src/dataflows/base_samples + diff --git a/samples/nodester_sample/resources/classic/nodester_customer_transform_pipeline.yml b/samples/nodester_sample/resources/classic/nodester_customer_transform_pipeline.yml new file mode 100644 index 0000000..aeab199 --- /dev/null +++ b/samples/nodester_sample/resources/classic/nodester_customer_transform_pipeline.yml @@ -0,0 +1,23 @@ +resources: + pipelines: + lakeflow_samples_nodester_customer_transform: + name: Lakeflow Framework - Nodester - Customer Transform (${var.logical_env}) + channel: CURRENT + clusters: + - ${var.pipeline_cluster_config} + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowIdFilter: nodester_customer_transform + root_path: ${workspace.file_path}/src/dataflows/base_samples + diff --git a/samples/nodester_sample/resources/serverless/jobs/nodester_samples_run_job.yml b/samples/nodester_sample/resources/serverless/jobs/nodester_samples_run_job.yml new file mode 100644 index 0000000..ce8ce29 --- /dev/null +++ b/samples/nodester_sample/resources/serverless/jobs/nodester_samples_run_job.yml @@ -0,0 +1,63 @@ +resources: + jobs: + nodester_samples_run_job: + name: Lakeflow Framework - Nodester Samples - Run (${var.logical_env}) + tasks: + # Base samples — individual pipelines + - task_key: nodester_customer_simple + pipeline_task: + pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_customer_simple.id} + full_refresh: true + + - task_key: nodester_customer_cloudfiles + pipeline_task: + pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_customer_cloudfiles.id} + full_refresh: true + + - task_key: nodester_customer_transform + pipeline_task: + pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_customer_transform.id} + full_refresh: true + + - task_key: nodester_customer_multi_target + pipeline_task: + pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_customer_multi_target.id} + full_refresh: true + + # Note: nodester_base_pipeline (combined) is excluded because individual + # base sample specs share view names which causes SDP "Cannot redefine" errors + # when loaded into a single pipeline. The individual pipelines above cover all base samples. + + # Feature samples — all run in parallel + - task_key: nodester_feature_general + pipeline_task: + pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_general_pipeline.id} + full_refresh: true + + - task_key: nodester_feature_python + pipeline_task: + pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_python_pipeline.id} + full_refresh: true + + - task_key: nodester_feature_data_quality + pipeline_task: + pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_data_quality_pipeline.id} + full_refresh: true + + - task_key: nodester_feature_snapshots + pipeline_task: + pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_snapshots_pipeline.id} + full_refresh: true + + - task_key: nodester_feature_table_migration + pipeline_task: + pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_table_migration_pipeline.id} + full_refresh: true + + - task_key: nodester_feature_materialized_views + pipeline_task: + pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_materialized_views_pipeline.id} + full_refresh: true + + queue: + enabled: true diff --git a/samples/nodester_sample/resources/serverless/nodester_base_samples_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_base_samples_pipeline.yml new file mode 100644 index 0000000..152eaff --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_base_samples_pipeline.yml @@ -0,0 +1,23 @@ +resources: + pipelines: + lakeflow_samples_nodester_base_pipeline: + name: Lakeflow Framework - Nodester - Base Pipeline (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + #pipeline.dataFlowIdFilter: nodester_customer_simple,nodester_customer_transform + pipeline.dataFlowGroupFilter: nodester_base_samples + root_path: ${workspace.file_path}/src/dataflows/base_samples + diff --git a/samples/nodester_sample/resources/serverless/nodester_customer_cloudfiles_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_customer_cloudfiles_pipeline.yml new file mode 100644 index 0000000..9fcfd60 --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_customer_cloudfiles_pipeline.yml @@ -0,0 +1,22 @@ +resources: + pipelines: + lakeflow_samples_nodester_customer_cloudfiles: + name: Lakeflow Framework - Nodester - Customer CloudFiles (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowIdFilter: nodester_customer_cloudfiles + root_path: ${workspace.file_path}/src/dataflows/base_samples + diff --git a/samples/nodester_sample/resources/serverless/nodester_customer_multi_target_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_customer_multi_target_pipeline.yml new file mode 100644 index 0000000..d747bc3 --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_customer_multi_target_pipeline.yml @@ -0,0 +1,22 @@ +resources: + pipelines: + lakeflow_samples_nodester_customer_multi_target: + name: Lakeflow Framework - Nodester - Customer Multi Target (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowIdFilter: nodester_customer_multi_target + root_path: ${workspace.file_path}/src/dataflows/base_samples + diff --git a/samples/nodester_sample/resources/serverless/nodester_customer_simple_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_customer_simple_pipeline.yml new file mode 100644 index 0000000..3cc5282 --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_customer_simple_pipeline.yml @@ -0,0 +1,22 @@ +resources: + pipelines: + lakeflow_samples_nodester_customer_simple: + name: Lakeflow Framework - Nodester - Customer Simple (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowIdFilter: nodester_customer_simple + root_path: ${workspace.file_path}/src/dataflows/base_samples + diff --git a/samples/nodester_sample/resources/serverless/nodester_customer_transform_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_customer_transform_pipeline.yml new file mode 100644 index 0000000..9a9d5fc --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_customer_transform_pipeline.yml @@ -0,0 +1,22 @@ +resources: + pipelines: + lakeflow_samples_nodester_customer_transform: + name: Lakeflow Framework - Nodester - Customer Transform (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowIdFilter: nodester_customer_transform + root_path: ${workspace.file_path}/src/dataflows/base_samples + diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_data_quality_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_feature_samples_data_quality_pipeline.yml new file mode 100644 index 0000000..47796e4 --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_feature_samples_data_quality_pipeline.yml @@ -0,0 +1,20 @@ +resources: + pipelines: + lakeflow_samples_nodester_feature_data_quality_pipeline: + name: Lakeflow Framework - Nodester - Feature Samples Data Quality Pipeline (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowGroupFilter: nodester_feature_samples_data_quality + root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_general_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_feature_samples_general_pipeline.yml new file mode 100644 index 0000000..3aa9d13 --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_feature_samples_general_pipeline.yml @@ -0,0 +1,20 @@ +resources: + pipelines: + lakeflow_samples_nodester_feature_general_pipeline: + name: Lakeflow Framework - Nodester - Feature Samples General Pipeline (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowGroupFilter: nodester_feature_samples_general + root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_materialized_views_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_feature_samples_materialized_views_pipeline.yml new file mode 100644 index 0000000..2c1107f --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_feature_samples_materialized_views_pipeline.yml @@ -0,0 +1,20 @@ +resources: + pipelines: + lakeflow_samples_nodester_feature_materialized_views_pipeline: + name: Lakeflow Framework - Nodester - Feature Samples Materialized Views Pipeline (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowGroupFilter: nodester_feature_samples_materialized_views + root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_python_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_feature_samples_python_pipeline.yml new file mode 100644 index 0000000..c15a961 --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_feature_samples_python_pipeline.yml @@ -0,0 +1,20 @@ +resources: + pipelines: + lakeflow_samples_nodester_feature_python_pipeline: + name: Lakeflow Framework - Nodester - Feature Samples Python Pipeline (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowGroupFilter: nodester_feature_samples_python + root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_snapshots_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_feature_samples_snapshots_pipeline.yml new file mode 100644 index 0000000..3c6b054 --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_feature_samples_snapshots_pipeline.yml @@ -0,0 +1,20 @@ +resources: + pipelines: + lakeflow_samples_nodester_feature_snapshots_pipeline: + name: Lakeflow Framework - Nodester - Feature Samples Snapshots Pipeline (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowGroupFilter: nodester_feature_samples_snapshots + root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_table_migration_pipeline.yml b/samples/nodester_sample/resources/serverless/nodester_feature_samples_table_migration_pipeline.yml new file mode 100644 index 0000000..345eaaa --- /dev/null +++ b/samples/nodester_sample/resources/serverless/nodester_feature_samples_table_migration_pipeline.yml @@ -0,0 +1,20 @@ +resources: + pipelines: + lakeflow_samples_nodester_feature_table_migration_pipeline: + name: Lakeflow Framework - Nodester - Feature Samples Table Migration Pipeline (${var.logical_env}) + channel: CURRENT + serverless: true + catalog: ${var.catalog} + schema: ${var.schema} + libraries: + - notebook: + path: ${var.framework_source_path}/dlt_pipeline + configuration: + bundle.sourcePath: ${workspace.file_path}/src + bundle.target: ${bundle.target} + framework.sourcePath: ${var.framework_source_path} + workspace.host: ${var.workspace_host} + pipeline.layer: ${var.layer} + logicalEnv: ${var.logical_env} + pipeline.dataFlowGroupFilter: nodester_feature_samples_table_migration + root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json b/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json new file mode 100644 index 0000000..5fcd882 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json @@ -0,0 +1,48 @@ +{ + "data_flow_id": "nodester_customer_cloudfiles", + "data_flow_group": "nodester_base_samples", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_source_files", + "node_type": "source", + "source_type": "cloudFiles", + "config": { + "path": "{sample_file_location}/customer/", + "mode": "stream", + "schema_path": "customer_source_schema.json", + "reader_options": { + "cloudFiles.format": "json", + "cloudFiles.inferColumnTypes": "true", + "cloudFiles.schemaEvolutionMode": "rescue" + } + } + }, + { + "name": "v_transform_normalize", + "node_type": "transformation", + "transformation_type": "sql", + "output_view_name": "v_normalized_customer", + "config": { + "sql_statement": "SELECT CAST(customer_id AS BIGINT) as CUSTOMER_ID, UPPER(customer_name) as CUSTOMER_NAME, LOWER(email) as EMAIL, TO_DATE(created_date) as CREATED_DATE, current_timestamp() as LOAD_TIMESTAMP FROM STREAM(live.v_source_files)" + } + }, + { + "name": "target_bronze", + "node_type": "target", + "config": { + "table": "customer_bronze_node", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cluster_by_auto": true, + "input": [ + { + "view": "v_transform_normalize", + "flow": "f_transform_normalize" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json b/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json new file mode 100644 index 0000000..cf8b0a8 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json @@ -0,0 +1,91 @@ +{ + "data_flow_id": "nodester_customer_multi_target", + "data_flow_group": "nodester_base_samples", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_source_customer", + "node_type": "source", + "source_type": "delta", + "config": { + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true, + "mode": "stream" + } + }, + { + "name": "v_transform_append", + "node_type": "transformation", + "transformation_type": "sql", + "output_view_name": "v_customer_with_timestamp", + "config": { + "sql_statement": "SELECT * EXCEPT(LOAD_TIMESTAMP), current_timestamp() as LOAD_TIMESTAMP FROM STREAM(live.v_source_customer)" + } + }, + { + "name": "staging_append", + "node_type": "target", + "config": { + "table": "customer_append_staging_node", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cluster_by_auto": true, + "data_quality_expectations_enabled": true, + "data_quality_expectations_path": "customer_staging_dqe.json", + "quarantine_mode": "flag", + "input": [ + { + "flow": "f_append_staging", + "view": "v_transform_append" + } + ] + } + }, + { + "name": "v_transform_merge", + "node_type": "transformation", + "transformation_type": "sql", + "output_view_name": "v_customer_for_merge", + "config": { + "sql_statement": "SELECT * FROM STREAM(live.customer_append_staging_node)" + } + }, + { + "name": "target_final", + "node_type": "target", + "config": { + "table": "customer_final_node", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cluster_by_columns": [ + "CUSTOMER_ID" + ], + "comment": "Final customer table with SCD2 history and data quality", + "cdc_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "sequence_by": "LOAD_TIMESTAMP", + "ignore_null_updates": true + }, + "data_quality_expectations_enabled": true, + "data_quality_expectations_path": "customer_final_dqe.json", + "quarantine_mode": "table", + "quarantine_target_details": { + "target_format": "delta", + "table": "customer_final_node_quarantine" + }, + "input": [ + { + "flow": "f_merge_final", + "view": "v_transform_merge" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json b/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json new file mode 100644 index 0000000..f207454 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json @@ -0,0 +1,35 @@ +{ + "data_flow_id": "nodester_customer_simple", + "data_flow_group": "nodester_base_samples", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_source_customer", + "node_type": "source", + "source_type": "delta", + "config": { + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true, + "mode": "stream" + } + }, + { + "name": "target_customer", + "node_type": "target", + "config": { + "table": "customer_silver_node", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cluster_by_auto": true, + "input": [ + { + "flow": "f_customer_silver_ingest", + "view": "v_source_customer" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json b/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json new file mode 100644 index 0000000..760cbdd --- /dev/null +++ b/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json @@ -0,0 +1,44 @@ +{ + "data_flow_id": "nodester_customer_st_with_mv", + "data_flow_group": "nodester_base_samples", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_source_customer_st_mv", + "node_type": "source", + "source_type": "delta", + "config": { + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true, + "mode": "stream" + } + }, + { + "name": "target_customer_streaming", + "node_type": "target", + "config": { + "table": "customer_st_with_mv_node", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cluster_by_auto": true, + "input": [ + { + "view": "v_source_customer_st_mv", + "flow": "f_source_customer_st_mv" + } + ] + } + }, + { + "name": "target_customer_summary", + "node_type": "target", + "config": { + "table": "customer_summary_mv_node", + "table_type": "mv", + "sql_statement": "SELECT count(*) as customer_count, count(CASE WHEN DELETE_FLAG IS NULL OR DELETE_FLAG = false THEN 1 END) as active_customer_count, max(LOAD_TIMESTAMP) as latest_load_timestamp FROM live.customer_st_with_mv_node" + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json b/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json new file mode 100644 index 0000000..71ee799 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json @@ -0,0 +1,143 @@ +{ + "data_flow_id": "nodester_customer_transform", + "data_flow_group": "nodester_base_samples", + "data_flow_type": "nodester", + "tags": { + "layer": "silver", + "source_project": "nodester" + }, + "nodes": [ + { + "name": "v_source_customer", + "node_type": "source", + "source_type": "delta", + "config": { + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true, + "mode": "stream" + } + }, + { + "name": "v_source_customer_address", + "node_type": "source", + "source_type": "delta", + "config": { + "database": "{staging_schema}", + "table": "customer_address", + "cdf_enabled": true, + "mode": "stream" + } + }, + { + "name": "staging_append_customer", + "node_type": "target", + "config": { + "table": "customer_transform_append_node", + "cluster_by_auto": true, + "input": [ + { + "flow": "f_append_customer", + "view": "v_source_customer" + } + ] + } + }, + { + "name": "staging_append_address", + "node_type": "target", + "config": { + "table": "customer_transform_append_node", + "cluster_by_auto": true, + "input": [ + { + "flow": "f_append_address", + "view": "v_source_customer_address" + } + ] + } + }, + { + "name": "v_source_append_cdf", + "node_type": "source", + "source_type": "delta", + "config": { + "table": "customer_transform_append_node", + "cdf_enabled": true, + "mode": "stream" + } + }, + { + "name": "staging_merge", + "node_type": "target", + "config": { + "table": "customer_transform_merge_node", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cluster_by_auto": true, + "cdc_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "sequence_by": "LOAD_TIMESTAMP", + "ignore_null_updates": true, + "scd_type": "2" + }, + "input": [ + { + "flow": "f_merge_append_cdf", + "view": "v_source_append_cdf" + } + ] + } + }, + { + "name": "v_source_merge_cdf", + "node_type": "source", + "source_type": "delta", + "config": { + "table": "customer_transform_merge_node", + "cdf_enabled": true, + "mode": "stream" + } + }, + { + "name": "v_transform_final", + "node_type": "transformation", + "transformation_type": "sql", + "output_view_name": "v_final_transform", + "config": { + "sql_statement": "SELECT CUSTOMER_ID, EMAIL, CITY, STATE, LOAD_TIMESTAMP FROM STREAM(live.v_source_merge_cdf)" + } + }, + { + "name": "target_customer_enriched", + "node_type": "target", + "config": { + "table": "customer_enriched_node", + "schema_path": "customer_enriched_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cluster_by_columns": [ + "CUSTOMER_ID" + ], + "cdc_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "sequence_by": "LOAD_TIMESTAMP", + "ignore_null_updates": true + }, + "input": [ + { + "flow": "f_enrich_final", + "view": "v_transform_final" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/base_samples/expectations/customer_final_dqe.json b/samples/nodester_sample/src/dataflows/base_samples/expectations/customer_final_dqe.json new file mode 100644 index 0000000..08136bc --- /dev/null +++ b/samples/nodester_sample/src/dataflows/base_samples/expectations/customer_final_dqe.json @@ -0,0 +1,21 @@ +{ + "expect": [ + { + "name": "customer_id_positive", + "constraint": "CUSTOMER_ID > 0", + "tag": "Validity" + } + ], + "expect_or_drop": [ + { + "name": "customer_id_not_null", + "constraint": "CUSTOMER_ID IS NOT NULL", + "tag": "Validity" + }, + { + "name": "load_timestamp_valid", + "constraint": "LOAD_TIMESTAMP IS NOT NULL", + "tag": "Completeness" + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/base_samples/expectations/customer_staging_dqe.json b/samples/nodester_sample/src/dataflows/base_samples/expectations/customer_staging_dqe.json new file mode 100644 index 0000000..3fca604 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/base_samples/expectations/customer_staging_dqe.json @@ -0,0 +1,14 @@ +{ + "expect_or_drop": [ + { + "name": "customer_id_not_null", + "constraint": "CUSTOMER_ID IS NOT NULL", + "tag": "Validity" + }, + { + "name": "email_not_empty", + "constraint": "EMAIL IS NOT NULL AND LENGTH(EMAIL) > 0", + "tag": "Completeness" + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_enriched_schema.json b/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_enriched_schema.json new file mode 100644 index 0000000..bfd1dd4 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_enriched_schema.json @@ -0,0 +1,35 @@ +{ + "type": "struct", + "fields": [ + { + "name": "CUSTOMER_ID", + "type": "integer", + "nullable": true, + "metadata": {} + }, + { + "name": "EMAIL", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LOAD_TIMESTAMP", + "type": "timestamp", + "nullable": true, + "metadata": {} + }, + { + "name": "CITY", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "STATE", + "type": "string", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_schema.json b/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_schema.json new file mode 100644 index 0000000..d543bcd --- /dev/null +++ b/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_schema.json @@ -0,0 +1,41 @@ +{ + "type": "struct", + "fields": [ + { + "name": "CUSTOMER_ID", + "type": "integer", + "nullable": true, + "metadata": {} + }, + { + "name": "FIRST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LAST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "EMAIL", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "DELETE_FLAG", + "type": "boolean", + "nullable": true, + "metadata": {} + }, + { + "name": "LOAD_TIMESTAMP", + "type": "timestamp", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_source_schema.json b/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_source_schema.json new file mode 100644 index 0000000..9267d4f --- /dev/null +++ b/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_source_schema.json @@ -0,0 +1,29 @@ +{ + "type": "struct", + "fields": [ + { + "name": "customer_id", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "customer_name", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "email", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "created_date", + "type": "string", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json new file mode 100644 index 0000000..a2bde00 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json @@ -0,0 +1,35 @@ +{ + "data_flow_id": "append_sql_flow", + "data_flow_group": "nodester_feature_samples_general", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_sql_f_customer_address_append_sql", + "node_type": "source", + "source_type": "sql", + "config": { + "sql_statement": "SELECT * FROM STREAM({staging_schema}.customer_address)" + } + }, + { + "name": "target_append_sql_flow", + "node_type": "target", + "config": { + "table": "append_sql_flow", + "schema_path": "target/customer_address_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "data_quality_expectations_enabled": true, + "data_quality_expectations_path": "./customer_address_dqe.json", + "quarantine_mode": "flag", + "input": [ + { + "view": "v_sql_f_customer_address_append_sql", + "flow": "f_sql_f_customer_address_append_sql" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json new file mode 100644 index 0000000..9203c8e --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json @@ -0,0 +1,34 @@ +{ + "data_flow_id": "append_view_flow", + "data_flow_group": "nodester_feature_samples_general", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_append_view_flow", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true + } + }, + { + "name": "target_append_view_flow", + "node_type": "target", + "config": { + "table": "append_view_flow", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "input": [ + { + "view": "v_append_view_flow", + "flow": "f_append_view_flow" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json new file mode 100644 index 0000000..f627f29 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json @@ -0,0 +1,34 @@ +{ + "data_flow_id": "append_view_once_flow", + "data_flow_group": "nodester_feature_samples_general", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_append_view_once_flow", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "batch", + "database": "{staging_schema}", + "table": "customer" + } + }, + { + "name": "target_append_view_once_flow", + "node_type": "target", + "config": { + "table": "append_view_once_flow", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "once": true, + "input": [ + { + "view": "v_append_view_once_flow", + "flow": "f_append_view_once_flow" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json new file mode 100644 index 0000000..73d1077 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json @@ -0,0 +1,35 @@ +{ + "data_flow_id": "feature_constraints", + "data_flow_group": "nodester_feature_samples_general", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_feature_constraints", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true + } + }, + { + "name": "target_feature_constraints", + "node_type": "target", + "config": { + "table": "feature_constraints", + "schema_path": "target/feature_ddl_schema.ddl", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "input": [ + { + "view": "v_feature_constraints", + "flow": "f_feature_constraints" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json new file mode 100644 index 0000000..ef15316 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json @@ -0,0 +1,41 @@ +{ + "data_flow_id": "feature_historical_files_snapshot_datetime", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "target_feature_historical_snapshot_files_datetime", + "node_type": "target", + "config": { + "table": "feature_historical_snapshot_files_datetime", + "schema_path": "target/customer_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "snapshot_type": "historical", + "source_type": "file", + "source": { + "format": "csv", + "path": "{sample_file_location}/snapshot_customer/customer_{version}.csv", + "reader_options": { + "header": "true" + }, + "version_type": "timestamp", + "datetime_format": "%Y_%m_%d" + }, + "track_history_except_column_list": [ + "LOAD_TIMESTAMP" + ] + } + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json new file mode 100644 index 0000000..50361ae --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json @@ -0,0 +1,41 @@ +{ + "data_flow_id": "feature_historical_files_snapshot_datetime_multifile", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "target_feature_historical_snapshot_files_datetime_multifile", + "node_type": "target", + "config": { + "table": "feature_historical_snapshot_files_datetime_multifile", + "schema_path": "target/customer_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "snapshot_type": "historical", + "source_type": "file", + "source": { + "format": "csv", + "path": "{sample_file_location}/snapshot_customer_multifile/customer_{version}_split_{fragment}.csv", + "reader_options": { + "header": "true" + }, + "version_type": "timestamp", + "datetime_format": "%Y_%m_%d" + }, + "track_history_except_column_list": [ + "LOAD_TIMESTAMP" + ] + } + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json new file mode 100644 index 0000000..0740a55 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json @@ -0,0 +1,42 @@ +{ + "data_flow_id": "feature_historical_files_snapshot_datetime_recursive_and_partitioned", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "target_feature_historical_snapshot_files_datetime_recursive_and_partitioned", + "node_type": "target", + "config": { + "table": "feature_historical_snapshot_files_datetime_recursive_and_partitioned", + "schema_path": "target/customer_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "snapshot_type": "historical", + "source_type": "file", + "source": { + "format": "csv", + "path": "{sample_file_location}/snapshot_customer_partitioned/{version}/customer.csv", + "reader_options": { + "header": "true" + }, + "version_type": "timestamp", + "datetime_format": "YEAR=%Y/MONTH=%m/DAY=%d", + "recursiveFileLookup": true + }, + "track_history_except_column_list": [ + "LOAD_TIMESTAMP" + ] + } + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json new file mode 100644 index 0000000..c837332 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json @@ -0,0 +1,40 @@ +{ + "data_flow_id": "feature_historical_files_snapshot_datetime_recursive_and_partitioned_parquet", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "target_feature_historical_snapshot_files_datetime_recursive_and_partitioned_parquet", + "node_type": "target", + "config": { + "table": "feature_historical_snapshot_files_datetime_recursive_and_partitioned_parquet", + "schema_path": "target/customer_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "snapshot_type": "historical", + "source_type": "file", + "source": { + "format": "parquet", + "path": "{sample_file_location}/snapshot_customer_partitioned_parquet/{version}/customer.parquet", + "version_type": "timestamp", + "datetime_format": "YEAR=%Y/MONTH=%m/DAY=%d", + "recursiveFileLookup": true, + "deduplicate_mode": "keys_only" + }, + "track_history_except_column_list": [ + "LOAD_TIMESTAMP" + ] + } + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json new file mode 100644 index 0000000..0b12b14 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json @@ -0,0 +1,42 @@ +{ + "data_flow_id": "feature_historical_files_snapshot_datetime_recursive_and_regex", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "target_feature_historical_snapshot_files_datetime_recursive_and_regex", + "node_type": "target", + "config": { + "table": "feature_historical_snapshot_files_datetime_recursive_and_regex", + "schema_path": "target/customer_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "snapshot_type": "historical", + "source_type": "file", + "source": { + "format": "csv", + "path": "{sample_file_location}/snapshot_customer_regex/(?P.+)/(?P.+)/data/customer_(?P.+).csv", + "reader_options": { + "header": "true" + }, + "version_type": "timestamp", + "datetime_format": "%Y%m%d", + "recursiveFileLookup": true + }, + "track_history_except_column_list": [ + "LOAD_TIMESTAMP" + ] + } + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json new file mode 100644 index 0000000..e9a59a2 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json @@ -0,0 +1,85 @@ +{ + "data_flow_id": "feature_historical_files_snapshot_flow", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "target_stg_feature_historical_files_snapshot_flow", + "node_type": "target", + "config": { + "table": "stg_feature_historical_files_snapshot_flow", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "1", + "snapshot_type": "historical", + "source_type": "file", + "source": { + "format": "csv", + "path": "{sample_file_location}/snapshot_customer/customer_{version}.csv", + "reader_options": { + "header": "true" + }, + "version_type": "timestamp", + "datetime_format": "%Y_%m_%d", + "schema_path": "source/feature_historic_snapshot_customer_schema.json", + "select_exp": [ + "CUSTOMER_ID", + "FIRST_NAME", + "LAST_NAME", + "EMAIL", + "TO_TIMESTAMP(LOAD_TIMESTAMP, 'yyyy-MM-dd HH:mm:ss') AS LOAD_TIMESTAMP" + ] + } + } + } + }, + { + "name": "v_feature_historical_files_snapshot_append", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "live", + "table": "stg_feature_historical_files_snapshot_flow", + "cdf_enabled": true, + "select_exp": [ + "CUSTOMER_ID", + "FIRST_NAME", + "LAST_NAME", + "EMAIL", + "_change_type AS META_CDC_CHANGE_TYPE" + ], + "starting_version_from_dlt_setup": true, + "cdf_change_type_override": [ + "insert", + "update_postimage", + "delete" + ] + } + }, + { + "name": "target_feature_historical_files_snapshot_flow", + "node_type": "target", + "config": { + "table": "feature_historical_files_snapshot_flow", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "input": [ + { + "view": "v_feature_historical_files_snapshot_append", + "flow": "f_feature_historical_files_snapshot_append" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json new file mode 100644 index 0000000..e2ea0a7 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json @@ -0,0 +1,40 @@ +{ + "data_flow_id": "feature_historical_files_snapshot_int", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "target_feature_historical_snapshot_files_int", + "node_type": "target", + "config": { + "table": "feature_historical_snapshot_files_int", + "schema_path": "target/customer_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "snapshot_type": "historical", + "source_type": "file", + "source": { + "format": "csv", + "path": "{sample_file_location}/customer/customer_{version}.csv", + "reader_options": { + "header": "true" + }, + "version_type": "integer" + }, + "track_history_except_column_list": [ + "LOAD_TIMESTAMP" + ] + } + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json new file mode 100644 index 0000000..5ce4383 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json @@ -0,0 +1,48 @@ +{ + "data_flow_id": "feature_historical_files_snapshot_schema_and_select", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "target_feature_historical_snapshot_files_schema_and_select", + "node_type": "target", + "config": { + "table": "feature_historical_snapshot_files_schema_and_select", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "snapshot_type": "historical", + "source_type": "file", + "source": { + "format": "csv", + "path": "{sample_file_location}/snapshot_customer/customer_{version}.csv", + "reader_options": { + "header": "true" + }, + "version_type": "timestamp", + "datetime_format": "%Y_%m_%d", + "schema_path": "source/feature_historic_snapshot_customer_schema.json", + "select_exp": [ + "CUSTOMER_ID", + "FIRST_NAME", + "LAST_NAME", + "EMAIL", + "LOAD_TIMESTAMP" + ] + }, + "track_history_except_column_list": [ + "LOAD_TIMESTAMP" + ] + } + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json new file mode 100644 index 0000000..efd70ef --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json @@ -0,0 +1,37 @@ +{ + "data_flow_id": "feature_historical_table_snapshot_datetime", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "target_feature_historical_snapshot_table_datetime", + "node_type": "target", + "config": { + "table": "feature_historical_snapshot_table_datetime", + "schema_path": "target/feature_historic_snapshot_customer_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "snapshot_type": "historical", + "source_type": "table", + "source": { + "table": "{staging_schema}.customer_historical_snapshot_source", + "version_column": "LOAD_TIMESTAMP", + "version_type": "timestamp" + }, + "track_history_except_column_list": [ + "LOAD_TIMESTAMP" + ] + } + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json new file mode 100644 index 0000000..6fbc6aa --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json @@ -0,0 +1,43 @@ +{ + "data_flow_id": "feature_historical_table_snapshot_select_expression", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "target_feature_historical_snapshot_table_select_expression", + "node_type": "target", + "config": { + "table": "feature_historical_snapshot_table_select_expression", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "snapshot_type": "historical", + "source_type": "table", + "source": { + "table": "{staging_schema}.customer_historical_snapshot_source", + "version_column": "LOAD_TIMESTAMP", + "version_type": "timestamp", + "select_exp": [ + "CUSTOMER_ID", + "FIRST_NAME", + "LAST_NAME", + "EMAIL", + "LOAD_TIMESTAMP" + ] + }, + "track_history_except_column_list": [ + "LOAD_TIMESTAMP" + ] + } + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json new file mode 100644 index 0000000..621bccd --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json @@ -0,0 +1,98 @@ +{ + "data_flow_id": "feature_materialized_views", + "data_flow_group": "nodester_feature_samples_materialized_views", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_feature_mv_source_view", + "node_type": "source", + "source_type": "delta", + "config": { + "database": "{staging_schema}", + "table": "customer", + "mode": "batch" + } + }, + { + "name": "target_feature_mv_source_view", + "node_type": "target", + "config": { + "table": "feature_mv_source_view", + "table_type": "mv", + "input": [ + { + "flow": "f_mv_source_view_load", + "view": "v_feature_mv_source_view" + } + ] + } + }, + { + "name": "target_feature_mv_sql_path", + "node_type": "target", + "config": { + "table": "feature_mv_sql_path", + "table_type": "mv", + "sql_path": "./feature_mv_sql_path.sql" + } + }, + { + "name": "target_feature_mv_sql_statement", + "node_type": "target", + "config": { + "table": "feature_mv_sql_statement", + "table_type": "mv", + "sql_statement": "SELECT * FROM {staging_schema}.customer", + "table_details": { + "private": true + } + } + }, + { + "name": "target_feature_mv_with_quarantine", + "node_type": "target", + "config": { + "table": "feature_mv_with_quarantine", + "table_type": "mv", + "sql_statement": "SELECT * FROM {staging_schema}.customer_address", + "data_quality_expectations_enabled": true, + "data_quality_expectations_path": "./customer_address_dqe.json", + "quarantine_mode": "table", + "quarantine_target_details": { + "target_format": "delta", + "cluster_by_auto": true + } + } + }, + { + "name": "target_feature_mv_chain_mvs", + "node_type": "target", + "config": { + "table": "feature_mv_chain_mvs", + "table_type": "mv", + "sql_statement": "SELECT * FROM live.feature_mv_sql_statement", + "table_details": { + "comment": "Test Config", + "spark_conf": { + "spark.sql.session.timeZone": "Australia/Sydney" + } + } + } + }, + { + "name": "target_feature_mv_with_refresh_policy", + "node_type": "target", + "config": { + "table": "feature_mv_with_refresh_policy", + "table_type": "mv", + "sql_statement": "SELECT * FROM {staging_schema}.customer", + "refresh_policy": "incremental_strict", + "table_details": { + "config_flags": [ + "disableOperationalMetadata" + ] + } + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json new file mode 100644 index 0000000..5636fba --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json @@ -0,0 +1,52 @@ +{ + "data_flow_id": "feature_periodic_snapshot_scd1", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_periodic_snapshot_customer_scd1", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "batch", + "database": "{staging_schema}", + "table": "customer_snapshot_source", + "cdf_enabled": false, + "select_exp": [ + "CUSTOMER_ID", + "FIRST_NAME", + "LAST_NAME", + "EMAIL", + "DELETE_FLAG" + ] + } + }, + { + "name": "target_feature_periodic_snapshot_scd1", + "node_type": "target", + "config": { + "table": "feature_periodic_snapshot_scd1", + "schema_path": "target/feature_periodic_snapshot_customer_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "snapshot_type": "periodic", + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "1" + }, + "input": [ + { + "view": "v_periodic_snapshot_customer_scd1", + "flow": "f_periodic_snapshot_customer_scd1" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json new file mode 100644 index 0000000..f054e27 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json @@ -0,0 +1,52 @@ +{ + "data_flow_id": "feature_periodic_snapshot_scd2", + "data_flow_group": "nodester_feature_samples_snapshots", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_periodic_snapshot_customer_scd2", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "batch", + "database": "{staging_schema}", + "table": "customer_snapshot_source", + "cdf_enabled": false, + "select_exp": [ + "CUSTOMER_ID", + "FIRST_NAME", + "LAST_NAME", + "EMAIL", + "DELETE_FLAG" + ] + } + }, + { + "name": "target_feature_periodic_snapshot_scd2", + "node_type": "target", + "config": { + "table": "feature_periodic_snapshot_scd2", + "schema_path": "target/feature_periodic_snapshot_customer_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "config_flags": [ + "disableOperationalMetadata" + ], + "cdc_snapshot_settings": { + "snapshot_type": "periodic", + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2" + }, + "input": [ + { + "view": "v_periodic_snapshot_customer_scd2", + "flow": "f_periodic_snapshot_customer_scd2" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json new file mode 100644 index 0000000..ca11de2 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json @@ -0,0 +1,36 @@ +{ + "data_flow_id": "feature_python_extension_source", + "data_flow_group": "nodester_feature_samples_python", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_feature_python_extension_source", + "node_type": "source", + "source_type": "python", + "config": { + "mode": "stream", + "python_module": "sources.get_customer_cdf", + "tokens": { + "sourceTable": "{staging_schema}.customer" + } + } + }, + { + "name": "target_feature_python_extension_source", + "node_type": "target", + "config": { + "table": "feature_python_extension_source", + "schema_path": "target/feature_python_function_source_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "input": [ + { + "view": "v_feature_python_extension_source", + "flow": "f_feature_python_extension_source" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json new file mode 100644 index 0000000..952295e --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json @@ -0,0 +1,36 @@ +{ + "data_flow_id": "feature_python_function_source", + "data_flow_group": "nodester_feature_samples_python", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_feature_python_function_source", + "node_type": "source", + "source_type": "python", + "config": { + "mode": "stream", + "function_path": "feature_python_function_source.py", + "tokens": { + "sourceTable": "{staging_schema}.customer" + } + } + }, + { + "name": "target_feature_python_function_source", + "node_type": "target", + "config": { + "table": "feature_python_function_source", + "schema_path": "target/feature_python_function_source_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "input": [ + { + "view": "v_feature_python_function_source", + "flow": "f_feature_python_function_source" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json new file mode 100644 index 0000000..beaae77 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json @@ -0,0 +1,43 @@ +{ + "data_flow_id": "feature_python_extension_transform", + "data_flow_group": "nodester_feature_samples_python", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_source_python_extension_transform", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true + } + }, + { + "name": "v_transform_python_extension", + "node_type": "transformation", + "transformation_type": "python", + "config": { + "python_module": "transforms.customer_aggregation" + } + }, + { + "name": "target_feature_python_extension_transform", + "node_type": "target", + "config": { + "table": "feature_python_extension_transform", + "schema_path": "target/feature_python_function_transform_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "input": [ + { + "view": "v_transform_python_extension", + "flow": "f_transform_python_extension" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json new file mode 100644 index 0000000..52e8ba1 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json @@ -0,0 +1,43 @@ +{ + "data_flow_id": "feature_python_function_transform_transform", + "data_flow_group": "nodester_feature_samples_python", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_source_python_function_transform", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true + } + }, + { + "name": "v_transform_python_function", + "node_type": "transformation", + "transformation_type": "python", + "config": { + "function_path": "feature_python_function_transform.py" + } + }, + { + "name": "target_feature_python_function_transform", + "node_type": "target", + "config": { + "table": "feature_python_function_transform", + "schema_path": "target/feature_python_function_transform_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "input": [ + { + "view": "v_transform_python_function", + "flow": "f_transform_python_function" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json new file mode 100644 index 0000000..3fb6f8b --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json @@ -0,0 +1,44 @@ +{ + "data_flow_id": "feature_quarantine_flag", + "data_flow_group": "nodester_feature_samples_data_quality", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_customer_address_feature_quarantine_flag", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer_address", + "cdf_enabled": true, + "select_exp": [ + "CUSTOMER_ID", + "CITY", + "STATE", + "LOAD_TIMESTAMP" + ] + } + }, + { + "name": "target_feature_quarantine_flag", + "node_type": "target", + "config": { + "table": "feature_quarantine_flag", + "schema_path": "target/customer_address_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "data_quality_expectations_enabled": true, + "data_quality_expectations_path": "./customer_address_dqe.json", + "quarantine_mode": "flag", + "input": [ + { + "view": "v_customer_address_feature_quarantine_flag", + "flow": "f_customer_address_feature_quarantine_flag" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json new file mode 100644 index 0000000..b287e35 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json @@ -0,0 +1,47 @@ +{ + "data_flow_id": "feature_quarantine_table", + "data_flow_group": "nodester_feature_samples_data_quality", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_customer_address_feature_quarantine_table", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer_address", + "cdf_enabled": true, + "select_exp": [ + "CUSTOMER_ID", + "CITY", + "STATE", + "LOAD_TIMESTAMP" + ] + } + }, + { + "name": "target_feature_quarantine_table", + "node_type": "target", + "config": { + "table": "feature_quarantine_table", + "schema_path": "target/customer_address_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "data_quality_expectations_enabled": true, + "data_quality_expectations_path": "./customer_address_dqe.json", + "quarantine_mode": "table", + "quarantine_target_details": { + "target_format": "delta" + }, + "input": [ + { + "view": "v_customer_address_feature_quarantine_table", + "flow": "f_customer_address_feature_quarantine_table" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json new file mode 100644 index 0000000..1eb07f0 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json @@ -0,0 +1,54 @@ +{ + "data_flow_id": "feature_cdc_with_quarantine_table", + "data_flow_group": "nodester_feature_samples", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_customer_address_feature_quarantine_table_cdc", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer_address", + "cdf_enabled": true, + "select_exp": [ + "CUSTOMER_ID", + "CITY", + "STATE", + "LOAD_TIMESTAMP" + ] + } + }, + { + "name": "target_feature_cdc_with_quarantine_table", + "node_type": "target", + "config": { + "table": "feature_cdc_with_quarantine_table", + "schema_path": "target/customer_address_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cdc_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "sequence_by": "LOAD_TIMESTAMP", + "scd_type": "2" + }, + "data_quality_expectations_enabled": true, + "data_quality_expectations_path": "./customer_address_dqe.json", + "quarantine_mode": "table", + "quarantine_target_details": { + "target_format": "delta" + }, + "input": [ + { + "view": "v_customer_address_feature_quarantine_table_cdc", + "flow": "f_customer_address_feature_quarantine_table_cdc" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json new file mode 100644 index 0000000..82b0248 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json @@ -0,0 +1,35 @@ +{ + "data_flow_id": "sink_delta_uc_table", + "data_flow_group": "nodester_feature_samples_general_DISABLED", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_customer_delta_sink_uc_table", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true + } + }, + { + "name": "target_sink_delta_uc_table", + "node_type": "target", + "target_type": "delta_sink", + "config": { + "name": "sink_delta_uc_table", + "sink_options": { + "table_name": "{bronze_schema}.feature_sink_delta_uc_table" + }, + "input": [ + { + "view": "v_customer_delta_sink_uc_table", + "flow": "f_customer_delta_sink_uc_table" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json new file mode 100644 index 0000000..04d604b --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json @@ -0,0 +1,35 @@ +{ + "data_flow_id": "sink_delta_path_table", + "data_flow_group": "nodester_feature_samples_general_DISABLED", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_customer_delta_sink_path_table", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true + } + }, + { + "name": "target_sink_delta_path_table", + "node_type": "target", + "config": { + "name": "sink_delta_path_table", + "sink_options": { + "path": "{staging_volume}/feature_sink_delta_path_table" + }, + "input": [ + { + "view": "v_customer_delta_sink_path_table", + "flow": "f_customer_delta_sink_path_table" + } + ] + }, + "target_type": "delta_sink" + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json new file mode 100644 index 0000000..176afff --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json @@ -0,0 +1,35 @@ +{ + "data_flow_id": "sink_delta_uc_table", + "data_flow_group": "nodester_feature_samples_general_DISABLED", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_customer_delta_sink_uc_table", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true + } + }, + { + "name": "target_sink_delta_uc_table", + "node_type": "target", + "config": { + "name": "sink_delta_uc_table", + "sink_options": { + "table_name": "{bronze_schema}.feature_sink_delta_uc_table" + }, + "input": [ + { + "view": "v_customer_delta_sink_uc_table", + "flow": "f_customer_delta_sink_uc_table" + } + ] + }, + "target_type": "delta_sink" + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json new file mode 100644 index 0000000..dda0a35 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json @@ -0,0 +1,41 @@ +{ + "data_flow_id": "for_each_batch_basic_sql", + "data_flow_group": "nodester_feature_samples_general", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_customer_purchase_basic_sql", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer_purchase", + "cdf_enabled": true + } + }, + { + "name": "target_for_each_batch_basic_sql", + "node_type": "target", + "config": { + "name": "for_each_batch_basic_sql", + "sink_type": "basic_sql", + "sink_config": { + "database": "{staging_schema}", + "table": "feature_foreach_batch_single_basic_sql", + "sql_path": "./customer_purchase_transformed.sql", + "table_properties": { + "delta.enableChangeDataFeed": "true" + } + }, + "input": [ + { + "view": "v_customer_purchase_basic_sql", + "flow": "f_customer_purchase_basic_sql" + } + ] + }, + "target_type": "foreach_batch_sink" + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json new file mode 100644 index 0000000..a3afde2 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json @@ -0,0 +1,41 @@ +{ + "data_flow_id": "for_each_batch_python_function", + "data_flow_group": "nodester_feature_samples_general", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_customer_purchase_python_function", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer_purchase", + "cdf_enabled": true + } + }, + { + "name": "target_for_each_batch_python_function", + "node_type": "target", + "config": { + "name": "for_each_batch_python_function", + "sink_type": "python_function", + "sink_config": { + "function_path": "./feature_foreach_batch_python.py", + "tokens": { + "staging_schema": "{staging_schema}", + "bronze_schema": "{bronze_schema}", + "staging_volume": "{staging_volume}" + } + }, + "input": [ + { + "view": "v_customer_purchase_python_function", + "flow": "f_customer_purchase_python_function" + } + ] + }, + "target_type": "foreach_batch_sink" + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json new file mode 100644 index 0000000..a941d1e --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json @@ -0,0 +1,50 @@ +{ + "data_flow_id": "table_migration_append_only", + "data_flow_group": "nodester_feature_samples_table_migration", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_feature_tm_customer_scd0", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true, + "select_exp": [ + "CUSTOMER_ID", + "FIRST_NAME", + "LAST_NAME", + "EMAIL", + "LOAD_TIMESTAMP" + ] + } + }, + { + "name": "target_feature_migrated_table_append_only", + "node_type": "target", + "config": { + "table": "feature_migrated_table_append_only", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "table_migration_details": { + "catalog_type": "uc", + "enabled": true, + "auto_starting_versions_enabled": true, + "source_details": { + "database": "{bronze_schema}", + "table": "table_to_migrate_scd0" + } + }, + "input": [ + { + "view": "v_feature_tm_customer_scd0", + "flow": "f_feature_tm_customer_scd0" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json new file mode 100644 index 0000000..c6b06ec --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json @@ -0,0 +1,68 @@ +{ + "data_flow_id": "table_migration_scd2", + "data_flow_group": "nodester_feature_samples_table_migration", + "data_flow_type": "nodester", + "nodes": [ + { + "name": "v_feature_tm_customer_scd2", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true, + "select_exp": [ + "CUSTOMER_ID", + "FIRST_NAME", + "LAST_NAME", + "EMAIL", + "LOAD_TIMESTAMP" + ] + } + }, + { + "name": "target_feature_migrated_table_scd2", + "node_type": "target", + "config": { + "table": "feature_migrated_table_scd2", + "table_properties": { + "delta.enableChangeDataFeed": "false" + }, + "cdc_settings": { + "keys": [ + "CUSTOMER_ID" + ], + "scd_type": "2", + "sequence_by": "LOAD_TIMESTAMP", + "except_column_list": [], + "ignore_null_updates": true, + "apply_as_deletes": "" + }, + "table_migration_details": { + "catalog_type": "uc", + "enabled": true, + "auto_starting_versions_enabled": true, + "source_details": { + "database": "{bronze_schema}", + "table": "table_to_migrate_scd2", + "select_exp": [ + "CUSTOMER_ID", + "FIRST_NAME", + "LAST_NAME", + "EMAIL", + "EFFECTIVE_FROM AS __START_AT", + "EFFECTIVE_TO AS __END_AT" + ] + } + }, + "input": [ + { + "view": "v_feature_tm_customer_scd2", + "flow": "f_feature_tm_customer_scd2" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json new file mode 100644 index 0000000..a683302 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json @@ -0,0 +1,80 @@ +{ + "data_flow_id": "version_mapping_flows", + "data_flow_group": "nodester_feature_samples_general", + "data_flow_type": "nodester", + "data_flow_version": "0.1.0", + "nodes": [ + { + "name": "target_feature_version_mapping_stage", + "node_type": "target", + "config": { + "table": "feature_version_mapping_stage", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cdc_apply_changes": { + "keys": [ + "CUSTOMER_ID" + ], + "sequence_by": "LOAD_TIMESTAMP", + "ignore_null_updates": true, + "scd_type": "1" + }, + "input": [ + { + "view": "v_flows_version_mapping", + "flow": "f_flows_version_mapping" + } + ] + } + }, + { + "name": "v_flows_version_mapping", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true + } + }, + { + "name": "v_version_mapping_final", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "live", + "table": "feature_version_mapping_stage", + "cdf_enabled": true + } + }, + { + "name": "target_feature_version_mapping_flows", + "node_type": "target", + "config": { + "table": "feature_version_mapping_flows", + "schema_path": "target/feature_version_mapping_flows_schema.json", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cdc_apply_changes": { + "keys": [ + "CUSTOMER_ID" + ], + "sequence_by": "LOAD_TIMESTAMP", + "where": "", + "ignore_null_updates": false, + "scd_type": "1" + }, + "input": [ + { + "view": "v_version_mapping_final", + "flow": "f_version_mapping_final" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json new file mode 100644 index 0000000..24435aa --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json @@ -0,0 +1,44 @@ +{ + "data_flow_id": "version_mapping", + "data_flow_group": "nodester_feature_samples_general", + "data_flow_type": "nodester", + "data_flow_version": "0.1.0", + "nodes": [ + { + "name": "v_version_mapping_standard", + "node_type": "source", + "source_type": "delta", + "config": { + "mode": "stream", + "database": "{staging_schema}", + "table": "customer", + "cdf_enabled": true + } + }, + { + "name": "target_v_version_mapping_standard", + "node_type": "target", + "config": { + "table": "v_version_mapping_standard", + "table_properties": { + "delta.enableChangeDataFeed": "true" + }, + "cdc_apply_changes": { + "keys": [ + "CUSTOMER_ID" + ], + "sequence_by": "LOAD_TIMESTAMP", + "where": "", + "ignore_null_updates": false, + "scd_type": "1" + }, + "input": [ + { + "view": "v_version_mapping_standard", + "flow": "f_version_mapping_standard" + } + ] + } + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dml/customer_purchase_transformed.sql b/samples/nodester_sample/src/dataflows/feature_samples/dml/customer_purchase_transformed.sql new file mode 100644 index 0000000..9a4e59b --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dml/customer_purchase_transformed.sql @@ -0,0 +1,9 @@ +SELECT * +FROM ( + SELECT customer_id, product, quantity + FROM micro_batch_view +) src +PIVOT ( + SUM(quantity) + FOR product IN ('apples', 'bananas', 'oranges', 'pears') +) \ No newline at end of file diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dml/feature_mv_sql_path.sql b/samples/nodester_sample/src/dataflows/feature_samples/dml/feature_mv_sql_path.sql new file mode 100644 index 0000000..98bc595 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/dml/feature_mv_sql_path.sql @@ -0,0 +1 @@ +SELECT * FROM {staging_schema}.customer \ No newline at end of file diff --git a/samples/nodester_sample/src/dataflows/feature_samples/expectations/customer_address_dqe.json b/samples/nodester_sample/src/dataflows/feature_samples/expectations/customer_address_dqe.json new file mode 100644 index 0000000..f35d49b --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/expectations/customer_address_dqe.json @@ -0,0 +1,15 @@ +{ + "expect_or_drop": [ + { + "name": "PK not null", + "constraint": "CUSTOMER_ID IS NOT NULL", + "tag": "Validity" + }, + { + "name": "enabledTest", + "constraint": "CUSTOMER_ID = 1", + "tag": "Validity", + "enabled": false + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_foreach_batch_python.py b/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_foreach_batch_python.py new file mode 100644 index 0000000..9fd61b9 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_foreach_batch_python.py @@ -0,0 +1,46 @@ +from pyspark.sql import DataFrame, SparkSession +from pyspark.sql import functions as F +from typing import Dict + +def micro_batch_function(df: DataFrame, batch_id: int, tokens: Dict) -> DataFrame: + bronze_schema = tokens["bronze_schema"] + staging_schema = tokens["staging_schema"] + staging_volume = tokens["staging_volume"] + volume_root_file_path = f"/Volumes/{staging_schema}/{staging_volume}".replace(".", "/") + spark = df.sparkSession + + df_transformed = ( + df.groupBy("customer_id") + .pivot("product", ["apples", "bananas", "oranges", "pears"]) + .agg(F.sum("quantity")) + ) + + # Support multiple writes without multiple reads + df_transformed.persist() + + # New UC Delta Table example + table_name = f"{bronze_schema}.feature_foreach_batch_python" + + write_command = df_transformed.write.format("delta") \ + .mode("append") + # You can add partitionBy and clusterBy here + + try: + spark.sql(f"DESCRIBE TABLE {table_name}") + write_command.saveAsTable(table_name) + except Exception: + # Create table if it does not exist + write_command.saveAsTable(table_name) + spark.sql(f"ALTER TABLE {table_name} SET TBLPROPERTIES (delta.enableChangeDataFeed = 'true')") + + # External Delta Table + df_transformed.write \ + .format("delta") \ + .mode("append") \ + .save(f"{volume_root_file_path}/feature_foreach_batch_python/delta_target") + + # JSON file location + df_transformed.write \ + .format("json") \ + .mode("append") \ + .save(f"{volume_root_file_path}/feature_foreach_batch_python/json_target") diff --git a/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_source.py b/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_source.py new file mode 100644 index 0000000..12bd91d --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_source.py @@ -0,0 +1,15 @@ +from pyspark.sql import DataFrame, SparkSession +from pyspark.sql import functions as F +from typing import Dict + +def get_df(spark: SparkSession, tokens: Dict) -> DataFrame: + """ + Get a DataFrame from the source details with applied transformations. + """ + source_table = tokens["sourceTable"] + reader_options = { + "readChangeFeed": "true" + } + + df = spark.readStream.options(**reader_options).table(source_table) + return df.withColumn("TEST_COLUMN", F.lit("testing...")) diff --git a/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_transform.py b/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_transform.py new file mode 100644 index 0000000..a4b15fd --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_transform.py @@ -0,0 +1,12 @@ +from pyspark.sql import DataFrame +from pyspark.sql import functions as F + +def apply_transform(df: DataFrame) -> DataFrame: + """ + Apply a transformation to the DataFrame. + """ + return ( + df.withWatermark("load_timestamp", "10 minutes") + .groupBy("CUSTOMER_ID") + .agg(F.count("*").alias("COUNT")) + ) diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/source/feature_historic_snapshot_customer_schema.json b/samples/nodester_sample/src/dataflows/feature_samples/schemas/source/feature_historic_snapshot_customer_schema.json new file mode 100644 index 0000000..1ba5997 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/schemas/source/feature_historic_snapshot_customer_schema.json @@ -0,0 +1,35 @@ +{ + "type": "struct", + "fields": [ + { + "name": "CUSTOMER_ID", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "FIRST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LAST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "EMAIL", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LOAD_TIMESTAMP", + "type": "string", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_address_schema.json b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_address_schema.json new file mode 100644 index 0000000..c8460d9 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_address_schema.json @@ -0,0 +1,29 @@ +{ + "type": "struct", + "fields": [ + { + "name": "CUSTOMER_ID", + "type": "integer", + "nullable": true, + "metadata": {} + }, + { + "name": "CITY", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "STATE", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LOAD_TIMESTAMP", + "type": "timestamp", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_schema.json b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_schema.json new file mode 100644 index 0000000..30b679e --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_schema.json @@ -0,0 +1,41 @@ +{ + "type": "struct", + "fields": [ + { + "name": "CUSTOMER_ID", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "FIRST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LAST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "EMAIL", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "DELETE_FLAG", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LOAD_TIMESTAMP", + "type": "string", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_ddl_schema.ddl b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_ddl_schema.ddl new file mode 100644 index 0000000..d525792 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_ddl_schema.ddl @@ -0,0 +1,8 @@ +CUSTOMER_ID integer NOT NULL, +FIRST_NAME string, +LAST_NAME string, +EMAIL string, +DELETE_FLAG boolean, +LOAD_TIMESTAMP timestamp, +YEAR_TEST INT GENERATED ALWAYS AS (YEAR(LOAD_TIMESTAMP)) +-- CONSTRAINT pk_customer PRIMARY KEY(CUSTOMER_ID) \ No newline at end of file diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_historic_snapshot_customer_schema.json b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_historic_snapshot_customer_schema.json new file mode 100644 index 0000000..a54c2b2 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_historic_snapshot_customer_schema.json @@ -0,0 +1,35 @@ +{ + "type": "struct", + "fields": [ + { + "name": "CUSTOMER_ID", + "type": "integer", + "nullable": true, + "metadata": {} + }, + { + "name": "FIRST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LAST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "EMAIL", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LOAD_TIMESTAMP", + "type": "timestamp", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_periodic_snapshot_customer_schema.json b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_periodic_snapshot_customer_schema.json new file mode 100644 index 0000000..d543bcd --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_periodic_snapshot_customer_schema.json @@ -0,0 +1,41 @@ +{ + "type": "struct", + "fields": [ + { + "name": "CUSTOMER_ID", + "type": "integer", + "nullable": true, + "metadata": {} + }, + { + "name": "FIRST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LAST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "EMAIL", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "DELETE_FLAG", + "type": "boolean", + "nullable": true, + "metadata": {} + }, + { + "name": "LOAD_TIMESTAMP", + "type": "timestamp", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_source_schema.json b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_source_schema.json new file mode 100644 index 0000000..1e17a11 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_source_schema.json @@ -0,0 +1,47 @@ +{ + "type": "struct", + "fields": [ + { + "name": "CUSTOMER_ID", + "type": "integer", + "nullable": true, + "metadata": {} + }, + { + "name": "FIRST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LAST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "EMAIL", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "DELETE_FLAG", + "type": "boolean", + "nullable": true, + "metadata": {} + }, + { + "name": "TEST_COLUMN", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LOAD_TIMESTAMP", + "type": "timestamp", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_transform_schema.json b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_transform_schema.json new file mode 100644 index 0000000..81366e8 --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_transform_schema.json @@ -0,0 +1,17 @@ +{ + "type": "struct", + "fields": [ + { + "name": "CUSTOMER_ID", + "type": "integer", + "nullable": true, + "metadata": {} + }, + { + "name": "COUNT", + "type": "long", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_version_mapping_flows_schema.json b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_version_mapping_flows_schema.json new file mode 100644 index 0000000..d543bcd --- /dev/null +++ b/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_version_mapping_flows_schema.json @@ -0,0 +1,41 @@ +{ + "type": "struct", + "fields": [ + { + "name": "CUSTOMER_ID", + "type": "integer", + "nullable": true, + "metadata": {} + }, + { + "name": "FIRST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "LAST_NAME", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "EMAIL", + "type": "string", + "nullable": true, + "metadata": {} + }, + { + "name": "DELETE_FLAG", + "type": "boolean", + "nullable": true, + "metadata": {} + }, + { + "name": "LOAD_TIMESTAMP", + "type": "timestamp", + "nullable": true, + "metadata": {} + } + ] +} diff --git a/samples/nodester_sample/src/extensions/sources.py b/samples/nodester_sample/src/extensions/sources.py new file mode 100644 index 0000000..72538eb --- /dev/null +++ b/samples/nodester_sample/src/extensions/sources.py @@ -0,0 +1,31 @@ +""" +Python source extensions for the bronze sample pipeline. + +These functions are loaded via the pythonModule reference in dataflow specs +and are available because the extensions directory is added to sys.path +during pipeline initialization. +""" +from pyspark.sql import DataFrame, SparkSession +from pyspark.sql import functions as F +from typing import Dict + + +def get_customer_cdf(spark: SparkSession, tokens: Dict) -> DataFrame: + """ + Get customer data with Change Data Feed enabled. + + Args: + spark: SparkSession instance + tokens: Dictionary of tokens from the dataflow spec + + Returns: + DataFrame with customer data and a TEST_COLUMN added + """ + source_table = tokens["sourceTable"] + reader_options = { + "readChangeFeed": "true" + } + + df = spark.readStream.options(**reader_options).table(source_table) + return df.withColumn("TEST_COLUMN", F.lit("testing from extension...")) + diff --git a/samples/nodester_sample/src/extensions/transforms.py b/samples/nodester_sample/src/extensions/transforms.py new file mode 100644 index 0000000..0fff8df --- /dev/null +++ b/samples/nodester_sample/src/extensions/transforms.py @@ -0,0 +1,57 @@ +""" +Python transform extensions for the bronze sample pipeline. + +These functions are loaded via the pythonTransform.module reference in dataflow specs +and are available because the extensions directory is added to sys.path +during pipeline initialization. + +Transform functions receive a DataFrame and optionally tokens, and return a DataFrame. +""" +from pyspark.sql import DataFrame +from pyspark.sql import functions as F +from typing import Dict + + +def customer_aggregation(df: DataFrame) -> DataFrame: + """ + Apply customer aggregation transformation. + + Groups by CUSTOMER_ID and counts records within a 10-minute watermark window. + + Args: + df: Input DataFrame with customer data + + Returns: + DataFrame with CUSTOMER_ID and COUNT columns + """ + return ( + df.withWatermark("load_timestamp", "10 minutes") + .groupBy("CUSTOMER_ID") + .agg(F.count("*").alias("COUNT")) + ) + + +def customer_aggregation_with_tokens(df: DataFrame, tokens: Dict) -> DataFrame: + """ + Apply customer aggregation transformation with configurable parameters. + + Args: + df: Input DataFrame with customer data + tokens: Configuration tokens with: + - watermark_column: Column to use for watermark (default: load_timestamp) + - watermark_delay: Watermark delay duration (default: 10 minutes) + - group_by_column: Column to group by (default: CUSTOMER_ID) + + Returns: + DataFrame with grouped counts + """ + watermark_column = tokens.get("watermarkColumn", "load_timestamp") + watermark_delay = tokens.get("watermarkDelay", "10 minutes") + group_by_column = tokens.get("groupByColumn", "CUSTOMER_ID") + + return ( + df.withWatermark(watermark_column, watermark_delay) + .groupBy(group_by_column) + .agg(F.count("*").alias("COUNT")) + ) + diff --git a/samples/nodester_sample/src/pipeline_configs/dev_substitutions.json b/samples/nodester_sample/src/pipeline_configs/dev_substitutions.json new file mode 100644 index 0000000..5ce2264 --- /dev/null +++ b/samples/nodester_sample/src/pipeline_configs/dev_substitutions.json @@ -0,0 +1,13 @@ +{ + "tokens": { + "staging_schema": "main.lakeflow_samples_staging{logical_env}", + "bronze_schema": "main.lakeflow_samples_bronze{logical_env}", + "silver_schema": "main.lakeflow_samples_silver{logical_env}", + "staging_volume": "stg_volume", + "sample_file_location": "/Volumes/main/lakeflow_samples_staging{logical_env}/stg_volume", + "kafka_sink_servers": "your_kafka_sink_server_urls", + "kafka_sink_topic": "your_topic_with_{logical_env}", + "kafka_source_servers": "your_kafka_source_server_urls", + "kafka_source_topic": "your_kafka_source_topic" + } +} diff --git a/samples/nodester_sample/src/pipeline_configs/global.json b/samples/nodester_sample/src/pipeline_configs/global.json new file mode 100644 index 0000000..3554532 --- /dev/null +++ b/samples/nodester_sample/src/pipeline_configs/global.json @@ -0,0 +1,6 @@ +{ + "logLevel": "INFO", + "specFileFormat": "json", + "table_migration_state_volume_path": "/Volumes/main/lakeflow_samples_staging{logical_env}/stg_volume/checkpoint_state" +} + diff --git a/samples/nodester_sample/tests/__init__.py b/samples/nodester_sample/tests/__init__.py new file mode 100644 index 0000000..1028a47 --- /dev/null +++ b/samples/nodester_sample/tests/__init__.py @@ -0,0 +1 @@ +# Nodester Sample Tests diff --git a/samples/nodester_sample/tests/test_placeholder.py b/samples/nodester_sample/tests/test_placeholder.py new file mode 100644 index 0000000..9686711 --- /dev/null +++ b/samples/nodester_sample/tests/test_placeholder.py @@ -0,0 +1,11 @@ +""" +Placeholder tests for the Nodester sample. + +To run integration tests, deploy this bundle and verify the pipelines +execute successfully using the Databricks CLI or UI. +""" + + +def test_placeholder(): + """Placeholder test to verify pytest configuration.""" + assert True diff --git a/samples/pattern-samples/src/notebooks/initialize.ipynb b/samples/pattern-samples/src/notebooks/initialize.ipynb index e4c7d64..1f61618 100644 --- a/samples/pattern-samples/src/notebooks/initialize.ipynb +++ b/samples/pattern-samples/src/notebooks/initialize.ipynb @@ -35,6 +35,7 @@ "silver_schema = f'{catalog}.{schema_namespace}_silver{logical_env}'\n", "gold_schema = f'{catalog}.{schema_namespace}_gold{logical_env}'\n", "dpm_schema = f'{catalog}.{schema_namespace}_dpm{logical_env}'\n", + "yaml_schema = f'{catalog}.{schema_namespace}_yaml{logical_env}'\n", "staging_volume = \"stg_volume\"\n", "\n", "volume_root_file_path = f\"/Volumes/{staging_schema}/{staging_volume}\".replace(\".\", \"/\")\n", diff --git a/samples/pattern-samples/src/notebooks/validate_run_1.ipynb b/samples/pattern-samples/src/notebooks/validate_run_1.ipynb new file mode 100644 index 0000000..135382d --- /dev/null +++ b/samples/pattern-samples/src/notebooks/validate_run_1.ipynb @@ -0,0 +1,606 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run 1 Validation\n", + "\n", + "This notebook validates the output tables after Run 1 (Day 1) data load.\n", + "\n", + "## Expected State After Run 1:\n", + "- **Staging**: Initial customer data loaded (3 customers, 5 addresses, purchases)\n", + "- **Bronze**: Streaming tables populated from staging sources\n", + "- **Silver**: SCD2 tables with initial versions (all active, no history)\n", + "- **Gold**: Dimension tables populated from silver\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%run \"./initialize\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import validation utilities\n", + "from validation_utils import ValidationRunner\n", + "\n", + "# Initialize validation runner with spark session\n", + "v = ValidationRunner(spark)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bronze Layer Validations\n", + "\n", + "### Base Samples Pipeline\n", + "- `customer`: Streaming from staging.customer (3 rows: John, Jane, Richard)\n", + "- `customer_address`: Streaming with data quality, quarantine mode=table\n", + " - 5 rows in staging, but 1 has NULL CUSTOMER_ID which should be quarantined\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=\" * 60)\n", + "print(\"BRONZE LAYER - Base Samples\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Bronze customer table - should have 3 rows (John, Jane, Richard)\n", + "v.validate_row_count(f\"{bronze_schema}.customer\", 3, \"Initial customer load\")\n", + "\n", + "# Validate customer IDs present\n", + "v.validate_values_exist(f\"{bronze_schema}.customer\", \"CUSTOMER_ID\", [1, 2, 10], \"Customer IDs\")\n", + "\n", + "# Validate specific customer data\n", + "v.validate_column_value(\n", + " f\"{bronze_schema}.customer\",\n", + " \"CUSTOMER_ID = 1\",\n", + " \"EMAIL\",\n", + " \"john.doe@example.com\",\n", + " \"John's email\"\n", + ")\n", + "\n", + "# Bronze customer_address - should have 4 valid rows (NULL CUSTOMER_ID quarantined)\n", + "v.validate_row_count(f\"{bronze_schema}.customer_address\", 4, \"Valid addresses (1 quarantined)\")\n", + "\n", + "# Quarantine table should have 1 row (NULL CUSTOMER_ID)\n", + "v.validate_quarantine_count(f\"{bronze_schema}.customer_address_quarantine\", 1)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Feature Samples - Data Quality Pipeline\n", + "\n", + "- `feature_quarantine_table`: Same source as customer_address, quarantine mode=table\n", + "- `feature_quarantine_flag`: Quarantine mode=flag (adds __IS_QUARANTINED column)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Samples (Data Quality)\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Feature quarantine table - same as customer_address\n", + "v.validate_row_count(f\"{bronze_schema}.feature_quarantine_table\", 4, \"Valid records in quarantine table feature\")\n", + "v.validate_quarantine_count(f\"{bronze_schema}.feature_quarantine_table_quarantine\", 1)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Feature Samples - Snapshots Pipeline\n", + "\n", + "- Historical snapshots from files and tables\n", + "- Periodic snapshot SCD2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Samples (Snapshots)\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Historical snapshot from table - processes all historical snapshots\n", + "# Initial load has data with 3 timestamps (2024-01-01, 2024-01-04, 2024-02-10)\n", + "# SCD2 should track changes across snapshots\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_table_datetime\", \n", + " 3, # At minimum 3 customers from initial snapshot\n", + " \"Historical snapshot from table\"\n", + ")\n", + "\n", + "# Periodic snapshot SCD2 - initial snapshot with 3 customers\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.feature_periodic_snapshot_scd2\",\n", + " 3,\n", + " \"__END_AT\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Silver Layer Validations\n", + "\n", + "### Base Samples Pipeline\n", + "- `customer`: SCD2 with CDC from bronze.customer\n", + "- `customer_address`: SCD2 with CDC from bronze.customer_address\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"SILVER LAYER - Base Samples\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Silver customer - SCD2, all records should be active (no changes yet)\n", + "v.validate_row_count(f\"{silver_schema}.customer\", 3, \"Initial SCD2 records\")\n", + "v.validate_active_scd2_count(f\"{silver_schema}.customer\", 3)\n", + "v.validate_closed_scd2_count(f\"{silver_schema}.customer\", 0)\n", + "\n", + "# Silver customer_address - SCD2, 4 valid records from bronze (quarantine filtered)\n", + "v.validate_row_count(f\"{silver_schema}.customer_address\", 4, \"Initial SCD2 records\")\n", + "v.validate_active_scd2_count(f\"{silver_schema}.customer_address\", 4)\n", + "v.validate_closed_scd2_count(f\"{silver_schema}.customer_address\", 0)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Multi-Source Streaming Pipeline\n", + "\n", + "- `customer_ms_basic`: Merges customer and customer_address streams\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"SILVER LAYER - Multi-Source Streaming\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Multi-source streaming combines customer and customer_address\n", + "# Customer IDs: 1, 2, 10 from customer + 1, 2, 4, 10 from address (deduplicated)\n", + "# Unique IDs: 1, 2, 4, 10 = 4 active records\n", + "v.validate_active_scd2_count(\n", + " f\"{silver_schema}.customer_ms_basic\",\n", + " 4, # Unique customer IDs from both sources\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"OPERATIONAL METADATA - meta_load_details Validation\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Validate meta_load_details nested column fields are not null\n", + "# Using wildcard to check all fields in the struct\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer meta_load_details\"\n", + ")\n", + "\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer_address\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer_address meta_load_details\"\n", + ")\n", + "\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer_ms_basic\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer_ms_basic meta_load_details\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Gold Layer Validations\n", + "\n", + "### Stream-Static Pipeline\n", + "- `dim_customer_sql_sample`: Dimension built from silver customer data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"GOLD LAYER - Stream-Static\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Gold dimension - should have SCD2 records from silver\n", + "v.validate_active_scd2_count(\n", + " f\"{gold_schema}.dim_customer_sql_sample\",\n", + " 4, # Combined unique keys from customer and customer_address\n", + " \"__END_AT\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## YAML Sample Validations\n", + "\n", + "The YAML sample pipeline mirrors the bronze base samples but uses YAML format\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"YAML SAMPLE\")\n", + "print(\"=\" * 60)\n", + "\n", + "# YAML customer - same as bronze.customer\n", + "v.validate_row_count(f\"{yaml_schema}.customer\", 3, \"YAML customer table\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feature General Pipeline Validations\n", + "\n", + "### Append Flows\n", + "- `append_sql_flow`: All 5 customer_address rows inserted (quarantineMode=flag, 1 row gets __IS_QUARANTINED=true)\n", + "- `append_view_flow`: All 3 customer rows appended from a view\n", + "- `feature_constraints`: 3 rows from customer with DDL schema/constraints applied\n", + "- `feature_version_mapping_flows`: SCD1 with staging table pattern - 3 rows\n", + "- `v_version_mapping_standard`: SCD1 standard pattern - 3 rows\n", + "\n", + "### Materialized Views\n", + "- `feature_mv_source_view`: 3 rows from customer (view-based MV)\n", + "- `feature_mv_sql_path`: 3 rows from customer (inline SQL MV)\n", + "- `feature_mv_chain_mvs`: 3 rows (chained MV)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Template Samples Pipeline\")\n", + "print(\"=\" * 60)\n", + "\n", + "# customer_template: SCD2 from snapshot files via template\n", + "# 3 snapshot files loaded in Run 1: 2024-01-01 (John, Jane), 2024-01-04 (John jdoe@, Alice, Joe), 2024-02-10 (John jdoe@, Alice, Joe, Sarah)\n", + "# SCD2: John has 2 versions (email changed), Jane removed from later snapshots\n", + "# Actual: 5 active in Run 1\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.customer_template\",\n", + " 5,\n", + " \"Template customer SCD2 (at least 5 rows after 3 snapshot files)\"\n", + ")\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.customer_template\",\n", + " 5,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# customer_address_template: SCD2 from customer_address snapshot files\n", + "# 3 snapshot files: 2024-01-01 (2 rows), 2024-01-04 (3 rows, ID 1 still Melbourne), 2024-02-10 (4 rows, ID 1 moves to Sydney)\n", + "# SCD2: at least 4 unique IDs, ID 1 has 2 versions\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.customer_address_template\",\n", + " 4,\n", + " \"Template customer_address SCD2 (at least 4 rows from snapshot files)\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bronze Template Samples Pipeline Validations\n", + "\n", + "The template samples pipeline instantiates `cdc_stream_from_snapshot_template` for two parameter sets:\n", + "- `customer_template`: SCD2 from customer snapshot CSV files (historical, datetime-versioned)\n", + " - Run 1 loads 3 snapshot files → SCD2 result with at least 2 active customers\n", + "- `customer_address_template`: SCD2 from customer_address snapshot CSV files\n", + " - Run 1 loads 3 snapshot files with address changes across time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"SILVER LAYER - DPM Pipeline\")\n", + "print(\"=\" * 60)\n", + "\n", + "# customer_dpm: SCD2, merges bronze.customer + bronze.customer_address\n", + "# Unique customer IDs from both streams: 1, 2, 4, 10 = 4 active\n", + "v.validate_active_scd2_count(\n", + " f\"{dpm_schema}.customer_dpm\",\n", + " 4,\n", + " \"__END_AT\"\n", + ")\n", + "v.validate_closed_scd2_count(\n", + " f\"{dpm_schema}.customer_dpm\",\n", + " 0,\n", + " \"__END_AT\"\n", + ")\n", + "v.validate_values_exist(\n", + " f\"{dpm_schema}.customer_dpm\",\n", + " \"CUSTOMER_ID\",\n", + " [1, 2, 4, 10],\n", + " \"DPM customer IDs (union of customer + address sources)\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Silver DPM Pipeline Validations\n", + "\n", + "- `customer_dpm` (in dpm_schema): SCD2 built from merging bronze.customer (IDs 1, 2, 10) + bronze.customer_address (IDs 1, 2, 4, 10) streams\n", + " - Union of unique customer IDs: 1, 2, 4, 10 = 4 active records\n", + " - No changes yet, so all 4 should be active with no closed records" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Table Migration Pipeline\")\n", + "print(\"=\" * 60)\n", + "\n", + "# feature_migrated_table_scd2: migrated from bronze.table_to_migrate_scd2\n", + "# Actual Run 1 state: 5 active, 1 closed\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", + " 5,\n", + " \"__END_AT\"\n", + ")\n", + "v.validate_closed_scd2_count(\n", + " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", + " 1,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# Validate migrated IDs are present (including ID 40 from the pre-migration table)\n", + "v.validate_values_exist(\n", + " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", + " \"CUSTOMER_ID\",\n", + " [1, 2, 10, 30, 40],\n", + " \"Migrated customer IDs (including 30 and 40 from legacy table)\"\n", + ")\n", + "\n", + "# feature_migrated_table_append_only: migrated from bronze.table_to_migrate_scd0 (3 rows)\n", + "# Streaming from staging.customer then appends new inserts\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_migrated_table_append_only\",\n", + " 3,\n", + " \"Append-only migration (at least 3 migrated rows)\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feature Table Migration Pipeline Validations\n", + "\n", + "- `feature_migrated_table_scd2`: Migrated SCD2 data from `bronze.table_to_migrate_scd2` + streaming CDC from staging.customer\n", + " - Migration source has: IDs 1, 2, 10 (active) + ID 30 (closed same day) + ID 40 (2 rows: 1 closed, 1 active) = 4 active, 2 closed\n", + " - Streaming adds CDC from staging.customer (IDs 1, 2, 10), but autoStartingVersionsEnabled means no duplicate migration data\n", + "- `feature_migrated_table_append_only`: Migrated append-only table from `bronze.table_to_migrate_scd0` (IDs 1, 2, 10) + streaming inserts from staging.customer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Snapshots (Additional Tables)\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Periodic snapshot SCD1: full batch overwrite each run, no history\n", + "# Run 1: customer_snapshot_source has 3 rows (John, Jane, Richard)\n", + "v.validate_row_count(\n", + " f\"{bronze_schema}.feature_periodic_snapshot_scd1\",\n", + " 3,\n", + " \"Periodic snapshot SCD1 (3 customers, no history)\"\n", + ")\n", + "\n", + "# Historical file snapshots (datetime-based versioning from snapshot_customer/ files)\n", + "# Run 1 loads 3 snapshot files: 2024-01-01 (2 rows), 2024-01-04 (3 rows), 2024-02-10 (4 rows)\n", + "# SCD2 result: 4 unique customers, John has 2 versions (email changed), Jane closed after removed from later snapshots\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_files_datetime\",\n", + " 4,\n", + " \"Historical snapshot files (datetime) - at least 4 rows after 3 snapshot files\"\n", + ")\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_files_datetime\",\n", + " 4, # John (jdoe@), Alice, Joe, Sarah still active in latest snapshot\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# Multifile variant: 3 files (2 splits for 2024-01-01, 1 for 2024-12-12)\n", + "# Actual Run 1 state: 3 total rows, 1 active, 2 closed\n", + "v.validate_row_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_files_datetime_multifile\",\n", + " 3,\n", + " \"Historical snapshot multifile (3 total rows)\"\n", + ")\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_files_datetime_multifile\",\n", + " 1,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# Partitioned variant (same data as datetime but in YEAR/MONTH/DAY directories)\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_files_datetime_recursive_and_partitioned\",\n", + " 4,\n", + " \"Historical snapshot partitioned (datetime) - at least 4 rows\"\n", + ")\n", + "\n", + "# Parquet partitioned variant (same data, parquet format)\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_files_datetime_recursive_and_partitioned_parquet\",\n", + " 4,\n", + " \"Historical snapshot partitioned parquet - at least 4 rows\"\n", + ")\n", + "\n", + "# Flow-based historical snapshot: uses staging table + append_view flow pattern\n", + "# Sources from the same snapshot CSV files\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_historical_files_snapshot_flow\",\n", + " 3,\n", + " \"Historical snapshot flow - at least 3 rows (initial customers)\"\n", + ")\n", + "\n", + "# Integer-versioned historical snapshot: uses customer_1.csv and customer_2.csv\n", + "# Run 1 only loads customer_1.csv: John (john.doe@), Jane = 2 rows active\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_files_int\",\n", + " 2,\n", + " \"Historical snapshot files (integer versioning) - at least 2 rows from customer_1.csv\"\n", + ")\n", + "\n", + "# Table-based historical snapshot with select expression\n", + "# Same source as feature_historical_snapshot_table_datetime but with custom selectExp\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_table_select_expression\",\n", + " 3,\n", + " \"Historical snapshot table (select expression) - at least 3 rows\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feature Snapshots Pipeline - Additional Tables\n", + "\n", + "Beyond `feature_periodic_snapshot_scd2` and `feature_historical_snapshot_table_datetime` (validated above), the snapshots pipeline also produces:\n", + "- `feature_periodic_snapshot_scd1`: SCD1 periodic snapshot (no history, just current state)\n", + "- File-based historical snapshots in several variants (datetime, multifile, partitioned, parquet, flow, int, schema_and_select)\n", + "- `feature_historical_snapshot_table_select_expression`: table-based historical snapshot with custom select" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature General Pipeline\")\n", + "print(\"=\" * 60)\n", + "\n", + "# append_sql_flow: quarantineMode=flag, all 5 address rows written (1 gets __IS_QUARANTINED=true)\n", + "v.validate_row_count(f\"{bronze_schema}.append_sql_flow\", 5, \"All address rows including quarantine-flagged\")\n", + "\n", + "# append_view_flow: 3 customer rows appended via view\n", + "v.validate_row_count(f\"{bronze_schema}.append_view_flow\", 3, \"Customer rows appended via view\")\n", + "\n", + "# feature_constraints: 3 customer rows with DDL-defined constraints applied\n", + "v.validate_row_count(f\"{bronze_schema}.feature_constraints\", 3, \"Customer rows with DDL constraints\")\n", + "\n", + "# feature_version_mapping_flows: SCD1 from customer via staging table flow\n", + "# Run 1: 3 customers (John, Jane, Richard)\n", + "v.validate_row_count(f\"{bronze_schema}.feature_version_mapping_flows\", 3, \"SCD1 version mapping via flow (3 customers)\")\n", + "\n", + "# v_version_mapping_standard: SCD1 from customer - standard (non-flow) pattern\n", + "v.validate_row_count(f\"{bronze_schema}.v_version_mapping_standard\", 3, \"SCD1 version mapping standard (3 customers)\")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Materialized Views\")\n", + "print(\"=\" * 60)\n", + "\n", + "# feature_mv_source_view: MV backed by a DLT view over staging.customer\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_source_view\", 3, \"Materialized view from source view\")\n", + "\n", + "# feature_mv_sql_path: MV using inline SQL SELECT * FROM staging.customer\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_sql_path\", 3, \"Materialized view from SQL path\")\n", + "\n", + "# feature_mv_chain_mvs: chained materialized view\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_chain_mvs\", 3, \"Chained materialized view\")\n", + "\n", + "# feature_mv_with_quarantine: MV over customer_address with quarantine (table mode)\n", + "# 4 valid rows pass DQ, 1 NULL CUSTOMER_ID goes to quarantine table\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_with_quarantine\", 4, \"MV with quarantine (4 valid address rows)\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validation Summary\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "v.print_summary()\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/samples/pattern-samples/src/notebooks/validate_run_2.ipynb b/samples/pattern-samples/src/notebooks/validate_run_2.ipynb new file mode 100644 index 0000000..c71f794 --- /dev/null +++ b/samples/pattern-samples/src/notebooks/validate_run_2.ipynb @@ -0,0 +1,411 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run 2 Validation\n", + "\n", + "This notebook validates the output tables after Run 2 (Day 2) data load.\n", + "\n", + "## Expected Changes in Run 2:\n", + "- **Customer Updates**:\n", + " - John (ID=1): Email changed from john.doe@example.com to jdoe@example.com\n", + " - Richard (ID=10): Marked for deletion (DELETE_FLAG=1)\n", + " - New customers: Alice (ID=3), Joe (ID=4)\n", + "- **Customer Address Updates**:\n", + " - Jane (ID=2): City changed to Perth, WA\n", + " - New: Alice (ID=3) in Sydney, NSW\n", + "- **SCD2 Behavior**: Previous versions should be closed, new versions created\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%run \"./initialize\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import validation utilities\n", + "from validation_utils import ValidationRunner\n", + "\n", + "# Initialize validation runner with spark session\n", + "v = ValidationRunner(spark)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bronze Layer Validations\n", + "\n", + "After Run 2:\n", + "- `customer`: 7 rows total (3 original + 4 new CDC records)\n", + "- `customer_address`: 6 rows (4 original + 2 new)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=\" * 60)\n", + "print(\"BRONZE LAYER - Base Samples\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Bronze customer - 3 original + 4 new (John update, Alice, Joe, Richard delete marker)\n", + "v.validate_row_count(f\"{bronze_schema}.customer\", 7, \"Total customer CDC records\")\n", + "\n", + "# All customer IDs including new ones\n", + "v.validate_values_exist(f\"{bronze_schema}.customer\", \"CUSTOMER_ID\", [1, 2, 3, 4, 10], \"All customer IDs\")\n", + "\n", + "# Bronze customer_address - 4 original + 2 new (Jane update, Alice new)\n", + "v.validate_row_count(f\"{bronze_schema}.customer_address\", 6, \"Total address CDC records\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Silver Layer Validations\n", + "\n", + "### SCD2 Behavior After Run 2:\n", + "- **customer**: \n", + " - John (ID=1): 1 closed (old email), 1 active (new email)\n", + " - Jane (ID=2): 1 active (unchanged)\n", + " - Alice (ID=3): 1 active (new)\n", + " - Joe (ID=4): 1 active (new)\n", + " - Richard (ID=10): 1 closed (deleted via apply_as_deletes)\n", + " - Total: 4 active, 2 closed = 6 rows\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"SILVER LAYER - Base Samples\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Silver customer SCD2 after Day 2:\n", + "# Active: Jane (unchanged), Alice (new), Joe (new), John (updated) = 4\n", + "# Closed: John (old version), Richard (deleted) = 2\n", + "v.validate_active_scd2_count(f\"{silver_schema}.customer\", 4, \"__END_AT\")\n", + "v.validate_closed_scd2_count(f\"{silver_schema}.customer\", 2, \"__END_AT\")\n", + "\n", + "# Validate John's current email is updated\n", + "v.validate_column_value(\n", + " f\"{silver_schema}.customer\",\n", + " \"CUSTOMER_ID = 1 AND __END_AT IS NULL\",\n", + " \"EMAIL\",\n", + " \"jdoe@example.com\",\n", + " \"John's updated email (active record)\"\n", + ")\n", + "\n", + "# Validate new customers exist\n", + "v.validate_values_exist(f\"{silver_schema}.customer\", \"CUSTOMER_ID\", [1, 2, 3, 4], \"Active customer IDs\")\n", + "\n", + "# Silver customer_address SCD2:\n", + "# Jane (ID=2): 1 closed (Melbourne), 1 active (Perth)\n", + "# Alice (ID=3): 1 active (new)\n", + "# Original 1, 4, 10 still active\n", + "v.validate_min_closed_scd2_count(f\"{silver_schema}.customer_address\", 1, \"__END_AT\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Multi-Source Streaming Pipeline\n", + "\n", + "The multi-source streaming table merges customer and customer_address data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"SILVER LAYER - Multi-Source Streaming\")\n", + "print(\"=\" * 60)\n", + "\n", + "# After Run 2, multi-source should have:\n", + "# Active IDs: 1, 2, 3, 4 (10 was deleted)\n", + "# Should have historical records from changes\n", + "v.validate_active_scd2_count(\n", + " f\"{silver_schema}.customer_ms_basic\",\n", + " 4,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# Should have at least 1 closed record from changes\n", + "v.validate_min_closed_scd2_count(\n", + " f\"{silver_schema}.customer_ms_basic\",\n", + " 1,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"OPERATIONAL METADATA - meta_load_details Validation\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Validate meta_load_details nested column fields are not null\n", + "# Using wildcard to check all fields in the struct\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer meta_load_details\"\n", + ")\n", + "\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer_address\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer_address meta_load_details\"\n", + ")\n", + "\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer_ms_basic\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer_ms_basic meta_load_details\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Gold Layer Validations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"GOLD LAYER - Stream-Static\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Gold dimension should reflect silver layer changes\n", + "v.validate_min_row_count(\n", + " f\"{gold_schema}.dim_customer_sql_sample\",\n", + " 4, # At least 4 rows (active records)\n", + " \"Dimension records\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feature Samples - Snapshots\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Samples (Snapshots)\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Periodic snapshot should show SCD2 changes after snapshot update\n", + "# Run 2 overwrites snapshot source, so periodic snapshot should detect changes\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_periodic_snapshot_scd2\",\n", + " 3, # At least 3 records from initial + any changes\n", + " \"Periodic snapshot records\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validation Summary\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Validation summary runs at the end of the notebook after all checks complete\n", + "pass" + ] + }, + { + "cell_type": "code", + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Snapshots (Additional Tables) Run 2\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Periodic snapshot SCD1: customer_snapshot_source OVERWRITTEN in Run 2 to 5 rows\n", + "# (John jdoe@, Jane, Alice, Joe, Richard) - SCD1 full replacement\n", + "v.validate_row_count(\n", + " f\"{bronze_schema}.feature_periodic_snapshot_scd1\",\n", + " 5,\n", + " \"Periodic snapshot SCD1 (5 customers after snapshot overwrite)\"\n", + ")\n", + "\n", + "# Historical file snapshots (datetime): new snapshot file 2024-03-01 adds customer 6 (Someone)\n", + "# Active now includes customer 6; at least 7 rows total (was 6, now +1 active)\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_files_datetime\",\n", + " 7,\n", + " \"Historical snapshot files (datetime) - at least 7 rows after 4th snapshot\"\n", + ")\n", + "\n", + "# Integer-versioned historical snapshot: customer_2.csv now loaded (John jdoe@, Alice, Joe)\n", + "# SCD2: John gets new version (email changed), Alice/Joe added, customer_1.csv history preserved\n", + "# Active: John (jdoe@), Alice, Joe = 3; Closed: John (john.doe@), Jane = 2; Total >= 5\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_historical_snapshot_files_int\",\n", + " 4,\n", + " \"Historical snapshot int (at least 4 rows after customer_2.csv loaded)\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Table Migration Pipeline Run 2\")\n", + "print(\"=\" * 60)\n", + "\n", + "# feature_migrated_table_scd2: Run 2 adds streaming CDC records (John update, Alice, Joe)\n", + "# Active: 5 records after Run 2 streaming\n", + "# Closed: at least 1 (from SCD2 version history)\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", + " 5,\n", + " \"__END_AT\"\n", + ")\n", + "v.validate_min_closed_scd2_count(\n", + " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", + " 1,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# feature_migrated_table_append_only: rows accumulated from streaming after Run 2\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_migrated_table_append_only\",\n", + " 3,\n", + " \"Append-only migration (at least 3 rows after Run 2 streaming)\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Template Samples After Run 2\")\n", + "print(\"=\" * 60)\n", + "\n", + "# customer_template: Run 2 adds 2024-03-01 snapshot file with customer 6\n", + "# Active: 6 records after Run 2 snapshot ingestion\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.customer_template\",\n", + " 6,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# customer_address_template: Run 2 adds 2024-03-01 file (Jane Perth, Alice Sydney, Someone Adelaide)\n", + "# ID 2 (Jane) moves to Perth \u2192 new version; new IDs 3 (Sydney), 6 (Adelaide)\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.customer_address_template\",\n", + " 5,\n", + " \"Template customer_address SCD2 (at least 5 rows after Run 2 snapshot)\"\n", + ")" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Feature Snapshots Pipeline - Additional Tables (Run 2)\n", + "\n", + "- `feature_periodic_snapshot_scd1`: customer_snapshot_source OVERWRITTEN to 5 rows (John, Jane, Alice, Joe, Richard)\n", + "- `feature_historical_snapshot_files_datetime`: new snapshot file 2024-03-01 (customer 6) adds a row, total at least 7\n", + "- `feature_historical_snapshot_files_int`: customer_2.csv now loaded: John (jdoe@), Alice, Joe \u2192 at least 4 rows (history + new)\n", + "- `feature_migrated_table_scd2`: John and Richard get new versions; Alice and Joe added" + ], + "metadata": {} + }, + { + "cell_type": "code", + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature General Pipeline\")\n", + "print(\"=\" * 60)\n", + "\n", + "# append_sql_flow: 5 from Run 1 + 2 new (Jane Perth, Alice Sydney) = 7 total\n", + "v.validate_row_count(f\"{bronze_schema}.append_sql_flow\", 7, \"Address rows after Run 2 (+2 new)\")\n", + "\n", + "# append_view_flow: 3 from Run 1 + 4 new customer CDC records = 7 total\n", + "v.validate_row_count(f\"{bronze_schema}.append_view_flow\", 7, \"Customer rows after Run 2 (+4 new CDC)\")\n", + "\n", + "# feature_constraints: 3 original + 4 new = 7 total rows (streaming append from customer CDC)\n", + "v.validate_row_count(f\"{bronze_schema}.feature_constraints\", 7, \"Customer rows with DDL constraints after Run 2\")\n", + "\n", + "# feature_version_mapping_flows: SCD1, updated in-place\n", + "# John updated to jdoe@, Alice + Joe inserted, Richard updated (DELETE_FLAG=True)\n", + "v.validate_row_count(f\"{bronze_schema}.feature_version_mapping_flows\", 5, \"SCD1 version mapping (5 customers after Run 2)\")\n", + "v.validate_column_value(\n", + " f\"{bronze_schema}.feature_version_mapping_flows\",\n", + " \"CUSTOMER_ID = 1\",\n", + " \"EMAIL\",\n", + " \"jdoe@example.com\",\n", + " \"John's updated email in SCD1 table\"\n", + ")\n", + "\n", + "# v_version_mapping_standard: same SCD1 behaviour\n", + "v.validate_row_count(f\"{bronze_schema}.v_version_mapping_standard\", 5, \"SCD1 standard (5 customers after Run 2)\")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Materialized Views After Run 2\")\n", + "print(\"=\" * 60)\n", + "\n", + "# MVs source from staging.customer (7 rows after Run 2: 3 original + 4 new CDC)\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_source_view\", 7, \"MV source view (7 rows after Run 2)\")\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_sql_path\", 7, \"MV SQL path (7 rows after Run 2)\")\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_chain_mvs\", 7, \"Chained MV (7 rows after Run 2)\")\n", + "\n", + "# feature_mv_with_quarantine: sources from staging.customer_address\n", + "# Run 1: 5 rows; Run 2: +2 (Jane Perth, Alice Sydney) = 7 total, 1 NULL quarantined \u2192 6 valid\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_with_quarantine\", 6, \"MV with quarantine (6 valid after Run 2)\")" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "v.print_summary()" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/samples/pattern-samples/src/notebooks/validate_run_3.ipynb b/samples/pattern-samples/src/notebooks/validate_run_3.ipynb new file mode 100644 index 0000000..a48b924 --- /dev/null +++ b/samples/pattern-samples/src/notebooks/validate_run_3.ipynb @@ -0,0 +1,312 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run 3 Validation\n", + "\n", + "This notebook validates the output tables after Run 3 (Day 3) data load.\n", + "\n", + "## Expected Changes in Run 3:\n", + "- **Snapshot Source Updates**:\n", + " - customer_snapshot_source is overwritten (John ID=1 removed, only Jane, Alice, Joe, Richard remain)\n", + " - New address for customer ID=1 (Brisbane, QLD)\n", + "- **SCD2 Behavior**: \n", + " - Periodic snapshots should detect the removal of John from snapshot\n", + " - Customer address should show new version for ID=1\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%run \"./initialize\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import validation utilities\n", + "from validation_utils import ValidationRunner\n", + "\n", + "# Initialize validation runner with spark session\n", + "v = ValidationRunner(spark)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bronze Layer Validations\n", + "\n", + "Run 3 only adds 1 new address record for customer ID=1 (Brisbane, QLD)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=\" * 60)\n", + "print(\"BRONZE LAYER - Base Samples\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Bronze customer - no new customer records in Run 3, still 7\n", + "v.validate_row_count(f\"{bronze_schema}.customer\", 7, \"Customer CDC records (no change)\")\n", + "\n", + "# Bronze customer_address - 6 from Run 2 + 1 new (ID=1 Brisbane)\n", + "v.validate_row_count(f\"{bronze_schema}.customer_address\", 7, \"Address CDC records (+1 from Run 3)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Silver Layer Validations\n", + "\n", + "### SCD2 Changes:\n", + "- Customer table unchanged (no new customer data in Run 3)\n", + "- Customer address: ID=1 gets new version (Brisbane), old version (Melbourne) closed\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"SILVER LAYER - Base Samples\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Silver customer - unchanged from Run 2\n", + "v.validate_active_scd2_count(f\"{silver_schema}.customer\", 4, \"__END_AT\")\n", + "\n", + "# Silver customer_address - ID=1 now has new active version\n", + "# Should have at least 2 closed records now (Jane's Melbourne + ID=1's original Melbourne)\n", + "v.validate_min_closed_scd2_count(f\"{silver_schema}.customer_address\", 2, \"__END_AT\")\n", + "\n", + "# Validate ID=1's current city is Brisbane\n", + "v.validate_column_value(\n", + " f\"{silver_schema}.customer_address\",\n", + " \"CUSTOMER_ID = 1 AND __END_AT IS NULL\",\n", + " \"CITY\",\n", + " \"Brisbane\",\n", + " \"Customer 1's updated city (active record)\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"OPERATIONAL METADATA - meta_load_details Validation\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Validate meta_load_details nested column fields are not null\n", + "# Using wildcard to check all fields in the struct\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer meta_load_details\"\n", + ")\n", + "\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer_address\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer_address meta_load_details\"\n", + ")\n", + "\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer_ms_basic\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer_ms_basic meta_load_details\"\n", + ")\n", + "\n", + "# TODO: Add validation for all operational metadata fields. Currently fails on pipeline_update_id sometimes \n", + "# v.validate_column_not_null(\n", + "# f\"{silver_schema}.customer_ms_basic\",\n", + "# \"meta_load_details.*\",\n", + "# \"Silver customer_ms_basic meta_load_details\"\n", + "# )\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Feature Samples - Periodic Snapshot\n", + "\n", + "The periodic snapshot should detect the change in customer_snapshot_source\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Samples (Snapshots)\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Periodic snapshot should have processed another snapshot\n", + "# John (ID=1) was removed from snapshot source in Run 3\n", + "# This should result in a delete operation in SCD2\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_periodic_snapshot_scd2\",\n", + " 4, # At least 4 rows (original + changes from snapshots)\n", + " \"Periodic snapshot records after Run 3\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Gold Layer Validations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"GOLD LAYER - Stream-Static\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Gold dimension should have more history from address changes\n", + "v.validate_min_row_count(\n", + " f\"{gold_schema}.dim_customer_sql_sample\",\n", + " 5, # At least 5 rows including history\n", + " \"Dimension records with history\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validation Summary\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Validation summary runs at the end of the notebook after all checks complete\n", + "pass" + ] + }, + { + "cell_type": "code", + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature General Pipeline (Run 3)\")\n", + "print(\"=\" * 60)\n", + "\n", + "# append_sql_flow: 7 from Run 2 + 1 new (Brisbane QLD for ID=1) = 8\n", + "v.validate_row_count(f\"{bronze_schema}.append_sql_flow\", 8, \"Address rows after Run 3 (+1 Brisbane)\")\n", + "\n", + "# append_view_flow: unchanged - no new customer CDC in Run 3\n", + "v.validate_row_count(f\"{bronze_schema}.append_view_flow\", 7, \"Customer rows after Run 3 (unchanged)\")\n", + "\n", + "# feature_constraints: unchanged\n", + "v.validate_row_count(f\"{bronze_schema}.feature_constraints\", 7, \"Customer rows with DDL constraints (unchanged in Run 3)\")\n", + "\n", + "# feature_version_mapping_flows: SCD1 unchanged in Run 3 (no customer updates)\n", + "v.validate_row_count(f\"{bronze_schema}.feature_version_mapping_flows\", 5, \"SCD1 version mapping (unchanged in Run 3)\")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Snapshots (Additional Tables) Run 3\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Periodic snapshot SCD1: customer_snapshot_source OVERWRITTEN to 4 rows in Run 3\n", + "# John (ID=1) removed from snapshot; remaining: Jane, Alice, Joe, Richard\n", + "v.validate_row_count(\n", + " f\"{bronze_schema}.feature_periodic_snapshot_scd1\",\n", + " 4,\n", + " \"Periodic snapshot SCD1 (4 customers, John removed from snapshot source)\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Table Migration Pipeline Run 3\")\n", + "print(\"=\" * 60)\n", + "\n", + "# feature_migrated_table_scd2: no new customer CDC in Run 3, unchanged from Run 2\n", + "# Active: 5 records (same as Run 2)\n", + "# Closed: at least 1 (from SCD2 version history)\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", + " 5,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# feature_migrated_table_append_only: no new customer inserts in Run 3, still accumulating\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_migrated_table_append_only\",\n", + " 3,\n", + " \"Append-only migration (at least 3 rows, unchanged in Run 3)\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Template Samples After Run 3\")\n", + "print(\"=\" * 60)\n", + "\n", + "# customer_template: no new customer snapshot files in Run 3; unchanged from Run 2\n", + "# Active: 6 records (same as Run 2)\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.customer_template\",\n", + " 6,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# customer_address_template: no new address snapshot files in Run 3; unchanged\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.customer_address_template\",\n", + " 5,\n", + " \"Template customer_address SCD2 (at least 5 rows, unchanged in Run 3)\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Materialized Views After Run 3\")\n", + "print(\"=\" * 60)\n", + "\n", + "# MVs source from staging.customer (unchanged from Run 2: no new customer CDC in Run 3)\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_source_view\", 7, \"MV source view (7 rows, unchanged in Run 3)\")\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_sql_path\", 7, \"MV SQL path (7 rows, unchanged in Run 3)\")\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_chain_mvs\", 7, \"Chained MV (7 rows, unchanged in Run 3)\")\n", + "\n", + "# feature_mv_with_quarantine: sources from staging.customer_address\n", + "# Run 2 had 6 valid; Run 3 adds Brisbane for ID=1 \u2192 7 total rows, 1 NULL quarantined \u2192 7 valid\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_with_quarantine\", 7, \"MV with quarantine (7 valid after Run 3)\")" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "v.print_summary()" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/samples/pattern-samples/src/notebooks/validate_run_4.ipynb b/samples/pattern-samples/src/notebooks/validate_run_4.ipynb new file mode 100644 index 0000000..22952ae --- /dev/null +++ b/samples/pattern-samples/src/notebooks/validate_run_4.ipynb @@ -0,0 +1,340 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run 4 Validation\n", + "\n", + "This notebook validates the output tables after Run 4 (Day 4) data load.\n", + "\n", + "## Expected Changes in Run 4:\n", + "- **Customer Updates**:\n", + " - John (ID=1): Email changed from jdoe@example.com to john.doe@another.example.com\n", + " - DELETE_FLAG set to False (was NULL)\n", + "- **SCD2 Behavior**: \n", + " - John should have another version created in silver.customer\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%run \"./initialize\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import validation utilities\n", + "from validation_utils import ValidationRunner\n", + "\n", + "# Initialize validation runner with spark session\n", + "v = ValidationRunner(spark)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bronze Layer Validations\n", + "\n", + "Run 4 adds 1 new customer record (John's email update)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"=\" * 60)\n", + "print(\"BRONZE LAYER - Base Samples\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Bronze customer - 7 from Run 3 + 1 new (John's email update)\n", + "v.validate_row_count(f\"{bronze_schema}.customer\", 8, \"Customer CDC records (+1 John update)\")\n", + "\n", + "# Bronze customer_address - unchanged from Run 3\n", + "v.validate_row_count(f\"{bronze_schema}.customer_address\", 7, \"Address CDC records (unchanged)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Silver Layer Validations\n", + "\n", + "### SCD2 Changes:\n", + "- John (ID=1): Another version created (email changed to john.doe@another.example.com)\n", + "- Now John has 3 versions: original, jdoe, and john.doe@another.example.com\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"SILVER LAYER - Base Samples\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Silver customer - still 4 active (John, Jane, Alice, Joe)\n", + "# Richard was deleted in Run 2\n", + "v.validate_active_scd2_count(f\"{silver_schema}.customer\", 4, \"__END_AT\")\n", + "\n", + "# Silver customer - should now have 3 closed records:\n", + "# 1. John's original version\n", + "# 2. John's jdoe version\n", + "# 3. Richard (deleted)\n", + "v.validate_min_closed_scd2_count(f\"{silver_schema}.customer\", 3, \"__END_AT\")\n", + "\n", + "# Validate John's current email is the new one\n", + "v.validate_column_value(\n", + " f\"{silver_schema}.customer\",\n", + " \"CUSTOMER_ID = 1 AND __END_AT IS NULL\",\n", + " \"EMAIL\",\n", + " \"john.doe@another.example.com\",\n", + " \"John's latest email (active record)\"\n", + ")\n", + "\n", + "# Customer address unchanged from Run 3\n", + "v.validate_min_closed_scd2_count(f\"{silver_schema}.customer_address\", 2, \"__END_AT\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Gold Layer Validations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"GOLD LAYER - Stream-Static\")\n", + "print(\"=\" * 60)\n", + "\n", + "# TODO - Fix Gold pattern for DWH, currently does not have intended behavior\n", + "# # Gold dimension should have accumulated more history\n", + "# v.validate_min_row_count(\n", + "# f\"{gold_schema}.dim_customer_sql_sample\",\n", + "# 6, # At least 6 rows including all history\n", + "# \"Final dimension records with full history\"\n", + "# )\n", + "\n", + "# # Verify active record count\n", + "# v.validate_active_scd2_count(\n", + "# f\"{gold_schema}.dim_customer_sql_sample\",\n", + "# 4, # 4 active customers (John, Jane, Alice, Joe) - Richard deleted\n", + "# \"__END_AT\"\n", + "# )\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-Source Streaming Final State\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"SILVER LAYER - Multi-Source Streaming Final State\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Multi-source streaming should have accumulated history\n", + "v.validate_active_scd2_count(\n", + " f\"{silver_schema}.customer_ms_basic\",\n", + " 4, # 4 active (John, Jane, Alice, Joe)\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# Should have multiple closed records from all the changes\n", + "v.validate_min_closed_scd2_count(\n", + " f\"{silver_schema}.customer_ms_basic\",\n", + " 2, # At least 2 closed (Richard deleted, plus changes)\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"OPERATIONAL METADATA - meta_load_details Validation\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Validate meta_load_details nested column fields are not null\n", + "# Using wildcard to check all fields in the struct\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer meta_load_details\"\n", + ")\n", + "\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer_address\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer_address meta_load_details\"\n", + ")\n", + "\n", + "v.validate_column_not_null(\n", + " f\"{silver_schema}.customer_ms_basic\",\n", + " \"meta_load_details.pipeline_start_timestamp\",\n", + " \"Silver customer_ms_basic meta_load_details\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validation Summary\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Validation summary runs at the end of the notebook after all checks complete\n", + "pass" + ] + }, + { + "cell_type": "code", + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature General Pipeline (Run 4)\")\n", + "print(\"=\" * 60)\n", + "\n", + "# append_sql_flow: unchanged from Run 3 (no new addresses in Run 4)\n", + "v.validate_row_count(f\"{bronze_schema}.append_sql_flow\", 8, \"Address rows after Run 4 (unchanged)\")\n", + "\n", + "# append_view_flow: 7 from Run 3 + 1 new (John email update) = 8\n", + "v.validate_row_count(f\"{bronze_schema}.append_view_flow\", 8, \"Customer rows after Run 4 (+1 John update)\")\n", + "\n", + "# feature_constraints: 7 + 1 = 8 rows\n", + "v.validate_row_count(f\"{bronze_schema}.feature_constraints\", 8, \"Customer rows with DDL constraints after Run 4\")\n", + "\n", + "# feature_version_mapping_flows: SCD1, John updated to new email\n", + "v.validate_row_count(f\"{bronze_schema}.feature_version_mapping_flows\", 5, \"SCD1 version mapping (5 rows, John updated)\")\n", + "v.validate_column_value(\n", + " f\"{bronze_schema}.feature_version_mapping_flows\",\n", + " \"CUSTOMER_ID = 1\",\n", + " \"EMAIL\",\n", + " \"john.doe@another.example.com\",\n", + " \"John's final email in SCD1 table\"\n", + ")\n", + "\n", + "# v_version_mapping_standard: same SCD1 behaviour\n", + "v.validate_row_count(f\"{bronze_schema}.v_version_mapping_standard\", 5, \"SCD1 standard (5 rows after Run 4)\")\n", + "v.validate_column_value(\n", + " f\"{bronze_schema}.v_version_mapping_standard\",\n", + " \"CUSTOMER_ID = 1\",\n", + " \"EMAIL\",\n", + " \"john.doe@another.example.com\",\n", + " \"John's final email in SCD1 standard\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Snapshots (Additional Tables) Run 4\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Periodic snapshot SCD1: no snapshot source change in Run 4 (same 4 rows as Run 3)\n", + "v.validate_row_count(\n", + " f\"{bronze_schema}.feature_periodic_snapshot_scd1\",\n", + " 4,\n", + " \"Periodic snapshot SCD1 (4 customers, unchanged from Run 3)\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Feature Table Migration Pipeline Run 4\")\n", + "print(\"=\" * 60)\n", + "\n", + "# feature_migrated_table_scd2: Run 4 CDC does not produce additional SCD2 versions in this table\n", + "# Active: still 5 keys; Closed: still 1 (same as Run 2/3)\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", + " 5,\n", + " \"__END_AT\"\n", + ")\n", + "v.validate_min_closed_scd2_count(\n", + " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", + " 1,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# feature_migrated_table_append_only: accumulates rows from streaming CDC\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.feature_migrated_table_append_only\",\n", + " 3,\n", + " \"Append-only migration (at least 3 rows after Run 4)\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Template Samples After Run 4\")\n", + "print(\"=\" * 60)\n", + "\n", + "# customer_template: no new snapshot files in Run 4; unchanged from Run 2\n", + "# Active: 6 records (same as Run 2/3)\n", + "v.validate_active_scd2_count(\n", + " f\"{bronze_schema}.customer_template\",\n", + " 6,\n", + " \"__END_AT\"\n", + ")\n", + "\n", + "# customer_address_template: no new address snapshot files in Run 4; unchanged\n", + "v.validate_min_row_count(\n", + " f\"{bronze_schema}.customer_address_template\",\n", + " 5,\n", + " \"Template customer_address SCD2 (at least 5 rows, unchanged in Run 4)\"\n", + ")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"BRONZE LAYER - Materialized Views After Run 4\")\n", + "print(\"=\" * 60)\n", + "\n", + "# MVs source from staging.customer (8 rows after Run 4: 7 from Run 2/3 + 1 John update)\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_source_view\", 8, \"MV source view (8 rows after Run 4)\")\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_sql_path\", 8, \"MV SQL path (8 rows after Run 4)\")\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_chain_mvs\", 8, \"Chained MV (8 rows after Run 4)\")\n", + "\n", + "# feature_mv_with_quarantine: no new addresses in Run 4 (unchanged from Run 3)\n", + "v.validate_row_count(f\"{bronze_schema}.feature_mv_with_quarantine\", 7, \"MV with quarantine (7 valid, unchanged in Run 4)\")" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "v.print_summary()" + ], + "metadata": {}, + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/samples/pattern-samples/src/notebooks/validation_utils.py b/samples/pattern-samples/src/notebooks/validation_utils.py new file mode 100644 index 0000000..2b5c8e1 --- /dev/null +++ b/samples/pattern-samples/src/notebooks/validation_utils.py @@ -0,0 +1,306 @@ +# Validation helper functions for Lakeflow Framework sample pipelines +from pyspark.sql import DataFrame, SparkSession +from typing import List, Dict, Any, Optional + + +class ValidationResult: + """Represents the result of a validation check.""" + + def __init__(self, test_name: str, passed: bool, message: str, details: Optional[Dict] = None): + self.test_name = test_name + self.passed = passed + self.message = message + self.details = details or {} + + def __repr__(self): + status = "✅ PASS" if self.passed else "❌ FAIL" + return f"{status}: {self.test_name} - {self.message}" + + +class ValidationRunner: + """Runs validation checks and tracks results.""" + + def __init__(self, spark: SparkSession): + self.spark = spark + self.results: List[ValidationResult] = [] + + def validate_row_count(self, table_name: str, expected_count: int, description: str = "") -> ValidationResult: + """Validate that a table has the expected row count.""" + try: + actual_count = self.spark.table(table_name).count() + passed = actual_count == expected_count + message = f"Expected {expected_count} rows, got {actual_count}" + if description: + message = f"{description}: {message}" + result = ValidationResult(f"Row Count: {table_name}", passed, message, {"expected": expected_count, "actual": actual_count}) + except Exception as e: + result = ValidationResult(f"Row Count: {table_name}", False, f"Error: {str(e)}") + self.results.append(result) + print(result) + return result + + def validate_min_row_count(self, table_name: str, min_count: int, description: str = "") -> ValidationResult: + """Validate that a table has at least the minimum row count.""" + try: + actual_count = self.spark.table(table_name).count() + passed = actual_count >= min_count + message = f"Expected at least {min_count} rows, got {actual_count}" + if description: + message = f"{description}: {message}" + result = ValidationResult(f"Min Row Count: {table_name}", passed, message, {"min_expected": min_count, "actual": actual_count}) + except Exception as e: + result = ValidationResult(f"Min Row Count: {table_name}", False, f"Error: {str(e)}") + self.results.append(result) + print(result) + return result + + def validate_active_scd2_count(self, table_name: str, expected_count: int, end_column: str = "__END_AT") -> ValidationResult: + """Validate count of active (current) SCD2 records.""" + try: + df = self.spark.table(table_name) + actual_count = df.filter(f"{end_column} IS NULL").count() + passed = actual_count == expected_count + message = f"Expected {expected_count} active records, got {actual_count}" + result = ValidationResult(f"Active SCD2 Records: {table_name}", passed, message, {"expected": expected_count, "actual": actual_count}) + except Exception as e: + result = ValidationResult(f"Active SCD2 Records: {table_name}", False, f"Error: {str(e)}") + self.results.append(result) + print(result) + return result + + def validate_closed_scd2_count(self, table_name: str, expected_count: int, end_column: str = "__END_AT") -> ValidationResult: + """Validate count of closed (historical) SCD2 records.""" + try: + df = self.spark.table(table_name) + actual_count = df.filter(f"{end_column} IS NOT NULL").count() + passed = actual_count == expected_count + message = f"Expected {expected_count} closed records, got {actual_count}" + result = ValidationResult(f"Closed SCD2 Records: {table_name}", passed, message, {"expected": expected_count, "actual": actual_count}) + except Exception as e: + result = ValidationResult(f"Closed SCD2 Records: {table_name}", False, f"Error: {str(e)}") + self.results.append(result) + print(result) + return result + + def validate_min_closed_scd2_count(self, table_name: str, min_count: int, end_column: str = "__END_AT") -> ValidationResult: + """Validate that at least min_count closed SCD2 records exist.""" + try: + df = self.spark.table(table_name) + actual_count = df.filter(f"{end_column} IS NOT NULL").count() + passed = actual_count >= min_count + message = f"Expected at least {min_count} closed records, got {actual_count}" + result = ValidationResult(f"Min Closed SCD2 Records: {table_name}", passed, message, {"min_expected": min_count, "actual": actual_count}) + except Exception as e: + result = ValidationResult(f"Min Closed SCD2 Records: {table_name}", False, f"Error: {str(e)}") + self.results.append(result) + print(result) + return result + + def validate_values_exist(self, table_name: str, column: str, expected_values: List[Any], description: str = "") -> ValidationResult: + """Validate that specific values exist in a column.""" + try: + df = self.spark.table(table_name) + actual_values = [row[column] for row in df.select(column).distinct().collect()] + missing = [v for v in expected_values if v not in actual_values] + passed = len(missing) == 0 + message = f"All expected values found" if passed else f"Missing values: {missing}" + if description: + message = f"{description}: {message}" + result = ValidationResult(f"Values Exist: {table_name}.{column}", passed, message, {"expected": expected_values, "missing": missing}) + except Exception as e: + result = ValidationResult(f"Values Exist: {table_name}.{column}", False, f"Error: {str(e)}") + self.results.append(result) + print(result) + return result + + def validate_column_value(self, table_name: str, filter_condition: str, column: str, expected_value: Any, description: str = "") -> ValidationResult: + """Validate a specific column value for filtered rows.""" + try: + df = self.spark.table(table_name) + filtered = df.filter(filter_condition) + if filtered.count() == 0: + result = ValidationResult(f"Column Value: {table_name}", False, f"No rows found for filter: {filter_condition}") + else: + actual_value = filtered.select(column).first()[0] + passed = actual_value == expected_value + message = f"Expected '{expected_value}', got '{actual_value}'" + if description: + message = f"{description}: {message}" + result = ValidationResult(f"Column Value: {table_name}.{column}", passed, message) + except Exception as e: + result = ValidationResult(f"Column Value: {table_name}.{column}", False, f"Error: {str(e)}") + self.results.append(result) + print(result) + return result + + def validate_quarantine_count(self, quarantine_table: str, expected_count: int) -> ValidationResult: + """Validate quarantine table row count.""" + return self.validate_row_count(quarantine_table, expected_count, "Quarantine records") + + def _get_column_paths_to_check(self, df: "DataFrame", column_spec: str) -> List[str]: + """ + Resolve a column specification to a list of column paths to check. + + Supports: + - Simple column: "CUSTOMER_ID" -> ["CUSTOMER_ID"] + - Nested column with dot notation: "meta_load_details.record_insert_timestamp" -> ["meta_load_details.record_insert_timestamp"] + - All fields in a struct: "meta_load_details.*" -> ["meta_load_details.field1", "meta_load_details.field2", ...] + + Args: + df: The DataFrame to inspect schema from + column_spec: The column specification string + + Returns: + List of fully qualified column paths to check + """ + from pyspark.sql.types import StructType + + # Handle wildcard for struct expansion + if column_spec.endswith(".*"): + struct_name = column_spec[:-2] # Remove .* + + # Navigate to the struct in the schema + schema = df.schema + parts = struct_name.split(".") + current_type = None + + for i, part in enumerate(parts): + field = schema[part] if i == 0 else current_type[part] + current_type = field.dataType + + if not isinstance(current_type, StructType): + raise ValueError(f"Column '{struct_name}' is not a struct type, cannot use wildcard") + + return [f"{struct_name}.{field.name}" for field in current_type.fields] + + # Handle simple or dot-notation column + return [column_spec] + + def validate_column_not_null( + self, + table_name: str, + columns: "str | List[str]", + description: str = "" + ) -> ValidationResult: + """ + Validate that column(s) do not contain null values. + + This is a generic validation that works with: + - Simple columns: "CUSTOMER_ID" + - Nested columns with dot notation: "meta_load_details.record_insert_timestamp" + - All fields in a struct using wildcard: "meta_load_details.*" + - Multiple columns as a list: ["col1", "meta_load_details.field1", "nested.*"] + + Args: + table_name: The fully qualified table name + columns: Column specification - can be: + - A single column name: "CUSTOMER_ID" + - A nested column path: "meta_load_details.record_insert_timestamp" + - A struct with wildcard to check all fields: "meta_load_details.*" + - A list of any combination of the above + description: Optional description for the validation + + Returns: + ValidationResult indicating if all specified columns have non-null values + + Examples: + # Check a single column + v.validate_column_not_null("schema.table", "CUSTOMER_ID") + + # Check a nested field + v.validate_column_not_null("schema.table", "meta_load_details.record_insert_timestamp") + + # Check all fields in a struct + v.validate_column_not_null("schema.table", "meta_load_details.*") + + # Check multiple columns at once + v.validate_column_not_null("schema.table", ["CUSTOMER_ID", "meta_load_details.*"]) + """ + try: + from pyspark.sql import functions as F + + df = self.spark.table(table_name) + + # Normalize columns to a list + if isinstance(columns, str): + column_specs = [columns] + else: + column_specs = columns + + # Resolve all column specs to actual column paths + all_column_paths = [] + for spec in column_specs: + try: + paths = self._get_column_paths_to_check(df, spec) + all_column_paths.extend(paths) + except Exception as e: + result = ValidationResult( + f"Column Not Null: {table_name}", + False, + f"Error resolving column spec '{spec}': {str(e)}" + ) + self.results.append(result) + print(result) + return result + + # Check each column for null values + null_columns = [] + null_counts = {} + + for col_path in all_column_paths: + null_count = df.filter(F.col(col_path).isNull()).count() + if null_count > 0: + null_columns.append(col_path) + null_counts[col_path] = null_count + + passed = len(null_columns) == 0 + + if passed: + message = f"All {len(all_column_paths)} column(s) have non-null values" + else: + null_details = ", ".join([f"{c}({null_counts[c]} nulls)" for c in null_columns]) + message = f"Found null values in: {null_details}" + + if description: + message = f"{description}: {message}" + + # Create a display name for the columns being checked + col_display = ", ".join(column_specs) if len(column_specs) <= 3 else f"{len(column_specs)} columns" + + result = ValidationResult( + f"Column Not Null: {table_name} [{col_display}]", + passed, + message, + {"columns_checked": all_column_paths, "null_columns": null_columns, "null_counts": null_counts} + ) + except Exception as e: + col_display = columns if isinstance(columns, str) else ", ".join(columns[:3]) + result = ValidationResult( + f"Column Not Null: {table_name} [{col_display}]", + False, + f"Error: {str(e)}" + ) + + self.results.append(result) + print(result) + return result + + def print_summary(self): + """Print validation summary and raise AssertionError if any tests failed.""" + passed = sum(1 for r in self.results if r.passed) + failed = sum(1 for r in self.results if not r.passed) + total = len(self.results) + + print("\n" + "="*60) + print(f"VALIDATION SUMMARY: {passed}/{total} tests passed") + print("="*60) + + if failed > 0: + print("\nFailed Tests:") + for r in self.results: + if not r.passed: + print(f" - {r.test_name}: {r.message}") + raise AssertionError(f"{failed} validation(s) failed!") + else: + print("\n✅ All validations passed!") + diff --git a/scripts/migrate_to_nodester.py b/scripts/migrate_to_nodester.py new file mode 100644 index 0000000..fedd461 --- /dev/null +++ b/scripts/migrate_to_nodester.py @@ -0,0 +1,539 @@ +#!/usr/bin/env python3 +""" +Migrate Lakeflow Framework dataflow specs to Nodester format. + +Converts standard, flow, and materialized_view specs into the +node-based Nodester specification format. + +Usage: + python migrate_to_nodester.py [--output ] + python migrate_to_nodester.py --output-dir [--recursive] + +Examples: + # Single file + python migrate_to_nodester.py spec_main.json --output nodester_spec_main.json + + # Directory (all *_main.json files) + python migrate_to_nodester.py ./dataflowspec/ --output-dir ./nodester_dataflowspec/ +""" +import argparse +import json +import os +import sys +from typing import Dict, List, Optional, Any, Tuple + + +def migrate_spec(spec: Dict) -> Dict: + """Convert a dataflow spec to nodester format based on its dataFlowType.""" + spec_type = spec.get("dataFlowType", "standard").lower() + + if spec_type == "standard": + return _migrate_standard(spec) + elif spec_type == "flow": + return _migrate_flow(spec) + elif spec_type == "materialized_view": + return _migrate_materialized_view(spec) + else: + print(f" Warning: Unknown dataFlowType '{spec_type}', treating as standard") + return _migrate_standard(spec) + + +# ─── Standard Spec Migration ───────────────────────────────────────────────── + +def _migrate_standard(spec: Dict) -> Dict: + """Convert a standard spec to nodester.""" + nodes = [] + + # Build source node + source_id = spec.get("sourceViewName", "source").replace("v_", "", 1) + if not source_id: + source_id = "source" + source_node = _build_source_node_from_standard(source_id, spec) + nodes.append(source_node) + + # Build target node + target_config, target_type = _build_target_config_from_standard(spec) + target_id = f"target_{target_config.get('table', target_config.get('name', 'output'))}" + target_config["input"] = [source_id] + target_node: Dict[str, Any] = { + "name": target_id, + "node_type": "target", + } + if target_type != "delta": + target_node["target_type"] = target_type + target_node["config"] = target_config + nodes.append(target_node) + + # Build nodester spec + result = { + "data_flow_id": spec.get("dataFlowId"), + "data_flow_group": spec.get("dataFlowGroup"), + "data_flow_type": "nodester", + "nodes": nodes + } + + if spec.get("dataFlowVersion"): + result["data_flow_version"] = spec["data_flow_version"] + if spec.get("tags"): + result["tags"] = spec["tags"] + if spec.get("features"): + result["features"] = spec["features"] + + return result + + +def _build_source_node_from_standard(source_id: str, spec: Dict) -> Dict: + """Build a source node from a standard spec's source fields.""" + source_type = spec.get("sourceType", "delta") + source_details = spec.get("sourceDetails", {}) + mode = spec.get("mode", "stream") + + config: Dict[str, Any] = {} + + if mode: + config["mode"] = mode + + # Copy source details based on type + if source_type == "delta": + for key in ["database", "table", "cdfEnabled", "tablePath"]: + if key in source_details: + config[key] = source_details[key] + elif source_type in ("cloudFiles", "batchFiles"): + for key in ["path", "readerOptions"]: + if key in source_details: + config[key] = source_details[key] + elif source_type == "python": + for key in ["functionPath", "pythonModule", "tokens"]: + if key in source_details: + config[key] = source_details[key] + elif source_type == "sql": + for key in ["sqlPath", "sqlStatement"]: + if key in source_details: + config[key] = source_details[key] + elif source_type == "kafka": + if "readerOptions" in source_details: + config["reader_options"] = source_details["readerOptions"] + + # Common optional source properties + for key in ["selectExp", "whereClause", "schemaPath"]: + if key in source_details: + config[key] = source_details[key] + + node: Dict[str, Any] = {"name": source_id, "node_type": "source", "source_type": source_type} + if config: + node["config"] = config + return node + + +def _build_target_config_from_standard(spec: Dict) -> Tuple[Dict, str]: + """Build target node config from standard spec's target and spec-level settings. + + Returns (config_dict, target_type_string). + """ + target_details = spec.get("targetDetails", {}) + target_format = spec.get("targetFormat", "delta") + + config: Dict[str, Any] = {} + + if target_format != "delta": + # For sinks, copy all target details directly + for key, val in target_details.items(): + config[key] = val + else: + # Delta target + for key in ["table", "database", "schemaPath", "tableProperties", "path", + "partitionColumns", "clusterByColumns", "clusterByAuto", + "comment", "sparkConf", "rowFilter", "configFlags"]: + if key in target_details: + config[key] = target_details[key] + + # Move spec-level settings to target config + for key in ["cdc_settings", "cdc_apply_changes", "cdc_snapshot_settings"]: + if spec.get(key): + config[key] = spec[key] + + if spec.get("dataQualityExpectationsEnabled"): + config["data_quality_expectations_enabled"] = spec["data_quality_expectations_enabled"] + if spec.get("dataQualityExpectationsPath"): + config["data_quality_expectations_path"] = spec["data_quality_expectations_path"] + if spec.get("quarantineMode"): + config["quarantine_mode"] = spec["quarantine_mode"] + if spec.get("quarantineTargetDetails"): + config["quarantine_target_details"] = spec["quarantine_target_details"] + if spec.get("tableMigrationDetails"): + config["table_migration_details"] = spec["table_migration_details"] + + return config, target_format + + +# ─── Flow Spec Migration ───────────────────────────────────────────────────── + +def _migrate_flow(spec: Dict) -> Dict: + """Convert a flow spec to nodester.""" + nodes = [] + node_ids = set() + + flow_groups = spec.get("flowGroups", []) + + # Collect all staging table names (these become target nodes) + staging_table_names = set() + for fg in flow_groups: + for name in fg.get("stagingTables", {}).keys(): + staging_table_names.add(name) + + # Process each flow group + for fg in flow_groups: + staging_tables = fg.get("stagingTables", {}) + flows = fg.get("flows", {}) + + # Create target nodes for staging tables + for stg_name, stg_config in staging_tables.items(): + target_id = f"target_{stg_name}" + if target_id in node_ids: + continue + node_ids.add(target_id) + + target_config = _build_target_config_from_staging(stg_name, stg_config) + target_config["input"] = [] # placeholder, filled when processing flows + nodes.append({ + "name": target_id, + "node_type": "target", + "config": target_config, + }) + + # Process flows + for flow_name, flow_config in flows.items(): + flow_type = flow_config.get("flowType") + flow_details = flow_config.get("flowDetails", {}) + target_table = flow_details.get("targetTable") + views = flow_config.get("views", {}) + + # Create source/transform nodes from views + source_node_id = None + for view_name, view_config in views.items(): + if view_name in node_ids: + continue + node_ids.add(view_name) + + view_source_details = view_config.get("sourceDetails", {}) + view_source_type = view_config.get("sourceType", "delta") + view_mode = view_config.get("mode", "stream") + + # Check if this view reads from a staging table (internal source) + view_db = view_source_details.get("database", "") + view_table = view_source_details.get("table", "") + is_internal = ( + view_db == "live" and view_table in staging_table_names + ) + + # Build source node config + src_config: Dict[str, Any] = {} + if view_mode: + src_config["mode"] = view_mode + + # Copy source details + for key, val in view_source_details.items(): + src_config[key] = val + + src_node: Dict[str, Any] = { + "name": view_name, + "node_type": "source", + "source_type": view_source_type, + } + if src_config: + src_node["config"] = src_config + nodes.append(src_node) + source_node_id = view_name + + # Handle append_sql flows (SQL is in flowDetails, not views) + if flow_type == "append_sql": + sql_source_id = f"sql_{flow_name}" + if sql_source_id not in node_ids: + node_ids.add(sql_source_id) + sql_config: Dict[str, Any] = {} + if flow_details.get("sqlStatement"): + sql_config["sql_statement"] = flow_details["sql_statement"] + elif flow_details.get("sqlPath"): + sql_config["sql_path"] = flow_details["sql_path"] + sql_node: Dict[str, Any] = { + "name": sql_source_id, + "node_type": "source", + "source_type": "sql", + } + if sql_config: + sql_node["config"] = sql_config + nodes.append(sql_node) + source_node_id = sql_source_id + + # If no views and not append_sql, use the sourceView reference + if not source_node_id and flow_details.get("sourceView"): + source_node_id = flow_details["source_view"] + + # Connect flow to target + if target_table and source_node_id: + # Find the target node + target_node_id = f"target_{target_table}" + target_found = False + for node in nodes: + if node["name"] == target_node_id: + if source_node_id not in node.get("inputs", []): + node.setdefault("config", {}).setdefault("input", []).append(source_node_id) + target_found = True + break + + if not target_found: + # This is the main target + if target_node_id not in node_ids: + node_ids.add(target_node_id) + main_target_config = _build_target_config_from_spec_level(spec) + # Add once flag if present + if flow_details.get("once"): + main_target_config["once"] = True + main_target_config["input"] = [source_node_id] + nodes.append({ + "name": target_node_id, + "node_type": "target", + "config": main_target_config + }) + else: + # Find existing and add input + for node in nodes: + if node["name"] == target_node_id: + node.setdefault("config", {}).setdefault("input", []).append(source_node_id) + break + + # Clean up empty inputSources + for node in nodes: + config = node.get("config", {}) + if config.get("input") == []: + del config["input"] + + result = { + "data_flow_id": spec.get("dataFlowId"), + "data_flow_group": spec.get("dataFlowGroup"), + "data_flow_type": "nodester", + "nodes": nodes + } + + if spec.get("dataFlowVersion"): + result["data_flow_version"] = spec["data_flow_version"] + if spec.get("tags"): + result["tags"] = spec["tags"] + if spec.get("features"): + result["features"] = spec["features"] + + return result + + +def _build_target_config_from_staging(table_name: str, stg_config: Dict) -> Dict: + """Build target config from staging table definition.""" + config: Dict[str, Any] = {"table": table_name} + + for key in ["tableProperties", "schemaPath", "clusterByColumns", + "clusterByAuto", "partitionColumns", "database", "configFlags"]: + if key in stg_config: + config[key] = stg_config[key] + + # CDC settings + for key in ["cdc_settings", "cdc_apply_changes", "cdc_snapshot_settings"]: + if key in stg_config: + config[key] = stg_config[key] + + # DQ settings + if stg_config.get("dataQualityExpectationsEnabled"): + config["data_quality_expectations_enabled"] = stg_config["data_quality_expectations_enabled"] + if stg_config.get("dataQualityExpectationsPath"): + config["data_quality_expectations_path"] = stg_config["data_quality_expectations_path"] + + # Quarantine + if stg_config.get("quarantineMode"): + config["quarantine_mode"] = stg_config["quarantine_mode"] + if stg_config.get("quarantineTargetDetails"): + config["quarantine_target_details"] = stg_config["quarantine_target_details"] + + return config + + +def _build_target_config_from_spec_level(spec: Dict) -> Dict: + """Build main target config from spec-level target details and settings.""" + target_details = spec.get("targetDetails", {}) + config: Dict[str, Any] = {} + + for key in ["table", "database", "schema_path", "table_properties", "path", + "partition_columns", "cluster_by_columns", "cluster_by_auto", + "comment", "spark_conf", "row_filter", "config_flags"]: + if key in target_details: + config[key] = target_details[key] + + # Spec-level settings → target config + for key in ["cdc_settings", "cdc_apply_changes", "cdc_snapshot_settings"]: + if spec.get(key): + config[key] = spec[key] + + if spec.get("dataQualityExpectationsEnabled"): + config["data_quality_expectations_enabled"] = spec["data_quality_expectations_enabled"] + if spec.get("dataQualityExpectationsPath"): + config["data_quality_expectations_path"] = spec["data_quality_expectations_path"] + if spec.get("quarantineMode"): + config["quarantine_mode"] = spec["quarantine_mode"] + if spec.get("quarantineTargetDetails"): + config["quarantine_target_details"] = spec["quarantine_target_details"] + if spec.get("tableMigrationDetails"): + config["table_migration_details"] = spec["table_migration_details"] + + return config + + +# ─── Materialized View Spec Migration ───────────────────────────────────────── + +def _migrate_materialized_view(spec: Dict) -> Dict: + """Convert a materialized_view spec to nodester.""" + materialized_views = spec.get("materializedViews", {}) + nodes = [] + + for mv_name, mv_config in materialized_views.items(): + target_config: Dict[str, Any] = { + "table": mv_name, + "table_type": "mv" + } + + # Copy MV properties + for key in ["sql_path", "sql_statement", "refresh_policy"]: + if key in mv_config: + target_config[key] = mv_config[key] + + # Copy table details + if mv_config.get("tableDetails"): + target_config["table_details"] = mv_config["table_details"] + + # Copy DQ settings + if mv_config.get("dataQualityExpectationsEnabled"): + target_config["data_quality_expectations_enabled"] = mv_config["data_quality_expectations_enabled"] + if mv_config.get("dataQualityExpectationsPath"): + target_config["data_quality_expectations_path"] = mv_config["data_quality_expectations_path"] + if mv_config.get("quarantineMode"): + target_config["quarantine_mode"] = mv_config["quarantine_mode"] + if mv_config.get("quarantineTargetDetails"): + target_config["quarantine_target_details"] = mv_config["quarantine_target_details"] + + # Handle source view: MV source views are no longer inlined on the + # target. Emit a source node and chain it into the MV via `input`. + source_view = mv_config.get("source_view") + input_sources = [] + if source_view: + source_id = source_view.get("sourceViewName", f"source_{mv_name}") + # Create a source node for the view + src_config: Dict[str, Any] = {"mode": "stream"} + for key, val in source_view.get("sourceDetails", {}).items(): + src_config[key] = val + nodes.append({ + "name": source_id, + "node_type": "source", + "source_type": source_view.get("sourceType", "delta"), + "config": src_config + }) + input_sources = [source_id] + + target_node: Dict[str, Any] = { + "name": f"target_{mv_name}", + "node_type": "target", + "config": target_config + } + if input_sources: + target_config["input"] = input_sources + nodes.append(target_node) + + result = { + "data_flow_id": spec.get("dataFlowId"), + "data_flow_group": spec.get("dataFlowGroup"), + "data_flow_type": "nodester", + "nodes": nodes + } + + if spec.get("tags"): + result["tags"] = spec["tags"] + if spec.get("features"): + result["features"] = spec["features"] + + return result + + +# ─── CLI ────────────────────────────────────────────────────────────────────── + +def process_file(input_path: str, output_path: Optional[str] = None) -> str: + """Process a single spec file.""" + with open(input_path, "r") as f: + spec = json.load(f) + + result = migrate_spec(spec) + + if output_path is None: + output_path = input_path + + with open(output_path, "w") as f: + json.dump(result, f, indent=4) + f.write("\n") + + return output_path + + +def main(): + parser = argparse.ArgumentParser( + description="Migrate Lakeflow Framework specs to Nodester format" + ) + parser.add_argument("input", help="Input spec file or directory") + parser.add_argument("--output", "-o", help="Output file (for single file mode)") + parser.add_argument("--output-dir", "-d", help="Output directory (for directory mode)") + parser.add_argument("--recursive", "-r", action="store_true", + help="Process directories recursively") + parser.add_argument("--dry-run", action="store_true", + help="Show what would be done without writing files") + + args = parser.parse_args() + + if os.path.isfile(args.input): + output = args.output or args.input.replace(".json", "_nodester.json") + if args.dry_run: + print(f"Would convert: {args.input} → {output}") + else: + result = process_file(args.input, output) + print(f"Converted: {args.input} → {result}") + + elif os.path.isdir(args.input): + output_dir = args.output_dir or args.input + os.makedirs(output_dir, exist_ok=True) + + pattern = "*_main.json" + count = 0 + for root, dirs, files in os.walk(args.input): + for fname in sorted(files): + if fname.endswith("_main.json") or fname.endswith(".json"): + input_path = os.path.join(root, fname) + + # Compute relative path for output + rel = os.path.relpath(input_path, args.input) + output_path = os.path.join(output_dir, rel) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + if args.dry_run: + print(f"Would convert: {input_path} → {output_path}") + else: + try: + process_file(input_path, output_path) + print(f" ✓ {fname}") + count += 1 + except Exception as e: + print(f" ✗ {fname}: {e}") + + if not args.recursive: + break + + if not args.dry_run: + print(f"\nConverted {count} files to {output_dir}") + else: + print(f"Error: {args.input} not found") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/dataflow/dataflow.py b/src/dataflow/dataflow.py index 7cef5c8..f057563 100644 --- a/src/dataflow/dataflow.py +++ b/src/dataflow/dataflow.py @@ -39,15 +39,18 @@ class DataFlow: cdc_settings (CDCSettings): The CDC settings. cdc_snapshot_settings (CDCSnapshotSettings): The CDC snapshot settings. - # expectations - expectations_enabled (bool): Whether expectations are enabled. - expectations (DataQualityExpectations): The data quality expectations. + # expectations (for main target) + expectations_enabled (bool): Whether expectations are enabled for main target. + expectations (DataQualityExpectations): The data quality expectations for main target. expectations_clause (Dict): The expectations clause for the SDP create table api. - # quarantine settings - quarantine_enabled (bool): Whether quarantine is enabled. - quarantine_manager (QuarantineManager): The quarantine manager. - quarantine_mode (str): The quarantine mode. + # quarantine settings (for main target) + quarantine_enabled (bool): Whether quarantine is enabled for main target. + quarantine_manager (QuarantineManager): The quarantine manager for main target. + quarantine_mode (str): The quarantine mode for main target. + + # per-target quarantine managers + staging_quarantine_managers (Dict): Quarantine managers per staging table. # table migration settings table_migration_manager (TableMigrationManager): The table migration manager. @@ -92,6 +95,9 @@ def __init__( self._init_expectations() self._init_quarantine() self._init_table_migration() + + # Initialize per-staging-table quarantine managers (populated during flow group creation) + self.staging_quarantine_managers: Dict[str, QuarantineManager] = {} def _init_target_details(self): """init target details from the dataflow specification.""" @@ -268,7 +274,14 @@ def _create_flow_group(self, flow_group: FlowGroup): if staging_tables: self.logger.info("Creating Staging Tables...") for staging_table in staging_tables.values(): - staging_table.create_table() + # Initialize DQ and quarantine for this staging table + staging_expectations = self._init_staging_table_expectations(staging_table) + self._init_staging_table_quarantine(staging_table, staging_expectations) + # Add SCD columns to staging table schema if it has a schema and CDC/CDC Snapshot SCD2 + self._init_staging_table_cdc_settings(staging_table) + + # Create the staging table with expectations if enabled + staging_table.create_table(staging_expectations) # Support direct historical snapshots into Staging Tables in Flows cdc_snapshot_settings = staging_table.get_cdc_snapshot_settings() @@ -294,6 +307,93 @@ def _create_flow_group(self, flow_group: FlowGroup): else: self.logger.info("Flow Disabled: %s", flow.flowName) + def _init_staging_table_expectations(self, staging_table: StagingTable) -> Dict: + """Initialize data quality expectations for a staging table.""" + if not staging_table.dataQualityExpectationsEnabled: + return None + + self.logger.info("Initializing DQ expectations for staging table: %s", staging_table.table) + + expectations = staging_table.get_data_quality_expectations() + if expectations is None: + self.logger.warning( + "DQ enabled for staging table %s but no expectations loaded", + staging_table.table + ) + return None + + self.logger.debug("Expectations for %s: %s", staging_table.table, expectations.__dict__) + + # For quarantine FLAG mode, return expect_all format + if (staging_table.has_quarantine_enabled() + and staging_table.quarantineMode == QuarantineMode.FLAG): + return expectations.get_expectations_as_expect_all() + + return expectations.get_expectations() + + def _init_staging_table_quarantine( + self, + staging_table: StagingTable, + expectations: Dict + ) -> None: + """Initialize quarantine manager for a staging table if enabled.""" + if not staging_table.has_quarantine_enabled(): + return + + # Get the DQ expectations object for quarantine rules + dq_expectations = staging_table.get_data_quality_expectations() + if dq_expectations is None or not dq_expectations.all_rules: + self.logger.warning( + "Quarantine enabled for staging table %s but no DQ rules available", + staging_table.table + ) + return + + self.logger.info( + "Initializing quarantine for staging table: %s, mode: %s", + staging_table.table, staging_table.quarantineMode + ) + + # Create quarantine manager for this staging table + quarantine_manager = QuarantineManager( + quarantine_mode=staging_table.quarantineMode, + data_quality_rules=dq_expectations.all_rules, + target_format=TargetType.DELTA, + target_details=staging_table, + quarantine_target_details=staging_table.quarantineTargetDetails or {} + ) + + # Store the quarantine manager for this staging table + self.staging_quarantine_managers[staging_table.table] = quarantine_manager + + # Add quarantine columns to staging table schema if FLAG mode + if staging_table.quarantineMode == QuarantineMode.FLAG: + quarantine_manager.add_quarantine_columns_delta(staging_table) + + def _init_staging_table_cdc_settings(self, staging_table: StagingTable): + """Add SCD2 columns to staging table schema if it has a defined schema and CDC/CDC Snapshot SCD type 2.""" + if not hasattr(staging_table, 'schema') or not staging_table.schema: + return + + def get_scd2_columns(sequence_by_data_type: T.DataType) -> List[T.StructField]: + return [ + T.StructField(SystemColumns.SCD2Columns.SCD2_START_AT.value, sequence_by_data_type), + T.StructField(SystemColumns.SCD2Columns.SCD2_END_AT.value, sequence_by_data_type) + ] + + cdc_settings = staging_table.get_cdc_settings() + cdc_snapshot_settings = staging_table.get_cdc_snapshot_settings() + + scd2_cfg = None + if cdc_settings and cdc_settings.scd_type == "2": + scd2_cfg = cdc_settings + elif cdc_snapshot_settings and cdc_snapshot_settings.scd_type == "2": + scd2_cfg = cdc_snapshot_settings + if scd2_cfg: + sequence_by_data_type = scd2_cfg.sequence_by_data_type + scd2_columns = get_scd2_columns(sequence_by_data_type) + staging_table.add_columns(scd2_columns) + def _create_flow(self, flow: BaseFlow, staging_tables: Dict[str, StagingTable]): """Create a flow and its associated views.""" self.logger.info("Creating Views...") @@ -301,49 +401,93 @@ def _create_flow(self, flow: BaseFlow, staging_tables: Dict[str, StagingTable]): # Prepare Flow Configuration is_target = self.is_target(flow.targetTable) flow_config = self._prepare_flow_config(flow, staging_tables) + + # Get quarantine settings for this flow's target + quarantine_enabled, quarantine_mode, quarantine_manager = self._get_target_quarantine_settings( + flow.targetTable, staging_tables + ) if isinstance(flow, BaseFlowWithViews): views = flow.get_views() or {} # Create views - self._create_views(views, flow.sourceView, is_target, flow_config.target_config_flags) + self._create_views( + views, flow.sourceView, is_target, flow_config.target_config_flags, + quarantine_enabled, quarantine_mode, quarantine_manager + ) # Create Flow flow.create_flow(self.dataflow_config, flow_config) # Handle Table Quarantine Mode - if (self.quarantine_enabled - and self.quarantine_mode == QuarantineMode.TABLE - and is_target + if (quarantine_enabled + and quarantine_mode == QuarantineMode.TABLE + and quarantine_manager ): - self.quarantine_manager.create_quarantine_flow(flow.sourceView) + quarantine_manager.create_quarantine_flow(flow.sourceView) else: # Get quarantine rules if needed. note in table mode we don't apply them to the source view, # they are applied to a quarantine view that passes to the the quarantine target table. quarantine_rules = None - if (self.quarantine_enabled - and self.quarantine_mode != QuarantineMode.TABLE - and is_target + if (quarantine_enabled + and quarantine_mode != QuarantineMode.TABLE + and quarantine_manager ): - quarantine_rules = self.quarantine_manager.quarantine_rules + quarantine_rules = quarantine_manager.quarantine_rules # Create Flow flow.create_flow(self.dataflow_config, flow_config, quarantine_rules) - def _create_views(self, views: Dict[str, View], flow_source_view: str, is_target: bool, target_config_flags: List[str]) -> None: + def _get_target_quarantine_settings( + self, + target_table: str, + staging_tables: Dict[str, StagingTable] + ) -> tuple: + """Get quarantine settings for a target (main target or staging table).""" + is_target = self.is_target(target_table) + + if is_target: + # Main target uses spec-level quarantine settings + return ( + self.quarantine_enabled, + self.quarantine_mode, + self.quarantine_manager if self.quarantine_enabled else None + ) + else: + # Staging table uses its own quarantine settings + staging_table = staging_tables.get(target_table) + if staging_table and staging_table.has_quarantine_enabled(): + quarantine_manager = self.staging_quarantine_managers.get(staging_table.table) + return ( + True, + staging_table.quarantineMode, + quarantine_manager + ) + return (False, None, None) + + def _create_views( + self, + views: Dict[str, View], + flow_source_view: str, + is_target: bool, + target_config_flags: List[str], + quarantine_enabled: bool = False, + quarantine_mode: str = None, + quarantine_manager: QuarantineManager = None + ) -> None: """Create views for the flow, handling quarantine as needed.""" for view in views.values(): # Get quarantine rules if needed. note in table mode we don't apply them to the source view, # they are applied to a quarantine view that passes to the the quarantine target table. quarantine_rules = None - if (self.quarantine_enabled - and self.quarantine_mode != QuarantineMode.TABLE - and is_target + if (quarantine_enabled + and quarantine_mode != QuarantineMode.TABLE and flow_source_view == view.viewName + and quarantine_manager ): - quarantine_rules = self.quarantine_manager.quarantine_rules + quarantine_rules = quarantine_manager.quarantine_rules # Create the view view.create_view( diff --git a/src/dataflow/targets/staging_table.py b/src/dataflow/targets/staging_table.py index dcb0551..03106ba 100644 --- a/src/dataflow/targets/staging_table.py +++ b/src/dataflow/targets/staging_table.py @@ -1,8 +1,9 @@ from dataclasses import dataclass, field -from typing import Dict +from typing import Dict, Optional from ..cdc import CDCSettings from ..cdc_snapshot import CDCSnapshotSettings +from ..expectations import DataQualityExpectations from .delta_streaming_table import TargetDeltaStreamingTable @@ -15,14 +16,32 @@ class StagingTable(TargetDeltaStreamingTable): Attributes: cdcSettings (Dict, optional): CDC settings. cdcSnapshotSettings (Dict, optional): CDC snapshot settings. + dataQualityExpectationsEnabled (bool, optional): Whether DQ expectations are enabled. + dataQualityExpectationsPath (str, optional): Path to DQ expectations file. + dataQualityExpectations (Dict, optional): Loaded DQ expectations. + quarantineMode (str, optional): Quarantine mode (off/flag/table). + quarantineTargetDetails (Dict, optional): Quarantine target configuration. Methods: get_cdc_settings() -> CDCSettings: Get CDC settings. get_cdc_snapshot_settings() -> CDCSnapshotSettings: Get CDC snapshot settings. + get_data_quality_expectations() -> DataQualityExpectations: Get DQ expectations. + has_quarantine_enabled() -> bool: Check if quarantine is enabled. """ cdcSettings: Dict = field(default_factory=dict) cdcSnapshotSettings: Dict = field(default_factory=dict) + dataQualityExpectationsEnabled: bool = False + dataQualityExpectationsPath: str = None + dataQualityExpectations: Dict = field(default_factory=dict) + quarantineMode: str = None + quarantineTargetDetails: Dict = field(default_factory=dict) + + def __post_init__(self): + """Post-init processing.""" + super().__post_init__() + if self.quarantineMode: + self.quarantineMode = self.quarantineMode.lower() def get_cdc_settings(self) -> CDCSettings: """Get CDC configuration.""" @@ -33,3 +52,16 @@ def get_cdc_snapshot_settings(self) -> CDCSnapshotSettings: """Get CDC snapshot settings.""" return CDCSnapshotSettings(**self.cdcSnapshotSettings) \ if self.cdcSnapshotSettings else None + + def get_data_quality_expectations(self) -> Optional[DataQualityExpectations]: + """Get data quality expectations for this staging table.""" + return DataQualityExpectations(**self.dataQualityExpectations) \ + if self.dataQualityExpectationsEnabled and self.dataQualityExpectations else None + + def has_quarantine_enabled(self) -> bool: + """Check if quarantine is enabled for this staging table.""" + return ( + self.dataQualityExpectationsEnabled + and self.quarantineMode is not None + and self.quarantineMode != "off" + ) diff --git a/src/dataflow_spec_builder/dataflow_spec_builder.py b/src/dataflow_spec_builder/dataflow_spec_builder.py index 8199310..e49df75 100644 --- a/src/dataflow_spec_builder/dataflow_spec_builder.py +++ b/src/dataflow_spec_builder/dataflow_spec_builder.py @@ -203,8 +203,25 @@ def _read_dataflow_specs(self) -> Dict: main_specs = {} flow_specs = {} + # Map of snake_case → camelCase for core metadata keys. + # Specs may use either convention; we normalise to camelCase in-place + # so downstream processing (filtering, validation, transformation) works uniformly. + _METADATA_SNAKE_TO_CAMEL = { + "data_flow_id": self.Keys.DATA_FLOW_ID, + "data_flow_group": self.Keys.DATA_FLOW_GROUP, + "data_flow_type": self.Keys.DATA_FLOW_TYPE, + "data_flow_version": self.Keys.DATA_FLOW_VERSION, + } + + def _normalise_spec_metadata(spec: Dict) -> None: + """Normalise snake_case metadata keys to camelCase in-place.""" + for snake, camel in _METADATA_SNAKE_TO_CAMEL.items(): + if snake in spec and camel not in spec: + spec[camel] = spec.pop(snake) + def _extract_spec(spec: Dict) -> Dict: """Extract metadata from a dataflow specification.""" + _normalise_spec_metadata(spec) return { "fileType": "main", self.Keys.DATA_FLOW_ID: spec.get(self.Keys.DATA_FLOW_ID, None), @@ -421,7 +438,18 @@ def _validate_json(spec_item: tuple) -> tuple: spec_path, spec_payload = spec_item file_type = spec_payload.get("fileType", "main") json_data = spec_payload.get(self.Keys.DATA) - if file_type == "main": + spec_type = spec_payload.get(self.Keys.DATA_FLOW_TYPE, "") + + # Nodester specs use their own schema directly — the main.json + # if/else chain doesn't route cleanly for non-standard types. + if spec_type == "nodester": + nodester_schema_path = os.path.join(self.framework_path, "schemas", "spec_nodester.json") + if os.path.exists(nodester_schema_path): + nodester_validator = utility.JSONValidator(nodester_schema_path) + errors = nodester_validator.validate(json_data) + else: + errors = [] + elif file_type == "main": errors = self.main_validator.validate(json_data) else: errors = self.flow_validator.validate(json_data) @@ -698,6 +726,12 @@ def _get_base_path(self, file_path: str) -> str: def _get_expectations(self, dataflow_spec: Dict, base_path: str) -> Dict: """Set the expectation validator path in the dataflow specification.""" dqe_validator_path = os.path.join(self.framework_path,FrameworkPaths.EXPECTATIONS_SPEC_SCHEMA_PATH) + dqe_builder = DataQualityExpectationBuilder( + self.logger, + dqe_validator_path, + self.spec_file_format + ) + if dataflow_spec.get(self.Keys.DATA_QUALITY_EXPECTATIONS_ENABLED, False): dqe_path = dataflow_spec.get(self.Keys.DATA_QUALITY_EXPECTATIONS_PATH, None) if dqe_path is None or dqe_path.strip() == "": @@ -705,10 +739,21 @@ def _get_expectations(self, dataflow_spec: Dict, base_path: str) -> Dict: dqe_path = f"{base_path}/{PipelineBundlePaths.DQE_PATH}/{dqe_path}" dataflow_spec[self.Keys.DATA_QUALITY_EXPECTATIONS] = ( - DataQualityExpectationBuilder( - self.logger, - dqe_validator_path, - self.spec_file_format - ).get_expectations(dqe_path).__dict__) + dqe_builder.get_expectations(dqe_path).__dict__) + + # Also load DQ expectations for staging tables within flow groups + for flow_group in dataflow_spec.get(self.Keys.FLOW_GROUPS, []): + staging_tables = flow_group.get("stagingTables", {}) + for staging_config in staging_tables.values(): + if staging_config.get(self.Keys.DATA_QUALITY_EXPECTATIONS_ENABLED, False): + staging_dqe_path = staging_config.get(self.Keys.DATA_QUALITY_EXPECTATIONS_PATH, None) + if staging_dqe_path is None or staging_dqe_path.strip() == "": + self.logger.warning( + "DQ expectations enabled for staging table but no path specified" + ) + continue + full_staging_dqe_path = f"{base_path}/{PipelineBundlePaths.DQE_PATH}/{staging_dqe_path}" + staging_config[self.Keys.DATA_QUALITY_EXPECTATIONS] = ( + dqe_builder.get_expectations(full_staging_dqe_path).__dict__) return dataflow_spec diff --git a/src/dataflow_spec_builder/spec_mapper.py b/src/dataflow_spec_builder/spec_mapper.py index 724350e..21df221 100644 --- a/src/dataflow_spec_builder/spec_mapper.py +++ b/src/dataflow_spec_builder/spec_mapper.py @@ -2,7 +2,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Dict, Tuple, Optional, Any -from config_resolver import resolve_framework_config_dir +from config_resolver import resolve_framework_config_dir, resolve_framework_config_path from constants import FrameworkPaths import pipeline_config import utility @@ -45,6 +45,7 @@ def __init__(self, framework_path: str, max_workers: int = 1): max_workers: Maximum parallel workers for processing """ self.framework_path = framework_path + self._framework_config_path = resolve_framework_config_path(framework_path) self.max_workers = max_workers self._mapping_cache: Dict[str, Dict] = {} diff --git a/src/dataflow_spec_builder/transformer/__init__.py b/src/dataflow_spec_builder/transformer/__init__.py index d182f34..3ed7fb0 100644 --- a/src/dataflow_spec_builder/transformer/__init__.py +++ b/src/dataflow_spec_builder/transformer/__init__.py @@ -2,6 +2,7 @@ from .standard import StandardSpecTransformer from .flow import FlowSpecTransformer from .materialized_views import MaterializedViewSpecTransformer +from .nodester import NodesterSpecTransformer from .factory import SpecTransformerFactory __all__ = [ @@ -9,5 +10,6 @@ 'StandardSpecTransformer', 'FlowSpecTransformer', 'MaterializedViewSpecTransformer', + 'NodesterSpecTransformer', 'SpecTransformerFactory' ] diff --git a/src/dataflow_spec_builder/transformer/base.py b/src/dataflow_spec_builder/transformer/base.py index c250b2e..d2a6d6a 100644 --- a/src/dataflow_spec_builder/transformer/base.py +++ b/src/dataflow_spec_builder/transformer/base.py @@ -7,7 +7,14 @@ class BaseSpecTransformer(ABC): """Base class for dataflow spec transformers.""" - + + # When True, transform() inspects the produced flow spec and warns on + # source definitions that embed SQL/Python transformation logic. Spec + # types that can already detect this on their own input (e.g. nodester, + # which distinguishes source nodes from transformation nodes) set this to + # False and emit the warning themselves. + WARN_INLINE_SOURCE_TRANSFORMATIONS_FROM_OUTPUT = True + def __init__(self): self.logger = pipeline_config.get_logger() @@ -19,8 +26,53 @@ def _process_spec(self, spec_data: Dict) -> Union[Dict, List[Dict]]: def transform(self, spec_data: Dict) -> Union[Dict, List[Dict]]: """Transform the spec data. Returns either a single Dict or List[Dict].""" spec_data = self._apply_features_and_limitations(spec_data) - return self._process_spec(spec_data) - + result = self._process_spec(spec_data) + if self.WARN_INLINE_SOURCE_TRANSFORMATIONS_FROM_OUTPUT: + for flow_spec in (result if isinstance(result, list) else [result]): + self._warn_inline_source_transformations(flow_spec) + return result + + def _warn_inline_source_transformations(self, flow_spec: Dict) -> None: + """Warn when a source definition embeds SQL/Python transformation logic. + + Defining a source whose type is SQL or Python (or appending the result + of an inline SQL statement) folds transformation logic into the source + definition. This is still supported for backward compatibility, but is + discouraged: it conflates "where the data comes from" with "how the data + is transformed". The recommended pattern is to declare a plain source + and model transformation logic as a separate transformation step. This + behaviour may not be supported in a future release. + """ + if not isinstance(flow_spec, dict): + return + + data_flow_id = flow_spec.get("dataFlowId") + for flow_group in flow_spec.get("flowGroups", []) or []: + flows = flow_group.get("flows", {}) or {} + for flow_name, flow in flows.items(): + if str(flow.get("flowType", "")).lower() == "append_sql": + self.logger.warning( + "Flow '%s' (dataFlowId '%s') uses an inline SQL source " + "(flowType 'append_sql'). This is still supported but " + "discouraged and may not be supported in a future " + "release. Model transformation logic as a separate " + "transformation step instead.", + flow_name, data_flow_id, + ) + for view_name, view in (flow.get("views", {}) or {}).items(): + source_type = str(view.get("sourceType", "")).lower() + if source_type in ("sql", "python"): + self.logger.warning( + "Source view '%s' (dataFlowId '%s') defines an " + "inline %s transformation (sourceType '%s'). This " + "is still supported but discouraged and may not be " + "supported in a future release. Declare a plain " + "source and model transformation logic as a " + "separate transformation step instead.", + view_name, data_flow_id, source_type, source_type, + ) + + def _apply_features_and_limitations(self, dataflow_spec: Dict) -> Dict: """Apply common features and limitations transformations.""" # Operational MetadataSnapshot diff --git a/src/dataflow_spec_builder/transformer/factory.py b/src/dataflow_spec_builder/transformer/factory.py index a1c8b8b..e79f1a2 100644 --- a/src/dataflow_spec_builder/transformer/factory.py +++ b/src/dataflow_spec_builder/transformer/factory.py @@ -3,6 +3,7 @@ from .standard import StandardSpecTransformer from .flow import FlowSpecTransformer from .materialized_views import MaterializedViewSpecTransformer +from .nodester import NodesterSpecTransformer class SpecTransformerFactory: @@ -11,7 +12,8 @@ class SpecTransformerFactory: _transformers = { "standard": StandardSpecTransformer, "flow": FlowSpecTransformer, - "materialized_view": MaterializedViewSpecTransformer + "materialized_view": MaterializedViewSpecTransformer, + "nodester": NodesterSpecTransformer } @classmethod @@ -26,4 +28,4 @@ def create_transformer(cls, dataflow_type: str, dataflow_spec_mapping_path: str @classmethod def get_supported_types(cls) -> List[str]: """Return list of supported dataflow types.""" - return list(cls._transformers.keys()) \ No newline at end of file + return list(cls._transformers.keys()) diff --git a/src/dataflow_spec_builder/transformer/nodester.py b/src/dataflow_spec_builder/transformer/nodester.py new file mode 100644 index 0000000..dae6be4 --- /dev/null +++ b/src/dataflow_spec_builder/transformer/nodester.py @@ -0,0 +1,748 @@ +""" +Nodester Spec Transformer + +Converts a node-based dataflow spec into the Lakeflow Framework's flow-based +spec. A nodester spec is a graph of three node kinds: + + source -> transformation -> target + +- source nodes become views (or direct table references for internal tables) +- transformation nodes become SQL/Python views +- target nodes become the spec target table or staging tables, each + carrying its own settings (CDC, data quality, quarantine, ...) + +How nodes connect: +- A target node lists what feeds it in `config.input`. Each input is either a + view/node name (string) or `{"view": ..., "flow": ...}` to set the flow name. +- Source and transformation nodes reference their upstream inside their own + definition (for example a transform's SQL names the view it reads). + +Spec target selection: +- The backend needs one `targetDetails`. It is auto-selected as the terminal + target (a target not consumed by any other node). The remaining targets become + staging tables. With multiple terminal targets the last one is used. + +Materialized views: +- Targets with `table_type: "mv"` each become their own flow spec, so a spec with + both streaming and MV targets returns a list of flow specs. + +Casing: nodester input is snake_case; the emitted flow spec is the backend's +camelCase. `_rename_keys` / `_to_backend_keys` do that translation. +""" +from collections import defaultdict +from typing import Any, Dict, List, Optional, Tuple, Set, Union + +from .base import BaseSpecTransformer +from dataflow.enums import FlowType, Mode, TableType, TargetType + + +# Explicit snake_case -> backend key map for nested blobs that are copied +# wholesale (CDC snapshot settings, table migration, sink config). The backend +# mixes camelCase and snake_case, so only the keys listed here are renamed. +_NODESTER_TO_BACKEND_KEYS = { + # CDC snapshot settings + "snapshot_type": "snapshotType", + "source_type": "sourceType", + "version_type": "versionType", + "version_column": "versionColumn", + "starting_version": "startingVersion", + "datetime_format": "datetimeFormat", + "reader_options": "readerOptions", + "schema_path": "schemaPath", + "select_exp": "selectExp", + "deduplicate_mode": "deduplicateMode", + # Table migration details + "catalog_type": "catalogType", + "auto_starting_versions_enabled": "autoStartingVersionsEnabled", + "source_details": "sourceDetails", + "table_name": "tableName", + # Source nested fields / sink config + "function_path": "functionPath", + "python_module": "pythonModule", + "sql_path": "sqlPath", + "sql_statement": "sqlStatement", + "table_properties": "tableProperties", +} + + +def _to_backend_keys(obj: Any) -> Any: + """Recursively rename known snake_case keys to their backend equivalents.""" + if isinstance(obj, dict): + return {_NODESTER_TO_BACKEND_KEYS.get(k, k): _to_backend_keys(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_to_backend_keys(i) for i in obj] + return obj + + +def _rename_keys(source: Dict, mapping: Dict[str, str]) -> Dict: + """Copy keys present in `source` into a new dict, renaming src -> dst.""" + return {dst: source[src] for src, dst in mapping.items() if src in source} + + +class NodesterSpecTransformer(BaseSpecTransformer): + """Transform a nodester node-graph spec into a flow spec (or list of them).""" + + # Nodester warns about inline source transformations at the node level (where + # source and transformation nodes are distinguishable), so the base class + # should not re-derive that warning from the produced flow spec. + WARN_INLINE_SOURCE_TRANSFORMATIONS_FROM_OUTPUT = False + + FLOW_GROUP_ID = "nodester_main" + FLOW_PREFIX = "f_" + VIEW_PREFIX = "v_" + + SOURCE = "source" + TRANSFORMATION = "transformation" + TARGET = "target" + + # ------------------------------------------------------------------ # + # Entry point + # ------------------------------------------------------------------ # + def _process_spec(self, spec_data: Dict) -> Union[Dict, List[Dict]]: + """Transform a nodester spec into one flow spec, or a list when it + produces both streaming-table and materialized-view targets.""" + nodes = spec_data.get("nodes", []) + if not nodes: + raise ValueError("Nodester spec must contain at least one node") + + sources, transforms, targets = self._categorize_nodes(nodes) + self._warn_inline_source_transformations(sources) + self._validate_node_graph(sources, targets, nodes) + + mv_targets = [t for t in targets if self._is_mv(t)] + st_targets = [t for t in targets if not self._is_mv(t)] + + specs: List[Dict] = [] + if st_targets: + specs.append(self._build_streaming_spec(spec_data, st_targets, sources, nodes)) + if mv_targets: + specs.extend(self._build_mv_specs(spec_data, mv_targets, sources, nodes)) + + if not specs: + raise ValueError("Nodester spec must contain at least one target node") + return specs if len(specs) > 1 else specs[0] + + @staticmethod + def _is_mv(target: Dict) -> bool: + return target.get("config", {}).get("table_type") == "mv" + + # ------------------------------------------------------------------ # + # Node organisation, validation, warnings + # ------------------------------------------------------------------ # + def _categorize_nodes(self, nodes: List[Dict]) -> Tuple[List[Dict], List[Dict], List[Dict]]: + """Split nodes into (sources, transformations, targets).""" + sources, transforms, targets = [], [], [] + buckets = {self.SOURCE: sources, self.TRANSFORMATION: transforms, self.TARGET: targets} + for node in nodes: + bucket = buckets.get(node.get("node_type", "").lower()) + if bucket is None: + self.logger.warning( + f"Unknown node type '{node.get('node_type')}' for node '{node.get('name')}'" + ) + else: + bucket.append(node) + return sources, transforms, targets + + def _warn_inline_source_transformations(self, source_nodes: List[Dict]) -> None: + """Warn when a source node embeds SQL/Python transformation logic. + + Declaring a SQL or Python transformation directly on a source node (via + ``source_type: "sql"`` / ``"python"``) is still supported for backward + compatibility but discouraged: it hides transformation logic inside the + data-origin node. The recommended pattern is a plain source node chained + into a dedicated transformation node. + """ + for node in source_nodes: + source_type = node.get("source_type", "delta") + if source_type in ("sql", "python"): + self.logger.warning( + "Source node '%s' defines an inline %s transformation " + "(source_type: '%s'). This is still supported but " + "discouraged and may not be supported in a future release. " + "Define a source node and chain a dedicated transformation " + "node off it instead.", + node.get("name"), source_type, source_type, + ) + + def _validate_node_graph(self, sources: List[Dict], targets: List[Dict], all_nodes: List[Dict]) -> None: + """Validate references, the removed MV source_view, and source presence.""" + node_names = {node.get("name") for node in all_nodes} + + for target in targets: + for view in self._input_views(target): + if view not in node_names: + raise ValueError( + f"Node '{target.get('name')}' references non-existent input '{view}'" + ) + + # Inline source views on MV targets are no longer supported: authors must + # declare a source node and chain it into the MV target via `input`. + for target in targets: + if self._is_mv(target) and "source_view" in target.get("config", {}): + raise ValueError( + f"Materialized view target '{target.get('name')}' defines an inline " + "'source_view'. This is no longer supported. Declare a source node and " + "chain it into the materialized view target via its 'input' array instead." + ) + + if not targets: + raise ValueError("Nodester spec must contain at least one target node") + + # A source node is unnecessary only when every target reads its data + # without one: an MV defined by inline SQL, or a snapshot CDC target. + all_mv_inline_sql = all( + self._is_mv(t) and (t["config"].get("sql_path") or t["config"].get("sql_statement")) + for t in targets + ) + all_snapshot = all(t.get("config", {}).get("cdc_snapshot_settings") for t in targets) + if not sources and not (all_mv_inline_sql or all_snapshot): + raise ValueError("Nodester spec must contain at least one source node") + + # ------------------------------------------------------------------ # + # input handling + # ------------------------------------------------------------------ # + def _normalize_inputs(self, inputs: List) -> List[Tuple[Optional[str], str]]: + """Normalize a target's ``input`` list into (flow_name, view_name) pairs. + + Each item is either a string (upstream node name; flow name auto-generated) + or ``{"view": ..., "flow": ...}`` where ``flow`` defines the SDP flow name. + Defining the flow name keeps it stable across edits, which matters because + renaming a flow forces a full refresh in SDP. + """ + pairs: List[Tuple[Optional[str], str]] = [] + for item in inputs or []: + if isinstance(item, dict): + if item.get("view"): + pairs.append((item.get("flow"), item["view"])) + else: + pairs.append((None, item)) + return pairs + + def _input_views(self, node: Dict) -> List[str]: + """Return the upstream view/node names a node's ``input`` references.""" + return [view for _, view in self._normalize_inputs(node.get("config", {}).get("input", []))] + + # ------------------------------------------------------------------ # + # Streaming-table targets + # ------------------------------------------------------------------ # + def _build_streaming_spec( + self, spec_data: Dict, targets: List[Dict], sources: List[Dict], all_nodes: List[Dict] + ) -> Dict: + """Build a single flow spec for the streaming-table targets.""" + internal_sources = self._identify_internal_sources(sources, targets) + spec_target, other_targets = self._select_spec_target(targets, sources, all_nodes) + return self._build_flow_spec(spec_data, spec_target, other_targets, all_nodes, internal_sources) + + def _select_spec_target( + self, targets: List[Dict], sources: List[Dict], all_nodes: List[Dict] + ) -> Tuple[Dict, List[Dict]]: + """Pick the terminal target as the spec target; the rest are staging.""" + if len(targets) == 1: + return targets[0], [] + + consumed = {view for node in all_nodes for view in self._input_views(node)} + source_tables = {s.get("config", {}).get("table") for s in sources} + source_tables.discard(None) + + terminal, other = [], [] + for target in targets: + table = target.get("config", {}).get("table") + is_consumed = target.get("name") in consumed or (table in source_tables) + (other if is_consumed else terminal).append(target) + + if not terminal: + raise ValueError( + "Could not determine spec target: all target nodes are consumed by other nodes" + ) + if len(terminal) > 1: + self.logger.warning( + "Multiple terminal targets found %s. Using last target '%s' as spec target.", + [t.get("name") for t in terminal], terminal[-1].get("name"), + ) + return terminal[-1], other + terminal[:-1] + + def _build_flow_spec( + self, spec_data: Dict, spec_target: Dict, other_targets: List[Dict], + all_nodes: List[Dict], internal_sources: Dict[str, str], + ) -> Dict: + """Assemble the flow spec for the streaming targets (camelCase output).""" + node_lookup = {node.get("name"): node for node in all_nodes} + config = spec_target.get("config", {}) + target_format = spec_target.get("target_type", "delta") + + if target_format == "delta": + target_details = self._build_target_details(config) + else: + target_details = self._build_sink_target_details(config) + + cdc_settings = config.get("cdc_settings") or config.get("cdc_apply_changes") + cdc_snapshot_settings = config.get("cdc_snapshot_settings") + has_cdc = cdc_settings is not None or cdc_snapshot_settings is not None + + flow_spec = { + "dataFlowId": spec_data.get("dataFlowId"), + "dataFlowGroup": spec_data.get("dataFlowGroup"), + "dataFlowType": spec_data.get("dataFlowType"), + "targetFormat": target_format, + "targetDetails": target_details, + "tags": spec_data.get("tags", {}), + "features": spec_data.get("features", {}), + } + if spec_data.get("dataFlowVersion"): + flow_spec["dataFlowVersion"] = spec_data["dataFlowVersion"] + if cdc_settings: + flow_spec["cdcSettings"] = cdc_settings + if cdc_snapshot_settings: + flow_spec["cdcSnapshotSettings"] = _to_backend_keys(cdc_snapshot_settings) + + flow_spec["dataQualityExpectationsEnabled"] = config.get("data_quality_expectations_enabled", False) + if config.get("data_quality_expectations_path"): + flow_spec["dataQualityExpectationsPath"] = config["data_quality_expectations_path"] + if config.get("quarantine_mode"): + flow_spec["quarantineMode"] = config["quarantine_mode"] + if config.get("quarantine_target_details"): + flow_spec["quarantineTargetDetails"] = config["quarantine_target_details"] + if config.get("table_migration_details"): + flow_spec["tableMigrationDetails"] = _to_backend_keys(config["table_migration_details"]) + + flow_spec["flowGroups"] = [ + self._build_flow_group(spec_target, other_targets, node_lookup, has_cdc, internal_sources) + ] + return flow_spec + + def _build_target_details(self, config: Dict) -> Dict: + """Build targetDetails for a delta (streaming table) target.""" + target_details = {"table": config.get("table"), "type": TableType.STREAMING} + target_details.update(_rename_keys(config, { + "database": "database", "schema_path": "schemaPath", + "table_properties": "tableProperties", "path": "path", + "partition_columns": "partitionColumns", + "cluster_by_columns": "clusterByColumns", "cluster_by_auto": "clusterByAuto", + "comment": "comment", "spark_conf": "sparkConf", + "row_filter": "rowFilter", "config_flags": "configFlags", + })) + return target_details + + def _build_sink_target_details(self, config: Dict) -> Dict: + """Build targetDetails for a sink target (delta_sink / foreach_batch / custom).""" + target_details = _rename_keys(config, {"name": "name", "sink_type": "type", "sink_options": "sinkOptions"}) + if "sink_config" in config: + target_details["config"] = _to_backend_keys(config["sink_config"]) + return target_details + + def _build_flow_group( + self, spec_target: Dict, other_targets: List[Dict], node_lookup: Dict[str, Dict], + has_cdc: bool, internal_sources: Dict[str, str], + ) -> Dict: + """Build the single flow group: staging tables plus one flow per input.""" + flow_group = {"flowGroupId": self.FLOW_GROUP_ID, "flows": {}} + + staging_tables = self._build_staging_tables(other_targets) + if staging_tables: + flow_group["stagingTables"] = staging_tables + + # Views already attached to a flow, so we don't redefine them (SDP errors + # on "Cannot redefine"). Shared across all flows in the group. + registered_views: Set[str] = set() + + # Group targets by output table, preserving first-seen order. + table_to_targets: Dict[str, List[Dict]] = defaultdict(list) + for target in other_targets + [spec_target]: + config = target.get("config", {}) + table = config.get("table") or config.get("name") + if table: + table_to_targets[table].append(target) + + flow_counter = 0 + for table_name, targets in table_to_targets.items(): + for target in targets: + flow_counter = self._add_target_flows( + flow_group, target, table_name, spec_target, node_lookup, + has_cdc, internal_sources, registered_views, flow_counter, + ) + return flow_group + + def _build_staging_tables(self, other_targets: List[Dict]) -> Dict: + """Build the stagingTables map from the non-spec targets.""" + staging: Dict[str, Dict] = {} + for target in other_targets: + config = target.get("config", {}) + table = config.get("table") + if not table or table in staging: + continue + + entry = {"type": "ST"} + entry.update(_rename_keys(config, { + "database": "database", "schema_path": "schemaPath", + "table_properties": "tableProperties", + "cluster_by_columns": "clusterByColumns", "cluster_by_auto": "clusterByAuto", + "partition_columns": "partitionColumns", "config_flags": "configFlags", + })) + + cdc = config.get("cdc_settings") or config.get("cdc_apply_changes") + if cdc: + entry["cdcSettings"] = cdc + if config.get("cdc_snapshot_settings"): + entry["cdcSnapshotSettings"] = _to_backend_keys(config["cdc_snapshot_settings"]) + if config.get("data_quality_expectations_enabled"): + entry["dataQualityExpectationsEnabled"] = config["data_quality_expectations_enabled"] + if config.get("data_quality_expectations_path"): + entry["dataQualityExpectationsPath"] = config["data_quality_expectations_path"] + if config.get("quarantine_mode"): + entry["quarantineMode"] = config["quarantine_mode"] + if config.get("quarantine_target_details"): + entry["quarantineTargetDetails"] = config["quarantine_target_details"] + + staging[table] = entry + return staging + + def _add_target_flows( + self, flow_group: Dict, target: Dict, table_name: str, spec_target: Dict, + node_lookup: Dict[str, Dict], has_cdc: bool, internal_sources: Dict[str, str], + registered_views: Set[str], flow_counter: int, + ) -> int: + """Add the flow(s) that write into `target`. Returns the next flow counter.""" + config = target.get("config", {}) + target_name = target.get("name") + inputs = self._normalize_inputs(config.get("input", [])) + enabled = target.get("enabled", True) + + # No inputs: only valid for snapshot CDC targets (the snapshot system + # reads its source directly), otherwise there is nothing to write. + if not inputs: + if config.get("cdc_snapshot_settings"): + flow_group["flows"][f"{self.FLOW_PREFIX}{target_name}_{flow_counter}"] = { + "flowType": FlowType.MERGE, + "flowDetails": {"targetTable": table_name}, + "enabled": enabled, + } + flow_counter += 1 + else: + self.logger.warning(f"Target '{target_name}' has no inputs, skipping") + return flow_counter + + target_cdc = config.get("cdc_settings") or config.get("cdc_apply_changes") + for explicit_flow, view in inputs: + input_node = node_lookup.get(view, {}) + is_sql_source = ( + input_node.get("node_type", "").lower() == self.SOURCE + and input_node.get("source_type") == "sql" + ) + + if target_cdc or (has_cdc and target is spec_target): + flow_type = FlowType.MERGE + elif is_sql_source: + flow_type = FlowType.APPEND_SQL + else: + flow_type = FlowType.APPEND_VIEW + + flow_name = explicit_flow or f"{self.FLOW_PREFIX}{view}_{flow_counter}" + flow_counter += 1 + + if flow_type == FlowType.APPEND_SQL: + flow = { + "flowType": flow_type, + "flowDetails": {"targetTable": table_name}, + "enabled": enabled, + } + input_config = input_node.get("config", {}) + if input_config.get("sql_statement"): + flow["flowDetails"]["sqlStatement"] = input_config["sql_statement"] + elif input_config.get("sql_path"): + flow["flowDetails"]["sqlPath"] = input_config["sql_path"] + else: + source_view, views = self._trace_inputs_to_views([view], node_lookup, internal_sources) + if not source_view: + self.logger.warning( + f"Could not determine source view for target '{target_name}' input '{view}'" + ) + continue + flow = { + "flowType": flow_type, + "flowDetails": {"sourceView": source_view, "targetTable": table_name}, + "enabled": enabled, + } + new_views = {k: v for k, v in views.items() if k not in registered_views} + if new_views: + flow["views"] = new_views + registered_views.update(views.keys()) + + if config.get("once"): + flow["flowDetails"]["once"] = True + + flow_group["flows"][flow_name] = flow + return flow_counter + + # ------------------------------------------------------------------ # + # Materialized-view targets + # ------------------------------------------------------------------ # + def _build_mv_specs( + self, spec_data: Dict, mv_targets: List[Dict], sources: List[Dict], all_nodes: List[Dict] + ) -> List[Dict]: + """Build one flow spec per materialized-view target.""" + node_lookup = {node.get("name"): node for node in all_nodes} + return [self._build_mv_spec(spec_data, t, sources, node_lookup) for t in mv_targets] + + def _build_mv_spec( + self, spec_data: Dict, mv_target: Dict, sources: List[Dict], node_lookup: Dict[str, Dict] + ) -> Dict: + """Build the flow spec for a single materialized-view target.""" + config = mv_target.get("config", {}) + mv_name = config.get("table") + + target_details = {"table": mv_name, "type": TableType.MATERIALIZED_VIEW} + target_details.update(_rename_keys(config, { + "sql_path": "sqlPath", "sql_statement": "sqlStatement", "refresh_policy": "refreshPolicy", + })) + # table_details wins over the same key on config (back-compat fallback). + detail_map = { + "private": "private", "comment": "comment", "spark_conf": "sparkConf", + "config_flags": "configFlags", "table_properties": "tableProperties", + "cluster_by_columns": "clusterByColumns", "cluster_by_auto": "clusterByAuto", + } + target_details.update({ + **_rename_keys(config, detail_map), + **_rename_keys(config.get("table_details", {}), detail_map), + }) + + flow_spec = { + "dataFlowId": spec_data.get("dataFlowId"), + "dataFlowGroup": spec_data.get("dataFlowGroup"), + "dataFlowType": spec_data.get("dataFlowType"), + "features": spec_data.get("features", {}), + "targetFormat": TargetType.DELTA, + "targetDetails": target_details, + "dataQualityExpectationsEnabled": config.get("data_quality_expectations_enabled", False), + "localPath": spec_data.get("localPath"), + } + if config.get("data_quality_expectations_path"): + flow_spec["dataQualityExpectationsPath"] = config["data_quality_expectations_path"] + if config.get("quarantine_mode"): + flow_spec["quarantineMode"] = config["quarantine_mode"] + if config.get("quarantine_target_details"): + flow_spec["quarantineTargetDetails"] = config["quarantine_target_details"] + + flow_group = {"flowGroupId": self.FLOW_GROUP_ID, "flows": {}} + inputs = self._normalize_inputs(config.get("input", [])) + if inputs: + internal_sources = self._identify_internal_sources(sources, [mv_target]) + explicit_flow, view = inputs[0] + source_view, views = self._trace_inputs_to_views([view], node_lookup, internal_sources) + if source_view: + target_details["sourceView"] = source_view + flow = {"flowType": FlowType.MATERIALIZED_VIEW, "flowDetails": {"targetTable": mv_name}} + if views: + for v in views.values(): + v["mode"] = Mode.BATCH # MVs use batch reads (spark.sql) + flow["views"] = views + flow_group["flows"][explicit_flow or f"{self.FLOW_PREFIX}{mv_name}"] = flow + + flow_spec["flowGroups"] = [flow_group] + return flow_spec + + # ------------------------------------------------------------------ # + # Views and internal sources + # ------------------------------------------------------------------ # + def _identify_internal_sources( + self, source_nodes: List[Dict], target_nodes: List[Dict] + ) -> Dict[str, str]: + """Map source-node name -> table for sources that read a table produced by + a target in this same spec. Such sources reference the produced table + directly instead of getting their own view.""" + target_tables = { + t["config"]["table"]: (t["config"].get("database") or None) + for t in target_nodes if t.get("config", {}).get("table") + } + + internal: Dict[str, str] = {} + for source in source_nodes: + if source.get("source_type", "delta") != "delta": + continue + config = source.get("config", {}) + table = config.get("table", "") + if table not in target_tables: + continue + source_db = config.get("database") or None + target_db = target_tables[table] + if not source_db or not target_db or source_db == target_db: + internal[source.get("name")] = table + return internal + + def _trace_inputs_to_views( + self, inputs: List[str], node_lookup: Dict[str, Dict], internal_sources: Dict[str, str] + ) -> Tuple[Optional[str], Dict]: + """Resolve an input to its source view name and the views it needs. + + When the input is a transformation, every source node in the spec is also + emitted as a view so SDP can resolve the references inside that + transformation's SQL/Python. + """ + views: Dict[str, Dict] = {} + immediate_input = inputs[0] if inputs else None + if not immediate_input: + return None, views + + # An internal source reads the produced table directly: no view needed. + if immediate_input in internal_sources: + return internal_sources[immediate_input], views + + source_view_name = None + references_transformation = False + for node_name in inputs: + if node_name in internal_sources: + continue + node = node_lookup.get(node_name) + if not node: + continue + + node_type = node.get("node_type", "").lower() + view_name = node.get("output_view_name") or self._view_name(node_name) + if node_type == self.SOURCE: + views[view_name] = self._build_source_view(node) + elif node_type == self.TRANSFORMATION: + views[view_name] = self._build_transform_view(node, node_lookup) + references_transformation = True + else: + continue + if source_view_name is None and node_name == immediate_input: + source_view_name = view_name + + # Register every source node so SDP can resolve the transformation's refs. + if references_transformation: + for name, node in node_lookup.items(): + if node.get("node_type", "").lower() != self.SOURCE: + continue + view_name = node.get("output_view_name") or self._view_name(name) + if view_name in views: + continue + if name in internal_sources: + # Internal sources need database "live" so SDP resolves them + # within the pipeline context. + node = {**node, "config": {**node.get("config", {}), "database": "live"}} + views[view_name] = self._build_source_view(node) + + if source_view_name is None: + source_view_name = self._view_name(immediate_input) + return source_view_name, views + + def _view_name(self, node_name: str) -> str: + """View name for a node: the node name, prefixed with v_ if needed.""" + return node_name if node_name.startswith(self.VIEW_PREFIX) else f"{self.VIEW_PREFIX}{node_name}" + + def _build_source_view(self, node: Dict) -> Dict: + """Build a view definition from a source node (camelCase output).""" + source_type = node.get("source_type", "delta") + config = node.get("config", {}) + details: Dict[str, Any] = {} + + if source_type == "delta": + if "database" in config: + details["database"] = config["database"] + if "table" in config: + details["table"] = config["table"] + details["cdfEnabled"] = config.get("cdf_enabled", False) + if "table_path" in config: + details["tablePath"] = config["table_path"] + elif source_type in ("cloudFiles", "batchFiles"): + details.update(_rename_keys(config, {"path": "path", "reader_options": "readerOptions"})) + elif source_type == "sql": + details.update(self._first_present(config, [("sql_path", "sqlPath"), ("sql_statement", "sqlStatement")])) + elif source_type == "python": + details.update(self._first_present(config, [("function_path", "functionPath"), ("python_module", "pythonModule")])) + details.update(_rename_keys(config, {"tokens": "tokens"})) + elif source_type == "kafka": + details.update(_rename_keys(config, {"reader_options": "readerOptions"})) + + details.update(_rename_keys(config, { + "select_exp": "selectExp", "where_clause": "whereClause", "schema_path": "schemaPath", + "starting_version_from_dlt_setup": "startingVersionFromDLTSetup", + "cdf_change_type_override": "cdfChangeTypeOverride", + })) + + # Backward compatibility: a source may carry an inline python transform + # that post-processes the read DataFrame via apply_transform(df). + python_transform = config.get("python_transform") + if python_transform: + mapped: Dict[str, Any] = {} + if python_transform.get("function_path"): + mapped["functionPath"] = python_transform["function_path"] + elif python_transform.get("functionPath"): + mapped["functionPath"] = python_transform["functionPath"] + if python_transform.get("python_module"): + mapped["module"] = python_transform["python_module"] + elif python_transform.get("module"): + mapped["module"] = python_transform["module"] + if python_transform.get("tokens"): + mapped["tokens"] = python_transform["tokens"] + details["pythonTransform"] = mapped + + return {"mode": config.get("mode", Mode.STREAM), "sourceType": source_type, "sourceDetails": details} + + def _build_transform_view(self, node: Dict, node_lookup: Dict[str, Dict]) -> Dict: + """Build a view definition from a transformation node (camelCase output).""" + transform_type = node.get("transformation_type", "sql") + config = node.get("config", {}) + + if transform_type == "passthrough": + return {"mode": Mode.STREAM, "sourceType": "delta", "sourceDetails": {}} + + if transform_type == "python": + # A python transformation is its own view: it reads its upstream view + # and applies apply_transform(df) via the framework's pythonTransform. + # The upstream is inferred from the graph; SDP resolves the live ref. + upstream_view, upstream_mode = self._infer_python_transform_upstream(node, node_lookup) + python_transform: Dict[str, Any] = {} + if config.get("function_path"): + python_transform["functionPath"] = config["function_path"] + elif config.get("python_module"): + python_transform["module"] = config["python_module"] + if config.get("tokens"): + python_transform["tokens"] = config["tokens"] + return { + "mode": upstream_mode, + "sourceType": "delta", + "sourceDetails": { + "database": "live", + "table": upstream_view, + "pythonTransform": python_transform, + }, + } + + # sql + details = self._first_present(config, [("sql_path", "sqlPath"), ("sql_statement", "sqlStatement")]) + return {"mode": Mode.BATCH, "sourceType": "sql", "sourceDetails": details} + + def _infer_python_transform_upstream(self, node: Dict, node_lookup: Dict[str, Dict]) -> Tuple[str, str]: + """Infer the upstream view a python transformation reads, plus its mode. + + Python transforms operate on a DataFrame, so (unlike SQL transforms) they + carry no textual reference to their input. SDP resolves view dependencies + automatically, so the upstream is inferred from the graph: the spec's + source feeds the transform. With several sources the first is used and a + warning is logged. + """ + sources = [n for n in node_lookup.values() + if n.get("node_type", "").lower() == self.SOURCE] + if not sources: + self.logger.warning( + "Python transformation '%s' has no upstream source node to read from.", + node.get("name"), + ) + return self._view_name(node.get("name")), Mode.STREAM + if len(sources) > 1: + self.logger.warning( + "Python transformation '%s' has multiple upstream sources %s; reading from '%s'.", + node.get("name"), [s.get("name") for s in sources], sources[0].get("name"), + ) + upstream = sources[0] + view = upstream.get("output_view_name") or self._view_name(upstream.get("name")) + mode = upstream.get("config", {}).get("mode", Mode.STREAM) + return view, mode + + @staticmethod + def _first_present(config: Dict, candidates: List[Tuple[str, str]]) -> Dict: + """Return {dst: config[src]} for the first (src, dst) whose src is set.""" + for src, dst in candidates: + if src in config: + return {dst: config[src]} + return {} diff --git a/src/schemas/flow_group.json b/src/schemas/flow_group.json index 738c09d..e9f5f09 100644 --- a/src/schemas/flow_group.json +++ b/src/schemas/flow_group.json @@ -50,6 +50,10 @@ "clusterByAuto": {"type": "boolean"}, "cdcSettings": {"$ref": "./definitions_main.json#/definitions/cdcSettings"}, "cdcSnapshotSettings": {"$ref": "./definitions_main.json#/definitions/cdcSnapshotSettings"}, + "dataQualityExpectationsEnabled": {"type": "boolean", "default": false}, + "dataQualityExpectationsPath": {"type": "string"}, + "quarantineMode": {"type": "string", "enum": ["off", "flag", "table"], "default": "off"}, + "quarantineTargetDetails": {"$ref": "./definitions_main.json#/definitions/quarantineTargetDetails"}, "configFlags": {"$ref": "./definitions_targets.json#/$defs/configFlags"} }, "additionalProperties": false diff --git a/src/schemas/main.json b/src/schemas/main.json index 1d34ce6..69dd9c3 100644 --- a/src/schemas/main.json +++ b/src/schemas/main.json @@ -4,7 +4,7 @@ "properties": { "dataFlowId": {"type": "string"}, "dataFlowGroup": {"type": "string"}, - "dataFlowType": {"type": "string", "enum": ["flow", "standard", "materialized_view"]}, + "dataFlowType": {"type": "string", "enum": ["flow", "standard", "materialized_view", "nodester"]}, "dataFlowVersion": {"type": "string"}, "tags": {"type": "object", "additionalProperties": true}, "features": { @@ -33,8 +33,16 @@ "dataFlowType": { "const": "materialized_view" } } }, - "then": { "$ref": "./spec_materialized_views.json#/$defs/materializedViewsSpec" } + "then": { "$ref": "./spec_materialized_views.json#/$defs/materializedViewsSpec" }, + "else": { + "if": { + "properties": { + "dataFlowType": { "const": "nodester" } + } + }, + "then": { "$ref": "./spec_nodester.json#/$defs/nodesterSpec" } + } } }, "required": ["dataFlowId", "dataFlowGroup", "dataFlowType"] -} \ No newline at end of file +} diff --git a/src/schemas/spec_nodester.json b/src/schemas/spec_nodester.json new file mode 100644 index 0000000..2dfefe1 --- /dev/null +++ b/src/schemas/spec_nodester.json @@ -0,0 +1,504 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://dlt-framework/schemas/spec_nodester.json", + "title": "Nodester Dataflow Specification", + "description": "Dataflow specification using a node-based architecture. Converts node graphs into Lakeflow Framework's flow-based format. All target-specific settings (CDC, data quality, quarantine) are specified on target nodes directly.", + "$defs": { + "nodesterSpec": { + "type": "object", + "properties": { + "dataFlowId": { + "type": "string" + }, + "dataFlowGroup": { + "type": "string" + }, + "dataFlowType": { + "const": "nodester" + }, + "dataFlowVersion": { + "type": "string" + }, + "tags": { + "type": "object", + "additionalProperties": true + }, + "features": { + "type": "object", + "additionalProperties": true + }, + "nodes": { + "type": "array", + "description": "Array of nodes representing sources, transformations, and targets in a pipeline graph", + "items": { + "$ref": "#/$defs/node" + }, + "minItems": 1 + } + }, + "required": [ + "dataFlowId", + "dataFlowGroup", + "dataFlowType", + "nodes" + ], + "additionalProperties": false + }, + "node": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique name for the node within the pipeline" + }, + "node_type": { + "type": "string", + "enum": [ + "source", + "transformation", + "target" + ], + "description": "Type of node: source (data input), transformation (processing), or target (data output)" + }, + "source_type": { + "type": "string", + "enum": [ + "batchFiles", + "cloudFiles", + "delta", + "deltaJoin", + "kafka", + "python", + "sql" + ], + "description": "Type of data source (required for source nodes)" + }, + "transformation_type": { + "type": "string", + "enum": [ + "sql", + "python", + "passthrough" + ], + "description": "Type of transformation to apply (required for transformation nodes)" + }, + "output_view_name": { + "type": "string", + "description": "Optional name for the output view. If omitted, defaults to the node name. Applicable to source and transformation nodes." + }, + "target_type": { + "type": "string", + "enum": [ + "delta", + "delta_sink", + "foreach_batch_sink", + "custom_python_sink" + ], + "default": "delta", + "description": "Target format type (required for target nodes). Default: delta" + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Whether this node is enabled in the pipeline" + }, + "config": { + "type": "object", + "description": "Node-specific configuration" + } + }, + "required": [ + "name", + "node_type" + ], + "allOf": [ + { + "if": { + "properties": { + "node_type": { + "const": "source" + } + } + }, + "then": { + "required": [ + "source_type" + ], + "properties": { + "config": { + "$ref": "#/$defs/source_node_config" + } + } + } + }, + { + "if": { + "properties": { + "node_type": { + "const": "transformation" + } + } + }, + "then": { + "required": [ + "transformation_type" + ], + "properties": { + "config": { + "$ref": "#/$defs/transformation_node_config" + } + } + } + }, + { + "if": { + "properties": { + "node_type": { + "const": "target" + } + } + }, + "then": { + "properties": { + "config": { + "$ref": "#/$defs/target_node_config" + } + } + } + } + ] + }, + "source_node_config": { + "type": "object", + "description": "Configuration for source nodes. The sourceType field has moved to the node level.", + "properties": { + "mode": { + "type": "string", + "enum": [ + "stream", + "batch" + ], + "default": "stream", + "description": "Processing mode for the source" + }, + "database": { + "type": "string" + }, + "table": { + "type": "string" + }, + "path": { + "type": "string" + }, + "cdf_enabled": { + "type": "boolean" + }, + "schema_path": { + "type": "string" + }, + "select_exp": { + "type": "array", + "items": { + "type": "string" + } + }, + "where_clause": { + "type": "array", + "items": { + "type": "string" + } + }, + "reader_options": { + "type": "object", + "additionalProperties": true + }, + "function_path": { + "type": "string", + "description": "Path to Python source function file" + }, + "python_module": { + "type": "string", + "description": "Python module path for source function" + }, + "tokens": { + "type": "object", + "description": "Token dictionary passed to Python sources" + }, + "sql_path": { + "type": "string", + "description": "Path to SQL file for SQL sources" + }, + "sql_statement": { + "type": "string", + "description": "Inline SQL statement for SQL sources" + }, + "starting_version_from_dlt_setup": { + "type": "boolean" + }, + "cdf_change_type_override": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": true + }, + "transformation_node_config": { + "type": "object", + "description": "Configuration for transformation nodes. The transformationType and outputViewName fields have moved to the node level.", + "properties": { + "sql_path": { + "type": "string" + }, + "sql_statement": { + "type": "string" + }, + "function_path": { + "type": "string" + }, + "python_module": { + "type": "string" + }, + "tokens": { + "type": "object" + } + }, + "additionalProperties": true + }, + "target_node_config": { + "type": "object", + "description": "Configuration for target nodes. All target-specific settings including CDC, data quality, quarantine, table migration, and sink config are specified here.", + "properties": { + "input": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string", + "description": "Upstream source/transformation node name. The SDP flow name is auto-generated." + }, + { + "type": "object", + "properties": { + "view": { + "type": "string", + "description": "Upstream source or transformation node name feeding this target" + }, + "flow": { + "type": "string", + "description": "Explicit SDP flow name. Pin this to keep flow names stable across changes (renaming a flow forces a full refresh in SDP)." + } + }, + "required": [ + "view" + ], + "additionalProperties": false + } + ] + }, + "description": "Inputs feeding this target. Each item is either an upstream node name (string, flow name auto-generated) or an object {view, flow} that pins the SDP flow name." + }, + "table": { + "type": "string", + "description": "Target table name (for delta targets)" + }, + "database": { + "type": "string", + "description": "Target database/schema name" + }, + "table_type": { + "type": "string", + "enum": [ + "st", + "mv" + ], + "default": "st", + "description": "Table type: st (streaming table) or mv (materialized view). Default: st" + }, + "schema_path": { + "type": "string", + "pattern": "\\.(json|ddl)$" + }, + "table_properties": { + "type": "object" + }, + "path": { + "type": "string" + }, + "partition_columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "cluster_by_columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "cluster_by_auto": { + "type": "boolean" + }, + "comment": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "row_filter": { + "type": "string" + }, + "config_flags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Configuration flags, e.g. ['disableOperationalMetadata']" + }, + "once": { + "type": "boolean", + "default": false, + "description": "Execute flow to this target only once (batch mode)" + }, + "cdc_settings": { + "$ref": "./definitions_main.json#/definitions/cdcSettings" + }, + "cdc_apply_changes": { + "$ref": "./definitions_main.json#/definitions/cdcSettings" + }, + "cdc_snapshot_settings": { + "$ref": "./definitions_main.json#/definitions/cdcSnapshotSettings" + }, + "data_quality_expectations_enabled": { + "type": "boolean", + "default": false, + "description": "Enable data quality expectations for this target" + }, + "data_quality_expectations_path": { + "type": "string", + "description": "Path to the data quality expectations file" + }, + "quarantine_mode": { + "type": "string", + "enum": [ + "off", + "flag", + "table" + ], + "default": "off", + "description": "Quarantine mode for this target" + }, + "quarantine_target_details": { + "$ref": "./definitions_main.json#/definitions/quarantineTargetDetails", + "description": "Quarantine target configuration when quarantineMode is 'table'" + }, + "table_migration_details": { + "type": "object", + "description": "Table migration configuration", + "properties": { + "catalog_type": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "auto_starting_versions_enabled": { + "type": "boolean" + }, + "source_details": { + "type": "object" + } + } + }, + "name": { + "type": "string", + "description": "Sink name (for sink target formats)" + }, + "sink_type": { + "type": "string", + "description": "Sink sub-type: basic_sql, python_function" + }, + "sink_config": { + "type": "object", + "description": "Sink-specific configuration (sqlPath, functionPath, tokens, etc.)" + }, + "sink_options": { + "type": "object", + "description": "Delta sink options (path, tableName)" + }, + "sql_path": { + "type": "string", + "description": "SQL file path (for materialized view targets)" + }, + "sql_statement": { + "type": "string", + "description": "Inline SQL statement (for materialized view targets)" + }, + "refresh_policy": { + "type": "string", + "description": "Materialized view refresh policy" + }, + "table_details": { + "type": "object", + "description": "Additional table details (for materialized view targets)", + "properties": { + "private": { + "type": "boolean" + }, + "comment": { + "type": "string" + }, + "spark_conf": { + "type": "object" + }, + "config_flags": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "data_quality_expectations_enabled": { + "const": true + } + }, + "required": [ + "data_quality_expectations_enabled" + ] + }, + "then": { + "required": [ + "data_quality_expectations_path" + ] + } + }, + { + "if": { + "properties": { + "quarantine_mode": { + "const": "table" + } + }, + "required": [ + "quarantine_mode" + ] + }, + "then": { + "required": [ + "quarantine_target_details" + ] + } + } + ] + } + } +} From 9c9c74a16e41d022fd102e4eee1a9cf774989d15 Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 26 Jun 2026 17:05:32 +1000 Subject: [PATCH 02/19] update json schema for nodester --- .../dataflow_spec_ref_main_nodester.rst | 90 ++- docs/source/feature_auto_complete.rst | 7 + scripts/migrate_to_nodester.py | 641 +++++++++--------- .../dataflow_spec_builder.py | 14 +- .../transformer/nodester.py | 3 - src/schemas/main.json | 55 +- src/schemas/spec_nodester.json | 144 +++- 7 files changed, 575 insertions(+), 379 deletions(-) diff --git a/docs/source/dataflow_spec_ref_main_nodester.rst b/docs/source/dataflow_spec_ref_main_nodester.rst index 71a3fa0..4fd332d 100644 --- a/docs/source/dataflow_spec_ref_main_nodester.rst +++ b/docs/source/dataflow_spec_ref_main_nodester.rst @@ -116,6 +116,17 @@ view it reads; the target lists the transformation in its ``input``: ] } +Editor autocomplete and validation +================================== + +Nodester specs have a JSON Schema (``src/schemas/spec_nodester.json``), so editors +can offer key/value autocomplete and inline validation while you author them. The +schema is wired through ``main.json``, which routes any ``*_main.json`` file to +the correct spec schema based on its ``data_flow_type``. Add the ``json.schemas`` +mapping described in :doc:`feature_auto_complete` — the same ``*_main.json`` +mapping enables IntelliSense for nodester (no nodester-specific configuration is +required). + .. _dataflow-spec-nodester-metadata-configuration: Dataflow metadata configuration @@ -254,7 +265,7 @@ Transformation node configuration - **Description** * - **transformation_type** - ``string`` - - ``sql``, ``python``, or ``passthrough``. + - ``sql`` or ``python``. * - **sql_path** / **sql_statement** - ``string`` - For ``sql`` transforms. The SQL names the view(s) it reads, e.g. @@ -344,6 +355,83 @@ Defining the flow name keeps it stable across edits. This matters because renaming a flow forces a full refresh in SDP, so a defined name avoids triggering one. The string and object forms can be mixed in the same list. +Snapshot CDC targets +------------------- + +A target's ``cdc_snapshot_settings`` configures snapshot-based CDC. There are two +modes, set via ``snapshot_type``: + +- **historical** — the snapshot source is built **inside** the settings via + ``source_type`` (``file`` or ``table``) and a ``source`` object. No source node + is required; the framework reads the files/table directly. +- **periodic** — the target reads from an upstream **source node** chained via + ``input``; ``source_type`` / ``source`` are not used. + +For historical snapshots the ``source`` object fields depend on ``source_type``: + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - **Field** + - **source_type** + - **Description** + * - **path** + - ``file`` + - Snapshot file path/pattern; may contain a ``{version}`` token. + * - **format** + - ``file`` + - File format, e.g. ``csv``, ``parquet``, ``json``. + * - **reader_options** + - ``file`` + - Options passed to the file reader, e.g. ``{"header": "true"}``. + * - **datetime_format** + - ``file`` + - Format used to parse the ``{version}`` token for timestamp versioning. + * - **schema_path** + - ``file`` + - Schema file applied when reading the snapshot files. + * - **recursiveFileLookup** + - ``file`` + - Recurse into subdirectories when discovering snapshot files. + * - **table** + - ``table`` + - Source table, e.g. ``{schema}.snapshot_source``. + * - **version_column** + - ``table`` + - Column carrying the snapshot version. + * - **version_type** + - both + - How versions are interpreted, e.g. ``timestamp`` or ``integer``. + * - **select_exp** + - both + - Select expressions applied to the snapshot source. + +Example — historical file snapshot: + +.. code-block:: json + + { + "name": "target_customer_snapshot", + "node_type": "target", + "config": { + "table": "customer_snapshot", + "cdc_snapshot_settings": { + "keys": ["CUSTOMER_ID"], + "scd_type": "2", + "snapshot_type": "historical", + "source_type": "file", + "source": { + "format": "csv", + "path": "{sample_file_location}/snapshot_customer/customer_{version}.csv", + "reader_options": { "header": "true" }, + "version_type": "timestamp", + "datetime_format": "%Y_%m_%d" + } + } + } + } + .. _dataflow-spec-nodester-mv: Materialized view targets diff --git a/docs/source/feature_auto_complete.rst b/docs/source/feature_auto_complete.rst index a116155..7950971 100644 --- a/docs/source/feature_auto_complete.rst +++ b/docs/source/feature_auto_complete.rst @@ -19,6 +19,13 @@ The framework uses JSON Schema-based IntelliSense when creating or editing Data - Validation (checking for errors based on the schema) - Quick Fixes (where applicable) +The ``*_main.json`` mapping below covers **all** Data Flow Spec types — standard, +flow, materialized view, and :doc:`nodester `. +``main.json`` inspects the ``data_flow_type`` / ``dataFlowType`` field and routes +to the matching schema automatically, so a single mapping enables IntelliSense +for every spec type. Nodester specs are authored in ``snake_case`` and the schema +validates them as written. + Autocompletion Example ~~~~~~~~~~~~~~~~~~~~~~ *Suggesting keys* diff --git a/scripts/migrate_to_nodester.py b/scripts/migrate_to_nodester.py index fedd461..3a22b74 100644 --- a/scripts/migrate_to_nodester.py +++ b/scripts/migrate_to_nodester.py @@ -2,8 +2,14 @@ """ Migrate Lakeflow Framework dataflow specs to Nodester format. -Converts standard, flow, and materialized_view specs into the -node-based Nodester specification format. +Converts standard, flow, and materialized_view specs into the node-based +Nodester specification format. + +Nodester specs are snake_case throughout, so this script converts the +camelCase field names used by the standard/flow/materialized_view formats +(e.g. ``cdfEnabled`` -> ``cdf_enabled``, ``tableProperties`` -> +``table_properties``) while leaving opaque value maps (table properties, +reader options, spark conf, tokens) untouched. Usage: python migrate_to_nodester.py [--output ] @@ -23,9 +29,160 @@ from typing import Dict, List, Optional, Any, Tuple +# ─── camelCase -> snake_case key maps ──────────────────────────────────────── +# Only structural keys are renamed. Values of opaque maps (table properties, +# reader options, spark conf, tokens) are copied verbatim — never recursed into. + +_SOURCE_DETAIL_MAP = { + "cdfEnabled": "cdf_enabled", + "tablePath": "table_path", + "readerOptions": "reader_options", + "functionPath": "function_path", + "pythonModule": "python_module", + "sqlPath": "sql_path", + "sqlStatement": "sql_statement", + "selectExp": "select_exp", + "whereClause": "where_clause", + "schemaPath": "schema_path", + "startingVersionFromDLTSetup": "starting_version_from_dlt_setup", + "cdfChangeTypeOverride": "cdf_change_type_override", + "pythonTransform": "python_transform", +} + +_TARGET_DETAIL_MAP = { + "schemaPath": "schema_path", + "tableProperties": "table_properties", + "partitionColumns": "partition_columns", + "clusterByColumns": "cluster_by_columns", + "clusterByAuto": "cluster_by_auto", + "sparkConf": "spark_conf", + "rowFilter": "row_filter", + "configFlags": "config_flags", +} + +# Snapshot settings carry camelCase keys at both the top level and inside the +# nested `source` object. `recursiveFileLookup` is intentionally left camelCase +# (the framework reads it as-is). +_SNAPSHOT_MAP = { + "snapshotType": "snapshot_type", + "sourceType": "source_type", + "versionType": "version_type", + "versionColumn": "version_column", + "startingVersion": "starting_version", + "datetimeFormat": "datetime_format", + "readerOptions": "reader_options", + "schemaPath": "schema_path", + "selectExp": "select_exp", + "deduplicateMode": "deduplicate_mode", +} + +_QUARANTINE_MAP = { + "targetFormat": "target_format", + "clusterByAuto": "cluster_by_auto", + "clusterByColumns": "cluster_by_columns", + "partitionColumns": "partition_columns", +} + +_TABLE_MIGRATION_MAP = { + "catalogType": "catalog_type", + "autoStartingVersionsEnabled": "auto_starting_versions_enabled", + "sourceDetails": "source_details", + "tableName": "table_name", +} + +_PYTHON_TRANSFORM_MAP = { + "functionPath": "function_path", + "pythonModule": "python_module", + "module": "module", +} + + +def _rename(d: Dict, mapping: Dict[str, str]) -> Dict: + """Return a new dict with top-level keys renamed per mapping; values as-is.""" + if not isinstance(d, dict): + return d + return {mapping.get(k, k): v for k, v in d.items()} + + +def _convert_source_details(details: Dict) -> Dict: + """camelCase -> snake_case for a source/view sourceDetails block.""" + out = _rename(details, _SOURCE_DETAIL_MAP) + if isinstance(out.get("python_transform"), dict): + out["python_transform"] = _rename(out["python_transform"], _PYTHON_TRANSFORM_MAP) + return out + + +def _convert_snapshot(cs: Dict) -> Dict: + """camelCase -> snake_case for cdcSnapshotSettings, including nested source.""" + out = _rename(cs, _SNAPSHOT_MAP) + if isinstance(out.get("source"), dict): + out["source"] = _rename(out["source"], _SNAPSHOT_MAP) + return out + + +def _get(src: Dict, camel: str, snake: str): + """Read a value that may be present under either camelCase or snake_case.""" + if camel in src: + return src[camel] + return src.get(snake) + + +def _add_target_settings(config: Dict, src: Dict) -> None: + """Copy CDC / DQ / quarantine / table-migration settings onto a target config. + + `src` is the spec (standard), a staging-table config (flow), or an MV config. + Reads either casing; writes snake_case. + """ + cdc = _get(src, "cdcSettings", "cdc_settings") + if cdc: + config["cdc_settings"] = cdc + cdc_apply = _get(src, "cdcApplyChanges", "cdc_apply_changes") + if cdc_apply: + config["cdc_apply_changes"] = cdc_apply + snapshot = _get(src, "cdcSnapshotSettings", "cdc_snapshot_settings") + if snapshot: + config["cdc_snapshot_settings"] = _convert_snapshot(snapshot) + + dq_enabled = _get(src, "dataQualityExpectationsEnabled", "data_quality_expectations_enabled") + if dq_enabled: + config["data_quality_expectations_enabled"] = dq_enabled + dq_path = _get(src, "dataQualityExpectationsPath", "data_quality_expectations_path") + if dq_path: + config["data_quality_expectations_path"] = dq_path + + quarantine_mode = _get(src, "quarantineMode", "quarantine_mode") + if quarantine_mode: + config["quarantine_mode"] = quarantine_mode + quarantine_details = _get(src, "quarantineTargetDetails", "quarantine_target_details") + if quarantine_details: + config["quarantine_target_details"] = _rename(quarantine_details, _QUARANTINE_MAP) + + table_migration = _get(src, "tableMigrationDetails", "table_migration_details") + if table_migration: + config["table_migration_details"] = _rename(table_migration, _TABLE_MIGRATION_MAP) + + +def _result_envelope(spec: Dict, nodes: List[Dict]) -> Dict: + """Build the top-level nodester spec (snake_case metadata).""" + result = { + "data_flow_id": spec.get("dataFlowId") or spec.get("data_flow_id"), + "data_flow_group": spec.get("dataFlowGroup") or spec.get("data_flow_group"), + "data_flow_type": "nodester", + "nodes": nodes, + } + version = _get(spec, "dataFlowVersion", "data_flow_version") + if version: + result["data_flow_version"] = version + if spec.get("tags"): + result["tags"] = spec["tags"] + if spec.get("features"): + result["features"] = spec["features"] + return result + + def migrate_spec(spec: Dict) -> Dict: """Convert a dataflow spec to nodester format based on its dataFlowType.""" - spec_type = spec.get("dataFlowType", "standard").lower() + spec_type = (spec.get("dataFlowType") or spec.get("data_flow_type") or "standard").lower() if spec_type == "standard": return _migrate_standard(spec) @@ -33,6 +190,8 @@ def migrate_spec(spec: Dict) -> Dict: return _migrate_flow(spec) elif spec_type == "materialized_view": return _migrate_materialized_view(spec) + elif spec_type == "nodester": + return spec # already nodester else: print(f" Warning: Unknown dataFlowType '{spec_type}', treating as standard") return _migrate_standard(spec) @@ -42,127 +201,55 @@ def migrate_spec(spec: Dict) -> Dict: def _migrate_standard(spec: Dict) -> Dict: """Convert a standard spec to nodester.""" - nodes = [] - - # Build source node - source_id = spec.get("sourceViewName", "source").replace("v_", "", 1) - if not source_id: - source_id = "source" - source_node = _build_source_node_from_standard(source_id, spec) - nodes.append(source_node) - - # Build target node - target_config, target_type = _build_target_config_from_standard(spec) - target_id = f"target_{target_config.get('table', target_config.get('name', 'output'))}" + source_id = spec.get("sourceViewName") or "v_source" + source_node = _build_source_node(source_id, spec.get("sourceType", "delta"), + spec.get("sourceDetails", {}), spec.get("mode", "stream")) + + target_config, target_type = _build_spec_target(spec) target_config["input"] = [source_id] - target_node: Dict[str, Any] = { - "name": target_id, - "node_type": "target", - } + target_name = target_config.get("table") or target_config.get("name") or "output" + target_node: Dict[str, Any] = {"name": f"target_{target_name}", "node_type": "target"} if target_type != "delta": target_node["target_type"] = target_type target_node["config"] = target_config - nodes.append(target_node) - - # Build nodester spec - result = { - "data_flow_id": spec.get("dataFlowId"), - "data_flow_group": spec.get("dataFlowGroup"), - "data_flow_type": "nodester", - "nodes": nodes - } - if spec.get("dataFlowVersion"): - result["data_flow_version"] = spec["data_flow_version"] - if spec.get("tags"): - result["tags"] = spec["tags"] - if spec.get("features"): - result["features"] = spec["features"] + return _result_envelope(spec, [source_node, target_node]) - return result - - -def _build_source_node_from_standard(source_id: str, spec: Dict) -> Dict: - """Build a source node from a standard spec's source fields.""" - source_type = spec.get("sourceType", "delta") - source_details = spec.get("sourceDetails", {}) - mode = spec.get("mode", "stream") +def _build_source_node(name: str, source_type: str, source_details: Dict, mode: str) -> Dict: + """Build a nodester source node (snake_case) from camelCase source details.""" config: Dict[str, Any] = {} - if mode: config["mode"] = mode + config.update(_convert_source_details(source_details)) - # Copy source details based on type - if source_type == "delta": - for key in ["database", "table", "cdfEnabled", "tablePath"]: - if key in source_details: - config[key] = source_details[key] - elif source_type in ("cloudFiles", "batchFiles"): - for key in ["path", "readerOptions"]: - if key in source_details: - config[key] = source_details[key] - elif source_type == "python": - for key in ["functionPath", "pythonModule", "tokens"]: - if key in source_details: - config[key] = source_details[key] - elif source_type == "sql": - for key in ["sqlPath", "sqlStatement"]: - if key in source_details: - config[key] = source_details[key] - elif source_type == "kafka": - if "readerOptions" in source_details: - config["reader_options"] = source_details["readerOptions"] - - # Common optional source properties - for key in ["selectExp", "whereClause", "schemaPath"]: - if key in source_details: - config[key] = source_details[key] - - node: Dict[str, Any] = {"name": source_id, "node_type": "source", "source_type": source_type} + node: Dict[str, Any] = {"name": name, "node_type": "source", "source_type": source_type} if config: node["config"] = config return node -def _build_target_config_from_standard(spec: Dict) -> Tuple[Dict, str]: - """Build target node config from standard spec's target and spec-level settings. - - Returns (config_dict, target_type_string). - """ +def _build_spec_target(spec: Dict) -> Tuple[Dict, str]: + """Build the spec-level target config (delta or sink). Returns (config, target_type).""" target_details = spec.get("targetDetails", {}) target_format = spec.get("targetFormat", "delta") - config: Dict[str, Any] = {} - if target_format != "delta": - # For sinks, copy all target details directly - for key, val in target_details.items(): - config[key] = val + # Sink target: map the known sink fields to snake_case. + config = {} + if "name" in target_details: + config["name"] = target_details["name"] + if "type" in target_details: + config["sink_type"] = target_details["type"] + if "config" in target_details: + config["sink_config"] = target_details["config"] + sink_options = _get(target_details, "sinkOptions", "sink_options") + if sink_options is not None: + config["sink_options"] = sink_options else: - # Delta target - for key in ["table", "database", "schemaPath", "tableProperties", "path", - "partitionColumns", "clusterByColumns", "clusterByAuto", - "comment", "sparkConf", "rowFilter", "configFlags"]: - if key in target_details: - config[key] = target_details[key] - - # Move spec-level settings to target config - for key in ["cdc_settings", "cdc_apply_changes", "cdc_snapshot_settings"]: - if spec.get(key): - config[key] = spec[key] - - if spec.get("dataQualityExpectationsEnabled"): - config["data_quality_expectations_enabled"] = spec["data_quality_expectations_enabled"] - if spec.get("dataQualityExpectationsPath"): - config["data_quality_expectations_path"] = spec["data_quality_expectations_path"] - if spec.get("quarantineMode"): - config["quarantine_mode"] = spec["quarantine_mode"] - if spec.get("quarantineTargetDetails"): - config["quarantine_target_details"] = spec["quarantine_target_details"] - if spec.get("tableMigrationDetails"): - config["table_migration_details"] = spec["table_migration_details"] + config = _rename(target_details, _TARGET_DETAIL_MAP) + _add_target_settings(config, spec) return config, target_format @@ -170,217 +257,140 @@ def _build_target_config_from_standard(spec: Dict) -> Tuple[Dict, str]: def _migrate_flow(spec: Dict) -> Dict: """Convert a flow spec to nodester.""" - nodes = [] - node_ids = set() + nodes: List[Dict] = [] + node_ids: set = set() flow_groups = spec.get("flowGroups", []) - # Collect all staging table names (these become target nodes) staging_table_names = set() for fg in flow_groups: - for name in fg.get("stagingTables", {}).keys(): - staging_table_names.add(name) + staging_table_names.update(fg.get("stagingTables", {}).keys()) + + def find_target_node(target_name: str) -> Optional[Dict]: + for n in nodes: + if n["name"] == f"target_{target_name}": + return n + return None - # Process each flow group for fg in flow_groups: staging_tables = fg.get("stagingTables", {}) flows = fg.get("flows", {}) - # Create target nodes for staging tables + # Staging tables -> target nodes (inputs filled while processing flows). for stg_name, stg_config in staging_tables.items(): target_id = f"target_{stg_name}" if target_id in node_ids: continue node_ids.add(target_id) + config = _build_target_config_from_staging(stg_name, stg_config) + config["input"] = [] + nodes.append({"name": target_id, "node_type": "target", "config": config}) - target_config = _build_target_config_from_staging(stg_name, stg_config) - target_config["input"] = [] # placeholder, filled when processing flows - nodes.append({ - "name": target_id, - "node_type": "target", - "config": target_config, - }) - - # Process flows for flow_name, flow_config in flows.items(): flow_type = flow_config.get("flowType") flow_details = flow_config.get("flowDetails", {}) target_table = flow_details.get("targetTable") views = flow_config.get("views", {}) - # Create source/transform nodes from views source_node_id = None + + # Views -> source nodes. for view_name, view_config in views.items(): - if view_name in node_ids: - continue - node_ids.add(view_name) - - view_source_details = view_config.get("sourceDetails", {}) - view_source_type = view_config.get("sourceType", "delta") - view_mode = view_config.get("mode", "stream") - - # Check if this view reads from a staging table (internal source) - view_db = view_source_details.get("database", "") - view_table = view_source_details.get("table", "") - is_internal = ( - view_db == "live" and view_table in staging_table_names - ) - - # Build source node config - src_config: Dict[str, Any] = {} - if view_mode: - src_config["mode"] = view_mode - - # Copy source details - for key, val in view_source_details.items(): - src_config[key] = val - - src_node: Dict[str, Any] = { - "name": view_name, - "node_type": "source", - "source_type": view_source_type, - } - if src_config: - src_node["config"] = src_config - nodes.append(src_node) + if view_name not in node_ids: + node_ids.add(view_name) + nodes.append(_build_source_node( + view_name, + view_config.get("sourceType", "delta"), + view_config.get("sourceDetails", {}), + view_config.get("mode", "stream"), + )) source_node_id = view_name - # Handle append_sql flows (SQL is in flowDetails, not views) + # append_sql flows carry SQL directly in flowDetails (no view). if flow_type == "append_sql": - sql_source_id = f"sql_{flow_name}" + sql_source_id = f"v_sql_{flow_name}" if sql_source_id not in node_ids: node_ids.add(sql_source_id) sql_config: Dict[str, Any] = {} if flow_details.get("sqlStatement"): - sql_config["sql_statement"] = flow_details["sql_statement"] + sql_config["sql_statement"] = flow_details["sqlStatement"] elif flow_details.get("sqlPath"): - sql_config["sql_path"] = flow_details["sql_path"] - sql_node: Dict[str, Any] = { - "name": sql_source_id, - "node_type": "source", - "source_type": "sql", - } + sql_config["sql_path"] = flow_details["sqlPath"] + node: Dict[str, Any] = {"name": sql_source_id, "node_type": "source", "source_type": "sql"} if sql_config: - sql_node["config"] = sql_config - nodes.append(sql_node) - source_node_id = sql_source_id - - # If no views and not append_sql, use the sourceView reference + node["config"] = sql_config + nodes.append(node) + source_node_id = sql_source_id + + # Fall back to an explicit sourceView reference. When that reference + # is a staging table (a table produced by another target in this + # spec), nodester models it as an explicit "internal" source node + # that reads the staging table; the transformer auto-detects it. if not source_node_id and flow_details.get("sourceView"): - source_node_id = flow_details["source_view"] - - # Connect flow to target - if target_table and source_node_id: - # Find the target node - target_node_id = f"target_{target_table}" - target_found = False - for node in nodes: - if node["name"] == target_node_id: - if source_node_id not in node.get("inputs", []): - node.setdefault("config", {}).setdefault("input", []).append(source_node_id) - target_found = True - break - - if not target_found: - # This is the main target - if target_node_id not in node_ids: - node_ids.add(target_node_id) - main_target_config = _build_target_config_from_spec_level(spec) - # Add once flag if present - if flow_details.get("once"): - main_target_config["once"] = True - main_target_config["input"] = [source_node_id] + source_view = flow_details["sourceView"] + staging_key = source_view.split(".")[-1] + if staging_key in staging_table_names: + internal_id = source_view if source_view.startswith("v_") else f"v_{staging_key}" + if internal_id not in node_ids: + node_ids.add(internal_id) nodes.append({ - "name": target_node_id, - "node_type": "target", - "config": main_target_config + "name": internal_id, + "node_type": "source", + "source_type": "delta", + "config": {"mode": "stream", "table": staging_key}, }) - else: - # Find existing and add input - for node in nodes: - if node["name"] == target_node_id: - node.setdefault("config", {}).setdefault("input", []).append(source_node_id) - break + source_node_id = internal_id + else: + source_node_id = source_view - # Clean up empty inputSources + if not (target_table and source_node_id): + continue + + # Strip any schema qualifier from the target table name when matching. + target_key = target_table.split(".")[-1] + target_node = find_target_node(target_key) + if target_node is not None: + target_node["config"].setdefault("input", []).append(source_node_id) + else: + # Main (spec-level) target (delta or sink). + target_id = f"target_{target_key}" + if target_id not in node_ids: + node_ids.add(target_id) + main_config, target_type = _build_spec_target(spec) + if flow_details.get("once"): + main_config["once"] = True + main_config["input"] = [source_node_id] + main_node: Dict[str, Any] = {"name": target_id, "node_type": "target"} + if target_type != "delta": + main_node["target_type"] = target_type + main_node["config"] = main_config + nodes.append(main_node) + + # Drop placeholder empty input lists. for node in nodes: config = node.get("config", {}) if config.get("input") == []: del config["input"] - result = { - "data_flow_id": spec.get("dataFlowId"), - "data_flow_group": spec.get("dataFlowGroup"), - "data_flow_type": "nodester", - "nodes": nodes - } - - if spec.get("dataFlowVersion"): - result["data_flow_version"] = spec["data_flow_version"] - if spec.get("tags"): - result["tags"] = spec["tags"] - if spec.get("features"): - result["features"] = spec["features"] - - return result + return _result_envelope(spec, nodes) def _build_target_config_from_staging(table_name: str, stg_config: Dict) -> Dict: - """Build target config from staging table definition.""" + """Build a target config from a flow staging-table definition.""" config: Dict[str, Any] = {"table": table_name} - - for key in ["tableProperties", "schemaPath", "clusterByColumns", - "clusterByAuto", "partitionColumns", "database", "configFlags"]: - if key in stg_config: - config[key] = stg_config[key] - - # CDC settings - for key in ["cdc_settings", "cdc_apply_changes", "cdc_snapshot_settings"]: - if key in stg_config: - config[key] = stg_config[key] - - # DQ settings - if stg_config.get("dataQualityExpectationsEnabled"): - config["data_quality_expectations_enabled"] = stg_config["data_quality_expectations_enabled"] - if stg_config.get("dataQualityExpectationsPath"): - config["data_quality_expectations_path"] = stg_config["data_quality_expectations_path"] - - # Quarantine - if stg_config.get("quarantineMode"): - config["quarantine_mode"] = stg_config["quarantine_mode"] - if stg_config.get("quarantineTargetDetails"): - config["quarantine_target_details"] = stg_config["quarantine_target_details"] - - return config - - -def _build_target_config_from_spec_level(spec: Dict) -> Dict: - """Build main target config from spec-level target details and settings.""" - target_details = spec.get("targetDetails", {}) - config: Dict[str, Any] = {} - - for key in ["table", "database", "schema_path", "table_properties", "path", - "partition_columns", "cluster_by_columns", "cluster_by_auto", - "comment", "spark_conf", "row_filter", "config_flags"]: - if key in target_details: - config[key] = target_details[key] - - # Spec-level settings → target config - for key in ["cdc_settings", "cdc_apply_changes", "cdc_snapshot_settings"]: - if spec.get(key): - config[key] = spec[key] - - if spec.get("dataQualityExpectationsEnabled"): - config["data_quality_expectations_enabled"] = spec["data_quality_expectations_enabled"] - if spec.get("dataQualityExpectationsPath"): - config["data_quality_expectations_path"] = spec["data_quality_expectations_path"] - if spec.get("quarantineMode"): - config["quarantine_mode"] = spec["quarantine_mode"] - if spec.get("quarantineTargetDetails"): - config["quarantine_target_details"] = spec["quarantine_target_details"] - if spec.get("tableMigrationDetails"): - config["table_migration_details"] = spec["table_migration_details"] - + config.update(_rename( + {k: v for k, v in stg_config.items() if k != "type"}, + _TARGET_DETAIL_MAP, + )) + # Drop settings handled separately so they get correct snake_case + conversion. + for k in ("cdcSettings", "cdc_settings", "cdcApplyChanges", "cdc_apply_changes", + "cdcSnapshotSettings", "cdc_snapshot_settings", + "dataQualityExpectationsEnabled", "data_quality_expectations_enabled", + "dataQualityExpectationsPath", "data_quality_expectations_path", + "quarantineMode", "quarantine_mode", + "quarantineTargetDetails", "quarantine_target_details"): + config.pop(k, None) + _add_target_settings(config, stg_config) return config @@ -389,73 +399,41 @@ def _build_target_config_from_spec_level(spec: Dict) -> Dict: def _migrate_materialized_view(spec: Dict) -> Dict: """Convert a materialized_view spec to nodester.""" materialized_views = spec.get("materializedViews", {}) - nodes = [] + nodes: List[Dict] = [] for mv_name, mv_config in materialized_views.items(): - target_config: Dict[str, Any] = { - "table": mv_name, - "table_type": "mv" - } - - # Copy MV properties - for key in ["sql_path", "sql_statement", "refresh_policy"]: - if key in mv_config: - target_config[key] = mv_config[key] - - # Copy table details - if mv_config.get("tableDetails"): - target_config["table_details"] = mv_config["table_details"] - - # Copy DQ settings - if mv_config.get("dataQualityExpectationsEnabled"): - target_config["data_quality_expectations_enabled"] = mv_config["data_quality_expectations_enabled"] - if mv_config.get("dataQualityExpectationsPath"): - target_config["data_quality_expectations_path"] = mv_config["data_quality_expectations_path"] - if mv_config.get("quarantineMode"): - target_config["quarantine_mode"] = mv_config["quarantine_mode"] - if mv_config.get("quarantineTargetDetails"): - target_config["quarantine_target_details"] = mv_config["quarantine_target_details"] - - # Handle source view: MV source views are no longer inlined on the - # target. Emit a source node and chain it into the MV via `input`. - source_view = mv_config.get("source_view") - input_sources = [] - if source_view: - source_id = source_view.get("sourceViewName", f"source_{mv_name}") - # Create a source node for the view - src_config: Dict[str, Any] = {"mode": "stream"} - for key, val in source_view.get("sourceDetails", {}).items(): - src_config[key] = val - nodes.append({ - "name": source_id, - "node_type": "source", - "source_type": source_view.get("sourceType", "delta"), - "config": src_config - }) - input_sources = [source_id] - - target_node: Dict[str, Any] = { - "name": f"target_{mv_name}", - "node_type": "target", - "config": target_config - } - if input_sources: - target_config["input"] = input_sources - nodes.append(target_node) + target_config: Dict[str, Any] = {"table": mv_name, "table_type": "mv"} - result = { - "data_flow_id": spec.get("dataFlowId"), - "data_flow_group": spec.get("dataFlowGroup"), - "data_flow_type": "nodester", - "nodes": nodes - } + for camel, snake in (("sqlPath", "sql_path"), ("sqlStatement", "sql_statement"), + ("refreshPolicy", "refresh_policy")): + val = _get(mv_config, camel, snake) + if val is not None: + target_config[snake] = val - if spec.get("tags"): - result["tags"] = spec["tags"] - if spec.get("features"): - result["features"] = spec["features"] + table_details = _get(mv_config, "tableDetails", "table_details") + if table_details: + target_config["table_details"] = _rename(table_details, _TARGET_DETAIL_MAP) - return result + _add_target_settings(target_config, mv_config) + + # MV source views are no longer inlined on the target: emit a source node + # and chain it into the MV via `input`. + source_view = _get(mv_config, "sourceView", "source_view") + if isinstance(source_view, dict) and source_view: + source_id = (source_view.get("sourceViewName") + or source_view.get("source_view_name") + or f"v_source_{mv_name}") + nodes.append(_build_source_node( + source_id, + source_view.get("sourceType") or source_view.get("source_type", "delta"), + source_view.get("sourceDetails") or source_view.get("source_details", {}), + "batch", + )) + target_config["input"] = [source_id] + + nodes.append({"name": f"target_{mv_name}", "node_type": "target", "config": target_config}) + + return _result_envelope(spec, nodes) # ─── CLI ────────────────────────────────────────────────────────────────────── @@ -503,14 +481,11 @@ def main(): output_dir = args.output_dir or args.input os.makedirs(output_dir, exist_ok=True) - pattern = "*_main.json" count = 0 for root, dirs, files in os.walk(args.input): for fname in sorted(files): - if fname.endswith("_main.json") or fname.endswith(".json"): + if fname.endswith("_main.json"): input_path = os.path.join(root, fname) - - # Compute relative path for output rel = os.path.relpath(input_path, args.input) output_path = os.path.join(output_dir, rel) os.makedirs(os.path.dirname(output_path), exist_ok=True) diff --git a/src/dataflow_spec_builder/dataflow_spec_builder.py b/src/dataflow_spec_builder/dataflow_spec_builder.py index e49df75..5ebca5b 100644 --- a/src/dataflow_spec_builder/dataflow_spec_builder.py +++ b/src/dataflow_spec_builder/dataflow_spec_builder.py @@ -446,7 +446,19 @@ def _validate_json(spec_item: tuple) -> tuple: nodester_schema_path = os.path.join(self.framework_path, "schemas", "spec_nodester.json") if os.path.exists(nodester_schema_path): nodester_validator = utility.JSONValidator(nodester_schema_path) - errors = nodester_validator.validate(json_data) + # Nodester field names are snake_case. The metadata keys were + # normalised to camelCase at read time for uniform downstream + # processing, so present them as snake_case for validation. + camel_to_snake = { + self.Keys.DATA_FLOW_ID: "data_flow_id", + self.Keys.DATA_FLOW_GROUP: "data_flow_group", + self.Keys.DATA_FLOW_TYPE: "data_flow_type", + self.Keys.DATA_FLOW_VERSION: "data_flow_version", + } + validation_data = { + camel_to_snake.get(k, k): v for k, v in json_data.items() + } + errors = nodester_validator.validate(validation_data) else: errors = [] elif file_type == "main": diff --git a/src/dataflow_spec_builder/transformer/nodester.py b/src/dataflow_spec_builder/transformer/nodester.py index dae6be4..49a533c 100644 --- a/src/dataflow_spec_builder/transformer/nodester.py +++ b/src/dataflow_spec_builder/transformer/nodester.py @@ -683,9 +683,6 @@ def _build_transform_view(self, node: Dict, node_lookup: Dict[str, Dict]) -> Dic transform_type = node.get("transformation_type", "sql") config = node.get("config", {}) - if transform_type == "passthrough": - return {"mode": Mode.STREAM, "sourceType": "delta", "sourceDetails": {}} - if transform_type == "python": # A python transformation is its own view: it reads its upstream view # and applies apply_transform(df) via the framework's pythonTransform. diff --git a/src/schemas/main.json b/src/schemas/main.json index 69dd9c3..a53a66e 100644 --- a/src/schemas/main.json +++ b/src/schemas/main.json @@ -1,48 +1,47 @@ { "title": "Main DataFlowSpec", - "type": "object", - "properties": { - "dataFlowId": {"type": "string"}, - "dataFlowGroup": {"type": "string"}, - "dataFlowType": {"type": "string", "enum": ["flow", "standard", "materialized_view", "nodester"]}, - "dataFlowVersion": {"type": "string"}, - "tags": {"type": "object", "additionalProperties": true}, - "features": { - "type": "object", - "properties": { - "operationalMetadataEnabled": {"type": "boolean"} - } - } - }, "if": { - "properties": { - "dataFlowType": { "const": "standard" } - } + "properties": { "data_flow_type": { "const": "nodester" } }, + "required": ["data_flow_type"] }, - "then": { "$ref": "./spec_standard.json#/$defs/standardSpec" }, + "then": { "$ref": "./spec_nodester.json#/$defs/nodesterSpec" }, "else": { + "type": "object", + "properties": { + "dataFlowId": {"type": "string"}, + "dataFlowGroup": {"type": "string"}, + "dataFlowType": {"type": "string", "enum": ["flow", "standard", "materialized_view"]}, + "dataFlowVersion": {"type": "string"}, + "tags": {"type": "object", "additionalProperties": true}, + "features": { + "type": "object", + "properties": { + "operationalMetadataEnabled": {"type": "boolean"} + } + } + }, "if": { "properties": { - "dataFlowType": { "const": "flow" } + "dataFlowType": { "const": "standard" } } }, - "then": { "$ref": "./spec_flows.json#/$defs/flowsSpec" }, - "else": { + "then": { "$ref": "./spec_standard.json#/$defs/standardSpec" }, + "else": { "if": { "properties": { - "dataFlowType": { "const": "materialized_view" } + "dataFlowType": { "const": "flow" } } }, - "then": { "$ref": "./spec_materialized_views.json#/$defs/materializedViewsSpec" }, + "then": { "$ref": "./spec_flows.json#/$defs/flowsSpec" }, "else": { "if": { "properties": { - "dataFlowType": { "const": "nodester" } + "dataFlowType": { "const": "materialized_view" } } }, - "then": { "$ref": "./spec_nodester.json#/$defs/nodesterSpec" } + "then": { "$ref": "./spec_materialized_views.json#/$defs/materializedViewsSpec" } } - } - }, - "required": ["dataFlowId", "dataFlowGroup", "dataFlowType"] + }, + "required": ["dataFlowId", "dataFlowGroup", "dataFlowType"] + } } diff --git a/src/schemas/spec_nodester.json b/src/schemas/spec_nodester.json index 2dfefe1..a2154ff 100644 --- a/src/schemas/spec_nodester.json +++ b/src/schemas/spec_nodester.json @@ -6,17 +6,18 @@ "$defs": { "nodesterSpec": { "type": "object", + "description": "Nodester specs use snake_case field names throughout.", "properties": { - "dataFlowId": { + "data_flow_id": { "type": "string" }, - "dataFlowGroup": { + "data_flow_group": { "type": "string" }, - "dataFlowType": { + "data_flow_type": { "const": "nodester" }, - "dataFlowVersion": { + "data_flow_version": { "type": "string" }, "tags": { @@ -37,9 +38,9 @@ } }, "required": [ - "dataFlowId", - "dataFlowGroup", - "dataFlowType", + "data_flow_id", + "data_flow_group", + "data_flow_type", "nodes" ], "additionalProperties": false @@ -77,8 +78,7 @@ "type": "string", "enum": [ "sql", - "python", - "passthrough" + "python" ], "description": "Type of transformation to apply (required for transformation nodes)" }, @@ -92,7 +92,8 @@ "delta", "delta_sink", "foreach_batch_sink", - "custom_python_sink" + "custom_python_sink", + "kafka_sink" ], "default": "delta", "description": "Target format type (required for target nodes). Default: delta" @@ -369,7 +370,7 @@ "$ref": "./definitions_main.json#/definitions/cdcSettings" }, "cdc_snapshot_settings": { - "$ref": "./definitions_main.json#/definitions/cdcSnapshotSettings" + "$ref": "#/$defs/snapshotCdcConfig" }, "data_quality_expectations_enabled": { "type": "boolean", @@ -391,8 +392,8 @@ "description": "Quarantine mode for this target" }, "quarantine_target_details": { - "$ref": "./definitions_main.json#/definitions/quarantineTargetDetails", - "description": "Quarantine target configuration when quarantineMode is 'table'" + "$ref": "#/$defs/quarantineTargetDetails", + "description": "Quarantine target configuration when quarantine_mode is 'table'" }, "table_migration_details": { "type": "object", @@ -499,6 +500,123 @@ } } ] + }, + "snapshotCdcConfig": { + "type": "object", + "description": "Snapshot CDC configuration for a target (snake_case). The framework normalises these keys to the backend's camelCase at build time.", + "properties": { + "keys": { + "type": "array", + "items": { "type": "string" } + }, + "scd_type": { + "type": ["string", "integer"] + }, + "snapshot_type": { + "type": "string", + "enum": ["historical", "periodic"], + "description": "Historical snapshots read files/tables directly and need no source node; periodic snapshots read from a source." + }, + "source_type": { + "type": "string", + "enum": ["file", "table"], + "description": "Historical snapshot source kind: 'file' (read snapshot files) or 'table' (read a versioned table). Not used for periodic snapshots." + }, + "source": { + "type": "object", + "description": "Historical snapshot source specification — the source is built here (not via a source node). Fields depend on source_type. Periodic snapshots omit this and read from an upstream source node.", + "properties": { + "table": { + "type": "string", + "description": "Source table (source_type 'table'), e.g. '{schema}.snapshot_source'." + }, + "path": { + "type": "string", + "description": "Snapshot file path/pattern (source_type 'file'); may contain a {version} token." + }, + "format": { + "type": "string", + "description": "File format for file sources (e.g. 'csv', 'parquet', 'json')." + }, + "reader_options": { + "type": "object", + "additionalProperties": true, + "description": "Reader options passed to the file reader (e.g. {\"header\": \"true\"})." + }, + "schema_path": { + "type": "string", + "description": "Path to a schema file applied when reading file sources." + }, + "select_exp": { + "type": "array", + "items": { "type": "string" }, + "description": "Select expressions applied to the snapshot source." + }, + "version_type": { + "type": "string", + "description": "How snapshot versions are interpreted (e.g. 'timestamp', 'integer')." + }, + "version_column": { + "type": "string", + "description": "Column carrying the snapshot version (source_type 'table')." + }, + "datetime_format": { + "type": "string", + "description": "Datetime format used to parse the {version} token for timestamp-versioned file sources (e.g. '%Y_%m_%d')." + }, + "deduplicate_mode": { + "type": "string", + "description": "Deduplication mode applied to snapshot rows." + }, + "recursiveFileLookup": { + "type": ["boolean", "string"], + "description": "Recurse into subdirectories when discovering snapshot files." + } + }, + "additionalProperties": true + }, + "track_history_column_list": { + "type": "array", + "items": { "type": "string" } + }, + "track_history_except_column_list": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["keys", "scd_type", "snapshot_type"], + "additionalProperties": true + }, + "quarantineTargetDetails": { + "type": "object", + "description": "Quarantine target configuration (snake_case).", + "properties": { + "target_format": { + "type": "string" + }, + "table": { + "type": "string" + }, + "database": { + "type": "string" + }, + "path": { + "type": "string" + }, + "cluster_by_auto": { + "type": "boolean" + }, + "cluster_by_columns": { + "type": "array", + "items": { "type": "string" } + }, + "partition_columns": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["target_format"], + "additionalProperties": true } } } From d570cbc0cf4d4635f8eab6298ff44b8bce943473 Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 26 Jun 2026 17:18:23 +1000 Subject: [PATCH 03/19] chore: gitignore .claude and stop tracking it The local .claude directory (Claude Code commands/settings) should not be part of the repository. Add .claude/ to .gitignore and remove the previously tracked command files from version control. The files remain on disk locally (now ignored), so they persist across branch switches. Co-authored-by: Isaac --- .claude/commands/add_feature_sample.md | 153 -------------------- .claude/commands/deploy-nodester-samples.md | 138 ------------------ .gitignore | 2 +- 3 files changed, 1 insertion(+), 292 deletions(-) delete mode 100644 .claude/commands/add_feature_sample.md delete mode 100644 .claude/commands/deploy-nodester-samples.md diff --git a/.claude/commands/add_feature_sample.md b/.claude/commands/add_feature_sample.md deleted file mode 100644 index b95ccd1..0000000 --- a/.claude/commands/add_feature_sample.md +++ /dev/null @@ -1,153 +0,0 @@ -Add a new feature sample and its validations to the Lakeflow Framework samples. - -## Context - -You are helping add a new feature sample to the Lakeflow Framework. A feature sample demonstrates a specific framework capability end-to-end — from a dataflowspec JSON/YAML file through to validated output tables. - -The framework root is the current working directory (the repo root). - -## Your Task - -The user will describe what feature they want to add a sample for (e.g. "add a sample for the new append_dedup flow type" or "add a sample for custom Python aggregation transforms"). Follow all steps below. - ---- - -## Step 1 — Understand the Feature - -1. Read the relevant framework source code in `src/` to understand what the feature does, its configuration keys, and what output tables it produces. -2. Read at least 2 similar existing dataflowspecs in `samples/bronze_sample/src/dataflows/feature_samples/dataflowspec/` to understand the naming convention (`{feature_name}_main.json`) and structure. -3. Decide which pipeline group the new sample belongs to: - - `feature_samples_general` → goes in the General Pipeline - - `feature_samples_data_quality` → goes in the Data Quality Pipeline - - `feature_samples_snapshots` → goes in the Snapshots Pipeline - - `feature_samples_table_migration` → goes in the Table Migration Pipeline - - `feature_samples_python` → goes in the Python Pipeline - - If it's a new group, you'll also need to create a new pipeline resource YAML (see Step 4). - ---- - -## Step 2 — Create the Dataflowspec - -Create `samples/bronze_sample/src/dataflows/feature_samples/dataflowspec/{feature_name}_main.json` (or `.yaml`). - -Follow the exact naming pattern: `{feature_name}_main.json`. - -Rules: -- Set `dataFlowGroup` to the correct pipeline group from Step 1 -- Use `{staging_schema}`, `{bronze_schema}`, `{silver_schema}` etc. as placeholders (these are substituted at runtime) -- Use a descriptive `dataFlowId` that starts with `feature_` to distinguish it as a sample -- Set `targetDetails.table` to `feature_{feature_name}` or a similarly descriptive name -- Enable `delta.enableChangeDataFeed` in tableProperties where applicable -- Validate the spec using: `python scripts/validate_dataflows.py samples/bronze_sample/src/dataflows/feature_samples/dataflowspec/{feature_name}_main.json` - ---- - -## Step 3 — Add Test Data (if needed) - -If the feature requires specific source data that doesn't already exist: -1. Check `samples/test_data_and_orchestrator/src/create_schemas_and_tables.ipynb` — add DDL if a new staging table is needed -2. Check `samples/test_data_and_orchestrator/src/run_1_staging_load.ipynb` — add initial data inserts -3. Check `run_2_staging_load.ipynb`, `run_3_staging_load.ipynb`, `run_4_staging_load.ipynb` — add incremental data to exercise SCD2 history, updates, deletes, snapshot changes, etc. - -Data loading conventions: -- Run 1: initial state (all active, no history) -- Run 2: first set of changes (updates, new records, deletions via flag) -- Run 3: second set of changes (more updates, snapshot overwrites) -- Run 4: final changes (additional updates to test multiple SCD2 versions) - ---- - -## Step 4 — Register in a Pipeline (only if new pipeline group) - -If the feature uses a new `dataFlowGroup` that doesn't map to an existing pipeline, create pipeline resource YAMLs: -- `samples/bronze_sample/resources/serverless/{pipeline_name}.yml` -- `samples/bronze_sample/resources/classic/{pipeline_name}.yml` - -Copy the structure from an existing pipeline YAML (e.g. `bronze_feature_samples_pipeline_general.yml`) and set `pipeline.dataFlowGroupFilter` to the new group. - -Then add the pipeline lookup variable to `samples/test_data_and_orchestrator/databricks.yml`: -```yaml -lakeflow_samples_{pipeline_key}_id: - lookup: - pipeline: "${var.name_prefix}Lakeflow Framework - {Pipeline Display Name} (${var.logical_env})" -``` - ---- - -## Step 5 — Add Pipeline Tasks to Orchestrator Jobs - -Add the pipeline as a task in all 4 run job YAMLs (both classic and serverless): -- `samples/test_data_and_orchestrator/resources/serverless/run_1_load_and_schema_initialization_job.yml` -- `samples/test_data_and_orchestrator/resources/serverless/run_2_load_job.yml` -- `samples/test_data_and_orchestrator/resources/serverless/run_3_load_job.yml` -- `samples/test_data_and_orchestrator/resources/serverless/run_4_load_job.yml` -- Same 4 files under `resources/classic/` - -Use `full_refresh: true` in run_1 and `full_refresh: false` in run_2/3/4. - -Also add the new pipeline task key to the `depends_on` list of the `validate_run_N` task in all 8 files. - ---- - -## Step 6 — Write Validations - -Add validation cells to all 4 validate notebooks: -`samples/test_data_and_orchestrator/src/validate_run_{1,2,3,4}.ipynb` - -For each notebook, insert new markdown + code cells **before the final `v.print_summary()` cell**. - -**Key principles for good validations:** - -1. **Exact counts for deterministic tables**: Use `v.validate_row_count(table, N, description)` when you know exactly how many rows should be present after each run (e.g. append-only, SCD1 batch-overwrite, small deterministic CDC streams). - -2. **Min counts for complex tables**: Use `v.validate_min_row_count(table, N, description)` for tables where exact counts are harder to predict (e.g. file-based historical snapshots, complex SCD2 with many interacting sources). - -3. **SCD2 active/closed counts**: Use these validators for CDC/SCD2 tables: - - `v.validate_active_scd2_count(table, N, end_at_col)` — records where `end_at_col IS NULL` - - `v.validate_closed_scd2_count(table, N, end_at_col)` — records where `end_at_col IS NOT NULL` - - `v.validate_min_closed_scd2_count(table, N, end_at_col)` — at least N closed records - -4. **Value checks**: Use `v.validate_column_value(table, where_clause, column, expected_value, description)` to verify specific field values (e.g. check that John's email updated correctly across runs). - -5. **Existence checks**: Use `v.validate_values_exist(table, column, [list_of_values], description)` to verify that specific IDs or keys exist. - -6. **Null checks**: Use `v.validate_column_not_null(table, column_expr, description)` for operational metadata or required columns. - -**Per-run validation logic:** -- **Run 1**: Initial state. All streaming tables show data from the first load. SCD2 tables show all-active records (0 closed). SCD1 tables show the first snapshot. Historical snapshot tables show SCD2 history from all pre-loaded files. -- **Run 2**: First incremental. CDC tables grow by the Run 2 inserts. SCD2 tables gain closed records for updated keys. SCD1 tables update in-place. Snapshot sources are overwritten. -- **Run 3**: Second incremental. Similar pattern; note which snapshot sources are overwritten and which tables are unchanged. -- **Run 4**: Final incremental. Verify the last expected state, including checking exact final values (e.g. latest email address). - -**Add a markdown cell** before each code cell explaining what tables are being validated and the expected state. - ---- - -## Step 7 — Verify - -1. Run the dataflowspec validator to check the spec is valid: - ```bash - python scripts/validate_dataflows.py samples/bronze_sample/src/dataflows/feature_samples/dataflowspec/{feature_name}_main.json -v - ``` - -2. If Databricks CLI is configured, you can deploy and test with a logical environment alias: - ```bash - cd samples - ./deploy_and_test.sh -l "_test_{feature_name}" -u "" -h "" -p "" --runs 4 - ``` - The `logical_env` parameter (e.g. `_test_my_feature`) namespaces all schemas so you don't affect the main deployment. - -3. After a successful test run, review the validation notebook output to confirm all assertions pass. - ---- - -## Checklist - -Before finishing, confirm: -- [ ] Dataflowspec file created and validates cleanly with `validate_dataflows.py` -- [ ] Test data added (or confirmed that existing data covers the feature) -- [ ] Pipeline registered in orchestrator databricks.yml (if new group) -- [ ] Pipeline tasks added to all 8 run_N job YAMLs (classic + serverless) with correct depends_on -- [ ] Validation cells added to all 4 validate_run_N notebooks covering: Run 1 (initial), Run 2 (first incremental), Run 3 (second incremental), Run 4 (final state) -- [ ] Each validation uses the most appropriate validator method (exact count vs min count vs SCD2 active/closed) -- [ ] All validate task `depends_on` lists in the job YAMLs include the new pipeline task key diff --git a/.claude/commands/deploy-nodester-samples.md b/.claude/commands/deploy-nodester-samples.md deleted file mode 100644 index 153ca54..0000000 --- a/.claude/commands/deploy-nodester-samples.md +++ /dev/null @@ -1,138 +0,0 @@ -# Deploy and Test Nodester Samples - -This skill deploys and tests the Lakeflow Framework nodester samples in a Databricks workspace. - -## Overview - -The nodester samples demonstrate a node-based graph approach to defining DLT pipelines. They live in `samples/nodester_sample/` and include 4 sample dataflows: -- **customer_simple**: Single source → single target (basic streaming) -- **customer_cloudfiles**: Cloud Files → bronze table (file ingestion) -- **customer_with_transform**: Multi-source with transform chain and SCD2 merge -- **customer_multi_target**: Staging append → merge → SCD2 with quarantine table - -## Prerequisites - -- Databricks CLI installed and authenticated (`databricks auth login`) -- Access to a Databricks workspace with Unity Catalog enabled -- The repo cloned locally — all paths below are relative to the repo root - -## Step-by-Step Deployment - -### 1. Deploy the Framework (Required First) - -The framework source code must be deployed to the workspace before any sample pipelines can run. - -```bash -# From the repo root -databricks bundle deploy -``` - -### 2. Deploy All Samples - -To deploy all samples (bronze, silver, gold, yaml, nodester, orchestrator): - -```bash -cd samples -./deploy.sh -h -u -l -``` - -To deploy only the nodester sample: - -```bash -cd samples -./deploy_nodester.sh -h -u -l -``` - -**Parameters:** - -| Flag | Description | Example | -|------|-------------|---------| -| `-h` / `--host` | Databricks workspace URL | `https://my-workspace.azuredatabricks.net/` | -| `-u` / `--user` | Your Databricks user email | `alice@example.com` | -| `-l` / `--logical_env` | Logical environment suffix (prefixed with `_`) | `_alice` | -| `--catalog` | Unity Catalog catalog name (default: `main`) | `main` | -| `--schema_namespace` | Schema namespace prefix (default: `lakeflow_samples`) | `lakeflow_samples` | -| `-p` / `--profile` | Databricks CLI profile to use | `my-profile` | - -The resulting schemas will be: -- Bronze: `._bronze` -- Silver: `._silver` -- Staging: `._staging` - -### 3. Deploy and Run Tests (Run 1 - Full Refresh) - -To deploy everything and run the Day 1 initialization job: - -```bash -cd samples -./deploy_and_test.sh -h -u -l --runs 1 -``` - -This will: -1. Deploy all sample bundles (bronze, silver, gold, yaml, nodester, orchestrator) -2. Run the "Lakeflow Framework - Run 1 - Load and Schema Initialization" job -3. The job runs all pipelines including the 4 nodester pipelines - -### 4. Run Subsequent Tests (Incremental Loads) - -```bash -./deploy_and_test.sh -h -u -l --runs 2 -./deploy_and_test.sh -h -u -l --runs 3 -./deploy_and_test.sh -h -u -l --runs 4 -``` - -## Manual Pipeline Execution via Databricks CLI - -To manually trigger a specific nodester pipeline: - -```bash -# Find the pipeline ID by name -databricks pipelines list-pipelines --profile | grep -i nodester - -# Run a full refresh -databricks pipelines start-update --full-refresh --profile - -# Run an incremental update -databricks pipelines start-update --profile - -# Check update status -databricks pipelines get-update --profile -``` - -## Key File Locations - -| File | Purpose | -|------|---------| -| `samples/nodester_sample/databricks.yml` | Bundle definition for nodester sample | -| `samples/nodester_sample/src/dataflows/base_samples/dataflowspec/` | Nodester JSON spec files | -| `samples/nodester_sample/src/dataflows/base_samples/expectations/` | DQ expectation files | -| `samples/nodester_sample/src/dataflows/base_samples/schemas/` | Table schema files | -| `samples/nodester_sample/src/pipeline_configs/dev_substitutions.json` | Token substitutions | -| `samples/deploy_nodester.sh` | Nodester-only deploy script | -| `samples/deploy.sh` | Full samples deploy script | -| `samples/deploy_and_test.sh` | Deploy + run test job | -| `src/dataflow_spec_builder/transformer/nodester.py` | Nodester spec transformer | - -## Troubleshooting - -### Pipeline fails with "view not found" or "Failed to analyze flow" -- Check that `inputs` arrays reference **node IDs** (not table/view names) -- If a source reads from an internal pipeline table, ensure its `table` field matches the producing target's `table` — the framework auto-detects internal sources by graph topology - -### Quarantine table is empty -- Verify `dataQualityExpectationsEnabled: true` and `dataQualityExpectationsPath` are set on the target node -- For staging tables with quarantine, ensure the DQ path points to a valid expectations file - -### Framework source path error -- Re-run `databricks bundle deploy` from the repo root before deploying samples - -### Schema not found -- Ensure the `create_schemas_and_tables` notebook ran successfully (it is part of the Run 1 job) - -## Architecture Notes - -The nodester spec transformer (`src/dataflow_spec_builder/transformer/nodester.py`) converts node-based specs to the standard framework flow spec format. Key rules: -- `inputs` arrays reference **node IDs**, not view or table names -- The primary (terminal) target is auto-detected from the graph — no `isPrimary` flag needed -- Internal pipeline sources are detected by matching `table` against target node tables in the spec; no `database: "live"` needed in user specs -- Views are auto-named `v_{node_id}` unless `outputName` is specified on a transformation node diff --git a/.gitignore b/.gitignore index f67b990..cb9943b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ scratch/** !scratch/README.md .DS_Store **/.DS_Store -.claude/settings.local.json +.claude/ From d8680d7f358b9417fb8519e3f3f4ac64ddac6449 Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 26 Jun 2026 17:30:14 +1000 Subject: [PATCH 04/19] chore: remove feature-test workflow and validation notebooks Move the feature-test GitHub workflow and the pattern-samples validation notebooks (validate_run_1..4 + validation_utils) out of this branch. They are kept on a local-only branch and intentionally not published upstream. Co-authored-by: Isaac --- .github/workflows/feature-test.yml | 79 --- .../src/notebooks/validate_run_1.ipynb | 606 ------------------ .../src/notebooks/validate_run_2.ipynb | 411 ------------ .../src/notebooks/validate_run_3.ipynb | 312 --------- .../src/notebooks/validate_run_4.ipynb | 340 ---------- .../src/notebooks/validation_utils.py | 306 --------- 6 files changed, 2054 deletions(-) delete mode 100644 .github/workflows/feature-test.yml delete mode 100644 samples/pattern-samples/src/notebooks/validate_run_1.ipynb delete mode 100644 samples/pattern-samples/src/notebooks/validate_run_2.ipynb delete mode 100644 samples/pattern-samples/src/notebooks/validate_run_3.ipynb delete mode 100644 samples/pattern-samples/src/notebooks/validate_run_4.ipynb delete mode 100644 samples/pattern-samples/src/notebooks/validation_utils.py diff --git a/.github/workflows/feature-test.yml b/.github/workflows/feature-test.yml deleted file mode 100644 index 01fc43c..0000000 --- a/.github/workflows/feature-test.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Feature Branch - Deploy and Test Samples - -on: - push: - branches: - - feature/sample-validations - workflow_dispatch: - inputs: - logical_env: - description: 'Logical environment suffix (e.g. _hw)' - required: true - default: '_hw' - type: string - compute: - description: 'Compute type (0=Classic, 1=Serverless)' - required: true - default: '1' - type: choice - options: - - '0' - - '1' - catalog: - description: 'Unity Catalog name' - required: true - default: 'main' - type: string - schema_namespace: - description: 'Schema namespace' - required: true - default: 'lakeflow_samples' - type: string - num_runs: - description: 'Number of test runs (1-4)' - required: true - default: '4' - type: choice - options: - - '1' - - '2' - - '3' - - '4' - -permissions: - contents: read - -jobs: - deploy-and-test: - runs-on: - group: databricks-solutions-protected-runner-group - env: - DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install Databricks CLI - uses: databricks/setup-cli@main - - - name: Verify Databricks authentication - run: databricks auth env - - - name: Deploy framework - run: databricks bundle deploy - - - name: Deploy and Test Samples - working-directory: samples - run: | - chmod +x deploy_and_test.sh common.sh deploy.sh - ./deploy_and_test.sh \ - -h "${{ secrets.DATABRICKS_HOST }}" \ - -u "${{ secrets.DATABRICKS_USER }}" \ - -l "${{ inputs.logical_env || '_ci_test' }}" \ - -c "${{ inputs.compute || '1' }}" \ - -p "DEFAULT" \ - --catalog "${{ inputs.catalog || 'main' }}" \ - --schema_namespace "${{ inputs.schema_namespace || 'lakeflow_samples' }}" \ - --runs "${{ inputs.num_runs || '4' }}" diff --git a/samples/pattern-samples/src/notebooks/validate_run_1.ipynb b/samples/pattern-samples/src/notebooks/validate_run_1.ipynb deleted file mode 100644 index 135382d..0000000 --- a/samples/pattern-samples/src/notebooks/validate_run_1.ipynb +++ /dev/null @@ -1,606 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Run 1 Validation\n", - "\n", - "This notebook validates the output tables after Run 1 (Day 1) data load.\n", - "\n", - "## Expected State After Run 1:\n", - "- **Staging**: Initial customer data loaded (3 customers, 5 addresses, purchases)\n", - "- **Bronze**: Streaming tables populated from staging sources\n", - "- **Silver**: SCD2 tables with initial versions (all active, no history)\n", - "- **Gold**: Dimension tables populated from silver\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%run \"./initialize\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import validation utilities\n", - "from validation_utils import ValidationRunner\n", - "\n", - "# Initialize validation runner with spark session\n", - "v = ValidationRunner(spark)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bronze Layer Validations\n", - "\n", - "### Base Samples Pipeline\n", - "- `customer`: Streaming from staging.customer (3 rows: John, Jane, Richard)\n", - "- `customer_address`: Streaming with data quality, quarantine mode=table\n", - " - 5 rows in staging, but 1 has NULL CUSTOMER_ID which should be quarantined\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"=\" * 60)\n", - "print(\"BRONZE LAYER - Base Samples\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Bronze customer table - should have 3 rows (John, Jane, Richard)\n", - "v.validate_row_count(f\"{bronze_schema}.customer\", 3, \"Initial customer load\")\n", - "\n", - "# Validate customer IDs present\n", - "v.validate_values_exist(f\"{bronze_schema}.customer\", \"CUSTOMER_ID\", [1, 2, 10], \"Customer IDs\")\n", - "\n", - "# Validate specific customer data\n", - "v.validate_column_value(\n", - " f\"{bronze_schema}.customer\",\n", - " \"CUSTOMER_ID = 1\",\n", - " \"EMAIL\",\n", - " \"john.doe@example.com\",\n", - " \"John's email\"\n", - ")\n", - "\n", - "# Bronze customer_address - should have 4 valid rows (NULL CUSTOMER_ID quarantined)\n", - "v.validate_row_count(f\"{bronze_schema}.customer_address\", 4, \"Valid addresses (1 quarantined)\")\n", - "\n", - "# Quarantine table should have 1 row (NULL CUSTOMER_ID)\n", - "v.validate_quarantine_count(f\"{bronze_schema}.customer_address_quarantine\", 1)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Feature Samples - Data Quality Pipeline\n", - "\n", - "- `feature_quarantine_table`: Same source as customer_address, quarantine mode=table\n", - "- `feature_quarantine_flag`: Quarantine mode=flag (adds __IS_QUARANTINED column)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Samples (Data Quality)\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Feature quarantine table - same as customer_address\n", - "v.validate_row_count(f\"{bronze_schema}.feature_quarantine_table\", 4, \"Valid records in quarantine table feature\")\n", - "v.validate_quarantine_count(f\"{bronze_schema}.feature_quarantine_table_quarantine\", 1)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Feature Samples - Snapshots Pipeline\n", - "\n", - "- Historical snapshots from files and tables\n", - "- Periodic snapshot SCD2\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Samples (Snapshots)\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Historical snapshot from table - processes all historical snapshots\n", - "# Initial load has data with 3 timestamps (2024-01-01, 2024-01-04, 2024-02-10)\n", - "# SCD2 should track changes across snapshots\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_table_datetime\", \n", - " 3, # At minimum 3 customers from initial snapshot\n", - " \"Historical snapshot from table\"\n", - ")\n", - "\n", - "# Periodic snapshot SCD2 - initial snapshot with 3 customers\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.feature_periodic_snapshot_scd2\",\n", - " 3,\n", - " \"__END_AT\"\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Silver Layer Validations\n", - "\n", - "### Base Samples Pipeline\n", - "- `customer`: SCD2 with CDC from bronze.customer\n", - "- `customer_address`: SCD2 with CDC from bronze.customer_address\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"SILVER LAYER - Base Samples\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Silver customer - SCD2, all records should be active (no changes yet)\n", - "v.validate_row_count(f\"{silver_schema}.customer\", 3, \"Initial SCD2 records\")\n", - "v.validate_active_scd2_count(f\"{silver_schema}.customer\", 3)\n", - "v.validate_closed_scd2_count(f\"{silver_schema}.customer\", 0)\n", - "\n", - "# Silver customer_address - SCD2, 4 valid records from bronze (quarantine filtered)\n", - "v.validate_row_count(f\"{silver_schema}.customer_address\", 4, \"Initial SCD2 records\")\n", - "v.validate_active_scd2_count(f\"{silver_schema}.customer_address\", 4)\n", - "v.validate_closed_scd2_count(f\"{silver_schema}.customer_address\", 0)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Multi-Source Streaming Pipeline\n", - "\n", - "- `customer_ms_basic`: Merges customer and customer_address streams\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"SILVER LAYER - Multi-Source Streaming\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Multi-source streaming combines customer and customer_address\n", - "# Customer IDs: 1, 2, 10 from customer + 1, 2, 4, 10 from address (deduplicated)\n", - "# Unique IDs: 1, 2, 4, 10 = 4 active records\n", - "v.validate_active_scd2_count(\n", - " f\"{silver_schema}.customer_ms_basic\",\n", - " 4, # Unique customer IDs from both sources\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"OPERATIONAL METADATA - meta_load_details Validation\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Validate meta_load_details nested column fields are not null\n", - "# Using wildcard to check all fields in the struct\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer meta_load_details\"\n", - ")\n", - "\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer_address\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer_address meta_load_details\"\n", - ")\n", - "\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer_ms_basic\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer_ms_basic meta_load_details\"\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Gold Layer Validations\n", - "\n", - "### Stream-Static Pipeline\n", - "- `dim_customer_sql_sample`: Dimension built from silver customer data\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"GOLD LAYER - Stream-Static\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Gold dimension - should have SCD2 records from silver\n", - "v.validate_active_scd2_count(\n", - " f\"{gold_schema}.dim_customer_sql_sample\",\n", - " 4, # Combined unique keys from customer and customer_address\n", - " \"__END_AT\"\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## YAML Sample Validations\n", - "\n", - "The YAML sample pipeline mirrors the bronze base samples but uses YAML format\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"YAML SAMPLE\")\n", - "print(\"=\" * 60)\n", - "\n", - "# YAML customer - same as bronze.customer\n", - "v.validate_row_count(f\"{yaml_schema}.customer\", 3, \"YAML customer table\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Feature General Pipeline Validations\n", - "\n", - "### Append Flows\n", - "- `append_sql_flow`: All 5 customer_address rows inserted (quarantineMode=flag, 1 row gets __IS_QUARANTINED=true)\n", - "- `append_view_flow`: All 3 customer rows appended from a view\n", - "- `feature_constraints`: 3 rows from customer with DDL schema/constraints applied\n", - "- `feature_version_mapping_flows`: SCD1 with staging table pattern - 3 rows\n", - "- `v_version_mapping_standard`: SCD1 standard pattern - 3 rows\n", - "\n", - "### Materialized Views\n", - "- `feature_mv_source_view`: 3 rows from customer (view-based MV)\n", - "- `feature_mv_sql_path`: 3 rows from customer (inline SQL MV)\n", - "- `feature_mv_chain_mvs`: 3 rows (chained MV)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Template Samples Pipeline\")\n", - "print(\"=\" * 60)\n", - "\n", - "# customer_template: SCD2 from snapshot files via template\n", - "# 3 snapshot files loaded in Run 1: 2024-01-01 (John, Jane), 2024-01-04 (John jdoe@, Alice, Joe), 2024-02-10 (John jdoe@, Alice, Joe, Sarah)\n", - "# SCD2: John has 2 versions (email changed), Jane removed from later snapshots\n", - "# Actual: 5 active in Run 1\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.customer_template\",\n", - " 5,\n", - " \"Template customer SCD2 (at least 5 rows after 3 snapshot files)\"\n", - ")\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.customer_template\",\n", - " 5,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# customer_address_template: SCD2 from customer_address snapshot files\n", - "# 3 snapshot files: 2024-01-01 (2 rows), 2024-01-04 (3 rows, ID 1 still Melbourne), 2024-02-10 (4 rows, ID 1 moves to Sydney)\n", - "# SCD2: at least 4 unique IDs, ID 1 has 2 versions\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.customer_address_template\",\n", - " 4,\n", - " \"Template customer_address SCD2 (at least 4 rows from snapshot files)\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bronze Template Samples Pipeline Validations\n", - "\n", - "The template samples pipeline instantiates `cdc_stream_from_snapshot_template` for two parameter sets:\n", - "- `customer_template`: SCD2 from customer snapshot CSV files (historical, datetime-versioned)\n", - " - Run 1 loads 3 snapshot files → SCD2 result with at least 2 active customers\n", - "- `customer_address_template`: SCD2 from customer_address snapshot CSV files\n", - " - Run 1 loads 3 snapshot files with address changes across time" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"SILVER LAYER - DPM Pipeline\")\n", - "print(\"=\" * 60)\n", - "\n", - "# customer_dpm: SCD2, merges bronze.customer + bronze.customer_address\n", - "# Unique customer IDs from both streams: 1, 2, 4, 10 = 4 active\n", - "v.validate_active_scd2_count(\n", - " f\"{dpm_schema}.customer_dpm\",\n", - " 4,\n", - " \"__END_AT\"\n", - ")\n", - "v.validate_closed_scd2_count(\n", - " f\"{dpm_schema}.customer_dpm\",\n", - " 0,\n", - " \"__END_AT\"\n", - ")\n", - "v.validate_values_exist(\n", - " f\"{dpm_schema}.customer_dpm\",\n", - " \"CUSTOMER_ID\",\n", - " [1, 2, 4, 10],\n", - " \"DPM customer IDs (union of customer + address sources)\"\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Silver DPM Pipeline Validations\n", - "\n", - "- `customer_dpm` (in dpm_schema): SCD2 built from merging bronze.customer (IDs 1, 2, 10) + bronze.customer_address (IDs 1, 2, 4, 10) streams\n", - " - Union of unique customer IDs: 1, 2, 4, 10 = 4 active records\n", - " - No changes yet, so all 4 should be active with no closed records" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Table Migration Pipeline\")\n", - "print(\"=\" * 60)\n", - "\n", - "# feature_migrated_table_scd2: migrated from bronze.table_to_migrate_scd2\n", - "# Actual Run 1 state: 5 active, 1 closed\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", - " 5,\n", - " \"__END_AT\"\n", - ")\n", - "v.validate_closed_scd2_count(\n", - " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", - " 1,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# Validate migrated IDs are present (including ID 40 from the pre-migration table)\n", - "v.validate_values_exist(\n", - " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", - " \"CUSTOMER_ID\",\n", - " [1, 2, 10, 30, 40],\n", - " \"Migrated customer IDs (including 30 and 40 from legacy table)\"\n", - ")\n", - "\n", - "# feature_migrated_table_append_only: migrated from bronze.table_to_migrate_scd0 (3 rows)\n", - "# Streaming from staging.customer then appends new inserts\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_migrated_table_append_only\",\n", - " 3,\n", - " \"Append-only migration (at least 3 migrated rows)\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Feature Table Migration Pipeline Validations\n", - "\n", - "- `feature_migrated_table_scd2`: Migrated SCD2 data from `bronze.table_to_migrate_scd2` + streaming CDC from staging.customer\n", - " - Migration source has: IDs 1, 2, 10 (active) + ID 30 (closed same day) + ID 40 (2 rows: 1 closed, 1 active) = 4 active, 2 closed\n", - " - Streaming adds CDC from staging.customer (IDs 1, 2, 10), but autoStartingVersionsEnabled means no duplicate migration data\n", - "- `feature_migrated_table_append_only`: Migrated append-only table from `bronze.table_to_migrate_scd0` (IDs 1, 2, 10) + streaming inserts from staging.customer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Snapshots (Additional Tables)\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Periodic snapshot SCD1: full batch overwrite each run, no history\n", - "# Run 1: customer_snapshot_source has 3 rows (John, Jane, Richard)\n", - "v.validate_row_count(\n", - " f\"{bronze_schema}.feature_periodic_snapshot_scd1\",\n", - " 3,\n", - " \"Periodic snapshot SCD1 (3 customers, no history)\"\n", - ")\n", - "\n", - "# Historical file snapshots (datetime-based versioning from snapshot_customer/ files)\n", - "# Run 1 loads 3 snapshot files: 2024-01-01 (2 rows), 2024-01-04 (3 rows), 2024-02-10 (4 rows)\n", - "# SCD2 result: 4 unique customers, John has 2 versions (email changed), Jane closed after removed from later snapshots\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_files_datetime\",\n", - " 4,\n", - " \"Historical snapshot files (datetime) - at least 4 rows after 3 snapshot files\"\n", - ")\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_files_datetime\",\n", - " 4, # John (jdoe@), Alice, Joe, Sarah still active in latest snapshot\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# Multifile variant: 3 files (2 splits for 2024-01-01, 1 for 2024-12-12)\n", - "# Actual Run 1 state: 3 total rows, 1 active, 2 closed\n", - "v.validate_row_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_files_datetime_multifile\",\n", - " 3,\n", - " \"Historical snapshot multifile (3 total rows)\"\n", - ")\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_files_datetime_multifile\",\n", - " 1,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# Partitioned variant (same data as datetime but in YEAR/MONTH/DAY directories)\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_files_datetime_recursive_and_partitioned\",\n", - " 4,\n", - " \"Historical snapshot partitioned (datetime) - at least 4 rows\"\n", - ")\n", - "\n", - "# Parquet partitioned variant (same data, parquet format)\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_files_datetime_recursive_and_partitioned_parquet\",\n", - " 4,\n", - " \"Historical snapshot partitioned parquet - at least 4 rows\"\n", - ")\n", - "\n", - "# Flow-based historical snapshot: uses staging table + append_view flow pattern\n", - "# Sources from the same snapshot CSV files\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_historical_files_snapshot_flow\",\n", - " 3,\n", - " \"Historical snapshot flow - at least 3 rows (initial customers)\"\n", - ")\n", - "\n", - "# Integer-versioned historical snapshot: uses customer_1.csv and customer_2.csv\n", - "# Run 1 only loads customer_1.csv: John (john.doe@), Jane = 2 rows active\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_files_int\",\n", - " 2,\n", - " \"Historical snapshot files (integer versioning) - at least 2 rows from customer_1.csv\"\n", - ")\n", - "\n", - "# Table-based historical snapshot with select expression\n", - "# Same source as feature_historical_snapshot_table_datetime but with custom selectExp\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_table_select_expression\",\n", - " 3,\n", - " \"Historical snapshot table (select expression) - at least 3 rows\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Feature Snapshots Pipeline - Additional Tables\n", - "\n", - "Beyond `feature_periodic_snapshot_scd2` and `feature_historical_snapshot_table_datetime` (validated above), the snapshots pipeline also produces:\n", - "- `feature_periodic_snapshot_scd1`: SCD1 periodic snapshot (no history, just current state)\n", - "- File-based historical snapshots in several variants (datetime, multifile, partitioned, parquet, flow, int, schema_and_select)\n", - "- `feature_historical_snapshot_table_select_expression`: table-based historical snapshot with custom select" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature General Pipeline\")\n", - "print(\"=\" * 60)\n", - "\n", - "# append_sql_flow: quarantineMode=flag, all 5 address rows written (1 gets __IS_QUARANTINED=true)\n", - "v.validate_row_count(f\"{bronze_schema}.append_sql_flow\", 5, \"All address rows including quarantine-flagged\")\n", - "\n", - "# append_view_flow: 3 customer rows appended via view\n", - "v.validate_row_count(f\"{bronze_schema}.append_view_flow\", 3, \"Customer rows appended via view\")\n", - "\n", - "# feature_constraints: 3 customer rows with DDL-defined constraints applied\n", - "v.validate_row_count(f\"{bronze_schema}.feature_constraints\", 3, \"Customer rows with DDL constraints\")\n", - "\n", - "# feature_version_mapping_flows: SCD1 from customer via staging table flow\n", - "# Run 1: 3 customers (John, Jane, Richard)\n", - "v.validate_row_count(f\"{bronze_schema}.feature_version_mapping_flows\", 3, \"SCD1 version mapping via flow (3 customers)\")\n", - "\n", - "# v_version_mapping_standard: SCD1 from customer - standard (non-flow) pattern\n", - "v.validate_row_count(f\"{bronze_schema}.v_version_mapping_standard\", 3, \"SCD1 version mapping standard (3 customers)\")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Materialized Views\")\n", - "print(\"=\" * 60)\n", - "\n", - "# feature_mv_source_view: MV backed by a DLT view over staging.customer\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_source_view\", 3, \"Materialized view from source view\")\n", - "\n", - "# feature_mv_sql_path: MV using inline SQL SELECT * FROM staging.customer\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_sql_path\", 3, \"Materialized view from SQL path\")\n", - "\n", - "# feature_mv_chain_mvs: chained materialized view\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_chain_mvs\", 3, \"Chained materialized view\")\n", - "\n", - "# feature_mv_with_quarantine: MV over customer_address with quarantine (table mode)\n", - "# 4 valid rows pass DQ, 1 NULL CUSTOMER_ID goes to quarantine table\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_with_quarantine\", 4, \"MV with quarantine (4 valid address rows)\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Validation Summary\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "v.print_summary()\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/samples/pattern-samples/src/notebooks/validate_run_2.ipynb b/samples/pattern-samples/src/notebooks/validate_run_2.ipynb deleted file mode 100644 index c71f794..0000000 --- a/samples/pattern-samples/src/notebooks/validate_run_2.ipynb +++ /dev/null @@ -1,411 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Run 2 Validation\n", - "\n", - "This notebook validates the output tables after Run 2 (Day 2) data load.\n", - "\n", - "## Expected Changes in Run 2:\n", - "- **Customer Updates**:\n", - " - John (ID=1): Email changed from john.doe@example.com to jdoe@example.com\n", - " - Richard (ID=10): Marked for deletion (DELETE_FLAG=1)\n", - " - New customers: Alice (ID=3), Joe (ID=4)\n", - "- **Customer Address Updates**:\n", - " - Jane (ID=2): City changed to Perth, WA\n", - " - New: Alice (ID=3) in Sydney, NSW\n", - "- **SCD2 Behavior**: Previous versions should be closed, new versions created\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%run \"./initialize\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import validation utilities\n", - "from validation_utils import ValidationRunner\n", - "\n", - "# Initialize validation runner with spark session\n", - "v = ValidationRunner(spark)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bronze Layer Validations\n", - "\n", - "After Run 2:\n", - "- `customer`: 7 rows total (3 original + 4 new CDC records)\n", - "- `customer_address`: 6 rows (4 original + 2 new)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"=\" * 60)\n", - "print(\"BRONZE LAYER - Base Samples\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Bronze customer - 3 original + 4 new (John update, Alice, Joe, Richard delete marker)\n", - "v.validate_row_count(f\"{bronze_schema}.customer\", 7, \"Total customer CDC records\")\n", - "\n", - "# All customer IDs including new ones\n", - "v.validate_values_exist(f\"{bronze_schema}.customer\", \"CUSTOMER_ID\", [1, 2, 3, 4, 10], \"All customer IDs\")\n", - "\n", - "# Bronze customer_address - 4 original + 2 new (Jane update, Alice new)\n", - "v.validate_row_count(f\"{bronze_schema}.customer_address\", 6, \"Total address CDC records\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Silver Layer Validations\n", - "\n", - "### SCD2 Behavior After Run 2:\n", - "- **customer**: \n", - " - John (ID=1): 1 closed (old email), 1 active (new email)\n", - " - Jane (ID=2): 1 active (unchanged)\n", - " - Alice (ID=3): 1 active (new)\n", - " - Joe (ID=4): 1 active (new)\n", - " - Richard (ID=10): 1 closed (deleted via apply_as_deletes)\n", - " - Total: 4 active, 2 closed = 6 rows\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"SILVER LAYER - Base Samples\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Silver customer SCD2 after Day 2:\n", - "# Active: Jane (unchanged), Alice (new), Joe (new), John (updated) = 4\n", - "# Closed: John (old version), Richard (deleted) = 2\n", - "v.validate_active_scd2_count(f\"{silver_schema}.customer\", 4, \"__END_AT\")\n", - "v.validate_closed_scd2_count(f\"{silver_schema}.customer\", 2, \"__END_AT\")\n", - "\n", - "# Validate John's current email is updated\n", - "v.validate_column_value(\n", - " f\"{silver_schema}.customer\",\n", - " \"CUSTOMER_ID = 1 AND __END_AT IS NULL\",\n", - " \"EMAIL\",\n", - " \"jdoe@example.com\",\n", - " \"John's updated email (active record)\"\n", - ")\n", - "\n", - "# Validate new customers exist\n", - "v.validate_values_exist(f\"{silver_schema}.customer\", \"CUSTOMER_ID\", [1, 2, 3, 4], \"Active customer IDs\")\n", - "\n", - "# Silver customer_address SCD2:\n", - "# Jane (ID=2): 1 closed (Melbourne), 1 active (Perth)\n", - "# Alice (ID=3): 1 active (new)\n", - "# Original 1, 4, 10 still active\n", - "v.validate_min_closed_scd2_count(f\"{silver_schema}.customer_address\", 1, \"__END_AT\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Multi-Source Streaming Pipeline\n", - "\n", - "The multi-source streaming table merges customer and customer_address data.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"SILVER LAYER - Multi-Source Streaming\")\n", - "print(\"=\" * 60)\n", - "\n", - "# After Run 2, multi-source should have:\n", - "# Active IDs: 1, 2, 3, 4 (10 was deleted)\n", - "# Should have historical records from changes\n", - "v.validate_active_scd2_count(\n", - " f\"{silver_schema}.customer_ms_basic\",\n", - " 4,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# Should have at least 1 closed record from changes\n", - "v.validate_min_closed_scd2_count(\n", - " f\"{silver_schema}.customer_ms_basic\",\n", - " 1,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"OPERATIONAL METADATA - meta_load_details Validation\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Validate meta_load_details nested column fields are not null\n", - "# Using wildcard to check all fields in the struct\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer meta_load_details\"\n", - ")\n", - "\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer_address\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer_address meta_load_details\"\n", - ")\n", - "\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer_ms_basic\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer_ms_basic meta_load_details\"\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Gold Layer Validations\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"GOLD LAYER - Stream-Static\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Gold dimension should reflect silver layer changes\n", - "v.validate_min_row_count(\n", - " f\"{gold_schema}.dim_customer_sql_sample\",\n", - " 4, # At least 4 rows (active records)\n", - " \"Dimension records\"\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Feature Samples - Snapshots\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Samples (Snapshots)\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Periodic snapshot should show SCD2 changes after snapshot update\n", - "# Run 2 overwrites snapshot source, so periodic snapshot should detect changes\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_periodic_snapshot_scd2\",\n", - " 3, # At least 3 records from initial + any changes\n", - " \"Periodic snapshot records\"\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Validation Summary\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Validation summary runs at the end of the notebook after all checks complete\n", - "pass" - ] - }, - { - "cell_type": "code", - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Snapshots (Additional Tables) Run 2\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Periodic snapshot SCD1: customer_snapshot_source OVERWRITTEN in Run 2 to 5 rows\n", - "# (John jdoe@, Jane, Alice, Joe, Richard) - SCD1 full replacement\n", - "v.validate_row_count(\n", - " f\"{bronze_schema}.feature_periodic_snapshot_scd1\",\n", - " 5,\n", - " \"Periodic snapshot SCD1 (5 customers after snapshot overwrite)\"\n", - ")\n", - "\n", - "# Historical file snapshots (datetime): new snapshot file 2024-03-01 adds customer 6 (Someone)\n", - "# Active now includes customer 6; at least 7 rows total (was 6, now +1 active)\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_files_datetime\",\n", - " 7,\n", - " \"Historical snapshot files (datetime) - at least 7 rows after 4th snapshot\"\n", - ")\n", - "\n", - "# Integer-versioned historical snapshot: customer_2.csv now loaded (John jdoe@, Alice, Joe)\n", - "# SCD2: John gets new version (email changed), Alice/Joe added, customer_1.csv history preserved\n", - "# Active: John (jdoe@), Alice, Joe = 3; Closed: John (john.doe@), Jane = 2; Total >= 5\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_historical_snapshot_files_int\",\n", - " 4,\n", - " \"Historical snapshot int (at least 4 rows after customer_2.csv loaded)\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Table Migration Pipeline Run 2\")\n", - "print(\"=\" * 60)\n", - "\n", - "# feature_migrated_table_scd2: Run 2 adds streaming CDC records (John update, Alice, Joe)\n", - "# Active: 5 records after Run 2 streaming\n", - "# Closed: at least 1 (from SCD2 version history)\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", - " 5,\n", - " \"__END_AT\"\n", - ")\n", - "v.validate_min_closed_scd2_count(\n", - " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", - " 1,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# feature_migrated_table_append_only: rows accumulated from streaming after Run 2\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_migrated_table_append_only\",\n", - " 3,\n", - " \"Append-only migration (at least 3 rows after Run 2 streaming)\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Template Samples After Run 2\")\n", - "print(\"=\" * 60)\n", - "\n", - "# customer_template: Run 2 adds 2024-03-01 snapshot file with customer 6\n", - "# Active: 6 records after Run 2 snapshot ingestion\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.customer_template\",\n", - " 6,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# customer_address_template: Run 2 adds 2024-03-01 file (Jane Perth, Alice Sydney, Someone Adelaide)\n", - "# ID 2 (Jane) moves to Perth \u2192 new version; new IDs 3 (Sydney), 6 (Adelaide)\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.customer_address_template\",\n", - " 5,\n", - " \"Template customer_address SCD2 (at least 5 rows after Run 2 snapshot)\"\n", - ")" - ], - "metadata": {}, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Feature Snapshots Pipeline - Additional Tables (Run 2)\n", - "\n", - "- `feature_periodic_snapshot_scd1`: customer_snapshot_source OVERWRITTEN to 5 rows (John, Jane, Alice, Joe, Richard)\n", - "- `feature_historical_snapshot_files_datetime`: new snapshot file 2024-03-01 (customer 6) adds a row, total at least 7\n", - "- `feature_historical_snapshot_files_int`: customer_2.csv now loaded: John (jdoe@), Alice, Joe \u2192 at least 4 rows (history + new)\n", - "- `feature_migrated_table_scd2`: John and Richard get new versions; Alice and Joe added" - ], - "metadata": {} - }, - { - "cell_type": "code", - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature General Pipeline\")\n", - "print(\"=\" * 60)\n", - "\n", - "# append_sql_flow: 5 from Run 1 + 2 new (Jane Perth, Alice Sydney) = 7 total\n", - "v.validate_row_count(f\"{bronze_schema}.append_sql_flow\", 7, \"Address rows after Run 2 (+2 new)\")\n", - "\n", - "# append_view_flow: 3 from Run 1 + 4 new customer CDC records = 7 total\n", - "v.validate_row_count(f\"{bronze_schema}.append_view_flow\", 7, \"Customer rows after Run 2 (+4 new CDC)\")\n", - "\n", - "# feature_constraints: 3 original + 4 new = 7 total rows (streaming append from customer CDC)\n", - "v.validate_row_count(f\"{bronze_schema}.feature_constraints\", 7, \"Customer rows with DDL constraints after Run 2\")\n", - "\n", - "# feature_version_mapping_flows: SCD1, updated in-place\n", - "# John updated to jdoe@, Alice + Joe inserted, Richard updated (DELETE_FLAG=True)\n", - "v.validate_row_count(f\"{bronze_schema}.feature_version_mapping_flows\", 5, \"SCD1 version mapping (5 customers after Run 2)\")\n", - "v.validate_column_value(\n", - " f\"{bronze_schema}.feature_version_mapping_flows\",\n", - " \"CUSTOMER_ID = 1\",\n", - " \"EMAIL\",\n", - " \"jdoe@example.com\",\n", - " \"John's updated email in SCD1 table\"\n", - ")\n", - "\n", - "# v_version_mapping_standard: same SCD1 behaviour\n", - "v.validate_row_count(f\"{bronze_schema}.v_version_mapping_standard\", 5, \"SCD1 standard (5 customers after Run 2)\")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Materialized Views After Run 2\")\n", - "print(\"=\" * 60)\n", - "\n", - "# MVs source from staging.customer (7 rows after Run 2: 3 original + 4 new CDC)\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_source_view\", 7, \"MV source view (7 rows after Run 2)\")\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_sql_path\", 7, \"MV SQL path (7 rows after Run 2)\")\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_chain_mvs\", 7, \"Chained MV (7 rows after Run 2)\")\n", - "\n", - "# feature_mv_with_quarantine: sources from staging.customer_address\n", - "# Run 1: 5 rows; Run 2: +2 (Jane Perth, Alice Sydney) = 7 total, 1 NULL quarantined \u2192 6 valid\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_with_quarantine\", 6, \"MV with quarantine (6 valid after Run 2)\")" - ], - "metadata": {}, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "v.print_summary()" - ], - "metadata": {}, - "execution_count": null, - "outputs": [] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file diff --git a/samples/pattern-samples/src/notebooks/validate_run_3.ipynb b/samples/pattern-samples/src/notebooks/validate_run_3.ipynb deleted file mode 100644 index a48b924..0000000 --- a/samples/pattern-samples/src/notebooks/validate_run_3.ipynb +++ /dev/null @@ -1,312 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Run 3 Validation\n", - "\n", - "This notebook validates the output tables after Run 3 (Day 3) data load.\n", - "\n", - "## Expected Changes in Run 3:\n", - "- **Snapshot Source Updates**:\n", - " - customer_snapshot_source is overwritten (John ID=1 removed, only Jane, Alice, Joe, Richard remain)\n", - " - New address for customer ID=1 (Brisbane, QLD)\n", - "- **SCD2 Behavior**: \n", - " - Periodic snapshots should detect the removal of John from snapshot\n", - " - Customer address should show new version for ID=1\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%run \"./initialize\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import validation utilities\n", - "from validation_utils import ValidationRunner\n", - "\n", - "# Initialize validation runner with spark session\n", - "v = ValidationRunner(spark)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bronze Layer Validations\n", - "\n", - "Run 3 only adds 1 new address record for customer ID=1 (Brisbane, QLD)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"=\" * 60)\n", - "print(\"BRONZE LAYER - Base Samples\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Bronze customer - no new customer records in Run 3, still 7\n", - "v.validate_row_count(f\"{bronze_schema}.customer\", 7, \"Customer CDC records (no change)\")\n", - "\n", - "# Bronze customer_address - 6 from Run 2 + 1 new (ID=1 Brisbane)\n", - "v.validate_row_count(f\"{bronze_schema}.customer_address\", 7, \"Address CDC records (+1 from Run 3)\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Silver Layer Validations\n", - "\n", - "### SCD2 Changes:\n", - "- Customer table unchanged (no new customer data in Run 3)\n", - "- Customer address: ID=1 gets new version (Brisbane), old version (Melbourne) closed\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"SILVER LAYER - Base Samples\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Silver customer - unchanged from Run 2\n", - "v.validate_active_scd2_count(f\"{silver_schema}.customer\", 4, \"__END_AT\")\n", - "\n", - "# Silver customer_address - ID=1 now has new active version\n", - "# Should have at least 2 closed records now (Jane's Melbourne + ID=1's original Melbourne)\n", - "v.validate_min_closed_scd2_count(f\"{silver_schema}.customer_address\", 2, \"__END_AT\")\n", - "\n", - "# Validate ID=1's current city is Brisbane\n", - "v.validate_column_value(\n", - " f\"{silver_schema}.customer_address\",\n", - " \"CUSTOMER_ID = 1 AND __END_AT IS NULL\",\n", - " \"CITY\",\n", - " \"Brisbane\",\n", - " \"Customer 1's updated city (active record)\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"OPERATIONAL METADATA - meta_load_details Validation\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Validate meta_load_details nested column fields are not null\n", - "# Using wildcard to check all fields in the struct\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer meta_load_details\"\n", - ")\n", - "\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer_address\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer_address meta_load_details\"\n", - ")\n", - "\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer_ms_basic\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer_ms_basic meta_load_details\"\n", - ")\n", - "\n", - "# TODO: Add validation for all operational metadata fields. Currently fails on pipeline_update_id sometimes \n", - "# v.validate_column_not_null(\n", - "# f\"{silver_schema}.customer_ms_basic\",\n", - "# \"meta_load_details.*\",\n", - "# \"Silver customer_ms_basic meta_load_details\"\n", - "# )\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Feature Samples - Periodic Snapshot\n", - "\n", - "The periodic snapshot should detect the change in customer_snapshot_source\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Samples (Snapshots)\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Periodic snapshot should have processed another snapshot\n", - "# John (ID=1) was removed from snapshot source in Run 3\n", - "# This should result in a delete operation in SCD2\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_periodic_snapshot_scd2\",\n", - " 4, # At least 4 rows (original + changes from snapshots)\n", - " \"Periodic snapshot records after Run 3\"\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Gold Layer Validations\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"GOLD LAYER - Stream-Static\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Gold dimension should have more history from address changes\n", - "v.validate_min_row_count(\n", - " f\"{gold_schema}.dim_customer_sql_sample\",\n", - " 5, # At least 5 rows including history\n", - " \"Dimension records with history\"\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Validation Summary\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Validation summary runs at the end of the notebook after all checks complete\n", - "pass" - ] - }, - { - "cell_type": "code", - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature General Pipeline (Run 3)\")\n", - "print(\"=\" * 60)\n", - "\n", - "# append_sql_flow: 7 from Run 2 + 1 new (Brisbane QLD for ID=1) = 8\n", - "v.validate_row_count(f\"{bronze_schema}.append_sql_flow\", 8, \"Address rows after Run 3 (+1 Brisbane)\")\n", - "\n", - "# append_view_flow: unchanged - no new customer CDC in Run 3\n", - "v.validate_row_count(f\"{bronze_schema}.append_view_flow\", 7, \"Customer rows after Run 3 (unchanged)\")\n", - "\n", - "# feature_constraints: unchanged\n", - "v.validate_row_count(f\"{bronze_schema}.feature_constraints\", 7, \"Customer rows with DDL constraints (unchanged in Run 3)\")\n", - "\n", - "# feature_version_mapping_flows: SCD1 unchanged in Run 3 (no customer updates)\n", - "v.validate_row_count(f\"{bronze_schema}.feature_version_mapping_flows\", 5, \"SCD1 version mapping (unchanged in Run 3)\")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Snapshots (Additional Tables) Run 3\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Periodic snapshot SCD1: customer_snapshot_source OVERWRITTEN to 4 rows in Run 3\n", - "# John (ID=1) removed from snapshot; remaining: Jane, Alice, Joe, Richard\n", - "v.validate_row_count(\n", - " f\"{bronze_schema}.feature_periodic_snapshot_scd1\",\n", - " 4,\n", - " \"Periodic snapshot SCD1 (4 customers, John removed from snapshot source)\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Table Migration Pipeline Run 3\")\n", - "print(\"=\" * 60)\n", - "\n", - "# feature_migrated_table_scd2: no new customer CDC in Run 3, unchanged from Run 2\n", - "# Active: 5 records (same as Run 2)\n", - "# Closed: at least 1 (from SCD2 version history)\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", - " 5,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# feature_migrated_table_append_only: no new customer inserts in Run 3, still accumulating\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_migrated_table_append_only\",\n", - " 3,\n", - " \"Append-only migration (at least 3 rows, unchanged in Run 3)\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Template Samples After Run 3\")\n", - "print(\"=\" * 60)\n", - "\n", - "# customer_template: no new customer snapshot files in Run 3; unchanged from Run 2\n", - "# Active: 6 records (same as Run 2)\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.customer_template\",\n", - " 6,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# customer_address_template: no new address snapshot files in Run 3; unchanged\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.customer_address_template\",\n", - " 5,\n", - " \"Template customer_address SCD2 (at least 5 rows, unchanged in Run 3)\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Materialized Views After Run 3\")\n", - "print(\"=\" * 60)\n", - "\n", - "# MVs source from staging.customer (unchanged from Run 2: no new customer CDC in Run 3)\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_source_view\", 7, \"MV source view (7 rows, unchanged in Run 3)\")\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_sql_path\", 7, \"MV SQL path (7 rows, unchanged in Run 3)\")\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_chain_mvs\", 7, \"Chained MV (7 rows, unchanged in Run 3)\")\n", - "\n", - "# feature_mv_with_quarantine: sources from staging.customer_address\n", - "# Run 2 had 6 valid; Run 3 adds Brisbane for ID=1 \u2192 7 total rows, 1 NULL quarantined \u2192 7 valid\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_with_quarantine\", 7, \"MV with quarantine (7 valid after Run 3)\")" - ], - "metadata": {}, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "v.print_summary()" - ], - "metadata": {}, - "execution_count": null, - "outputs": [] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file diff --git a/samples/pattern-samples/src/notebooks/validate_run_4.ipynb b/samples/pattern-samples/src/notebooks/validate_run_4.ipynb deleted file mode 100644 index 22952ae..0000000 --- a/samples/pattern-samples/src/notebooks/validate_run_4.ipynb +++ /dev/null @@ -1,340 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Run 4 Validation\n", - "\n", - "This notebook validates the output tables after Run 4 (Day 4) data load.\n", - "\n", - "## Expected Changes in Run 4:\n", - "- **Customer Updates**:\n", - " - John (ID=1): Email changed from jdoe@example.com to john.doe@another.example.com\n", - " - DELETE_FLAG set to False (was NULL)\n", - "- **SCD2 Behavior**: \n", - " - John should have another version created in silver.customer\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%run \"./initialize\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import validation utilities\n", - "from validation_utils import ValidationRunner\n", - "\n", - "# Initialize validation runner with spark session\n", - "v = ValidationRunner(spark)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bronze Layer Validations\n", - "\n", - "Run 4 adds 1 new customer record (John's email update)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"=\" * 60)\n", - "print(\"BRONZE LAYER - Base Samples\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Bronze customer - 7 from Run 3 + 1 new (John's email update)\n", - "v.validate_row_count(f\"{bronze_schema}.customer\", 8, \"Customer CDC records (+1 John update)\")\n", - "\n", - "# Bronze customer_address - unchanged from Run 3\n", - "v.validate_row_count(f\"{bronze_schema}.customer_address\", 7, \"Address CDC records (unchanged)\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Silver Layer Validations\n", - "\n", - "### SCD2 Changes:\n", - "- John (ID=1): Another version created (email changed to john.doe@another.example.com)\n", - "- Now John has 3 versions: original, jdoe, and john.doe@another.example.com\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"SILVER LAYER - Base Samples\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Silver customer - still 4 active (John, Jane, Alice, Joe)\n", - "# Richard was deleted in Run 2\n", - "v.validate_active_scd2_count(f\"{silver_schema}.customer\", 4, \"__END_AT\")\n", - "\n", - "# Silver customer - should now have 3 closed records:\n", - "# 1. John's original version\n", - "# 2. John's jdoe version\n", - "# 3. Richard (deleted)\n", - "v.validate_min_closed_scd2_count(f\"{silver_schema}.customer\", 3, \"__END_AT\")\n", - "\n", - "# Validate John's current email is the new one\n", - "v.validate_column_value(\n", - " f\"{silver_schema}.customer\",\n", - " \"CUSTOMER_ID = 1 AND __END_AT IS NULL\",\n", - " \"EMAIL\",\n", - " \"john.doe@another.example.com\",\n", - " \"John's latest email (active record)\"\n", - ")\n", - "\n", - "# Customer address unchanged from Run 3\n", - "v.validate_min_closed_scd2_count(f\"{silver_schema}.customer_address\", 2, \"__END_AT\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Gold Layer Validations\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"GOLD LAYER - Stream-Static\")\n", - "print(\"=\" * 60)\n", - "\n", - "# TODO - Fix Gold pattern for DWH, currently does not have intended behavior\n", - "# # Gold dimension should have accumulated more history\n", - "# v.validate_min_row_count(\n", - "# f\"{gold_schema}.dim_customer_sql_sample\",\n", - "# 6, # At least 6 rows including all history\n", - "# \"Final dimension records with full history\"\n", - "# )\n", - "\n", - "# # Verify active record count\n", - "# v.validate_active_scd2_count(\n", - "# f\"{gold_schema}.dim_customer_sql_sample\",\n", - "# 4, # 4 active customers (John, Jane, Alice, Joe) - Richard deleted\n", - "# \"__END_AT\"\n", - "# )\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multi-Source Streaming Final State\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"SILVER LAYER - Multi-Source Streaming Final State\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Multi-source streaming should have accumulated history\n", - "v.validate_active_scd2_count(\n", - " f\"{silver_schema}.customer_ms_basic\",\n", - " 4, # 4 active (John, Jane, Alice, Joe)\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# Should have multiple closed records from all the changes\n", - "v.validate_min_closed_scd2_count(\n", - " f\"{silver_schema}.customer_ms_basic\",\n", - " 2, # At least 2 closed (Richard deleted, plus changes)\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"OPERATIONAL METADATA - meta_load_details Validation\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Validate meta_load_details nested column fields are not null\n", - "# Using wildcard to check all fields in the struct\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer meta_load_details\"\n", - ")\n", - "\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer_address\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer_address meta_load_details\"\n", - ")\n", - "\n", - "v.validate_column_not_null(\n", - " f\"{silver_schema}.customer_ms_basic\",\n", - " \"meta_load_details.pipeline_start_timestamp\",\n", - " \"Silver customer_ms_basic meta_load_details\"\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Validation Summary\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Validation summary runs at the end of the notebook after all checks complete\n", - "pass" - ] - }, - { - "cell_type": "code", - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature General Pipeline (Run 4)\")\n", - "print(\"=\" * 60)\n", - "\n", - "# append_sql_flow: unchanged from Run 3 (no new addresses in Run 4)\n", - "v.validate_row_count(f\"{bronze_schema}.append_sql_flow\", 8, \"Address rows after Run 4 (unchanged)\")\n", - "\n", - "# append_view_flow: 7 from Run 3 + 1 new (John email update) = 8\n", - "v.validate_row_count(f\"{bronze_schema}.append_view_flow\", 8, \"Customer rows after Run 4 (+1 John update)\")\n", - "\n", - "# feature_constraints: 7 + 1 = 8 rows\n", - "v.validate_row_count(f\"{bronze_schema}.feature_constraints\", 8, \"Customer rows with DDL constraints after Run 4\")\n", - "\n", - "# feature_version_mapping_flows: SCD1, John updated to new email\n", - "v.validate_row_count(f\"{bronze_schema}.feature_version_mapping_flows\", 5, \"SCD1 version mapping (5 rows, John updated)\")\n", - "v.validate_column_value(\n", - " f\"{bronze_schema}.feature_version_mapping_flows\",\n", - " \"CUSTOMER_ID = 1\",\n", - " \"EMAIL\",\n", - " \"john.doe@another.example.com\",\n", - " \"John's final email in SCD1 table\"\n", - ")\n", - "\n", - "# v_version_mapping_standard: same SCD1 behaviour\n", - "v.validate_row_count(f\"{bronze_schema}.v_version_mapping_standard\", 5, \"SCD1 standard (5 rows after Run 4)\")\n", - "v.validate_column_value(\n", - " f\"{bronze_schema}.v_version_mapping_standard\",\n", - " \"CUSTOMER_ID = 1\",\n", - " \"EMAIL\",\n", - " \"john.doe@another.example.com\",\n", - " \"John's final email in SCD1 standard\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Snapshots (Additional Tables) Run 4\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Periodic snapshot SCD1: no snapshot source change in Run 4 (same 4 rows as Run 3)\n", - "v.validate_row_count(\n", - " f\"{bronze_schema}.feature_periodic_snapshot_scd1\",\n", - " 4,\n", - " \"Periodic snapshot SCD1 (4 customers, unchanged from Run 3)\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Feature Table Migration Pipeline Run 4\")\n", - "print(\"=\" * 60)\n", - "\n", - "# feature_migrated_table_scd2: Run 4 CDC does not produce additional SCD2 versions in this table\n", - "# Active: still 5 keys; Closed: still 1 (same as Run 2/3)\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", - " 5,\n", - " \"__END_AT\"\n", - ")\n", - "v.validate_min_closed_scd2_count(\n", - " f\"{bronze_schema}.feature_migrated_table_scd2\",\n", - " 1,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# feature_migrated_table_append_only: accumulates rows from streaming CDC\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.feature_migrated_table_append_only\",\n", - " 3,\n", - " \"Append-only migration (at least 3 rows after Run 4)\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Template Samples After Run 4\")\n", - "print(\"=\" * 60)\n", - "\n", - "# customer_template: no new snapshot files in Run 4; unchanged from Run 2\n", - "# Active: 6 records (same as Run 2/3)\n", - "v.validate_active_scd2_count(\n", - " f\"{bronze_schema}.customer_template\",\n", - " 6,\n", - " \"__END_AT\"\n", - ")\n", - "\n", - "# customer_address_template: no new address snapshot files in Run 4; unchanged\n", - "v.validate_min_row_count(\n", - " f\"{bronze_schema}.customer_address_template\",\n", - " 5,\n", - " \"Template customer_address SCD2 (at least 5 rows, unchanged in Run 4)\"\n", - ")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"BRONZE LAYER - Materialized Views After Run 4\")\n", - "print(\"=\" * 60)\n", - "\n", - "# MVs source from staging.customer (8 rows after Run 4: 7 from Run 2/3 + 1 John update)\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_source_view\", 8, \"MV source view (8 rows after Run 4)\")\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_sql_path\", 8, \"MV SQL path (8 rows after Run 4)\")\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_chain_mvs\", 8, \"Chained MV (8 rows after Run 4)\")\n", - "\n", - "# feature_mv_with_quarantine: no new addresses in Run 4 (unchanged from Run 3)\n", - "v.validate_row_count(f\"{bronze_schema}.feature_mv_with_quarantine\", 7, \"MV with quarantine (7 valid, unchanged in Run 4)\")" - ], - "metadata": {}, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "v.print_summary()" - ], - "metadata": {}, - "execution_count": null, - "outputs": [] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file diff --git a/samples/pattern-samples/src/notebooks/validation_utils.py b/samples/pattern-samples/src/notebooks/validation_utils.py deleted file mode 100644 index 2b5c8e1..0000000 --- a/samples/pattern-samples/src/notebooks/validation_utils.py +++ /dev/null @@ -1,306 +0,0 @@ -# Validation helper functions for Lakeflow Framework sample pipelines -from pyspark.sql import DataFrame, SparkSession -from typing import List, Dict, Any, Optional - - -class ValidationResult: - """Represents the result of a validation check.""" - - def __init__(self, test_name: str, passed: bool, message: str, details: Optional[Dict] = None): - self.test_name = test_name - self.passed = passed - self.message = message - self.details = details or {} - - def __repr__(self): - status = "✅ PASS" if self.passed else "❌ FAIL" - return f"{status}: {self.test_name} - {self.message}" - - -class ValidationRunner: - """Runs validation checks and tracks results.""" - - def __init__(self, spark: SparkSession): - self.spark = spark - self.results: List[ValidationResult] = [] - - def validate_row_count(self, table_name: str, expected_count: int, description: str = "") -> ValidationResult: - """Validate that a table has the expected row count.""" - try: - actual_count = self.spark.table(table_name).count() - passed = actual_count == expected_count - message = f"Expected {expected_count} rows, got {actual_count}" - if description: - message = f"{description}: {message}" - result = ValidationResult(f"Row Count: {table_name}", passed, message, {"expected": expected_count, "actual": actual_count}) - except Exception as e: - result = ValidationResult(f"Row Count: {table_name}", False, f"Error: {str(e)}") - self.results.append(result) - print(result) - return result - - def validate_min_row_count(self, table_name: str, min_count: int, description: str = "") -> ValidationResult: - """Validate that a table has at least the minimum row count.""" - try: - actual_count = self.spark.table(table_name).count() - passed = actual_count >= min_count - message = f"Expected at least {min_count} rows, got {actual_count}" - if description: - message = f"{description}: {message}" - result = ValidationResult(f"Min Row Count: {table_name}", passed, message, {"min_expected": min_count, "actual": actual_count}) - except Exception as e: - result = ValidationResult(f"Min Row Count: {table_name}", False, f"Error: {str(e)}") - self.results.append(result) - print(result) - return result - - def validate_active_scd2_count(self, table_name: str, expected_count: int, end_column: str = "__END_AT") -> ValidationResult: - """Validate count of active (current) SCD2 records.""" - try: - df = self.spark.table(table_name) - actual_count = df.filter(f"{end_column} IS NULL").count() - passed = actual_count == expected_count - message = f"Expected {expected_count} active records, got {actual_count}" - result = ValidationResult(f"Active SCD2 Records: {table_name}", passed, message, {"expected": expected_count, "actual": actual_count}) - except Exception as e: - result = ValidationResult(f"Active SCD2 Records: {table_name}", False, f"Error: {str(e)}") - self.results.append(result) - print(result) - return result - - def validate_closed_scd2_count(self, table_name: str, expected_count: int, end_column: str = "__END_AT") -> ValidationResult: - """Validate count of closed (historical) SCD2 records.""" - try: - df = self.spark.table(table_name) - actual_count = df.filter(f"{end_column} IS NOT NULL").count() - passed = actual_count == expected_count - message = f"Expected {expected_count} closed records, got {actual_count}" - result = ValidationResult(f"Closed SCD2 Records: {table_name}", passed, message, {"expected": expected_count, "actual": actual_count}) - except Exception as e: - result = ValidationResult(f"Closed SCD2 Records: {table_name}", False, f"Error: {str(e)}") - self.results.append(result) - print(result) - return result - - def validate_min_closed_scd2_count(self, table_name: str, min_count: int, end_column: str = "__END_AT") -> ValidationResult: - """Validate that at least min_count closed SCD2 records exist.""" - try: - df = self.spark.table(table_name) - actual_count = df.filter(f"{end_column} IS NOT NULL").count() - passed = actual_count >= min_count - message = f"Expected at least {min_count} closed records, got {actual_count}" - result = ValidationResult(f"Min Closed SCD2 Records: {table_name}", passed, message, {"min_expected": min_count, "actual": actual_count}) - except Exception as e: - result = ValidationResult(f"Min Closed SCD2 Records: {table_name}", False, f"Error: {str(e)}") - self.results.append(result) - print(result) - return result - - def validate_values_exist(self, table_name: str, column: str, expected_values: List[Any], description: str = "") -> ValidationResult: - """Validate that specific values exist in a column.""" - try: - df = self.spark.table(table_name) - actual_values = [row[column] for row in df.select(column).distinct().collect()] - missing = [v for v in expected_values if v not in actual_values] - passed = len(missing) == 0 - message = f"All expected values found" if passed else f"Missing values: {missing}" - if description: - message = f"{description}: {message}" - result = ValidationResult(f"Values Exist: {table_name}.{column}", passed, message, {"expected": expected_values, "missing": missing}) - except Exception as e: - result = ValidationResult(f"Values Exist: {table_name}.{column}", False, f"Error: {str(e)}") - self.results.append(result) - print(result) - return result - - def validate_column_value(self, table_name: str, filter_condition: str, column: str, expected_value: Any, description: str = "") -> ValidationResult: - """Validate a specific column value for filtered rows.""" - try: - df = self.spark.table(table_name) - filtered = df.filter(filter_condition) - if filtered.count() == 0: - result = ValidationResult(f"Column Value: {table_name}", False, f"No rows found for filter: {filter_condition}") - else: - actual_value = filtered.select(column).first()[0] - passed = actual_value == expected_value - message = f"Expected '{expected_value}', got '{actual_value}'" - if description: - message = f"{description}: {message}" - result = ValidationResult(f"Column Value: {table_name}.{column}", passed, message) - except Exception as e: - result = ValidationResult(f"Column Value: {table_name}.{column}", False, f"Error: {str(e)}") - self.results.append(result) - print(result) - return result - - def validate_quarantine_count(self, quarantine_table: str, expected_count: int) -> ValidationResult: - """Validate quarantine table row count.""" - return self.validate_row_count(quarantine_table, expected_count, "Quarantine records") - - def _get_column_paths_to_check(self, df: "DataFrame", column_spec: str) -> List[str]: - """ - Resolve a column specification to a list of column paths to check. - - Supports: - - Simple column: "CUSTOMER_ID" -> ["CUSTOMER_ID"] - - Nested column with dot notation: "meta_load_details.record_insert_timestamp" -> ["meta_load_details.record_insert_timestamp"] - - All fields in a struct: "meta_load_details.*" -> ["meta_load_details.field1", "meta_load_details.field2", ...] - - Args: - df: The DataFrame to inspect schema from - column_spec: The column specification string - - Returns: - List of fully qualified column paths to check - """ - from pyspark.sql.types import StructType - - # Handle wildcard for struct expansion - if column_spec.endswith(".*"): - struct_name = column_spec[:-2] # Remove .* - - # Navigate to the struct in the schema - schema = df.schema - parts = struct_name.split(".") - current_type = None - - for i, part in enumerate(parts): - field = schema[part] if i == 0 else current_type[part] - current_type = field.dataType - - if not isinstance(current_type, StructType): - raise ValueError(f"Column '{struct_name}' is not a struct type, cannot use wildcard") - - return [f"{struct_name}.{field.name}" for field in current_type.fields] - - # Handle simple or dot-notation column - return [column_spec] - - def validate_column_not_null( - self, - table_name: str, - columns: "str | List[str]", - description: str = "" - ) -> ValidationResult: - """ - Validate that column(s) do not contain null values. - - This is a generic validation that works with: - - Simple columns: "CUSTOMER_ID" - - Nested columns with dot notation: "meta_load_details.record_insert_timestamp" - - All fields in a struct using wildcard: "meta_load_details.*" - - Multiple columns as a list: ["col1", "meta_load_details.field1", "nested.*"] - - Args: - table_name: The fully qualified table name - columns: Column specification - can be: - - A single column name: "CUSTOMER_ID" - - A nested column path: "meta_load_details.record_insert_timestamp" - - A struct with wildcard to check all fields: "meta_load_details.*" - - A list of any combination of the above - description: Optional description for the validation - - Returns: - ValidationResult indicating if all specified columns have non-null values - - Examples: - # Check a single column - v.validate_column_not_null("schema.table", "CUSTOMER_ID") - - # Check a nested field - v.validate_column_not_null("schema.table", "meta_load_details.record_insert_timestamp") - - # Check all fields in a struct - v.validate_column_not_null("schema.table", "meta_load_details.*") - - # Check multiple columns at once - v.validate_column_not_null("schema.table", ["CUSTOMER_ID", "meta_load_details.*"]) - """ - try: - from pyspark.sql import functions as F - - df = self.spark.table(table_name) - - # Normalize columns to a list - if isinstance(columns, str): - column_specs = [columns] - else: - column_specs = columns - - # Resolve all column specs to actual column paths - all_column_paths = [] - for spec in column_specs: - try: - paths = self._get_column_paths_to_check(df, spec) - all_column_paths.extend(paths) - except Exception as e: - result = ValidationResult( - f"Column Not Null: {table_name}", - False, - f"Error resolving column spec '{spec}': {str(e)}" - ) - self.results.append(result) - print(result) - return result - - # Check each column for null values - null_columns = [] - null_counts = {} - - for col_path in all_column_paths: - null_count = df.filter(F.col(col_path).isNull()).count() - if null_count > 0: - null_columns.append(col_path) - null_counts[col_path] = null_count - - passed = len(null_columns) == 0 - - if passed: - message = f"All {len(all_column_paths)} column(s) have non-null values" - else: - null_details = ", ".join([f"{c}({null_counts[c]} nulls)" for c in null_columns]) - message = f"Found null values in: {null_details}" - - if description: - message = f"{description}: {message}" - - # Create a display name for the columns being checked - col_display = ", ".join(column_specs) if len(column_specs) <= 3 else f"{len(column_specs)} columns" - - result = ValidationResult( - f"Column Not Null: {table_name} [{col_display}]", - passed, - message, - {"columns_checked": all_column_paths, "null_columns": null_columns, "null_counts": null_counts} - ) - except Exception as e: - col_display = columns if isinstance(columns, str) else ", ".join(columns[:3]) - result = ValidationResult( - f"Column Not Null: {table_name} [{col_display}]", - False, - f"Error: {str(e)}" - ) - - self.results.append(result) - print(result) - return result - - def print_summary(self): - """Print validation summary and raise AssertionError if any tests failed.""" - passed = sum(1 for r in self.results if r.passed) - failed = sum(1 for r in self.results if not r.passed) - total = len(self.results) - - print("\n" + "="*60) - print(f"VALIDATION SUMMARY: {passed}/{total} tests passed") - print("="*60) - - if failed > 0: - print("\nFailed Tests:") - for r in self.results: - if not r.passed: - print(f" - {r.test_name}: {r.message}") - raise AssertionError(f"{failed} validation(s) failed!") - else: - print("\n✅ All validations passed!") - From 2c109fdc5e07c09006189bb442a700652b87049e Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 26 Jun 2026 17:43:58 +1000 Subject: [PATCH 05/19] refactor: rename nodester to nodespec everywhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the spec type, schema (spec_nodespec.json + nodespecSpec), transformer (NodespecSpecTransformer), migration script, sample bundle (nodespec_sample), pipelines, docs, and the data_flow_type value from "nodester" to "nodespec". No behavioural change — purely a rename. Co-authored-by: Isaac --- ...=> 0007-unified-nodespec-dataflow-spec.md} | 20 +++--- ...md => nodespec_feature_samples_changes.md} | 66 +++++++++---------- ...st => dataflow_spec_ref_main_nodespec.rst} | 46 ++++++------- docs/source/dataflow_spec_reference.rst | 2 +- docs/source/feature_auto_complete.rst | 4 +- samples/deploy.sh | 4 +- samples/deploy_and_test.sh | 20 +++--- ...{deploy_nodester.sh => deploy_nodespec.sh} | 12 ++-- samples/destroy.sh | 6 +- .../.gitignore | 0 .../.vscode/settings.json | 0 .../README.md | 40 +++++------ .../databricks.yml | 4 +- .../pytest.ini | 0 .../nodespec_base_samples_pipeline.yml} | 8 +-- ...nodespec_customer_cloudfiles_pipeline.yml} | 6 +- ...despec_customer_multi_target_pipeline.yml} | 6 +- .../nodespec_customer_simple_pipeline.yml} | 6 +- .../nodespec_customer_transform_pipeline.yml} | 6 +- .../jobs/nodespec_samples_run_job.yml} | 46 ++++++------- .../nodespec_base_samples_pipeline.yml} | 8 +-- ...nodespec_customer_cloudfiles_pipeline.yml} | 6 +- ...despec_customer_multi_target_pipeline.yml} | 6 +- .../nodespec_customer_simple_pipeline.yml} | 6 +- .../nodespec_customer_transform_pipeline.yml} | 6 +- ...feature_samples_data_quality_pipeline.yml} | 6 +- ...spec_feature_samples_general_pipeline.yml} | 6 +- ...e_samples_materialized_views_pipeline.yml} | 6 +- ...espec_feature_samples_python_pipeline.yml} | 6 +- ...ec_feature_samples_snapshots_pipeline.yml} | 6 +- ...ture_samples_table_migration_pipeline.yml} | 6 +- .../customer_cloudfiles_main.json | 6 +- .../customer_multi_target_main.json | 6 +- .../dataflowspec/customer_simple_main.json | 6 +- .../customer_st_with_mv_main.json | 6 +- .../customer_with_transform_main.json | 8 +-- .../expectations/customer_final_dqe.json | 0 .../expectations/customer_staging_dqe.json | 0 .../schemas/customer_enriched_schema.json | 0 .../base_samples/schemas/customer_schema.json | 0 .../schemas/customer_source_schema.json | 0 .../dataflowspec/append_sql_flow_main.json | 4 +- .../dataflowspec/append_view_flow_main.json | 4 +- .../append_view_once_flow_main.json | 4 +- .../dataflowspec/ddl_schema_main.json | 4 +- ...storical_snapshot_files_datetime_main.json | 4 +- ...napshot_files_datetime_multifile_main.json | 4 +- ...tetime_recursive_and_partitioned_main.json | 4 +- ...ecursive_and_partitioned_parquet_main.json | 4 +- ..._recursive_and_partitioned_regex_main.json | 4 +- .../historical_snapshot_files_flow_main.json | 4 +- .../historical_snapshot_files_int_main.json | 4 +- ...snapshot_files_schema_and_select_main.json | 4 +- ...storical_snapshot_table_datetime_main.json | 4 +- ...snapshot_table_select_expression_main.json | 4 +- .../dataflowspec/materialized_views_main.json | 4 +- .../periodic_snapshot_scd1_main.json | 4 +- .../periodic_snapshot_scd2_main.json | 4 +- .../python_source_extension_main.json | 4 +- .../dataflowspec/python_source_main.json | 4 +- .../python_transform_extension_main.json | 4 +- .../dataflowspec/python_transform_main.json | 4 +- .../dataflowspec/quarantine_flag_main.json | 4 +- .../dataflowspec/quarantine_table_main.json | 4 +- .../quarantine_table_with_cdc_main.json | 4 +- .../sink_custom_python_generate_cdc_feed.json | 4 +- .../sink_delta_path_table_main.json | 4 +- .../sink_delta_uc_table_main.json | 4 +- ...k_foreach_batch_single_basic_sql_main.json | 4 +- ...ach_batch_single_python_function_main.json | 4 +- .../table_migration_append_only_main.json | 4 +- .../table_migration_scd2_main.json | 4 +- ...rsion_mapping_flows_dataflowspec_main.json | 4 +- ...on_mapping_standard_dataflowspec_main.json | 4 +- .../dml/customer_purchase_transformed.sql | 0 .../dml/feature_mv_sql_path.sql | 0 .../expectations/customer_address_dqe.json | 0 .../feature_foreach_batch_python.py | 0 .../feature_python_function_source.py | 0 .../feature_python_function_transform.py | 0 ...ure_historic_snapshot_customer_schema.json | 0 .../target/customer_address_schema.json | 0 .../schemas/target/customer_schema.json | 0 .../schemas/target/feature_ddl_schema.ddl | 0 ...ure_historic_snapshot_customer_schema.json | 0 ...ure_periodic_snapshot_customer_schema.json | 0 ...feature_python_function_source_schema.json | 0 ...ture_python_function_transform_schema.json | 0 .../feature_version_mapping_flows_schema.json | 0 .../src/extensions/sources.py | 0 .../src/extensions/transforms.py | 0 .../pipeline_configs/dev_substitutions.json | 0 .../src/pipeline_configs/global.json | 0 samples/nodespec_sample/tests/__init__.py | 1 + .../tests/test_placeholder.py | 2 +- samples/nodester_sample/tests/__init__.py | 1 - ..._to_nodester.py => migrate_to_nodespec.py} | 38 +++++------ .../dataflow_spec_builder.py | 14 ++-- .../transformer/__init__.py | 4 +- src/dataflow_spec_builder/transformer/base.py | 2 +- .../transformer/factory.py | 4 +- .../transformer/{nodester.py => nodespec.py} | 28 ++++---- src/schemas/main.json | 4 +- ...{spec_nodester.json => spec_nodespec.json} | 10 +-- 104 files changed, 321 insertions(+), 321 deletions(-) rename docs/decisions/{0007-unified-nodester-dataflow-spec.md => 0007-unified-nodespec-dataflow-spec.md} (96%) rename docs/{nodester_feature_samples_changes.md => nodespec_feature_samples_changes.md} (84%) rename docs/source/{dataflow_spec_ref_main_nodester.rst => dataflow_spec_ref_main_nodespec.rst} (93%) rename samples/{deploy_nodester.sh => deploy_nodespec.sh} (80%) rename samples/{nodester_sample => nodespec_sample}/.gitignore (100%) rename samples/{nodester_sample => nodespec_sample}/.vscode/settings.json (100%) rename samples/{nodester_sample => nodespec_sample}/README.md (83%) rename samples/{nodester_sample => nodespec_sample}/databricks.yml (91%) rename samples/{nodester_sample => nodespec_sample}/pytest.ini (100%) rename samples/{nodester_sample/resources/classic/nodester_base_samples_pipeline.yml => nodespec_sample/resources/classic/nodespec_base_samples_pipeline.yml} (71%) rename samples/{nodester_sample/resources/classic/nodester_customer_cloudfiles_pipeline.yml => nodespec_sample/resources/classic/nodespec_customer_cloudfiles_pipeline.yml} (78%) rename samples/{nodester_sample/resources/classic/nodester_customer_multi_target_pipeline.yml => nodespec_sample/resources/classic/nodespec_customer_multi_target_pipeline.yml} (78%) rename samples/{nodester_sample/resources/classic/nodester_customer_simple_pipeline.yml => nodespec_sample/resources/classic/nodespec_customer_simple_pipeline.yml} (78%) rename samples/{nodester_sample/resources/classic/nodester_customer_transform_pipeline.yml => nodespec_sample/resources/classic/nodespec_customer_transform_pipeline.yml} (78%) rename samples/{nodester_sample/resources/serverless/jobs/nodester_samples_run_job.yml => nodespec_sample/resources/serverless/jobs/nodespec_samples_run_job.yml} (60%) rename samples/{nodester_sample/resources/serverless/nodester_base_samples_pipeline.yml => nodespec_sample/resources/serverless/nodespec_base_samples_pipeline.yml} (70%) rename samples/{nodester_sample/resources/serverless/nodester_customer_cloudfiles_pipeline.yml => nodespec_sample/resources/serverless/nodespec_customer_cloudfiles_pipeline.yml} (77%) rename samples/{nodester_sample/resources/serverless/nodester_customer_multi_target_pipeline.yml => nodespec_sample/resources/serverless/nodespec_customer_multi_target_pipeline.yml} (77%) rename samples/{nodester_sample/resources/serverless/nodester_customer_simple_pipeline.yml => nodespec_sample/resources/serverless/nodespec_customer_simple_pipeline.yml} (78%) rename samples/{nodester_sample/resources/serverless/nodester_customer_transform_pipeline.yml => nodespec_sample/resources/serverless/nodespec_customer_transform_pipeline.yml} (77%) rename samples/{nodester_sample/resources/serverless/nodester_feature_samples_data_quality_pipeline.yml => nodespec_sample/resources/serverless/nodespec_feature_samples_data_quality_pipeline.yml} (77%) rename samples/{nodester_sample/resources/serverless/nodester_feature_samples_general_pipeline.yml => nodespec_sample/resources/serverless/nodespec_feature_samples_general_pipeline.yml} (77%) rename samples/{nodester_sample/resources/serverless/nodester_feature_samples_materialized_views_pipeline.yml => nodespec_sample/resources/serverless/nodespec_feature_samples_materialized_views_pipeline.yml} (77%) rename samples/{nodester_sample/resources/serverless/nodester_feature_samples_python_pipeline.yml => nodespec_sample/resources/serverless/nodespec_feature_samples_python_pipeline.yml} (77%) rename samples/{nodester_sample/resources/serverless/nodester_feature_samples_snapshots_pipeline.yml => nodespec_sample/resources/serverless/nodespec_feature_samples_snapshots_pipeline.yml} (77%) rename samples/{nodester_sample/resources/serverless/nodester_feature_samples_table_migration_pipeline.yml => nodespec_sample/resources/serverless/nodespec_feature_samples_table_migration_pipeline.yml} (77%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json (92%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json (95%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/base_samples/dataflowspec/customer_simple_main.json (87%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json (91%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json (96%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/base_samples/expectations/customer_final_dqe.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/base_samples/expectations/customer_staging_dqe.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/base_samples/schemas/customer_enriched_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/base_samples/schemas/customer_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/base_samples/schemas/customer_source_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json (92%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json (90%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json (90%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json (91%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json (93%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json (93%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json (94%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json (94%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json (94%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json (97%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json (93%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json (94%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json (93%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json (94%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json (97%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json (94%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json (94%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json (92%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/python_source_main.json (92%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json (93%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/python_transform_main.json (93%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json (93%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json (94%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json (95%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json (91%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json (91%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json (91%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json (93%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json (93%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json (94%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json (95%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json (96%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json (93%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dml/customer_purchase_transformed.sql (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/dml/feature_mv_sql_path.sql (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/expectations/customer_address_dqe.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/python_functions/feature_foreach_batch_python.py (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/python_functions/feature_python_function_source.py (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/python_functions/feature_python_function_transform.py (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/schemas/source/feature_historic_snapshot_customer_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/schemas/target/customer_address_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/schemas/target/customer_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/schemas/target/feature_ddl_schema.ddl (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/schemas/target/feature_historic_snapshot_customer_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/schemas/target/feature_periodic_snapshot_customer_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/schemas/target/feature_python_function_source_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/schemas/target/feature_python_function_transform_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/dataflows/feature_samples/schemas/target/feature_version_mapping_flows_schema.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/extensions/sources.py (100%) rename samples/{nodester_sample => nodespec_sample}/src/extensions/transforms.py (100%) rename samples/{nodester_sample => nodespec_sample}/src/pipeline_configs/dev_substitutions.json (100%) rename samples/{nodester_sample => nodespec_sample}/src/pipeline_configs/global.json (100%) create mode 100644 samples/nodespec_sample/tests/__init__.py rename samples/{nodester_sample => nodespec_sample}/tests/test_placeholder.py (84%) delete mode 100644 samples/nodester_sample/tests/__init__.py rename scripts/{migrate_to_nodester.py => migrate_to_nodespec.py} (94%) rename src/dataflow_spec_builder/transformer/{nodester.py => nodespec.py} (97%) rename src/schemas/{spec_nodester.json => spec_nodespec.json} (99%) diff --git a/docs/decisions/0007-unified-nodester-dataflow-spec.md b/docs/decisions/0007-unified-nodespec-dataflow-spec.md similarity index 96% rename from docs/decisions/0007-unified-nodester-dataflow-spec.md rename to docs/decisions/0007-unified-nodespec-dataflow-spec.md index 53c289a..2232c13 100644 --- a/docs/decisions/0007-unified-nodester-dataflow-spec.md +++ b/docs/decisions/0007-unified-nodespec-dataflow-spec.md @@ -1,4 +1,4 @@ -# ADR-0007: A unified, node-based dataflow spec (`nodester`) +# ADR-0007: A unified, node-based dataflow spec (`nodespec`) **Date:** 2026-06-19 **Status:** Accepted @@ -60,10 +60,10 @@ framework, and the current formats work against it. ## Decision -Introduce a single, unified dataflow spec, called **`nodester`**, and make it the +Introduce a single, unified dataflow spec, called **`nodespec`**, and make it the way pipelines are described going forward. -A `nodester` spec is a **graph of nodes**. There are exactly three node types, and +A `nodespec` spec is a **graph of nodes**. There are exactly three node types, and they compose by **chaining**: ``` @@ -88,7 +88,7 @@ lives. That uniform model replaces all three legacy formats. Every pipeline, whether a one-hop ingest, a CDC merge, a multi-step transform, or a materialized view, is expressed the same way: declare nodes, then chain them together. -Because output type is just a property of a target node, a single `nodester` spec +Because output type is just a property of a target node, a single `nodespec` spec can declare both streaming-table and materialized-view targets at once, including chains where one feeds the other. A logical pipeline that mixes the two no longer has to be split across separate specs by output type. @@ -206,7 +206,7 @@ is buried several levels deep: } ``` -The same pipeline as a `nodester` graph is a flat list of nodes that reads +The same pipeline as a `nodespec` graph is a flat list of nodes that reads top-to-bottom in the order the data flows. Each concern sits on the node it belongs to, and the connections are stated rather than inferred: @@ -214,7 +214,7 @@ belongs to, and the connections are stated rather than inferred: { "data_flow_id": "multi_source_streaming_decomp_staging", "data_flow_group": "multi_source_streaming_decomposed_staging", - "data_flow_type": "nodester", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_customer", @@ -271,7 +271,7 @@ in the chain rather than another layer of nesting. ### Convergence behaviour -`nodester` does not change the engine. The transformer lowers a node graph into +`nodespec` does not change the engine. The transformer lowers a node graph into the framework's existing flow-spec representation, so all current capabilities (CDC, snapshots, data quality, quarantine, sinks, table migration, materialized views) are available. The legacy formats continue to transform into the same @@ -285,7 +285,7 @@ To steer everyone toward the chaining model, two behaviours change: transformation logic into a source definition, meaning a source whose type is `sql` or `python`, or an `append_sql` flow, conflates "where data comes from" with "how it is transformed". This still works, but now emits a warning, in - both the legacy and `nodester` formats. The recommended replacement is a plain + both the legacy and `nodespec` formats. The recommended replacement is a plain source chained into a dedicated transformation node. - **Inline source views on materialized view targets are removed (breaking).** A @@ -318,7 +318,7 @@ To steer everyone toward the chaining model, two behaviours change: representation, so every current feature remains available. - **Smooth adoption.** Legacy formats keep working. A migration script converts - existing specs into `nodester`. The inline-source-transformation warning gives + existing specs into `nodespec`. The inline-source-transformation warning gives consumers time to align before any future removal. - **One breaking change.** Materialized view targets no longer accept an inline @@ -326,7 +326,7 @@ To steer everyone toward the chaining model, two behaviours change: ## Key design decisions -The decisions below define how the `nodester` model behaves. They are the design +The decisions below define how the `nodespec` model behaves. They are the design choices made while building out the format and translating the full feature set onto it. diff --git a/docs/nodester_feature_samples_changes.md b/docs/nodespec_feature_samples_changes.md similarity index 84% rename from docs/nodester_feature_samples_changes.md rename to docs/nodespec_feature_samples_changes.md index 36e6892..9905f09 100644 --- a/docs/nodester_feature_samples_changes.md +++ b/docs/nodespec_feature_samples_changes.md @@ -1,18 +1,18 @@ -# Nodester Feature Samples — Changes, Design & Considerations +# Nodespec Feature Samples — Changes, Design & Considerations ## Overview -The Nodester dataflow spec type introduces a **node-based graph architecture** for defining data pipelines. Instead of the traditional flat (standard) or pre-built flow (flow) spec formats, Nodester expresses pipelines as directed graphs of **source**, **transformation**, and **target** nodes connected through `inputs` arrays. +The Nodespec dataflow spec type introduces a **node-based graph architecture** for defining data pipelines. Instead of the traditional flat (standard) or pre-built flow (flow) spec formats, Nodespec expresses pipelines as directed graphs of **source**, **transformation**, and **target** nodes connected through `inputs` arrays. -This document covers the changes made to support all 33 feature samples from the bronze_sample in the Nodester format, the migration tooling created, and the design decisions and assumptions behind the implementation. +This document covers the changes made to support all 33 feature samples from the bronze_sample in the Nodespec format, the migration tooling created, and the design decisions and assumptions behind the implementation. --- ## What Changed -### 1. Transformer Extensions (`src/dataflow_spec_builder/transformer/nodester.py`) +### 1. Transformer Extensions (`src/dataflow_spec_builder/transformer/nodespec.py`) -The Nodester transformer was extended to support all feature types: +The Nodespec transformer was extended to support all feature types: | Feature | Change | |---------|--------| @@ -28,22 +28,22 @@ The Nodester transformer was extended to support all feature types: | **Materialized view targets** | Targets with `tableType: "mv"` produce separate MV flow specs. Source views for MVs are forced to batch mode (MVs use `spark.sql()` which can't read streaming views). Returns `List[Dict]` for multi-MV specs. | | **Targets without inputs** | Targets with `cdcSnapshotSettings` and no `inputs` create a placeholder MERGE flow. The snapshot system handles file/table reading directly. | -### 2. Schema Extensions (`src/schemas/spec_nodester.json`) +### 2. Schema Extensions (`src/schemas/spec_nodespec.json`) - Removed `isPrimary` from the node schema - Added to `targetNodeConfig`: `targetFormat`, `tableType`, `tableMigrationDetails`, `configFlags`, `once`, `name`, `sinkType`, `sinkConfig`, `sinkOptions`, `sqlPath`, `sqlStatement`, `sourceView`, `refreshPolicy`, `tableDetails`, `cdcApplyChanges` - Added to `sourceNodeConfig`: `functionPath`, `pythonModule`, `tokens`, `sqlPath`, `sqlStatement`, `startingVersionFromDLTSetup`, `cdfChangeTypeOverride` -### 3. Migration Script (`scripts/migrate_to_nodester.py`) +### 3. Migration Script (`scripts/migrate_to_nodespec.py`) -CLI tool that converts any existing spec to Nodester format: +CLI tool that converts any existing spec to Nodespec format: ```bash # Single file -python scripts/migrate_to_nodester.py spec_main.json --output nodester_spec.json +python scripts/migrate_to_nodespec.py spec_main.json --output nodespec_spec.json # Entire directory -python scripts/migrate_to_nodester.py ./dataflowspec/ --output-dir ./nodester_dataflowspec/ +python scripts/migrate_to_nodespec.py ./dataflowspec/ --output-dir ./nodespec_dataflowspec/ ``` Handles all three spec types: @@ -51,19 +51,19 @@ Handles all three spec types: - **Flow** → source nodes (from views) + target nodes (from staging tables + main target) with connections from flow inputs - **Materialized View** → target nodes with `tableType: "mv"`, optional source nodes for sourceView patterns -### 4. Feature Samples (`samples/nodester_sample/src/dataflows/feature_samples/`) +### 4. Feature Samples (`samples/nodespec_sample/src/dataflows/feature_samples/`) -All 33 specs translated with supporting files (schemas, expectations, SQL, Python functions) copied from `bronze_sample`. Each spec uses `dataFlowGroup: "nodester_feature_samples_*"` prefixes. +All 33 specs translated with supporting files (schemas, expectations, SQL, Python functions) copied from `bronze_sample`. Each spec uses `dataFlowGroup: "nodespec_feature_samples_*"` prefixes. -### 5. Pipeline Resources (`samples/nodester_sample/resources/serverless/`) +### 5. Pipeline Resources (`samples/nodespec_sample/resources/serverless/`) Six new pipeline definitions: -- `nodester_feature_samples_general_pipeline.yml` — 9 specs -- `nodester_feature_samples_data_quality_pipeline.yml` — 2 specs -- `nodester_feature_samples_snapshots_pipeline.yml` — 11 specs -- `nodester_feature_samples_python_pipeline.yml` — 4 specs -- `nodester_feature_samples_table_migration_pipeline.yml` — 2 specs -- `nodester_feature_samples_materialized_views_pipeline.yml` — 6 MVs +- `nodespec_feature_samples_general_pipeline.yml` — 9 specs +- `nodespec_feature_samples_data_quality_pipeline.yml` — 2 specs +- `nodespec_feature_samples_snapshots_pipeline.yml` — 11 specs +- `nodespec_feature_samples_python_pipeline.yml` — 4 specs +- `nodespec_feature_samples_table_migration_pipeline.yml` — 2 specs +- `nodespec_feature_samples_materialized_views_pipeline.yml` — 6 MVs ### 6. Framework Bug Fix (`src/dlt_pipeline_builder.py`) @@ -71,11 +71,11 @@ Six new pipeline definitions: --- -## How Nodester Works +## How Nodespec Works ### Node Graph Model -A Nodester spec defines a pipeline as a graph of nodes: +A Nodespec spec defines a pipeline as a graph of nodes: ``` Source Nodes ──→ Transformation Nodes ──→ Target Nodes @@ -91,10 +91,10 @@ Each node has: ### Transformation Pipeline ``` -Nodester JSON Spec +Nodespec JSON Spec │ ▼ -NodesterSpecTransformer._process_spec() +NodespecSpecTransformer._process_spec() │ ├─ Categorize nodes by type ├─ Validate graph (references, cycles, required nodes) @@ -180,7 +180,7 @@ When a source node reads from a table that's also a target in the same spec, it' ### 6. Migration Script Assumptions -- The script preserves `dataFlowId` and `dataFlowGroup` from the original spec (the group is prefixed with `nodester_` separately) +- The script preserves `dataFlowId` and `dataFlowGroup` from the original spec (the group is prefixed with `nodespec_` separately) - Source node IDs are derived from `sourceViewName` (stripping `v_` prefix) - Target node IDs follow the pattern `target_{table_name}` - For flow specs, view names from the original spec become source node IDs directly @@ -202,7 +202,7 @@ When a source node reads from a table that's also a target in the same spec, it' **Status:** Not tested (disabled in the original bronze_sample too). -The `sink_delta_path_table` and `sink_delta_uc_table` specs have `dataFlowGroup: "nodester_feature_samples_general_DISABLED"` — they're excluded from all pipelines, matching the original behavior. +The `sink_delta_path_table` and `sink_delta_uc_table` specs have `dataFlowGroup: "nodespec_feature_samples_general_DISABLED"` — they're excluded from all pipelines, matching the original behavior. --- @@ -223,13 +223,13 @@ The `sink_delta_path_table` and `sink_delta_uc_table` specs have `dataFlowGroup: | File | Action | |------|--------| -| `src/dataflow_spec_builder/transformer/nodester.py` | Extended transformer with all feature support | -| `src/schemas/spec_nodester.json` | Extended schema, removed isPrimary | +| `src/dataflow_spec_builder/transformer/nodespec.py` | Extended transformer with all feature support | +| `src/schemas/spec_nodespec.json` | Extended schema, removed isPrimary | | `src/dlt_pipeline_builder.py` | Moved table_migration init after SubstitutionManager | | `samples/bronze_sample/src/pipeline_configs/global.json` | Changed hardcoded `_es` to `{logical_env}` token | -| `scripts/migrate_to_nodester.py` | New migration script | -| `samples/nodester_sample/src/dataflows/feature_samples/` | 33 translated specs + supporting files | -| `samples/nodester_sample/src/extensions/` | Copied Python extension modules | -| `samples/nodester_sample/src/pipeline_configs/global.json` | Added table_migration_state_volume_path | -| `samples/nodester_sample/resources/serverless/` | 6 new pipeline resource YAMLs | -| `samples/nodester_sample/databricks.yml` | Added serverless resource includes | +| `scripts/migrate_to_nodespec.py` | New migration script | +| `samples/nodespec_sample/src/dataflows/feature_samples/` | 33 translated specs + supporting files | +| `samples/nodespec_sample/src/extensions/` | Copied Python extension modules | +| `samples/nodespec_sample/src/pipeline_configs/global.json` | Added table_migration_state_volume_path | +| `samples/nodespec_sample/resources/serverless/` | 6 new pipeline resource YAMLs | +| `samples/nodespec_sample/databricks.yml` | Added serverless resource includes | diff --git a/docs/source/dataflow_spec_ref_main_nodester.rst b/docs/source/dataflow_spec_ref_main_nodespec.rst similarity index 93% rename from docs/source/dataflow_spec_ref_main_nodester.rst rename to docs/source/dataflow_spec_ref_main_nodespec.rst index 4fd332d..4a4b859 100644 --- a/docs/source/dataflow_spec_ref_main_nodester.rst +++ b/docs/source/dataflow_spec_ref_main_nodespec.rst @@ -1,7 +1,7 @@ -Creating a Nodester Data Flow Spec Reference +Creating a Nodespec Data Flow Spec Reference ############################################ -The **Nodester** Data Flow Spec describes a pipeline as a graph of nodes that +The **Nodespec** Data Flow Spec describes a pipeline as a graph of nodes that chain together: .. code-block:: text @@ -9,8 +9,8 @@ chain together: source -> transformation -> target Instead of the target-table-driven layout of the standard and flow specs, a -Nodester spec is a flat list of nodes that reads top-to-bottom in the order the -data flows. The framework converts a Nodester spec into its internal flow-based +Nodespec spec is a flat list of nodes that reads top-to-bottom in the order the +data flows. The framework converts a Nodespec spec into its internal flow-based format at build time, so every existing capability (CDC, data quality, quarantine, snapshots, table migration, sinks, materialized views) is available. @@ -32,7 +32,7 @@ reads in its SQL (for example ``FROM STREAM(live.v_source_customer)``), and a source names the table, path, or stream it reads. The ``input`` list is therefore a target-node construct. -**Casing.** Nodester specs use ``snake_case`` field names. +**Casing.** Nodespec specs use ``snake_case`` field names. Example: simple data flow ========================= @@ -42,9 +42,9 @@ The simplest spec connects a source to a target: .. code-block:: json { - "data_flow_id": "nodester_customer_simple", - "data_flow_group": "nodester_base_samples", - "data_flow_type": "nodester", + "data_flow_id": "nodespec_customer_simple", + "data_flow_group": "nodespec_base_samples", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_source_customer", @@ -80,9 +80,9 @@ view it reads; the target lists the transformation in its ``input``: .. code-block:: json { - "data_flow_id": "nodester_customer_enriched", - "data_flow_group": "nodester_base_samples", - "data_flow_type": "nodester", + "data_flow_id": "nodespec_customer_enriched", + "data_flow_group": "nodespec_base_samples", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_source_customer", @@ -119,15 +119,15 @@ view it reads; the target lists the transformation in its ``input``: Editor autocomplete and validation ================================== -Nodester specs have a JSON Schema (``src/schemas/spec_nodester.json``), so editors +Nodespec specs have a JSON Schema (``src/schemas/spec_nodespec.json``), so editors can offer key/value autocomplete and inline validation while you author them. The schema is wired through ``main.json``, which routes any ``*_main.json`` file to the correct spec schema based on its ``data_flow_type``. Add the ``json.schemas`` mapping described in :doc:`feature_auto_complete` — the same ``*_main.json`` -mapping enables IntelliSense for nodester (no nodester-specific configuration is +mapping enables IntelliSense for nodespec (no nodespec-specific configuration is required). -.. _dataflow-spec-nodester-metadata-configuration: +.. _dataflow-spec-nodespec-metadata-configuration: Dataflow metadata configuration =============================== @@ -147,7 +147,7 @@ Dataflow metadata configuration - The group to which the data flow belongs. * - **data_flow_type** - ``string`` - - Must be ``"nodester"``. + - Must be ``"nodespec"``. * - **data_flow_version** (*optional*) - ``string`` - Version of the dataflow spec for migration purposes. @@ -158,7 +158,7 @@ Dataflow metadata configuration - ``object`` - Feature flags (e.g., ``operationalMetadataEnabled``). -.. _dataflow-spec-nodester-nodes: +.. _dataflow-spec-nodespec-nodes: Node configuration ================== @@ -204,7 +204,7 @@ Each entry of the ``nodes`` array has: spec target (the backend's ``targetDetails``) as the terminal target — the one not consumed by any other node. Any other targets become staging tables. -.. _dataflow-spec-nodester-source-nodes: +.. _dataflow-spec-nodespec-source-nodes: Source node configuration ========================= @@ -251,7 +251,7 @@ Source node configuration **discouraged** and emit a warning at build time. Model the logic as a dedicated transformation node instead. -.. _dataflow-spec-nodester-transformation-nodes: +.. _dataflow-spec-nodespec-transformation-nodes: Transformation node configuration ================================= @@ -283,7 +283,7 @@ Transformation node configuration view and applies ``apply_transform``. The upstream is inferred from the graph, so no explicit reference is required. -.. _dataflow-spec-nodester-target-nodes: +.. _dataflow-spec-nodespec-target-nodes: Target node configuration ========================= @@ -432,7 +432,7 @@ Example — historical file snapshot: } } -.. _dataflow-spec-nodester-mv: +.. _dataflow-spec-nodespec-mv: Materialized view targets ========================= @@ -477,9 +477,9 @@ A single spec may contain both streaming-table and materialized-view targets, including chains where one feeds the other, so a pipeline that mixes the two does not have to be split across separate specs. -.. _dataflow-spec-nodester-conversion: +.. _dataflow-spec-nodespec-conversion: -How Nodester specs are converted +How Nodespec specs are converted ================================ 1. The terminal target (not consumed by any other node) becomes ``targetDetails``; @@ -503,7 +503,7 @@ Comparison with other spec types * - **Aspect** - **Standard** - **Flow** - - **Nodester** + - **Nodespec** * - **Paradigm** - Single source → target - Target-table-driven with flow groups diff --git a/docs/source/dataflow_spec_reference.rst b/docs/source/dataflow_spec_reference.rst index a555434..a69b2f7 100644 --- a/docs/source/dataflow_spec_reference.rst +++ b/docs/source/dataflow_spec_reference.rst @@ -17,4 +17,4 @@ A Data Flow Spec is a JSON file that defines the structure of a single data flow dataflow_spec_ref_main_standard dataflow_spec_ref_main_flows dataflow_spec_ref_main_materialized_views - dataflow_spec_ref_main_nodester \ No newline at end of file + dataflow_spec_ref_main_nodespec \ No newline at end of file diff --git a/docs/source/feature_auto_complete.rst b/docs/source/feature_auto_complete.rst index 7950971..1b91db5 100644 --- a/docs/source/feature_auto_complete.rst +++ b/docs/source/feature_auto_complete.rst @@ -20,10 +20,10 @@ The framework uses JSON Schema-based IntelliSense when creating or editing Data - Quick Fixes (where applicable) The ``*_main.json`` mapping below covers **all** Data Flow Spec types — standard, -flow, materialized view, and :doc:`nodester `. +flow, materialized view, and :doc:`nodespec `. ``main.json`` inspects the ``data_flow_type`` / ``dataFlowType`` field and routes to the matching schema automatically, so a single mapping enables IntelliSense -for every spec type. Nodester specs are authored in ``snake_case`` and the schema +for every spec type. Nodespec specs are authored in ``snake_case`` and the schema validates them as written. Autocompletion Example diff --git a/samples/deploy.sh b/samples/deploy.sh index ea757c5..ac169e5 100755 --- a/samples/deploy.sh +++ b/samples/deploy.sh @@ -40,8 +40,8 @@ echo "" # Deploy Pattern Samples (end-to-end medallion patterns: bronze → silver → gold) ./deploy_pattern_samples.sh -u "$user" -h "$host" -p "$profile" -c "$compute" -l "$logical_env" --catalog "$catalog" --schema_namespace "$schema_namespace" -# Deploy Nodester Samples (node-based dataflow spec) -./deploy_nodester.sh -u "$user" -h "$host" -p "$profile" -c "$compute" -l "$logical_env" --catalog "$catalog" --schema_namespace "$schema_namespace" +# Deploy Nodespec Samples (node-based dataflow spec) +./deploy_nodespec.sh -u "$user" -h "$host" -p "$profile" -c "$compute" -l "$logical_env" --catalog "$catalog" --schema_namespace "$schema_namespace" # Note: Kafka samples are deployed as part of feature-samples but require external Kafka infrastructure. # Run ./deploy_feature_samples.sh separately and then trigger the kafka_samples_run_job manually. diff --git a/samples/deploy_and_test.sh b/samples/deploy_and_test.sh index 1b3e9e0..577e09f 100755 --- a/samples/deploy_and_test.sh +++ b/samples/deploy_and_test.sh @@ -146,26 +146,26 @@ done cd "$SCRIPT_DIR" || exit 1 -# Step 4: Execute nodester samples run job (reads from staging/bronze created by pattern run 1) +# Step 4: Execute nodespec samples run job (reads from staging/bronze created by pattern run 1) echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo "Executing Nodester Samples Run Job" +echo "Executing Nodespec Samples Run Job" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" -nodester_schema="${schema_namespace}_silver${logical_env}" -setup_bundle_env "Nodester Samples Test Run" "$nodester_schema" +nodespec_schema="${schema_namespace}_silver${logical_env}" +setup_bundle_env "Nodespec Samples Test Run" "$nodespec_schema" -cd "$SCRIPT_DIR/nodester_sample" || { - log_error "Failed to change directory to nodester_sample" +cd "$SCRIPT_DIR/nodespec_sample" || { + log_error "Failed to change directory to nodespec_sample" exit 1 } -log_info "Running nodester_samples_run_job..." -if databricks bundle run nodester_samples_run_job -t dev --profile "$profile" 2>&1; then - log_success "Nodester samples run completed successfully" +log_info "Running nodespec_samples_run_job..." +if databricks bundle run nodespec_samples_run_job -t dev --profile "$profile" 2>&1; then + log_success "Nodespec samples run completed successfully" else - log_error "Nodester samples run failed" + log_error "Nodespec samples run failed" exit 1 fi diff --git a/samples/deploy_nodester.sh b/samples/deploy_nodespec.sh similarity index 80% rename from samples/deploy_nodester.sh rename to samples/deploy_nodespec.sh index 040e3b6..04a3826 100755 --- a/samples/deploy_nodester.sh +++ b/samples/deploy_nodespec.sh @@ -1,8 +1,8 @@ #!/bin/bash -# Nodester Sample Bundle Deployment +# Nodespec Sample Bundle Deployment # Sample-specific constants -BUNDLE_NAME="Nodester Sample Bundle" +BUNDLE_NAME="Nodespec Sample Bundle" SCHEMA="${DEFAULT_SCHEMA_NAMESPACE}_silver" # Source common library and configuration @@ -33,13 +33,13 @@ fi setup_bundle_env "$BUNDLE_NAME" "$schema" # Update substitutions file with catalog and schema namespace -if ! update_substitutions_file "nodester_sample/src/pipeline_configs/dev_substitutions.json"; then +if ! update_substitutions_file "nodespec_sample/src/pipeline_configs/dev_substitutions.json"; then log_error "Failed to update substitutions file. Exiting." exit 1 fi -# Change to nodester_sample directory for deployment -cd nodester_sample +# Change to nodespec_sample directory for deployment +cd nodespec_sample # Deploy the bundle deploy_bundle "$BUNDLE_NAME" @@ -48,5 +48,5 @@ deploy_bundle "$BUNDLE_NAME" cd .. # Restore original substitutions file -restore_substitutions_file "nodester_sample/src/pipeline_configs/dev_substitutions.json" +restore_substitutions_file "nodespec_sample/src/pipeline_configs/dev_substitutions.json" diff --git a/samples/destroy.sh b/samples/destroy.sh index 0419a09..7fe88ae 100755 --- a/samples/destroy.sh +++ b/samples/destroy.sh @@ -25,11 +25,11 @@ echo "" cd .. ########## -# Destroy Nodester Samples -echo "Destroying Nodester Sample Bundle" +# Destroy Nodespec Samples +echo "Destroying Nodespec Sample Bundle" export BUNDLE_VAR_schema="${schema_namespace}_silver${logical_env}" echo "BUNDLE_VAR_schema: $BUNDLE_VAR_schema" -cd nodester_sample +cd nodespec_sample databricks bundle destroy -t dev --profile "$profile" --auto-approve echo "" cd .. diff --git a/samples/nodester_sample/.gitignore b/samples/nodespec_sample/.gitignore similarity index 100% rename from samples/nodester_sample/.gitignore rename to samples/nodespec_sample/.gitignore diff --git a/samples/nodester_sample/.vscode/settings.json b/samples/nodespec_sample/.vscode/settings.json similarity index 100% rename from samples/nodester_sample/.vscode/settings.json rename to samples/nodespec_sample/.vscode/settings.json diff --git a/samples/nodester_sample/README.md b/samples/nodespec_sample/README.md similarity index 83% rename from samples/nodester_sample/README.md rename to samples/nodespec_sample/README.md index bb33ec9..a7be717 100644 --- a/samples/nodester_sample/README.md +++ b/samples/nodespec_sample/README.md @@ -1,36 +1,36 @@ -# Nodester Sample +# Nodespec Sample This sample demonstrates how to use node-based node-based dataflow specifications with the Lakeflow Framework. ## Overview -The Nodester dataflow type (`dataFlowType: "nodester"`) provides compatibility with [node-based](https://github.com/Mmodarre/Lakehouse_Nodester) style specifications. This allows users familiar with Nodester's graph-based approach to define pipelines using source, transformation, and target nodes. +The Nodespec dataflow type (`dataFlowType: "nodespec"`) provides compatibility with [node-based](https://github.com/Mmodarre/Lakehouse_Nodespec) style specifications. This allows users familiar with Nodespec's graph-based approach to define pipelines using source, transformation, and target nodes. ## Directory Structure ``` -nodester_sample/ +nodespec_sample/ ├── databricks.yml # DABs bundle configuration ├── .gitignore ├── pytest.ini ├── README.md ├── resources/ │ ├── classic/ # Classic compute pipeline configs -│ │ ├── nodester_base_samples_pipeline.yml -│ │ ├── nodester_customer_simple_pipeline.yml -│ │ ├── nodester_customer_transform_pipeline.yml -│ │ ├── nodester_customer_multi_target_pipeline.yml -│ │ └── nodester_customer_cloudfiles_pipeline.yml +│ │ ├── nodespec_base_samples_pipeline.yml +│ │ ├── nodespec_customer_simple_pipeline.yml +│ │ ├── nodespec_customer_transform_pipeline.yml +│ │ ├── nodespec_customer_multi_target_pipeline.yml +│ │ └── nodespec_customer_cloudfiles_pipeline.yml │ └── serverless/ # Serverless pipeline configs -│ ├── nodester_base_samples_pipeline.yml -│ ├── nodester_customer_simple_pipeline.yml -│ ├── nodester_customer_transform_pipeline.yml -│ ├── nodester_customer_multi_target_pipeline.yml -│ └── nodester_customer_cloudfiles_pipeline.yml +│ ├── nodespec_base_samples_pipeline.yml +│ ├── nodespec_customer_simple_pipeline.yml +│ ├── nodespec_customer_transform_pipeline.yml +│ ├── nodespec_customer_multi_target_pipeline.yml +│ └── nodespec_customer_cloudfiles_pipeline.yml ├── src/ │ ├── dataflows/ │ │ └── base_samples/ -│ │ ├── dataflowspec/ # Nodester-style dataflow specs +│ │ ├── dataflowspec/ # Nodespec-style dataflow specs │ │ └── schemas/ # Table schemas │ └── pipeline_configs/ │ ├── dev_substitutions.json @@ -130,10 +130,10 @@ SELECT * FROM v_source_customer WHERE status = 'active' ```bash # For classic compute -cp resources/classic/nodester_base_samples_pipeline.yml scratch/resources/ +cp resources/classic/nodespec_base_samples_pipeline.yml scratch/resources/ # For serverless -cp resources/serverless/nodester_base_samples_pipeline.yml scratch/resources/ +cp resources/serverless/nodespec_base_samples_pipeline.yml scratch/resources/ ``` 2. Deploy the bundle: @@ -149,7 +149,7 @@ databricks bundle deploy \ 3. Run the pipeline: ```bash -databricks bundle run lakeflow_samples_nodester_base_pipeline +databricks bundle run lakeflow_samples_nodespec_base_pipeline ``` ### Environment Variables @@ -179,14 +179,14 @@ Edit `src/pipeline_configs/dev_substitutions.json` to add your own substitution } ``` -### Creating New Nodester Specs +### Creating New Nodespec Specs 1. Create a new `*_main.json` file in `src/dataflows/base_samples/dataflowspec/` -2. Use `dataFlowType: "nodester"` and define your nodes +2. Use `dataFlowType: "nodespec"` and define your nodes 3. Add corresponding schema files if needed 4. Create a pipeline resource file in `resources/classic/` and `resources/serverless/` -The Nodester specs are automatically converted to flow specs at runtime, so all framework features (CDC, data quality, secrets, etc.) work seamlessly. +The Nodespec specs are automatically converted to flow specs at runtime, so all framework features (CDC, data quality, secrets, etc.) work seamlessly. ## Backend Behavior Notes diff --git a/samples/nodester_sample/databricks.yml b/samples/nodespec_sample/databricks.yml similarity index 91% rename from samples/nodester_sample/databricks.yml rename to samples/nodespec_sample/databricks.yml index d026347..0d7406a 100644 --- a/samples/nodester_sample/databricks.yml +++ b/samples/nodespec_sample/databricks.yml @@ -1,7 +1,7 @@ -# This is a Databricks asset bundle definition for nodester_sample. +# This is a Databricks asset bundle definition for nodespec_sample. # See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. bundle: - name: nodester_sample + name: nodespec_sample include: - scratch/resources/*.yml diff --git a/samples/nodester_sample/pytest.ini b/samples/nodespec_sample/pytest.ini similarity index 100% rename from samples/nodester_sample/pytest.ini rename to samples/nodespec_sample/pytest.ini diff --git a/samples/nodester_sample/resources/classic/nodester_base_samples_pipeline.yml b/samples/nodespec_sample/resources/classic/nodespec_base_samples_pipeline.yml similarity index 71% rename from samples/nodester_sample/resources/classic/nodester_base_samples_pipeline.yml rename to samples/nodespec_sample/resources/classic/nodespec_base_samples_pipeline.yml index 1674b23..c91fb69 100644 --- a/samples/nodester_sample/resources/classic/nodester_base_samples_pipeline.yml +++ b/samples/nodespec_sample/resources/classic/nodespec_base_samples_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_base_pipeline: - name: Lakeflow Framework - Nodester - Base Pipeline (${var.logical_env}) + lakeflow_samples_nodespec_base_pipeline: + name: Lakeflow Framework - Nodespec - Base Pipeline (${var.logical_env}) channel: CURRENT clusters: - ${var.pipeline_cluster_config} @@ -18,7 +18,7 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - #pipeline.dataFlowIdFilter: nodester_customer_simple,nodester_customer_transform - pipeline.dataFlowGroupFilter: nodester_base_samples + #pipeline.dataFlowIdFilter: nodespec_customer_simple,nodespec_customer_transform + pipeline.dataFlowGroupFilter: nodespec_base_samples root_path: ${workspace.file_path}/src/dataflows/base_samples diff --git a/samples/nodester_sample/resources/classic/nodester_customer_cloudfiles_pipeline.yml b/samples/nodespec_sample/resources/classic/nodespec_customer_cloudfiles_pipeline.yml similarity index 78% rename from samples/nodester_sample/resources/classic/nodester_customer_cloudfiles_pipeline.yml rename to samples/nodespec_sample/resources/classic/nodespec_customer_cloudfiles_pipeline.yml index e6d6dc2..881a2b7 100644 --- a/samples/nodester_sample/resources/classic/nodester_customer_cloudfiles_pipeline.yml +++ b/samples/nodespec_sample/resources/classic/nodespec_customer_cloudfiles_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_customer_cloudfiles: - name: Lakeflow Framework - Nodester - Customer CloudFiles (${var.logical_env}) + lakeflow_samples_nodespec_customer_cloudfiles: + name: Lakeflow Framework - Nodespec - Customer CloudFiles (${var.logical_env}) channel: CURRENT clusters: - ${var.pipeline_cluster_config} @@ -18,6 +18,6 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowIdFilter: nodester_customer_cloudfiles + pipeline.dataFlowIdFilter: nodespec_customer_cloudfiles root_path: ${workspace.file_path}/src/dataflows/base_samples diff --git a/samples/nodester_sample/resources/classic/nodester_customer_multi_target_pipeline.yml b/samples/nodespec_sample/resources/classic/nodespec_customer_multi_target_pipeline.yml similarity index 78% rename from samples/nodester_sample/resources/classic/nodester_customer_multi_target_pipeline.yml rename to samples/nodespec_sample/resources/classic/nodespec_customer_multi_target_pipeline.yml index 5c77ca4..d241e56 100644 --- a/samples/nodester_sample/resources/classic/nodester_customer_multi_target_pipeline.yml +++ b/samples/nodespec_sample/resources/classic/nodespec_customer_multi_target_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_customer_multi_target: - name: Lakeflow Framework - Nodester - Customer Multi Target (${var.logical_env}) + lakeflow_samples_nodespec_customer_multi_target: + name: Lakeflow Framework - Nodespec - Customer Multi Target (${var.logical_env}) channel: CURRENT clusters: - ${var.pipeline_cluster_config} @@ -18,6 +18,6 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowIdFilter: nodester_customer_multi_target + pipeline.dataFlowIdFilter: nodespec_customer_multi_target root_path: ${workspace.file_path}/src/dataflows/base_samples diff --git a/samples/nodester_sample/resources/classic/nodester_customer_simple_pipeline.yml b/samples/nodespec_sample/resources/classic/nodespec_customer_simple_pipeline.yml similarity index 78% rename from samples/nodester_sample/resources/classic/nodester_customer_simple_pipeline.yml rename to samples/nodespec_sample/resources/classic/nodespec_customer_simple_pipeline.yml index e5a7c23..430fd5e 100644 --- a/samples/nodester_sample/resources/classic/nodester_customer_simple_pipeline.yml +++ b/samples/nodespec_sample/resources/classic/nodespec_customer_simple_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_customer_simple: - name: Lakeflow Framework - Nodester - Customer Simple (${var.logical_env}) + lakeflow_samples_nodespec_customer_simple: + name: Lakeflow Framework - Nodespec - Customer Simple (${var.logical_env}) channel: CURRENT clusters: - ${var.pipeline_cluster_config} @@ -18,6 +18,6 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowIdFilter: nodester_customer_simple + pipeline.dataFlowIdFilter: nodespec_customer_simple root_path: ${workspace.file_path}/src/dataflows/base_samples diff --git a/samples/nodester_sample/resources/classic/nodester_customer_transform_pipeline.yml b/samples/nodespec_sample/resources/classic/nodespec_customer_transform_pipeline.yml similarity index 78% rename from samples/nodester_sample/resources/classic/nodester_customer_transform_pipeline.yml rename to samples/nodespec_sample/resources/classic/nodespec_customer_transform_pipeline.yml index aeab199..66adb78 100644 --- a/samples/nodester_sample/resources/classic/nodester_customer_transform_pipeline.yml +++ b/samples/nodespec_sample/resources/classic/nodespec_customer_transform_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_customer_transform: - name: Lakeflow Framework - Nodester - Customer Transform (${var.logical_env}) + lakeflow_samples_nodespec_customer_transform: + name: Lakeflow Framework - Nodespec - Customer Transform (${var.logical_env}) channel: CURRENT clusters: - ${var.pipeline_cluster_config} @@ -18,6 +18,6 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowIdFilter: nodester_customer_transform + pipeline.dataFlowIdFilter: nodespec_customer_transform root_path: ${workspace.file_path}/src/dataflows/base_samples diff --git a/samples/nodester_sample/resources/serverless/jobs/nodester_samples_run_job.yml b/samples/nodespec_sample/resources/serverless/jobs/nodespec_samples_run_job.yml similarity index 60% rename from samples/nodester_sample/resources/serverless/jobs/nodester_samples_run_job.yml rename to samples/nodespec_sample/resources/serverless/jobs/nodespec_samples_run_job.yml index ce8ce29..fc8fb14 100644 --- a/samples/nodester_sample/resources/serverless/jobs/nodester_samples_run_job.yml +++ b/samples/nodespec_sample/resources/serverless/jobs/nodespec_samples_run_job.yml @@ -1,62 +1,62 @@ resources: jobs: - nodester_samples_run_job: - name: Lakeflow Framework - Nodester Samples - Run (${var.logical_env}) + nodespec_samples_run_job: + name: Lakeflow Framework - Nodespec Samples - Run (${var.logical_env}) tasks: # Base samples — individual pipelines - - task_key: nodester_customer_simple + - task_key: nodespec_customer_simple pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_customer_simple.id} + pipeline_id: ${resources.pipelines.lakeflow_samples_nodespec_customer_simple.id} full_refresh: true - - task_key: nodester_customer_cloudfiles + - task_key: nodespec_customer_cloudfiles pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_customer_cloudfiles.id} + pipeline_id: ${resources.pipelines.lakeflow_samples_nodespec_customer_cloudfiles.id} full_refresh: true - - task_key: nodester_customer_transform + - task_key: nodespec_customer_transform pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_customer_transform.id} + pipeline_id: ${resources.pipelines.lakeflow_samples_nodespec_customer_transform.id} full_refresh: true - - task_key: nodester_customer_multi_target + - task_key: nodespec_customer_multi_target pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_customer_multi_target.id} + pipeline_id: ${resources.pipelines.lakeflow_samples_nodespec_customer_multi_target.id} full_refresh: true - # Note: nodester_base_pipeline (combined) is excluded because individual + # Note: nodespec_base_pipeline (combined) is excluded because individual # base sample specs share view names which causes SDP "Cannot redefine" errors # when loaded into a single pipeline. The individual pipelines above cover all base samples. # Feature samples — all run in parallel - - task_key: nodester_feature_general + - task_key: nodespec_feature_general pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_general_pipeline.id} + pipeline_id: ${resources.pipelines.lakeflow_samples_nodespec_feature_general_pipeline.id} full_refresh: true - - task_key: nodester_feature_python + - task_key: nodespec_feature_python pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_python_pipeline.id} + pipeline_id: ${resources.pipelines.lakeflow_samples_nodespec_feature_python_pipeline.id} full_refresh: true - - task_key: nodester_feature_data_quality + - task_key: nodespec_feature_data_quality pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_data_quality_pipeline.id} + pipeline_id: ${resources.pipelines.lakeflow_samples_nodespec_feature_data_quality_pipeline.id} full_refresh: true - - task_key: nodester_feature_snapshots + - task_key: nodespec_feature_snapshots pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_snapshots_pipeline.id} + pipeline_id: ${resources.pipelines.lakeflow_samples_nodespec_feature_snapshots_pipeline.id} full_refresh: true - - task_key: nodester_feature_table_migration + - task_key: nodespec_feature_table_migration pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_table_migration_pipeline.id} + pipeline_id: ${resources.pipelines.lakeflow_samples_nodespec_feature_table_migration_pipeline.id} full_refresh: true - - task_key: nodester_feature_materialized_views + - task_key: nodespec_feature_materialized_views pipeline_task: - pipeline_id: ${resources.pipelines.lakeflow_samples_nodester_feature_materialized_views_pipeline.id} + pipeline_id: ${resources.pipelines.lakeflow_samples_nodespec_feature_materialized_views_pipeline.id} full_refresh: true queue: diff --git a/samples/nodester_sample/resources/serverless/nodester_base_samples_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_base_samples_pipeline.yml similarity index 70% rename from samples/nodester_sample/resources/serverless/nodester_base_samples_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_base_samples_pipeline.yml index 152eaff..c85641b 100644 --- a/samples/nodester_sample/resources/serverless/nodester_base_samples_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_base_samples_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_base_pipeline: - name: Lakeflow Framework - Nodester - Base Pipeline (${var.logical_env}) + lakeflow_samples_nodespec_base_pipeline: + name: Lakeflow Framework - Nodespec - Base Pipeline (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -17,7 +17,7 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - #pipeline.dataFlowIdFilter: nodester_customer_simple,nodester_customer_transform - pipeline.dataFlowGroupFilter: nodester_base_samples + #pipeline.dataFlowIdFilter: nodespec_customer_simple,nodespec_customer_transform + pipeline.dataFlowGroupFilter: nodespec_base_samples root_path: ${workspace.file_path}/src/dataflows/base_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_customer_cloudfiles_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_customer_cloudfiles_pipeline.yml similarity index 77% rename from samples/nodester_sample/resources/serverless/nodester_customer_cloudfiles_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_customer_cloudfiles_pipeline.yml index 9fcfd60..373834e 100644 --- a/samples/nodester_sample/resources/serverless/nodester_customer_cloudfiles_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_customer_cloudfiles_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_customer_cloudfiles: - name: Lakeflow Framework - Nodester - Customer CloudFiles (${var.logical_env}) + lakeflow_samples_nodespec_customer_cloudfiles: + name: Lakeflow Framework - Nodespec - Customer CloudFiles (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -17,6 +17,6 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowIdFilter: nodester_customer_cloudfiles + pipeline.dataFlowIdFilter: nodespec_customer_cloudfiles root_path: ${workspace.file_path}/src/dataflows/base_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_customer_multi_target_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_customer_multi_target_pipeline.yml similarity index 77% rename from samples/nodester_sample/resources/serverless/nodester_customer_multi_target_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_customer_multi_target_pipeline.yml index d747bc3..253b1b2 100644 --- a/samples/nodester_sample/resources/serverless/nodester_customer_multi_target_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_customer_multi_target_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_customer_multi_target: - name: Lakeflow Framework - Nodester - Customer Multi Target (${var.logical_env}) + lakeflow_samples_nodespec_customer_multi_target: + name: Lakeflow Framework - Nodespec - Customer Multi Target (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -17,6 +17,6 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowIdFilter: nodester_customer_multi_target + pipeline.dataFlowIdFilter: nodespec_customer_multi_target root_path: ${workspace.file_path}/src/dataflows/base_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_customer_simple_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_customer_simple_pipeline.yml similarity index 78% rename from samples/nodester_sample/resources/serverless/nodester_customer_simple_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_customer_simple_pipeline.yml index 3cc5282..25f848d 100644 --- a/samples/nodester_sample/resources/serverless/nodester_customer_simple_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_customer_simple_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_customer_simple: - name: Lakeflow Framework - Nodester - Customer Simple (${var.logical_env}) + lakeflow_samples_nodespec_customer_simple: + name: Lakeflow Framework - Nodespec - Customer Simple (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -17,6 +17,6 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowIdFilter: nodester_customer_simple + pipeline.dataFlowIdFilter: nodespec_customer_simple root_path: ${workspace.file_path}/src/dataflows/base_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_customer_transform_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_customer_transform_pipeline.yml similarity index 77% rename from samples/nodester_sample/resources/serverless/nodester_customer_transform_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_customer_transform_pipeline.yml index 9a9d5fc..6b33ce8 100644 --- a/samples/nodester_sample/resources/serverless/nodester_customer_transform_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_customer_transform_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_customer_transform: - name: Lakeflow Framework - Nodester - Customer Transform (${var.logical_env}) + lakeflow_samples_nodespec_customer_transform: + name: Lakeflow Framework - Nodespec - Customer Transform (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -17,6 +17,6 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowIdFilter: nodester_customer_transform + pipeline.dataFlowIdFilter: nodespec_customer_transform root_path: ${workspace.file_path}/src/dataflows/base_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_data_quality_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_data_quality_pipeline.yml similarity index 77% rename from samples/nodester_sample/resources/serverless/nodester_feature_samples_data_quality_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_feature_samples_data_quality_pipeline.yml index 47796e4..f0f2fd1 100644 --- a/samples/nodester_sample/resources/serverless/nodester_feature_samples_data_quality_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_data_quality_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_feature_data_quality_pipeline: - name: Lakeflow Framework - Nodester - Feature Samples Data Quality Pipeline (${var.logical_env}) + lakeflow_samples_nodespec_feature_data_quality_pipeline: + name: Lakeflow Framework - Nodespec - Feature Samples Data Quality Pipeline (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -16,5 +16,5 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowGroupFilter: nodester_feature_samples_data_quality + pipeline.dataFlowGroupFilter: nodespec_feature_samples_data_quality root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_general_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_general_pipeline.yml similarity index 77% rename from samples/nodester_sample/resources/serverless/nodester_feature_samples_general_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_feature_samples_general_pipeline.yml index 3aa9d13..2f96d88 100644 --- a/samples/nodester_sample/resources/serverless/nodester_feature_samples_general_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_general_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_feature_general_pipeline: - name: Lakeflow Framework - Nodester - Feature Samples General Pipeline (${var.logical_env}) + lakeflow_samples_nodespec_feature_general_pipeline: + name: Lakeflow Framework - Nodespec - Feature Samples General Pipeline (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -16,5 +16,5 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowGroupFilter: nodester_feature_samples_general + pipeline.dataFlowGroupFilter: nodespec_feature_samples_general root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_materialized_views_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_materialized_views_pipeline.yml similarity index 77% rename from samples/nodester_sample/resources/serverless/nodester_feature_samples_materialized_views_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_feature_samples_materialized_views_pipeline.yml index 2c1107f..4b9a720 100644 --- a/samples/nodester_sample/resources/serverless/nodester_feature_samples_materialized_views_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_materialized_views_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_feature_materialized_views_pipeline: - name: Lakeflow Framework - Nodester - Feature Samples Materialized Views Pipeline (${var.logical_env}) + lakeflow_samples_nodespec_feature_materialized_views_pipeline: + name: Lakeflow Framework - Nodespec - Feature Samples Materialized Views Pipeline (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -16,5 +16,5 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowGroupFilter: nodester_feature_samples_materialized_views + pipeline.dataFlowGroupFilter: nodespec_feature_samples_materialized_views root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_python_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_python_pipeline.yml similarity index 77% rename from samples/nodester_sample/resources/serverless/nodester_feature_samples_python_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_feature_samples_python_pipeline.yml index c15a961..03e60fd 100644 --- a/samples/nodester_sample/resources/serverless/nodester_feature_samples_python_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_python_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_feature_python_pipeline: - name: Lakeflow Framework - Nodester - Feature Samples Python Pipeline (${var.logical_env}) + lakeflow_samples_nodespec_feature_python_pipeline: + name: Lakeflow Framework - Nodespec - Feature Samples Python Pipeline (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -16,5 +16,5 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowGroupFilter: nodester_feature_samples_python + pipeline.dataFlowGroupFilter: nodespec_feature_samples_python root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_snapshots_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_snapshots_pipeline.yml similarity index 77% rename from samples/nodester_sample/resources/serverless/nodester_feature_samples_snapshots_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_feature_samples_snapshots_pipeline.yml index 3c6b054..733cf73 100644 --- a/samples/nodester_sample/resources/serverless/nodester_feature_samples_snapshots_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_snapshots_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_feature_snapshots_pipeline: - name: Lakeflow Framework - Nodester - Feature Samples Snapshots Pipeline (${var.logical_env}) + lakeflow_samples_nodespec_feature_snapshots_pipeline: + name: Lakeflow Framework - Nodespec - Feature Samples Snapshots Pipeline (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -16,5 +16,5 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowGroupFilter: nodester_feature_samples_snapshots + pipeline.dataFlowGroupFilter: nodespec_feature_samples_snapshots root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/resources/serverless/nodester_feature_samples_table_migration_pipeline.yml b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_table_migration_pipeline.yml similarity index 77% rename from samples/nodester_sample/resources/serverless/nodester_feature_samples_table_migration_pipeline.yml rename to samples/nodespec_sample/resources/serverless/nodespec_feature_samples_table_migration_pipeline.yml index 345eaaa..416e000 100644 --- a/samples/nodester_sample/resources/serverless/nodester_feature_samples_table_migration_pipeline.yml +++ b/samples/nodespec_sample/resources/serverless/nodespec_feature_samples_table_migration_pipeline.yml @@ -1,7 +1,7 @@ resources: pipelines: - lakeflow_samples_nodester_feature_table_migration_pipeline: - name: Lakeflow Framework - Nodester - Feature Samples Table Migration Pipeline (${var.logical_env}) + lakeflow_samples_nodespec_feature_table_migration_pipeline: + name: Lakeflow Framework - Nodespec - Feature Samples Table Migration Pipeline (${var.logical_env}) channel: CURRENT serverless: true catalog: ${var.catalog} @@ -16,5 +16,5 @@ resources: workspace.host: ${var.workspace_host} pipeline.layer: ${var.layer} logicalEnv: ${var.logical_env} - pipeline.dataFlowGroupFilter: nodester_feature_samples_table_migration + pipeline.dataFlowGroupFilter: nodespec_feature_samples_table_migration root_path: ${workspace.file_path}/src/dataflows/feature_samples diff --git a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json similarity index 92% rename from samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json rename to samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json index 5fcd882..761fb54 100644 --- a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json @@ -1,7 +1,7 @@ { - "data_flow_id": "nodester_customer_cloudfiles", - "data_flow_group": "nodester_base_samples", - "data_flow_type": "nodester", + "data_flow_id": "nodespec_customer_cloudfiles", + "data_flow_group": "nodespec_base_samples", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_source_files", diff --git a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json similarity index 95% rename from samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json rename to samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json index cf8b0a8..4280f10 100644 --- a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json @@ -1,7 +1,7 @@ { - "data_flow_id": "nodester_customer_multi_target", - "data_flow_group": "nodester_base_samples", - "data_flow_type": "nodester", + "data_flow_id": "nodespec_customer_multi_target", + "data_flow_group": "nodespec_base_samples", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_source_customer", diff --git a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json similarity index 87% rename from samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json rename to samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json index f207454..559e1c2 100644 --- a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json @@ -1,7 +1,7 @@ { - "data_flow_id": "nodester_customer_simple", - "data_flow_group": "nodester_base_samples", - "data_flow_type": "nodester", + "data_flow_id": "nodespec_customer_simple", + "data_flow_group": "nodespec_base_samples", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_source_customer", diff --git a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json similarity index 91% rename from samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json rename to samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json index 760cbdd..62d64bb 100644 --- a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json @@ -1,7 +1,7 @@ { - "data_flow_id": "nodester_customer_st_with_mv", - "data_flow_group": "nodester_base_samples", - "data_flow_type": "nodester", + "data_flow_id": "nodespec_customer_st_with_mv", + "data_flow_group": "nodespec_base_samples", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_source_customer_st_mv", diff --git a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json similarity index 96% rename from samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json rename to samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json index 71ee799..9fcfadd 100644 --- a/samples/nodester_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json @@ -1,10 +1,10 @@ { - "data_flow_id": "nodester_customer_transform", - "data_flow_group": "nodester_base_samples", - "data_flow_type": "nodester", + "data_flow_id": "nodespec_customer_transform", + "data_flow_group": "nodespec_base_samples", + "data_flow_type": "nodespec", "tags": { "layer": "silver", - "source_project": "nodester" + "source_project": "nodespec" }, "nodes": [ { diff --git a/samples/nodester_sample/src/dataflows/base_samples/expectations/customer_final_dqe.json b/samples/nodespec_sample/src/dataflows/base_samples/expectations/customer_final_dqe.json similarity index 100% rename from samples/nodester_sample/src/dataflows/base_samples/expectations/customer_final_dqe.json rename to samples/nodespec_sample/src/dataflows/base_samples/expectations/customer_final_dqe.json diff --git a/samples/nodester_sample/src/dataflows/base_samples/expectations/customer_staging_dqe.json b/samples/nodespec_sample/src/dataflows/base_samples/expectations/customer_staging_dqe.json similarity index 100% rename from samples/nodester_sample/src/dataflows/base_samples/expectations/customer_staging_dqe.json rename to samples/nodespec_sample/src/dataflows/base_samples/expectations/customer_staging_dqe.json diff --git a/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_enriched_schema.json b/samples/nodespec_sample/src/dataflows/base_samples/schemas/customer_enriched_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/base_samples/schemas/customer_enriched_schema.json rename to samples/nodespec_sample/src/dataflows/base_samples/schemas/customer_enriched_schema.json diff --git a/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_schema.json b/samples/nodespec_sample/src/dataflows/base_samples/schemas/customer_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/base_samples/schemas/customer_schema.json rename to samples/nodespec_sample/src/dataflows/base_samples/schemas/customer_schema.json diff --git a/samples/nodester_sample/src/dataflows/base_samples/schemas/customer_source_schema.json b/samples/nodespec_sample/src/dataflows/base_samples/schemas/customer_source_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/base_samples/schemas/customer_source_schema.json rename to samples/nodespec_sample/src/dataflows/base_samples/schemas/customer_source_schema.json diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json similarity index 92% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json index a2bde00..2f28339 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "append_sql_flow", - "data_flow_group": "nodester_feature_samples_general", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_sql_f_customer_address_append_sql", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json similarity index 90% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json index 9203c8e..2154c6d 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "append_view_flow", - "data_flow_group": "nodester_feature_samples_general", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_append_view_flow", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json similarity index 90% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json index f627f29..d5407f6 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "append_view_once_flow", - "data_flow_group": "nodester_feature_samples_general", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_append_view_once_flow", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json similarity index 91% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json index 73d1077..9665fae 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_constraints", - "data_flow_group": "nodester_feature_samples_general", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_feature_constraints", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json similarity index 93% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json index ef15316..cdbece1 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_historical_files_snapshot_datetime", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "target_feature_historical_snapshot_files_datetime", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json similarity index 93% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json index 50361ae..104af52 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_historical_files_snapshot_datetime_multifile", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "target_feature_historical_snapshot_files_datetime_multifile", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json similarity index 94% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json index 0740a55..b658a72 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_historical_files_snapshot_datetime_recursive_and_partitioned", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "target_feature_historical_snapshot_files_datetime_recursive_and_partitioned", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json similarity index 94% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json index c837332..98676cb 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_historical_files_snapshot_datetime_recursive_and_partitioned_parquet", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "target_feature_historical_snapshot_files_datetime_recursive_and_partitioned_parquet", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json similarity index 94% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json index 0b12b14..1e86f51 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_historical_files_snapshot_datetime_recursive_and_regex", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "target_feature_historical_snapshot_files_datetime_recursive_and_regex", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json similarity index 97% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json index e9a59a2..847b65a 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_historical_files_snapshot_flow", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "target_stg_feature_historical_files_snapshot_flow", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json similarity index 93% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json index e2ea0a7..edafdd4 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_historical_files_snapshot_int", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "target_feature_historical_snapshot_files_int", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json similarity index 94% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json index 5ce4383..22c4b44 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_historical_files_snapshot_schema_and_select", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "target_feature_historical_snapshot_files_schema_and_select", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json similarity index 93% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json index efd70ef..3523917 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_historical_table_snapshot_datetime", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "target_feature_historical_snapshot_table_datetime", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json similarity index 94% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json index 6fbc6aa..d739359 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_historical_table_snapshot_select_expression", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "target_feature_historical_snapshot_table_select_expression", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json similarity index 97% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json index 621bccd..519a8aa 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_materialized_views", - "data_flow_group": "nodester_feature_samples_materialized_views", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_materialized_views", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_feature_mv_source_view", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json similarity index 94% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json index 5636fba..5332a40 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_periodic_snapshot_scd1", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_periodic_snapshot_customer_scd1", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json similarity index 94% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json index f054e27..fa54474 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_periodic_snapshot_scd2", - "data_flow_group": "nodester_feature_samples_snapshots", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_snapshots", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_periodic_snapshot_customer_scd2", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json similarity index 92% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json index ca11de2..155cb38 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_python_extension_source", - "data_flow_group": "nodester_feature_samples_python", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_python", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_feature_python_extension_source", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json similarity index 92% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json index 952295e..00de723 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_python_function_source", - "data_flow_group": "nodester_feature_samples_python", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_python", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_feature_python_function_source", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json similarity index 93% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json index beaae77..a0d08e2 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_python_extension_transform", - "data_flow_group": "nodester_feature_samples_python", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_python", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_source_python_extension_transform", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json similarity index 93% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json index 52e8ba1..b83f333 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_python_function_transform_transform", - "data_flow_group": "nodester_feature_samples_python", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_python", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_source_python_function_transform", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json similarity index 93% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json index 3fb6f8b..1acc9c7 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_quarantine_flag", - "data_flow_group": "nodester_feature_samples_data_quality", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_data_quality", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_customer_address_feature_quarantine_flag", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json similarity index 94% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json index b287e35..b8edec2 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_quarantine_table", - "data_flow_group": "nodester_feature_samples_data_quality", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_data_quality", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_customer_address_feature_quarantine_table", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json similarity index 95% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json index 1eb07f0..0d9ef9e 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "feature_cdc_with_quarantine_table", - "data_flow_group": "nodester_feature_samples", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_customer_address_feature_quarantine_table_cdc", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json similarity index 91% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json index 82b0248..35600cd 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json @@ -1,7 +1,7 @@ { "data_flow_id": "sink_delta_uc_table", - "data_flow_group": "nodester_feature_samples_general_DISABLED", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general_DISABLED", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_customer_delta_sink_uc_table", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json similarity index 91% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json index 04d604b..18bca23 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "sink_delta_path_table", - "data_flow_group": "nodester_feature_samples_general_DISABLED", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general_DISABLED", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_customer_delta_sink_path_table", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json similarity index 91% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json index 176afff..8d9ef2c 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "sink_delta_uc_table", - "data_flow_group": "nodester_feature_samples_general_DISABLED", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general_DISABLED", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_customer_delta_sink_uc_table", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json similarity index 93% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json index dda0a35..969dc13 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "for_each_batch_basic_sql", - "data_flow_group": "nodester_feature_samples_general", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_customer_purchase_basic_sql", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json similarity index 93% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json index a3afde2..a95201d 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "for_each_batch_python_function", - "data_flow_group": "nodester_feature_samples_general", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_customer_purchase_python_function", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json similarity index 94% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json index a941d1e..650ce79 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "table_migration_append_only", - "data_flow_group": "nodester_feature_samples_table_migration", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_table_migration", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_feature_tm_customer_scd0", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json similarity index 95% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json index c6b06ec..be73011 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "table_migration_scd2", - "data_flow_group": "nodester_feature_samples_table_migration", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_table_migration", + "data_flow_type": "nodespec", "nodes": [ { "name": "v_feature_tm_customer_scd2", diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json similarity index 96% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json index a683302..04d8903 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "version_mapping_flows", - "data_flow_group": "nodester_feature_samples_general", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general", + "data_flow_type": "nodespec", "data_flow_version": "0.1.0", "nodes": [ { diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json similarity index 93% rename from samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json rename to samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json index 24435aa..7803173 100644 --- a/samples/nodester_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json @@ -1,7 +1,7 @@ { "data_flow_id": "version_mapping", - "data_flow_group": "nodester_feature_samples_general", - "data_flow_type": "nodester", + "data_flow_group": "nodespec_feature_samples_general", + "data_flow_type": "nodespec", "data_flow_version": "0.1.0", "nodes": [ { diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dml/customer_purchase_transformed.sql b/samples/nodespec_sample/src/dataflows/feature_samples/dml/customer_purchase_transformed.sql similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/dml/customer_purchase_transformed.sql rename to samples/nodespec_sample/src/dataflows/feature_samples/dml/customer_purchase_transformed.sql diff --git a/samples/nodester_sample/src/dataflows/feature_samples/dml/feature_mv_sql_path.sql b/samples/nodespec_sample/src/dataflows/feature_samples/dml/feature_mv_sql_path.sql similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/dml/feature_mv_sql_path.sql rename to samples/nodespec_sample/src/dataflows/feature_samples/dml/feature_mv_sql_path.sql diff --git a/samples/nodester_sample/src/dataflows/feature_samples/expectations/customer_address_dqe.json b/samples/nodespec_sample/src/dataflows/feature_samples/expectations/customer_address_dqe.json similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/expectations/customer_address_dqe.json rename to samples/nodespec_sample/src/dataflows/feature_samples/expectations/customer_address_dqe.json diff --git a/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_foreach_batch_python.py b/samples/nodespec_sample/src/dataflows/feature_samples/python_functions/feature_foreach_batch_python.py similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_foreach_batch_python.py rename to samples/nodespec_sample/src/dataflows/feature_samples/python_functions/feature_foreach_batch_python.py diff --git a/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_source.py b/samples/nodespec_sample/src/dataflows/feature_samples/python_functions/feature_python_function_source.py similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_source.py rename to samples/nodespec_sample/src/dataflows/feature_samples/python_functions/feature_python_function_source.py diff --git a/samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_transform.py b/samples/nodespec_sample/src/dataflows/feature_samples/python_functions/feature_python_function_transform.py similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/python_functions/feature_python_function_transform.py rename to samples/nodespec_sample/src/dataflows/feature_samples/python_functions/feature_python_function_transform.py diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/source/feature_historic_snapshot_customer_schema.json b/samples/nodespec_sample/src/dataflows/feature_samples/schemas/source/feature_historic_snapshot_customer_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/schemas/source/feature_historic_snapshot_customer_schema.json rename to samples/nodespec_sample/src/dataflows/feature_samples/schemas/source/feature_historic_snapshot_customer_schema.json diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_address_schema.json b/samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/customer_address_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_address_schema.json rename to samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/customer_address_schema.json diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_schema.json b/samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/customer_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/schemas/target/customer_schema.json rename to samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/customer_schema.json diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_ddl_schema.ddl b/samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_ddl_schema.ddl similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_ddl_schema.ddl rename to samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_ddl_schema.ddl diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_historic_snapshot_customer_schema.json b/samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_historic_snapshot_customer_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_historic_snapshot_customer_schema.json rename to samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_historic_snapshot_customer_schema.json diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_periodic_snapshot_customer_schema.json b/samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_periodic_snapshot_customer_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_periodic_snapshot_customer_schema.json rename to samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_periodic_snapshot_customer_schema.json diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_source_schema.json b/samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_source_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_source_schema.json rename to samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_source_schema.json diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_transform_schema.json b/samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_transform_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_transform_schema.json rename to samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_python_function_transform_schema.json diff --git a/samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_version_mapping_flows_schema.json b/samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_version_mapping_flows_schema.json similarity index 100% rename from samples/nodester_sample/src/dataflows/feature_samples/schemas/target/feature_version_mapping_flows_schema.json rename to samples/nodespec_sample/src/dataflows/feature_samples/schemas/target/feature_version_mapping_flows_schema.json diff --git a/samples/nodester_sample/src/extensions/sources.py b/samples/nodespec_sample/src/extensions/sources.py similarity index 100% rename from samples/nodester_sample/src/extensions/sources.py rename to samples/nodespec_sample/src/extensions/sources.py diff --git a/samples/nodester_sample/src/extensions/transforms.py b/samples/nodespec_sample/src/extensions/transforms.py similarity index 100% rename from samples/nodester_sample/src/extensions/transforms.py rename to samples/nodespec_sample/src/extensions/transforms.py diff --git a/samples/nodester_sample/src/pipeline_configs/dev_substitutions.json b/samples/nodespec_sample/src/pipeline_configs/dev_substitutions.json similarity index 100% rename from samples/nodester_sample/src/pipeline_configs/dev_substitutions.json rename to samples/nodespec_sample/src/pipeline_configs/dev_substitutions.json diff --git a/samples/nodester_sample/src/pipeline_configs/global.json b/samples/nodespec_sample/src/pipeline_configs/global.json similarity index 100% rename from samples/nodester_sample/src/pipeline_configs/global.json rename to samples/nodespec_sample/src/pipeline_configs/global.json diff --git a/samples/nodespec_sample/tests/__init__.py b/samples/nodespec_sample/tests/__init__.py new file mode 100644 index 0000000..ebcebe0 --- /dev/null +++ b/samples/nodespec_sample/tests/__init__.py @@ -0,0 +1 @@ +# Nodespec Sample Tests diff --git a/samples/nodester_sample/tests/test_placeholder.py b/samples/nodespec_sample/tests/test_placeholder.py similarity index 84% rename from samples/nodester_sample/tests/test_placeholder.py rename to samples/nodespec_sample/tests/test_placeholder.py index 9686711..816a6ba 100644 --- a/samples/nodester_sample/tests/test_placeholder.py +++ b/samples/nodespec_sample/tests/test_placeholder.py @@ -1,5 +1,5 @@ """ -Placeholder tests for the Nodester sample. +Placeholder tests for the Nodespec sample. To run integration tests, deploy this bundle and verify the pipelines execute successfully using the Databricks CLI or UI. diff --git a/samples/nodester_sample/tests/__init__.py b/samples/nodester_sample/tests/__init__.py deleted file mode 100644 index 1028a47..0000000 --- a/samples/nodester_sample/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Nodester Sample Tests diff --git a/scripts/migrate_to_nodester.py b/scripts/migrate_to_nodespec.py similarity index 94% rename from scripts/migrate_to_nodester.py rename to scripts/migrate_to_nodespec.py index 3a22b74..b8da03f 100644 --- a/scripts/migrate_to_nodester.py +++ b/scripts/migrate_to_nodespec.py @@ -1,26 +1,26 @@ #!/usr/bin/env python3 """ -Migrate Lakeflow Framework dataflow specs to Nodester format. +Migrate Lakeflow Framework dataflow specs to Nodespec format. Converts standard, flow, and materialized_view specs into the node-based -Nodester specification format. +Nodespec specification format. -Nodester specs are snake_case throughout, so this script converts the +Nodespec specs are snake_case throughout, so this script converts the camelCase field names used by the standard/flow/materialized_view formats (e.g. ``cdfEnabled`` -> ``cdf_enabled``, ``tableProperties`` -> ``table_properties``) while leaving opaque value maps (table properties, reader options, spark conf, tokens) untouched. Usage: - python migrate_to_nodester.py [--output ] - python migrate_to_nodester.py --output-dir [--recursive] + python migrate_to_nodespec.py [--output ] + python migrate_to_nodespec.py --output-dir [--recursive] Examples: # Single file - python migrate_to_nodester.py spec_main.json --output nodester_spec_main.json + python migrate_to_nodespec.py spec_main.json --output nodespec_spec_main.json # Directory (all *_main.json files) - python migrate_to_nodester.py ./dataflowspec/ --output-dir ./nodester_dataflowspec/ + python migrate_to_nodespec.py ./dataflowspec/ --output-dir ./nodespec_dataflowspec/ """ import argparse import json @@ -163,11 +163,11 @@ def _add_target_settings(config: Dict, src: Dict) -> None: def _result_envelope(spec: Dict, nodes: List[Dict]) -> Dict: - """Build the top-level nodester spec (snake_case metadata).""" + """Build the top-level nodespec spec (snake_case metadata).""" result = { "data_flow_id": spec.get("dataFlowId") or spec.get("data_flow_id"), "data_flow_group": spec.get("dataFlowGroup") or spec.get("data_flow_group"), - "data_flow_type": "nodester", + "data_flow_type": "nodespec", "nodes": nodes, } version = _get(spec, "dataFlowVersion", "data_flow_version") @@ -181,7 +181,7 @@ def _result_envelope(spec: Dict, nodes: List[Dict]) -> Dict: def migrate_spec(spec: Dict) -> Dict: - """Convert a dataflow spec to nodester format based on its dataFlowType.""" + """Convert a dataflow spec to nodespec format based on its dataFlowType.""" spec_type = (spec.get("dataFlowType") or spec.get("data_flow_type") or "standard").lower() if spec_type == "standard": @@ -190,8 +190,8 @@ def migrate_spec(spec: Dict) -> Dict: return _migrate_flow(spec) elif spec_type == "materialized_view": return _migrate_materialized_view(spec) - elif spec_type == "nodester": - return spec # already nodester + elif spec_type == "nodespec": + return spec # already nodespec else: print(f" Warning: Unknown dataFlowType '{spec_type}', treating as standard") return _migrate_standard(spec) @@ -200,7 +200,7 @@ def migrate_spec(spec: Dict) -> Dict: # ─── Standard Spec Migration ───────────────────────────────────────────────── def _migrate_standard(spec: Dict) -> Dict: - """Convert a standard spec to nodester.""" + """Convert a standard spec to nodespec.""" source_id = spec.get("sourceViewName") or "v_source" source_node = _build_source_node(source_id, spec.get("sourceType", "delta"), spec.get("sourceDetails", {}), spec.get("mode", "stream")) @@ -217,7 +217,7 @@ def _migrate_standard(spec: Dict) -> Dict: def _build_source_node(name: str, source_type: str, source_details: Dict, mode: str) -> Dict: - """Build a nodester source node (snake_case) from camelCase source details.""" + """Build a nodespec source node (snake_case) from camelCase source details.""" config: Dict[str, Any] = {} if mode: config["mode"] = mode @@ -256,7 +256,7 @@ def _build_spec_target(spec: Dict) -> Tuple[Dict, str]: # ─── Flow Spec Migration ───────────────────────────────────────────────────── def _migrate_flow(spec: Dict) -> Dict: - """Convert a flow spec to nodester.""" + """Convert a flow spec to nodespec.""" nodes: List[Dict] = [] node_ids: set = set() @@ -324,7 +324,7 @@ def find_target_node(target_name: str) -> Optional[Dict]: # Fall back to an explicit sourceView reference. When that reference # is a staging table (a table produced by another target in this - # spec), nodester models it as an explicit "internal" source node + # spec), nodespec models it as an explicit "internal" source node # that reads the staging table; the transformer auto-detects it. if not source_node_id and flow_details.get("sourceView"): source_view = flow_details["sourceView"] @@ -397,7 +397,7 @@ def _build_target_config_from_staging(table_name: str, stg_config: Dict) -> Dict # ─── Materialized View Spec Migration ───────────────────────────────────────── def _migrate_materialized_view(spec: Dict) -> Dict: - """Convert a materialized_view spec to nodester.""" + """Convert a materialized_view spec to nodespec.""" materialized_views = spec.get("materializedViews", {}) nodes: List[Dict] = [] @@ -457,7 +457,7 @@ def process_file(input_path: str, output_path: Optional[str] = None) -> str: def main(): parser = argparse.ArgumentParser( - description="Migrate Lakeflow Framework specs to Nodester format" + description="Migrate Lakeflow Framework specs to Nodespec format" ) parser.add_argument("input", help="Input spec file or directory") parser.add_argument("--output", "-o", help="Output file (for single file mode)") @@ -470,7 +470,7 @@ def main(): args = parser.parse_args() if os.path.isfile(args.input): - output = args.output or args.input.replace(".json", "_nodester.json") + output = args.output or args.input.replace(".json", "_nodespec.json") if args.dry_run: print(f"Would convert: {args.input} → {output}") else: diff --git a/src/dataflow_spec_builder/dataflow_spec_builder.py b/src/dataflow_spec_builder/dataflow_spec_builder.py index 5ebca5b..a561ef9 100644 --- a/src/dataflow_spec_builder/dataflow_spec_builder.py +++ b/src/dataflow_spec_builder/dataflow_spec_builder.py @@ -440,13 +440,13 @@ def _validate_json(spec_item: tuple) -> tuple: json_data = spec_payload.get(self.Keys.DATA) spec_type = spec_payload.get(self.Keys.DATA_FLOW_TYPE, "") - # Nodester specs use their own schema directly — the main.json + # Nodespec specs use their own schema directly — the main.json # if/else chain doesn't route cleanly for non-standard types. - if spec_type == "nodester": - nodester_schema_path = os.path.join(self.framework_path, "schemas", "spec_nodester.json") - if os.path.exists(nodester_schema_path): - nodester_validator = utility.JSONValidator(nodester_schema_path) - # Nodester field names are snake_case. The metadata keys were + if spec_type == "nodespec": + nodespec_schema_path = os.path.join(self.framework_path, "schemas", "spec_nodespec.json") + if os.path.exists(nodespec_schema_path): + nodespec_validator = utility.JSONValidator(nodespec_schema_path) + # Nodespec field names are snake_case. The metadata keys were # normalised to camelCase at read time for uniform downstream # processing, so present them as snake_case for validation. camel_to_snake = { @@ -458,7 +458,7 @@ def _validate_json(spec_item: tuple) -> tuple: validation_data = { camel_to_snake.get(k, k): v for k, v in json_data.items() } - errors = nodester_validator.validate(validation_data) + errors = nodespec_validator.validate(validation_data) else: errors = [] elif file_type == "main": diff --git a/src/dataflow_spec_builder/transformer/__init__.py b/src/dataflow_spec_builder/transformer/__init__.py index 3ed7fb0..b1aa804 100644 --- a/src/dataflow_spec_builder/transformer/__init__.py +++ b/src/dataflow_spec_builder/transformer/__init__.py @@ -2,7 +2,7 @@ from .standard import StandardSpecTransformer from .flow import FlowSpecTransformer from .materialized_views import MaterializedViewSpecTransformer -from .nodester import NodesterSpecTransformer +from .nodespec import NodespecSpecTransformer from .factory import SpecTransformerFactory __all__ = [ @@ -10,6 +10,6 @@ 'StandardSpecTransformer', 'FlowSpecTransformer', 'MaterializedViewSpecTransformer', - 'NodesterSpecTransformer', + 'NodespecSpecTransformer', 'SpecTransformerFactory' ] diff --git a/src/dataflow_spec_builder/transformer/base.py b/src/dataflow_spec_builder/transformer/base.py index d2a6d6a..f0f04cf 100644 --- a/src/dataflow_spec_builder/transformer/base.py +++ b/src/dataflow_spec_builder/transformer/base.py @@ -10,7 +10,7 @@ class BaseSpecTransformer(ABC): # When True, transform() inspects the produced flow spec and warns on # source definitions that embed SQL/Python transformation logic. Spec - # types that can already detect this on their own input (e.g. nodester, + # types that can already detect this on their own input (e.g. nodespec, # which distinguishes source nodes from transformation nodes) set this to # False and emit the warning themselves. WARN_INLINE_SOURCE_TRANSFORMATIONS_FROM_OUTPUT = True diff --git a/src/dataflow_spec_builder/transformer/factory.py b/src/dataflow_spec_builder/transformer/factory.py index e79f1a2..04cc30d 100644 --- a/src/dataflow_spec_builder/transformer/factory.py +++ b/src/dataflow_spec_builder/transformer/factory.py @@ -3,7 +3,7 @@ from .standard import StandardSpecTransformer from .flow import FlowSpecTransformer from .materialized_views import MaterializedViewSpecTransformer -from .nodester import NodesterSpecTransformer +from .nodespec import NodespecSpecTransformer class SpecTransformerFactory: @@ -13,7 +13,7 @@ class SpecTransformerFactory: "standard": StandardSpecTransformer, "flow": FlowSpecTransformer, "materialized_view": MaterializedViewSpecTransformer, - "nodester": NodesterSpecTransformer + "nodespec": NodespecSpecTransformer } @classmethod diff --git a/src/dataflow_spec_builder/transformer/nodester.py b/src/dataflow_spec_builder/transformer/nodespec.py similarity index 97% rename from src/dataflow_spec_builder/transformer/nodester.py rename to src/dataflow_spec_builder/transformer/nodespec.py index 49a533c..f739196 100644 --- a/src/dataflow_spec_builder/transformer/nodester.py +++ b/src/dataflow_spec_builder/transformer/nodespec.py @@ -1,8 +1,8 @@ """ -Nodester Spec Transformer +Nodespec Spec Transformer Converts a node-based dataflow spec into the Lakeflow Framework's flow-based -spec. A nodester spec is a graph of three node kinds: +spec. A nodespec spec is a graph of three node kinds: source -> transformation -> target @@ -26,7 +26,7 @@ - Targets with `table_type: "mv"` each become their own flow spec, so a spec with both streaming and MV targets returns a list of flow specs. -Casing: nodester input is snake_case; the emitted flow spec is the backend's +Casing: nodespec input is snake_case; the emitted flow spec is the backend's camelCase. `_rename_keys` / `_to_backend_keys` do that translation. """ from collections import defaultdict @@ -39,7 +39,7 @@ # Explicit snake_case -> backend key map for nested blobs that are copied # wholesale (CDC snapshot settings, table migration, sink config). The backend # mixes camelCase and snake_case, so only the keys listed here are renamed. -_NODESTER_TO_BACKEND_KEYS = { +_NODESPEC_TO_BACKEND_KEYS = { # CDC snapshot settings "snapshot_type": "snapshotType", "source_type": "sourceType", @@ -68,7 +68,7 @@ def _to_backend_keys(obj: Any) -> Any: """Recursively rename known snake_case keys to their backend equivalents.""" if isinstance(obj, dict): - return {_NODESTER_TO_BACKEND_KEYS.get(k, k): _to_backend_keys(v) for k, v in obj.items()} + return {_NODESPEC_TO_BACKEND_KEYS.get(k, k): _to_backend_keys(v) for k, v in obj.items()} if isinstance(obj, list): return [_to_backend_keys(i) for i in obj] return obj @@ -79,15 +79,15 @@ def _rename_keys(source: Dict, mapping: Dict[str, str]) -> Dict: return {dst: source[src] for src, dst in mapping.items() if src in source} -class NodesterSpecTransformer(BaseSpecTransformer): - """Transform a nodester node-graph spec into a flow spec (or list of them).""" +class NodespecSpecTransformer(BaseSpecTransformer): + """Transform a nodespec node-graph spec into a flow spec (or list of them).""" - # Nodester warns about inline source transformations at the node level (where + # Nodespec warns about inline source transformations at the node level (where # source and transformation nodes are distinguishable), so the base class # should not re-derive that warning from the produced flow spec. WARN_INLINE_SOURCE_TRANSFORMATIONS_FROM_OUTPUT = False - FLOW_GROUP_ID = "nodester_main" + FLOW_GROUP_ID = "nodespec_main" FLOW_PREFIX = "f_" VIEW_PREFIX = "v_" @@ -99,11 +99,11 @@ class NodesterSpecTransformer(BaseSpecTransformer): # Entry point # ------------------------------------------------------------------ # def _process_spec(self, spec_data: Dict) -> Union[Dict, List[Dict]]: - """Transform a nodester spec into one flow spec, or a list when it + """Transform a nodespec spec into one flow spec, or a list when it produces both streaming-table and materialized-view targets.""" nodes = spec_data.get("nodes", []) if not nodes: - raise ValueError("Nodester spec must contain at least one node") + raise ValueError("Nodespec spec must contain at least one node") sources, transforms, targets = self._categorize_nodes(nodes) self._warn_inline_source_transformations(sources) @@ -119,7 +119,7 @@ def _process_spec(self, spec_data: Dict) -> Union[Dict, List[Dict]]: specs.extend(self._build_mv_specs(spec_data, mv_targets, sources, nodes)) if not specs: - raise ValueError("Nodester spec must contain at least one target node") + raise ValueError("Nodespec spec must contain at least one target node") return specs if len(specs) > 1 else specs[0] @staticmethod @@ -186,7 +186,7 @@ def _validate_node_graph(self, sources: List[Dict], targets: List[Dict], all_nod ) if not targets: - raise ValueError("Nodester spec must contain at least one target node") + raise ValueError("Nodespec spec must contain at least one target node") # A source node is unnecessary only when every target reads its data # without one: an MV defined by inline SQL, or a snapshot CDC target. @@ -196,7 +196,7 @@ def _validate_node_graph(self, sources: List[Dict], targets: List[Dict], all_nod ) all_snapshot = all(t.get("config", {}).get("cdc_snapshot_settings") for t in targets) if not sources and not (all_mv_inline_sql or all_snapshot): - raise ValueError("Nodester spec must contain at least one source node") + raise ValueError("Nodespec spec must contain at least one source node") # ------------------------------------------------------------------ # # input handling diff --git a/src/schemas/main.json b/src/schemas/main.json index a53a66e..9bb0e84 100644 --- a/src/schemas/main.json +++ b/src/schemas/main.json @@ -1,10 +1,10 @@ { "title": "Main DataFlowSpec", "if": { - "properties": { "data_flow_type": { "const": "nodester" } }, + "properties": { "data_flow_type": { "const": "nodespec" } }, "required": ["data_flow_type"] }, - "then": { "$ref": "./spec_nodester.json#/$defs/nodesterSpec" }, + "then": { "$ref": "./spec_nodespec.json#/$defs/nodespecSpec" }, "else": { "type": "object", "properties": { diff --git a/src/schemas/spec_nodester.json b/src/schemas/spec_nodespec.json similarity index 99% rename from src/schemas/spec_nodester.json rename to src/schemas/spec_nodespec.json index a2154ff..4b6b1d1 100644 --- a/src/schemas/spec_nodester.json +++ b/src/schemas/spec_nodespec.json @@ -1,12 +1,12 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://dlt-framework/schemas/spec_nodester.json", - "title": "Nodester Dataflow Specification", + "$id": "https://dlt-framework/schemas/spec_nodespec.json", + "title": "Nodespec Dataflow Specification", "description": "Dataflow specification using a node-based architecture. Converts node graphs into Lakeflow Framework's flow-based format. All target-specific settings (CDC, data quality, quarantine) are specified on target nodes directly.", "$defs": { - "nodesterSpec": { + "nodespecSpec": { "type": "object", - "description": "Nodester specs use snake_case field names throughout.", + "description": "Nodespec specs use snake_case field names throughout.", "properties": { "data_flow_id": { "type": "string" @@ -15,7 +15,7 @@ "type": "string" }, "data_flow_type": { - "const": "nodester" + "const": "nodespec" }, "data_flow_version": { "type": "string" From eb1751bed77f2b4b419b5f95cd0487e0b49f1968 Mon Sep 17 00:00:00 2001 From: Haille W Date: Mon, 29 Jun 2026 13:30:17 +1000 Subject: [PATCH 06/19] refactor(nodespec): centralise casing translation, simplify transformer Rewrite the nodespec transformer around a single snake_case -> camelCase key map plus flat/recursive converters, replacing the per-context allowlist maps and per-source-type branches. Builders now copy every non-structural config key as passthrough detail. Output is unchanged (verified identical across all 38 samples and end to end on Databricks); ~593 -> 440 lines. Co-authored-by: Isaac --- .../transformer/nodespec.py | 1015 ++++++----------- 1 file changed, 355 insertions(+), 660 deletions(-) diff --git a/src/dataflow_spec_builder/transformer/nodespec.py b/src/dataflow_spec_builder/transformer/nodespec.py index f739196..fe47d3e 100644 --- a/src/dataflow_spec_builder/transformer/nodespec.py +++ b/src/dataflow_spec_builder/transformer/nodespec.py @@ -1,33 +1,18 @@ """ -Nodespec Spec Transformer - -Converts a node-based dataflow spec into the Lakeflow Framework's flow-based -spec. A nodespec spec is a graph of three node kinds: - - source -> transformation -> target - -- source nodes become views (or direct table references for internal tables) -- transformation nodes become SQL/Python views -- target nodes become the spec target table or staging tables, each - carrying its own settings (CDC, data quality, quarantine, ...) - -How nodes connect: -- A target node lists what feeds it in `config.input`. Each input is either a - view/node name (string) or `{"view": ..., "flow": ...}` to set the flow name. -- Source and transformation nodes reference their upstream inside their own - definition (for example a transform's SQL names the view it reads). - -Spec target selection: -- The backend needs one `targetDetails`. It is auto-selected as the terminal - target (a target not consumed by any other node). The remaining targets become - staging tables. With multiple terminal targets the last one is used. - -Materialized views: -- Targets with `table_type: "mv"` each become their own flow spec, so a spec with - both streaming and MV targets returns a list of flow specs. - -Casing: nodespec input is snake_case; the emitted flow spec is the backend's -camelCase. `_rename_keys` / `_to_backend_keys` do that translation. +Nodespec Spec Transformer. + +Converts a node-based dataflow spec (source -> transformation -> target nodes) +into the framework's flow-based spec. Sources/transformations become views; +targets become the spec target table or staging tables (each carrying its own +CDC / data quality / quarantine settings). The terminal target (not consumed by +another node) becomes the backend ``targetDetails``; the rest become staging +tables. ``table_type: "mv"`` targets each become their own flow spec, so a mixed +spec returns a list. + +The snake_case -> camelCase translation is centralised: one global key map +(``_KEYS``) plus ``_camel`` (flat) / ``_deep_camel`` (nested blobs). Each builder +copies every config key that isn't "structural" (``_HANDLED`` — input, cdc_*, +quarantine_*, sink_*, ...); everything else is passthrough table/source detail. """ from collections import defaultdict from typing import Any, Dict, List, Optional, Tuple, Set, Union @@ -36,87 +21,91 @@ from dataflow.enums import FlowType, Mode, TableType, TargetType -# Explicit snake_case -> backend key map for nested blobs that are copied -# wholesale (CDC snapshot settings, table migration, sink config). The backend -# mixes camelCase and snake_case, so only the keys listed here are renamed. -_NODESPEC_TO_BACKEND_KEYS = { - # CDC snapshot settings - "snapshot_type": "snapshotType", - "source_type": "sourceType", - "version_type": "versionType", - "version_column": "versionColumn", - "starting_version": "startingVersion", - "datetime_format": "datetimeFormat", - "reader_options": "readerOptions", - "schema_path": "schemaPath", - "select_exp": "selectExp", - "deduplicate_mode": "deduplicateMode", - # Table migration details - "catalog_type": "catalogType", - "auto_starting_versions_enabled": "autoStartingVersionsEnabled", - "source_details": "sourceDetails", - "table_name": "tableName", - # Source nested fields / sink config - "function_path": "functionPath", - "python_module": "pythonModule", - "sql_path": "sqlPath", - "sql_statement": "sqlStatement", - "table_properties": "tableProperties", +# The single snake_case -> camelCase map. Identity keys (database, table, path, +# comment, name, tokens, private, ...) need no entry. Used both for flat detail +# conversion and recursive conversion of nested blobs (snapshot CDC, table +# migration, sink config). +_KEYS = { + "schema_path": "schemaPath", "table_properties": "tableProperties", + "partition_columns": "partitionColumns", "cluster_by_columns": "clusterByColumns", + "cluster_by_auto": "clusterByAuto", "spark_conf": "sparkConf", "row_filter": "rowFilter", + "config_flags": "configFlags", "cdf_enabled": "cdfEnabled", "table_path": "tablePath", + "reader_options": "readerOptions", "sql_path": "sqlPath", "sql_statement": "sqlStatement", + "function_path": "functionPath", "python_module": "pythonModule", "select_exp": "selectExp", + "where_clause": "whereClause", "refresh_policy": "refreshPolicy", + "starting_version_from_dlt_setup": "startingVersionFromDLTSetup", + "cdf_change_type_override": "cdfChangeTypeOverride", + # nested snapshot-CDC / table-migration keys + "snapshot_type": "snapshotType", "source_type": "sourceType", "version_type": "versionType", + "version_column": "versionColumn", "starting_version": "startingVersion", + "datetime_format": "datetimeFormat", "deduplicate_mode": "deduplicateMode", + "catalog_type": "catalogType", "auto_starting_versions_enabled": "autoStartingVersionsEnabled", + "source_details": "sourceDetails", "table_name": "tableName", +} + +# Keys a target config handles specially, so they are NOT copied into details. +_HANDLED = { + "input", "table", "table_type", "enabled", "once", "name", + "cdc_settings", "cdc_apply_changes", "cdc_snapshot_settings", + "data_quality_expectations_enabled", "data_quality_expectations_path", + "quarantine_mode", "quarantine_target_details", "table_migration_details", + "sink_type", "sink_config", "sink_options", "table_details", "source_view", } -def _to_backend_keys(obj: Any) -> Any: - """Recursively rename known snake_case keys to their backend equivalents.""" +def _camel(cfg: Dict, drop: Set[str] = frozenset()) -> Dict: + """Flat snake->camel rename of top-level keys (values copied verbatim).""" + return {_KEYS.get(k, k): v for k, v in cfg.items() if k not in drop} + + +def _deep_camel(obj: Any) -> Any: + """Recursive snake->camel for nested blobs (snapshot CDC, migration, sink config).""" if isinstance(obj, dict): - return {_NODESPEC_TO_BACKEND_KEYS.get(k, k): _to_backend_keys(v) for k, v in obj.items()} + return {_KEYS.get(k, k): _deep_camel(v) for k, v in obj.items()} if isinstance(obj, list): - return [_to_backend_keys(i) for i in obj] + return [_deep_camel(i) for i in obj] return obj -def _rename_keys(source: Dict, mapping: Dict[str, str]) -> Dict: - """Copy keys present in `source` into a new dict, renaming src -> dst.""" - return {dst: source[src] for src, dst in mapping.items() if src in source} +def _first(cfg: Dict, *keys: str) -> Dict: + """{camel(key): value} for the first present snake key (e.g. sql_path|sql_statement).""" + for k in keys: + if k in cfg: + return {_KEYS.get(k, k): cfg[k]} + return {} class NodespecSpecTransformer(BaseSpecTransformer): """Transform a nodespec node-graph spec into a flow spec (or list of them).""" - # Nodespec warns about inline source transformations at the node level (where - # source and transformation nodes are distinguishable), so the base class - # should not re-derive that warning from the produced flow spec. WARN_INLINE_SOURCE_TRANSFORMATIONS_FROM_OUTPUT = False - FLOW_GROUP_ID = "nodespec_main" FLOW_PREFIX = "f_" VIEW_PREFIX = "v_" + SOURCE, TRANSFORMATION, TARGET = "source", "transformation", "target" - SOURCE = "source" - TRANSFORMATION = "transformation" - TARGET = "target" - - # ------------------------------------------------------------------ # - # Entry point - # ------------------------------------------------------------------ # def _process_spec(self, spec_data: Dict) -> Union[Dict, List[Dict]]: - """Transform a nodespec spec into one flow spec, or a list when it - produces both streaming-table and materialized-view targets.""" + """Transform a nodespec spec into one flow spec, or a list (streaming + MV).""" nodes = spec_data.get("nodes", []) if not nodes: raise ValueError("Nodespec spec must contain at least one node") - sources, transforms, targets = self._categorize_nodes(nodes) - self._warn_inline_source_transformations(sources) - self._validate_node_graph(sources, targets, nodes) + sources = [n for n in nodes if n.get("node_type", "").lower() == self.SOURCE] + targets = [n for n in nodes if n.get("node_type", "").lower() == self.TARGET] + self._warn_inline_sources(sources) + self._validate(sources, targets, nodes) - mv_targets = [t for t in targets if self._is_mv(t)] - st_targets = [t for t in targets if not self._is_mv(t)] + self.lookup = {n.get("name"): n for n in nodes} + st = [t for t in targets if not self._is_mv(t)] + mv = [t for t in targets if self._is_mv(t)] specs: List[Dict] = [] - if st_targets: - specs.append(self._build_streaming_spec(spec_data, st_targets, sources, nodes)) - if mv_targets: - specs.extend(self._build_mv_specs(spec_data, mv_targets, sources, nodes)) + if st: + self.internal = self._internal_sources(sources, st) + spec_target, others = self._select_spec_target(st, sources, nodes) + specs.append(self._streaming_spec(spec_data, spec_target, others)) + for target in mv: + specs.append(self._mv_spec(spec_data, target, sources)) if not specs: raise ValueError("Nodespec spec must contain at least one target node") @@ -126,620 +115,326 @@ def _process_spec(self, spec_data: Dict) -> Union[Dict, List[Dict]]: def _is_mv(target: Dict) -> bool: return target.get("config", {}).get("table_type") == "mv" - # ------------------------------------------------------------------ # - # Node organisation, validation, warnings - # ------------------------------------------------------------------ # - def _categorize_nodes(self, nodes: List[Dict]) -> Tuple[List[Dict], List[Dict], List[Dict]]: - """Split nodes into (sources, transformations, targets).""" - sources, transforms, targets = [], [], [] - buckets = {self.SOURCE: sources, self.TRANSFORMATION: transforms, self.TARGET: targets} - for node in nodes: - bucket = buckets.get(node.get("node_type", "").lower()) - if bucket is None: - self.logger.warning( - f"Unknown node type '{node.get('node_type')}' for node '{node.get('name')}'" - ) - else: - bucket.append(node) - return sources, transforms, targets - - def _warn_inline_source_transformations(self, source_nodes: List[Dict]) -> None: - """Warn when a source node embeds SQL/Python transformation logic. - - Declaring a SQL or Python transformation directly on a source node (via - ``source_type: "sql"`` / ``"python"``) is still supported for backward - compatibility but discouraged: it hides transformation logic inside the - data-origin node. The recommended pattern is a plain source node chained - into a dedicated transformation node. - """ - for node in source_nodes: - source_type = node.get("source_type", "delta") - if source_type in ("sql", "python"): + # -- validation / warnings -- + + def _warn_inline_sources(self, sources: List[Dict]) -> None: + for node in sources: + st = node.get("source_type", "delta") + if st in ("sql", "python"): self.logger.warning( - "Source node '%s' defines an inline %s transformation " - "(source_type: '%s'). This is still supported but " - "discouraged and may not be supported in a future release. " - "Define a source node and chain a dedicated transformation " - "node off it instead.", - node.get("name"), source_type, source_type, - ) - - def _validate_node_graph(self, sources: List[Dict], targets: List[Dict], all_nodes: List[Dict]) -> None: - """Validate references, the removed MV source_view, and source presence.""" - node_names = {node.get("name") for node in all_nodes} - - for target in targets: - for view in self._input_views(target): - if view not in node_names: - raise ValueError( - f"Node '{target.get('name')}' references non-existent input '{view}'" - ) - - # Inline source views on MV targets are no longer supported: authors must - # declare a source node and chain it into the MV target via `input`. - for target in targets: - if self._is_mv(target) and "source_view" in target.get("config", {}): + "Source node '%s' defines an inline %s transformation (source_type: '%s'). " + "This is still supported but discouraged and may not be supported in a future " + "release. Define a source node and chain a dedicated transformation node off it " + "instead.", node.get("name"), st, st) + + def _validate(self, sources: List[Dict], targets: List[Dict], nodes: List[Dict]) -> None: + names = {n.get("name") for n in nodes} + for t in targets: + for view in self._inputs(t): + if view[1] not in names: + raise ValueError(f"Node '{t.get('name')}' references non-existent input '{view[1]}'") + if self._is_mv(t) and "source_view" in t.get("config", {}): raise ValueError( - f"Materialized view target '{target.get('name')}' defines an inline " - "'source_view'. This is no longer supported. Declare a source node and " - "chain it into the materialized view target via its 'input' array instead." - ) - + f"Materialized view target '{t.get('name')}' defines an inline 'source_view'. " + "This is no longer supported. Declare a source node and chain it into the " + "materialized view target via its 'input' array instead.") if not targets: raise ValueError("Nodespec spec must contain at least one target node") - # A source node is unnecessary only when every target reads its data - # without one: an MV defined by inline SQL, or a snapshot CDC target. - all_mv_inline_sql = all( - self._is_mv(t) and (t["config"].get("sql_path") or t["config"].get("sql_statement")) - for t in targets - ) - all_snapshot = all(t.get("config", {}).get("cdc_snapshot_settings") for t in targets) - if not sources and not (all_mv_inline_sql or all_snapshot): + mv_inline_sql = all( + self._is_mv(t) and (t["config"].get("sql_path") or t["config"].get("sql_statement")) for t in targets) + snapshot = all(t.get("config", {}).get("cdc_snapshot_settings") for t in targets) + if not sources and not (mv_inline_sql or snapshot): raise ValueError("Nodespec spec must contain at least one source node") - # ------------------------------------------------------------------ # - # input handling - # ------------------------------------------------------------------ # - def _normalize_inputs(self, inputs: List) -> List[Tuple[Optional[str], str]]: - """Normalize a target's ``input`` list into (flow_name, view_name) pairs. - - Each item is either a string (upstream node name; flow name auto-generated) - or ``{"view": ..., "flow": ...}`` where ``flow`` defines the SDP flow name. - Defining the flow name keeps it stable across edits, which matters because - renaming a flow forces a full refresh in SDP. - """ - pairs: List[Tuple[Optional[str], str]] = [] - for item in inputs or []: - if isinstance(item, dict): - if item.get("view"): - pairs.append((item.get("flow"), item["view"])) - else: + # -- inputs -- + + def _inputs(self, node: Dict) -> List[Tuple[Optional[str], str]]: + """``input`` items as (flow_name, view_name); strings auto-name the flow.""" + pairs = [] + for item in node.get("config", {}).get("input", []) or []: + if isinstance(item, dict) and item.get("view"): + pairs.append((item.get("flow"), item["view"])) + elif not isinstance(item, dict): pairs.append((None, item)) return pairs - def _input_views(self, node: Dict) -> List[str]: - """Return the upstream view/node names a node's ``input`` references.""" - return [view for _, view in self._normalize_inputs(node.get("config", {}).get("input", []))] - - # ------------------------------------------------------------------ # - # Streaming-table targets - # ------------------------------------------------------------------ # - def _build_streaming_spec( - self, spec_data: Dict, targets: List[Dict], sources: List[Dict], all_nodes: List[Dict] - ) -> Dict: - """Build a single flow spec for the streaming-table targets.""" - internal_sources = self._identify_internal_sources(sources, targets) - spec_target, other_targets = self._select_spec_target(targets, sources, all_nodes) - return self._build_flow_spec(spec_data, spec_target, other_targets, all_nodes, internal_sources) - - def _select_spec_target( - self, targets: List[Dict], sources: List[Dict], all_nodes: List[Dict] - ) -> Tuple[Dict, List[Dict]]: - """Pick the terminal target as the spec target; the rest are staging.""" - if len(targets) == 1: - return targets[0], [] + # -- shared settings -- + + def _settings(self, dst: Dict, cfg: Dict, *, cdc=False, migration=False, dq_default=False) -> None: + """Copy CDC / data-quality / quarantine / table-migration settings onto `dst`.""" + if cdc: + c = cfg.get("cdc_settings") or cfg.get("cdc_apply_changes") + if c: + dst["cdcSettings"] = c + if cfg.get("cdc_snapshot_settings"): + dst["cdcSnapshotSettings"] = _deep_camel(cfg["cdc_snapshot_settings"]) + if dq_default: + dst["dataQualityExpectationsEnabled"] = cfg.get("data_quality_expectations_enabled", False) + elif cfg.get("data_quality_expectations_enabled"): + dst["dataQualityExpectationsEnabled"] = cfg["data_quality_expectations_enabled"] + if cfg.get("data_quality_expectations_path"): + dst["dataQualityExpectationsPath"] = cfg["data_quality_expectations_path"] + if cfg.get("quarantine_mode"): + dst["quarantineMode"] = cfg["quarantine_mode"] + if cfg.get("quarantine_target_details"): + dst["quarantineTargetDetails"] = cfg["quarantine_target_details"] + if migration and cfg.get("table_migration_details"): + dst["tableMigrationDetails"] = _deep_camel(cfg["table_migration_details"]) + + def _base(self, spec_data: Dict) -> Dict: + base = {k: spec_data.get(c) for k, c in + (("dataFlowId", "dataFlowId"), ("dataFlowGroup", "dataFlowGroup"), ("dataFlowType", "dataFlowType"))} + base["features"] = spec_data.get("features", {}) + if spec_data.get("dataFlowVersion"): + base["dataFlowVersion"] = spec_data["dataFlowVersion"] + return base - consumed = {view for node in all_nodes for view in self._input_views(node)} - source_tables = {s.get("config", {}).get("table") for s in sources} - source_tables.discard(None) + # -- streaming targets -- + def _select_spec_target(self, targets, sources, nodes) -> Tuple[Dict, List[Dict]]: + """Terminal target (not consumed by another node) becomes the spec target.""" + if len(targets) == 1: + return targets[0], [] + consumed = {v for n in nodes for _, v in self._inputs(n)} + source_tables = {s.get("config", {}).get("table") for s in sources} - {None} terminal, other = [], [] - for target in targets: - table = target.get("config", {}).get("table") - is_consumed = target.get("name") in consumed or (table in source_tables) - (other if is_consumed else terminal).append(target) - + for t in targets: + table = t.get("config", {}).get("table") + (other if t.get("name") in consumed or table in source_tables else terminal).append(t) if not terminal: - raise ValueError( - "Could not determine spec target: all target nodes are consumed by other nodes" - ) + raise ValueError("Could not determine spec target: all target nodes are consumed by other nodes") if len(terminal) > 1: - self.logger.warning( - "Multiple terminal targets found %s. Using last target '%s' as spec target.", - [t.get("name") for t in terminal], terminal[-1].get("name"), - ) + self.logger.warning("Multiple terminal targets found %s. Using last target '%s' as spec target.", + [t.get("name") for t in terminal], terminal[-1].get("name")) return terminal[-1], other + terminal[:-1] - def _build_flow_spec( - self, spec_data: Dict, spec_target: Dict, other_targets: List[Dict], - all_nodes: List[Dict], internal_sources: Dict[str, str], - ) -> Dict: - """Assemble the flow spec for the streaming targets (camelCase output).""" - node_lookup = {node.get("name"): node for node in all_nodes} - config = spec_target.get("config", {}) - target_format = spec_target.get("target_type", "delta") - - if target_format == "delta": - target_details = self._build_target_details(config) - else: - target_details = self._build_sink_target_details(config) - - cdc_settings = config.get("cdc_settings") or config.get("cdc_apply_changes") - cdc_snapshot_settings = config.get("cdc_snapshot_settings") - has_cdc = cdc_settings is not None or cdc_snapshot_settings is not None - - flow_spec = { - "dataFlowId": spec_data.get("dataFlowId"), - "dataFlowGroup": spec_data.get("dataFlowGroup"), - "dataFlowType": spec_data.get("dataFlowType"), - "targetFormat": target_format, - "targetDetails": target_details, - "tags": spec_data.get("tags", {}), - "features": spec_data.get("features", {}), - } - if spec_data.get("dataFlowVersion"): - flow_spec["dataFlowVersion"] = spec_data["dataFlowVersion"] - if cdc_settings: - flow_spec["cdcSettings"] = cdc_settings - if cdc_snapshot_settings: - flow_spec["cdcSnapshotSettings"] = _to_backend_keys(cdc_snapshot_settings) - - flow_spec["dataQualityExpectationsEnabled"] = config.get("data_quality_expectations_enabled", False) - if config.get("data_quality_expectations_path"): - flow_spec["dataQualityExpectationsPath"] = config["data_quality_expectations_path"] - if config.get("quarantine_mode"): - flow_spec["quarantineMode"] = config["quarantine_mode"] - if config.get("quarantine_target_details"): - flow_spec["quarantineTargetDetails"] = config["quarantine_target_details"] - if config.get("table_migration_details"): - flow_spec["tableMigrationDetails"] = _to_backend_keys(config["table_migration_details"]) - - flow_spec["flowGroups"] = [ - self._build_flow_group(spec_target, other_targets, node_lookup, has_cdc, internal_sources) - ] - return flow_spec - - def _build_target_details(self, config: Dict) -> Dict: - """Build targetDetails for a delta (streaming table) target.""" - target_details = {"table": config.get("table"), "type": TableType.STREAMING} - target_details.update(_rename_keys(config, { - "database": "database", "schema_path": "schemaPath", - "table_properties": "tableProperties", "path": "path", - "partition_columns": "partitionColumns", - "cluster_by_columns": "clusterByColumns", "cluster_by_auto": "clusterByAuto", - "comment": "comment", "spark_conf": "sparkConf", - "row_filter": "rowFilter", "config_flags": "configFlags", - })) - return target_details - - def _build_sink_target_details(self, config: Dict) -> Dict: - """Build targetDetails for a sink target (delta_sink / foreach_batch / custom).""" - target_details = _rename_keys(config, {"name": "name", "sink_type": "type", "sink_options": "sinkOptions"}) - if "sink_config" in config: - target_details["config"] = _to_backend_keys(config["sink_config"]) - return target_details - - def _build_flow_group( - self, spec_target: Dict, other_targets: List[Dict], node_lookup: Dict[str, Dict], - has_cdc: bool, internal_sources: Dict[str, str], - ) -> Dict: - """Build the single flow group: staging tables plus one flow per input.""" - flow_group = {"flowGroupId": self.FLOW_GROUP_ID, "flows": {}} - - staging_tables = self._build_staging_tables(other_targets) - if staging_tables: - flow_group["stagingTables"] = staging_tables - - # Views already attached to a flow, so we don't redefine them (SDP errors - # on "Cannot redefine"). Shared across all flows in the group. - registered_views: Set[str] = set() - - # Group targets by output table, preserving first-seen order. - table_to_targets: Dict[str, List[Dict]] = defaultdict(list) - for target in other_targets + [spec_target]: - config = target.get("config", {}) - table = config.get("table") or config.get("name") + def _streaming_spec(self, spec_data: Dict, spec_target: Dict, others: List[Dict]) -> Dict: + cfg = spec_target.get("config", {}) + fmt = spec_target.get("target_type", "delta") + has_cdc = bool(cfg.get("cdc_settings") or cfg.get("cdc_apply_changes") or cfg.get("cdc_snapshot_settings")) + + spec = self._base(spec_data) + spec["targetFormat"] = fmt + spec["targetDetails"] = self._target_details(cfg) if fmt == "delta" else self._sink_details(cfg) + spec["tags"] = spec_data.get("tags", {}) + self._settings(spec, cfg, cdc=True, migration=True, dq_default=True) + spec["flowGroups"] = [self._flow_group(spec_target, others, has_cdc)] + return spec + + def _target_details(self, cfg: Dict) -> Dict: + return {"table": cfg.get("table"), "type": TableType.STREAMING, **_camel(cfg, _HANDLED)} + + def _sink_details(self, cfg: Dict) -> Dict: + details: Dict[str, Any] = {} + if "name" in cfg: + details["name"] = cfg["name"] + if "sink_type" in cfg: + details["type"] = cfg["sink_type"] + if "sink_options" in cfg: + details["sinkOptions"] = cfg["sink_options"] + if "sink_config" in cfg: + details["config"] = _deep_camel(cfg["sink_config"]) + return details + + def _flow_group(self, spec_target: Dict, others: List[Dict], has_cdc: bool) -> Dict: + group = {"flowGroupId": self.FLOW_GROUP_ID, "flows": {}} + staging = {} + for t in others: + cfg = t.get("config", {}) + table = cfg.get("table") + if table and table not in staging: + entry = {"type": "ST", **_camel(cfg, _HANDLED)} + self._settings(entry, cfg, cdc=True) + staging[table] = entry + if staging: + group["stagingTables"] = staging + + registered: Set[str] = set() # views already attached (avoid SDP "Cannot redefine") + by_table: Dict[str, List[Dict]] = defaultdict(list) + for t in others + [spec_target]: + cfg = t.get("config", {}) + table = cfg.get("table") or cfg.get("name") if table: - table_to_targets[table].append(target) - - flow_counter = 0 - for table_name, targets in table_to_targets.items(): - for target in targets: - flow_counter = self._add_target_flows( - flow_group, target, table_name, spec_target, node_lookup, - has_cdc, internal_sources, registered_views, flow_counter, - ) - return flow_group - - def _build_staging_tables(self, other_targets: List[Dict]) -> Dict: - """Build the stagingTables map from the non-spec targets.""" - staging: Dict[str, Dict] = {} - for target in other_targets: - config = target.get("config", {}) - table = config.get("table") - if not table or table in staging: - continue + by_table[table].append(t) + + counter = 0 + for table, ts in by_table.items(): + for t in ts: + counter = self._add_flows(group, t, table, spec_target, has_cdc, registered, counter) + return group - entry = {"type": "ST"} - entry.update(_rename_keys(config, { - "database": "database", "schema_path": "schemaPath", - "table_properties": "tableProperties", - "cluster_by_columns": "clusterByColumns", "cluster_by_auto": "clusterByAuto", - "partition_columns": "partitionColumns", "config_flags": "configFlags", - })) - - cdc = config.get("cdc_settings") or config.get("cdc_apply_changes") - if cdc: - entry["cdcSettings"] = cdc - if config.get("cdc_snapshot_settings"): - entry["cdcSnapshotSettings"] = _to_backend_keys(config["cdc_snapshot_settings"]) - if config.get("data_quality_expectations_enabled"): - entry["dataQualityExpectationsEnabled"] = config["data_quality_expectations_enabled"] - if config.get("data_quality_expectations_path"): - entry["dataQualityExpectationsPath"] = config["data_quality_expectations_path"] - if config.get("quarantine_mode"): - entry["quarantineMode"] = config["quarantine_mode"] - if config.get("quarantine_target_details"): - entry["quarantineTargetDetails"] = config["quarantine_target_details"] - - staging[table] = entry - return staging - - def _add_target_flows( - self, flow_group: Dict, target: Dict, table_name: str, spec_target: Dict, - node_lookup: Dict[str, Dict], has_cdc: bool, internal_sources: Dict[str, str], - registered_views: Set[str], flow_counter: int, - ) -> int: - """Add the flow(s) that write into `target`. Returns the next flow counter.""" - config = target.get("config", {}) - target_name = target.get("name") - inputs = self._normalize_inputs(config.get("input", [])) + def _add_flows(self, group, target, table, spec_target, has_cdc, registered, counter) -> int: + cfg = target.get("config", {}) + name = target.get("name") enabled = target.get("enabled", True) + inputs = self._inputs(target) - # No inputs: only valid for snapshot CDC targets (the snapshot system - # reads its source directly), otherwise there is nothing to write. - if not inputs: - if config.get("cdc_snapshot_settings"): - flow_group["flows"][f"{self.FLOW_PREFIX}{target_name}_{flow_counter}"] = { - "flowType": FlowType.MERGE, - "flowDetails": {"targetTable": table_name}, - "enabled": enabled, - } - flow_counter += 1 + if not inputs: # only valid for snapshot CDC targets (snapshot reads its source directly) + if cfg.get("cdc_snapshot_settings"): + group["flows"][f"{self.FLOW_PREFIX}{name}_{counter}"] = { + "flowType": FlowType.MERGE, "flowDetails": {"targetTable": table}, "enabled": enabled} + counter += 1 else: - self.logger.warning(f"Target '{target_name}' has no inputs, skipping") - return flow_counter - - target_cdc = config.get("cdc_settings") or config.get("cdc_apply_changes") - for explicit_flow, view in inputs: - input_node = node_lookup.get(view, {}) - is_sql_source = ( - input_node.get("node_type", "").lower() == self.SOURCE - and input_node.get("source_type") == "sql" - ) - - if target_cdc or (has_cdc and target is spec_target): - flow_type = FlowType.MERGE - elif is_sql_source: - flow_type = FlowType.APPEND_SQL + self.logger.warning(f"Target '{name}' has no inputs, skipping") + return counter + + merge = bool(cfg.get("cdc_settings") or cfg.get("cdc_apply_changes") or (has_cdc and target is spec_target)) + for flow_name, view in inputs: + node = self.lookup.get(view, {}) + sql_source = node.get("node_type", "").lower() == self.SOURCE and node.get("source_type") == "sql" + ftype = FlowType.MERGE if merge else (FlowType.APPEND_SQL if sql_source else FlowType.APPEND_VIEW) + key = flow_name or f"{self.FLOW_PREFIX}{view}_{counter}" + counter += 1 + + if ftype == FlowType.APPEND_SQL: + flow = {"flowType": ftype, "flowDetails": {"targetTable": table}, "enabled": enabled} + flow["flowDetails"].update(_first(node.get("config", {}), "sql_statement", "sql_path")) else: - flow_type = FlowType.APPEND_VIEW - - flow_name = explicit_flow or f"{self.FLOW_PREFIX}{view}_{flow_counter}" - flow_counter += 1 - - if flow_type == FlowType.APPEND_SQL: - flow = { - "flowType": flow_type, - "flowDetails": {"targetTable": table_name}, - "enabled": enabled, - } - input_config = input_node.get("config", {}) - if input_config.get("sql_statement"): - flow["flowDetails"]["sqlStatement"] = input_config["sql_statement"] - elif input_config.get("sql_path"): - flow["flowDetails"]["sqlPath"] = input_config["sql_path"] - else: - source_view, views = self._trace_inputs_to_views([view], node_lookup, internal_sources) + source_view, views = self._views_for(view) if not source_view: - self.logger.warning( - f"Could not determine source view for target '{target_name}' input '{view}'" - ) + self.logger.warning(f"Could not determine source view for target '{name}' input '{view}'") continue - flow = { - "flowType": flow_type, - "flowDetails": {"sourceView": source_view, "targetTable": table_name}, - "enabled": enabled, - } - new_views = {k: v for k, v in views.items() if k not in registered_views} - if new_views: - flow["views"] = new_views - registered_views.update(views.keys()) - - if config.get("once"): + flow = {"flowType": ftype, "flowDetails": {"sourceView": source_view, "targetTable": table}, + "enabled": enabled} + fresh = {k: v for k, v in views.items() if k not in registered} + if fresh: + flow["views"] = fresh + registered.update(views) + + if cfg.get("once"): flow["flowDetails"]["once"] = True - - flow_group["flows"][flow_name] = flow - return flow_counter - - # ------------------------------------------------------------------ # - # Materialized-view targets - # ------------------------------------------------------------------ # - def _build_mv_specs( - self, spec_data: Dict, mv_targets: List[Dict], sources: List[Dict], all_nodes: List[Dict] - ) -> List[Dict]: - """Build one flow spec per materialized-view target.""" - node_lookup = {node.get("name"): node for node in all_nodes} - return [self._build_mv_spec(spec_data, t, sources, node_lookup) for t in mv_targets] - - def _build_mv_spec( - self, spec_data: Dict, mv_target: Dict, sources: List[Dict], node_lookup: Dict[str, Dict] - ) -> Dict: - """Build the flow spec for a single materialized-view target.""" - config = mv_target.get("config", {}) - mv_name = config.get("table") - - target_details = {"table": mv_name, "type": TableType.MATERIALIZED_VIEW} - target_details.update(_rename_keys(config, { - "sql_path": "sqlPath", "sql_statement": "sqlStatement", "refresh_policy": "refreshPolicy", - })) - # table_details wins over the same key on config (back-compat fallback). - detail_map = { - "private": "private", "comment": "comment", "spark_conf": "sparkConf", - "config_flags": "configFlags", "table_properties": "tableProperties", - "cluster_by_columns": "clusterByColumns", "cluster_by_auto": "clusterByAuto", - } - target_details.update({ - **_rename_keys(config, detail_map), - **_rename_keys(config.get("table_details", {}), detail_map), - }) - - flow_spec = { - "dataFlowId": spec_data.get("dataFlowId"), - "dataFlowGroup": spec_data.get("dataFlowGroup"), - "dataFlowType": spec_data.get("dataFlowType"), - "features": spec_data.get("features", {}), - "targetFormat": TargetType.DELTA, - "targetDetails": target_details, - "dataQualityExpectationsEnabled": config.get("data_quality_expectations_enabled", False), - "localPath": spec_data.get("localPath"), - } - if config.get("data_quality_expectations_path"): - flow_spec["dataQualityExpectationsPath"] = config["data_quality_expectations_path"] - if config.get("quarantine_mode"): - flow_spec["quarantineMode"] = config["quarantine_mode"] - if config.get("quarantine_target_details"): - flow_spec["quarantineTargetDetails"] = config["quarantine_target_details"] - - flow_group = {"flowGroupId": self.FLOW_GROUP_ID, "flows": {}} - inputs = self._normalize_inputs(config.get("input", [])) + group["flows"][key] = flow + return counter + + # -- materialized views -- + + def _mv_spec(self, spec_data: Dict, target: Dict, sources: List[Dict]) -> Dict: + cfg = target.get("config", {}) + mv = cfg.get("table") + # config-level details + table_details (the latter overrides on conflict). + details = {"table": mv, "type": TableType.MATERIALIZED_VIEW, + **_camel(cfg, _HANDLED), **_camel(cfg.get("table_details", {}))} + + spec = self._base(spec_data) + spec.pop("dataFlowVersion", None) # MV specs do not carry a version + spec["targetFormat"] = TargetType.DELTA + spec["targetDetails"] = details + spec["localPath"] = spec_data.get("localPath") + self._settings(spec, cfg, dq_default=True) + + group = {"flowGroupId": self.FLOW_GROUP_ID, "flows": {}} + inputs = self._inputs(target) if inputs: - internal_sources = self._identify_internal_sources(sources, [mv_target]) - explicit_flow, view = inputs[0] - source_view, views = self._trace_inputs_to_views([view], node_lookup, internal_sources) + self.internal = self._internal_sources(sources, [target]) + flow_name, view = inputs[0] + source_view, views = self._views_for(view) if source_view: - target_details["sourceView"] = source_view - flow = {"flowType": FlowType.MATERIALIZED_VIEW, "flowDetails": {"targetTable": mv_name}} + details["sourceView"] = source_view + flow = {"flowType": FlowType.MATERIALIZED_VIEW, "flowDetails": {"targetTable": mv}} if views: for v in views.values(): v["mode"] = Mode.BATCH # MVs use batch reads (spark.sql) flow["views"] = views - flow_group["flows"][explicit_flow or f"{self.FLOW_PREFIX}{mv_name}"] = flow - - flow_spec["flowGroups"] = [flow_group] - return flow_spec - - # ------------------------------------------------------------------ # - # Views and internal sources - # ------------------------------------------------------------------ # - def _identify_internal_sources( - self, source_nodes: List[Dict], target_nodes: List[Dict] - ) -> Dict[str, str]: - """Map source-node name -> table for sources that read a table produced by - a target in this same spec. Such sources reference the produced table - directly instead of getting their own view.""" - target_tables = { - t["config"]["table"]: (t["config"].get("database") or None) - for t in target_nodes if t.get("config", {}).get("table") - } - - internal: Dict[str, str] = {} - for source in source_nodes: - if source.get("source_type", "delta") != "delta": + group["flows"][flow_name or f"{self.FLOW_PREFIX}{mv}"] = flow + spec["flowGroups"] = [group] + return spec + + # -- views / internal sources -- + + def _internal_sources(self, sources: List[Dict], targets: List[Dict]) -> Dict[str, str]: + """Delta sources reading a table produced by a target here (referenced directly, no view).""" + produced = {t["config"]["table"]: (t["config"].get("database") or None) + for t in targets if t.get("config", {}).get("table")} + internal = {} + for s in sources: + if s.get("source_type", "delta") != "delta": continue - config = source.get("config", {}) - table = config.get("table", "") - if table not in target_tables: - continue - source_db = config.get("database") or None - target_db = target_tables[table] - if not source_db or not target_db or source_db == target_db: - internal[source.get("name")] = table + cfg = s.get("config", {}) + table = cfg.get("table", "") + if table in produced: + sdb, tdb = cfg.get("database") or None, produced[table] + if not sdb or not tdb or sdb == tdb: + internal[s.get("name")] = table return internal - def _trace_inputs_to_views( - self, inputs: List[str], node_lookup: Dict[str, Dict], internal_sources: Dict[str, str] - ) -> Tuple[Optional[str], Dict]: - """Resolve an input to its source view name and the views it needs. - - When the input is a transformation, every source node in the spec is also - emitted as a view so SDP can resolve the references inside that - transformation's SQL/Python. - """ - views: Dict[str, Dict] = {} - immediate_input = inputs[0] if inputs else None - if not immediate_input: - return None, views - - # An internal source reads the produced table directly: no view needed. - if immediate_input in internal_sources: - return internal_sources[immediate_input], views - - source_view_name = None - references_transformation = False - for node_name in inputs: - if node_name in internal_sources: - continue - node = node_lookup.get(node_name) - if not node: - continue - - node_type = node.get("node_type", "").lower() - view_name = node.get("output_view_name") or self._view_name(node_name) - if node_type == self.SOURCE: - views[view_name] = self._build_source_view(node) - elif node_type == self.TRANSFORMATION: - views[view_name] = self._build_transform_view(node, node_lookup) - references_transformation = True - else: - continue - if source_view_name is None and node_name == immediate_input: - source_view_name = view_name - - # Register every source node so SDP can resolve the transformation's refs. - if references_transformation: - for name, node in node_lookup.items(): - if node.get("node_type", "").lower() != self.SOURCE: + def _view_name(self, name: str) -> str: + return name if name.startswith(self.VIEW_PREFIX) else f"{self.VIEW_PREFIX}{name}" + + def _views_for(self, view: str) -> Tuple[Optional[str], Dict]: + """Resolve an input to (source_view_name, views). A transformation also pulls + in every source node so SDP can resolve its references.""" + if view in self.internal: # internal source reads the produced table directly + return self.internal[view], {} + node = self.lookup.get(view) + if not node: + return self._view_name(view), {} + + name = node.get("output_view_name") or self._view_name(view) + ntype = node.get("node_type", "").lower() + views = {} + if ntype == self.SOURCE: + views[name] = self._source_view(node) + elif ntype == self.TRANSFORMATION: + views[name] = self._transform_view(node) + for n2, node2 in self.lookup.items(): # register all sources for SDP resolution + if node2.get("node_type", "").lower() != self.SOURCE: continue - view_name = node.get("output_view_name") or self._view_name(name) - if view_name in views: + vn = node2.get("output_view_name") or self._view_name(n2) + if vn in views: continue - if name in internal_sources: - # Internal sources need database "live" so SDP resolves them - # within the pipeline context. - node = {**node, "config": {**node.get("config", {}), "database": "live"}} - views[view_name] = self._build_source_view(node) - - if source_view_name is None: - source_view_name = self._view_name(immediate_input) - return source_view_name, views - - def _view_name(self, node_name: str) -> str: - """View name for a node: the node name, prefixed with v_ if needed.""" - return node_name if node_name.startswith(self.VIEW_PREFIX) else f"{self.VIEW_PREFIX}{node_name}" - - def _build_source_view(self, node: Dict) -> Dict: - """Build a view definition from a source node (camelCase output).""" - source_type = node.get("source_type", "delta") - config = node.get("config", {}) - details: Dict[str, Any] = {} + if n2 in self.internal: # internal sources need database "live" + node2 = {**node2, "config": {**node2.get("config", {}), "database": "live"}} + views[vn] = self._source_view(node2) + return name, views + + def _source_view(self, node: Dict) -> Dict: + st = node.get("source_type", "delta") + cfg = node.get("config", {}) + details = _camel(cfg, drop={"mode", "python_transform"}) + if st == "delta": + details.setdefault("cdfEnabled", False) + if cfg.get("python_transform"): + details["pythonTransform"] = self._py_transform(cfg["python_transform"]) + return {"mode": cfg.get("mode", Mode.STREAM), "sourceType": st, "sourceDetails": details} + + def _transform_view(self, node: Dict) -> Dict: + cfg = node.get("config", {}) + if node.get("transformation_type", "sql") != "python": + return {"mode": Mode.BATCH, "sourceType": "sql", + "sourceDetails": _first(cfg, "sql_path", "sql_statement")} + # python transform: its own view reading the inferred upstream via apply_transform(df). + view, mode = self._python_upstream(node) + pt: Dict[str, Any] = {} + if cfg.get("function_path"): + pt["functionPath"] = cfg["function_path"] + elif cfg.get("python_module"): + pt["module"] = cfg["python_module"] + if cfg.get("tokens"): + pt["tokens"] = cfg["tokens"] + return {"mode": mode, "sourceType": "delta", + "sourceDetails": {"database": "live", "table": view, "pythonTransform": pt}} - if source_type == "delta": - if "database" in config: - details["database"] = config["database"] - if "table" in config: - details["table"] = config["table"] - details["cdfEnabled"] = config.get("cdf_enabled", False) - if "table_path" in config: - details["tablePath"] = config["table_path"] - elif source_type in ("cloudFiles", "batchFiles"): - details.update(_rename_keys(config, {"path": "path", "reader_options": "readerOptions"})) - elif source_type == "sql": - details.update(self._first_present(config, [("sql_path", "sqlPath"), ("sql_statement", "sqlStatement")])) - elif source_type == "python": - details.update(self._first_present(config, [("function_path", "functionPath"), ("python_module", "pythonModule")])) - details.update(_rename_keys(config, {"tokens": "tokens"})) - elif source_type == "kafka": - details.update(_rename_keys(config, {"reader_options": "readerOptions"})) - - details.update(_rename_keys(config, { - "select_exp": "selectExp", "where_clause": "whereClause", "schema_path": "schemaPath", - "starting_version_from_dlt_setup": "startingVersionFromDLTSetup", - "cdf_change_type_override": "cdfChangeTypeOverride", - })) - - # Backward compatibility: a source may carry an inline python transform - # that post-processes the read DataFrame via apply_transform(df). - python_transform = config.get("python_transform") - if python_transform: - mapped: Dict[str, Any] = {} - if python_transform.get("function_path"): - mapped["functionPath"] = python_transform["function_path"] - elif python_transform.get("functionPath"): - mapped["functionPath"] = python_transform["functionPath"] - if python_transform.get("python_module"): - mapped["module"] = python_transform["python_module"] - elif python_transform.get("module"): - mapped["module"] = python_transform["module"] - if python_transform.get("tokens"): - mapped["tokens"] = python_transform["tokens"] - details["pythonTransform"] = mapped - - return {"mode": config.get("mode", Mode.STREAM), "sourceType": source_type, "sourceDetails": details} - - def _build_transform_view(self, node: Dict, node_lookup: Dict[str, Dict]) -> Dict: - """Build a view definition from a transformation node (camelCase output).""" - transform_type = node.get("transformation_type", "sql") - config = node.get("config", {}) - - if transform_type == "python": - # A python transformation is its own view: it reads its upstream view - # and applies apply_transform(df) via the framework's pythonTransform. - # The upstream is inferred from the graph; SDP resolves the live ref. - upstream_view, upstream_mode = self._infer_python_transform_upstream(node, node_lookup) - python_transform: Dict[str, Any] = {} - if config.get("function_path"): - python_transform["functionPath"] = config["function_path"] - elif config.get("python_module"): - python_transform["module"] = config["python_module"] - if config.get("tokens"): - python_transform["tokens"] = config["tokens"] - return { - "mode": upstream_mode, - "sourceType": "delta", - "sourceDetails": { - "database": "live", - "table": upstream_view, - "pythonTransform": python_transform, - }, - } - - # sql - details = self._first_present(config, [("sql_path", "sqlPath"), ("sql_statement", "sqlStatement")]) - return {"mode": Mode.BATCH, "sourceType": "sql", "sourceDetails": details} - - def _infer_python_transform_upstream(self, node: Dict, node_lookup: Dict[str, Dict]) -> Tuple[str, str]: - """Infer the upstream view a python transformation reads, plus its mode. - - Python transforms operate on a DataFrame, so (unlike SQL transforms) they - carry no textual reference to their input. SDP resolves view dependencies - automatically, so the upstream is inferred from the graph: the spec's - source feeds the transform. With several sources the first is used and a - warning is logged. - """ - sources = [n for n in node_lookup.values() - if n.get("node_type", "").lower() == self.SOURCE] + @staticmethod + def _py_transform(pt: Dict) -> Dict: + out: Dict[str, Any] = {} + if pt.get("function_path") or pt.get("functionPath"): + out["functionPath"] = pt.get("function_path") or pt.get("functionPath") + if pt.get("python_module") or pt.get("module"): + out["module"] = pt.get("python_module") or pt.get("module") + if pt.get("tokens"): + out["tokens"] = pt["tokens"] + return out + + def _python_upstream(self, node: Dict) -> Tuple[str, str]: + """Infer the upstream view a python transform reads (it has no textual ref).""" + sources = [n for n in self.lookup.values() if n.get("node_type", "").lower() == self.SOURCE] if not sources: - self.logger.warning( - "Python transformation '%s' has no upstream source node to read from.", - node.get("name"), - ) + self.logger.warning("Python transformation '%s' has no upstream source node to read from.", + node.get("name")) return self._view_name(node.get("name")), Mode.STREAM if len(sources) > 1: - self.logger.warning( - "Python transformation '%s' has multiple upstream sources %s; reading from '%s'.", - node.get("name"), [s.get("name") for s in sources], sources[0].get("name"), - ) - upstream = sources[0] - view = upstream.get("output_view_name") or self._view_name(upstream.get("name")) - mode = upstream.get("config", {}).get("mode", Mode.STREAM) - return view, mode - - @staticmethod - def _first_present(config: Dict, candidates: List[Tuple[str, str]]) -> Dict: - """Return {dst: config[src]} for the first (src, dst) whose src is set.""" - for src, dst in candidates: - if src in config: - return {dst: config[src]} - return {} + self.logger.warning("Python transformation '%s' has multiple upstream sources %s; reading from '%s'.", + node.get("name"), [s.get("name") for s in sources], sources[0].get("name")) + up = sources[0] + return up.get("output_view_name") or self._view_name(up.get("name")), up.get("config", {}).get("mode", Mode.STREAM) From 4ceec4bce1a8567d852a4c7d6799575273ff7fdf Mon Sep 17 00:00:00 2001 From: Haille W Date: Thu, 2 Jul 2026 14:44:03 +1000 Subject: [PATCH 07/19] remove support for outdated cdc name --- .../version_mapping_flows_dataflowspec_main.json | 4 ++-- .../version_mapping_standard_dataflowspec_main.json | 2 +- scripts/migrate_to_nodespec.py | 8 ++++---- src/dataflow_spec_builder/transformer/nodespec.py | 11 +++++------ src/schemas/spec_nodespec.json | 3 --- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json index 04d8903..751a58b 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json @@ -12,7 +12,7 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "cdc_apply_changes": { + "cdc_settings": { "keys": [ "CUSTOMER_ID" ], @@ -59,7 +59,7 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "cdc_apply_changes": { + "cdc_settings": { "keys": [ "CUSTOMER_ID" ], diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json index 7803173..0155905 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json @@ -23,7 +23,7 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "cdc_apply_changes": { + "cdc_settings": { "keys": [ "CUSTOMER_ID" ], diff --git a/scripts/migrate_to_nodespec.py b/scripts/migrate_to_nodespec.py index b8da03f..0762cb8 100644 --- a/scripts/migrate_to_nodespec.py +++ b/scripts/migrate_to_nodespec.py @@ -133,12 +133,12 @@ def _add_target_settings(config: Dict, src: Dict) -> None: `src` is the spec (standard), a staging-table config (flow), or an MV config. Reads either casing; writes snake_case. """ - cdc = _get(src, "cdcSettings", "cdc_settings") + # nodespec has a single CDC field (cdc_settings). Legacy cdcApplyChanges is + # the same thing, so migrate either source key into cdc_settings. + cdc = (_get(src, "cdcSettings", "cdc_settings") + or _get(src, "cdcApplyChanges", "cdc_apply_changes")) if cdc: config["cdc_settings"] = cdc - cdc_apply = _get(src, "cdcApplyChanges", "cdc_apply_changes") - if cdc_apply: - config["cdc_apply_changes"] = cdc_apply snapshot = _get(src, "cdcSnapshotSettings", "cdc_snapshot_settings") if snapshot: config["cdc_snapshot_settings"] = _convert_snapshot(snapshot) diff --git a/src/dataflow_spec_builder/transformer/nodespec.py b/src/dataflow_spec_builder/transformer/nodespec.py index fe47d3e..21d4ac0 100644 --- a/src/dataflow_spec_builder/transformer/nodespec.py +++ b/src/dataflow_spec_builder/transformer/nodespec.py @@ -46,7 +46,7 @@ # Keys a target config handles specially, so they are NOT copied into details. _HANDLED = { "input", "table", "table_type", "enabled", "once", "name", - "cdc_settings", "cdc_apply_changes", "cdc_snapshot_settings", + "cdc_settings", "cdc_snapshot_settings", "data_quality_expectations_enabled", "data_quality_expectations_path", "quarantine_mode", "quarantine_target_details", "table_migration_details", "sink_type", "sink_config", "sink_options", "table_details", "source_view", @@ -164,9 +164,8 @@ def _inputs(self, node: Dict) -> List[Tuple[Optional[str], str]]: def _settings(self, dst: Dict, cfg: Dict, *, cdc=False, migration=False, dq_default=False) -> None: """Copy CDC / data-quality / quarantine / table-migration settings onto `dst`.""" if cdc: - c = cfg.get("cdc_settings") or cfg.get("cdc_apply_changes") - if c: - dst["cdcSettings"] = c + if cfg.get("cdc_settings"): + dst["cdcSettings"] = cfg["cdc_settings"] if cfg.get("cdc_snapshot_settings"): dst["cdcSnapshotSettings"] = _deep_camel(cfg["cdc_snapshot_settings"]) if dq_default: @@ -212,7 +211,7 @@ def _select_spec_target(self, targets, sources, nodes) -> Tuple[Dict, List[Dict] def _streaming_spec(self, spec_data: Dict, spec_target: Dict, others: List[Dict]) -> Dict: cfg = spec_target.get("config", {}) fmt = spec_target.get("target_type", "delta") - has_cdc = bool(cfg.get("cdc_settings") or cfg.get("cdc_apply_changes") or cfg.get("cdc_snapshot_settings")) + has_cdc = bool(cfg.get("cdc_settings") or cfg.get("cdc_snapshot_settings")) spec = self._base(spec_data) spec["targetFormat"] = fmt @@ -279,7 +278,7 @@ def _add_flows(self, group, target, table, spec_target, has_cdc, registered, cou self.logger.warning(f"Target '{name}' has no inputs, skipping") return counter - merge = bool(cfg.get("cdc_settings") or cfg.get("cdc_apply_changes") or (has_cdc and target is spec_target)) + merge = bool(cfg.get("cdc_settings") or (has_cdc and target is spec_target)) for flow_name, view in inputs: node = self.lookup.get(view, {}) sql_source = node.get("node_type", "").lower() == self.SOURCE and node.get("source_type") == "sql" diff --git a/src/schemas/spec_nodespec.json b/src/schemas/spec_nodespec.json index 4b6b1d1..904d25a 100644 --- a/src/schemas/spec_nodespec.json +++ b/src/schemas/spec_nodespec.json @@ -366,9 +366,6 @@ "cdc_settings": { "$ref": "./definitions_main.json#/definitions/cdcSettings" }, - "cdc_apply_changes": { - "$ref": "./definitions_main.json#/definitions/cdcSettings" - }, "cdc_snapshot_settings": { "$ref": "#/$defs/snapshotCdcConfig" }, From 383bc276fe0d54323a0388ca182b110e208ce813 Mon Sep 17 00:00:00 2001 From: Haille W Date: Wed, 8 Jul 2026 12:33:03 +1000 Subject: [PATCH 08/19] feat(dataflow-spec): default data_flow_type to nodespec when omitted nodespec is the framework's unified spec format, so specs may now omit data_flow_type entirely. - DataflowSpecBuilder defaults data_flow_type to "nodespec" at read time, so type selection, metadata validation, schema validation, and transformation all treat a type-less spec as nodespec. - main.json routes any spec without an explicit legacy dataFlowType (standard/flow/materialized_view) to the nodespec schema. - spec_nodespec.json no longer requires data_flow_type; the const remains so that, when present, it must be "nodespec". Co-authored-by: Isaac --- src/dataflow_spec_builder/dataflow_spec_builder.py | 10 ++++++++++ src/schemas/main.json | 11 ++++++----- src/schemas/spec_nodespec.json | 4 ++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/dataflow_spec_builder/dataflow_spec_builder.py b/src/dataflow_spec_builder/dataflow_spec_builder.py index a561ef9..c3e5430 100644 --- a/src/dataflow_spec_builder/dataflow_spec_builder.py +++ b/src/dataflow_spec_builder/dataflow_spec_builder.py @@ -46,6 +46,10 @@ class DataflowSpecBuilder: build(): Build dataflow specifications. """ + # The default spec type used when a spec omits data_flow_type. nodespec is + # the framework's unified spec format. + DEFAULT_DATA_FLOW_TYPE = "nodespec" + class Keys: """Constants for dictionary keys for the dataflow spec JSON files, and final dataflow spec format""" # Core dataflow specification keys @@ -222,6 +226,12 @@ def _normalise_spec_metadata(spec: Dict) -> None: def _extract_spec(spec: Dict) -> Dict: """Extract metadata from a dataflow specification.""" _normalise_spec_metadata(spec) + # nodespec is the default/unified spec type, so data_flow_type may be + # omitted. Default it here (in-place) so type selection, metadata + # validation, schema validation, and transformation all treat a + # type-less spec as nodespec. + if not spec.get(self.Keys.DATA_FLOW_TYPE): + spec[self.Keys.DATA_FLOW_TYPE] = self.DEFAULT_DATA_FLOW_TYPE return { "fileType": "main", self.Keys.DATA_FLOW_ID: spec.get(self.Keys.DATA_FLOW_ID, None), diff --git a/src/schemas/main.json b/src/schemas/main.json index 9bb0e84..c2bd41e 100644 --- a/src/schemas/main.json +++ b/src/schemas/main.json @@ -1,11 +1,11 @@ { "title": "Main DataFlowSpec", + "$comment": "nodespec is the default spec type: a spec is validated as a legacy format only when it carries an explicit legacy dataFlowType (standard/flow/materialized_view). Anything else — including specs that omit the type entirely — is treated as nodespec.", "if": { - "properties": { "data_flow_type": { "const": "nodespec" } }, - "required": ["data_flow_type"] + "properties": { "dataFlowType": { "enum": ["flow", "standard", "materialized_view"] } }, + "required": ["dataFlowType"] }, - "then": { "$ref": "./spec_nodespec.json#/$defs/nodespecSpec" }, - "else": { + "then": { "type": "object", "properties": { "dataFlowId": {"type": "string"}, @@ -43,5 +43,6 @@ } }, "required": ["dataFlowId", "dataFlowGroup", "dataFlowType"] - } + }, + "else": { "$ref": "./spec_nodespec.json#/$defs/nodespecSpec" } } diff --git a/src/schemas/spec_nodespec.json b/src/schemas/spec_nodespec.json index 904d25a..cf42d61 100644 --- a/src/schemas/spec_nodespec.json +++ b/src/schemas/spec_nodespec.json @@ -15,7 +15,8 @@ "type": "string" }, "data_flow_type": { - "const": "nodespec" + "const": "nodespec", + "description": "Optional. nodespec is the default spec type, so this may be omitted; when present it must be \"nodespec\"." }, "data_flow_version": { "type": "string" @@ -40,7 +41,6 @@ "required": [ "data_flow_id", "data_flow_group", - "data_flow_type", "nodes" ], "additionalProperties": false From 24545e4c35b3a18e4feb8a0997916f1605c22bca Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 10 Jul 2026 11:10:19 +1000 Subject: [PATCH 09/19] refactor(nodespec): finalize spec syntax, conversion, samples & docs; renumber ADR to 0008 Field syntax (discriminated config for IntelliSense + consistent nesting): - Targets wire inputs via `input_flows`. - Data quality is a nested `data_quality` object; quarantine is a nested `quarantine` object (mode + optional target). Sink targets stay flat. - Sources allow an optional inline `python_transform` (transformer already supports it). Transformer reads the new field names. migrate_to_nodespec.py emits the new shape, fixes historical-snapshot conversion (no spurious source node) and table-migration source_details casing, and writes configs in a canonical field order (idempotent normalise for existing nodespec specs). Samples reordered to the canonical field order. Reference doc, feature-changes doc updated. Renumber ADR 0007 -> 0008 (0007 is taken by scripted-versioned-docs on main), with an amendment mapping the original syntax to the shipped names. Co-authored-by: Isaac --- ...=> 0008-unified-nodespec-dataflow-spec.md} | 29 +- docs/nodespec_feature_samples_changes.md | 9 +- .../dataflow_spec_ref_main_nodespec.rst | 82 +- .../customer_cloudfiles_main.json | 4 +- .../customer_multi_target_main.json | 32 +- .../dataflowspec/customer_simple_main.json | 6 +- .../customer_st_with_mv_main.json | 6 +- .../customer_with_transform_main.json | 24 +- .../dataflowspec/append_sql_flow_main.json | 11 +- .../dataflowspec/append_view_flow_main.json | 2 +- .../append_view_once_flow_main.json | 2 +- .../dataflowspec/ddl_schema_main.json | 2 +- .../historical_snapshot_files_flow_main.json | 14 +- .../dataflowspec/materialized_views_main.json | 21 +- .../periodic_snapshot_scd1_main.json | 2 +- .../periodic_snapshot_scd2_main.json | 2 +- .../python_source_extension_main.json | 2 +- .../dataflowspec/python_source_main.json | 2 +- .../python_transform_extension_main.json | 2 +- .../dataflowspec/python_transform_main.json | 2 +- .../dataflowspec/quarantine_flag_main.json | 11 +- .../dataflowspec/quarantine_table_main.json | 15 +- .../quarantine_table_with_cdc_main.json | 15 +- .../sink_custom_python_generate_cdc_feed.json | 2 +- .../sink_delta_path_table_main.json | 6 +- .../sink_delta_uc_table_main.json | 6 +- ...k_foreach_batch_single_basic_sql_main.json | 6 +- ...ach_batch_single_python_function_main.json | 6 +- .../table_migration_append_only_main.json | 2 +- .../table_migration_scd2_main.json | 2 +- ...rsion_mapping_flows_dataflowspec_main.json | 4 +- ...on_mapping_standard_dataflowspec_main.json | 2 +- scripts/migrate_to_nodespec.py | 153 ++- .../transformer/nodespec.py | 36 +- src/schemas/spec_nodespec.json | 951 ++++++++---------- 35 files changed, 787 insertions(+), 686 deletions(-) rename docs/decisions/{0007-unified-nodespec-dataflow-spec.md => 0008-unified-nodespec-dataflow-spec.md} (92%) diff --git a/docs/decisions/0007-unified-nodespec-dataflow-spec.md b/docs/decisions/0008-unified-nodespec-dataflow-spec.md similarity index 92% rename from docs/decisions/0007-unified-nodespec-dataflow-spec.md rename to docs/decisions/0008-unified-nodespec-dataflow-spec.md index 2232c13..835155f 100644 --- a/docs/decisions/0007-unified-nodespec-dataflow-spec.md +++ b/docs/decisions/0008-unified-nodespec-dataflow-spec.md @@ -1,7 +1,30 @@ -# ADR-0007: A unified, node-based dataflow spec (`nodespec`) +# ADR-0008: A unified, node-based dataflow spec (`nodespec`) **Date:** 2026-06-19 -**Status:** Accepted +**Status:** Accepted (amended) + +--- + +## Amendment — post-acceptance refinements + +The decision below stands. Since acceptance, the authoring syntax has been +refined; the examples in this ADR use the **original** field names. The current +names (authoritative reference: ``docs/source/dataflow_spec_ref_main_nodespec.rst``) +differ as follows: + +- **`input` → `input_flows`** — the target-node wiring list was renamed for clarity. +- **Data quality → nested `data_quality`** object (`{ "expectations_path": ... }`), + replacing the `data_quality_expectations_enabled` / `_path` pair; presence implies + enabled. +- **Quarantine → nested `quarantine`** object (`{ "mode", "target" }`), replacing + `quarantine_mode` / `quarantine_target_details`. +- **`data_flow_type` is optional** — nodespec is the framework default, so a spec + that omits it is treated as nodespec. +- **`scd_type` is a string** in `cdc_settings` (e.g. `"2"`). +- **Sink targets keep flat fields** (`sink_type` / `sink_config` / `sink_options`); + a nested `sink` object was considered but deferred. +- **A node's `config` is schema-discriminated** by `node_type` + + `source_type`/`target_type`, so editors offer only the fields valid for that node. --- @@ -253,7 +276,7 @@ belongs to, and the connections are stated rather than inferred: "cdc_settings": { "keys": ["CUSTOMER_ID"], "sequence_by": "LOAD_TIMESTAMP", - "scd_type": 2, + "scd_type": "2", "except_column_list": ["LOAD_TIMESTAMP"], "ignore_null_updates": true }, diff --git a/docs/nodespec_feature_samples_changes.md b/docs/nodespec_feature_samples_changes.md index 9905f09..edc898f 100644 --- a/docs/nodespec_feature_samples_changes.md +++ b/docs/nodespec_feature_samples_changes.md @@ -1,8 +1,15 @@ # Nodespec Feature Samples — Changes, Design & Considerations +> **Note:** this is a historical design/implementation record. Some field names +> shown below reflect an earlier iteration; the current authoring syntax is in +> `docs/source/dataflow_spec_ref_main_nodespec.rst`. In particular: targets wire +> inputs via **`input_flows`**; data quality is a nested **`data_quality`** object; +> quarantine is a nested **`quarantine`** object; and `data_flow_type` is optional +> (nodespec is the default). + ## Overview -The Nodespec dataflow spec type introduces a **node-based graph architecture** for defining data pipelines. Instead of the traditional flat (standard) or pre-built flow (flow) spec formats, Nodespec expresses pipelines as directed graphs of **source**, **transformation**, and **target** nodes connected through `inputs` arrays. +The Nodespec dataflow spec type introduces a **node-based graph architecture** for defining data pipelines. Instead of the traditional flat (standard) or pre-built flow (flow) spec formats, Nodespec expresses pipelines as directed graphs of **source**, **transformation**, and **target** nodes connected through `input_flows` arrays (on target nodes). This document covers the changes made to support all 33 feature samples from the bronze_sample in the Nodespec format, the migration tooling created, and the design decisions and assumptions behind the implementation. diff --git a/docs/source/dataflow_spec_ref_main_nodespec.rst b/docs/source/dataflow_spec_ref_main_nodespec.rst index 4a4b859..a159ebd 100644 --- a/docs/source/dataflow_spec_ref_main_nodespec.rst +++ b/docs/source/dataflow_spec_ref_main_nodespec.rst @@ -26,14 +26,20 @@ There are three node types: (CDC, data quality, quarantine, clustering, and so on). **How nodes connect.** A *target* node declares what feeds it through an explicit -``input`` list. *Source* and *transformation* nodes are wired by explicit +``input_flows`` list. *Source* and *transformation* nodes are wired by explicit reference inside their own definition: a SQL transformation names the view it reads in its SQL (for example ``FROM STREAM(live.v_source_customer)``), and a -source names the table, path, or stream it reads. The ``input`` list is therefore -a target-node construct. +source names the table, path, or stream it reads. The ``input_flows`` list is +therefore a target-node construct. **Casing.** Nodespec specs use ``snake_case`` field names. +**Field order.** Within a node's ``config``, fields are written in a consistent +order — identity (``table``/``database``), then table/structural details, then +feature blocks (``cdc_settings``, ``data_quality``, ``quarantine``, +``table_migration_details``), and finally ``input_flows``. The order is +conventional only; it does not affect behaviour. + Example: simple data flow ========================= @@ -63,7 +69,7 @@ The simplest spec connects a source to a target: "config": { "table": "customer_silver", "cluster_by_auto": true, - "input": [ + "input_flows": [ { "view": "v_source_customer", "flow": "f_customer_ingest" } ] } @@ -71,11 +77,15 @@ The simplest spec connects a source to a target: ] } +``data_flow_type`` is optional — nodespec is the framework's default spec type, so +a spec that omits it is treated as nodespec. Include ``"data_flow_type": +"nodespec"`` only if you want to be explicit. + Example: multi-step transformation and CDC ========================================== Chaining a transformation into a CDC target. The SQL transformation names the -view it reads; the target lists the transformation in its ``input``: +view it reads; the target lists the transformation in its ``input_flows``: .. code-block:: json @@ -107,10 +117,10 @@ view it reads; the target lists the transformation in its ``input``: "cdc_settings": { "keys": ["CUSTOMER_ID"], "sequence_by": "LOAD_TIMESTAMP", - "scd_type": 2, + "scd_type": "2", "ignore_null_updates": true }, - "input": ["v_enrich"] + "input_flows": ["v_enrich"] } } ] @@ -145,9 +155,11 @@ Dataflow metadata configuration * - **data_flow_group** - ``string`` - The group to which the data flow belongs. - * - **data_flow_type** + * - **data_flow_type** (*optional*) - ``string`` - - Must be ``"nodespec"``. + - ``"nodespec"``. Optional — nodespec is the default spec type, so a spec + that omits ``data_flow_type`` is treated as nodespec. When present it must + be ``"nodespec"``. * - **data_flow_version** (*optional*) - ``string`` - Version of the dataflow spec for migration purposes. @@ -295,11 +307,6 @@ Target node configuration * - **Field** - **Type** - **Description** - * - **input** - - ``array`` - - What feeds this target. Each item is either an upstream node name - (``string``, flow name auto-generated) or an object - ``{ "view": , "flow": }`` that sets the flow name. * - **table** - ``string`` - Target table name. @@ -327,26 +334,45 @@ Target node configuration * - **cdc_settings** / **cdc_snapshot_settings** (*optional*) - ``object`` - CDC / snapshot-CDC settings for this target. - * - **data_quality_expectations_enabled** / **data_quality_expectations_path** (*optional*) - - ``boolean`` / ``string`` - - Enable and locate data quality expectations. - * - **quarantine_mode** / **quarantine_target_details** (*optional*) - - ``string`` / ``object`` - - ``off`` (default), ``flag``, or ``table``; quarantine table config when - ``table``. + * - **data_quality** (*optional*) + - ``object`` + - Data quality expectations: ``{ "expectations_path": }``. Presence + enables expectations (there is no separate enabled flag). + * - **quarantine** (*optional*) + - ``object`` + - Quarantine for rows failing expectations: ``{ "mode": "flag" | "table", + "target": { ... } }``. Omit for no quarantine; ``target`` is the quarantine + table config, required when ``mode`` is ``table``. * - **table_migration_details** (*optional*) - ``object`` - Table migration configuration. + * - **input_flows** + - ``array`` + - What feeds this target. Each item is either an upstream node name + (``string``, flow name auto-generated) or an object + ``{ "view": , "flow": }`` that sets the flow name. + Conventionally written last in the config. + +Sink targets +----------- + +When ``target_type`` is a sink (``delta_sink``, ``kafka_sink``, +``foreach_batch_sink``, ``custom_python_sink``) the target config uses the sink +fields instead of the delta-table fields: ``name``, ``sink_type`` (sink sub-type, +e.g. ``basic_sql`` / ``python_function`` for ``foreach_batch_sink``), +``sink_config`` (sink-specific configuration), and ``sink_options`` (e.g. +``table_name``/``path`` for a delta sink, or Kafka connection options), plus +``input_flows``. Defining flow names ------------------- By default the framework derives a flow name from the graph, so simple specs stay -simple. Use the object form of ``input`` to set a flow name explicitly: +simple. Use the object form of ``input_flows`` to set a flow name explicitly: .. code-block:: json - "input": [ + "input_flows": [ "v_source_a", { "view": "v_source_b", "flow": "f_append_b" } ] @@ -365,7 +391,7 @@ modes, set via ``snapshot_type``: ``source_type`` (``file`` or ``table``) and a ``source`` object. No source node is required; the framework reads the files/table directly. - **periodic** — the target reads from an upstream **source node** chained via - ``input``; ``source_type`` / ``source`` are not used. + ``input_flows``; ``source_type`` / ``source`` are not used. For historical snapshots the ``source`` object fields depend on ``source_type``: @@ -452,8 +478,8 @@ by inline SQL or by chaining a source node into it: } } -To feed a materialized view from a source node, chain it via ``input`` (an inline -``source_view`` on the target is **not** supported): +To feed a materialized view from a source node, chain it via ``input_flows`` (an +inline ``source_view`` on the target is **not** supported): .. code-block:: json @@ -469,7 +495,7 @@ To feed a materialized view from a source node, chain it via ``input`` (an inlin "config": { "table": "customer_mv", "table_type": "mv", - "input": ["v_mv_source"] + "input_flows": ["v_mv_source"] } } @@ -488,7 +514,7 @@ How Nodespec specs are converted 3. Source nodes become views; an internal source (one that reads a table produced by a target in the same spec) references that table directly. 4. Transformation nodes become SQL or Python views. -5. Each ``input`` entry becomes a flow into its target. The flow type is +5. Each ``input_flows`` entry becomes a flow into its target. The flow type is ``merge`` when the target has CDC, otherwise ``append_view`` (or ``append_sql`` for an inline SQL source). 6. Each ``table_type: "mv"`` target becomes its own materialized-view flow spec. diff --git a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json index 761fb54..730266d 100644 --- a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json @@ -8,8 +8,8 @@ "node_type": "source", "source_type": "cloudFiles", "config": { - "path": "{sample_file_location}/customer/", "mode": "stream", + "path": "{sample_file_location}/customer/", "schema_path": "customer_source_schema.json", "reader_options": { "cloudFiles.format": "json", @@ -36,7 +36,7 @@ "delta.enableChangeDataFeed": "true" }, "cluster_by_auto": true, - "input": [ + "input_flows": [ { "view": "v_transform_normalize", "flow": "f_transform_normalize" diff --git a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json index 4280f10..90d55ff 100644 --- a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_multi_target_main.json @@ -8,10 +8,10 @@ "node_type": "source", "source_type": "delta", "config": { + "mode": "stream", "database": "{staging_schema}", "table": "customer", - "cdf_enabled": true, - "mode": "stream" + "cdf_enabled": true } }, { @@ -32,10 +32,13 @@ "delta.enableChangeDataFeed": "true" }, "cluster_by_auto": true, - "data_quality_expectations_enabled": true, - "data_quality_expectations_path": "customer_staging_dqe.json", - "quarantine_mode": "flag", - "input": [ + "data_quality": { + "expectations_path": "customer_staging_dqe.json" + }, + "quarantine": { + "mode": "flag" + }, + "input_flows": [ { "flow": "f_append_staging", "view": "v_transform_append" @@ -72,14 +75,17 @@ "sequence_by": "LOAD_TIMESTAMP", "ignore_null_updates": true }, - "data_quality_expectations_enabled": true, - "data_quality_expectations_path": "customer_final_dqe.json", - "quarantine_mode": "table", - "quarantine_target_details": { - "target_format": "delta", - "table": "customer_final_node_quarantine" + "data_quality": { + "expectations_path": "customer_final_dqe.json" + }, + "quarantine": { + "mode": "table", + "target": { + "target_format": "delta", + "table": "customer_final_node_quarantine" + } }, - "input": [ + "input_flows": [ { "flow": "f_merge_final", "view": "v_transform_merge" diff --git a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json index 559e1c2..1796646 100644 --- a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_simple_main.json @@ -8,10 +8,10 @@ "node_type": "source", "source_type": "delta", "config": { + "mode": "stream", "database": "{staging_schema}", "table": "customer", - "cdf_enabled": true, - "mode": "stream" + "cdf_enabled": true } }, { @@ -23,7 +23,7 @@ "delta.enableChangeDataFeed": "true" }, "cluster_by_auto": true, - "input": [ + "input_flows": [ { "flow": "f_customer_silver_ingest", "view": "v_source_customer" diff --git a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json index 62d64bb..005eb12 100644 --- a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_st_with_mv_main.json @@ -8,10 +8,10 @@ "node_type": "source", "source_type": "delta", "config": { + "mode": "stream", "database": "{staging_schema}", "table": "customer", - "cdf_enabled": true, - "mode": "stream" + "cdf_enabled": true } }, { @@ -23,7 +23,7 @@ "delta.enableChangeDataFeed": "true" }, "cluster_by_auto": true, - "input": [ + "input_flows": [ { "view": "v_source_customer_st_mv", "flow": "f_source_customer_st_mv" diff --git a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json index 9fcfadd..e85e0ef 100644 --- a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_with_transform_main.json @@ -12,10 +12,10 @@ "node_type": "source", "source_type": "delta", "config": { + "mode": "stream", "database": "{staging_schema}", "table": "customer", - "cdf_enabled": true, - "mode": "stream" + "cdf_enabled": true } }, { @@ -23,10 +23,10 @@ "node_type": "source", "source_type": "delta", "config": { + "mode": "stream", "database": "{staging_schema}", "table": "customer_address", - "cdf_enabled": true, - "mode": "stream" + "cdf_enabled": true } }, { @@ -35,7 +35,7 @@ "config": { "table": "customer_transform_append_node", "cluster_by_auto": true, - "input": [ + "input_flows": [ { "flow": "f_append_customer", "view": "v_source_customer" @@ -49,7 +49,7 @@ "config": { "table": "customer_transform_append_node", "cluster_by_auto": true, - "input": [ + "input_flows": [ { "flow": "f_append_address", "view": "v_source_customer_address" @@ -62,9 +62,9 @@ "node_type": "source", "source_type": "delta", "config": { + "mode": "stream", "table": "customer_transform_append_node", - "cdf_enabled": true, - "mode": "stream" + "cdf_enabled": true } }, { @@ -84,7 +84,7 @@ "ignore_null_updates": true, "scd_type": "2" }, - "input": [ + "input_flows": [ { "flow": "f_merge_append_cdf", "view": "v_source_append_cdf" @@ -97,9 +97,9 @@ "node_type": "source", "source_type": "delta", "config": { + "mode": "stream", "table": "customer_transform_merge_node", - "cdf_enabled": true, - "mode": "stream" + "cdf_enabled": true } }, { @@ -131,7 +131,7 @@ "sequence_by": "LOAD_TIMESTAMP", "ignore_null_updates": true }, - "input": [ + "input_flows": [ { "flow": "f_enrich_final", "view": "v_transform_final" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json index 2f28339..172c48f 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_sql_flow_main.json @@ -20,10 +20,13 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "data_quality_expectations_enabled": true, - "data_quality_expectations_path": "./customer_address_dqe.json", - "quarantine_mode": "flag", - "input": [ + "data_quality": { + "expectations_path": "./customer_address_dqe.json" + }, + "quarantine": { + "mode": "flag" + }, + "input_flows": [ { "view": "v_sql_f_customer_address_append_sql", "flow": "f_sql_f_customer_address_append_sql" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json index 2154c6d..f1caff0 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_flow_main.json @@ -22,7 +22,7 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "input": [ + "input_flows": [ { "view": "v_append_view_flow", "flow": "f_append_view_flow" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json index d5407f6..90fa667 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/append_view_once_flow_main.json @@ -22,7 +22,7 @@ "delta.enableChangeDataFeed": "true" }, "once": true, - "input": [ + "input_flows": [ { "view": "v_append_view_once_flow", "flow": "f_append_view_once_flow" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json index 9665fae..467f351 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/ddl_schema_main.json @@ -23,7 +23,7 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "input": [ + "input_flows": [ { "view": "v_feature_constraints", "flow": "f_feature_constraints" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json index 847b65a..dfa7b88 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json @@ -50,18 +50,18 @@ "database": "live", "table": "stg_feature_historical_files_snapshot_flow", "cdf_enabled": true, + "cdf_change_type_override": [ + "insert", + "update_postimage", + "delete" + ], + "starting_version_from_dlt_setup": true, "select_exp": [ "CUSTOMER_ID", "FIRST_NAME", "LAST_NAME", "EMAIL", "_change_type AS META_CDC_CHANGE_TYPE" - ], - "starting_version_from_dlt_setup": true, - "cdf_change_type_override": [ - "insert", - "update_postimage", - "delete" ] } }, @@ -73,7 +73,7 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "input": [ + "input_flows": [ { "view": "v_feature_historical_files_snapshot_append", "flow": "f_feature_historical_files_snapshot_append" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json index 519a8aa..eaab6b5 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json @@ -8,9 +8,9 @@ "node_type": "source", "source_type": "delta", "config": { + "mode": "batch", "database": "{staging_schema}", - "table": "customer", - "mode": "batch" + "table": "customer" } }, { @@ -19,7 +19,7 @@ "config": { "table": "feature_mv_source_view", "table_type": "mv", - "input": [ + "input_flows": [ { "flow": "f_mv_source_view_load", "view": "v_feature_mv_source_view" @@ -55,12 +55,15 @@ "table": "feature_mv_with_quarantine", "table_type": "mv", "sql_statement": "SELECT * FROM {staging_schema}.customer_address", - "data_quality_expectations_enabled": true, - "data_quality_expectations_path": "./customer_address_dqe.json", - "quarantine_mode": "table", - "quarantine_target_details": { - "target_format": "delta", - "cluster_by_auto": true + "data_quality": { + "expectations_path": "./customer_address_dqe.json" + }, + "quarantine": { + "mode": "table", + "target": { + "target_format": "delta", + "cluster_by_auto": true + } } } }, diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json index 5332a40..5c6c912 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json @@ -40,7 +40,7 @@ ], "scd_type": "1" }, - "input": [ + "input_flows": [ { "view": "v_periodic_snapshot_customer_scd1", "flow": "f_periodic_snapshot_customer_scd1" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json index fa54474..666fab7 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json @@ -40,7 +40,7 @@ ], "scd_type": "2" }, - "input": [ + "input_flows": [ { "view": "v_periodic_snapshot_customer_scd2", "flow": "f_periodic_snapshot_customer_scd2" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json index 155cb38..421c831 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_extension_main.json @@ -24,7 +24,7 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "input": [ + "input_flows": [ { "view": "v_feature_python_extension_source", "flow": "f_feature_python_extension_source" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json index 00de723..ae3cccb 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_source_main.json @@ -24,7 +24,7 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "input": [ + "input_flows": [ { "view": "v_feature_python_function_source", "flow": "f_feature_python_function_source" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json index a0d08e2..5268d51 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_extension_main.json @@ -31,7 +31,7 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "input": [ + "input_flows": [ { "view": "v_transform_python_extension", "flow": "f_transform_python_extension" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json index b83f333..fade5cd 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/python_transform_main.json @@ -31,7 +31,7 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "input": [ + "input_flows": [ { "view": "v_transform_python_function", "flow": "f_transform_python_function" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json index 1acc9c7..02677d2 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_flag_main.json @@ -29,10 +29,13 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "data_quality_expectations_enabled": true, - "data_quality_expectations_path": "./customer_address_dqe.json", - "quarantine_mode": "flag", - "input": [ + "data_quality": { + "expectations_path": "./customer_address_dqe.json" + }, + "quarantine": { + "mode": "flag" + }, + "input_flows": [ { "view": "v_customer_address_feature_quarantine_flag", "flow": "f_customer_address_feature_quarantine_flag" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json index b8edec2..db6d0a9 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_main.json @@ -29,13 +29,16 @@ "table_properties": { "delta.enableChangeDataFeed": "true" }, - "data_quality_expectations_enabled": true, - "data_quality_expectations_path": "./customer_address_dqe.json", - "quarantine_mode": "table", - "quarantine_target_details": { - "target_format": "delta" + "data_quality": { + "expectations_path": "./customer_address_dqe.json" }, - "input": [ + "quarantine": { + "mode": "table", + "target": { + "target_format": "delta" + } + }, + "input_flows": [ { "view": "v_customer_address_feature_quarantine_table", "flow": "f_customer_address_feature_quarantine_table" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json index 0d9ef9e..5c78dec 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/quarantine_table_with_cdc_main.json @@ -36,13 +36,16 @@ "sequence_by": "LOAD_TIMESTAMP", "scd_type": "2" }, - "data_quality_expectations_enabled": true, - "data_quality_expectations_path": "./customer_address_dqe.json", - "quarantine_mode": "table", - "quarantine_target_details": { - "target_format": "delta" + "data_quality": { + "expectations_path": "./customer_address_dqe.json" }, - "input": [ + "quarantine": { + "mode": "table", + "target": { + "target_format": "delta" + } + }, + "input_flows": [ { "view": "v_customer_address_feature_quarantine_table_cdc", "flow": "f_customer_address_feature_quarantine_table_cdc" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json index 35600cd..ac8d828 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_custom_python_generate_cdc_feed.json @@ -23,7 +23,7 @@ "sink_options": { "table_name": "{bronze_schema}.feature_sink_delta_uc_table" }, - "input": [ + "input_flows": [ { "view": "v_customer_delta_sink_uc_table", "flow": "f_customer_delta_sink_uc_table" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json index 18bca23..40e6c6f 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_path_table_main.json @@ -17,19 +17,19 @@ { "name": "target_sink_delta_path_table", "node_type": "target", + "target_type": "delta_sink", "config": { "name": "sink_delta_path_table", "sink_options": { "path": "{staging_volume}/feature_sink_delta_path_table" }, - "input": [ + "input_flows": [ { "view": "v_customer_delta_sink_path_table", "flow": "f_customer_delta_sink_path_table" } ] - }, - "target_type": "delta_sink" + } } ] } diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json index 8d9ef2c..ac8d828 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_delta_uc_table_main.json @@ -17,19 +17,19 @@ { "name": "target_sink_delta_uc_table", "node_type": "target", + "target_type": "delta_sink", "config": { "name": "sink_delta_uc_table", "sink_options": { "table_name": "{bronze_schema}.feature_sink_delta_uc_table" }, - "input": [ + "input_flows": [ { "view": "v_customer_delta_sink_uc_table", "flow": "f_customer_delta_sink_uc_table" } ] - }, - "target_type": "delta_sink" + } } ] } diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json index 969dc13..4170a75 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_basic_sql_main.json @@ -17,6 +17,7 @@ { "name": "target_for_each_batch_basic_sql", "node_type": "target", + "target_type": "foreach_batch_sink", "config": { "name": "for_each_batch_basic_sql", "sink_type": "basic_sql", @@ -28,14 +29,13 @@ "delta.enableChangeDataFeed": "true" } }, - "input": [ + "input_flows": [ { "view": "v_customer_purchase_basic_sql", "flow": "f_customer_purchase_basic_sql" } ] - }, - "target_type": "foreach_batch_sink" + } } ] } diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json index a95201d..5be3db7 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/sink_foreach_batch_single_python_function_main.json @@ -17,6 +17,7 @@ { "name": "target_for_each_batch_python_function", "node_type": "target", + "target_type": "foreach_batch_sink", "config": { "name": "for_each_batch_python_function", "sink_type": "python_function", @@ -28,14 +29,13 @@ "staging_volume": "{staging_volume}" } }, - "input": [ + "input_flows": [ { "view": "v_customer_purchase_python_function", "flow": "f_customer_purchase_python_function" } ] - }, - "target_type": "foreach_batch_sink" + } } ] } diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json index 650ce79..f666af4 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_append_only_main.json @@ -38,7 +38,7 @@ "table": "table_to_migrate_scd0" } }, - "input": [ + "input_flows": [ { "view": "v_feature_tm_customer_scd0", "flow": "f_feature_tm_customer_scd0" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json index be73011..024f922 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/table_migration_scd2_main.json @@ -56,7 +56,7 @@ ] } }, - "input": [ + "input_flows": [ { "view": "v_feature_tm_customer_scd2", "flow": "f_feature_tm_customer_scd2" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json index 751a58b..1fe1362 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_flows_dataflowspec_main.json @@ -20,7 +20,7 @@ "ignore_null_updates": true, "scd_type": "1" }, - "input": [ + "input_flows": [ { "view": "v_flows_version_mapping", "flow": "f_flows_version_mapping" @@ -68,7 +68,7 @@ "ignore_null_updates": false, "scd_type": "1" }, - "input": [ + "input_flows": [ { "view": "v_version_mapping_final", "flow": "f_version_mapping_final" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json index 0155905..fbbeee0 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/version_mapping_standard_dataflowspec_main.json @@ -32,7 +32,7 @@ "ignore_null_updates": false, "scd_type": "1" }, - "input": [ + "input_flows": [ { "view": "v_version_mapping_standard", "flow": "f_version_mapping_standard" diff --git a/scripts/migrate_to_nodespec.py b/scripts/migrate_to_nodespec.py index 0762cb8..6a8b2a6 100644 --- a/scripts/migrate_to_nodespec.py +++ b/scripts/migrate_to_nodespec.py @@ -11,6 +11,15 @@ ``table_properties``) while leaving opaque value maps (table properties, reader options, spark conf, tokens) untouched. +Target wiring and feature settings use the current nodespec shape: targets +declare their inputs via ``input_flows``; data quality is a nested +``data_quality`` object; quarantine is a nested ``quarantine`` object +(``mode`` + optional ``target``). Sink targets keep their flat +``sink_type``/``sink_config``/``sink_options`` fields. Output is emitted in a +canonical field order (identity, then table/structural details, then feature +bundles, then ``input_flows``). Passing an existing nodespec spec normalises any +legacy fields and reorders it (idempotent). + Usage: python migrate_to_nodespec.py [--output ] python migrate_to_nodespec.py --output-dir [--recursive] @@ -90,6 +99,13 @@ "tableName": "table_name", } +# Inner keys of table_migration_details.source_details (a delta source shape). +_MIGRATION_SOURCE_MAP = { + "selectExp": "select_exp", + "whereClause": "where_clause", + "exceptColumns": "except_columns", +} + _PYTHON_TRANSFORM_MAP = { "functionPath": "function_path", "pythonModule": "python_module", @@ -104,6 +120,51 @@ def _rename(d: Dict, mapping: Dict[str, str]) -> Dict: return {mapping.get(k, k): v for k, v in d.items()} +# ─── canonical field ordering ──────────────────────────────────────────────── +# A single, readable order applied to emitted specs, nodes, and configs so that +# generated (and hand-authored) specs read consistently: identity first, then +# table/structural details, then feature bundles, with input_flows (the wiring) +# last. Keys not listed keep their original relative order, after the known ones. + +_SPEC_ORDER = ["data_flow_id", "data_flow_group", "data_flow_type", + "data_flow_version", "tags", "features", "nodes"] + +_NODE_ORDER = ["name", "node_type", "source_type", "transformation_type", + "target_type", "output_view_name", "enabled", "config"] + +_CONFIG_ORDER = [ + "mode", "database", "table", "table_path", "path", "format", + "cdf_enabled", "cdf_change_type_override", "starting_version_from_dlt_setup", + "table_type", + "function_path", "python_module", "sql_path", "sql_statement", "tokens", + "select_exp", "where_clause", "schema_path", "reader_options", "python_transform", + "table_properties", "partition_columns", "cluster_by_columns", "cluster_by_auto", + "comment", "spark_conf", "row_filter", "config_flags", + "refresh_policy", "table_details", "once", + "cdc_settings", "cdc_snapshot_settings", "data_quality", "quarantine", + "table_migration_details", + "name", "sink_type", "sink_config", "sink_options", + "input_flows", +] + + +def _order(d: Dict, order: List[str]) -> Dict: + """Return d with keys in `order` first (when present), remaining keys after.""" + if not isinstance(d, dict): + return d + known = [k for k in order if k in d] + rest = [k for k in d if k not in order] + return {k: d[k] for k in known + rest} + + +def _order_node(node: Dict) -> Dict: + """Order a node's keys and its config's keys canonically.""" + node = _order(node, _NODE_ORDER) + if isinstance(node.get("config"), dict): + node["config"] = _order(node["config"], _CONFIG_ORDER) + return node + + def _convert_source_details(details: Dict) -> Dict: """camelCase -> snake_case for a source/view sourceDetails block.""" out = _rename(details, _SOURCE_DETAIL_MAP) @@ -143,23 +204,28 @@ def _add_target_settings(config: Dict, src: Dict) -> None: if snapshot: config["cdc_snapshot_settings"] = _convert_snapshot(snapshot) - dq_enabled = _get(src, "dataQualityExpectationsEnabled", "data_quality_expectations_enabled") - if dq_enabled: - config["data_quality_expectations_enabled"] = dq_enabled + # Data quality collapses to a single nested object; presence implies enabled, + # so the standalone enabled flag is dropped. dq_path = _get(src, "dataQualityExpectationsPath", "data_quality_expectations_path") if dq_path: - config["data_quality_expectations_path"] = dq_path + config["data_quality"] = {"expectations_path": dq_path} + # Quarantine collapses mode + target details into one nested object. + # Mode "off" (or absent) means no quarantine, so no object is emitted. quarantine_mode = _get(src, "quarantineMode", "quarantine_mode") - if quarantine_mode: - config["quarantine_mode"] = quarantine_mode - quarantine_details = _get(src, "quarantineTargetDetails", "quarantine_target_details") - if quarantine_details: - config["quarantine_target_details"] = _rename(quarantine_details, _QUARANTINE_MAP) + if quarantine_mode and quarantine_mode != "off": + quarantine: Dict[str, Any] = {"mode": quarantine_mode} + quarantine_details = _get(src, "quarantineTargetDetails", "quarantine_target_details") + if quarantine_details: + quarantine["target"] = _rename(quarantine_details, _QUARANTINE_MAP) + config["quarantine"] = quarantine table_migration = _get(src, "tableMigrationDetails", "table_migration_details") if table_migration: - config["table_migration_details"] = _rename(table_migration, _TABLE_MIGRATION_MAP) + tm = _rename(table_migration, _TABLE_MIGRATION_MAP) + if isinstance(tm.get("source_details"), dict): + tm["source_details"] = _rename(tm["source_details"], _MIGRATION_SOURCE_MAP) + config["table_migration_details"] = tm def _result_envelope(spec: Dict, nodes: List[Dict]) -> Dict: @@ -177,7 +243,37 @@ def _result_envelope(spec: Dict, nodes: List[Dict]) -> Dict: result["tags"] = spec["tags"] if spec.get("features"): result["features"] = spec["features"] - return result + result["nodes"] = [_order_node(n) for n in nodes] + return _order(result, _SPEC_ORDER) + + +def _migrate_nodespec_config(cfg: Dict) -> None: + """Migrate a single node config's legacy fields to the current shape, in place.""" + if "input" in cfg and "input_flows" not in cfg: + cfg["input_flows"] = cfg.pop("input") + path = cfg.pop("data_quality_expectations_path", None) + cfg.pop("data_quality_expectations_enabled", None) + if path and "data_quality" not in cfg: + cfg["data_quality"] = {"expectations_path": path} + mode = cfg.pop("quarantine_mode", None) + qtd = cfg.pop("quarantine_target_details", None) + if mode and mode != "off" and "quarantine" not in cfg: + quarantine: Dict[str, Any] = {"mode": mode} + if qtd: + quarantine["target"] = qtd + cfg["quarantine"] = quarantine + + +def _migrate_nodespec(spec: Dict) -> Dict: + """Normalise an existing nodespec spec: migrate any legacy config fields + (input -> input_flows; data_quality_expectations_* -> data_quality; + quarantine_mode/quarantine_target_details -> quarantine) and apply the + canonical field order. Idempotent for already-current specs.""" + for node in spec.get("nodes", []): + if isinstance(node.get("config"), dict): + _migrate_nodespec_config(node["config"]) + spec["nodes"] = [_order_node(n) for n in spec.get("nodes", [])] + return _order(spec, _SPEC_ORDER) def migrate_spec(spec: Dict) -> Dict: @@ -191,7 +287,7 @@ def migrate_spec(spec: Dict) -> Dict: elif spec_type == "materialized_view": return _migrate_materialized_view(spec) elif spec_type == "nodespec": - return spec # already nodespec + return _migrate_nodespec(spec) # normalise legacy fields + canonical order else: print(f" Warning: Unknown dataFlowType '{spec_type}', treating as standard") return _migrate_standard(spec) @@ -201,19 +297,30 @@ def migrate_spec(spec: Dict) -> Dict: def _migrate_standard(spec: Dict) -> Dict: """Convert a standard spec to nodespec.""" - source_id = spec.get("sourceViewName") or "v_source" - source_node = _build_source_node(source_id, spec.get("sourceType", "delta"), - spec.get("sourceDetails", {}), spec.get("mode", "stream")) + # Historical snapshot targets read files/tables directly (via + # cdc_snapshot_settings) and have no source node — the standard spec carries + # no sourceDetails in that case, so don't synthesise one. + snapshot = _get(spec, "cdcSnapshotSettings", "cdc_snapshot_settings") or {} + is_historical_snapshot = _get(snapshot, "snapshotType", "snapshot_type") == "historical" + + nodes: List[Dict] = [] + source_id: Optional[str] = None + if not is_historical_snapshot: + source_id = spec.get("sourceViewName") or "v_source" + nodes.append(_build_source_node(source_id, spec.get("sourceType", "delta"), + spec.get("sourceDetails", {}), spec.get("mode", "stream"))) target_config, target_type = _build_spec_target(spec) - target_config["input"] = [source_id] + if source_id: + target_config["input_flows"] = [source_id] target_name = target_config.get("table") or target_config.get("name") or "output" target_node: Dict[str, Any] = {"name": f"target_{target_name}", "node_type": "target"} if target_type != "delta": target_node["target_type"] = target_type target_node["config"] = target_config + nodes.append(target_node) - return _result_envelope(spec, [source_node, target_node]) + return _result_envelope(spec, nodes) def _build_source_node(name: str, source_type: str, source_details: Dict, mode: str) -> Dict: @@ -283,7 +390,7 @@ def find_target_node(target_name: str) -> Optional[Dict]: continue node_ids.add(target_id) config = _build_target_config_from_staging(stg_name, stg_config) - config["input"] = [] + config["input_flows"] = [] nodes.append({"name": target_id, "node_type": "target", "config": config}) for flow_name, flow_config in flows.items(): @@ -350,7 +457,7 @@ def find_target_node(target_name: str) -> Optional[Dict]: target_key = target_table.split(".")[-1] target_node = find_target_node(target_key) if target_node is not None: - target_node["config"].setdefault("input", []).append(source_node_id) + target_node["config"].setdefault("input_flows", []).append(source_node_id) else: # Main (spec-level) target (delta or sink). target_id = f"target_{target_key}" @@ -359,7 +466,7 @@ def find_target_node(target_name: str) -> Optional[Dict]: main_config, target_type = _build_spec_target(spec) if flow_details.get("once"): main_config["once"] = True - main_config["input"] = [source_node_id] + main_config["input_flows"] = [source_node_id] main_node: Dict[str, Any] = {"name": target_id, "node_type": "target"} if target_type != "delta": main_node["target_type"] = target_type @@ -369,8 +476,8 @@ def find_target_node(target_name: str) -> Optional[Dict]: # Drop placeholder empty input lists. for node in nodes: config = node.get("config", {}) - if config.get("input") == []: - del config["input"] + if config.get("input_flows") == []: + del config["input_flows"] return _result_envelope(spec, nodes) @@ -429,7 +536,7 @@ def _migrate_materialized_view(spec: Dict) -> Dict: source_view.get("sourceDetails") or source_view.get("source_details", {}), "batch", )) - target_config["input"] = [source_id] + target_config["input_flows"] = [source_id] nodes.append({"name": f"target_{mv_name}", "node_type": "target", "config": target_config}) diff --git a/src/dataflow_spec_builder/transformer/nodespec.py b/src/dataflow_spec_builder/transformer/nodespec.py index 21d4ac0..59ccbe5 100644 --- a/src/dataflow_spec_builder/transformer/nodespec.py +++ b/src/dataflow_spec_builder/transformer/nodespec.py @@ -11,8 +11,9 @@ The snake_case -> camelCase translation is centralised: one global key map (``_KEYS``) plus ``_camel`` (flat) / ``_deep_camel`` (nested blobs). Each builder -copies every config key that isn't "structural" (``_HANDLED`` — input, cdc_*, -quarantine_*, sink_*, ...); everything else is passthrough table/source detail. +copies every config key that isn't "structural" (``_HANDLED`` — input_flows, +cdc_*, data_quality, quarantine, sink_*, ...); everything else is passthrough +table/source detail. """ from collections import defaultdict from typing import Any, Dict, List, Optional, Tuple, Set, Union @@ -45,10 +46,9 @@ # Keys a target config handles specially, so they are NOT copied into details. _HANDLED = { - "input", "table", "table_type", "enabled", "once", "name", + "input_flows", "table", "table_type", "enabled", "once", "name", "cdc_settings", "cdc_snapshot_settings", - "data_quality_expectations_enabled", "data_quality_expectations_path", - "quarantine_mode", "quarantine_target_details", "table_migration_details", + "data_quality", "quarantine", "table_migration_details", "sink_type", "sink_config", "sink_options", "table_details", "source_view", } @@ -137,7 +137,7 @@ def _validate(self, sources: List[Dict], targets: List[Dict], nodes: List[Dict]) raise ValueError( f"Materialized view target '{t.get('name')}' defines an inline 'source_view'. " "This is no longer supported. Declare a source node and chain it into the " - "materialized view target via its 'input' array instead.") + "materialized view target via its 'input_flows' array instead.") if not targets: raise ValueError("Nodespec spec must contain at least one target node") @@ -150,9 +150,9 @@ def _validate(self, sources: List[Dict], targets: List[Dict], nodes: List[Dict]) # -- inputs -- def _inputs(self, node: Dict) -> List[Tuple[Optional[str], str]]: - """``input`` items as (flow_name, view_name); strings auto-name the flow.""" + """``input_flows`` items as (flow_name, view_name); strings auto-name the flow.""" pairs = [] - for item in node.get("config", {}).get("input", []) or []: + for item in node.get("config", {}).get("input_flows", []) or []: if isinstance(item, dict) and item.get("view"): pairs.append((item.get("flow"), item["view"])) elif not isinstance(item, dict): @@ -168,16 +168,18 @@ def _settings(self, dst: Dict, cfg: Dict, *, cdc=False, migration=False, dq_defa dst["cdcSettings"] = cfg["cdc_settings"] if cfg.get("cdc_snapshot_settings"): dst["cdcSnapshotSettings"] = _deep_camel(cfg["cdc_snapshot_settings"]) + dq = cfg.get("data_quality") or {} if dq_default: - dst["dataQualityExpectationsEnabled"] = cfg.get("data_quality_expectations_enabled", False) - elif cfg.get("data_quality_expectations_enabled"): - dst["dataQualityExpectationsEnabled"] = cfg["data_quality_expectations_enabled"] - if cfg.get("data_quality_expectations_path"): - dst["dataQualityExpectationsPath"] = cfg["data_quality_expectations_path"] - if cfg.get("quarantine_mode"): - dst["quarantineMode"] = cfg["quarantine_mode"] - if cfg.get("quarantine_target_details"): - dst["quarantineTargetDetails"] = cfg["quarantine_target_details"] + dst["dataQualityExpectationsEnabled"] = bool(dq) + elif dq: + dst["dataQualityExpectationsEnabled"] = True + if dq.get("expectations_path"): + dst["dataQualityExpectationsPath"] = dq["expectations_path"] + quarantine = cfg.get("quarantine") or {} + if quarantine.get("mode"): + dst["quarantineMode"] = quarantine["mode"] + if quarantine.get("target"): + dst["quarantineTargetDetails"] = quarantine["target"] if migration and cfg.get("table_migration_details"): dst["tableMigrationDetails"] = _deep_camel(cfg["table_migration_details"]) diff --git a/src/schemas/spec_nodespec.json b/src/schemas/spec_nodespec.json index cf42d61..7640dfd 100644 --- a/src/schemas/spec_nodespec.json +++ b/src/schemas/spec_nodespec.json @@ -2,617 +2,532 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://dlt-framework/schemas/spec_nodespec.json", "title": "Nodespec Dataflow Specification", - "description": "Dataflow specification using a node-based architecture. Converts node graphs into Lakeflow Framework's flow-based format. All target-specific settings (CDC, data quality, quarantine) are specified on target nodes directly.", + "description": "Node-based dataflow spec (snake_case). A node's `config` is discriminated by node_type + source_type/target_type/table_type so editors offer only the fields valid for that node. Feature bundles (cdc, snapshot, data_quality, quarantine, migration) are single nested objects. Targets declare their inputs via `input_flows`. Sink targets keep their current flat form for now (sink redesign deferred).", + "$comment": "A node's `config` is discriminated at the node level via allOf/if-then keyed on node_type + source_type/target_type/table_type; each `then` selects one closed config $def, so editors offer only the fields valid for that node. Nested blobs are typed objects with descriptions. Sink targets keep their current flat form (sink_type/sink_config/sink_options) pending the sink redesign. Legacy definitions_sources.json / definitions_targets.json are camelCase; nodespec is snake_case, so source/target detail shapes are re-declared here. Literal Spark option keys (cloudFiles.*, kafka.*) are preserved verbatim.", "$defs": { "nodespecSpec": { "type": "object", - "description": "Nodespec specs use snake_case field names throughout.", + "description": "A nodespec pipeline: a graph of source -> transformation -> target nodes.", "properties": { - "data_flow_id": { - "type": "string" - }, - "data_flow_group": { - "type": "string" - }, - "data_flow_type": { - "const": "nodespec", - "description": "Optional. nodespec is the default spec type, so this may be omitted; when present it must be \"nodespec\"." - }, - "data_flow_version": { - "type": "string" - }, - "tags": { - "type": "object", - "additionalProperties": true - }, - "features": { - "type": "object", - "additionalProperties": true - }, + "data_flow_id": {"type": "string", "description": "Unique id for this dataflow."}, + "data_flow_group": {"type": "string", "description": "Group this dataflow belongs to (maps to a pipeline)."}, + "data_flow_type": {"const": "nodespec", "description": "Optional. nodespec is the default spec type, so this may be omitted; when present it must be \"nodespec\"."}, + "data_flow_version": {"type": "string"}, + "tags": {"type": "object", "additionalProperties": true}, + "features": {"type": "object", "additionalProperties": true}, "nodes": { "type": "array", - "description": "Array of nodes representing sources, transformations, and targets in a pipeline graph", - "items": { - "$ref": "#/$defs/node" - }, + "description": "Nodes in the pipeline graph.", + "items": {"$ref": "#/$defs/node"}, "minItems": 1 } }, - "required": [ - "data_flow_id", - "data_flow_group", - "nodes" - ], + "required": ["data_flow_id", "data_flow_group", "nodes"], "additionalProperties": false }, + "node": { "type": "object", "properties": { - "name": { - "type": "string", - "description": "Unique name for the node within the pipeline" - }, + "name": {"type": "string", "description": "Unique node name within the pipeline. Also the emitted view name for source/transformation nodes unless output_view_name is set."}, "node_type": { "type": "string", - "enum": [ - "source", - "transformation", - "target" - ], - "description": "Type of node: source (data input), transformation (processing), or target (data output)" + "enum": ["source", "transformation", "target"], + "description": "source (reads data) | transformation (reshapes a view) | target (writes an output)." }, "source_type": { "type": "string", - "enum": [ - "batchFiles", - "cloudFiles", - "delta", - "deltaJoin", - "kafka", - "python", - "sql" - ], - "description": "Type of data source (required for source nodes)" + "enum": ["batchFiles", "cloudFiles", "delta", "deltaJoin", "kafka", "python", "sql"], + "description": "Kind of source. Required when node_type=source; selects which config fields apply." }, "transformation_type": { "type": "string", - "enum": [ - "sql", - "python" - ], - "description": "Type of transformation to apply (required for transformation nodes)" - }, - "output_view_name": { - "type": "string", - "description": "Optional name for the output view. If omitted, defaults to the node name. Applicable to source and transformation nodes." + "enum": ["sql", "python"], + "description": "Kind of transformation. Required when node_type=transformation." }, "target_type": { "type": "string", - "enum": [ - "delta", - "delta_sink", - "foreach_batch_sink", - "custom_python_sink", - "kafka_sink" - ], + "enum": ["delta", "delta_sink", "foreach_batch_sink", "custom_python_sink", "kafka_sink"], "default": "delta", - "description": "Target format type (required for target nodes). Default: delta" + "description": "Output format for a target. Default: delta (a managed table). The *_sink values write to external sinks." }, - "enabled": { - "type": "boolean", - "default": true, - "description": "Whether this node is enabled in the pipeline" + "output_view_name": { + "type": "string", + "description": "Overrides the emitted view name (source/transformation nodes). Defaults to the node name." }, - "config": { - "type": "object", - "description": "Node-specific configuration" - } + "enabled": {"type": "boolean", "default": true, "description": "Whether this node is included in the pipeline."}, + "config": {"type": "object", "description": "Node configuration. Its valid fields are determined by node_type and the type field above."} }, - "required": [ - "name", - "node_type" - ], + "required": ["name", "node_type"], + "additionalProperties": false, "allOf": [ { - "if": { - "properties": { - "node_type": { - "const": "source" - } - } - }, - "then": { - "required": [ - "source_type" - ], - "properties": { - "config": { - "$ref": "#/$defs/source_node_config" - } - } - } + "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "delta"}}, "required": ["node_type", "source_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/sourceDeltaConfig"}}} }, { - "if": { - "properties": { - "node_type": { - "const": "transformation" - } - } - }, - "then": { - "required": [ - "transformation_type" - ], - "properties": { - "config": { - "$ref": "#/$defs/transformation_node_config" - } - } - } + "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "cloudFiles"}}, "required": ["node_type", "source_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/sourceCloudFilesConfig"}}} }, { - "if": { - "properties": { - "node_type": { - "const": "target" - } - } - }, - "then": { - "properties": { - "config": { - "$ref": "#/$defs/target_node_config" - } - } - } - } - ] - }, - "source_node_config": { - "type": "object", - "description": "Configuration for source nodes. The sourceType field has moved to the node level.", - "properties": { - "mode": { - "type": "string", - "enum": [ - "stream", - "batch" - ], - "default": "stream", - "description": "Processing mode for the source" - }, - "database": { - "type": "string" - }, - "table": { - "type": "string" - }, - "path": { - "type": "string" + "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "batchFiles"}}, "required": ["node_type", "source_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/sourceBatchFilesConfig"}}} }, - "cdf_enabled": { - "type": "boolean" - }, - "schema_path": { - "type": "string" - }, - "select_exp": { - "type": "array", - "items": { - "type": "string" - } + { + "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "kafka"}}, "required": ["node_type", "source_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/sourceKafkaConfig"}}} }, - "where_clause": { - "type": "array", - "items": { - "type": "string" - } + { + "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "deltaJoin"}}, "required": ["node_type", "source_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/sourceDeltaJoinConfig"}}} }, - "reader_options": { - "type": "object", - "additionalProperties": true + { + "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "python"}}, "required": ["node_type", "source_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/pythonLogicConfig"}}} }, - "function_path": { - "type": "string", - "description": "Path to Python source function file" + { + "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "sql"}}, "required": ["node_type", "source_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/sqlLogicConfig"}}} }, - "python_module": { - "type": "string", - "description": "Python module path for source function" + { + "if": {"properties": {"node_type": {"const": "source"}}, "required": ["node_type"]}, + "then": {"required": ["source_type"]} }, - "tokens": { - "type": "object", - "description": "Token dictionary passed to Python sources" + + { + "if": {"properties": {"node_type": {"const": "transformation"}, "transformation_type": {"const": "sql"}}, "required": ["node_type", "transformation_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/sqlLogicConfig"}}} }, - "sql_path": { - "type": "string", - "description": "Path to SQL file for SQL sources" + { + "if": {"properties": {"node_type": {"const": "transformation"}, "transformation_type": {"const": "python"}}, "required": ["node_type", "transformation_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/pythonLogicConfig"}}} }, - "sql_statement": { - "type": "string", - "description": "Inline SQL statement for SQL sources" + { + "if": {"properties": {"node_type": {"const": "transformation"}}, "required": ["node_type"]}, + "then": {"required": ["transformation_type"]} }, - "starting_version_from_dlt_setup": { - "type": "boolean" + + { + "if": {"properties": {"node_type": {"const": "target"}, "target_type": {"const": "delta"}}, "required": ["node_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/targetDeltaConfig"}}} }, - "cdf_change_type_override": { - "type": "array", - "items": { - "type": "string" - } + { + "if": {"properties": {"node_type": {"const": "target"}, "target_type": {"enum": ["delta_sink", "kafka_sink", "foreach_batch_sink", "custom_python_sink"]}}, "required": ["node_type", "target_type"]}, + "then": {"properties": {"config": {"$ref": "#/$defs/targetSinkConfig"}}} } + ] + }, + + "sourceCommon": { + "type": "object", + "properties": { + "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream", "description": "Read mode. Default: stream."}, + "select_exp": {"type": "array", "items": {"type": "string"}, "description": "SELECT expressions applied to the source."}, + "where_clause": {"type": "array", "items": {"type": "string"}, "description": "WHERE predicates applied to the source."}, + "schema_path": {"type": "string", "pattern": "\\.(json|ddl)$", "description": "Path to a schema file applied when reading."} + } + }, + + "pythonTransform": { + "type": "object", + "description": "Inline Python transform applied to a source. Provide function_path or python_module (module alias also accepted).", + "properties": { + "function_path": {"type": "string", "pattern": "\\.py$"}, + "python_module": {"type": "string"}, + "module": {"type": "string"}, + "tokens": {"type": "object", "additionalProperties": true} }, - "additionalProperties": true + "additionalProperties": false }, - "transformation_node_config": { + + "sourceDeltaConfig": { "type": "object", - "description": "Configuration for transformation nodes. The transformationType and outputViewName fields have moved to the node level.", + "description": "Delta table source.", + "allOf": [{"$ref": "#/$defs/sourceCommon"}], "properties": { - "sql_path": { - "type": "string" - }, - "sql_statement": { - "type": "string" - }, - "function_path": { - "type": "string" - }, - "python_module": { - "type": "string" - }, - "tokens": { - "type": "object" - } + "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, + "database": {"type": "string", "description": "Source schema/database."}, + "table": {"type": "string", "description": "Source table name."}, + "table_path": {"type": "string", "description": "Physical path to the Delta table (alternative to database/table)."}, + "cdf_enabled": {"type": "boolean", "description": "Read the table's Change Data Feed."}, + "cdf_change_type_override": {"type": "array", "items": {"type": "string", "enum": ["insert", "update_postimage", "delete"]}}, + "starting_version_from_dlt_setup": {"type": "boolean"}, + "select_exp": {"type": "array", "items": {"type": "string"}}, + "where_clause": {"type": "array", "items": {"type": "string"}}, + "schema_path": {"type": "string", "pattern": "\\.(json|ddl)$"}, + "reader_options": {"type": "object", "additionalProperties": true}, + "python_transform": {"$ref": "#/$defs/pythonTransform"} }, - "additionalProperties": true + "required": ["table"], + "additionalProperties": false }, - "target_node_config": { + + "sourceCloudFilesConfig": { "type": "object", - "description": "Configuration for target nodes. All target-specific settings including CDC, data quality, quarantine, table migration, and sink config are specified here.", + "description": "Auto Loader (cloudFiles) source.", "properties": { - "input": { - "type": "array", - "items": { - "oneOf": [ - { - "type": "string", - "description": "Upstream source/transformation node name. The SDP flow name is auto-generated." - }, - { - "type": "object", - "properties": { - "view": { - "type": "string", - "description": "Upstream source or transformation node name feeding this target" - }, - "flow": { - "type": "string", - "description": "Explicit SDP flow name. Pin this to keep flow names stable across changes (renaming a flow forces a full refresh in SDP)." - } - }, - "required": [ - "view" - ], - "additionalProperties": false - } - ] - }, - "description": "Inputs feeding this target. Each item is either an upstream node name (string, flow name auto-generated) or an object {view, flow} that pins the SDP flow name." - }, - "table": { - "type": "string", - "description": "Target table name (for delta targets)" - }, - "database": { - "type": "string", - "description": "Target database/schema name" - }, - "table_type": { - "type": "string", - "enum": [ - "st", - "mv" - ], - "default": "st", - "description": "Table type: st (streaming table) or mv (materialized view). Default: st" - }, - "schema_path": { - "type": "string", - "pattern": "\\.(json|ddl)$" - }, - "table_properties": { - "type": "object" - }, - "path": { - "type": "string" - }, - "partition_columns": { + "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, + "path": {"type": "string", "description": "Directory or glob to ingest."}, + "reader_options": {"$ref": "#/$defs/readerOptionsCloudFiles"}, + "select_exp": {"type": "array", "items": {"type": "string"}}, + "where_clause": {"type": "array", "items": {"type": "string"}}, + "schema_path": {"type": "string", "pattern": "\\.json$"}, + "python_transform": {"$ref": "#/$defs/pythonTransform"} + }, + "required": ["path", "reader_options"], + "additionalProperties": false + }, + + "sourceBatchFilesConfig": { + "type": "object", + "description": "Batch file source (read once, not streamed).", + "properties": { + "mode": {"type": "string", "enum": ["stream", "batch"], "default": "batch"}, + "format": {"type": "string", "enum": ["csv", "json", "parquet", "text", "xml"]}, + "path": {"type": "string"}, + "reader_options": {"type": "object", "additionalProperties": true}, + "select_exp": {"type": "array", "items": {"type": "string"}}, + "where_clause": {"type": "array", "items": {"type": "string"}}, + "schema_path": {"type": "string", "pattern": "\\.json$"}, + "python_transform": {"$ref": "#/$defs/pythonTransform"} + }, + "required": ["path", "reader_options"], + "additionalProperties": false + }, + + "sourceKafkaConfig": { + "type": "object", + "description": "Kafka source.", + "properties": { + "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, + "reader_options": {"$ref": "#/$defs/readerOptionsKafka"}, + "select_exp": {"type": "array", "items": {"type": "string"}}, + "where_clause": {"type": "array", "items": {"type": "string"}}, + "schema_path": {"type": "string", "pattern": "\\.json$"}, + "python_transform": {"$ref": "#/$defs/pythonTransform"} + }, + "required": ["reader_options"], + "additionalProperties": false + }, + + "sourceDeltaJoinConfig": { + "type": "object", + "description": "Join of multiple Delta sources into one view.", + "properties": { + "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, + "sources": { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "database": {"type": "string"}, + "table": {"type": "string"}, + "alias": {"type": "string"}, + "join_mode": {"type": "string", "enum": ["stream", "static"], "default": "stream"}, + "cdf_enabled": {"type": "boolean"}, + "table_path": {"type": "string"}, + "select_exp": {"type": "array", "items": {"type": "string"}}, + "where_clause": {"type": "array", "items": {"type": "string"}}, + "schema_path": {"type": "string", "pattern": "\\.json$"}, + "reader_options": {"type": "object", "additionalProperties": true}, + "python_transform": {"$ref": "#/$defs/pythonTransform"} + }, + "required": ["database", "table", "alias", "cdf_enabled", "join_mode"], + "additionalProperties": false } }, - "cluster_by_columns": { + "joins": { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "join_type": {"type": "string", "enum": ["left", "inner"], "default": "left"}, + "condition": {"type": "string"} + }, + "required": ["join_type", "condition"], + "additionalProperties": false } }, - "cluster_by_auto": { - "type": "boolean" - }, - "comment": { - "type": "string" - }, - "spark_conf": { - "type": "object" - }, - "row_filter": { - "type": "string" - }, - "config_flags": { - "type": "array", - "items": { - "type": "string" + "select_exp": {"type": "array", "items": {"type": "string"}}, + "where_clause": {"type": "array", "items": {"type": "string"}} + }, + "required": ["sources", "joins"], + "additionalProperties": false + }, + + "sqlLogicConfig": { + "type": "object", + "description": "SQL source or transformation. Provide exactly one of sql_path or sql_statement.", + "oneOf": [ + { + "properties": { + "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, + "sql_path": {"type": "string", "description": "Path to a .sql file."} }, - "description": "Configuration flags, e.g. ['disableOperationalMetadata']" - }, - "once": { - "type": "boolean", - "default": false, - "description": "Execute flow to this target only once (batch mode)" + "required": ["sql_path"], + "additionalProperties": false }, - "cdc_settings": { - "$ref": "./definitions_main.json#/definitions/cdcSettings" - }, - "cdc_snapshot_settings": { - "$ref": "#/$defs/snapshotCdcConfig" - }, - "data_quality_expectations_enabled": { - "type": "boolean", - "default": false, - "description": "Enable data quality expectations for this target" - }, - "data_quality_expectations_path": { - "type": "string", - "description": "Path to the data quality expectations file" - }, - "quarantine_mode": { - "type": "string", - "enum": [ - "off", - "flag", - "table" - ], - "default": "off", - "description": "Quarantine mode for this target" - }, - "quarantine_target_details": { - "$ref": "#/$defs/quarantineTargetDetails", - "description": "Quarantine target configuration when quarantine_mode is 'table'" - }, - "table_migration_details": { - "type": "object", - "description": "Table migration configuration", + { "properties": { - "catalog_type": { - "type": "string" - }, - "enabled": { - "type": "boolean" - }, - "auto_starting_versions_enabled": { - "type": "boolean" - }, - "source_details": { - "type": "object" - } - } - }, - "name": { - "type": "string", - "description": "Sink name (for sink target formats)" - }, - "sink_type": { - "type": "string", - "description": "Sink sub-type: basic_sql, python_function" - }, - "sink_config": { - "type": "object", - "description": "Sink-specific configuration (sqlPath, functionPath, tokens, etc.)" - }, - "sink_options": { - "type": "object", - "description": "Delta sink options (path, tableName)" - }, - "sql_path": { - "type": "string", - "description": "SQL file path (for materialized view targets)" - }, - "sql_statement": { - "type": "string", - "description": "Inline SQL statement (for materialized view targets)" - }, - "refresh_policy": { - "type": "string", - "description": "Materialized view refresh policy" - }, + "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, + "sql_statement": {"type": "string", "description": "Inline SQL. May reference upstream views via STREAM(live.)."} + }, + "required": ["sql_statement"], + "additionalProperties": false + } + ] + }, + + "pythonLogicConfig": { + "type": "object", + "description": "Python source or transformation. Provide exactly one of function_path or python_module.", + "properties": { + "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, + "function_path": {"type": "string", "pattern": "\\.py$", "description": "Path to a .py file exposing the function."}, + "python_module": {"type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_.]*\\.[a-zA-Z_][a-zA-Z0-9_]*$", "description": "Dotted module.function reference."}, + "tokens": {"type": "object", "additionalProperties": true, "description": "Tokens passed to the Python callable."} + }, + "oneOf": [ + {"required": ["function_path"]}, + {"required": ["python_module"]} + ], + "additionalProperties": false + }, + + "targetDeltaConfig": { + "type": "object", + "description": "Delta target (target_type=delta). A streaming table by default; set table_type=mv for a materialized view (then supply sql_path/sql_statement or an input_flows source).", + "properties": { + "input_flows": {"$ref": "#/$defs/inputFlows"}, + "table": {"type": "string", "description": "Target table name."}, + "database": {"type": "string", "description": "Target schema/database."}, + "table_type": {"type": "string", "enum": ["st", "mv"], "default": "st", "description": "st (streaming table) or mv (materialized view). Default: st."}, + "schema_path": {"type": "string", "pattern": "\\.(json|ddl)$"}, + "path": {"type": "string", "description": "External location for the table."}, + "table_properties": {"type": "object", "additionalProperties": {"type": "string"}, "description": "Delta table properties."}, + "partition_columns": {"type": "array", "items": {"type": "string"}}, + "cluster_by_columns": {"type": "array", "items": {"type": "string"}}, + "cluster_by_auto": {"type": "boolean", "description": "Enable Delta automatic liquid clustering."}, + "comment": {"type": "string"}, + "spark_conf": {"type": "object"}, + "row_filter": {"type": "string"}, + "config_flags": {"type": "array", "items": {"type": "string", "enum": ["disableOperationalMetadata"]}}, + "once": {"type": "boolean", "default": false, "description": "Run the flow to this target once (batch), then stop."}, + "cdc_settings": {"$ref": "./definitions_main.json#/definitions/cdcSettings"}, + "cdc_snapshot_settings": {"$ref": "#/$defs/snapshotCdcConfig"}, + "table_migration_details": {"$ref": "#/$defs/tableMigrationDetails"}, + "sql_path": {"type": "string", "description": "MV only: path to a .sql file defining the view."}, + "sql_statement": {"type": "string", "description": "MV only: inline SQL defining the view."}, + "refresh_policy": {"type": "string", "enum": ["auto", "incremental", "incremental_strict", "full"], "description": "MV only: refresh policy."}, "table_details": { "type": "object", - "description": "Additional table details (for materialized view targets)", + "description": "MV only: additional table details.", "properties": { - "private": { - "type": "boolean" - }, - "comment": { - "type": "string" - }, - "spark_conf": { - "type": "object" - }, - "config_flags": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - }, - "additionalProperties": false, - "allOf": [ - { - "if": { - "properties": { - "data_quality_expectations_enabled": { - "const": true - } - }, - "required": [ - "data_quality_expectations_enabled" - ] + "private": {"type": "boolean"}, + "comment": {"type": "string"}, + "spark_conf": {"type": "object"}, + "config_flags": {"type": "array", "items": {"type": "string", "enum": ["disableOperationalMetadata"]}} }, - "then": { - "required": [ - "data_quality_expectations_path" - ] - } + "additionalProperties": false }, - { - "if": { + "data_quality": {"$ref": "#/$defs/dataQuality"}, + "quarantine": {"$ref": "#/$defs/quarantine"} + }, + "required": ["table"], + "additionalProperties": false + }, + + "targetSinkConfig": { + "type": "object", + "description": "Sink target (target_type = delta_sink / kafka_sink / foreach_batch_sink / custom_python_sink). NOTE: the sink redesign is DEFERRED — sink fields remain in their current flat form (sink_type / sink_config / sink_options) rather than a nested `sink` object. Revisit when the sink redesign is taken up.", + "properties": { + "input_flows": {"$ref": "#/$defs/inputFlows"}, + "name": {"type": "string", "description": "Sink name."}, + "sink_type": {"type": "string", "description": "Sink sub-type, e.g. basic_sql or python_function (foreach_batch_sink)."}, + "sink_config": {"type": "object", "additionalProperties": true, "description": "Sink-specific configuration (e.g. sql_path, function_path, tokens)."}, + "sink_options": {"type": "object", "additionalProperties": true, "description": "Sink options (e.g. table_name/path for delta_sink; topic/bootstrap servers for kafka_sink)."} + }, + "additionalProperties": false + }, + + "inputFlows": { + "type": "array", + "description": "Upstream views feeding this target. Each item is a node name (flow auto-named) or {view, flow} to pin the SDP flow name.", + "items": { + "oneOf": [ + {"type": "string", "description": "Upstream node name; SDP flow name auto-generated."}, + { + "type": "object", "properties": { - "quarantine_mode": { - "const": "table" - } + "view": {"type": "string", "description": "Upstream node name feeding this target."}, + "flow": {"type": "string", "description": "Explicit SDP flow name. Pin to keep it stable (renaming forces a full refresh)."} }, - "required": [ - "quarantine_mode" - ] - }, - "then": { - "required": [ - "quarantine_target_details" - ] + "required": ["view"], + "additionalProperties": false } + ] + } + }, + + "dataQuality": { + "type": "object", + "description": "Data quality expectations. Presence enables expectations (no separate enabled flag).", + "properties": { + "expectations_path": {"type": "string", "description": "Path to the expectations file."} + }, + "required": ["expectations_path"], + "additionalProperties": false + }, + + "quarantine": { + "type": "object", + "description": "Quarantine for rows failing expectations. Omit the whole object for no quarantine.", + "properties": { + "mode": {"type": "string", "enum": ["flag", "table"], "description": "flag: mark rows in place; table: route bad rows to a side table."}, + "target": {"$ref": "#/$defs/quarantineTarget", "description": "Quarantine table config (required when mode=table)."} + }, + "required": ["mode"], + "additionalProperties": false, + "if": {"properties": {"mode": {"const": "table"}}, "required": ["mode"]}, + "then": {"required": ["target"]} + }, + + "quarantineTarget": { + "type": "object", + "properties": { + "target_format": {"type": "string", "enum": ["delta"], "default": "delta"}, + "table": {"type": "string"}, + "database": {"type": "string"}, + "table_properties": {"type": "object", "additionalProperties": {"type": "string"}}, + "path": {"type": "string"}, + "partition_columns": {"type": "array", "items": {"type": "string"}}, + "cluster_by_columns": {"type": "array", "items": {"type": "string"}}, + "cluster_by_auto": {"type": "boolean"} + }, + "additionalProperties": false + }, + + "tableMigrationDetails": { + "type": "object", + "description": "One-time migration of an existing table into this pipeline.", + "properties": { + "enabled": {"type": "boolean"}, + "auto_starting_versions_enabled": {"type": "boolean", "default": true}, + "catalog_type": {"type": "string", "enum": ["hms", "uc"]}, + "source_details": { + "type": "object", + "properties": { + "database": {"type": "string"}, + "table": {"type": "string"}, + "select_exp": {"type": "array", "items": {"type": "string"}}, + "where_clause": {"type": "array", "items": {"type": "string"}}, + "except_columns": {"type": "array", "items": {"type": "string"}} + }, + "required": ["database", "table"], + "additionalProperties": false } - ] + }, + "required": ["enabled", "catalog_type", "source_details"], + "additionalProperties": false }, + "snapshotCdcConfig": { "type": "object", - "description": "Snapshot CDC configuration for a target (snake_case). The framework normalises these keys to the backend's camelCase at build time.", + "description": "Snapshot CDC config for a target. Historical snapshots read files/tables directly (no source node); periodic snapshots read from an upstream source.", "properties": { - "keys": { - "type": "array", - "items": { "type": "string" } - }, - "scd_type": { - "type": ["string", "integer"] - }, - "snapshot_type": { - "type": "string", - "enum": ["historical", "periodic"], - "description": "Historical snapshots read files/tables directly and need no source node; periodic snapshots read from a source." - }, - "source_type": { - "type": "string", - "enum": ["file", "table"], - "description": "Historical snapshot source kind: 'file' (read snapshot files) or 'table' (read a versioned table). Not used for periodic snapshots." - }, + "keys": {"type": "array", "items": {"type": "string"}}, + "scd_type": {"type": ["string", "integer"]}, + "snapshot_type": {"type": "string", "enum": ["historical", "periodic"]}, + "source_type": {"type": "string", "enum": ["file", "table"], "description": "Historical source kind. Not used for periodic."}, "source": { "type": "object", - "description": "Historical snapshot source specification — the source is built here (not via a source node). Fields depend on source_type. Periodic snapshots omit this and read from an upstream source node.", + "description": "Historical snapshot source. Fields depend on source_type. Periodic snapshots omit this.", "properties": { - "table": { - "type": "string", - "description": "Source table (source_type 'table'), e.g. '{schema}.snapshot_source'." - }, - "path": { - "type": "string", - "description": "Snapshot file path/pattern (source_type 'file'); may contain a {version} token." - }, - "format": { - "type": "string", - "description": "File format for file sources (e.g. 'csv', 'parquet', 'json')." - }, - "reader_options": { - "type": "object", - "additionalProperties": true, - "description": "Reader options passed to the file reader (e.g. {\"header\": \"true\"})." - }, - "schema_path": { - "type": "string", - "description": "Path to a schema file applied when reading file sources." - }, - "select_exp": { - "type": "array", - "items": { "type": "string" }, - "description": "Select expressions applied to the snapshot source." - }, - "version_type": { - "type": "string", - "description": "How snapshot versions are interpreted (e.g. 'timestamp', 'integer')." - }, - "version_column": { - "type": "string", - "description": "Column carrying the snapshot version (source_type 'table')." - }, - "datetime_format": { - "type": "string", - "description": "Datetime format used to parse the {version} token for timestamp-versioned file sources (e.g. '%Y_%m_%d')." - }, - "deduplicate_mode": { - "type": "string", - "description": "Deduplication mode applied to snapshot rows." - }, - "recursiveFileLookup": { - "type": ["boolean", "string"], - "description": "Recurse into subdirectories when discovering snapshot files." - } + "table": {"type": "string"}, + "path": {"type": "string"}, + "format": {"type": "string"}, + "reader_options": {"type": "object", "additionalProperties": true}, + "schema_path": {"type": "string"}, + "select_exp": {"type": "array", "items": {"type": "string"}}, + "version_type": {"type": "string"}, + "version_column": {"type": "string"}, + "starting_version": {"type": ["string", "integer"]}, + "datetime_format": {"type": "string"}, + "deduplicate_mode": {"type": "string"}, + "recursiveFileLookup": {"type": ["boolean", "string"]} }, "additionalProperties": true }, - "track_history_column_list": { - "type": "array", - "items": { "type": "string" } - }, - "track_history_except_column_list": { - "type": "array", - "items": { "type": "string" } - } + "track_history_column_list": {"type": "array", "items": {"type": "string"}}, + "track_history_except_column_list": {"type": "array", "items": {"type": "string"}} }, "required": ["keys", "scd_type", "snapshot_type"], "additionalProperties": true }, - "quarantineTargetDetails": { + + "readerOptionsCloudFiles": { "type": "object", - "description": "Quarantine target configuration (snake_case).", + "description": "Auto Loader options. cloudFiles.format is required; format-specific options are offered once it is set.", "properties": { - "target_format": { - "type": "string" - }, - "table": { - "type": "string" - }, - "database": { - "type": "string" - }, - "path": { - "type": "string" - }, - "cluster_by_auto": { - "type": "boolean" - }, - "cluster_by_columns": { - "type": "array", - "items": { "type": "string" } + "cloudFiles.format": {"type": "string", "enum": ["avro", "binaryFile", "csv", "json", "orc", "parquet", "text", "xml"]}, + "cloudFiles.inferColumnTypes": {"type": "string", "enum": ["true", "false"], "default": "false"}, + "cloudFiles.schemaEvolutionMode": {"type": "string", "enum": ["addNewColumns", "rescue", "failOnNewColumns"]}, + "cloudFiles.schemaHints": {"type": "string"}, + "cloudFiles.schemaLocation": {"type": "string"}, + "cloudFiles.maxFilesPerTrigger": {"type": "string", "default": "1000"}, + "cloudFiles.maxBytesPerTrigger": {"type": "string"}, + "cloudFiles.includeExistingFiles": {"type": "string", "enum": ["true", "false"], "default": "true"}, + "cloudFiles.allowOverwrites": {"type": "string", "enum": ["true", "false"], "default": "false"}, + "cloudFiles.useStrictGlobber": {"type": "string", "enum": ["true", "false"], "default": "false"}, + "cloudFiles.validateOptions": {"type": "string", "enum": ["true", "false"], "default": "false"}, + "pathGlobFilter": {"type": "string"}, + "recursiveFileLookup": {"type": "string", "enum": ["true", "false"], "default": "false"} + }, + "required": ["cloudFiles.format"], + "additionalProperties": true, + "allOf": [ + { + "if": {"properties": {"cloudFiles.format": {"const": "csv"}}, "required": ["cloudFiles.format"]}, + "then": {"properties": { + "header": {"type": "string", "enum": ["true", "false"], "default": "true"}, + "inferSchema": {"type": "string", "enum": ["true", "false"], "default": "false"}, + "sep": {"type": "string", "default": ","}, + "quote": {"type": "string", "default": "\""}, + "mode": {"type": "string", "enum": ["PERMISSIVE", "DROPMALFORMED", "FAILFAST"], "default": "PERMISSIVE"}, + "multiLine": {"type": "string", "enum": ["true", "false"], "default": "false"}, + "rescuedDataColumn": {"type": "string", "default": "_rescued_data"} + }} }, - "partition_columns": { - "type": "array", - "items": { "type": "string" } + { + "if": {"properties": {"cloudFiles.format": {"const": "json"}}, "required": ["cloudFiles.format"]}, + "then": {"properties": { + "mode": {"type": "string", "enum": ["PERMISSIVE", "DROPMALFORMED", "FAILFAST"], "default": "PERMISSIVE"}, + "multiLine": {"type": "string", "enum": ["true", "false"], "default": "false"}, + "rescuedDataColumn": {"type": "string", "default": "_rescued_data"} + }} } + ] + }, + + "readerOptionsKafka": { + "type": "object", + "description": "Kafka reader options.", + "properties": { + "subscribe": {"type": "string"}, + "kafka.bootstrap.servers": {"type": "string"}, + "kafka.group.id": {"type": "string"}, + "kafka.security.protocol": {"type": "string", "default": "SASL_SSL"}, + "kafka.sasl.mechanism": {"type": "string", "default": "PLAIN"}, + "kafka.ssl.truststore.location": {"type": "string"}, + "kafka.ssl.truststore.password": {"type": "string"}, + "kafka.ssl.keystore.location": {"type": "string"}, + "kafka.ssl.keystore.password": {"type": "string"}, + "startingOffsets": {"type": "string"}, + "endingOffsets": {"type": "string"}, + "minPartitions": {"type": "string"}, + "failOnDataLoss": {"type": "string"} }, - "required": ["target_format"], + "required": ["kafka.bootstrap.servers"], "additionalProperties": true } } From b310e8ef7d38435d3b2170a3d366d64458ac0476 Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 10 Jul 2026 11:16:09 +1000 Subject: [PATCH 10/19] docs(nodespec): remove internal feature-samples changes notes docs/nodespec_feature_samples_changes.md was an internal implementation/notes doc that shouldn't live in the public repo. The authoritative user-facing reference remains docs/source/dataflow_spec_ref_main_nodespec.rst and the decision record docs/decisions/0008-unified-nodespec-dataflow-spec.md. Co-authored-by: Isaac --- docs/nodespec_feature_samples_changes.md | 242 ----------------------- 1 file changed, 242 deletions(-) delete mode 100644 docs/nodespec_feature_samples_changes.md diff --git a/docs/nodespec_feature_samples_changes.md b/docs/nodespec_feature_samples_changes.md deleted file mode 100644 index edc898f..0000000 --- a/docs/nodespec_feature_samples_changes.md +++ /dev/null @@ -1,242 +0,0 @@ -# Nodespec Feature Samples — Changes, Design & Considerations - -> **Note:** this is a historical design/implementation record. Some field names -> shown below reflect an earlier iteration; the current authoring syntax is in -> `docs/source/dataflow_spec_ref_main_nodespec.rst`. In particular: targets wire -> inputs via **`input_flows`**; data quality is a nested **`data_quality`** object; -> quarantine is a nested **`quarantine`** object; and `data_flow_type` is optional -> (nodespec is the default). - -## Overview - -The Nodespec dataflow spec type introduces a **node-based graph architecture** for defining data pipelines. Instead of the traditional flat (standard) or pre-built flow (flow) spec formats, Nodespec expresses pipelines as directed graphs of **source**, **transformation**, and **target** nodes connected through `input_flows` arrays (on target nodes). - -This document covers the changes made to support all 33 feature samples from the bronze_sample in the Nodespec format, the migration tooling created, and the design decisions and assumptions behind the implementation. - ---- - -## What Changed - -### 1. Transformer Extensions (`src/dataflow_spec_builder/transformer/nodespec.py`) - -The Nodespec transformer was extended to support all feature types: - -| Feature | Change | -|---------|--------| -| **Removed `isPrimary`** | No primary/secondary target concept. The transformer auto-selects one target for backend `targetDetails` from graph topology (terminal node detection). This is an internal detail invisible to spec authors. | -| **`targetFormat` on targets** | Targets can specify `targetFormat: "foreach_batch_sink"`, `"delta_sink"`, or `"custom_python_sink"` alongside `sinkType` and `sinkConfig` for non-delta outputs. | -| **`tableMigrationDetails`** | Targets can include table migration config, which the transformer copies to the spec level for the backend. | -| **`configFlags`** | Targets can specify `configFlags: ["disableOperationalMetadata"]`, propagated to `targetDetails`. | -| **`once` flag** | Targets can set `once: true` to execute the flow only once (batch mode). Added to `flowDetails` in the output. | -| **`append_sql` flow type** | When a source node has `sourceType: "sql"`, the transformer creates an `append_sql` flow with the SQL directly in `flowDetails` (no intermediate view). | -| **`cdcSnapshotSettings`** | Snapshot targets (periodic and historical) now correctly trigger MERGE flow type. Historical file/table snapshots work without source nodes since the CDC snapshot system reads files/tables directly. | -| **`cdcApplyChanges` alias** | Treated identically to `cdcSettings` for backward compatibility with version mapping specs. | -| **`dataFlowVersion`** | Passed through from spec level to output. | -| **Materialized view targets** | Targets with `tableType: "mv"` produce separate MV flow specs. Source views for MVs are forced to batch mode (MVs use `spark.sql()` which can't read streaming views). Returns `List[Dict]` for multi-MV specs. | -| **Targets without inputs** | Targets with `cdcSnapshotSettings` and no `inputs` create a placeholder MERGE flow. The snapshot system handles file/table reading directly. | - -### 2. Schema Extensions (`src/schemas/spec_nodespec.json`) - -- Removed `isPrimary` from the node schema -- Added to `targetNodeConfig`: `targetFormat`, `tableType`, `tableMigrationDetails`, `configFlags`, `once`, `name`, `sinkType`, `sinkConfig`, `sinkOptions`, `sqlPath`, `sqlStatement`, `sourceView`, `refreshPolicy`, `tableDetails`, `cdcApplyChanges` -- Added to `sourceNodeConfig`: `functionPath`, `pythonModule`, `tokens`, `sqlPath`, `sqlStatement`, `startingVersionFromDLTSetup`, `cdfChangeTypeOverride` - -### 3. Migration Script (`scripts/migrate_to_nodespec.py`) - -CLI tool that converts any existing spec to Nodespec format: - -```bash -# Single file -python scripts/migrate_to_nodespec.py spec_main.json --output nodespec_spec.json - -# Entire directory -python scripts/migrate_to_nodespec.py ./dataflowspec/ --output-dir ./nodespec_dataflowspec/ -``` - -Handles all three spec types: -- **Standard** → source node (from sourceDetails) + target node (from targetDetails + spec-level CDC/DQ/quarantine) -- **Flow** → source nodes (from views) + target nodes (from staging tables + main target) with connections from flow inputs -- **Materialized View** → target nodes with `tableType: "mv"`, optional source nodes for sourceView patterns - -### 4. Feature Samples (`samples/nodespec_sample/src/dataflows/feature_samples/`) - -All 33 specs translated with supporting files (schemas, expectations, SQL, Python functions) copied from `bronze_sample`. Each spec uses `dataFlowGroup: "nodespec_feature_samples_*"` prefixes. - -### 5. Pipeline Resources (`samples/nodespec_sample/resources/serverless/`) - -Six new pipeline definitions: -- `nodespec_feature_samples_general_pipeline.yml` — 9 specs -- `nodespec_feature_samples_data_quality_pipeline.yml` — 2 specs -- `nodespec_feature_samples_snapshots_pipeline.yml` — 11 specs -- `nodespec_feature_samples_python_pipeline.yml` — 4 specs -- `nodespec_feature_samples_table_migration_pipeline.yml` — 2 specs -- `nodespec_feature_samples_materialized_views_pipeline.yml` — 6 MVs - -### 6. Framework Bug Fix (`src/dlt_pipeline_builder.py`) - -`table_migration_state_volume_path` in `global.json` was hardcoded with `_es` suffix. Changed to use `{logical_env}` token and moved the initialization to after the `SubstitutionManager` is ready so the token gets resolved. - ---- - -## How Nodespec Works - -### Node Graph Model - -A Nodespec spec defines a pipeline as a graph of nodes: - -``` -Source Nodes ──→ Transformation Nodes ──→ Target Nodes -(data origins) (SQL/Python logic) (output tables) -``` - -Each node has: -- `id` — unique identifier -- `type` — `source`, `transformation`, or `target` -- `inputs` — array of node IDs this node reads from (source nodes have none) -- `config` — type-specific configuration - -### Transformation Pipeline - -``` -Nodespec JSON Spec - │ - ▼ -NodespecSpecTransformer._process_spec() - │ - ├─ Categorize nodes by type - ├─ Validate graph (references, cycles, required nodes) - ├─ Detect internal sources (source reading from target in same spec) - ├─ Auto-select spec target (terminal node in graph) - ├─ Build flow spec: - │ ├─ targetDetails from spec target - │ ├─ CDC/DQ/quarantine from spec target → spec level - │ ├─ Other targets → stagingTables - │ └─ Flows from node connections → flowGroups - ▼ -Lakeflow Framework Flow Spec (same format as standard/flow transformers) - │ - ▼ -Backend (DataFlow, FlowGroup, CDC, etc.) -``` - -### Spec Target Selection - -With no `isPrimary` flag, the transformer auto-selects which target becomes `targetDetails`: - -1. Build set of "consumed" targets — any target whose ID appears in another node's inputs, or whose table name is referenced by a source node -2. Terminal targets = targets that are NOT consumed -3. If multiple terminal targets exist, use the last one (with a warning) -4. All other targets become staging tables - -For single-target specs (the majority), the only target is automatically selected. - -### Internal Source Detection - -When a source node reads from a table that's also a target in the same spec, it's treated as "internal": -- The source references the staging table directly (no view created) -- Database is forced to `live` so DLT resolves it during pipeline analysis -- This enables staging patterns: source → staging_target → internal_source → final_target - -### Flow Type Determination - -| Condition | Flow Type | -|-----------|-----------| -| Target has `cdcSettings` or `cdcApplyChanges` | `merge` | -| Spec target has CDC and flow writes to spec target | `merge` | -| Target has `cdcSnapshotSettings` | `merge` | -| Source is SQL type (`sourceType: "sql"`) | `append_sql` | -| Otherwise | `append_view` | - ---- - -## Design Decisions & Assumptions - -### 1. No Primary Target Concept - -**Decision:** The spec author never needs to think about which target is "primary." All targets are equal — they each specify their own CDC, DQ, quarantine, and table migration settings directly on the target node config. - -**Why:** The "primary target" was a backend implementation detail leaking into the spec format. The backend requires `targetDetails` at the spec level, but this is now handled transparently by the transformer. - -**Trade-off:** The auto-selection heuristic (terminal node detection) could pick the wrong target in ambiguous multi-target specs. A warning is logged in this case. - -### 2. Per-Target Settings at Spec Level - -**Decision:** The spec target's CDC/DQ/quarantine settings are duplicated at the spec level for backend compatibility. - -**Assumption:** The backend reads these settings from the spec level for the main target. This duplication is an interim measure — the backend should eventually read all settings from target nodes directly. - -### 3. Materialized Views as Target Nodes - -**Decision:** MVs are target nodes with `tableType: "mv"`. Each MV target in a spec becomes a separate flow spec (the transformer returns `List[Dict]`). - -**Why:** MVs are fundamentally different from streaming tables — they use `spark.sql()` batch reads and have different creation ordering (flows first, then MV). Treating them as a target type keeps the node graph model consistent. - -**Constraint:** MV source views are forced to batch mode because DLT's `spark.sql()` cannot read from streaming views. - -### 4. SQL Sources → `append_sql` Flow Type - -**Decision:** When a source node has `sourceType: "sql"`, the transformer creates an `append_sql` flow with the SQL directly in `flowDetails`, bypassing view creation. - -**Why:** `append_sql` is a distinct flow type in the framework that handles DLT-specific SQL (like `STREAM()` functions) directly. Using `append_view` with an SQL source view would work for simple cases but wouldn't match the original spec behavior. - -### 5. Snapshot Targets Without Source Nodes - -**Decision:** Targets with `cdcSnapshotSettings` don't require source nodes. Historical file/table snapshots define their data source entirely within the snapshot settings. - -**Why:** The CDC snapshot system reads files/tables directly based on `cdcSnapshotSettings.source`. A traditional source node would create a view with nothing to read from. The transformer creates a placeholder MERGE flow with no `sourceView`. - -### 6. Migration Script Assumptions - -- The script preserves `dataFlowId` and `dataFlowGroup` from the original spec (the group is prefixed with `nodespec_` separately) -- Source node IDs are derived from `sourceViewName` (stripping `v_` prefix) -- Target node IDs follow the pattern `target_{table_name}` -- For flow specs, view names from the original spec become source node IDs directly -- The script doesn't validate the output against the JSON schema — the transformer handles validation at runtime - ---- - -## Known Limitations - -### 1. CDC Merge + Quarantine "table" Mode - -**Status:** Fixed. The `_get_exclude_columns` method in `dataflow.py` no longer adds `is_quarantined` to the exclude list for TABLE quarantine mode. - -**Root Cause (was):** When `quarantineMode: "table"` and `cdcSettings` coexist, the framework added `is_quarantined` to the flow's `exclude_columns` list. However, in table quarantine mode, the `is_quarantined` column is NOT added to the source view (only a separate quarantine view gets it). This created a column resolution failure when the CDC merge flow tried to reference it via `except_column_list`. - -**Fix:** Removed the logic that added `is_quarantined` to `exclude_columns` for TABLE mode. In table quarantine mode, the column never exists in the source view — the separate quarantine view/flow handles it independently. In FLAG mode, the column is part of the target schema and written through, so no exclusion is needed there either. - -### 2. Delta Sink Specs - -**Status:** Not tested (disabled in the original bronze_sample too). - -The `sink_delta_path_table` and `sink_delta_uc_table` specs have `dataFlowGroup: "nodespec_feature_samples_general_DISABLED"` — they're excluded from all pipelines, matching the original behavior. - ---- - -## Test Results - -| Pipeline | Flows | Status | -|----------|-------|--------| -| General | append_sql, append_view, append_view_once, ddl_schema, version_mapping (standard+flows), foreach_batch (sql+python) | 9/9 passed | -| Data Quality | quarantine_flag, quarantine_table | 3/3 passed | -| Snapshots | 7 historical file, 2 historical table, 1 historical flow, 2 periodic | 13/13 passed | -| Python | python_source, python_source_extension, python_transform, python_transform_extension | 4/4 passed | -| Table Migration | append_only, scd2 (with import flows) | 4/4 passed | -| Materialized Views | 6 MVs (source_view, sql_path, sql_statement, quarantine, chained, refresh_policy) | 8/8 passed | - ---- - -## Files Changed - -| File | Action | -|------|--------| -| `src/dataflow_spec_builder/transformer/nodespec.py` | Extended transformer with all feature support | -| `src/schemas/spec_nodespec.json` | Extended schema, removed isPrimary | -| `src/dlt_pipeline_builder.py` | Moved table_migration init after SubstitutionManager | -| `samples/bronze_sample/src/pipeline_configs/global.json` | Changed hardcoded `_es` to `{logical_env}` token | -| `scripts/migrate_to_nodespec.py` | New migration script | -| `samples/nodespec_sample/src/dataflows/feature_samples/` | 33 translated specs + supporting files | -| `samples/nodespec_sample/src/extensions/` | Copied Python extension modules | -| `samples/nodespec_sample/src/pipeline_configs/global.json` | Added table_migration_state_volume_path | -| `samples/nodespec_sample/resources/serverless/` | 6 new pipeline resource YAMLs | -| `samples/nodespec_sample/databricks.yml` | Added serverless resource includes | From f05dab41a6e4c3b77aba47acd16175307ec04289 Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 10 Jul 2026 13:45:37 +1000 Subject: [PATCH 11/19] refactor(nodespec schema): snake_case the internal $defs names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nodespec schema's $defs identifiers (sourceDeltaConfig, dataQuality, inputFlows, …) were camelCase while everything a nodespec spec author writes is snake_case. These names are schema-internal ref targets — they never appear in an authored spec — but renaming them to snake_case makes the nodespec schema fully self-consistent. - Renamed all nodespec $defs keys and their internal #/$defs refs to snake_case. - Updated main.json's external ref (#/$defs/nodespecSpec -> nodespec_spec). - The external ref out to definitions_main.json#/definitions/cdcSettings is unchanged (that def's name is definitions_main's concern). Legacy standardSpec/ flowsSpec/materializedViewsSpec refs are unaffected. No behavioural change: authored specs and the transformer are untouched; 37/37 nodespec samples still validate. Co-authored-by: Isaac --- src/schemas/main.json | 2 +- src/schemas/spec_nodespec.json | 88 +++++++++++++++++----------------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/src/schemas/main.json b/src/schemas/main.json index c2bd41e..1f09697 100644 --- a/src/schemas/main.json +++ b/src/schemas/main.json @@ -44,5 +44,5 @@ }, "required": ["dataFlowId", "dataFlowGroup", "dataFlowType"] }, - "else": { "$ref": "./spec_nodespec.json#/$defs/nodespecSpec" } + "else": { "$ref": "./spec_nodespec.json#/$defs/nodespec_spec" } } diff --git a/src/schemas/spec_nodespec.json b/src/schemas/spec_nodespec.json index 7640dfd..54dcda5 100644 --- a/src/schemas/spec_nodespec.json +++ b/src/schemas/spec_nodespec.json @@ -5,7 +5,7 @@ "description": "Node-based dataflow spec (snake_case). A node's `config` is discriminated by node_type + source_type/target_type/table_type so editors offer only the fields valid for that node. Feature bundles (cdc, snapshot, data_quality, quarantine, migration) are single nested objects. Targets declare their inputs via `input_flows`. Sink targets keep their current flat form for now (sink redesign deferred).", "$comment": "A node's `config` is discriminated at the node level via allOf/if-then keyed on node_type + source_type/target_type/table_type; each `then` selects one closed config $def, so editors offer only the fields valid for that node. Nested blobs are typed objects with descriptions. Sink targets keep their current flat form (sink_type/sink_config/sink_options) pending the sink redesign. Legacy definitions_sources.json / definitions_targets.json are camelCase; nodespec is snake_case, so source/target detail shapes are re-declared here. Literal Spark option keys (cloudFiles.*, kafka.*) are preserved verbatim.", "$defs": { - "nodespecSpec": { + "nodespec_spec": { "type": "object", "description": "A nodespec pipeline: a graph of source -> transformation -> target nodes.", "properties": { @@ -63,31 +63,31 @@ "allOf": [ { "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "delta"}}, "required": ["node_type", "source_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/sourceDeltaConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/source_delta_config"}}} }, { "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "cloudFiles"}}, "required": ["node_type", "source_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/sourceCloudFilesConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/source_cloud_files_config"}}} }, { "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "batchFiles"}}, "required": ["node_type", "source_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/sourceBatchFilesConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/source_batch_files_config"}}} }, { "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "kafka"}}, "required": ["node_type", "source_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/sourceKafkaConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/source_kafka_config"}}} }, { "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "deltaJoin"}}, "required": ["node_type", "source_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/sourceDeltaJoinConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/source_delta_join_config"}}} }, { "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "python"}}, "required": ["node_type", "source_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/pythonLogicConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/python_logic_config"}}} }, { "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "sql"}}, "required": ["node_type", "source_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/sqlLogicConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/sql_logic_config"}}} }, { "if": {"properties": {"node_type": {"const": "source"}}, "required": ["node_type"]}, @@ -96,11 +96,11 @@ { "if": {"properties": {"node_type": {"const": "transformation"}, "transformation_type": {"const": "sql"}}, "required": ["node_type", "transformation_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/sqlLogicConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/sql_logic_config"}}} }, { "if": {"properties": {"node_type": {"const": "transformation"}, "transformation_type": {"const": "python"}}, "required": ["node_type", "transformation_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/pythonLogicConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/python_logic_config"}}} }, { "if": {"properties": {"node_type": {"const": "transformation"}}, "required": ["node_type"]}, @@ -109,16 +109,16 @@ { "if": {"properties": {"node_type": {"const": "target"}, "target_type": {"const": "delta"}}, "required": ["node_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/targetDeltaConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/target_delta_config"}}} }, { "if": {"properties": {"node_type": {"const": "target"}, "target_type": {"enum": ["delta_sink", "kafka_sink", "foreach_batch_sink", "custom_python_sink"]}}, "required": ["node_type", "target_type"]}, - "then": {"properties": {"config": {"$ref": "#/$defs/targetSinkConfig"}}} + "then": {"properties": {"config": {"$ref": "#/$defs/target_sink_config"}}} } ] }, - "sourceCommon": { + "source_common": { "type": "object", "properties": { "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream", "description": "Read mode. Default: stream."}, @@ -128,7 +128,7 @@ } }, - "pythonTransform": { + "python_transform": { "type": "object", "description": "Inline Python transform applied to a source. Provide function_path or python_module (module alias also accepted).", "properties": { @@ -140,10 +140,10 @@ "additionalProperties": false }, - "sourceDeltaConfig": { + "source_delta_config": { "type": "object", "description": "Delta table source.", - "allOf": [{"$ref": "#/$defs/sourceCommon"}], + "allOf": [{"$ref": "#/$defs/source_common"}], "properties": { "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, "database": {"type": "string", "description": "Source schema/database."}, @@ -156,29 +156,29 @@ "where_clause": {"type": "array", "items": {"type": "string"}}, "schema_path": {"type": "string", "pattern": "\\.(json|ddl)$"}, "reader_options": {"type": "object", "additionalProperties": true}, - "python_transform": {"$ref": "#/$defs/pythonTransform"} + "python_transform": {"$ref": "#/$defs/python_transform"} }, "required": ["table"], "additionalProperties": false }, - "sourceCloudFilesConfig": { + "source_cloud_files_config": { "type": "object", "description": "Auto Loader (cloudFiles) source.", "properties": { "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, "path": {"type": "string", "description": "Directory or glob to ingest."}, - "reader_options": {"$ref": "#/$defs/readerOptionsCloudFiles"}, + "reader_options": {"$ref": "#/$defs/reader_options_cloud_files"}, "select_exp": {"type": "array", "items": {"type": "string"}}, "where_clause": {"type": "array", "items": {"type": "string"}}, "schema_path": {"type": "string", "pattern": "\\.json$"}, - "python_transform": {"$ref": "#/$defs/pythonTransform"} + "python_transform": {"$ref": "#/$defs/python_transform"} }, "required": ["path", "reader_options"], "additionalProperties": false }, - "sourceBatchFilesConfig": { + "source_batch_files_config": { "type": "object", "description": "Batch file source (read once, not streamed).", "properties": { @@ -189,28 +189,28 @@ "select_exp": {"type": "array", "items": {"type": "string"}}, "where_clause": {"type": "array", "items": {"type": "string"}}, "schema_path": {"type": "string", "pattern": "\\.json$"}, - "python_transform": {"$ref": "#/$defs/pythonTransform"} + "python_transform": {"$ref": "#/$defs/python_transform"} }, "required": ["path", "reader_options"], "additionalProperties": false }, - "sourceKafkaConfig": { + "source_kafka_config": { "type": "object", "description": "Kafka source.", "properties": { "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, - "reader_options": {"$ref": "#/$defs/readerOptionsKafka"}, + "reader_options": {"$ref": "#/$defs/reader_options_kafka"}, "select_exp": {"type": "array", "items": {"type": "string"}}, "where_clause": {"type": "array", "items": {"type": "string"}}, "schema_path": {"type": "string", "pattern": "\\.json$"}, - "python_transform": {"$ref": "#/$defs/pythonTransform"} + "python_transform": {"$ref": "#/$defs/python_transform"} }, "required": ["reader_options"], "additionalProperties": false }, - "sourceDeltaJoinConfig": { + "source_delta_join_config": { "type": "object", "description": "Join of multiple Delta sources into one view.", "properties": { @@ -230,7 +230,7 @@ "where_clause": {"type": "array", "items": {"type": "string"}}, "schema_path": {"type": "string", "pattern": "\\.json$"}, "reader_options": {"type": "object", "additionalProperties": true}, - "python_transform": {"$ref": "#/$defs/pythonTransform"} + "python_transform": {"$ref": "#/$defs/python_transform"} }, "required": ["database", "table", "alias", "cdf_enabled", "join_mode"], "additionalProperties": false @@ -255,7 +255,7 @@ "additionalProperties": false }, - "sqlLogicConfig": { + "sql_logic_config": { "type": "object", "description": "SQL source or transformation. Provide exactly one of sql_path or sql_statement.", "oneOf": [ @@ -278,7 +278,7 @@ ] }, - "pythonLogicConfig": { + "python_logic_config": { "type": "object", "description": "Python source or transformation. Provide exactly one of function_path or python_module.", "properties": { @@ -294,11 +294,11 @@ "additionalProperties": false }, - "targetDeltaConfig": { + "target_delta_config": { "type": "object", "description": "Delta target (target_type=delta). A streaming table by default; set table_type=mv for a materialized view (then supply sql_path/sql_statement or an input_flows source).", "properties": { - "input_flows": {"$ref": "#/$defs/inputFlows"}, + "input_flows": {"$ref": "#/$defs/input_flows"}, "table": {"type": "string", "description": "Target table name."}, "database": {"type": "string", "description": "Target schema/database."}, "table_type": {"type": "string", "enum": ["st", "mv"], "default": "st", "description": "st (streaming table) or mv (materialized view). Default: st."}, @@ -314,8 +314,8 @@ "config_flags": {"type": "array", "items": {"type": "string", "enum": ["disableOperationalMetadata"]}}, "once": {"type": "boolean", "default": false, "description": "Run the flow to this target once (batch), then stop."}, "cdc_settings": {"$ref": "./definitions_main.json#/definitions/cdcSettings"}, - "cdc_snapshot_settings": {"$ref": "#/$defs/snapshotCdcConfig"}, - "table_migration_details": {"$ref": "#/$defs/tableMigrationDetails"}, + "cdc_snapshot_settings": {"$ref": "#/$defs/snapshot_cdc_config"}, + "table_migration_details": {"$ref": "#/$defs/table_migration_details"}, "sql_path": {"type": "string", "description": "MV only: path to a .sql file defining the view."}, "sql_statement": {"type": "string", "description": "MV only: inline SQL defining the view."}, "refresh_policy": {"type": "string", "enum": ["auto", "incremental", "incremental_strict", "full"], "description": "MV only: refresh policy."}, @@ -330,18 +330,18 @@ }, "additionalProperties": false }, - "data_quality": {"$ref": "#/$defs/dataQuality"}, + "data_quality": {"$ref": "#/$defs/data_quality"}, "quarantine": {"$ref": "#/$defs/quarantine"} }, "required": ["table"], "additionalProperties": false }, - "targetSinkConfig": { + "target_sink_config": { "type": "object", "description": "Sink target (target_type = delta_sink / kafka_sink / foreach_batch_sink / custom_python_sink). NOTE: the sink redesign is DEFERRED — sink fields remain in their current flat form (sink_type / sink_config / sink_options) rather than a nested `sink` object. Revisit when the sink redesign is taken up.", "properties": { - "input_flows": {"$ref": "#/$defs/inputFlows"}, + "input_flows": {"$ref": "#/$defs/input_flows"}, "name": {"type": "string", "description": "Sink name."}, "sink_type": {"type": "string", "description": "Sink sub-type, e.g. basic_sql or python_function (foreach_batch_sink)."}, "sink_config": {"type": "object", "additionalProperties": true, "description": "Sink-specific configuration (e.g. sql_path, function_path, tokens)."}, @@ -350,7 +350,7 @@ "additionalProperties": false }, - "inputFlows": { + "input_flows": { "type": "array", "description": "Upstream views feeding this target. Each item is a node name (flow auto-named) or {view, flow} to pin the SDP flow name.", "items": { @@ -369,7 +369,7 @@ } }, - "dataQuality": { + "data_quality": { "type": "object", "description": "Data quality expectations. Presence enables expectations (no separate enabled flag).", "properties": { @@ -384,7 +384,7 @@ "description": "Quarantine for rows failing expectations. Omit the whole object for no quarantine.", "properties": { "mode": {"type": "string", "enum": ["flag", "table"], "description": "flag: mark rows in place; table: route bad rows to a side table."}, - "target": {"$ref": "#/$defs/quarantineTarget", "description": "Quarantine table config (required when mode=table)."} + "target": {"$ref": "#/$defs/quarantine_target", "description": "Quarantine table config (required when mode=table)."} }, "required": ["mode"], "additionalProperties": false, @@ -392,7 +392,7 @@ "then": {"required": ["target"]} }, - "quarantineTarget": { + "quarantine_target": { "type": "object", "properties": { "target_format": {"type": "string", "enum": ["delta"], "default": "delta"}, @@ -407,7 +407,7 @@ "additionalProperties": false }, - "tableMigrationDetails": { + "table_migration_details": { "type": "object", "description": "One-time migration of an existing table into this pipeline.", "properties": { @@ -431,7 +431,7 @@ "additionalProperties": false }, - "snapshotCdcConfig": { + "snapshot_cdc_config": { "type": "object", "description": "Snapshot CDC config for a target. Historical snapshots read files/tables directly (no source node); periodic snapshots read from an upstream source.", "properties": { @@ -465,7 +465,7 @@ "additionalProperties": true }, - "readerOptionsCloudFiles": { + "reader_options_cloud_files": { "type": "object", "description": "Auto Loader options. cloudFiles.format is required; format-specific options are offered once it is set.", "properties": { @@ -509,7 +509,7 @@ ] }, - "readerOptionsKafka": { + "reader_options_kafka": { "type": "object", "description": "Kafka reader options.", "properties": { From dc8825e13ded8d6dfd5614da6e050b206bdaa8d1 Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 10 Jul 2026 13:57:27 +1000 Subject: [PATCH 12/19] refactor(nodespec): snake_case the recursive_file_lookup snapshot field recursiveFileLookup was the last camelCase field an author writes in a nodespec spec. Renamed to recursive_file_lookup in the schema and samples; the transformer maps it back to the backend's `recursiveFileLookup` at build time (added to the snake->camel key map), and the conversion script snakes it during migration. Remaining camelCase in the schema is deliberately unchanged: JSON Schema keywords (additionalProperties, allOf, ...), literal Spark/Auto Loader option keys inside reader_options, framework-wide identifier values (source_type cloudFiles/ batchFiles/deltaJoin, disableOperationalMetadata), and the external cdcSettings ref. Co-authored-by: Isaac --- ...apshot_files_datetime_recursive_and_partitioned_main.json | 2 +- ...iles_datetime_recursive_and_partitioned_parquet_main.json | 4 ++-- ..._files_datetime_recursive_and_partitioned_regex_main.json | 2 +- scripts/migrate_to_nodespec.py | 5 +++-- src/dataflow_spec_builder/transformer/nodespec.py | 1 + src/schemas/spec_nodespec.json | 2 +- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json index b658a72..4b95385 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json @@ -30,7 +30,7 @@ }, "version_type": "timestamp", "datetime_format": "YEAR=%Y/MONTH=%m/DAY=%d", - "recursiveFileLookup": true + "recursive_file_lookup": true }, "track_history_except_column_list": [ "LOAD_TIMESTAMP" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json index 98676cb..7b189a7 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json @@ -27,8 +27,8 @@ "path": "{sample_file_location}/snapshot_customer_partitioned_parquet/{version}/customer.parquet", "version_type": "timestamp", "datetime_format": "YEAR=%Y/MONTH=%m/DAY=%d", - "recursiveFileLookup": true, - "deduplicate_mode": "keys_only" + "deduplicate_mode": "keys_only", + "recursive_file_lookup": true }, "track_history_except_column_list": [ "LOAD_TIMESTAMP" diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json index 1e86f51..5a85929 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json @@ -30,7 +30,7 @@ }, "version_type": "timestamp", "datetime_format": "%Y%m%d", - "recursiveFileLookup": true + "recursive_file_lookup": true }, "track_history_except_column_list": [ "LOAD_TIMESTAMP" diff --git a/scripts/migrate_to_nodespec.py b/scripts/migrate_to_nodespec.py index 6a8b2a6..008e480 100644 --- a/scripts/migrate_to_nodespec.py +++ b/scripts/migrate_to_nodespec.py @@ -70,8 +70,8 @@ } # Snapshot settings carry camelCase keys at both the top level and inside the -# nested `source` object. `recursiveFileLookup` is intentionally left camelCase -# (the framework reads it as-is). +# nested `source` object. All are converted to snake_case for nodespec; the +# transformer maps them back to the backend's camelCase at build time. _SNAPSHOT_MAP = { "snapshotType": "snapshot_type", "sourceType": "source_type", @@ -83,6 +83,7 @@ "schemaPath": "schema_path", "selectExp": "select_exp", "deduplicateMode": "deduplicate_mode", + "recursiveFileLookup": "recursive_file_lookup", } _QUARANTINE_MAP = { diff --git a/src/dataflow_spec_builder/transformer/nodespec.py b/src/dataflow_spec_builder/transformer/nodespec.py index 59ccbe5..492ce79 100644 --- a/src/dataflow_spec_builder/transformer/nodespec.py +++ b/src/dataflow_spec_builder/transformer/nodespec.py @@ -40,6 +40,7 @@ "snapshot_type": "snapshotType", "source_type": "sourceType", "version_type": "versionType", "version_column": "versionColumn", "starting_version": "startingVersion", "datetime_format": "datetimeFormat", "deduplicate_mode": "deduplicateMode", + "recursive_file_lookup": "recursiveFileLookup", "catalog_type": "catalogType", "auto_starting_versions_enabled": "autoStartingVersionsEnabled", "source_details": "sourceDetails", "table_name": "tableName", } diff --git a/src/schemas/spec_nodespec.json b/src/schemas/spec_nodespec.json index 54dcda5..d9687cc 100644 --- a/src/schemas/spec_nodespec.json +++ b/src/schemas/spec_nodespec.json @@ -454,7 +454,7 @@ "starting_version": {"type": ["string", "integer"]}, "datetime_format": {"type": "string"}, "deduplicate_mode": {"type": "string"}, - "recursiveFileLookup": {"type": ["boolean", "string"]} + "recursive_file_lookup": {"type": ["boolean", "string"]} }, "additionalProperties": true }, From ed46cf6e071f521c40c0d6938892a76c599cfb6e Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 10 Jul 2026 14:05:01 +1000 Subject: [PATCH 13/19] refactor(nodespec): snake_case enum VALUES (nodespec-wide) Authors now write snake_case for nodespec's own enum values: - source_type: cloudFiles -> cloud_files, batchFiles -> batch_files, deltaJoin -> delta_join (delta/kafka/python/sql unchanged). - config_flags: disableOperationalMetadata -> disable_operational_metadata. Nodespec-only: the transformer maps these back to the backend's expected values (cloudFiles / disableOperationalMetadata) on output, so the backend and the legacy standard/flow/materialized_view formats are unchanged. The conversion script snakes them during migration. Left as-is (not nodespec-owned): reader_options keys AND values are literal Spark/Auto Loader tokens (e.g. binaryFile, addNewColumns, cloudFiles.format); the engine reads them verbatim. Co-authored-by: Isaac --- .../customer_cloudfiles_main.json | 2 +- ...storical_snapshot_files_datetime_main.json | 2 +- ...napshot_files_datetime_multifile_main.json | 2 +- ...tetime_recursive_and_partitioned_main.json | 2 +- ...ecursive_and_partitioned_parquet_main.json | 2 +- ..._recursive_and_partitioned_regex_main.json | 2 +- .../historical_snapshot_files_flow_main.json | 2 +- .../historical_snapshot_files_int_main.json | 2 +- ...snapshot_files_schema_and_select_main.json | 2 +- ...storical_snapshot_table_datetime_main.json | 2 +- ...snapshot_table_select_expression_main.json | 2 +- .../dataflowspec/materialized_views_main.json | 2 +- .../periodic_snapshot_scd1_main.json | 2 +- .../periodic_snapshot_scd2_main.json | 2 +- scripts/migrate_to_nodespec.py | 24 ++++++++++++++++- .../transformer/nodespec.py | 27 +++++++++++++++++++ src/schemas/spec_nodespec.json | 12 ++++----- 17 files changed, 70 insertions(+), 21 deletions(-) diff --git a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json index 730266d..19b134b 100644 --- a/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json +++ b/samples/nodespec_sample/src/dataflows/base_samples/dataflowspec/customer_cloudfiles_main.json @@ -6,7 +6,7 @@ { "name": "v_source_files", "node_type": "source", - "source_type": "cloudFiles", + "source_type": "cloud_files", "config": { "mode": "stream", "path": "{sample_file_location}/customer/", diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json index cdbece1..5285204 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_main.json @@ -13,7 +13,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "keys": [ diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json index 104af52..8b07e59 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_multifile_main.json @@ -13,7 +13,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "keys": [ diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json index 4b95385..7276564 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_main.json @@ -13,7 +13,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "keys": [ diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json index 7b189a7..0575d49 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_parquet_main.json @@ -13,7 +13,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "keys": [ diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json index 5a85929..ac7d15c 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_datetime_recursive_and_partitioned_regex_main.json @@ -13,7 +13,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "keys": [ diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json index dfa7b88..53cc1f7 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_flow_main.json @@ -12,7 +12,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "keys": [ diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json index edafdd4..deccc3f 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_int_main.json @@ -13,7 +13,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "keys": [ diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json index 22c4b44..728a8ec 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_files_schema_and_select_main.json @@ -12,7 +12,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "keys": [ diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json index 3523917..4344a2e 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_datetime_main.json @@ -13,7 +13,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "keys": [ diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json index d739359..6f8e86e 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/historical_snapshot_table_select_expression_main.json @@ -12,7 +12,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "keys": [ diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json index eaab6b5..f583cb0 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json @@ -92,7 +92,7 @@ "refresh_policy": "incremental_strict", "table_details": { "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ] } } diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json index 5c6c912..7624ef9 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd1_main.json @@ -31,7 +31,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "snapshot_type": "periodic", diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json index 666fab7..76774b8 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/periodic_snapshot_scd2_main.json @@ -31,7 +31,7 @@ "delta.enableChangeDataFeed": "true" }, "config_flags": [ - "disableOperationalMetadata" + "disable_operational_metadata" ], "cdc_snapshot_settings": { "snapshot_type": "periodic", diff --git a/scripts/migrate_to_nodespec.py b/scripts/migrate_to_nodespec.py index 008e480..bddcc95 100644 --- a/scripts/migrate_to_nodespec.py +++ b/scripts/migrate_to_nodespec.py @@ -158,10 +158,32 @@ def _order(d: Dict, order: List[str]) -> Dict: return {k: d[k] for k in known + rest} +# nodespec authors enum VALUES in snake_case; legacy formats use camelCase for +# a few of them, so snake them during migration. +_SOURCE_TYPE_TO_SNAKE = {"cloudFiles": "cloud_files", "batchFiles": "batch_files", "deltaJoin": "delta_join"} +_CONFIG_FLAG_TO_SNAKE = {"disableOperationalMetadata": "disable_operational_metadata"} + + +def _snake_config_flag_values(obj) -> None: + """Recursively snake `config_flags` array values in place.""" + if isinstance(obj, dict): + for k, v in obj.items(): + if k == "config_flags" and isinstance(v, list): + obj[k] = [_CONFIG_FLAG_TO_SNAKE.get(x, x) if isinstance(x, str) else x for x in v] + else: + _snake_config_flag_values(v) + elif isinstance(obj, list): + for i in obj: + _snake_config_flag_values(i) + + def _order_node(node: Dict) -> Dict: - """Order a node's keys and its config's keys canonically.""" + """Order a node's keys and its config's keys canonically, and snake enum values.""" + if node.get("source_type") in _SOURCE_TYPE_TO_SNAKE: + node["source_type"] = _SOURCE_TYPE_TO_SNAKE[node["source_type"]] node = _order(node, _NODE_ORDER) if isinstance(node.get("config"), dict): + _snake_config_flag_values(node["config"]) node["config"] = _order(node["config"], _CONFIG_ORDER) return node diff --git a/src/dataflow_spec_builder/transformer/nodespec.py b/src/dataflow_spec_builder/transformer/nodespec.py index 492ce79..94e1310 100644 --- a/src/dataflow_spec_builder/transformer/nodespec.py +++ b/src/dataflow_spec_builder/transformer/nodespec.py @@ -68,6 +68,31 @@ def _deep_camel(obj: Any) -> Any: return obj +# nodespec authors enum VALUES in snake_case; the backend/legacy formats expect +# camelCase for a few of them. Map those back on the transformer's output. +_SOURCE_TYPE_TO_BACKEND = {"cloud_files": "cloudFiles", "batch_files": "batchFiles", "delta_join": "deltaJoin"} +_CONFIG_FLAG_TO_BACKEND = {"disable_operational_metadata": "disableOperationalMetadata"} + + +def _values_to_backend(obj: Any) -> Any: + """Recursively map nodespec snake_case enum values back to backend values + (sourceType: cloud_files->cloudFiles etc.; configFlags entries). Applied to + the final flow spec, so it also covers the file/table snapshot sourceType and + the sql/delta transform sourceType, which are simply left unchanged.""" + if isinstance(obj, dict): + for k, v in obj.items(): + if k == "sourceType" and isinstance(v, str): + obj[k] = _SOURCE_TYPE_TO_BACKEND.get(v, v) + elif k == "configFlags" and isinstance(v, list): + obj[k] = [_CONFIG_FLAG_TO_BACKEND.get(x, x) if isinstance(x, str) else x for x in v] + else: + _values_to_backend(v) + elif isinstance(obj, list): + for i in obj: + _values_to_backend(i) + return obj + + def _first(cfg: Dict, *keys: str) -> Dict: """{camel(key): value} for the first present snake key (e.g. sql_path|sql_statement).""" for k in keys: @@ -110,6 +135,8 @@ def _process_spec(self, spec_data: Dict) -> Union[Dict, List[Dict]]: if not specs: raise ValueError("Nodespec spec must contain at least one target node") + for s in specs: + _values_to_backend(s) return specs if len(specs) > 1 else specs[0] @staticmethod diff --git a/src/schemas/spec_nodespec.json b/src/schemas/spec_nodespec.json index d9687cc..ed2df55 100644 --- a/src/schemas/spec_nodespec.json +++ b/src/schemas/spec_nodespec.json @@ -37,7 +37,7 @@ }, "source_type": { "type": "string", - "enum": ["batchFiles", "cloudFiles", "delta", "deltaJoin", "kafka", "python", "sql"], + "enum": ["batch_files", "cloud_files", "delta", "delta_join", "kafka", "python", "sql"], "description": "Kind of source. Required when node_type=source; selects which config fields apply." }, "transformation_type": { @@ -66,11 +66,11 @@ "then": {"properties": {"config": {"$ref": "#/$defs/source_delta_config"}}} }, { - "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "cloudFiles"}}, "required": ["node_type", "source_type"]}, + "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "cloud_files"}}, "required": ["node_type", "source_type"]}, "then": {"properties": {"config": {"$ref": "#/$defs/source_cloud_files_config"}}} }, { - "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "batchFiles"}}, "required": ["node_type", "source_type"]}, + "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "batch_files"}}, "required": ["node_type", "source_type"]}, "then": {"properties": {"config": {"$ref": "#/$defs/source_batch_files_config"}}} }, { @@ -78,7 +78,7 @@ "then": {"properties": {"config": {"$ref": "#/$defs/source_kafka_config"}}} }, { - "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "deltaJoin"}}, "required": ["node_type", "source_type"]}, + "if": {"properties": {"node_type": {"const": "source"}, "source_type": {"const": "delta_join"}}, "required": ["node_type", "source_type"]}, "then": {"properties": {"config": {"$ref": "#/$defs/source_delta_join_config"}}} }, { @@ -311,7 +311,7 @@ "comment": {"type": "string"}, "spark_conf": {"type": "object"}, "row_filter": {"type": "string"}, - "config_flags": {"type": "array", "items": {"type": "string", "enum": ["disableOperationalMetadata"]}}, + "config_flags": {"type": "array", "items": {"type": "string", "enum": ["disable_operational_metadata"]}}, "once": {"type": "boolean", "default": false, "description": "Run the flow to this target once (batch), then stop."}, "cdc_settings": {"$ref": "./definitions_main.json#/definitions/cdcSettings"}, "cdc_snapshot_settings": {"$ref": "#/$defs/snapshot_cdc_config"}, @@ -326,7 +326,7 @@ "private": {"type": "boolean"}, "comment": {"type": "string"}, "spark_conf": {"type": "object"}, - "config_flags": {"type": "array", "items": {"type": "string", "enum": ["disableOperationalMetadata"]}} + "config_flags": {"type": "array", "items": {"type": "string", "enum": ["disable_operational_metadata"]}} }, "additionalProperties": false }, From 9f3ff56801b440403349bf9cc7b0db538dfb61a0 Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 10 Jul 2026 14:05:39 +1000 Subject: [PATCH 14/19] docs(nodespec): use snake_case source_type values in the reference Matches the schema/samples: source_type values are now cloud_files / batch_files / delta_join (delta/kafka/sql/python unchanged). Co-authored-by: Isaac --- docs/source/dataflow_spec_ref_main_nodespec.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/dataflow_spec_ref_main_nodespec.rst b/docs/source/dataflow_spec_ref_main_nodespec.rst index a159ebd..0e5958e 100644 --- a/docs/source/dataflow_spec_ref_main_nodespec.rst +++ b/docs/source/dataflow_spec_ref_main_nodespec.rst @@ -230,7 +230,7 @@ Source node configuration - **Description** * - **source_type** - ``string`` - - ``delta``, ``cloudFiles``, ``batchFiles``, ``kafka``, ``sql``, ``python``. + - ``delta``, ``cloud_files``, ``batch_files``, ``delta_join``, ``kafka``, ``sql``, ``python``. * - **mode** (*optional*) - ``string`` - ``stream`` or ``batch``. Default: ``stream``. @@ -239,7 +239,7 @@ Source node configuration - For ``delta`` sources. * - **path** - ``string`` - - For ``cloudFiles`` / ``batchFiles`` sources. + - For ``cloud_files`` / ``batch_files`` sources. * - **cdf_enabled** (*optional*) - ``boolean`` - Enable Change Data Feed for Delta sources. From 94bc5d15ebfe41c96e44d934828a0d0509f90052 Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 10 Jul 2026 14:08:26 +1000 Subject: [PATCH 15/19] feat(nodespec schema): gate MV-only vs ST-only target fields on table_type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Within target_delta_config, an intra-object if/then/else keyed on table_type now enforces which fields are valid: - table_type=mv: sql_path/sql_statement/refresh_policy/table_details allowed; cdc_settings/cdc_snapshot_settings/table_migration_details/once rejected. - table_type=st (or omitted): the reverse. table_type and the gated fields living in the same object is fine — if/then conditions on one property and constrains its siblings. Editors now flag, e.g., sql_path on a streaming table or cdc_settings on a materialized view. 37/37 samples still validate. Co-authored-by: Isaac --- src/schemas/spec_nodespec.json | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/schemas/spec_nodespec.json b/src/schemas/spec_nodespec.json index ed2df55..bc63b4c 100644 --- a/src/schemas/spec_nodespec.json +++ b/src/schemas/spec_nodespec.json @@ -334,7 +334,30 @@ "quarantine": {"$ref": "#/$defs/quarantine"} }, "required": ["table"], - "additionalProperties": false + "additionalProperties": false, + "allOf": [ + { + "if": {"properties": {"table_type": {"const": "mv"}}, "required": ["table_type"]}, + "then": { + "$comment": "Materialized view: streaming-table-only fields are not allowed.", + "properties": { + "cdc_settings": false, + "cdc_snapshot_settings": false, + "table_migration_details": false, + "once": false + } + }, + "else": { + "$comment": "Streaming table (table_type st/omitted): materialized-view-only fields are not allowed.", + "properties": { + "sql_path": false, + "sql_statement": false, + "refresh_policy": false, + "table_details": false + } + } + } + ] }, "target_sink_config": { From 0968d09c7564403a5edc65b501ce5fcd9566297b Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 10 Jul 2026 14:13:31 +1000 Subject: [PATCH 16/19] feat(nodespec schema): add conditional validation for dependent fields Extend the if/then pattern to other genuine field dependencies: - source_delta_config: starting_version_from_dlt_setup=true requires cdf_enabled=true. - target_delta_config: cluster_by_columns and partition_columns are mutually exclusive. - snapshot_cdc_config: periodic snapshots forbid inline source_type/source (they read an upstream source node); historical snapshots require them; a file source requires path+format; a table source requires table. Sinks are intentionally left flat/ungated for now (the sink redesign is deferred, and target_type lives at the node level rather than in the sink config). 37/37 samples still validate. Co-authored-by: Isaac --- src/schemas/spec_nodespec.json | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/schemas/spec_nodespec.json b/src/schemas/spec_nodespec.json index bc63b4c..bcc6aee 100644 --- a/src/schemas/spec_nodespec.json +++ b/src/schemas/spec_nodespec.json @@ -143,7 +143,13 @@ "source_delta_config": { "type": "object", "description": "Delta table source.", - "allOf": [{"$ref": "#/$defs/source_common"}], + "allOf": [ + {"$ref": "#/$defs/source_common"}, + { + "if": {"properties": {"starting_version_from_dlt_setup": {"const": true}}, "required": ["starting_version_from_dlt_setup"]}, + "then": {"properties": {"cdf_enabled": {"const": true}}} + } + ], "properties": { "mode": {"type": "string", "enum": ["stream", "batch"], "default": "stream"}, "database": {"type": "string", "description": "Source schema/database."}, @@ -335,6 +341,7 @@ }, "required": ["table"], "additionalProperties": false, + "not": {"required": ["cluster_by_columns", "partition_columns"]}, "allOf": [ { "if": {"properties": {"table_type": {"const": "mv"}}, "required": ["table_type"]}, @@ -485,7 +492,25 @@ "track_history_except_column_list": {"type": "array", "items": {"type": "string"}} }, "required": ["keys", "scd_type", "snapshot_type"], - "additionalProperties": true + "additionalProperties": true, + "allOf": [ + { + "$comment": "Periodic snapshots read from an upstream source node; no inline source_type/source.", + "if": {"properties": {"snapshot_type": {"const": "periodic"}}, "required": ["snapshot_type"]}, + "then": {"properties": {"source_type": false, "source": false}}, + "else": {"required": ["source_type", "source"]} + }, + { + "$comment": "Historical file source needs a path and format.", + "if": {"properties": {"source_type": {"const": "file"}}, "required": ["source_type"]}, + "then": {"properties": {"source": {"required": ["path", "format"]}}} + }, + { + "$comment": "Historical table source needs a table.", + "if": {"properties": {"source_type": {"const": "table"}}, "required": ["source_type"]}, + "then": {"properties": {"source": {"required": ["table"]}}} + } + ] }, "reader_options_cloud_files": { From 798eb1f9e0797e67f7acce0d2270ecb22cb97f0b Mon Sep 17 00:00:00 2001 From: Haille W Date: Fri, 10 Jul 2026 14:22:15 +1000 Subject: [PATCH 17/19] docs(nodespec): document table_type field gating and cluster/partition exclusivity Reflects the schema's conditional validation: streaming-table-only vs materialized-view-only target fields are gated on table_type, and partition_columns / cluster_by_columns are mutually exclusive. Co-authored-by: Isaac --- docs/source/dataflow_spec_ref_main_nodespec.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/source/dataflow_spec_ref_main_nodespec.rst b/docs/source/dataflow_spec_ref_main_nodespec.rst index 0e5958e..f20fceb 100644 --- a/docs/source/dataflow_spec_ref_main_nodespec.rst +++ b/docs/source/dataflow_spec_ref_main_nodespec.rst @@ -324,7 +324,8 @@ Target node configuration - Delta table properties. * - **partition_columns** / **cluster_by_columns** / **cluster_by_auto** (*optional*) - ``array`` / ``array`` / ``boolean`` - - Partitioning and liquid clustering. + - Partitioning and liquid clustering. ``partition_columns`` and + ``cluster_by_columns`` are mutually exclusive. * - **comment** / **spark_conf** / **row_filter** / **config_flags** (*optional*) - ``string`` / ``object`` / ``string`` / ``array`` - Additional table settings. @@ -353,6 +354,15 @@ Target node configuration ``{ "view": , "flow": }`` that sets the flow name. Conventionally written last in the config. +.. note:: + Which fields are valid depends on ``table_type``. Streaming-table settings + (``cdc_settings``, ``cdc_snapshot_settings``, ``table_migration_details``, + ``once``) are only allowed on streaming tables (``table_type`` ``st`` or + omitted); materialized-view settings (``sql_path``, ``sql_statement``, + ``refresh_policy``, ``table_details``) are only allowed on materialized views + (``table_type: "mv"``). The schema enforces this, so an editor flags, e.g., + ``sql_path`` on a streaming table or ``cdc_settings`` on a materialized view. + Sink targets ----------- From ed53eba309a47c682f0b70242162ff17ebbbe19b Mon Sep 17 00:00:00 2001 From: Haille W Date: Sun, 12 Jul 2026 12:48:52 +1000 Subject: [PATCH 18/19] refactor(nodespec): drop table_details; MV `private` is a top-level field table_details was an MV-only object that mostly duplicated top-level fields (comment/spark_conf/config_flags); its only unique field was `private`. Now that nodespec keeps table settings top-level for all targets, the object is removed and `private` becomes a top-level field gated to table_type=mv (like the other MV-only fields). MV and streaming tables configure table settings identically. - schema: remove table_details; add top-level `private` (MV-only, forbidden on ST). - transformer: MV targetDetails is built from the top-level config only. - migrate script: flatten legacy MV tableDetails onto the config; nodespec normaliser flattens any existing table_details. - samples/docs updated. 37/37 validate, 38/38 transform; `private` still reaches the backend targetDetails. Co-authored-by: Isaac --- .../dataflow_spec_ref_main_nodespec.rst | 2 +- .../dataflowspec/materialized_views_main.json | 22 +++++++------------ scripts/migrate_to_nodespec.py | 13 +++++++++-- .../transformer/nodespec.py | 7 +++--- src/schemas/spec_nodespec.json | 14 ++---------- 5 files changed, 25 insertions(+), 33 deletions(-) diff --git a/docs/source/dataflow_spec_ref_main_nodespec.rst b/docs/source/dataflow_spec_ref_main_nodespec.rst index f20fceb..f153349 100644 --- a/docs/source/dataflow_spec_ref_main_nodespec.rst +++ b/docs/source/dataflow_spec_ref_main_nodespec.rst @@ -359,7 +359,7 @@ Target node configuration (``cdc_settings``, ``cdc_snapshot_settings``, ``table_migration_details``, ``once``) are only allowed on streaming tables (``table_type`` ``st`` or omitted); materialized-view settings (``sql_path``, ``sql_statement``, - ``refresh_policy``, ``table_details``) are only allowed on materialized views + ``refresh_policy``, ``private``) are only allowed on materialized views (``table_type: "mv"``). The schema enforces this, so an editor flags, e.g., ``sql_path`` on a streaming table or ``cdc_settings`` on a materialized view. diff --git a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json index f583cb0..a9e6099 100644 --- a/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json +++ b/samples/nodespec_sample/src/dataflows/feature_samples/dataflowspec/materialized_views_main.json @@ -43,9 +43,7 @@ "table": "feature_mv_sql_statement", "table_type": "mv", "sql_statement": "SELECT * FROM {staging_schema}.customer", - "table_details": { - "private": true - } + "private": true } }, { @@ -74,11 +72,9 @@ "table": "feature_mv_chain_mvs", "table_type": "mv", "sql_statement": "SELECT * FROM live.feature_mv_sql_statement", - "table_details": { - "comment": "Test Config", - "spark_conf": { - "spark.sql.session.timeZone": "Australia/Sydney" - } + "comment": "Test Config", + "spark_conf": { + "spark.sql.session.timeZone": "Australia/Sydney" } } }, @@ -89,12 +85,10 @@ "table": "feature_mv_with_refresh_policy", "table_type": "mv", "sql_statement": "SELECT * FROM {staging_schema}.customer", - "refresh_policy": "incremental_strict", - "table_details": { - "config_flags": [ - "disable_operational_metadata" - ] - } + "config_flags": [ + "disable_operational_metadata" + ], + "refresh_policy": "incremental_strict" } } ] diff --git a/scripts/migrate_to_nodespec.py b/scripts/migrate_to_nodespec.py index bddcc95..f2cc501 100644 --- a/scripts/migrate_to_nodespec.py +++ b/scripts/migrate_to_nodespec.py @@ -141,7 +141,7 @@ def _rename(d: Dict, mapping: Dict[str, str]) -> Dict: "select_exp", "where_clause", "schema_path", "reader_options", "python_transform", "table_properties", "partition_columns", "cluster_by_columns", "cluster_by_auto", "comment", "spark_conf", "row_filter", "config_flags", - "refresh_policy", "table_details", "once", + "refresh_policy", "private", "once", "cdc_settings", "cdc_snapshot_settings", "data_quality", "quarantine", "table_migration_details", "name", "sink_type", "sink_config", "sink_options", @@ -285,6 +285,12 @@ def _migrate_nodespec_config(cfg: Dict) -> None: if qtd: quarantine["target"] = qtd cfg["quarantine"] = quarantine + # table_details is dropped: its fields (private + comment/spark_conf/config_flags) + # move to the config top level. + td = cfg.pop("table_details", None) + if isinstance(td, dict): + for k, v in td.items(): + cfg.setdefault(k, v) def _migrate_nodespec(spec: Dict) -> Dict: @@ -540,9 +546,12 @@ def _migrate_materialized_view(spec: Dict) -> Dict: if val is not None: target_config[snake] = val + # Legacy MV tableDetails settings are flattened onto the target config + # top-level (nodespec has no separate table_details object; MV-only + # `private` is a top-level field). table_details = _get(mv_config, "tableDetails", "table_details") if table_details: - target_config["table_details"] = _rename(table_details, _TARGET_DETAIL_MAP) + target_config.update(_rename(table_details, _TARGET_DETAIL_MAP)) _add_target_settings(target_config, mv_config) diff --git a/src/dataflow_spec_builder/transformer/nodespec.py b/src/dataflow_spec_builder/transformer/nodespec.py index 94e1310..cd6cda5 100644 --- a/src/dataflow_spec_builder/transformer/nodespec.py +++ b/src/dataflow_spec_builder/transformer/nodespec.py @@ -50,7 +50,7 @@ "input_flows", "table", "table_type", "enabled", "once", "name", "cdc_settings", "cdc_snapshot_settings", "data_quality", "quarantine", "table_migration_details", - "sink_type", "sink_config", "sink_options", "table_details", "source_view", + "sink_type", "sink_config", "sink_options", "source_view", } @@ -341,9 +341,8 @@ def _add_flows(self, group, target, table, spec_target, has_cdc, registered, cou def _mv_spec(self, spec_data: Dict, target: Dict, sources: List[Dict]) -> Dict: cfg = target.get("config", {}) mv = cfg.get("table") - # config-level details + table_details (the latter overrides on conflict). - details = {"table": mv, "type": TableType.MATERIALIZED_VIEW, - **_camel(cfg, _HANDLED), **_camel(cfg.get("table_details", {}))} + # All table settings (including MV-only `private`) live top-level on the config. + details = {"table": mv, "type": TableType.MATERIALIZED_VIEW, **_camel(cfg, _HANDLED)} spec = self._base(spec_data) spec.pop("dataFlowVersion", None) # MV specs do not carry a version diff --git a/src/schemas/spec_nodespec.json b/src/schemas/spec_nodespec.json index bcc6aee..d939fca 100644 --- a/src/schemas/spec_nodespec.json +++ b/src/schemas/spec_nodespec.json @@ -325,17 +325,7 @@ "sql_path": {"type": "string", "description": "MV only: path to a .sql file defining the view."}, "sql_statement": {"type": "string", "description": "MV only: inline SQL defining the view."}, "refresh_policy": {"type": "string", "enum": ["auto", "incremental", "incremental_strict", "full"], "description": "MV only: refresh policy."}, - "table_details": { - "type": "object", - "description": "MV only: additional table details.", - "properties": { - "private": {"type": "boolean"}, - "comment": {"type": "string"}, - "spark_conf": {"type": "object"}, - "config_flags": {"type": "array", "items": {"type": "string", "enum": ["disable_operational_metadata"]}} - }, - "additionalProperties": false - }, + "private": {"type": "boolean", "description": "MV only: create the materialized view as private (internal, not published to the catalog)."}, "data_quality": {"$ref": "#/$defs/data_quality"}, "quarantine": {"$ref": "#/$defs/quarantine"} }, @@ -360,7 +350,7 @@ "sql_path": false, "sql_statement": false, "refresh_policy": false, - "table_details": false + "private": false } } } From 169f83143fbfc440d57d8b16188070ee8eee6138 Mon Sep 17 00:00:00 2001 From: Haille W Date: Mon, 13 Jul 2026 09:56:01 +1000 Subject: [PATCH 19/19] chore(vscode): map spec JSON files to their schemas for editor validation Co-authored-by: Isaac --- .vscode/settings.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 6945bca..2d5e48b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -25,4 +25,22 @@ ], "cursorpyright.analysis.stubPath": ".vscode", "python.languageServer": "None", + "json.schemas": [ + { + "fileMatch": ["*_main.json"], + "url": "./src/schemas/main.json" + }, + { + "fileMatch": ["*_flow.json"], + "url": "./src/schemas/flow_group.json" + }, + { + "fileMatch": ["*_dqe.json"], + "url": "./src/schemas/expectations.json" + }, + { + "fileMatch": ["*_secrets.json"], + "url": "./src/schemas/secrets.json" + } + ] }