diff --git a/converters/databricks/README.md b/converters/databricks/README.md new file mode 100644 index 0000000..e2fe6cc --- /dev/null +++ b/converters/databricks/README.md @@ -0,0 +1,124 @@ + + +# Apache Ossie Databricks Converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and a Databricks +[Unity Catalog Metric View](https://docs.databricks.com/aws/en/metric-views/) (YAML +`1.1`). No Databricks connection required. + +- **Export** (`ossie-databricks export`): Apache Ossie -> Metric View (one fact + `source` with a nested `joins` tree and a flat `dimensions` list). +- **Import** (`ossie-databricks import`): Metric View -> Apache Ossie. Metric View features Apache Ossie has + no native field for are preserved in `custom_extensions[DATABRICKS]`, so + `MV -> Apache Ossie -> MV` is lossless. + +On **export** (Apache Ossie -> Metric View), Apache Ossie features with no Metric View slot -- relationship +`ai_context`, `dimension.is_time`, non-`DATABRICKS`/`ANSI_SQL` dialects, foreign-vendor +`custom_extensions` -- are **dropped with a warning**. On **import** (Metric View -> Apache Ossie), +Metric View only features (filter, window, format, rely, ...) are instead **preserved** in +`custom_extensions[DATABRICKS]`, so `MV -> Apache Ossie -> MV` is lossless. Any input that breaks a +[requirement](#requirements) **raises a `ConversionError`** -- the converter never +silently drops a field or produces an invalid result. + +## Installation + +```bash +pip install apache-ossie-databricks # once published to PyPI +# or, from a checkout of this directory: +pip install -e . +``` + +The only runtime dependency is `PyYAML`. Python 3.11+. + +## Usage + +### Command line + +```bash +ossie-databricks export -i model.yaml -o view.yaml [--source orders] # Apache Ossie -> Metric View +ossie-databricks import -i view.yaml -o model.yaml [--name my_model] # Metric View -> Apache Ossie +``` + +With no `-o`, output goes to stdout. `--source` (export) picks the fact/grain (default: +the FK-sink dataset; naming a coarser-grain dataset produces `one_to_many` joins); +`--name` (import) sets the Apache Ossie model name (default: the source's last identifier). + +### Python API + +```python +from ossie_databricks import convert_ossie_to_metric_view, convert_metric_view_to_ossie + +metric_view_yaml = convert_ossie_to_metric_view(ossie_yaml_str) # optionally choose the fact/grain, e.g. (ossie_yaml_str, source="orders") +ossie_yaml = convert_metric_view_to_ossie(metric_view_yaml_str, model_name="sales") +``` + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is specific to +**export** (Apache Ossie -> Metric View) or **import** (Metric View -> Apache Ossie). + +| Apache Ossie | Metric View (v1.1) | Notes | +|---|---|---| +| `semantic_model.description` | `comment` | Model-level description only. | +| root dataset | `source` | The fact/grain. | +| other `datasets` | nested `joins[]` | Export: the relationship graph is reassembled into the join tree; a dataset reached by two paths (a diamond) fans out into one aliased join per path. | +| `relationship` `from_columns`/`to_columns` | join `on` (differing names) / `using` (shared names) | Decomposed into columns on import; rebuilt into `on`/`using` on export. | +| `relationship.from`/`to` direction | join `cardinality` | Export: source on the many (`from`) side -> `many_to_one`; on the one (`to`) side -> `one_to_many`. | +| `dataset.primary_key` / `unique_keys` | join `rely.at_most_one_match` | Both directions: export sets `at_most_one_match` when a key covers the join columns; import recovers a `unique_keys` from it. | +| `dataset.fields[]` | `dimensions[]` | Export: fields flatten into one list and a joined column is qualified by its full join path (`customer.c_name`; `customer.region.r_name` when nested). | +| `field.expression.dialects[]` | `expr` | Export: prefer the `DATABRICKS` dialect, else `ANSI_SQL`. | +| `metrics[]` | `measures[]` | Export: fact columns are referenced bare (`SUM(amount)`). | +| `field.label` | `display_name` | | +| `field` / `metric` `description` | `comment` | | +| `ai_context.synonyms` | `synonyms` | | +| `custom_extensions[DATABRICKS]` | `filter`, `window`, `format`, `rely`, `materialization` | Import stashes Metric View only features here; export restores them -- keeping `MV -> Apache Ossie -> MV` lossless. | + +## Requirements + +Conversion raises a `ConversionError` (rather than guessing or emitting something +invalid) when an input breaks one of these: + +- the Metric View `version` is not `1.1`; +- a `source` is not a 3-part `catalog.schema.table` name or a `SELECT`/`WITH` subquery; +- the relationship graph is not acyclic and resolvable to a single fact -- a cycle, or + multiple candidate facts without `--source`, is rejected (a diamond is allowed and + fanned out); +- a join has no condition (a cross join has no Apache Ossie relationship form); +- a join condition is non-equi or otherwise can't be decomposed into equi-join columns + (Apache Ossie relationships are equi-joins, so the join has no Apache Ossie representation); +- the input YAML is malformed. + +## Development + +```bash +pip install -e ".[dev]" +python3 -m pytest tests/ +``` + +Example-based unit tests plus Hypothesis property-based round-trip tests +(`test_roundtrip_properties.py`, which skip if `hypothesis` is not installed). + +## Future effort + +Both the Apache Ossie specification and the Databricks Unity Catalog Metric View YAML are still +evolving. As either side adds or changes fields, this converter will be updated to track +them -- extending the mapping and coverage in both directions to keep the conversion +current and to support as much as each format allows over time. diff --git a/converters/databricks/pyproject.toml b/converters/databricks/pyproject.toml new file mode 100644 index 0000000..d4ed351 --- /dev/null +++ b/converters/databricks/pyproject.toml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "apache-ossie-databricks" +version = "0.2.0.dev0" +description = "Databricks Unity Catalog Metric View <> Apache Ossie converter" +requires-python = ">=3.11" +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "PyYAML>=6.0", +] + +[project.license] +text = "Apache-2.0" + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "hypothesis>=6.0", +] + +[project.scripts] +ossie-databricks = "ossie_databricks.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_databricks"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/converters/databricks/src/ossie_databricks/__init__.py b/converters/databricks/src/ossie_databricks/__init__.py new file mode 100644 index 0000000..379a2ae --- /dev/null +++ b/converters/databricks/src/ossie_databricks/__init__.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bidirectional converter between Apache Ossie semantic models and Databricks Unity Catalog +Metric Views (YAML v1.1). Pure offline string-in / string-out transforms. + + from ossie_databricks import convert_ossie_to_metric_view, convert_metric_view_to_ossie +""" + +from ._common import ConversionError +from .metric_view_to_ossie import convert_metric_view_to_ossie +from .ossie_to_metric_view import convert_ossie_to_metric_view + +__all__ = [ + "ConversionError", + "convert_metric_view_to_ossie", + "convert_ossie_to_metric_view", +] diff --git a/converters/databricks/src/ossie_databricks/_common.py b/converters/databricks/src/ossie_databricks/_common.py new file mode 100644 index 0000000..01f6263 --- /dev/null +++ b/converters/databricks/src/ossie_databricks/_common.py @@ -0,0 +1,278 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared helpers for the Apache Ossie <-> Databricks Metric View converters. + +Both directions are pure offline YAML transforms. The only cross-cutting concerns +live here: version constants, the dialect preference order, the `custom_extensions` +stash protocol, and small SQL-string helpers. +""" + +import json +import re + +import yaml + +# Apache Ossie semantic model spec version this converter targets (see core-spec). +OSSIE_VERSION = "0.2.0.dev0" + +# Databricks Unity Catalog Metric View YAML version. Only 1.1 supports joins, +# per-column comments, synonyms, and the format/window/parameters surface. +MV_VERSION = "1.1" + +# Vendor id used for the `custom_extensions` stash and for dialect selection. +VENDOR = "DATABRICKS" + +# Expression dialects this converter understands, in preference order. +DIALECT_DATABRICKS = "DATABRICKS" +DIALECT_ANSI = "ANSI_SQL" + +# Metric Views cap the number of synonyms per column. +SYNONYM_LIMIT = 10 + +# Bump when the shape of a stashed `data` blob changes. +STASH_VERSION = 1 + +# Metric View join cardinalities (the only two values v1.1 defines). Apache Ossie has no +# cardinality field; the value is implied by relationship direction -- `from` is the +# many side, `to` is the one side -- so the converter derives it from / writes it +# into the from/to orientation rather than relying on a dedicated field. +CARD_MANY_TO_ONE = "many_to_one" +CARD_ONE_TO_MANY = "one_to_many" + +# Model-level stash key recording which dataset was the Metric View `source` (its +# grain). Needed only when a one_to_many join puts the source on a relationship's +# `to` side, where the natural FK-sink heuristic would otherwise pick the wrong +# fact on re-export. Absent for plain many-to-one stars, so they stay clean. +STASH_SOURCE_KEY = "source_dataset" + +# A bare SQL identifier (single column reference), e.g. `c_name`. Used to decide +# whether an expression can be safely alias-prefixed on export / de-prefixed on +# import. +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class ConversionError(Exception): + """Raised when an input cannot be converted.""" + + +def require(obj, key, what): + """Return `obj[key]`, or raise a clean ConversionError if it's missing/empty -- so + malformed input surfaces as an error message rather than a raw KeyError traceback. + + Presence is tested by key (not truthiness), so a legitimately falsy value such as + `0` or `False` is returned; a missing key, a null, or an empty/whitespace string is + rejected. + """ + if not isinstance(obj, dict) or key not in obj or obj[key] is None: + raise ConversionError(f"{what} is missing required '{key}'") + value = obj[key] + if isinstance(value, str) and not value.strip(): + raise ConversionError(f"{what} has an empty '{key}'") + return value + + +def require_str(obj, key, what): + """Like require(), but also enforce the value is a string -- so a non-string scalar + (e.g. a YAML number for a name or expression) raises a clean ConversionError instead + of crashing later in a string operation.""" + value = require(obj, key, what) + if not isinstance(value, str): + raise ConversionError( + f"{what}: '{key}' must be a string, got {type(value).__name__}") + return value + + +# YAML 1.1 (PyYAML's default) treats bare on/off/yes/no/y/n as booleans, so a metric +# view join's `on:` key would parse as the boolean True and silently lose the join +# condition. Databricks (Jackson) uses YAML 1.2 booleans (only true/false). The Loader +# below uses 1.2 semantics, so it reads DBR's bare `on:` (and any "on"/"off" value) as a +# string. The Dumper additionally force-quotes those tokens on output (see below), so the +# YAML it emits round-trips the same way through a YAML 1.1 reader too (e.g. stock +# yaml.safe_load) rather than turning an "on"/"off" synonym/label into a boolean. +class _Yaml12Loader(yaml.SafeLoader): + """SafeLoader with YAML 1.2 boolean semantics.""" + + +class _Yaml12Dumper(yaml.SafeDumper): + """SafeDumper with YAML 1.2 boolean semantics.""" + + +_YAML12_BOOL = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$") +for _cls in (_Yaml12Loader, _Yaml12Dumper): + # Drop the YAML 1.1 bool resolver (yes/no/on/off/y/n) and re-add a 1.2 one. + _cls.yaml_implicit_resolvers = { + ch: [(tag, rx) for (tag, rx) in resolvers if tag != "tag:yaml.org,2002:bool"] + for ch, resolvers in _cls.yaml_implicit_resolvers.items() + } + _cls.add_implicit_resolver("tag:yaml.org,2002:bool", _YAML12_BOOL, list("tTfF")) + + +# Force-quote string scalars that a YAML 1.1 reader would otherwise interpret as booleans +# (yes/no/on/off/y/n/true/false, any case). Number- and null-like strings are already +# quoted by PyYAML's surviving resolvers; only these bool tokens need it. Without this, a +# synonym/label/comment like "on" emits bare and a 1.1 consumer reads it back as `True`. +_YAML11_BOOL_STRS = frozenset( + variant + for word in ("y", "n", "yes", "no", "on", "off", "true", "false") + for variant in (word, word.capitalize(), word.upper()) +) + + +def _represent_str(dumper, data): + style = "'" if data in _YAML11_BOOL_STRS else None + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +_Yaml12Dumper.add_representer(str, _represent_str) + + +def load_yaml(text): + """Parse YAML with 1.2 boolean semantics, so a join `on:` key stays the string + `on` rather than becoming the boolean True. A syntax error is surfaced as a + ConversionError so callers (and the CLI) get a clean message, not a raw traceback.""" + try: + return yaml.load(text, Loader=_Yaml12Loader) + except yaml.YAMLError as e: + raise ConversionError(f"Invalid YAML: {e}") from e + + +def dump_yaml(obj): + """Serialize to YAML with 1.2 boolean semantics. The bool-like token `on` -- whether + a join condition key or an "on"/"off"/"yes"/... string value -- is force-quoted as + `'on'` by the str representer (see `_represent_str`), so a YAML 1.1 reader of this + output reads it as the string, not the boolean True. Databricks' Jackson (1.2) parser + reads the quoted key/value correctly too.""" + return yaml.dump(obj, Dumper=_Yaml12Dumper, sort_keys=False, default_flow_style=False) + + +def is_simple_identifier(expr): + """True if `expr` is a single bare column reference (no operators/functions). + + A non-string input is simply not an identifier (returns False) rather than raising.""" + return isinstance(expr, str) and bool(_IDENTIFIER_RE.match(expr.strip())) + + +def read_stash(obj): + """Return the DATABRICKS stash dict on an Apache Ossie object, or {} if absent. + + The `_v` version marker is stripped from the returned dict. + """ + for ext in (obj or {}).get("custom_extensions") or []: + if ext.get("vendor_name") == VENDOR: + data = json.loads(ext.get("data") or "{}") + data.pop("_v", None) + return data + return {} + + +def write_stash(obj, data): + """Attach a DATABRICKS `custom_extensions` entry holding `data` (a dict). + + No-op when `data` is empty, so hand-authored Apache Ossie stays clean. Merges into an + existing DATABRICKS entry if one is already present. + """ + if not data: + return + payload = {"_v": STASH_VERSION} + payload.update(data) + blob = json.dumps(payload) + exts = obj.setdefault("custom_extensions", []) + for ext in exts: + if ext.get("vendor_name") == VENDOR: + ext["data"] = blob + return + exts.append({"vendor_name": VENDOR, "data": blob}) + + +def foreign_vendor_extensions(obj): + """Return non-DATABRICKS custom_extensions (dropped on export, with a warning).""" + return [ + ext + for ext in (obj or {}).get("custom_extensions") or [] + if ext.get("vendor_name") != VENDOR + ] + + +def pick_expression(ossie_expression): + """Choose the SQL string for an Apache Ossie expression: DATABRICKS, else ANSI_SQL. + + Returns None if neither dialect is present (the caller warns and skips). Does + not warn about other dialects here -- only the absence of a usable one matters. + """ + dialects = { + d.get("dialect"): d.get("expression") + for d in (ossie_expression or {}).get("dialects") or [] + } + expr = dialects.get(DIALECT_DATABRICKS) or dialects.get(DIALECT_ANSI) + if expr is not None and not isinstance(expr, str): + raise ConversionError( + f"expression must be a string, got {type(expr).__name__}") + return expr + + +def synonyms_of(ai_context): + """Extract the synonyms list from an Apache Ossie ai_context (object form only).""" + if isinstance(ai_context, dict): + return list(ai_context.get("synonyms") or []) + return [] + + +def merge_description(description, ai_context): + """Fold a string-form ai_context into a description. + + The Apache Ossie schema allows ai_context to be either a string or an object. A string + has no Metric View home of its own, so it is appended to the description + (which maps to `comment`). Object-form ai_context is handled separately + (synonyms map natively; instructions/examples are dropped). + """ + if isinstance(ai_context, str) and ai_context.strip(): + return f"{description}\n{ai_context}" if description else ai_context + return description + + +def validate_source(source, dataset_name): + """Validate and normalize a dataset source for a Metric View. + + Accepts a 3-part `catalog.schema.table` identifier or a `SELECT`/`WITH` + subquery. Raises ConversionError otherwise. + """ + if not source or not str(source).strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = str(source).strip() + # A SELECT/WITH subquery source. `\b` after the keyword matches `WITH(...)` (no + # space) too, but not an identifier like `WITHHELD`. + if re.match(r"(?i)(select|with)\b", s): + return s + # Exactly 3 parts, each a non-empty token with no whitespace -- so `.sch.tbl`, + # `cat..tbl`, `cat.sch.`, and `cat . sch . tbl` are all rejected (an empty or + # space-laden part is not a valid catalog/schema/table identifier). + parts = s.split(".") + if len(parts) == 3 and all(p and not any(ch.isspace() for ch in p) for p in parts): + return s + raise ConversionError( + f"Dataset '{dataset_name}': source '{source}' must be a 3-part " + f"catalog.schema.table identifier or a SELECT/WITH subquery" + ) + + +def last_identifier(source): + """Last dotted part of a table reference, e.g. `samples.tpch.lineitem` -> `lineitem`. + + Coerces to str so a malformed (non-string) source doesn't crash here -- it gets a + clean error from validate_source instead.""" + return str(source).strip().split(".")[-1].strip("`") if source else source diff --git a/converters/databricks/src/ossie_databricks/cli.py b/converters/databricks/src/ossie_databricks/cli.py new file mode 100644 index 0000000..a3751ba --- /dev/null +++ b/converters/databricks/src/ossie_databricks/cli.py @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the Apache Ossie <-> Databricks Metric View converter. + + ossie-databricks export -i model.yaml [-o view.yaml] [--source orders] + ossie-databricks import -i view.yaml [-o model.yaml] [--name my_model] + +`export` converts an Apache Ossie semantic model to a Databricks Metric View; `import` does the +reverse. With no `-o`, the result is written to stdout. Conversions that drop +information emit warnings to stderr. +""" + +import argparse +import sys + +from ._common import ConversionError +from .metric_view_to_ossie import convert_metric_view_to_ossie +from .ossie_to_metric_view import convert_ossie_to_metric_view + + +def _build_parser(): + parser = argparse.ArgumentParser(prog="ossie-databricks", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command") + sub.required = True # set as attribute (the add_subparsers kwarg is 3.7+) + + exp = sub.add_parser("export", help="Apache Ossie semantic model -> Databricks Metric View YAML") + exp.add_argument("-i", "--input", required=True, help="Apache Ossie YAML file") + exp.add_argument("-o", "--output", help="output Metric View YAML (default: stdout)") + exp.add_argument("-s", "--source", + help="dataset to use as the fact/grain (default: the FK-sink dataset); " + "naming a coarser-grain dataset unlocks one_to_many joins") + + imp = sub.add_parser("import", help="Databricks Metric View YAML -> Apache Ossie semantic model") + imp.add_argument("-i", "--input", required=True, help="Metric View YAML file") + imp.add_argument("-o", "--output", help="output Apache Ossie YAML (default: stdout)") + imp.add_argument("--name", help="Apache Ossie model name (default: derived from the source)") + return parser + + +def main(argv=None): + args = _build_parser().parse_args(argv) + try: + with open(args.input) as fh: + text = fh.read() + if args.command == "export": + out = convert_ossie_to_metric_view(text, source=args.source) + else: + out = convert_metric_view_to_ossie(text, model_name=args.name) + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + if args.output: + with open(args.output, "w") as fh: + fh.write(out) + else: + sys.stdout.write(out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py b/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py new file mode 100644 index 0000000..75cdb75 --- /dev/null +++ b/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py @@ -0,0 +1,366 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert a Databricks Unity Catalog Metric View (v1.1) to an Apache Ossie semantic model. + +Pure offline conversion. Accepts a Metric View (one `source` with a nested `joins` +tree). Metric View features Apache Ossie has no native field for -- filter, window, format, +rely, cardinality, parameters, materialization -- are preserved in +`custom_extensions[DATABRICKS]` so that converting back reproduces the original view. +A join condition an Apache Ossie relationship cannot represent (a non-equi or cross join) is +rejected, not stashed. See README.md. + +Usage (CLI): + ossie-databricks import -i view.yaml [-o model.yaml] [--name NAME] +""" + +import re +import warnings + +from ._common import ( + CARD_MANY_TO_ONE, + CARD_ONE_TO_MANY, + ConversionError, + DIALECT_DATABRICKS, + MV_VERSION, + OSSIE_VERSION, + STASH_SOURCE_KEY, + dump_yaml, + is_simple_identifier, + last_identifier, + load_yaml, + require, + require_str, + validate_source, + write_stash, +) + +# Metric View fields with no native Apache Ossie home -> stashed verbatim. +_MODEL_STASH_KEYS = ("filter", "parameters", "materialization") +_JOIN_STASH_KEYS = ("rely", "cardinality") +_COLUMN_STASH_KEYS = ("format", "window") + + +def _warn(scope, msg): + warnings.warn(f"[{scope}] {msg}") + + +# Operators that mean a join condition is NOT a simple equi-join (so it cannot be +# expressed as from_columns/to_columns, and the join is rejected on import). +_NON_EQUI_RE = re.compile(r"[<>!]=|<>|[<>]") + + +def _is_wildcard(col): + """A wildcard column (`expr: source.*`) is projected without a `name`; Apache Ossie has + no representation for it. Detected by the absence of a `name` key (a named column, + even one whose name is falsy like `0` or whose expression contains `*` as + multiplication, is not a wildcard).""" + return "name" not in col + + +def convert_metric_view_to_ossie(mv_yaml_str, model_name=None): + """Parse Metric View v1.1 YAML and return Apache Ossie semantic model YAML (string).""" + # load_yaml uses YAML 1.2 booleans, so a join `on:` key stays the string "on" + # (PyYAML's default 1.1 would parse it as the boolean True and drop the condition). + view = load_yaml(mv_yaml_str) + if not isinstance(view, dict): + raise ConversionError("Invalid Metric View YAML: expected a mapping at the root") + + version = str(view.get("version", "")) + if version != MV_VERSION: + raise ConversionError( + f"Unsupported Metric View version '{version}'. This converter targets " + f"v{MV_VERSION} only." + ) + + model = _convert_view(view, model_name) + return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}) + + +def _convert_view(view, model_name): + source = view.get("source") + if not source: + raise ConversionError("Metric View is missing required 'source'") + + # Derive the model/fact name from a table source's last identifier. A SELECT/WITH + # subquery source has no meaningful table name, so use a stable default instead of + # slicing a token out of the SQL text (override with --name). + is_sql = str(source).strip().split(None, 1)[0].upper() in ("SELECT", "WITH") + last_id = last_identifier(source) + fact_name = model_name or ( + last_id if (not is_sql and last_id and is_simple_identifier(last_id)) + else "metric_view" + ) + # Validate the source shape up front (3-part table or SELECT/WITH), mirroring the + # exporter -- so a malformed source fails here with a clean error instead of passing + # silently through to Apache Ossie and only erroring on a later re-export. + validate_source(source, fact_name) + + datasets = [{"name": fact_name, "source": source}] + relationships = [] + alias_to_dataset = {"source": fact_name, fact_name: fact_name} + # Names are compared case-insensitively (DBR identifiers are case-insensitive), so a + # `Fact`/`fact` or `dim`/`Dim` collision is caught here instead of producing two + # datasets DBR would reject on re-export. + seen_names = {fact_name.strip().lower()} + + # Walk the join tree, emitting one dataset + one relationship per join. + def walk(parent_name, parent_alias, joins): + for join in joins or []: + child = require_str(join, "name", "join") + # `source` is the reserved fact qualifier; reject any casing. + if child.strip().lower() == "source": + raise ConversionError( + "Join name 'source' is reserved for the fact source; rename the join." + ) + if child.strip().lower() in seen_names: + raise ConversionError( + f"Duplicate dataset/join name '{child}'; Metric View join names " + f"and the source must be distinct (case-insensitively)." + ) + seen_names.add(child.strip().lower()) + child_ds = {"name": child, "source": require_str(join, "source", f"join '{child}'")} + datasets.append(child_ds) + alias_to_dataset[child] = child + rel = _convert_join(join, parent_name, parent_alias, child) + relationships.append(rel) + # rely.at_most_one_match asserts the join key is unique on the joined + # (one) side, so record those columns as a unique key on the child dataset + # -- recovering key info Apache Ossie would otherwise lack. Only a many_to_one join + # has the child on the `to` side (one_to_many flips it), so this naturally + # skips one_to_many joins. + if (rel["to"] == child and rel.get("to_columns") + and (join.get("rely") or {}).get("at_most_one_match")): + child_ds["unique_keys"] = [list(rel["to_columns"])] + walk(child, child, join.get("joins")) + + walk(fact_name, "source", view.get("joins")) + + # Dimensions -> fields, grouped onto the dataset their alias points at. + # `fields` is a v1.1 alias for `dimensions` (and the form the DBR docs use), + # so accept either key. + if view.get("dimensions") and view.get("fields"): + _warn("view", "both 'dimensions' and 'fields' are set; 'fields' is a v1.1 alias " + "for 'dimensions', so the 'fields' list is ignored") + fields_by_dataset = {d["name"]: [] for d in datasets} + for dim in (view.get("dimensions") or view.get("fields") or []): + if _is_wildcard(dim): + _warn("dimension", f"wildcard column '{dim.get('expr')}' has no Apache Ossie field " + f"representation; skipped") + continue + ds_name, field = _convert_dimension(dim, alias_to_dataset, fact_name) + fields_by_dataset[ds_name].append(field) + for d in datasets: + flds = fields_by_dataset[d["name"]] + if flds: + d["fields"] = flds + + metrics = [] + for m in view.get("measures", []) or []: + if _is_wildcard(m): + _warn("measure", f"wildcard measure '{m.get('expr')}' has no Apache Ossie metric " + f"representation; skipped") + continue + metrics.append(_convert_measure(m, fact_name)) + + model = {"name": fact_name} + if view.get("comment"): + model["description"] = view["comment"] + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + + # Model-level stash: filter / parameters / materialization, plus the source + # dataset's identity when a one_to_many join is present -- without it the + # exporter's FK-sink heuristic would re-root at the wrong (many-side) dataset. + model_stash = {k: view[k] for k in _MODEL_STASH_KEYS if k in view} + if _has_otm(view.get("joins")): + model_stash[STASH_SOURCE_KEY] = fact_name + write_stash(model, model_stash) + return model + + +def _has_otm(joins): + """True if any join in the (nested) tree is one_to_many.""" + for j in joins or []: + if str(j.get("cardinality") or "").lower() == CARD_ONE_TO_MANY: + return True + if _has_otm(j.get("joins")): + return True + return False + + +def _convert_join(join, parent_name, parent_alias, child): + if not join.get("using") and not join.get("on"): + raise ConversionError( + f"Join '{child}' has no join condition (empty or absent 'on'/'using'); " + f"condition-less (cross) joins have no Apache Ossie relationship representation." + ) + # _decompose_on returns (parent-side columns, child-side columns). + parent_cols, child_cols, raw_on = _decompose_on(join, parent_alias, parent_name, child) + if raw_on is not None: + raise ConversionError( + f"Join '{child}' uses a non-equi or unsupported join condition ('on: {raw_on}') " + f"that an Apache Ossie relationship cannot represent. Apache Ossie joins are equi-joins of simple " + f"`alias.column` pairs (the fact side may be qualified with `source`, the source " + f"table name, or left bare). Cannot import." + ) + if "using" in join and not parent_cols: + # `using: [cols]` -> equal lists on both sides. Two distinct list objects, so the + # emitted YAML doesn't serialize one as an anchor/alias of the other. + parent_cols, child_cols = list(join["using"]), list(join["using"]) + + # Cardinality (default many_to_one) decides the Apache Ossie direction, since `from` is + # always the many side. many_to_one -> parent is many (from=parent); one_to_many + # -> the joined child is many (from=child, to=parent). Compared case-insensitively. + cardinality = join.get("cardinality") or CARD_MANY_TO_ONE + if str(cardinality).lower() == CARD_ONE_TO_MANY: + rel = {"name": f"{child}_to_{parent_name}", "from": child, "to": parent_name, + "from_columns": child_cols, "to_columns": parent_cols} + else: + rel = {"name": f"{parent_name}_to_{child}", "from": parent_name, "to": child, + "from_columns": parent_cols, "to_columns": child_cols} + + stash = {k: join[k] for k in _JOIN_STASH_KEYS if k in join} + write_stash(rel, stash) + return rel + + +def _decompose_on(join, parent_alias, parent_name, child_alias): + """Return (from_columns, to_columns, raw_on). + + raw_on is None when `on` decomposes cleanly into equi-join column pairs; it + holds the original string otherwise (a non-equi/complex condition the caller + rejects). `using` short-circuits to empty columns here and is handled by the caller. + + The child side of a clause is always referenced by its join name. The parent side + may be referenced by its alias (`source` at the top level, else the parent join + name) or by the parent dataset's own name. A bare (unqualified) operand is read as + the fact only at the top level; inside a nested join it is ambiguous (parent vs. + fact) and is rejected rather than guessed. + """ + if "using" in join: + return [], [], None + on = join.get("on") + if not on: + return [], [], None + + parent_aliases = {parent_alias, parent_name} + from_cols, to_cols = [], [] + for clause in re.split(r"\s+AND\s+", on, flags=re.IGNORECASE): + if _NON_EQUI_RE.search(clause): # >=, <=, !=, <>, <, > -> not an equi-join + return [], [], on + m = re.match(r"^\s*(.+?)\s*=\s*(.+?)\s*$", clause) + if not m: + return [], [], on + la, lc = _split_alias(m.group(1)) + ra, rc = _split_alias(m.group(2)) + # Both sides must be `.` (or a bare fact column). If an + # operand is a SQL fragment (e.g. `dim.b + 1`, or the trailing half of an + # OR/`=`-laden clause), `_split_alias` yields a non-identifier "column" -- + # that can't be an FK column pair, so stash the whole condition verbatim. + if not (is_simple_identifier(lc) and is_simple_identifier(rc)): + return [], [], on + # The parent side: its alias or the source table name. A *bare* (unqualified) + # operand is read as the fact only at the top level (`source`); inside a nested + # join an unqualified column is ambiguous (parent vs. fact), so don't guess -- + # leave it for rejection rather than silently attributing it to the parent. + allow_bare = parent_alias == "source" + l_parent = la in parent_aliases or (la is None and allow_bare) + r_parent = ra in parent_aliases or (ra is None and allow_bare) + if la == child_alias and r_parent: + from_cols.append(rc) + to_cols.append(lc) + elif ra == child_alias and l_parent: + from_cols.append(lc) + to_cols.append(rc) + else: + return [], [], on + return from_cols, to_cols, None + + +def _split_alias(operand): + """`customer.c_custkey` -> ('customer', 'c_custkey'); `x` -> (None, 'x').""" + operand = operand.strip() + if "." in operand: + alias, col = operand.split(".", 1) + return alias.strip(), col.strip() + return None, operand + + +def _convert_dimension(dim, alias_to_dataset, fact_name): + name = require_str(dim, "name", "dimension") + expr = require_str(dim, "expr", f"dimension '{name}'") + ds_name, ossie_expr = _resolve_column(expr, alias_to_dataset, fact_name) + + field = { + "name": name, + "expression": {"dialects": [{"dialect": DIALECT_DATABRICKS, "expression": ossie_expr}]}, + } + if dim.get("comment"): + field["description"] = dim["comment"] + if dim.get("display_name"): + field["label"] = dim["display_name"] + if dim.get("synonyms"): + field["ai_context"] = {"synonyms": list(dim["synonyms"])} + write_stash(field, {k: dim[k] for k in _COLUMN_STASH_KEYS if k in dim}) + return ds_name, field + + +def _resolve_column(expr, alias_to_dataset, fact_name): + """Map a dimension expression to (dataset_name, de-aliased_expression). + + A leading join path of known aliases files the field under the **deepest** one and + de-qualifies a bare column -- mirroring the exporter's nested-join qualification: + `partsupp.supplier.nation.n_name` -> `n_name` on `nation`; `customer.c_name` -> + `c_name` on `customer`; `source.x` -> `x` on the fact. A complex expression is filed + under that dataset but kept verbatim. A bare column (no leading alias) is a fact column. + """ + segments = [s.strip() for s in expr.split(".")] + ds = None + i = 0 + # Consume leading segments that are known join/source aliases (but never the last + # segment -- that is the column). The deepest alias is the owning dataset. + while i < len(segments) - 1 and segments[i] in alias_to_dataset: + ds = alias_to_dataset[segments[i]] + i += 1 + if ds is None: + return fact_name, expr + rest = ".".join(segments[i:]) + return (ds, rest) if is_simple_identifier(rest) else (ds, expr) + + +def _convert_measure(measure, fact_name): + name = require_str(measure, "name", "measure") + # Mirror the exporter's word-boundary handling: a `source.` fact qualifier (if + # an author used one) maps back to the fact dataset name. The replacement is a + # lambda so `fact_name` is inserted literally (a `--name` containing backslashes + # is not interpreted as a regex backreference). + raw_expr = require_str(measure, "expr", f"measure '{name}'") + expr = re.sub(r"\bsource\.", lambda _m: f"{fact_name}.", raw_expr) + metric = { + "name": name, + "expression": {"dialects": [{"dialect": DIALECT_DATABRICKS, "expression": expr}]}, + } + if measure.get("comment"): + metric["description"] = measure["comment"] + if measure.get("synonyms"): + metric["ai_context"] = {"synonyms": list(measure["synonyms"])} + write_stash(metric, {k: measure[k] for k in _COLUMN_STASH_KEYS if k in measure}) + return metric diff --git a/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py b/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py new file mode 100644 index 0000000..3c535ed --- /dev/null +++ b/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py @@ -0,0 +1,660 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an Apache Ossie semantic model to a Databricks Unity Catalog Metric View (v1.1). + +Pure offline conversion -- no Databricks connection required. Produces the +Metric View: one fact `source` with a nested `joins` tree and +all fields flattened into one `dimensions` list. See README.md for the +capability summary and limitations. + +Usage (CLI): + ossie-databricks export -i model.yaml [-o view.yaml] [--source orders] +""" + +import re +import warnings + +from ._common import ( + CARD_ONE_TO_MANY, + ConversionError, + MV_VERSION, + OSSIE_VERSION, + STASH_SOURCE_KEY, + SYNONYM_LIMIT, + dump_yaml, + foreign_vendor_extensions, + is_simple_identifier, + load_yaml, + merge_description, + pick_expression, + read_stash, + require, + require_str, + synonyms_of, + validate_source, +) + + +def _warn(scope, msg): + warnings.warn(f"[{scope}] {msg}") + + +# Fanning a diamond out into per-path joins can expand exponentially on a pathological +# lattice; real snowflakes are tiny, so cap total joins to catch runaway inputs. +_MAX_JOIN_NODES = 200 + + +def convert_ossie_to_metric_view(ossie_yaml_str, source=None): + """Parse Apache Ossie YAML and return Databricks Metric View v1.1 YAML (string). + + `source` names the dataset to use as the view's fact/grain. When omitted, the + fact is the dataset that is never a relationship `to` (the FK sink of a plain + many-to-one star). Naming a coarser-grain dataset as the source is what unlocks + `one_to_many` joins (the joined detail tables sit on the `from`/many side). + """ + root = load_yaml(ossie_yaml_str) + if not isinstance(root, dict): + raise ConversionError("Invalid Apache Ossie YAML: expected a mapping at the root") + + version = str(root.get("version", "")) + if version != OSSIE_VERSION: + raise ConversionError( + f"Unsupported Apache Ossie version '{version}'. Supported: {OSSIE_VERSION}" + ) + + models = root.get("semantic_model") + if not isinstance(models, list) or not models: + raise ConversionError("'semantic_model' must be a non-empty list") + if len(models) > 1: + _warn("model", "multiple semantic models found; converting only the first") + + view = _convert_model(models[0], explicit_source=source) + return dump_yaml(view) + + +def _convert_model(model, explicit_source=None): + name = model.get("name", "") + dataset_list = model.get("datasets", []) or [] + if not dataset_list: + raise ConversionError(f"Model '{name}' has no datasets") + + seen = set() + for d in dataset_list: + ds_name = require_str(d, "name", f"Model '{name}': dataset") + if ds_name.strip().lower() in seen: # case-insensitive: DBR identifiers are + raise ConversionError(f"Model '{name}': duplicate dataset name '{ds_name}'") + seen.add(ds_name.strip().lower()) + datasets = {d["name"]: d for d in dataset_list} + relationships = model.get("relationships", []) or [] + + # Model-level stash: filter / parameters / materialization, plus an optional + # `source_dataset` recording the original grain (written on import only when a + # one_to_many join made the fact ambiguous). An explicit `source` arg wins. + model_stash = read_stash(model) + fact_hint = explicit_source or model_stash.get(STASH_SOURCE_KEY) + root, fact = _build_join_tree(name, datasets, relationships, fact_hint) + counts = _assign_aliases(root, fact) + + # Mark one_to_many nodes (parent on the `to`/one side); their columns can't be + # dimensions. Also validates one_to_many subtree uniformity. + _mark_otm(name, root) + + fact_ds = datasets[fact] + view = {"version": MV_VERSION, "source": validate_source(fact_ds.get("source"), fact)} + + # The view comment is simply the model's top-level description -- the closest + # match. Model ai_context and dataset descriptions are not merged in (dropped; + # see _warn_dropped_model), which keeps model.description round-trippable. + comment = model.get("description") + if comment: + view["comment"] = comment + + if "filter" in model_stash: + view["filter"] = model_stash["filter"] + + joins = [_build_join(child, "source", datasets) for child in root["children"]] + if joins: + view["joins"] = joins + + # Dimensions: every field across every join instance, fact first then join order. + # A dataset joined under more than one alias (fanned out) is emitted once per + # instance with alias-prefixed names. Track dropped names so we can cascade-drop + # anything that references them. + dropped_dims, dropped_measures = set(), set() + dimensions = [] + seen_dims = set() + for node, join_path in _node_order(root): + is_fact = node is root + prefix = node["alias"] if counts[node["dataset"]] > 1 else None + for field in datasets[node["dataset"]].get("fields", []) or []: + fname = require_str(field, "name", f"dataset '{node['dataset']}': field") + if node["is_otm"]: + _warn( + f"field '{fname}'", + "column on a one-to-many-joined table cannot be a dimension " + "(must resolve to one value per source row); dropped", + ) + dropped_dims.add(fname) + continue + dim = _convert_field(field, fname, ".".join(join_path), is_fact, prefix) + if dim is None: + dropped_dims.add(fname) + continue + if dim["name"].lower() in seen_dims: # case-insensitive + raise ConversionError( + f"dataset '{node['dataset']}': dimension name '{dim['name']}' " + f"collides with another dimension/measure; Metric Views require " + f"unique dimension/measure names -- rename before use" + ) + seen_dims.add(dim["name"].lower()) + dimensions.append(dim) + + measures = [] + for metric in model.get("metrics", []) or []: + measure = _convert_metric(metric, fact, seen_dims) + if measure is None: + dropped_measures.add(metric.get("name")) + continue + measures.append(measure) + + # Cascade: drop any dimension/measure whose expression references a dropped + # name (transitively), so we never emit a dangling reference. + _cascade_drop(dimensions, measures, dropped_dims, dropped_measures) + + if dimensions: + view["dimensions"] = dimensions + if measures: + view["measures"] = measures + + if "parameters" in model_stash: + view["parameters"] = model_stash["parameters"] + if "materialization" in model_stash: + view["materialization"] = model_stash["materialization"] + + _warn_dropped_model(model) + return view + + +def _build_join_tree(model_name, datasets, relationships, fact_hint=None): + """Build the Metric View join tree from the Apache Ossie relationship graph; return + (root_node, fact_name). + + Each node is a dict: {alias, dataset, rel, parent_is_from, children, is_otm}. + Edges are oriented away from the fact (the nearer endpoint is the parent), so a + dataset reachable by more than one path -- a diamond, e.g. two facts sharing a + dimension, or a dimension reached via two parents -- is fanned out into one node + per path. Each instance is later given a unique alias, mirroring how a Metric View + joins the same table more than once. Non-tree (cyclic) shapes are rejected. + """ + for rel in relationships: + scope = f"Model '{model_name}': relationship '{rel.get('name', '')}'" + if require(rel, "from", scope) not in datasets or require(rel, "to", scope) not in datasets: + raise ConversionError( + f"Model '{model_name}': relationship '{rel.get('name')}' references " + f"an unknown dataset" + ) + + # Re-orient any relationship whose declared keys show `from`/`to` is mislabeled + # (the `from` columns are a unique key, the `to` columns are not). Done before + # fact selection so cardinality, columns, and fact choice all use the key-derived + # orientation. The join condition is unchanged (it is symmetric). + relationships = [_orient_by_key(model_name, rel, datasets) for rel in relationships] + + fact = _pick_fact(model_name, datasets, relationships, fact_hint) + + # BFS (undirected) measures each dataset's distance from the fact; that distance + # orients every edge away from the fact (parent = the nearer endpoint). + adj = {name: [] for name in datasets} + for rel in relationships: + adj[rel["from"]].append(rel["to"]) + adj[rel["to"]].append(rel["from"]) + dist = {fact: 0} + queue = [fact] + while queue: + cur = queue.pop(0) + for neighbor in adj[cur]: + if neighbor not in dist: + dist[neighbor] = dist[cur] + 1 + queue.append(neighbor) + + unreachable = set(datasets) - set(dist) + if unreachable: + raise ConversionError( + f"Model '{model_name}': datasets {sorted(unreachable)} are not reachable " + f"from fact '{fact}' via relationships." + ) + + # Orient each edge nearer->farther. An edge between two equidistant datasets has + # no fact-ward direction -- that only happens in a cyclic / non-tree graph. + children_of = {name: [] for name in datasets} + for rel in relationships: + a, b = rel["from"], rel["to"] + if dist[a] == dist[b]: + raise ConversionError( + f"Model '{model_name}': relationship '{rel.get('name')}' joins two " + f"datasets equidistant from the fact; the graph is not tree-shaped " + f"(it contains a cycle)." + ) + parent, child = (a, b) if dist[a] < dist[b] else (b, a) + children_of[parent].append((child, rel, parent == rel["from"])) + + counter = [0] + + def build(dataset, rel, parent_is_from): + counter[0] += 1 + if counter[0] > _MAX_JOIN_NODES: + raise ConversionError( + f"Model '{model_name}': join graph fans out to more than " + f"{_MAX_JOIN_NODES} joins; check for an unintended diamond explosion." + ) + node = {"alias": None, "dataset": dataset, "rel": rel, + "parent_is_from": parent_is_from, "children": [], "is_otm": False} + for child, crel, cfrom in children_of[dataset]: + node["children"].append(build(child, crel, cfrom)) + return node + + return build(fact, None, None), fact + + +def _assign_aliases(root, fact): + """Give every node a unique join alias and return per-dataset instance counts. + + A dataset with a single instance keeps its bare name (so non-diamond graphs are + unchanged); a fanned-out dataset's instances are disambiguated by parent alias + (e.g. `customers_regions` / `suppliers_regions`). The fact's alias is `source`. + """ + counts = {} + + def count(node): + counts[node["dataset"]] = counts.get(node["dataset"], 0) + 1 + for c in node["children"]: + count(c) + + count(root) + + used = {"source"} # reserved for the fact, so a dataset named `source` gets renamed + + def assign(node, parent_alias): + if node["dataset"] == fact: + alias = "source" + else: + # Single-instance datasets keep their bare name; fanned-out ones are + # qualified by the parent alias. Either way the result is deduped against + # `used` (which reserves `source`), so no two joins ever share an alias. + if counts[node["dataset"]] == 1: + base = node["dataset"] + else: + base = (f"{parent_alias}_{node['dataset']}" + if parent_alias and parent_alias != "source" else node["dataset"]) + alias, n = base, 2 + while alias in used: + alias, n = f"{base}_{n}", n + 1 + node["alias"] = alias + used.add(alias) + for c in node["children"]: + assign(c, alias) + + assign(root, None) + return counts + + +def _pick_fact(model_name, datasets, relationships, fact_hint): + """Choose the fact/root: an explicit hint if given, else the dataset that is + never a relationship `to` (the FK sink of a plain many-to-one star).""" + if fact_hint is not None: + if fact_hint not in datasets: + raise ConversionError( + f"Model '{model_name}': requested source '{fact_hint}' is not a dataset" + ) + return fact_hint + if len(datasets) > 1 and not relationships: + raise ConversionError( + f"Model '{model_name}': {len(datasets)} datasets but no relationships; " + f"cannot determine the fact table." + ) + incoming = {name: 0 for name in datasets} + for rel in relationships: + incoming[rel["to"]] += 1 + roots = [n for n, c in incoming.items() if c == 0] + if not roots: + raise ConversionError( + f"Model '{model_name}': join graph contains a cycle (no root dataset). " + f"A Metric View requires an acyclic, tree-shaped graph." + ) + if len(roots) > 1: + raise ConversionError( + f"Model '{model_name}': multiple candidate fact datasets {sorted(roots)}. " + f"Name the grain with --source -- e.g. for multiple facts sharing a " + f"dimension, name that dimension so each fact becomes a one_to_many join." + ) + return roots[0] + + +def _mark_otm(model_name, root): + """Mark each node `is_otm` (reached through a one_to_many join -- a parent on the + `to`/one side). Their columns can't be dimensions. Enforces the DBR rule that + every descendant of a one_to_many join is itself one_to_many.""" + + def visit(node, under_otm): + for child in node["children"]: + is_otm = not child["parent_is_from"] # parent on the `to` (one) side + if under_otm and not is_otm: + raise ConversionError( + f"Model '{model_name}': join '{child['alias']}' is many-to-one but " + f"descends from a one-to-many join; all descendants of a one-to-many " + f"join must also be one-to-many (Databricks Metric View rule)." + ) + child["is_otm"] = under_otm or is_otm + visit(child, child["is_otm"]) + + visit(root, False) + + +def _node_order(root): + """Fact first, then a stable depth-first walk of the join tree (one node per join + instance, so a fanned-out dataset appears once per path). Yields (node, join_path): + `join_path` is the tuple of join aliases from the source down to and including the + node (empty for the fact). A joined column is qualified in a dimension/measure by this + full path (`parent.child.col`) -- the Databricks nested-join rule -- which for a + depth-1 join is just the join's own name.""" + order = [] + + def visit(node, path): + order.append((node, path)) + for child in node["children"]: + visit(child, path + (child["alias"],)) + + visit(root, ()) + return order + + +def _build_join(node, parent_alias, datasets): + """Build one Metric View join entry from a tree node (recursively for nested joins). + + `node['parent_is_from']` is True when the parent is the relationship's `from` + (many) side -> a many_to_one join (the default, left implicit). When the parent is + the `to` (one) side the join is one_to_many and the column roles flip. + """ + rel, alias = node["rel"], node["alias"] + join = {"name": alias, + "source": validate_source(datasets[node["dataset"]].get("source"), node["dataset"])} + + stash = read_stash(rel) + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + # Apache Ossie relationships are equi-joins; a relationship without usable equi columns + # (e.g. a non-equi join the importer would have rejected) is rejected here too. + _validate_join_columns(rel, from_cols, to_cols) + # Write parent-side = child-side: the parent uses whichever list belongs to + # it -- from_columns when it is the `from`, to_columns when it is the `to`. + parent_cols, child_cols = ( + (from_cols, to_cols) if node["parent_is_from"] else (to_cols, from_cols)) + if parent_cols == child_cols: + # Equal column lists are an equi-join on shared names -> `using`, which + # round-trips faithfully (the importer maps `using` to equal lists). + join["using"] = list(parent_cols) + else: + join["on"] = " AND ".join( + f"{parent_alias}.{pc} = {alias}.{cc}" for pc, cc in zip(parent_cols, child_cols) + ) + # rely.at_most_one_match: a stashed value round-trips verbatim; otherwise derive it + # for a many_to_one join whose `to_columns` cover a declared primary/unique key of + # the joined dataset (joining on a key matches at most one row -- no fan-out). + if "rely" in stash: + join["rely"] = stash["rely"] + elif node["parent_is_from"] and _covers_unique_key(datasets[node["dataset"]], to_cols): + join["rely"] = {"at_most_one_match": True} + # Cardinality: an explicit stashed value round-trips verbatim; otherwise derive + # from orientation -- parent on the `to` (one) side means one_to_many. The + # many_to_one default is left implicit. + if "cardinality" in stash: + join["cardinality"] = stash["cardinality"] + elif not node["parent_is_from"]: + join["cardinality"] = CARD_ONE_TO_MANY + + nested = [_build_join(c, alias, datasets) for c in node["children"]] + if nested: + join["joins"] = nested + return join + + +def _covers_unique_key(dataset, join_cols): + """True if `join_cols` include a declared `primary_key` or one of `unique_keys` of + `dataset` -- i.e. joining on them matches at most one target row, so a many_to_one + join can assert `rely.at_most_one_match`.""" + cols = set(join_cols) + keys = [dataset.get("primary_key")] if dataset.get("primary_key") else [] + keys += dataset.get("unique_keys") or [] + return any(key and set(key) <= cols for key in keys) + + +def _orient_by_key(model_name, rel, datasets): + """`to` should be the unique 'one' side (per spec `to_columns` are key columns). If + the declared keys say otherwise -- the `from` columns are a unique key while the + `to` side declares keys its `to_columns` don't cover -- `from`/`to` is mislabeled. + Return a copy with `from`/`to` (and their columns) swapped, and warn. The swap is + symmetric, so the join condition is unchanged; only the orientation is corrected. + + When the `from` columns cover a unique key but the `to` side declares no key at all, + the orientation can't be verified either way (the `to` side may or may not be + unique); leave it as-is but warn, since the resulting cardinality may be inverted.""" + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + if not from_cols or not to_cols: + return rel # non-equi / column-less: nothing to deduce from + to_ds = datasets[rel["to"]] + to_has_keys = bool(to_ds.get("primary_key") or to_ds.get("unique_keys")) + from_covers = _covers_unique_key(datasets[rel["from"]], from_cols) + if from_covers and to_has_keys and not _covers_unique_key(to_ds, to_cols): + _warn( + f"relationship '{rel.get('name')}'", + "from/to looks mislabeled (the `from` columns are a declared key, the `to` " + "columns are not); re-orienting so the key side is the `to`/one side", + ) + return {**rel, "from": rel["to"], "to": rel["from"], + "from_columns": to_cols, "to_columns": from_cols} + if from_covers and not to_has_keys: + _warn( + f"relationship '{rel.get('name')}'", + "the `from` columns are a declared key but the `to` side declares none, so " + "from/to orientation can't be verified; using it as-is -- check the join " + "direction if the resulting cardinality looks inverted", + ) + return rel + + +def _validate_join_columns(rel, from_cols, to_cols): + if not from_cols or not to_cols: + raise ConversionError( + f"Relationship '{rel.get('name')}': from_columns and to_columns are required" + ) + if not isinstance(from_cols, list) or not isinstance(to_cols, list): + raise ConversionError( + f"Relationship '{rel.get('name')}': from_columns and to_columns must be lists" + ) + if len(from_cols) != len(to_cols): + raise ConversionError( + f"Relationship '{rel.get('name')}': from_columns ({len(from_cols)}) and " + f"to_columns ({len(to_cols)}) must have the same length" + ) + + +def _convert_field(field, name, qualifier, is_fact, prefix=None): + scope = f"field '{name}'" + expr = pick_expression(field.get("expression")) + if expr is None: + _warn(scope, "no DATABRICKS/ANSI_SQL dialect; dropping field") + return None + + # Requalify a joined-table column with its full join-name path from the source + # (`parent.child.col`); a depth-1 join is just its own name. Only safe for bare + # columns. A complex expression on a single join is emitted as-is (likely resolves; + # warned). On a fanned-out (diamond) dataset it cannot be attributed to one of the + # instances, so it is dropped rather than emitted as an ambiguous dimension. + if not is_fact: + if is_simple_identifier(expr): + expr = f"{qualifier}.{expr}" + elif prefix: + _warn(scope, "complex expression on a fanned-out (diamond) join cannot be " + "unambiguously qualified; dropped") + return None + else: + _warn(scope, "complex expression on a joined table; emitted as-is, verify qualification") + + # A fanned-out dataset (joined under more than one alias) needs unique dimension + # names, so prefix with the instance alias (e.g. customer_region's r_name -> + # customer_region_r_name). Single-instance datasets keep the bare field name. + if prefix: + name = f"{prefix}_{name}" + + dim = {"name": name, "expr": expr} + comment = merge_description(field.get("description"), field.get("ai_context")) + if comment: + dim["comment"] = comment + if field.get("label"): + dim["display_name"] = field["label"] + syns = synonyms_of(field.get("ai_context")) + if syns: + dim["synonyms"] = _truncate_synonyms(syns, scope) + + stash = read_stash(field) + if "format" in stash: + dim["format"] = stash["format"] + _warn_dropped_field(field, scope) + return dim + + +def _convert_metric(metric, fact, seen_names): + name = require_str(metric, "name", "metric") + scope = f"metric '{name}'" + if name.lower() in seen_names: # case-insensitive (shares seen_dims with dimensions) + raise ConversionError( + f"metric '{name}' collides with another dimension/measure; Metric Views " + f"require unique dimension/measure names -- rename before use") + seen_names.add(name.lower()) + expr = pick_expression(metric.get("expression")) + if expr is None: + _warn(scope, "no DATABRICKS/ANSI_SQL dialect; dropping metric") + return None + + # Fact-table columns are referenced by bare name in measure expressions (DBR + # idiom: `SUM(amount)`, not `SUM(source.amount)`); strip a `.` qualifier. + # Joined-table columns keep their alias. Word boundary avoids touching a table + # whose name merely ends with the fact name (e.g. fact 'sales' vs 'store_sales'). + expr = re.sub(r"\b" + re.escape(fact) + r"\.", "", expr) + + measure = {"name": name, "expr": expr} + comment = merge_description(metric.get("description"), metric.get("ai_context")) + if comment: + measure["comment"] = comment + syns = synonyms_of(metric.get("ai_context")) + if syns: + measure["synonyms"] = _truncate_synonyms(syns, scope) + + stash = read_stash(metric) + if "format" in stash: + measure["format"] = stash["format"] + if "window" in stash: + measure["window"] = stash["window"] + return measure + + +def _references_dropped(expr, self_name, dropped_dims, dropped_measures): + """Return a dropped name referenced by `expr`, or None. + + Measures are only referenceable via `measure()` (exact). Dimensions are + referenced by their bare, *unqualified* name: a name that is part of a qualified + path (`alias.name` or `name.col`) is ignored, so a join alias or joined column + that merely shares a dropped dimension's name is not over-dropped. The one + ambiguity the regex can't resolve without a SQL parser is a bare, unqualified + *source column* sharing a dropped dimension's name -- there it errs on dropping. + """ + for m in dropped_measures: + if re.search(r"measure\(\s*" + re.escape(m) + r"\s*\)", expr): + return m + for d in dropped_dims: + # Match only a bare, unqualified token: the negative look-behind/ahead for a + # word char or `.` excludes both substrings of a larger identifier and + # qualified paths (`alias.name` / `name.col`), so a join alias or joined + # column sharing a dropped name is not falsely cascade-dropped. + if d != self_name and re.search( + r"(? SYNONYM_LIMIT: + _warn(scope, f"{len(syns)} synonyms exceeds Metric View limit; keeping first {SYNONYM_LIMIT}") + return syns[:SYNONYM_LIMIT] + return syns + + +def _warn_dropped_model(model): + if foreign_vendor_extensions(model): + _warn("model", "foreign-vendor custom_extensions dropped") + if model.get("ai_context"): # string or object -- only the description maps to comment + _warn("model", "model-level ai_context dropped (only the description maps to the view comment)") + for ds in model.get("datasets", []) or []: + scope = f"dataset '{ds['name']}'" + if ds.get("primary_key") or ds.get("unique_keys"): + _warn(scope, "primary_key/unique_keys not stored as columns; used to set " + "rely.at_most_one_match on a matching many_to_one join where applicable") + if isinstance(ds.get("ai_context"), dict) and ds["ai_context"]: + _warn(scope, "dataset-level ai_context (object) dropped") + # Dataset descriptions are not merged into the view comment (a Metric View + # has no per-source comment); only the model description is used. + if ds.get("description"): + _warn(scope, "dataset-level description dropped (no per-source comment field)") + if foreign_vendor_extensions(ds): + _warn(scope, "foreign-vendor custom_extensions dropped") + for rel in model.get("relationships", []) or []: + if rel.get("ai_context"): + _warn(f"relationship '{rel.get('name', '')}'", "relationship ai_context dropped") + + +def _warn_dropped_field(field, scope): + dim = field.get("dimension") + if isinstance(dim, dict) and "is_time" in dim: + _warn(scope, "dimension.is_time has no Metric View counterpart; dropped") + if foreign_vendor_extensions(field): + _warn(scope, "foreign-vendor custom_extensions dropped") diff --git a/converters/databricks/tests/_roundtrip_helpers.py b/converters/databricks/tests/_roundtrip_helpers.py new file mode 100644 index 0000000..6c4ddfb --- /dev/null +++ b/converters/databricks/tests/_roundtrip_helpers.py @@ -0,0 +1,355 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared model builders and round-trip assertions for property-based tests. + +This module is deliberately free of any third-party test dependency (no hypothesis, +no pytest) so the generation + assertion logic can run two ways: + + - driven by Hypothesis strategies (see test_roundtrip_properties.py), and + - driven by a plain seeded `random.Random` (RandomRnd below), which is how the + logic is exercised in environments where hypothesis is not installed. + +Both drivers implement the small `Rnd` interface (chance/count/pick/text/colname); +the builders below depend only on that interface, so the generated model space is +identical regardless of driver. + +The builders intentionally generate within the *round-trippable subset* of models - +the shapes the converter reproduces exactly. Known normalizations are avoided by +construction (documented inline) rather than asserted around, e.g.: + - join `on` conditions use distinct parent/child column names, so an equi-join on + shared names is never silently rewritten to `using`; + - measure expressions are emitted without a `source.` qualifier, which the exporter + would otherwise strip; + - in the Apache Ossie direction, joined-dataset fields are bare identifiers, so they survive + the alias requalify/de-qualify trip on the same dataset. +Name fuzzing (reserved words, collisions) is left to the targeted unit tests, which +assert the converter *rejects* those inputs. +""" + +import random +import re +import string +import warnings + +from ossie_databricks import metric_view_to_ossie as importer +from ossie_databricks import ossie_to_metric_view as exporter +from ossie_databricks._common import MV_VERSION, OSSIE_VERSION, dump_yaml, load_yaml + +_AGGS = ["SUM", "COUNT", "AVG", "MIN", "MAX"] + + +# --- Rnd backend for offline (no hypothesis) runs -------------------------------- + +class RandomRnd: + """The `Rnd` interface backed by a seeded `random.Random`.""" + + def __init__(self, seed): + self.r = random.Random(seed) + + def chance(self, p=0.5): + return self.r.random() < p + + def count(self, lo, hi): + return self.r.randint(lo, hi) + + def pick(self, seq): + return self.r.choice(list(seq)) + + def text(self): + # Alphanumeric with optional interior spaces; no leading/trailing space and + # no YAML-special characters, so the value is preserved verbatim through a + # dump/load cycle (any failure then reflects the converter, not PyYAML). + alnum = string.ascii_letters + string.digits + n = self.r.randint(0, 10) + body = "".join(self.r.choice(alnum + " ") for _ in range(n)) + return (self.r.choice(alnum) + body).strip() or "x" + + def colname(self): + first = self.r.choice(string.ascii_lowercase + "_") + rest = "".join( + self.r.choice(string.ascii_lowercase + string.digits + "_") + for _ in range(self.r.randint(0, 7)) + ) + return first + rest + + +# --- Small generation helpers ---------------------------------------------------- + +class _Names: + """Hands out globally-unique names with a given prefix.""" + + def __init__(self): + self._n = {} + + def next(self, prefix): + i = self._n.get(prefix, 0) + self._n[prefix] = i + 1 + return f"{prefix}{i}" + + +def _maybe_meta(rnd, target): + """Attach optional comment/display_name/synonyms/format to a dim/measure dict.""" + if rnd.chance(0.4): + target["comment"] = rnd.text() + if rnd.chance(0.3): + target["display_name"] = rnd.text() + if rnd.chance(0.3): + target["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 3))] + if rnd.chance(0.25): + fmt = {"type": rnd.pick(["number", "currency", "date"])} + if fmt["type"] == "currency": + fmt["currency_code"] = "USD" + target["format"] = fmt + + +# --- Metric View builder (for MV -> Apache Ossie -> MV) ----------------------------------- + +def _build_join(rnd, names, parent_alias, depth, ancestor_path): + name = names.next("j") + # Full join-name path from the source -> how dimensions/measures qualify this join's + # columns (DBR nested-join rule). `on:` conditions instead use the immediate names. + qual = ".".join(ancestor_path + [name]) + join = {"name": name, "source": _three_part(rnd)} + if rnd.chance(0.5): + ncols = rnd.count(1, 2) + join["using"] = [f"u{i}_{rnd.colname()}" for i in range(ncols)] + else: + ncols = rnd.count(1, 2) + # distinct parent/child names so the equi-join stays `on`, not `using` + pairs = [(f"fk{i}_{rnd.colname()}", f"pk{i}_{rnd.colname()}") for i in range(ncols)] + join["on"] = " AND ".join(f"{parent_alias}.{pc} = {name}.{cc}" for pc, cc in pairs) + if rnd.chance(0.4): + join["cardinality"] = "many_to_one" # only the lossless cardinality (see module doc) + if rnd.chance(0.3): + join["rely"] = {"at_most_one_match": True} + # dimensions on this join (qualified by the full join path) + dims = [] + for _ in range(rnd.count(0, 2)): + col = rnd.colname() + expr = (f"{qual}.{col}" if rnd.chance(0.7) + else f"{qual}.{col} + {qual}.{rnd.colname()}") + dim = {"name": names.next("c"), "expr": expr} + _maybe_meta(rnd, dim) + dims.append(dim) + if depth < 2 and rnd.chance(0.35): + child, child_dims = _build_join(rnd, names, name, depth + 1, ancestor_path + [name]) + join["joins"] = [child] + dims.extend(child_dims) + return join, dims + + +def build_metric_view(rnd): + """Generate a Metric View YAML dict in the round-trippable subset.""" + names = _Names() + mv = {"version": MV_VERSION, "source": _three_part(rnd)} + if rnd.chance(0.4): + mv["comment"] = rnd.text() + if rnd.chance(0.3): + mv["filter"] = f"{rnd.colname()} > 0" + + fields, joins = [], [] + for _ in range(rnd.count(0, 3)): # source dimensions (bare or function exprs) + col = rnd.colname() + expr = col if rnd.chance(0.7) else f"UPPER({col})" + dim = {"name": names.next("c"), "expr": expr} + _maybe_meta(rnd, dim) + fields.append(dim) + for _ in range(rnd.count(0, 2)): # join subtrees + join, jdims = _build_join(rnd, names, "source", 0, []) + joins.append(join) + fields.extend(jdims) + + measures = [] + for _ in range(rnd.count(0, 2)): + m = {"name": names.next("c"), "expr": f"{rnd.pick(_AGGS)}({rnd.colname()})"} + if rnd.chance(0.4): + m["comment"] = rnd.text() + if rnd.chance(0.3): + m["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 3))] + if rnd.chance(0.3): + m["window"] = [{"order": rnd.colname(), "range": "trailing 7 day"}] + measures.append(m) + + if joins: + mv["joins"] = joins + if fields: + mv["fields"] = fields + if measures: + mv["measures"] = measures + if rnd.chance(0.2): + mv["materialization"] = {"schedule": "every 6 hours", + "mode": rnd.pick(["relaxed", "strict"])} + return mv + + +# --- Apache Ossie builder (for Apache Ossie -> MV -> Apache Ossie) ------------------------------------------ + +def _ossie_field(name, expr): + return {"name": name, + "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": expr}]}} + + +def build_ossie(rnd): + """Generate an Apache Ossie semantic model dict in the round-trippable subset.""" + names = _Names() + fact = "fact" # fact name must equal its source's last identifier to round-trip + datasets = [{"name": fact, "source": f"c.s.{fact}"}] + relationships = [] + + n_dims = rnd.count(0, 3) + dim_names = [names.next("dim") for _ in range(n_dims)] + reachable = [fact] + for i, dname in enumerate(dim_names): + parent = rnd.pick(reachable) # star, or snowflake off an earlier node + ds = {"name": dname, "source": f"c.s.{rnd.colname()}{i}"} + datasets.append(ds) + reachable.append(dname) + if rnd.chance(0.5): # equal column names -> `using`; else distinct -> `on` + cols = [rnd.colname() for _ in range(rnd.count(1, 2))] + relationships.append({"name": names.next("r"), "from": parent, "to": dname, + "from_columns": list(cols), "to_columns": list(cols)}) + else: + n = rnd.count(1, 2) + fcols = [f"fk{j}_{rnd.colname()}" for j in range(n)] + tcols = [f"pk{j}_{rnd.colname()}" for j in range(n)] + relationships.append({"name": names.next("r"), "from": parent, "to": dname, + "from_columns": fcols, "to_columns": tcols}) + + # fields, bare identifiers, filed onto a random dataset (globally unique names) + for ds in datasets: + flds = [_ossie_field(names.next("c"), rnd.colname()) for _ in range(rnd.count(0, 3))] + if flds: + ds["fields"] = flds + + metrics = [{"name": names.next("c"), + "expression": {"dialects": [{"dialect": "DATABRICKS", + "expression": f"{rnd.pick(_AGGS)}({rnd.colname()})"}]}} + for _ in range(rnd.count(0, 2))] + + model = {"name": names.next("m")} + if rnd.chance(0.4): + model["description"] = rnd.text() + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + return {"version": OSSIE_VERSION, "semantic_model": [model]} + + +def _three_part(rnd): + return f"{rnd.colname()}.{rnd.colname()}.{rnd.colname()}" + + +# --- Round-trip assertions ------------------------------------------------------- + +def _convert(fn, text): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(text) + + +def _cond_canon(join): + if join.get("using"): + return ("using", tuple(sorted(join["using"]))) + on = join.get("on") + if not on: + return (None, None) + pairs = set() + for clause in re.split(r"\s+AND\s+", on, flags=re.IGNORECASE): + left, right = clause.split("=", 1) + pairs.add((left.strip(), right.strip())) + return ("on", frozenset(pairs)) + + +def _flatten_joins(joins, parent="source", acc=None, edges=None): + acc = {} if acc is None else acc + edges = set() if edges is None else edges + for j in joins or []: + acc[j["name"]] = {"source": j["source"], "cond": _cond_canon(j), + "cardinality": j.get("cardinality"), "rely": j.get("rely")} + edges.add((parent, j["name"])) + _flatten_joins(j.get("joins"), j["name"], acc, edges) + return acc, edges + + +def _dims(mv): + # The exporter emits the canonical `dimensions:` key; the importer also accepts the + # `fields:` alias. Read either so the comparison is key-name agnostic. + return mv.get("dimensions") or mv.get("fields") or [] + + +def _dim_norm(d): + return (d["expr"], d.get("comment"), d.get("display_name"), + d.get("synonyms"), d.get("format")) + + +def _meas_norm(m): + return (m["expr"], m.get("comment"), m.get("synonyms"), m.get("format"), m.get("window")) + + +def assert_mv_roundtrip(mv): + """A Metric View dict survives MV -> Apache Ossie -> MV with content preserved.""" + ossie_yaml = _convert(importer.convert_metric_view_to_ossie, dump_yaml(mv)) + mv2 = load_yaml(_convert(exporter.convert_ossie_to_metric_view, ossie_yaml)) + + assert mv2["source"] == mv["source"], "source" + assert mv2.get("comment") == mv.get("comment"), "comment" + assert mv2.get("filter") == mv.get("filter"), "filter" + assert mv2.get("materialization") == mv.get("materialization"), "materialization" + + assert ({d["name"]: _dim_norm(d) for d in _dims(mv)} + == {d["name"]: _dim_norm(d) for d in _dims(mv2)}), "fields" + assert ({m["name"]: _meas_norm(m) for m in mv.get("measures", [])} + == {m["name"]: _meas_norm(m) for m in mv2.get("measures", [])}), "measures" + + a1, e1 = _flatten_joins(mv.get("joins")) + a2, e2 = _flatten_joins(mv2.get("joins")) + assert a1 == a2, "joins" + assert e1 == e2, "join nesting" + + +def _expr_of(obj): + for d in obj["expression"]["dialects"]: + if d["dialect"] == "DATABRICKS": + return d["expression"] + return None + + +def _fields_map(ds): + return {f["name"]: _expr_of(f) for f in ds.get("fields", [])} + + +def _rel_set(model): + return {(r["from"], r["to"], tuple(r.get("from_columns") or []), + tuple(r.get("to_columns") or [])) + for r in model.get("relationships", [])} + + +def assert_ossie_roundtrip(ossie): + """An Apache Ossie model dict survives Apache Ossie -> MV -> Apache Ossie with content preserved.""" + mv_yaml = _convert(exporter.convert_ossie_to_metric_view, dump_yaml(ossie)) + ossie2 = load_yaml(_convert(importer.convert_metric_view_to_ossie, mv_yaml)) + + m1, m2 = ossie["semantic_model"][0], ossie2["semantic_model"][0] + assert ({d["name"]: (d["source"], _fields_map(d)) for d in m1["datasets"]} + == {d["name"]: (d["source"], _fields_map(d)) for d in m2["datasets"]}), "datasets" + assert _rel_set(m1) == _rel_set(m2), "relationships" + assert ({x["name"]: _expr_of(x) for x in m1.get("metrics", [])} + == {x["name"]: _expr_of(x) for x in m2.get("metrics", [])}), "metrics" + assert m1.get("description") == m2.get("description"), "description" diff --git a/converters/databricks/tests/_util.py b/converters/databricks/tests/_util.py new file mode 100644 index 0000000..e82a1d0 --- /dev/null +++ b/converters/databricks/tests/_util.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared test helpers: fixture loading and structural normalization.""" + +import copy +import json +import pathlib + +from ossie_databricks._common import load_yaml # src is on sys.path via conftest.py + +FIXTURES = pathlib.Path(__file__).resolve().parent / "fixtures" + + +def load_fixture(name): + with open(FIXTURES / name) as fh: + return fh.read() + + +def parse(yaml_str): + # YAML 1.2 booleans so a join `on:` key parses as the string "on" (matching the + # converter's own load/dump), not the YAML-1.1 boolean True. + return load_yaml(yaml_str) + + +def canon(obj): + """Deep-copy with every `custom_extensions[].data` JSON string parsed into a + dict, so comparisons are insensitive to JSON key order / whitespace.""" + obj = copy.deepcopy(obj) + + def walk(node): + if isinstance(node, dict): + for ext in node.get("custom_extensions") or []: + if isinstance(ext.get("data"), str): + ext["data"] = json.loads(ext["data"]) + for v in node.values(): + walk(v) + elif isinstance(node, list): + for v in node: + walk(v) + + walk(obj) + return obj + + +def strip_dropped(ossie): + """Normalize away what the Apache Ossie -> MV -> Apache Ossie trip changes, so a round-trip + comparison reflects the documented limitations. Besides outright losses (model + name, descriptions), a declared key transforms across the trip: `primary_key` -> + `rely.at_most_one_match` (MV) -> `unique_keys` + a relationship rely-stash. We drop + both key forms and the relationship stash so the key info is compared as 'gone'.""" + ossie = copy.deepcopy(ossie) + for model in ossie.get("semantic_model", []): + model.pop("name", None) # MV carries no model name + model.pop("description", None) # model + fact descriptions merge into one comment + for ds in model.get("datasets", []): + ds.pop("primary_key", None) + ds.pop("unique_keys", None) + ds.pop("description", None) # no per-source comment in single-source MV + for rel in model.get("relationships", []): + rel.pop("custom_extensions", None) # derived rely-stash from a declared key + return ossie diff --git a/converters/databricks/tests/conftest.py b/converters/databricks/tests/conftest.py new file mode 100644 index 0000000..0bcde53 --- /dev/null +++ b/converters/databricks/tests/conftest.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib +import sys + +# Make the converter modules in ../src importable from the tests. +_SRC = pathlib.Path(__file__).resolve().parent.parent / "src" +sys.path.insert(0, str(_SRC)) diff --git a/converters/databricks/tests/fixtures/fixtureA_metric_view.yaml b/converters/databricks/tests/fixtures/fixtureA_metric_view.yaml new file mode 100644 index 0000000..4ffe681 --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureA_metric_view.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Fixture A -- expected UC Metric View (v1.1, single-source) produced from +# fixtureA_ossie.yaml. Must parse under the v1.1 strict schema. + +version: '1.1' +source: samples.tpch.orders +comment: Sales orders with customer attributes +joins: +- name: customer + source: samples.tpch.customer + on: source.o_custkey = customer.c_custkey + rely: + at_most_one_match: true +dimensions: +- name: o_orderkey + expr: o_orderkey + comment: Order identifier +- name: o_orderdate + expr: o_orderdate + display_name: Order Date + synonyms: + - order date + - date +- name: c_name + expr: customer.c_name + comment: Customer name +measures: +- name: total_revenue + expr: SUM(o_totalprice) + comment: Total order revenue + synonyms: + - revenue + - total revenue + - sales +- name: order_count + expr: COUNT(*) + comment: Number of orders diff --git a/converters/databricks/tests/fixtures/fixtureA_ossie.yaml b/converters/databricks/tests/fixtures/fixtureA_ossie.yaml new file mode 100644 index 0000000..a53942f --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureA_ossie.yaml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yaml-language-server: $schema=../../../../core-spec/osi-schema.json +# +# Fixture A -- all-native round-trip (Apache Ossie -> MV -> Apache Ossie). +# Star schema; every field maps to a native MV field. Documented losses on the +# Apache Ossie -> MV -> Apache Ossie trip: the model name (MV carries none) and primary_key. + +version: "0.2.0.dev0" + +semantic_model: + - name: sales + description: Sales orders with customer attributes + datasets: + - name: orders # fact: no incoming relationship -> becomes `source` + source: samples.tpch.orders + primary_key: [o_orderkey] # dropped on export (Apache Ossie-only) + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderkey + description: Order identifier + - name: o_orderdate + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderdate + label: Order Date + ai_context: + synonyms: [order date, date] + - name: customer + source: samples.tpch.customer + primary_key: [c_custkey] + fields: + - name: c_name + expression: + dialects: + - dialect: DATABRICKS + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: [o_custkey] + to_columns: [c_custkey] + metrics: + - name: total_revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(o_totalprice) # fact columns are bare in measures + description: Total order revenue + ai_context: + synonyms: [revenue, total revenue, sales] + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(*) + description: Number of orders diff --git a/converters/databricks/tests/fixtures/fixtureB_metric_view.yaml b/converters/databricks/tests/fixtures/fixtureB_metric_view.yaml new file mode 100644 index 0000000..bbbbabc --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureB_metric_view.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Fixture B -- stash round-trip (MV -> Apache Ossie -> MV, lossless). +# Exercises the stash at every placement level: model (filter), relationship +# (rely), dimension (format), measure (format). Must parse under v1.1. + +version: '1.1' +source: samples.tpch.lineitem +filter: l_returnflag = 'N' +comment: Line item shipping metrics +joins: +- name: orders + source: samples.tpch.orders + on: source.l_orderkey = orders.o_orderkey + rely: + at_most_one_match: true +dimensions: +- name: line_number + expr: l_linenumber + format: + type: number + decimal_places: + type: exact + places: 0 +measures: +- name: revenue + expr: SUM(l_extendedprice * (1 - l_discount)) + comment: Net revenue + format: + type: currency + currency_code: USD + decimal_places: + type: exact + places: 2 +- name: order_count + expr: COUNT(DISTINCT l_orderkey) diff --git a/converters/databricks/tests/fixtures/fixtureB_ossie.yaml b/converters/databricks/tests/fixtures/fixtureB_ossie.yaml new file mode 100644 index 0000000..f2c11d9 --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureB_ossie.yaml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yaml-language-server: $schema=../../../../core-spec/osi-schema.json +# +# Fixture B -- expected Apache Ossie produced from fixtureB_metric_view.yaml. MV-only +# features are stashed in custom_extensions[DATABRICKS], keyed by their exact v1.1 +# field name. Exporting this back must reproduce fixtureB_metric_view.yaml. +# Model name is derived from the fact table (`lineitem`). + +version: "0.2.0.dev0" + +semantic_model: + - name: lineitem + description: Line item shipping metrics + datasets: + - name: lineitem + source: samples.tpch.lineitem + fields: + - name: line_number + expression: + dialects: + - dialect: DATABRICKS + expression: l_linenumber + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "number", "decimal_places": {"type": "exact", "places": 0}}}' + - name: orders + source: samples.tpch.orders + unique_keys: + - [o_orderkey] + relationships: + - name: lineitem_to_orders + from: lineitem + to: orders + from_columns: [l_orderkey] + to_columns: [o_orderkey] + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "rely": {"at_most_one_match": true}}' + metrics: + - name: revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(l_extendedprice * (1 - l_discount)) + description: Net revenue + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD", "decimal_places": {"type": "exact", "places": 2}}}' + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(DISTINCT l_orderkey) + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "filter": "l_returnflag = ''N''"}' diff --git a/converters/databricks/tests/fixtures/tpcds_metric_view.yaml b/converters/databricks/tests/fixtures/tpcds_metric_view.yaml new file mode 100644 index 0000000..b22fc69 --- /dev/null +++ b/converters/databricks/tests/fixtures/tpcds_metric_view.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '1.1' +source: tpcds.public.store_sales +comment: Store sales enriched with date, item, and customer dimensions +filter: ss_net_profit > 0 +joins: +- name: date_dim + source: tpcds.public.date_dim + on: source.ss_sold_date_sk = date_dim.d_date_sk + rely: + at_most_one_match: true +- name: item + source: tpcds.public.item + on: source.ss_item_sk = item.i_item_sk + rely: + at_most_one_match: true +- name: customer + source: tpcds.public.customer + on: source.ss_customer_sk = customer.c_customer_sk + rely: + at_most_one_match: true +dimensions: +- name: ticket_number + expr: ss_ticket_number +- name: sold_year + expr: date_dim.d_year + display_name: Year + synonyms: + - year + - yr +- name: sold_date + expr: date_dim.d_date +- name: item_category + expr: item.i_category + synonyms: + - category + - product type +- name: item_brand + expr: item.i_brand +- name: birth_country + expr: customer.c_birth_country +measures: +- name: total_sales + expr: SUM(ss_ext_sales_price) + comment: Total sales revenue + format: + type: currency + currency_code: USD +- name: total_quantity + expr: SUM(ss_quantity) + comment: Total units sold diff --git a/converters/databricks/tests/fixtures/tpcds_ossie.yaml b/converters/databricks/tests/fixtures/tpcds_ossie.yaml new file mode 100644 index 0000000..e8fc601 --- /dev/null +++ b/converters/databricks/tests/fixtures/tpcds_ossie.yaml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: "0.2.0.dev0" +semantic_model: + - name: tpcds_store_sales + description: Store sales enriched with date, item, and customer dimensions + datasets: + - name: store_sales + source: tpcds.public.store_sales + fields: + - name: ticket_number + expression: + dialects: [{dialect: DATABRICKS, expression: ss_ticket_number}] + - name: date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] + fields: + - name: sold_year + expression: + dialects: [{dialect: DATABRICKS, expression: d_year}] + label: Year + ai_context: {synonyms: [year, yr]} + - name: sold_date + expression: + dialects: [{dialect: DATABRICKS, expression: d_date}] + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] + fields: + - name: item_category + expression: + dialects: [{dialect: DATABRICKS, expression: i_category}] + ai_context: {synonyms: [category, product type]} + - name: item_brand + expression: + dialects: [{dialect: DATABRICKS, expression: i_brand}] + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] + fields: + - name: birth_country + expression: + dialects: [{dialect: DATABRICKS, expression: c_birth_country}] + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] + metrics: + - name: total_sales + expression: + dialects: [{dialect: DATABRICKS, expression: SUM(ss_ext_sales_price)}] + description: Total sales revenue + custom_extensions: + - vendor_name: DATABRICKS + data: '{"format": {"type": "currency", "currency_code": "USD"}}' + - name: total_quantity + expression: + dialects: [{dialect: DATABRICKS, expression: SUM(ss_quantity)}] + description: Total units sold + custom_extensions: + - vendor_name: DATABRICKS + data: '{"filter": "ss_net_profit > 0"}' diff --git a/converters/databricks/tests/test_metric_view_to_ossie.py b/converters/databricks/tests/test_metric_view_to_ossie.py new file mode 100644 index 0000000..0289003 --- /dev/null +++ b/converters/databricks/tests/test_metric_view_to_ossie.py @@ -0,0 +1,347 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the Databricks Metric View -> Apache Ossie importer.""" + +import pytest + +from ossie_databricks import ConversionError +from ossie_databricks import metric_view_to_ossie as importer +from _util import canon, load_fixture, parse + + +def test_fixtureB_import_matches_expected(): + out = importer.convert_metric_view_to_ossie(load_fixture("fixtureB_metric_view.yaml")) + assert canon(parse(out)) == canon(parse(load_fixture("fixtureB_ossie.yaml"))) + + +def test_fields_is_accepted_as_alias_for_dimensions(): + """`fields:` is a v1.1 alias for `dimensions:` (the form the DBR docs use); the + importer must read it, not silently drop the columns.""" + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "fields:\n- {name: region, expr: region}\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + fields = ossie["semantic_model"][0]["datasets"][0].get("fields", []) + assert [f["name"] for f in fields] == ["region"] + + +def test_unsupported_version_rejected(): + with pytest.raises(ConversionError): + importer.convert_metric_view_to_ossie("version: '0.1'\nsource: c.s.t\n") + + +def test_both_dimensions_and_fields_present_warns_and_uses_dimensions(): + """`fields` is a v1.1 alias for `dimensions`; if a (malformed) view sets both, the + importer uses `dimensions` and warns that the `fields` list is ignored.""" + import warnings + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "dimensions:\n- {name: kept, expr: kept}\n" + "fields:\n- {name: ignored, expr: ignored}\n" + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + names = [f["name"] for f in ossie["semantic_model"][0]["datasets"][0].get("fields", [])] + assert names == ["kept"] + assert any("fields" in str(w.message) and "ignored" in str(w.message) for w in caught) + + +def test_stash_written_at_each_level(): + ossie = parse(importer.convert_metric_view_to_ossie(load_fixture("fixtureB_metric_view.yaml"))) + model = ossie["semantic_model"][0] + + # model-level filter + assert any(e["vendor_name"] == "DATABRICKS" and "filter" in e["data"] + for e in model["custom_extensions"]) + # relationship-level rely + rel = model["relationships"][0] + assert any("rely" in e["data"] for e in rel["custom_extensions"]) + # metric-level format + revenue = next(m for m in model["metrics"] if m["name"] == "revenue") + assert any("format" in e["data"] for e in revenue["custom_extensions"]) + + +def test_name_override(): + ossie = parse(importer.convert_metric_view_to_ossie( + load_fixture("fixtureB_metric_view.yaml"), model_name="custom")) + assert ossie["semantic_model"][0]["name"] == "custom" + + +def test_cross_join_rejected(): + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: dim\n source: c.s.dim\n" + with pytest.raises(ConversionError, match="cross"): + importer.convert_metric_view_to_ossie(mv) + + +def test_duplicate_join_name_rejected(): + # a join named like the fact (derived from the source's last identifier) collides + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: fact\n source: c.s.other\n using: [id]\n" + with pytest.raises(ConversionError, match="Duplicate"): + importer.convert_metric_view_to_ossie(mv) + + +def test_complex_joined_dimension_filed_under_join_dataset(): + mv = ( + "version: '1.1'\nsource: c.s.fact\n" + "joins:\n- name: cust\n source: c.s.cust\n on: source.cid = cust.id\n" + "dimensions:\n- name: full\n expr: cust.a || cust.b\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + cust = next(d for d in ossie["semantic_model"][0]["datasets"] if d["name"] == "cust") + assert any(f["name"] == "full" for f in cust.get("fields", [])) + + +def test_non_equi_on_rejected(): + """Apache Ossie relationships are equi-joins (from_columns/to_columns required, minItems 1), so a + non-equi `on` has no Apache Ossie representation and is rejected on import (rather than emitting + a relationship with empty column lists, which is invalid per the Apache Ossie schema).""" + mv = ( + "version: '1.1'\n" + "source: c.s.fact\n" + "joins:\n" + "- name: dim\n" + " source: c.s.dim\n" + " on: source.a >= dim.b\n" + ) + with pytest.raises(ConversionError, match="non-equi"): + importer.convert_metric_view_to_ossie(mv) + + +def test_complex_equi_on_rejected(): + """An equi `on` whose operand is a SQL fragment (OR, computed) can't be decomposed + into from/to columns, so it's rejected rather than producing schema-invalid Apache Ossie with + empty column lists.""" + for cond in ("source.a = dim.b OR source.c = dim.d", "source.a = dim.b + 1"): + mv = ( + "version: '1.1'\nsource: c.s.fact\n" + f"joins:\n- name: dim\n source: c.s.dim\n on: {cond}\n" + ) + with pytest.raises(ConversionError, match="non-equi"): + importer.convert_metric_view_to_ossie(mv) + + +def test_one_to_many_join_flips_from_to_and_stashes_source(): + """A one_to_many MV join becomes an Apache Ossie relationship with the MANY side as `from` + (the joined table), the source/grain on the `to` side, and the grain recorded in + the model-level stash so re-export re-roots correctly.""" + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: line_items\n source: c.s.line_items\n" + " on: source.order_id = line_items.l_order_id\n" + " cardinality: one_to_many\n" + "measures:\n- {name: order_count, expr: COUNT(*)}\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + model = ossie["semantic_model"][0] + rel = model["relationships"][0] + assert rel["from"] == "line_items" # many side (holds the FK) + assert rel["to"] == "orders" # one side (holds the PK) + assert rel["from_columns"] == ["l_order_id"] + assert rel["to_columns"] == ["order_id"] + assert any(e["vendor_name"] == "DATABRICKS" and "source_dataset" in e["data"] + for e in model["custom_extensions"]) + + +def test_at_most_one_match_recovers_unique_key(): + """A many_to_one join with rely.at_most_one_match records the join key as a + unique_keys entry on the joined dataset (recovering key info Apache Ossie would lack).""" + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n" + " on: source.cid = customer.id\n" + " rely: {at_most_one_match: true}\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + cust = next(d for d in ossie["semantic_model"][0]["datasets"] if d["name"] == "customer") + assert cust.get("unique_keys") == [["id"]] + + +def test_join_named_source_rejected(): + """`source` is reserved for the fact; a join named `source` is rejected (DBR + forbids it too) rather than silently overwriting the fact alias.""" + mv = ("version: '1.1'\nsource: c.s.fact\n" + "joins:\n- name: source\n source: c.s.dim\n using: [id]\n") + with pytest.raises(ConversionError, match="reserved"): + importer.convert_metric_view_to_ossie(mv) + + +def test_sql_source_name_defaults_to_metric_view(): + """A SELECT/WITH source has no table name, so the model name defaults to + `metric_view` (not a token sliced out of the SQL).""" + mv = "version: '1.1'\nsource: SELECT a, b FROM main.sales.orders\n" + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + assert ossie["semantic_model"][0]["name"] == "metric_view" + + +def test_join_missing_source_raises(): + """Missing required keys surface as ConversionError, not a raw KeyError.""" + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: dim\n using: [id]\n" + with pytest.raises(ConversionError, match="missing required 'source'"): + importer.convert_metric_view_to_ossie(mv) + + +def test_measure_rewrite_with_regex_special_name(): + r"""The `source.` -> fact-name rewrite in measures inserts the name literally, so a + --name containing regex backreference syntax (e.g. \1) does not raise a re.error.""" + mv = "version: '1.1'\nsource: c.s.fact\nmeasures:\n- {name: rev, expr: SUM(source.amount)}\n" + ossie = parse(importer.convert_metric_view_to_ossie(mv, model_name=r"a\1b")) + expr = ossie["semantic_model"][0]["metrics"][0]["expression"]["dialects"][0]["expression"] + assert expr == r"SUM(a\1b.amount)" + + +def test_invalid_source_rejected(): + """A malformed source (not 3-part / not SELECT) is rejected on import, matching the + exporter -- rather than passing through and only failing on a later re-export.""" + mv = "version: '1.1'\nsource: a.b\n" # 2-part, invalid + with pytest.raises(ConversionError, match="3-part"): + importer.convert_metric_view_to_ossie(mv) + + +def test_malformed_yaml_raises_conversion_error(): + """Invalid YAML surfaces as ConversionError, not a raw yaml.YAMLError traceback.""" + with pytest.raises(ConversionError, match="Invalid YAML"): + importer.convert_metric_view_to_ossie("source: c.s.t\njoins: [oops\n") + + +def test_empty_using_rejected(): + """An empty `using: []` is a condition-less join -- rejected at import rather than + silently producing a relationship with empty key columns (which would then fail on + re-export).""" + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: dim\n source: c.s.dim\n using: []\n" + with pytest.raises(ConversionError, match="cross"): + importer.convert_metric_view_to_ossie(mv) + + +def test_boollike_string_values_stay_strings_for_a_yaml_1_1_reader(): + """Bool-like string scalars (e.g. the synonyms `on`/`off`) must be emitted quoted so a + stock YAML 1.1 reader reads them back as strings, not booleans.""" + import yaml + mv = ("version: '1.1'\nsource: c.s.t\n" + "dimensions:\n- {name: status, expr: status, synonyms: [on, off]}\n") + ossie_out = importer.convert_metric_view_to_ossie(mv) + field = yaml.safe_load(ossie_out)["semantic_model"][0]["datasets"][0]["fields"][0] + assert field["ai_context"]["synonyms"] == ["on", "off"] + + +def test_fact_qualifier_variants_in_on_decompose(): + """The fact side of an `on` is valid MV YAML whether qualified with `source`, the + source table name, or left bare; all decompose to the same equi-join columns rather + than being wrongly rejected as a non-equi condition (bug-bash finding).""" + base = ("version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n on: {cond}\n") + for cond in ( + "source.o_custkey = customer.c_custkey", # `source` qualifier + "orders.o_custkey = customer.c_custkey", # source table name + "o_custkey = customer.c_custkey", # bare fact column + "customer.c_custkey = o_custkey", # reversed operand order, bare fact + ): + rel = parse(importer.convert_metric_view_to_ossie( + base.format(cond=cond)))["semantic_model"][0]["relationships"][0] + assert rel["from"] == "orders" and rel["to"] == "customer" + assert rel["from_columns"] == ["o_custkey"] + assert rel["to_columns"] == ["c_custkey"] + + +def test_multi_column_on_with_bare_and_tablename_fact(): + """Composite keys decompose with bare / source-table-name fact qualifiers too.""" + mv = ("version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n" + " on: o_a = customer.c_a AND orders.o_b = customer.c_b\n") + rel = parse(importer.convert_metric_view_to_ossie(mv))["semantic_model"][0]["relationships"][0] + assert rel["from_columns"] == ["o_a", "o_b"] + assert rel["to_columns"] == ["c_a", "c_b"] + + +def test_join_named_source_rejected_any_case(): + """`source` is reserved case-insensitively (DBR identifiers are case-insensitive), + so `Source`/`SOURCE` are rejected too (bug-bash finding).""" + for name in ("Source", "SOURCE", "SoUrCe"): + mv = (f"version: '1.1'\nsource: c.s.fact\n" + f"joins:\n- name: {name}\n source: c.s.dim\n using: [id]\n") + with pytest.raises(ConversionError, match="reserved"): + importer.convert_metric_view_to_ossie(mv) + + +def test_empty_source_part_rejected(): + """A 3-dot source with an empty part (`.s.t`, `c..t`, `c.s.`) is not a valid 3-part + identifier and is rejected -- the dot count alone is not enough (bug-bash finding).""" + for src in (".s.t", "c..t", "c.s."): + mv = f"version: '1.1'\nsource: {src}\n" + with pytest.raises(ConversionError, match="3-part"): + importer.convert_metric_view_to_ossie(mv) + + +def test_whitespace_source_part_rejected(): + """A 3-dot source with a whitespace-laden part (`cat . sch . tbl`) is rejected -- + a space is not part of a valid identifier (review finding).""" + with pytest.raises(ConversionError, match="3-part"): + importer.convert_metric_view_to_ossie("version: '1.1'\nsource: cat . sch . tbl\n") + + +def test_with_paren_subquery_source_accepted(): + """A `WITH(...)` subquery with no space after the keyword is recognized as SQL, + not mistaken for a (non-3-part) identifier (review finding).""" + mv = "version: '1.1'\nsource: WITH(t AS (SELECT 1 AS a)) SELECT a FROM t\n" + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + assert ossie["semantic_model"][0]["datasets"][0]["source"].startswith("WITH(") + + +def test_nested_join_bare_column_rejected(): + """A bare (unqualified) operand in a NESTED join's `on` is ambiguous (parent vs. + fact), so it is rejected rather than silently attributed to the immediate parent. + A bare fact column is still accepted at the top level (review finding).""" + mv = ("version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n on: source.ckey = customer.c_key\n" + " joins:\n - name: nation\n source: c.s.nation\n on: o_nkey = nation.n_key\n") + with pytest.raises(ConversionError, match="non-equi or unsupported"): + importer.convert_metric_view_to_ossie(mv) + + +def test_case_variant_duplicate_name_rejected(): + """DBR identifiers are case-insensitive, so `dim`/`Dim` collide and must be rejected + (consistent with the case-insensitive reserved-`source` check) (review finding).""" + mv = ("version: '1.1'\nsource: c.s.x\n" + "joins:\n- {name: dim, source: c.s.a, using: [id]}\n- {name: Dim, source: c.s.b, using: [id]}\n") + with pytest.raises(ConversionError, match="[Dd]uplicate"): + importer.convert_metric_view_to_ossie(mv) + + +def test_falsy_dimension_name_is_not_a_wildcard(): + """A present-but-falsy name (`0`) is a malformed column, not a wildcard projection; + it raises a clean error rather than being silently dropped (review finding).""" + with pytest.raises(ConversionError): + importer.convert_metric_view_to_ossie("version: '1.1'\nsource: c.s.o\ndimensions:\n- {name: 0, expr: x}\n") + + +def test_non_string_scalars_raise_clean_error(): + """Non-string scalars where a string is required (join name, measure expr) raise a + ConversionError, not a raw AttributeError/TypeError (review finding).""" + for mv in ("version: '1.1'\nsource: c.s.f\njoins:\n- {name: 5, source: c.s.d, using: [id]}\n", + "version: '1.1'\nsource: c.s.o\nmeasures:\n- {name: rev, expr: 5}\n"): + with pytest.raises(ConversionError): + importer.convert_metric_view_to_ossie(mv) + + +def test_using_join_emits_no_yaml_anchor(): + """`from_columns`/`to_columns` of a `using` join are distinct list objects, so the + output has no YAML anchor/alias (`&id`/`*id`) -- safer for other Apache Ossie consumers.""" + out = importer.convert_metric_view_to_ossie( + "version: '1.1'\nsource: c.s.a\njoins:\n- {name: b, source: c.s.b, using: [id]}\n") + assert "&id" not in out and "*id" not in out diff --git a/converters/databricks/tests/test_ossie_to_metric_view.py b/converters/databricks/tests/test_ossie_to_metric_view.py new file mode 100644 index 0000000..35e3f1b --- /dev/null +++ b/converters/databricks/tests/test_ossie_to_metric_view.py @@ -0,0 +1,652 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the Apache Ossie -> Databricks Metric View exporter.""" + +import json + +import pytest + +from ossie_databricks import ConversionError +from ossie_databricks import ossie_to_metric_view as exporter +from _util import canon, load_fixture, parse + + +def test_fixtureA_export_matches_expected(): + out = exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml")) + assert parse(out) == parse(load_fixture("fixtureA_metric_view.yaml")) + + +def test_tpcds_export_matches_expected(): + """A normalized TPC-DS star (fact + date/item/customer dims) exports to the expected + Metric View: a join tree, primary keys bridged to rely.at_most_one_match, joined + columns alias-qualified, and the filter/format stash carried through.""" + out = exporter.convert_ossie_to_metric_view(load_fixture("tpcds_ossie.yaml")) + assert parse(out) == parse(load_fixture("tpcds_metric_view.yaml")) + + +def test_unsupported_version_rejected(): + ossie = "version: '9.9.9'\nsemantic_model:\n - name: m\n datasets:\n - {name: d, source: c.s.t}\n" + with pytest.raises(ConversionError): + exporter.convert_ossie_to_metric_view(ossie) + + +def _model(rels): + return { + "version": exporter.OSSIE_VERSION, + "semantic_model": [ + { + "name": "m", + "datasets": [ + {"name": "a", "source": "c.s.a"}, + {"name": "b", "source": "c.s.b"}, + {"name": "x", "source": "c.s.x"}, + ], + "relationships": rels, + } + ], + } + + +def _rel(name, frm, to): + return {"name": name, "from": frm, "to": to, + "from_columns": ["k"], "to_columns": ["k"]} + + +def test_multiple_roots_raises(): + import yaml + # a->b leaves x as a second root. + ossie = yaml.safe_dump(_model([_rel("r1", "a", "b")])) + with pytest.raises(ConversionError, match="multiple candidate fact"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_triangle_is_rejected_as_cycle(): + import yaml + # a->b, a->x, b->x : b and x are equidistant from a (a triangle) -> not a tree. + ossie = yaml.safe_dump(_model([_rel("r1", "a", "b"), _rel("r2", "a", "x"), + _rel("r3", "b", "x")])) + with pytest.raises(ConversionError, match="cycle"): + exporter.convert_ossie_to_metric_view(ossie) + + +def _field(name, col): + return {"name": name, "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": col}]}} + + +def test_mto_diamond_fans_out(): + """A shared dimension reached by two parents (orders->customers->regions and + orders->suppliers->regions) is fanned out into two aliased joins, not rejected.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amount")]}, + {"name": "customers", "source": "c.s.customers"}, + {"name": "suppliers", "source": "c.s.suppliers"}, + {"name": "regions", "source": "c.s.regions", "fields": [_field("rname", "r_name")]}, + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, + {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + region_joins = [j for top in out["joins"] for j in top.get("joins", []) if j["source"] == "c.s.regions"] + assert {j["name"] for j in region_joins} == {"customers_regions", "suppliers_regions"} + dims = {d["name"]: d["expr"] for d in out["dimensions"]} + # the fanned `regions` is a depth-2 join, so its column is qualified by the full path + assert dims["customers_regions_rname"] == "customers.customers_regions.r_name" + assert dims["suppliers_regions_rname"] == "suppliers.suppliers_regions.r_name" + + +def test_otm_diamond_fans_out(): + """customers (fact) -> past_orders/future_orders -> line_items: the shared + line_items is fanned out, and every join is one_to_many.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "customers", "source": "c.s.customers"}, + {"name": "past_orders", "source": "c.s.past_orders"}, + {"name": "future_orders", "source": "c.s.future_orders"}, + {"name": "line_items", "source": "c.s.line_items"}, + ], + "relationships": [ + {"name": "r1", "from": "past_orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "future_orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r3", "from": "line_items", "to": "past_orders", "from_columns": ["oid"], "to_columns": ["id"]}, + {"name": "r4", "from": "line_items", "to": "future_orders", "from_columns": ["oid"], "to_columns": ["id"]}, + ], + "metrics": [{"name": "cnt", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie, source="customers")) + leaf_names = {j["name"] for top in out["joins"] for j in top.get("joins", [])} + assert leaf_names == {"past_orders_line_items", "future_orders_line_items"} + + def all_otm(joins): + return all(j.get("cardinality") == "one_to_many" and all_otm(j.get("joins", [])) + for j in joins) + assert all_otm(out["joins"]) + + +def test_cycle_raises(): + import yaml + # a->b->x->a : cycle, and no root. + ossie = yaml.safe_dump(_model([_rel("r1", "a", "b"), _rel("r2", "b", "x"), + _rel("r3", "x", "a")])) + with pytest.raises(ConversionError, match="cycle"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_no_unknown_keys_leak(): + """Exporter output must contain no key outside the v1.1 schema (the strict-parse + guard: no `custom_extensions`, no `sql_on`).""" + out = exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml")) + assert "custom_extensions" not in out + assert "sql_on" not in out + + +def test_primary_key_is_dropped(): + out = parse(exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml"))) + assert "primary_key" not in json.dumps(out) + + +def _single_fact_model(metric_expr): + import yaml + return yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [{"name": "orders", "source": "c.s.orders", + "fields": [{"name": "k", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "k"}]}}]}], + "metrics": [{"name": "rev", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": metric_expr}]}}], + }], + }) + + +def test_measure_strips_fact_prefix(): + """Fact columns are bare in measure expressions (DBR idiom), not `source.`-qualified.""" + out = parse(exporter.convert_ossie_to_metric_view(_single_fact_model("SUM(orders.amount)"))) + assert out["measures"][0]["expr"] == "SUM(amount)" + + +def test_measure_keeps_lookalike_table_prefix(): + """A table whose name merely ends with the fact name is not stripped.""" + out = parse(exporter.convert_ossie_to_metric_view(_single_fact_model("SUM(store_orders.amount)"))) + assert out["measures"][0]["expr"] == "SUM(store_orders.amount)" + + +def test_invalid_source_rejected(): + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{"name": "m", "datasets": [{"name": "d", "source": "justatable"}]}], + }) + with pytest.raises(ConversionError, match="source"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_duplicate_dimension_name_rejected(): + # Two datasets each contributing a field named `id` collide in the single flat + # `dimensions` list. Metric Views require unique dimension/measure names, so this + # is rejected (rather than emitting a duplicate the user would hand to Databricks). + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", + "fields": [{"name": "id", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}]}, + {"name": "customer", "source": "c.s.customer", + "fields": [{"name": "id", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}]}, + ], + "relationships": [{"name": "r", "from": "orders", "to": "customer", + "from_columns": ["cid"], "to_columns": ["id"]}], + }], + }) + with pytest.raises(ConversionError, match="collides"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_measure_name_collides_with_dimension_rejected(): + # A measure whose name matches an existing dimension (case-insensitively) collides + # in the shared name space; Metric Views require uniqueness, so this is rejected. + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", + "fields": [{"name": "total", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "total"}]}}]}, + ], + "metrics": [ + {"name": "Total", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "SUM(total)"}]}}, + ], + }], + }) + with pytest.raises(ConversionError, match="collides"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_cascade_drop_downstream_measure_reference(): + """A measure that references a dropped measure via measure() is itself dropped + (transitively), so no dangling reference is emitted.""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [{"name": "f", "source": "c.s.f"}], + "metrics": [ + {"name": "base", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "SUM(x)"}]}}, # dropped (no DBX/ANSI) + {"name": "derived", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "measure(base) * 2"}]}}, + {"name": "derived2", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "measure(derived) + 1"}]}}, # transitive + {"name": "ok", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}, + ], + }], + }) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + names = [m["name"] for m in out.get("measures", [])] + assert names == ["ok"] # base dropped; derived + derived2 cascade-dropped; ok survives + + +def test_cascade_drop_downstream_dimension_reference(): + """A field/measure referencing a dropped dimension by name is also dropped.""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [{"name": "f", "source": "c.s.f", "fields": [ + {"name": "region", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "r"}]}}, # dropped dim + {"name": "label", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "upper(region)"}]}}, # references region + {"name": "keep", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}, + ]}], + }], + }) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + dims = [d["name"] for d in out.get("dimensions", [])] + assert dims == ["keep"] # region dropped; label cascade-dropped; keep survives + + +def test_orientation_unverifiable_when_to_side_has_no_key_warns(): + """If the `from` columns are a declared key but the `to` side declares no key, the + from/to orientation can't be verified; the converter leaves it as-is (no reorient) + and warns, rather than silently producing a possibly-inverted cardinality.""" + import warnings + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "a", "source": "c.s.a", "primary_key": ["a_id"], "fields": [ + {"name": "a_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "a_name"}]}}]}, + {"name": "b", "source": "c.s.b", "fields": [ + {"name": "b_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "b_name"}]}}]}, + ], + # from columns cover a's PK, but b (the `to` side) declares no key + "relationships": [{"name": "a_to_b", "from": "a", "to": "b", + "from_columns": ["a_id"], "to_columns": ["b_x"]}], + }], + }) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exporter.convert_ossie_to_metric_view(ossie) + assert any("orientation can't be verified" in str(w.message) for w in caught) + + +def test_cascade_drop_skips_qualified_join_alias_collision(): + """A dropped field whose name collides with a join alias must NOT cascade-drop a + *qualified* `alias.col` reference. A genuine dimension reference is unqualified; + `region.r_name` points at the join `region`, a different thing than a dropped bare + `region`, so the cascade must leave it (and the joined column) alone.""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [ + # dropped (no DBX/ANSI dialect); its name collides with the `region` join + {"name": "region", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "r"}]}}, + # references the join alias `region`, not the dropped field -> must survive + {"name": "summary", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "region.r_name"}]}}, + ]}, + {"name": "region", "source": "c.s.region", "primary_key": ["r_key"], "fields": [ + {"name": "r_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "r_name"}]}}, + ]}, + ], + "relationships": [{"name": "orr", "from": "orders", "to": "region", + "from_columns": ["o_rkey"], "to_columns": ["r_key"]}], + }], + }) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + dims = [d["name"] for d in out.get("dimensions", [])] + # `region` field dropped; `summary` (refs alias region.r_name) and the joined + # `r_name` (qualified to region.r_name on export) both survive -- no false cascade. + assert "region" not in dims + assert "summary" in dims and "r_name" in dims + + +def _orders_lineitems_ossie(): + """Apache Ossie: line_items (many, FK l_order_id) -> orders (one, PK order_id).""" + import yaml + return yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "sales", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "primary_key": ["order_id"], + "fields": [{"name": "order_date", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "o_order_date"}]}}]}, + {"name": "line_items", "source": "c.s.line_items", + "fields": [{"name": "product_sk", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "l_product_sk"}]}}]}, + ], + "relationships": [{"name": "li_to_order", "from": "line_items", "to": "orders", + "from_columns": ["l_order_id"], "to_columns": ["order_id"]}], + "metrics": [{"name": "order_count", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], + }], + }) + + +def test_source_on_to_side_derives_one_to_many(): + """Naming the PK/one-side dataset as the source makes its join one_to_many, and + the many-side table's columns drop (a field must resolve to one value/source row).""" + out = parse(exporter.convert_ossie_to_metric_view(_orders_lineitems_ossie(), source="orders")) + assert out["source"] == "c.s.orders" + join = out["joins"][0] + assert join["name"] == "line_items" + assert join["cardinality"] == "one_to_many" + assert join["on"] == "source.order_id = line_items.l_order_id" + assert [d["name"] for d in out.get("dimensions", [])] == ["order_date"] # product_sk dropped + + +def test_default_fact_is_fk_sink_and_many_to_one(): + """Without an explicit source the fact is the FK-sink (line_items) and the join to + orders is the default many_to_one (no explicit cardinality).""" + out = parse(exporter.convert_ossie_to_metric_view(_orders_lineitems_ossie())) + assert out["source"] == "c.s.line_items" + join = out["joins"][0] + assert join["name"] == "orders" + assert "cardinality" not in join + assert join["on"] == "source.l_order_id = orders.order_id" + + +def test_unknown_source_rejected(): + with pytest.raises(ConversionError, match="not a dataset"): + exporter.convert_ossie_to_metric_view(_orders_lineitems_ossie(), source="nope") + + +def test_one_to_many_subtree_must_stay_one_to_many(): + """A many_to_one join descending from a one_to_many join is rejected (DBR rule).""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "line_items", "source": "c.s.line_items"}, + {"name": "product", "source": "c.s.product"}, + ], + "relationships": [ + {"name": "li_to_order", "from": "line_items", "to": "orders", # orders->li : OTM + "from_columns": ["l_order_id"], "to_columns": ["order_id"]}, + {"name": "li_to_product", "from": "line_items", "to": "product", # li->product : MTO + "from_columns": ["l_product_sk"], "to_columns": ["p_sk"]}, + ], + }], + }) + with pytest.raises(ConversionError, match="one-to-many"): + exporter.convert_ossie_to_metric_view(ossie, source="orders") + + +def test_primary_key_deduces_at_most_one_match(): + """A many_to_one join whose to_columns cover the target's declared primary_key + gets rely.at_most_one_match; a join to a key-less dataset does not.""" + import yaml + + def model(dim_extra): + dim = {"name": "customer", "source": "c.s.customer"} + dim.update(dim_extra) + return yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [{"name": "orders", "source": "c.s.orders"}, dim], + "relationships": [{"name": "r", "from": "orders", "to": "customer", + "from_columns": ["cid"], "to_columns": ["id"]}], + }]}) + + join = parse(exporter.convert_ossie_to_metric_view(model({"primary_key": ["id"]})))["joins"][0] + assert join.get("rely") == {"at_most_one_match": True} + join2 = parse(exporter.convert_ossie_to_metric_view(model({})))["joins"][0] + assert "rely" not in join2 + + +def test_mislabeled_from_to_reoriented_by_key(): + """When from/to is swapped but the declared keys show the real one-side, the + converter re-orients to the key side (warns) -- so fact selection and cardinality + come out identical to the well-formed model.""" + import warnings + import yaml + + def model(frm, to, from_cols, to_cols): + return yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "primary_key": ["order_id"], + "fields": [_field("amt", "amount")]}, + {"name": "customer", "source": "c.s.customer", "primary_key": ["c_id"], + "fields": [_field("cname", "c_name")]}, + ], + "relationships": [{"name": "r", "from": frm, "to": to, + "from_columns": from_cols, "to_columns": to_cols}], + }]}) + + well = parse(exporter.convert_ossie_to_metric_view( + model("orders", "customer", ["cust_id"], ["c_id"]))) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + swapped = parse(exporter.convert_ossie_to_metric_view( + model("customer", "orders", ["c_id"], ["cust_id"]))) + assert any("mislabeled" in str(w.message) for w in caught) + assert swapped["source"] == "c.s.orders" # fact selection corrected to the FK holder + assert swapped == well # identical to the well-formed model + + +def test_dataset_named_source_is_renamed(): + """A dataset literally named `source` must not collide with the fact's reserved + `source` alias (would otherwise emit an ambiguous join).""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "source", "source": "c.s.dim", "fields": [_field("x", "xcol")]}, + ], + "relationships": [{"name": "r", "from": "orders", "to": "source", + "from_columns": ["sid"], "to_columns": ["id"]}], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + join = out["joins"][0] + assert join["name"] != "source" + assert join["on"] == f"source.sid = {join['name']}.id" + assert out["dimensions"][0]["expr"] == f"{join['name']}.xcol" + + +def test_fanout_alias_collision_deduped(): + """A real dataset whose name equals a synthesized fan-out alias still gets a + distinct alias -- no two joins share a name.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "customers", "source": "c.s.customers"}, + {"name": "suppliers", "source": "c.s.suppliers"}, + {"name": "regions", "source": "c.s.regions"}, + {"name": "customers_regions", "source": "c.s.cr"}, # collides with fan-out alias + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, + {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r5", "from": "orders", "to": "customers_regions", "from_columns": ["xid"], "to_columns": ["id"]}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + names = [] + + def collect(joins): + for j in joins: + names.append(j["name"]) + collect(j.get("joins", [])) + + collect(out["joins"]) + assert len(names) == len(set(names)), names # all join names unique + + +def test_malformed_input_raises_conversion_error(): + """Missing required keys surface as ConversionError, not a raw KeyError traceback.""" + import yaml + bad = yaml.safe_dump({"version": exporter.OSSIE_VERSION, + "semantic_model": [{"name": "m", "datasets": [{"source": "c.s.t"}]}]}) + with pytest.raises(ConversionError, match="missing required 'name'"): + exporter.convert_ossie_to_metric_view(bad) + + +def test_nameless_relationship_with_ai_context_does_not_crash(): + """A relationship may omit `name`; the dropped-ai_context warning must not raise a + raw KeyError when it has ai_context but no name.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amt")]}, + {"name": "customers", "source": "c.s.customers"}, + ], + "relationships": [ + {"from": "orders", "to": "customers", "from_columns": ["cid"], + "to_columns": ["id"], "ai_context": "joins orders to customers"}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) # must not raise + assert out["joins"][0]["name"] == "customers" + + +def _cfield(name, expr): + return {"name": name, "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": expr}]}} + + +def test_fanout_complex_expr_dropped_not_emitted_ambiguously(): + """On a fanned-out (diamond) dataset, a simple column fans out into one aliased + dimension per instance, but a complex expression -- which cannot be attributed to a + single instance -- is dropped rather than emitted ambiguously.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "customers", "source": "c.s.customers"}, + {"name": "suppliers", "source": "c.s.suppliers"}, + {"name": "regions", "source": "c.s.regions", + "fields": [_cfield("r_name", "r_name"), _cfield("rfull", "r_a || r_b")]}, + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, + {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + ], + }]}) + dims = parse(exporter.convert_ossie_to_metric_view(ossie)).get("dimensions", []) + # the simple column fans out into two unambiguous, alias-qualified dimensions ... + assert sum(1 for d in dims if d["name"].endswith("_r_name")) == 2 + # ... while the ambiguous complex expression is dropped (never emitted unqualified) + assert not any("||" in d["expr"] for d in dims) + + +def test_malformed_yaml_raises_conversion_error(): + """Invalid YAML surfaces as ConversionError, not a raw yaml.YAMLError traceback.""" + with pytest.raises(ConversionError, match="Invalid YAML"): + exporter.convert_ossie_to_metric_view("semantic_model: [oops\n") + + +def test_nested_join_uses_full_path_qualification(): + """A snowflake (orders -> customer -> nation) qualifies a column from the nested + `nation` join by its full join path from the source (`customer.nation.n_name`) -- the + Databricks nested-join rule -- not the single-level `nation.n_name`. A depth-1 join + stays single-name.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amount")]}, + {"name": "customer", "source": "c.s.customer", "fields": [_field("cname", "c_name")]}, + {"name": "nation", "source": "c.s.nation", "fields": [_field("nname", "n_name")]}, + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customer", "from_columns": ["ckey"], "to_columns": ["c_key"]}, + {"name": "r2", "from": "customer", "to": "nation", "from_columns": ["nkey"], "to_columns": ["n_key"]}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + exprs = {d["name"]: d["expr"] for d in out["dimensions"]} + assert exprs["cname"] == "customer.c_name" # depth-1: the join's own name + assert exprs["nname"] == "customer.nation.n_name" # depth-2: full path from source + # the nested join's `on:` still uses immediate names (single-level) + nation_join = out["joins"][0]["joins"][0] + assert nation_join["on"] == "customer.nkey = nation.n_key" + + +def test_case_variant_dataset_name_rejected(): + """DBR identifiers are case-insensitive, so two datasets differing only in case + (`customer`/`Customer`) collide and are rejected (review finding).""" + ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" + " - {name: customer, source: c.s.c}\n - {name: Customer, source: c.s.c2}\n") + with pytest.raises(ConversionError, match="duplicate"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_non_string_field_expression_raises_clean_error(): + """A non-string dialect expression raises a ConversionError, not a raw crash + (review finding).""" + ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" + " - name: o\n source: c.s.o\n fields:\n - name: d\n expression:\n" + " dialects:\n - {dialect: DATABRICKS, expression: 123}\n") + with pytest.raises(ConversionError, match="must be a string"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_scalar_join_columns_rejected(): + """`from_columns`/`to_columns` given as a scalar string (not a list) raise a clear + 'must be lists' error rather than a misleading character-count length error.""" + ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" + " - {name: a, source: c.s.a}\n - {name: b, source: c.s.b}\n relationships:\n" + " - {name: ab, from: a, to: b, from_columns: cid, to_columns: id}\n") + with pytest.raises(ConversionError, match="must be lists"): + exporter.convert_ossie_to_metric_view(ossie) diff --git a/converters/databricks/tests/test_roundtrip.py b/converters/databricks/tests/test_roundtrip.py new file mode 100644 index 0000000..c9d5629 --- /dev/null +++ b/converters/databricks/tests/test_roundtrip.py @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Round-trip tests in both directions.""" + +from ossie_databricks import metric_view_to_ossie as importer +from ossie_databricks import ossie_to_metric_view as exporter +from _util import canon, load_fixture, parse, strip_dropped + + +def test_ossie_to_mv_to_ossie(): + """Apache Ossie -> MV -> Apache Ossie preserves everything except the documented drops + (model name, primary_key/unique_keys).""" + ossie_in = load_fixture("fixtureA_ossie.yaml") + mv = exporter.convert_ossie_to_metric_view(ossie_in) + ossie_out = importer.convert_metric_view_to_ossie(mv) + assert strip_dropped(parse(ossie_out)) == strip_dropped(parse(ossie_in)) + + +def test_using_clause_round_trips(): + """A `using` join survives MV -> Apache Ossie -> MV (equal column lists re-emit as `using`).""" + mv_in = ( + "version: '1.1'\nsource: c.s.fact\n" + "joins:\n- name: dim\n source: c.s.dim\n using: [id]\n" + "measures:\n- name: n\n expr: count(*)\n" + ) + ossie = importer.convert_metric_view_to_ossie(mv_in) + join = parse(exporter.convert_ossie_to_metric_view(ossie))["joins"][0] + assert join.get("using") == ["id"] + assert "on" not in join + + +def test_mv_to_ossie_to_mv_is_lossless(): + """MV -> Apache Ossie -> MV is byte-faithful (structurally): the stash carries every + MV-only feature through Apache Ossie and back.""" + mv_in = load_fixture("fixtureB_metric_view.yaml") + ossie = importer.convert_metric_view_to_ossie(mv_in) + mv_out = exporter.convert_ossie_to_metric_view(ossie) + assert parse(mv_out) == parse(mv_in) + + +def test_tpcds_mv_round_trips(): + """The TPC-DS Metric View (multi-join star with rely/filter/format) survives + MV -> Apache Ossie -> MV unchanged.""" + mv_in = load_fixture("tpcds_metric_view.yaml") + ossie = importer.convert_metric_view_to_ossie(mv_in) + mv_out = exporter.convert_ossie_to_metric_view(ossie) + assert parse(mv_out) == parse(mv_in) + + +def test_one_to_many_round_trips_mv_ossie_mv(): + """A one_to_many Metric View survives MV -> Apache Ossie -> MV: cardinality rides the + relationship direction (+ stash), and the source/grain rides the model stash, so + the exporter re-roots at `orders` rather than the FK-sink `line_items`.""" + mv_in = ( + "version: '1.1'\nsource: c.s.orders\ncomment: Orders\n" + "joins:\n- name: line_items\n source: c.s.line_items\n" + " on: source.order_id = line_items.l_order_id\n" + " cardinality: one_to_many\n" + "dimensions:\n- {name: order_date, expr: o_order_date}\n" + "measures:\n- {name: order_count, expr: COUNT(*)}\n" + ) + ossie = importer.convert_metric_view_to_ossie(mv_in) + mv_out = exporter.convert_ossie_to_metric_view(ossie) + assert parse(mv_out) == parse(mv_in) + + +# Property-based round-trip coverage. The Hypothesis driver lives in +# test_roundtrip_properties.py; these run the same generators/assertions under a plain +# seeded RNG so the property coverage also holds where Hypothesis is not installed. + +def test_property_mv_to_ossie_to_mv_seeded(): + from _roundtrip_helpers import RandomRnd, assert_mv_roundtrip, build_metric_view + for seed in range(250): + assert_mv_roundtrip(build_metric_view(RandomRnd(seed))) + + +def test_property_ossie_to_mv_to_ossie_seeded(): + from _roundtrip_helpers import RandomRnd, assert_ossie_roundtrip, build_ossie + for seed in range(250): + assert_ossie_roundtrip(build_ossie(RandomRnd(1_000_000 + seed))) diff --git a/converters/databricks/tests/test_roundtrip_properties.py b/converters/databricks/tests/test_roundtrip_properties.py new file mode 100644 index 0000000..d022ebb --- /dev/null +++ b/converters/databricks/tests/test_roundtrip_properties.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Property-based round-trip tests (Hypothesis). + +For any generated model in the round-trippable subset, converting one direction and +back preserves content: + + - MV -> Apache Ossie -> MV : source, every dimension/measure name+expression+metadata, every + join (name, source, condition, cardinality, rely) and the join nesting, and the + model-level filter/comment/materialization. + - Apache Ossie -> MV -> Apache Ossie : dataset names+sources+fields, relationship from/to/columns, + metric name+expression, and the model description. + +The model generation and the assertions live in `_roundtrip_helpers` (no test-framework +dependency) so the exact same logic also runs under a plain seeded RNG where Hypothesis +is unavailable (see the `*_seeded` tests in test_roundtrip.py). This file is the thin +Hypothesis driver: it maps draws into the shared `Rnd` interface and runs the properties. + +Run: `pytest test_roundtrip_properties.py` (needs `hypothesis`). +""" + +import pytest + +pytest.importorskip("hypothesis") # skip cleanly if hypothesis is not installed + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from _roundtrip_helpers import ( + assert_mv_roundtrip, + assert_ossie_roundtrip, + build_metric_view, + build_ossie, +) + +# Alphanumeric metadata text with optional interior spaces (no leading/trailing space, +# no YAML-special characters), so values survive a dump/load cycle verbatim. +_safe_text = st.from_regex(r"[A-Za-z0-9]([A-Za-z0-9 ]{0,18}[A-Za-z0-9])?", fullmatch=True) +# A SQL column-style identifier. +_colident = st.from_regex(r"[a-z_][a-z0-9_]{0,7}", fullmatch=True) + +_SETTINGS = settings( + max_examples=300, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) + + +class _HypothesisRnd: + """The `Rnd` interface backed by a Hypothesis `draw`. `chance(p)` ignores `p` + (Hypothesis explores both branches regardless).""" + + def __init__(self, draw): + self._draw = draw + + def chance(self, p=0.5): + return self._draw(st.booleans()) + + def count(self, lo, hi): + return self._draw(st.integers(min_value=lo, max_value=hi)) + + def pick(self, seq): + return self._draw(st.sampled_from(list(seq))) + + def text(self): + return self._draw(_safe_text) + + def colname(self): + return self._draw(_colident) + + +@st.composite +def metric_views(draw): + return build_metric_view(_HypothesisRnd(draw)) + + +@st.composite +def ossie_models(draw): + return build_ossie(_HypothesisRnd(draw)) + + +class TestMetricViewRoundTrip: + @given(mv=metric_views()) + @_SETTINGS + def test_mv_to_ossie_to_mv(self, mv): + assert_mv_roundtrip(mv) + + +class TestOSIRoundTrip: + @given(ossie=ossie_models()) + @_SETTINGS + def test_ossie_to_mv_to_ossie(self, ossie): + assert_ossie_roundtrip(ossie)