From ad19bfd2a829c85cf9c8146a672cf049a66baec3 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Fri, 10 Jul 2026 15:47:58 -0700 Subject: [PATCH 1/3] Add WisdomAI domain export converter Adds converters/wisdom, a one-way converter from a WisdomAI domain export JSON (format 1.0) to an Ossie semantic model YAML, following the layout of the dbt converter. Mapping: domain -> semantic model; domain knowledge and system instructions -> model-level ai_context; tables/columns/formulas -> datasets/fields; relationship graph -> relationships (cardinality folded into edge direction, compound AND-joins flattened to composite keys); per-table measures -> model-level metrics. Expressions are emitted verbatim under the dialect mapped from the table's connection (snowflake, databricks), falling back to ANSI_SQL with a warning for unsupported dialects. Information loss is reported through the same ConverterResult/ConverterIssue pattern as ossie-dbt. Co-Authored-By: Claude Fable 5 --- converters/wisdom/README.md | 89 +++++ converters/wisdom/pyproject.toml | 51 +++ .../wisdom/src/ossie_wisdom/__init__.py | 26 ++ converters/wisdom/src/ossie_wisdom/cli.py | 80 ++++ .../src/ossie_wisdom/converter_issues.py | 50 +++ .../wisdom/src/ossie_wisdom/wisdom_to_osi.py | 358 ++++++++++++++++++ converters/wisdom/tests/__init__.py | 16 + .../wisdom/tests/fixtures/sample_export.json | 227 +++++++++++ converters/wisdom/tests/test_wisdom_to_osi.py | 167 ++++++++ 9 files changed, 1064 insertions(+) create mode 100644 converters/wisdom/README.md create mode 100644 converters/wisdom/pyproject.toml create mode 100644 converters/wisdom/src/ossie_wisdom/__init__.py create mode 100644 converters/wisdom/src/ossie_wisdom/cli.py create mode 100644 converters/wisdom/src/ossie_wisdom/converter_issues.py create mode 100644 converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py create mode 100644 converters/wisdom/tests/__init__.py create mode 100644 converters/wisdom/tests/fixtures/sample_export.json create mode 100644 converters/wisdom/tests/test_wisdom_to_osi.py diff --git a/converters/wisdom/README.md b/converters/wisdom/README.md new file mode 100644 index 0000000..166c356 --- /dev/null +++ b/converters/wisdom/README.md @@ -0,0 +1,89 @@ + + +# WisdomAI → Apache Ossie Converter + +Converts a [WisdomAI](https://wisdom.ai) domain export (format `1.0`, produced by wisdom's +`exportDomain` API) into an Ossie semantic model YAML document. + +## Usage + +```bash +pip install -e ../../python -e . +ossie-wisdom wisdom-to-osi -i domain-export.json -o semantic_model.yaml +``` + +Conversion warnings (information loss) are printed to stderr; the output YAML validates +against the [Ossie JSON Schema](../../core-spec/osi-schema.json): + +```bash +python ../../validation/validate.py semantic_model.yaml --schema ../../core-spec/osi-schema.json +``` + +## Field mapping + +| Ossie | Wisdom | +|-------|--------| +| `semantic_model[].name` | domain `ref.name` | +| `semantic_model[].description` | domain `description` | +| `semantic_model[].ai_context` | `domainSystemInstructions` + each domain `knowledge[].content` as a bulleted list | +| `datasets[].name` | table `ref.name` | +| `datasets[].source` | table `location.database.schema.dbTable` | +| `datasets[].description` | table `description` | +| `datasets[].primary_key` | table `primaryKey.columns`, else columns flagged `isPrimaryKey` | +| `datasets[].fields[]` | table `columns[]` (expression = bare column name) and `formulas[]` (expression verbatim) | +| `fields[].label` | column/formula `properties.displayName` | +| `fields[].dimension.is_time` | set when `properties.dataType` is `DATE`, `DATETIME`, or `TIMESTAMP` | +| `relationships[]` | domain `relationshipGraph.relationships[]` (see cardinality below) | +| `metrics[]` | every table's `measures[]`, hoisted to the model level | + +Expressions are emitted verbatim under the Ossie dialect mapped from the table's connection +(`snowflake → SNOWFLAKE`, `databricks → DATABRICKS`). Connections with any other dialect fall +back to `ANSI_SQL` verbatim with an `UNSUPPORTED_DIALECT` warning. + +### Relationship cardinality + +Ossie encodes cardinality by direction (`from` = many side, `to` = one side), so wisdom's +`relationshipType` is folded into the edge direction: + +| Wisdom `relationshipType` | Ossie | +|---------------------------|-------| +| `MANY_TO_ONE` | `from` = left, `to` = right | +| `ONE_TO_MANY` | `from` = right, `to` = left | +| `ONE_TO_ONE` | `from` = left, plus an `ai_context` note (direction is arbitrary) | +| `MANY_TO_MANY` | `from` = left, plus an `ai_context` note and a `CARDINALITY_LOSS` warning | + +Compound join conditions that are an `AND` of equality conditions are flattened into +positional `from_columns`/`to_columns` arrays; any other compound condition (e.g. `OR`) +drops the relationship with a `RELATIONSHIP_DROPPED` warning. + +## Known limitations + +- One-way only (wisdom → Ossie). No Ossie → wisdom export yet. +- Hidden columns and stale measures are converted anyway (with a `STALE_MEASURE` warning + for the latter), so the output may reference columns wisdom itself hides from querying. +- Out of scope for now: reviewed queries, recommended questions, synonym sets, row-level + security config, per-knowledge schema annotations, column enum values, and LLM prompts. + +## Development + +```bash +pip install -e ../../python -e . pytest +pytest tests/ +``` diff --git a/converters/wisdom/pyproject.toml b/converters/wisdom/pyproject.toml new file mode 100644 index 0000000..7cb7701 --- /dev/null +++ b/converters/wisdom/pyproject.toml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "apache-ossie-wisdom" +version = "0.2.0.dev0" +description = "WisdomAI domain export -> Apache Ossie converter" +requires-python = ">=3.11" +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "apache-ossie>=0.2.0.dev0", + "PyYAML>=6.0", +] + +[project.license] +text = "Apache-2.0" + +[project.scripts] +ossie-wisdom = "ossie_wisdom.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_wisdom"] + +[tool.uv] +dev-dependencies = [ + "pytest>=8.0", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/converters/wisdom/src/ossie_wisdom/__init__.py b/converters/wisdom/src/ossie_wisdom/__init__.py new file mode 100644 index 0000000..94153f4 --- /dev/null +++ b/converters/wisdom/src/ossie_wisdom/__init__.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult +from ossie_wisdom.wisdom_to_osi import WisdomToOSIConverter + +__all__ = [ + "ConverterIssue", + "ConverterIssueType", + "ConverterResult", + "WisdomToOSIConverter", +] diff --git a/converters/wisdom/src/ossie_wisdom/cli.py b/converters/wisdom/src/ossie_wisdom/cli.py new file mode 100644 index 0000000..0274e62 --- /dev/null +++ b/converters/wisdom/src/ossie_wisdom/cli.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""CLI entry point for the ossie-wisdom converter. + +Usage: + ossie-wisdom wisdom-to-osi -i domain-export.json -o output.yaml +""" + +import argparse +import json +import sys +from pathlib import Path + +from ossie_wisdom.converter_issues import ConverterIssueType +from ossie_wisdom.wisdom_to_osi import WisdomToOSIConverter + +_ISSUE_REASON: dict[ConverterIssueType, str] = { + ConverterIssueType.UNSUPPORTED_DIALECT: "the connection dialect has no Ossie equivalent; expressions were emitted verbatim as ANSI_SQL", + ConverterIssueType.CARDINALITY_LOSS: "Ossie relationships encode cardinality by direction and cannot represent many-to-many", + ConverterIssueType.RELATIONSHIP_DROPPED: "the relationship uses a join Ossie cannot represent (non-table source, OR/non-equi condition, or unknown dataset)", + ConverterIssueType.METRIC_NAME_COLLISION: "another table defines a measure with the same name; this one was prefixed with its table name", + ConverterIssueType.STALE_MEASURE: "wisdom marked this measure stale; it was converted anyway", + ConverterIssueType.DUPLICATE_FIELD_DROPPED: "the dataset already has a field with this name", +} + +_DROPPED_ISSUE_TYPES = { + ConverterIssueType.RELATIONSHIP_DROPPED, + ConverterIssueType.DUPLICATE_FIELD_DROPPED, +} + + +def _cmd_wisdom_to_osi(args: argparse.Namespace) -> None: + input_path = Path(args.input) + output_path = Path(args.output) + + export = json.loads(input_path.read_text()) + result = WisdomToOSIConverter().convert(export) + + for issue in result.issues: + verb = "was dropped" if issue.issue_type in _DROPPED_ISSUE_TYPES else "was converted with loss" + reason = _ISSUE_REASON[issue.issue_type] + print(f"[WARNING] {issue.issue_type.value}: {issue.element_name} {verb} during conversion because {reason}", file=sys.stderr) + + output_path.write_text(result.output.to_osi_yaml()) + print(f"Written to {output_path}", file=sys.stderr) + + +def main() -> None: + parser = argparse.ArgumentParser( + prog="ossie-wisdom", + description="Convert a WisdomAI domain export JSON to Ossie YAML.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + wisdom_to_osi = subparsers.add_parser("wisdom-to-osi", help="Convert domain-export.json → Ossie YAML") + wisdom_to_osi.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to wisdom domain export JSON") + wisdom_to_osi.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output Ossie YAML") + + args = parser.parse_args() + if args.command == "wisdom-to-osi": + _cmd_wisdom_to_osi(args) + + +if __name__ == "__main__": + main() diff --git a/converters/wisdom/src/ossie_wisdom/converter_issues.py b/converters/wisdom/src/ossie_wisdom/converter_issues.py new file mode 100644 index 0000000..f8c8fb2 --- /dev/null +++ b/converters/wisdom/src/ossie_wisdom/converter_issues.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass +from enum import Enum +from typing import Generic, List, TypeVar + + +class ConverterIssueType(Enum): + """Identifies the kind of information loss that occurred during conversion.""" + + UNSUPPORTED_DIALECT = "UNSUPPORTED_DIALECT" + CARDINALITY_LOSS = "CARDINALITY_LOSS" + RELATIONSHIP_DROPPED = "RELATIONSHIP_DROPPED" + METRIC_NAME_COLLISION = "METRIC_NAME_COLLISION" + STALE_MEASURE = "STALE_MEASURE" + DUPLICATE_FIELD_DROPPED = "DUPLICATE_FIELD_DROPPED" + + +@dataclass(frozen=True) +class ConverterIssue: + """Records a single instance of information loss during conversion.""" + + issue_type: ConverterIssueType + element_name: str + + +T = TypeVar("T") + + +@dataclass(frozen=True) +class ConverterResult(Generic[T]): + """Return value of a converter's convert() method, pairing the output with any conversion issues.""" + + output: T + issues: List[ConverterIssue] diff --git a/converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py b/converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py new file mode 100644 index 0000000..855641d --- /dev/null +++ b/converters/wisdom/src/ossie_wisdom/wisdom_to_osi.py @@ -0,0 +1,358 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Converts a WisdomAI domain export (format 1.0) into an Ossie Document. + +The input is the JSON produced by wisdom's ``exportDomain`` RPC: a wrapper +holding the domain ZSheet, one ZSheet per table, and connection metadata, +all serialized as protobuf-JSON (camelCase keys). +""" + +import re +from typing import Dict, List, Optional, Set, Tuple + +from ossie import ( + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDimension, + OSIDocument, + OSIExpression, + OSIField, + OSIMetric, + OSIRelationship, + OSISemanticModel, +) +from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult + +_DIALECT_MAP: Dict[str, OSIDialect] = { + "snowflake": OSIDialect.SNOWFLAKE, + "databricks": OSIDialect.DATABRICKS, + "ansi": OSIDialect.ANSI_SQL, + "ansi_sql": OSIDialect.ANSI_SQL, +} + +_TIME_DATA_TYPES = {"DATE", "DATETIME", "TIMESTAMP"} + +_SIMPLE_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class WisdomToOSIConverter: + """Converts a wisdom domain export dict into an Ossie Document.""" + + def convert(self, export: dict) -> ConverterResult[OSIDocument]: + issues: List[ConverterIssue] = [] + + domain = export.get("domain", {}).get("zsheet_json", {}) + dialect_by_connection = self._build_dialect_index(export, issues) + + datasets = [] + metrics: List[OSIMetric] = [] + for table in export.get("tables", []): + zsheet = table.get("zsheet_json", {}) + dialect = self._resolve_dialect(zsheet, dialect_by_connection) + datasets.append(self._convert_table(zsheet, dialect, issues)) + metrics.extend(self._convert_measures(zsheet, dialect, issues)) + if not datasets: + raise ValueError("Export contains no tables; an Ossie semantic model requires at least one dataset.") + metrics = self._dedupe_metrics(metrics, issues) + + dataset_names = {d.name for d in datasets} + relationships = self._convert_relationships(domain, dataset_names, issues) + + model = OSISemanticModel( + name=domain.get("ref", {}).get("name") or export.get("export_metadata", {}).get("domain_name", "domain"), + description=domain.get("description") or None, + ai_context=self._build_ai_context(domain), + datasets=datasets, + relationships=relationships or None, + metrics=[metric for _, metric in metrics] or None, + ) + return ConverterResult(output=OSIDocument(semantic_model=[model]), issues=issues) + + def _build_dialect_index(self, export: dict, issues: List[ConverterIssue]) -> Dict[str, OSIDialect]: + index: Dict[str, OSIDialect] = {} + for connection in export.get("connections", []): + dialect_name = connection.get("dialect", "").lower() + dialect = _DIALECT_MAP.get(dialect_name) + if dialect is None: + dialect = OSIDialect.ANSI_SQL + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.UNSUPPORTED_DIALECT, + element_name=f"{connection.get('name', connection.get('connection_id', '?'))} ({dialect_name})", + ) + ) + index[connection.get("connection_id", "")] = dialect + return index + + def _resolve_dialect(self, zsheet: dict, dialect_by_connection: Dict[str, OSIDialect]) -> OSIDialect: + connection_id = zsheet.get("location", {}).get("connectionId", "") + return dialect_by_connection.get(connection_id, OSIDialect.ANSI_SQL) + + def _build_ai_context(self, domain: dict) -> Optional[str]: + parts: List[str] = [] + system_instructions = domain.get("domainSystemInstructions", "").strip() + if system_instructions: + parts.append(system_instructions) + for knowledge in domain.get("knowledge", []): + content = knowledge.get("content", "").strip() + if content: + parts.append(f"- {content}") + return "\n".join(parts) or None + + def _convert_table(self, zsheet: dict, dialect: OSIDialect, issues: List[ConverterIssue]) -> OSIDataset: + name = zsheet.get("ref", {}).get("name", "") + location = zsheet.get("location", {}) + source = ".".join( + part for part in (location.get("database"), location.get("schema"), location.get("dbTable")) if part + ) + + fields: List[OSIField] = [] + seen: Set[str] = set() + for column in zsheet.get("columns", []): + field = self._convert_column(column, dialect) + if field.name in seen: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.DUPLICATE_FIELD_DROPPED, element_name=f"{name}.{field.name}" + ) + ) + continue + seen.add(field.name) + fields.append(field) + for formula in zsheet.get("formulas", []): + field = self._convert_formula(formula, dialect) + if field is None: + continue + if field.name in seen: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.DUPLICATE_FIELD_DROPPED, element_name=f"{name}.{field.name}" + ) + ) + continue + seen.add(field.name) + fields.append(field) + + return OSIDataset( + name=name, + source=source, + primary_key=self._extract_primary_key(zsheet), + description=zsheet.get("description") or None, + fields=fields or None, + ) + + def _extract_primary_key(self, zsheet: dict) -> Optional[List[str]]: + primary_key = zsheet.get("primaryKey", {}).get("columns") + if primary_key: + return list(primary_key) + flagged = [ + column["name"] + for column in zsheet.get("columns", []) + if column.get("properties", {}).get("isPrimaryKey") + ] + return flagged or None + + def _convert_column(self, column: dict, dialect: OSIDialect) -> OSIField: + properties = column.get("properties", {}) + return OSIField( + name=column.get("name", ""), + expression=self._make_expression(self._quote_identifier(column.get("name", ""), dialect), dialect), + dimension=OSIDimension(is_time=True) if properties.get("dataType") in _TIME_DATA_TYPES else None, + label=properties.get("displayName") or None, + description=column.get("description") or None, + ) + + def _convert_formula(self, formula: dict, dialect: OSIDialect) -> Optional[OSIField]: + name = formula.get("name", "") + expression = formula.get("expression", "") + if not name or not expression: + return None + properties = formula.get("properties", {}) + return OSIField( + name=name, + expression=self._make_expression(expression, dialect), + dimension=OSIDimension(is_time=True) if properties.get("dataType") in _TIME_DATA_TYPES else None, + label=properties.get("displayName") or None, + description=formula.get("description") or None, + ) + + def _convert_measures( + self, zsheet: dict, dialect: OSIDialect, issues: List[ConverterIssue] + ) -> List[Tuple[str, OSIMetric]]: + table_name = zsheet.get("ref", {}).get("name", "") + metrics: List[Tuple[str, OSIMetric]] = [] + for measure in zsheet.get("measures", []): + name = measure.get("name", "") + expression = measure.get("expression", "") + if not name or not expression: + continue + if measure.get("staleReason"): + issues.append( + ConverterIssue(issue_type=ConverterIssueType.STALE_MEASURE, element_name=f"{table_name}.{name}") + ) + metrics.append( + ( + table_name, + OSIMetric( + name=name, + expression=self._make_expression(expression, dialect), + description=measure.get("description") or None, + ), + ) + ) + return metrics + + def _dedupe_metrics( + self, metrics: List[Tuple[str, OSIMetric]], issues: List[ConverterIssue] + ) -> List[Tuple[str, OSIMetric]]: + result: List[Tuple[str, OSIMetric]] = [] + seen: Set[str] = set() + for table_name, metric in metrics: + name = metric.name + if name in seen: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.METRIC_NAME_COLLISION, element_name=f"{table_name}.{name}" + ) + ) + name = f"{table_name}_{name}" + suffix = 2 + while name in seen: + name = f"{table_name}_{metric.name}_{suffix}" + suffix += 1 + metric = metric.model_copy(update={"name": name}) + seen.add(name) + result.append((table_name, metric)) + return result + + def _convert_relationships( + self, domain: dict, dataset_names: Set[str], issues: List[ConverterIssue] + ) -> List[OSIRelationship]: + relationships: List[OSIRelationship] = [] + seen_names: Set[str] = set() + edges = domain.get("relationshipGraph", {}).get("relationships", []) + for edge in edges: + left = edge.get("leftDataSource", {}).get("zsheet", {}).get("name") + right = edge.get("rightDataSource", {}).get("zsheet", {}).get("name") + properties = edge.get("properties", {}) + column_pairs = self._extract_column_pairs(properties) + if not left or not right or column_pairs is None: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_DROPPED, + element_name=f"{left or '?'} <-> {right or '?'}", + ) + ) + continue + if left not in dataset_names or right not in dataset_names: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_DROPPED, element_name=f"{left} <-> {right}" + ) + ) + continue + + relationship_type = properties.get("relationshipType", "") + ai_context: Optional[str] = None + # Ossie encodes cardinality by direction: `from` is the many side, `to` the one side. + if relationship_type == "ONE_TO_MANY": + from_dataset, to_dataset = right, left + from_columns = [pair[1] for pair in column_pairs] + to_columns = [pair[0] for pair in column_pairs] + else: + from_dataset, to_dataset = left, right + from_columns = [pair[0] for pair in column_pairs] + to_columns = [pair[1] for pair in column_pairs] + if relationship_type == "ONE_TO_ONE": + ai_context = "one-to-one relationship" + elif relationship_type == "MANY_TO_MANY": + ai_context = "many-to-many relationship; cardinality is not representable in Ossie" + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.CARDINALITY_LOSS, element_name=f"{left} <-> {right}" + ) + ) + + name = f"{from_dataset}_to_{to_dataset}" + suffix = 2 + while name in seen_names: + name = f"{from_dataset}_to_{to_dataset}_{suffix}" + suffix += 1 + seen_names.add(name) + + relationships.append( + OSIRelationship( + name=name, + from_dataset=from_dataset, + to=to_dataset, + from_columns=from_columns, + to_columns=to_columns, + ai_context=ai_context, + ) + ) + return relationships + + def _extract_column_pairs(self, properties: dict) -> Optional[List[Tuple[str, str]]]: + """Returns (left_column, right_column) pairs, or None when the condition cannot be represented.""" + condition = properties.get("joinCondition") + if condition is not None: + pair = self._simple_condition_pair(condition) + return [pair] if pair else None + compound = properties.get("compoundJoinCondition") + if compound is not None: + return self._flatten_compound_condition(compound) + return None + + def _simple_condition_pair(self, condition: dict) -> Optional[Tuple[str, str]]: + left = condition.get("leftColumn", {}).get("name") + right = condition.get("rightColumn", {}).get("name") + # The only join operator wisdom emits is EQUAL (proto default, omitted in JSON). + if condition.get("operator") not in (None, "EQUAL") or not left or not right: + return None + return (left, right) + + def _flatten_compound_condition(self, compound: dict) -> Optional[List[Tuple[str, str]]]: + """Flattens an AND-of-equals compound condition into column pairs; None for anything else.""" + simple = compound.get("simpleCondition") + if simple is not None: + pair = self._simple_condition_pair(simple) + return [pair] if pair else None + nested = compound.get("nestedCondition") + if nested is not None: + if nested.get("logicalOperator") != "AND": + return None + pairs: List[Tuple[str, str]] = [] + for child in nested.get("conditions", []): + child_pairs = self._flatten_compound_condition(child) + if child_pairs is None: + return None + pairs.extend(child_pairs) + return pairs or None + return None + + def _quote_identifier(self, name: str, dialect: OSIDialect) -> str: + """Quotes a column name for use as a SQL expression when it is not a plain identifier.""" + if _SIMPLE_IDENTIFIER.match(name): + return name + if dialect is OSIDialect.DATABRICKS: + return "`" + name.replace("`", "``") + "`" + return '"' + name.replace('"', '""') + '"' + + def _make_expression(self, expression: str, dialect: OSIDialect) -> OSIExpression: + return OSIExpression(dialects=[OSIDialectExpression(dialect=dialect, expression=expression)]) diff --git a/converters/wisdom/tests/__init__.py b/converters/wisdom/tests/__init__.py new file mode 100644 index 0000000..13a8339 --- /dev/null +++ b/converters/wisdom/tests/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/wisdom/tests/fixtures/sample_export.json b/converters/wisdom/tests/fixtures/sample_export.json new file mode 100644 index 0000000..ebe256c --- /dev/null +++ b/converters/wisdom/tests/fixtures/sample_export.json @@ -0,0 +1,227 @@ +{ + "version": "1.0", + "export_metadata": { + "exported_at": "2026-07-10T00:00:00+00:00", + "source_domain_id": "ET_DOMAIN_sample", + "domain_name": "Sample Sales" + }, + "domain": { + "zsheet_json": { + "ref": {"uuid": "ET_DOMAIN_sample", "name": "Sample Sales", "version": "1"}, + "description": "Synthetic sales domain used for converter tests", + "zsheetType": "DOMAIN", + "domainSystemInstructions": "Only answer questions about sales data.", + "knowledge": [ + { + "name": "Fiscal year", + "content": "The fiscal year starts in February.", + "id": "ET_UNSTRUCTURED_KNOWLEDGE_1" + }, + { + "name": "Pipeline", + "content": "Pipeline refers to open orders expected to close this quarter.", + "id": "ET_UNSTRUCTURED_KNOWLEDGE_2", + "schemaAnnotation": {"relevance": "NOT_SCHEMA_RELATED"} + } + ], + "relationshipGraph": { + "zsheets": [ + {"uuid": "ET_ZSHEET_orders", "name": "orders", "version": "1"}, + {"uuid": "ET_ZSHEET_customers", "name": "customers", "version": "1"}, + {"uuid": "ET_ZSHEET_tags", "name": "tags", "version": "1"} + ], + "relationships": [ + { + "properties": { + "joinCondition": { + "leftColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_customers", "name": "customers"}} + }, + "relationshipType": "MANY_TO_ONE" + }, + "leftDataSource": {"zsheet": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightDataSource": {"zsheet": {"uuid": "ET_ZSHEET_customers", "name": "customers"}} + }, + { + "properties": { + "joinCondition": { + "leftColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_customers", "name": "customers"}}, + "rightColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}} + }, + "relationshipType": "ONE_TO_MANY" + }, + "leftDataSource": {"zsheet": {"uuid": "ET_ZSHEET_customers", "name": "customers"}}, + "rightDataSource": {"zsheet": {"uuid": "ET_ZSHEET_orders", "name": "orders"}} + }, + { + "properties": { + "joinCondition": { + "leftColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + }, + "relationshipType": "MANY_TO_MANY" + }, + "leftDataSource": {"zsheet": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightDataSource": {"zsheet": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + }, + { + "properties": { + "compoundJoinCondition": { + "nestedCondition": { + "logicalOperator": "AND", + "conditions": [ + { + "simpleCondition": { + "leftColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + } + }, + { + "simpleCondition": { + "leftColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightColumn": {"name": "customer_id", "zsheetRef": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + } + } + ] + } + }, + "relationshipType": "MANY_TO_ONE" + }, + "leftDataSource": {"zsheet": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightDataSource": {"zsheet": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + }, + { + "properties": { + "compoundJoinCondition": { + "nestedCondition": { + "logicalOperator": "OR", + "conditions": [ + { + "simpleCondition": { + "leftColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightColumn": {"name": "order_id", "zsheetRef": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + } + } + ] + } + }, + "relationshipType": "MANY_TO_ONE" + }, + "leftDataSource": {"zsheet": {"uuid": "ET_ZSHEET_orders", "name": "orders"}}, + "rightDataSource": {"zsheet": {"uuid": "ET_ZSHEET_tags", "name": "tags"}} + } + ] + } + } + }, + "tables": [ + { + "zsheet_uuid": "ET_ZSHEET_orders", + "zsheet_json": { + "ref": {"uuid": "ET_ZSHEET_orders", "name": "orders", "version": "1"}, + "description": "Customer orders", + "location": { + "database": "analytics", + "schema": "sales", + "dbTable": "orders", + "connectionId": "et-connection-snowflake" + }, + "columns": [ + {"name": "order_id", "properties": {"dataType": "INT64", "isPrimaryKey": true}}, + {"name": "customer_id", "properties": {"dataType": "INT64"}}, + {"name": "order_date", "properties": {"dataType": "DATE"}}, + {"name": "amount", "properties": {"dataType": "DOUBLE", "visibility": "HIDDEN"}}, + { + "name": "status", + "description": "Current order status", + "properties": {"dataType": "VARCHAR", "displayName": "Order Status", "isEnum": true} + }, + {"name": "Discount - Percent", "properties": {"dataType": "DOUBLE"}} + ], + "formulas": [ + { + "name": "is_large", + "expression": "CASE WHEN \"orders\".\"amount\" > 100 THEN TRUE ELSE FALSE END", + "description": "true when the order amount exceeds 100", + "properties": {"dataType": "BOOL", "displayName": "Is Large"}, + "id": "FORMULA_1" + } + ], + "measures": [ + { + "name": "total_amount", + "expression": "SUM(\"orders\".\"amount\")", + "description": "Total order amount", + "staleReason": "Column orders.amount is hidden in this domain and cannot be queried.", + "id": "MEASURE_1" + } + ] + } + }, + { + "zsheet_uuid": "ET_ZSHEET_customers", + "zsheet_json": { + "ref": {"uuid": "ET_ZSHEET_customers", "name": "customers", "version": "1"}, + "description": "Customer master data", + "location": { + "database": "analytics", + "schema": "sales", + "dbTable": "customers", + "connectionId": "et-connection-snowflake" + }, + "primaryKey": {"columns": ["customer_id"]}, + "columns": [ + {"name": "customer_id", "properties": {"dataType": "INT64"}}, + {"name": "name", "properties": {"dataType": "VARCHAR"}}, + {"name": "region", "properties": {"dataType": "VARCHAR"}} + ], + "formulas": [ + { + "name": "region", + "expression": "UPPER(\"customers\".\"region\")", + "properties": {"dataType": "VARCHAR"}, + "id": "FORMULA_2" + } + ], + "measures": [ + { + "name": "total_amount", + "expression": "COUNT(DISTINCT \"customers\".\"customer_id\")", + "description": "Distinct customer count (name collides with the orders measure)", + "id": "MEASURE_2" + } + ] + } + }, + { + "zsheet_uuid": "ET_ZSHEET_tags", + "zsheet_json": { + "ref": {"uuid": "ET_ZSHEET_tags", "name": "tags", "version": "1"}, + "description": "Order tags", + "location": { + "database": "tagdb", + "schema": "public", + "dbTable": "tags", + "connectionId": "et-connection-postgres" + }, + "columns": [ + {"name": "tag_id", "properties": {"dataType": "INT64"}}, + {"name": "order_id", "properties": {"dataType": "INT64"}}, + {"name": "customer_id", "properties": {"dataType": "INT64"}} + ] + } + } + ], + "connections": [ + {"connection_id": "et-connection-snowflake", "dialect": "snowflake", "name": "Snowflake"}, + {"connection_id": "et-connection-postgres", "dialect": "postgres", "name": "Postgres"} + ], + "reviewed_queries": {"ref": null, "items_json": "{}"}, + "synonym_sets": {"ref": null, "items_json": "{}"}, + "table_metadata": [ + {"zsheet_uuid": "ET_ZSHEET_orders", "connection_id": "et-connection-snowflake", "database": "analytics", "schema": "sales", "table_name": "orders"}, + {"zsheet_uuid": "ET_ZSHEET_customers", "connection_id": "et-connection-snowflake", "database": "analytics", "schema": "sales", "table_name": "customers"}, + {"zsheet_uuid": "ET_ZSHEET_tags", "connection_id": "et-connection-postgres", "database": "tagdb", "schema": "public", "table_name": "tags"} + ], + "recommended_questions": ["What is the total order amount by region?"] +} diff --git a/converters/wisdom/tests/test_wisdom_to_osi.py b/converters/wisdom/tests/test_wisdom_to_osi.py new file mode 100644 index 0000000..eaa1c2d --- /dev/null +++ b/converters/wisdom/tests/test_wisdom_to_osi.py @@ -0,0 +1,167 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from pathlib import Path + +import pytest +import yaml + +from ossie import OSIDialect, OSIDocument +from ossie_wisdom import ConverterIssueType, WisdomToOSIConverter + +FIXTURE = Path(__file__).parent / "fixtures" / "sample_export.json" + + +@pytest.fixture(scope="module") +def result(): + export = json.loads(FIXTURE.read_text()) + return WisdomToOSIConverter().convert(export) + + +@pytest.fixture(scope="module") +def model(result): + assert len(result.output.semantic_model) == 1 + return result.output.semantic_model[0] + + +def _issues_of(result, issue_type): + return [issue for issue in result.issues if issue.issue_type is issue_type] + + +def test_model_name_and_description(model): + assert model.name == "Sample Sales" + assert model.description == "Synthetic sales domain used for converter tests" + + +def test_knowledge_becomes_model_ai_context(model): + assert model.ai_context == ( + "Only answer questions about sales data.\n" + "- The fiscal year starts in February.\n" + "- Pipeline refers to open orders expected to close this quarter." + ) + + +def test_datasets(model): + datasets = {dataset.name: dataset for dataset in model.datasets} + assert set(datasets) == {"orders", "customers", "tags"} + assert datasets["orders"].source == "analytics.sales.orders" + assert datasets["orders"].description == "Customer orders" + # Explicit primaryKey wins; otherwise per-column isPrimaryKey flags are collected. + assert datasets["customers"].primary_key == ["customer_id"] + assert datasets["orders"].primary_key == ["order_id"] + assert datasets["tags"].primary_key is None + + +def test_columns_become_fields(model): + orders = next(dataset for dataset in model.datasets if dataset.name == "orders") + fields = {field.name: field for field in orders.fields} + assert fields["order_id"].expression.dialects[0].expression == "order_id" + assert fields["order_id"].expression.dialects[0].dialect is OSIDialect.SNOWFLAKE + assert fields["order_date"].dimension.is_time is True + assert fields["order_id"].dimension is None + assert fields["status"].label == "Order Status" + assert fields["status"].description == "Current order status" + # Hidden columns are included. + assert "amount" in fields + # Non-identifier column names are quoted in the expression so they parse as SQL. + assert fields["Discount - Percent"].expression.dialects[0].expression == '"Discount - Percent"' + + +def test_formulas_become_fields(model): + orders = next(dataset for dataset in model.datasets if dataset.name == "orders") + fields = {field.name: field for field in orders.fields} + is_large = fields["is_large"] + assert is_large.expression.dialects[0].expression == 'CASE WHEN "orders"."amount" > 100 THEN TRUE ELSE FALSE END' + assert is_large.expression.dialects[0].dialect is OSIDialect.SNOWFLAKE + assert is_large.label == "Is Large" + assert is_large.description == "true when the order amount exceeds 100" + + +def test_formula_colliding_with_column_is_dropped(result, model): + customers = next(dataset for dataset in model.datasets if dataset.name == "customers") + assert [field.name for field in customers.fields].count("region") == 1 + dropped = _issues_of(result, ConverterIssueType.DUPLICATE_FIELD_DROPPED) + assert [issue.element_name for issue in dropped] == ["customers.region"] + + +def test_unsupported_dialect_falls_back_to_ansi(result, model): + tags = next(dataset for dataset in model.datasets if dataset.name == "tags") + assert all(field.expression.dialects[0].dialect is OSIDialect.ANSI_SQL for field in tags.fields) + unsupported = _issues_of(result, ConverterIssueType.UNSUPPORTED_DIALECT) + assert len(unsupported) == 1 + assert "postgres" in unsupported[0].element_name + + +def test_relationship_directions(model): + relationships = {relationship.name: relationship for relationship in model.relationships} + # MANY_TO_ONE keeps left as the many side. + many_to_one = relationships["orders_to_customers"] + assert (many_to_one.from_dataset, many_to_one.to) == ("orders", "customers") + assert many_to_one.from_columns == ["customer_id"] + assert many_to_one.to_columns == ["customer_id"] + # ONE_TO_MANY is flipped so `from` is the many side; the name is deduped. + flipped = relationships["orders_to_customers_2"] + assert (flipped.from_dataset, flipped.to) == ("orders", "customers") + + +def test_many_to_many_is_kept_with_cardinality_loss(result, model): + relationships = {relationship.name: relationship for relationship in model.relationships} + many_to_many = relationships["orders_to_tags"] + assert many_to_many.ai_context == "many-to-many relationship; cardinality is not representable in Ossie" + losses = _issues_of(result, ConverterIssueType.CARDINALITY_LOSS) + assert [issue.element_name for issue in losses] == ["orders <-> tags"] + + +def test_compound_and_join_is_flattened(model): + relationships = {relationship.name: relationship for relationship in model.relationships} + compound = relationships["orders_to_tags_2"] + assert compound.from_columns == ["order_id", "customer_id"] + assert compound.to_columns == ["order_id", "customer_id"] + + +def test_or_join_is_dropped(result, model): + assert len(model.relationships) == 4 + dropped = _issues_of(result, ConverterIssueType.RELATIONSHIP_DROPPED) + assert [issue.element_name for issue in dropped] == ["orders <-> tags"] + + +def test_measures_become_metrics(result, model): + metrics = {metric.name: metric for metric in model.metrics} + assert set(metrics) == {"total_amount", "customers_total_amount"} + total = metrics["total_amount"] + assert total.expression.dialects[0].expression == 'SUM("orders"."amount")' + assert total.expression.dialects[0].dialect is OSIDialect.SNOWFLAKE + assert total.description == "Total order amount" + collisions = _issues_of(result, ConverterIssueType.METRIC_NAME_COLLISION) + assert [issue.element_name for issue in collisions] == ["customers.total_amount"] + + +def test_stale_measure_is_kept_with_warning(result, model): + assert any(metric.name == "total_amount" for metric in model.metrics) + stale = _issues_of(result, ConverterIssueType.STALE_MEASURE) + assert [issue.element_name for issue in stale] == ["orders.total_amount"] + + +def test_output_round_trips_through_osi_yaml(result): + document = OSIDocument.model_validate(yaml.safe_load(result.output.to_osi_yaml())) + assert document == result.output + + +def test_export_without_tables_is_rejected(): + with pytest.raises(ValueError, match="no tables"): + WisdomToOSIConverter().convert({"domain": {"zsheet_json": {"ref": {"name": "empty"}}}, "tables": []}) From a584961881e5cfd24e472adaee6a9a45805b6d1b Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Fri, 10 Jul 2026 16:01:57 -0700 Subject: [PATCH 2/3] Add WISDOM to well-known custom-extension vendors Adds WISDOM to the OSIVendor enum, the vendor examples in the spec documents and JSON schema, and the supported-vendors table in the converter guide. Co-Authored-By: Claude Fable 5 --- converters/index.md | 1 + core-spec/osi-schema.json | 2 +- core-spec/spec.md | 1 + core-spec/spec.yaml | 2 +- python/src/ossie/models.py | 1 + 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/converters/index.md b/converters/index.md index 1ceb3b8..b835531 100644 --- a/converters/index.md +++ b/converters/index.md @@ -73,6 +73,7 @@ The Ossie specification currently defines extensions for the following vendors: | `SALESFORCE` | Salesforce / Tableau semantic layer | | `DBT` | dbt semantic models | | `DATABRICKS` | Databricks semantic layer | +| `WISDOM` | WisdomAI domain | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/core-spec/osi-schema.json b/core-spec/osi-schema.json index 1732d4d..30ee7f6 100644 --- a/core-spec/osi-schema.json +++ b/core-spec/osi-schema.json @@ -28,7 +28,7 @@ }, "Vendor": { "type": "string", - "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA"], + "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM"], "description": "Vendor name for custom extensions. Any string value is accepted." }, "AIContext": { diff --git a/core-spec/spec.md b/core-spec/spec.md index b544042..be965dc 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -385,6 +385,7 @@ The following are well-known examples: | `DBT` | dbt-specific attributes | | `DATABRICKS` | Databricks-specific attributes | | `GOODDATA` | GoodData-specific attributes | +| `WISDOM` | WisdomAI-specific attributes | ### Examples diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 316d784..097e158 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -40,7 +40,7 @@ dialects: # Vendor name for custom extensions (free-form string) -# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA" +# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM" vendor_name: string diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index cc4aad5..5a9457c 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -42,6 +42,7 @@ class OSIVendor(str, Enum): DBT = "DBT" DATABRICKS = "DATABRICKS" GOODDATA = "GOODDATA" + WISDOM = "WISDOM" class OSIAIContextObject(BaseModel): From ba8672bb66582de66b32e7c7c6680acab06cfe2a Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Fri, 10 Jul 2026 16:12:47 -0700 Subject: [PATCH 3/3] Add Ossie -> wisdom domain export direction Adds osi-to-wisdom, the reverse of the wisdom converter: an Ossie document becomes a wisdom domain export (format 1.0) consumable by wisdom's importDomain API. The mapping inverts wisdom-to-osi so round-trips are stable in both directions: model ai_context splits back into system instructions and knowledge items, fields split into columns and formulas, relationship direction is read as many-to-one with ai_context notes restoring one-to-one/many-to-many and composite keys becoming compound AND join conditions, and metrics attach to the table their expression references. IDs are deterministic (derived from names) and connections are per-dialect placeholders remapped at import time. Verified: Ossie -> wisdom -> Ossie round-trips of two real domain exports are byte-identical. Co-Authored-By: Claude Fable 5 --- converters/wisdom/README.md | 32 +- .../wisdom/src/ossie_wisdom/__init__.py | 2 + converters/wisdom/src/ossie_wisdom/cli.py | 48 ++- .../src/ossie_wisdom/converter_issues.py | 6 + .../wisdom/src/ossie_wisdom/osi_to_wisdom.py | 393 ++++++++++++++++++ converters/wisdom/tests/test_osi_to_wisdom.py | 229 ++++++++++ 6 files changed, 700 insertions(+), 10 deletions(-) create mode 100644 converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py create mode 100644 converters/wisdom/tests/test_osi_to_wisdom.py diff --git a/converters/wisdom/README.md b/converters/wisdom/README.md index 166c356..4b4064f 100644 --- a/converters/wisdom/README.md +++ b/converters/wisdom/README.md @@ -17,16 +17,18 @@ under the License. --> -# WisdomAI → Apache Ossie Converter +# WisdomAI ↔ Apache Ossie Converter -Converts a [WisdomAI](https://wisdom.ai) domain export (format `1.0`, produced by wisdom's -`exportDomain` API) into an Ossie semantic model YAML document. +Converts between a [WisdomAI](https://wisdom.ai) domain export (format `1.0`, produced by +wisdom's `exportDomain` API and consumed by `importDomain`) and an Ossie semantic model +YAML document. ## Usage ```bash pip install -e ../../python -e . ossie-wisdom wisdom-to-osi -i domain-export.json -o semantic_model.yaml +ossie-wisdom osi-to-wisdom -i semantic_model.yaml -o domain-export.json ``` Conversion warnings (information loss) are printed to stderr; the output YAML validates @@ -73,9 +75,31 @@ Compound join conditions that are an `AND` of equality conditions are flattened positional `from_columns`/`to_columns` arrays; any other compound condition (e.g. `OR`) drops the relationship with a `RELATIONSHIP_DROPPED` warning. +## Ossie → wisdom (export direction) + +`osi-to-wisdom` emits a domain export mirroring wisdom's `exportDomain` JSON, inverting +the mapping above so that wisdom → Ossie → wisdom and Ossie → wisdom → Ossie round-trips +are stable: + +- Model `ai_context` splits back into `domainSystemInstructions` (leading text) and one + knowledge item per `- ` bullet. +- A field whose expression is just its own (possibly quoted) name becomes a column; + anything else becomes a formula. `dimension.is_time` becomes a `TIMESTAMP` data type + (wisdom re-derives exact types from the warehouse). +- Relationships become `MANY_TO_ONE` edges (Ossie's `from` is the many side); the + `ai_context` notes written by `wisdom-to-osi` restore `ONE_TO_ONE`/`MANY_TO_MANY`, and + composite keys become compound `AND` join conditions. +- Metrics attach to the first dataset their expression references (a + `METRIC_TABLE_UNRESOLVED` warning falls back to the first dataset). +- Connections are per-dialect placeholders (`et-connection-snowflake`, ...) expected to be + remapped when the domain is imported; all IDs are derived deterministically from names, + so re-runs produce identical output. +- Not representable in wisdom (dropped with warnings): extra semantic models beyond the + first, `unique_keys`, `custom_extensions`, and `ai_context` on fields/metrics (plus + synonyms/examples anywhere). + ## Known limitations -- One-way only (wisdom → Ossie). No Ossie → wisdom export yet. - Hidden columns and stale measures are converted anyway (with a `STALE_MEASURE` warning for the latter), so the output may reference columns wisdom itself hides from querying. - Out of scope for now: reviewed queries, recommended questions, synonym sets, row-level diff --git a/converters/wisdom/src/ossie_wisdom/__init__.py b/converters/wisdom/src/ossie_wisdom/__init__.py index 94153f4..337704d 100644 --- a/converters/wisdom/src/ossie_wisdom/__init__.py +++ b/converters/wisdom/src/ossie_wisdom/__init__.py @@ -16,11 +16,13 @@ # under the License. from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult +from ossie_wisdom.osi_to_wisdom import OSIToWisdomConverter from ossie_wisdom.wisdom_to_osi import WisdomToOSIConverter __all__ = [ "ConverterIssue", "ConverterIssueType", "ConverterResult", + "OSIToWisdomConverter", "WisdomToOSIConverter", ] diff --git a/converters/wisdom/src/ossie_wisdom/cli.py b/converters/wisdom/src/ossie_wisdom/cli.py index 0274e62..6248697 100644 --- a/converters/wisdom/src/ossie_wisdom/cli.py +++ b/converters/wisdom/src/ossie_wisdom/cli.py @@ -19,6 +19,7 @@ Usage: ossie-wisdom wisdom-to-osi -i domain-export.json -o output.yaml + ossie-wisdom osi-to-wisdom -i input.yaml -o domain-export.json """ import argparse @@ -26,24 +27,45 @@ import sys from pathlib import Path +import yaml + +from ossie import OSIDocument from ossie_wisdom.converter_issues import ConverterIssueType +from ossie_wisdom.osi_to_wisdom import OSIToWisdomConverter from ossie_wisdom.wisdom_to_osi import WisdomToOSIConverter _ISSUE_REASON: dict[ConverterIssueType, str] = { ConverterIssueType.UNSUPPORTED_DIALECT: "the connection dialect has no Ossie equivalent; expressions were emitted verbatim as ANSI_SQL", ConverterIssueType.CARDINALITY_LOSS: "Ossie relationships encode cardinality by direction and cannot represent many-to-many", - ConverterIssueType.RELATIONSHIP_DROPPED: "the relationship uses a join Ossie cannot represent (non-table source, OR/non-equi condition, or unknown dataset)", + ConverterIssueType.RELATIONSHIP_DROPPED: "the relationship uses a join that cannot be represented (non-table source, OR/non-equi condition, or unknown dataset)", ConverterIssueType.METRIC_NAME_COLLISION: "another table defines a measure with the same name; this one was prefixed with its table name", ConverterIssueType.STALE_MEASURE: "wisdom marked this measure stale; it was converted anyway", ConverterIssueType.DUPLICATE_FIELD_DROPPED: "the dataset already has a field with this name", + ConverterIssueType.EXTRA_MODEL_DROPPED: "a wisdom domain export holds a single domain; only the first semantic model was converted", + ConverterIssueType.AI_CONTEXT_DROPPED: "wisdom has no equivalent for ai_context at this level (or for synonyms/examples)", + ConverterIssueType.METRIC_TABLE_UNRESOLVED: "the metric expression references no known dataset; it was attached to the first dataset", + ConverterIssueType.MISSING_DIALECT_EXPRESSION: "no expression was available in the dataset's dialect or ANSI_SQL; the first available dialect was used", + ConverterIssueType.UNIQUE_KEYS_DROPPED: "wisdom has no unique-key construct", + ConverterIssueType.CUSTOM_EXTENSION_DROPPED: "wisdom cannot store Ossie custom extensions", } _DROPPED_ISSUE_TYPES = { ConverterIssueType.RELATIONSHIP_DROPPED, ConverterIssueType.DUPLICATE_FIELD_DROPPED, + ConverterIssueType.EXTRA_MODEL_DROPPED, + ConverterIssueType.AI_CONTEXT_DROPPED, + ConverterIssueType.UNIQUE_KEYS_DROPPED, + ConverterIssueType.CUSTOM_EXTENSION_DROPPED, } +def _print_issues(result) -> None: + for issue in result.issues: + verb = "was dropped" if issue.issue_type in _DROPPED_ISSUE_TYPES else "was converted with loss" + reason = _ISSUE_REASON[issue.issue_type] + print(f"[WARNING] {issue.issue_type.value}: {issue.element_name} {verb} during conversion because {reason}", file=sys.stderr) + + def _cmd_wisdom_to_osi(args: argparse.Namespace) -> None: input_path = Path(args.input) output_path = Path(args.output) @@ -51,15 +73,23 @@ def _cmd_wisdom_to_osi(args: argparse.Namespace) -> None: export = json.loads(input_path.read_text()) result = WisdomToOSIConverter().convert(export) - for issue in result.issues: - verb = "was dropped" if issue.issue_type in _DROPPED_ISSUE_TYPES else "was converted with loss" - reason = _ISSUE_REASON[issue.issue_type] - print(f"[WARNING] {issue.issue_type.value}: {issue.element_name} {verb} during conversion because {reason}", file=sys.stderr) - + _print_issues(result) output_path.write_text(result.output.to_osi_yaml()) print(f"Written to {output_path}", file=sys.stderr) +def _cmd_osi_to_wisdom(args: argparse.Namespace) -> None: + input_path = Path(args.input) + output_path = Path(args.output) + + document = OSIDocument.model_validate(yaml.safe_load(input_path.read_text())) + result = OSIToWisdomConverter().convert(document) + + _print_issues(result) + output_path.write_text(json.dumps(result.output, indent=2) + "\n") + print(f"Written to {output_path}", file=sys.stderr) + + def main() -> None: parser = argparse.ArgumentParser( prog="ossie-wisdom", @@ -71,9 +101,15 @@ def main() -> None: wisdom_to_osi.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to wisdom domain export JSON") wisdom_to_osi.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output Ossie YAML") + osi_to_wisdom = subparsers.add_parser("osi-to-wisdom", help="Convert Ossie YAML → domain-export.json") + osi_to_wisdom.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to Ossie YAML") + osi_to_wisdom.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output wisdom domain export JSON") + args = parser.parse_args() if args.command == "wisdom-to-osi": _cmd_wisdom_to_osi(args) + elif args.command == "osi-to-wisdom": + _cmd_osi_to_wisdom(args) if __name__ == "__main__": diff --git a/converters/wisdom/src/ossie_wisdom/converter_issues.py b/converters/wisdom/src/ossie_wisdom/converter_issues.py index f8c8fb2..fcef92f 100644 --- a/converters/wisdom/src/ossie_wisdom/converter_issues.py +++ b/converters/wisdom/src/ossie_wisdom/converter_issues.py @@ -29,6 +29,12 @@ class ConverterIssueType(Enum): METRIC_NAME_COLLISION = "METRIC_NAME_COLLISION" STALE_MEASURE = "STALE_MEASURE" DUPLICATE_FIELD_DROPPED = "DUPLICATE_FIELD_DROPPED" + EXTRA_MODEL_DROPPED = "EXTRA_MODEL_DROPPED" + AI_CONTEXT_DROPPED = "AI_CONTEXT_DROPPED" + METRIC_TABLE_UNRESOLVED = "METRIC_TABLE_UNRESOLVED" + MISSING_DIALECT_EXPRESSION = "MISSING_DIALECT_EXPRESSION" + UNIQUE_KEYS_DROPPED = "UNIQUE_KEYS_DROPPED" + CUSTOM_EXTENSION_DROPPED = "CUSTOM_EXTENSION_DROPPED" @dataclass(frozen=True) diff --git a/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py b/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py new file mode 100644 index 0000000..cc245d1 --- /dev/null +++ b/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py @@ -0,0 +1,393 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Converts an Ossie Document into a WisdomAI domain export (format 1.0). + +The output mirrors the JSON produced by wisdom's ``exportDomain`` RPC so it +can be fed to ``importDomain``. Inverse of :mod:`ossie_wisdom.wisdom_to_osi`: +model ``ai_context`` splits back into system instructions and knowledge items, +fields split back into columns and formulas, relationship direction is read as +many-to-one (with the ai_context notes restoring one-to-one/many-to-many), and +metrics attach to the table their expression references. + +IDs are derived deterministically from element names, and connections are +per-dialect placeholders expected to be remapped when the domain is imported. +""" + +import hashlib +import re +from datetime import datetime, timezone +from typing import Dict, List, Optional, Tuple + +from ossie import ( + OSIAIContextObject, + OSIDataset, + OSIDialect, + OSIDocument, + OSIExpression, + OSISemanticModel, +) +from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult + +_WISDOM_DIALECT = { + OSIDialect.SNOWFLAKE: "snowflake", + OSIDialect.DATABRICKS: "databricks", + OSIDialect.ANSI_SQL: "ansi", +} + + +def _stable_id(prefix: str, value: str) -> str: + return f"{prefix}_{hashlib.md5(value.encode('utf-8')).hexdigest()}" + + +class OSIToWisdomConverter: + """Converts an Ossie Document into a wisdom domain export dict.""" + + def convert(self, document: OSIDocument, exported_at: Optional[str] = None) -> ConverterResult[dict]: + issues: List[ConverterIssue] = [] + + model = document.semantic_model[0] + for extra in document.semantic_model[1:]: + issues.append(ConverterIssue(issue_type=ConverterIssueType.EXTRA_MODEL_DROPPED, element_name=extra.name)) + self._report_custom_extensions(model, issues) + + domain_uuid = _stable_id("ET_DOMAIN", model.name) + zsheet_refs = { + dataset.name: {"uuid": _stable_id("ET_ZSHEET", dataset.name), "name": dataset.name, "version": "1"} + for dataset in model.datasets + } + dataset_dialects = {dataset.name: self._infer_dataset_dialect(dataset) for dataset in model.datasets} + measures_by_dataset = self._assign_metrics(model, dataset_dialects, issues) + + tables = [] + table_metadata = [] + for dataset in model.datasets: + zsheet = self._convert_dataset( + dataset, zsheet_refs, dataset_dialects[dataset.name], measures_by_dataset.get(dataset.name, []), issues + ) + tables.append({"zsheet_uuid": zsheet_refs[dataset.name]["uuid"], "zsheet_json": zsheet}) + location = zsheet["location"] + table_metadata.append( + { + "zsheet_uuid": zsheet_refs[dataset.name]["uuid"], + "connection_id": location["connectionId"], + "database": location["database"], + "schema": location["schema"], + "table_name": location["dbTable"], + } + ) + + instructions, knowledge = self._split_ai_context(model, issues) + domain_zsheet: dict = { + "ref": {"uuid": domain_uuid, "name": model.name, "version": "1"}, + "zsheetType": "DOMAIN", + "relationshipGraph": { + "zsheets": list(zsheet_refs.values()), + "relationships": self._convert_relationships(model, zsheet_refs, issues), + }, + } + if model.description: + domain_zsheet["description"] = model.description + if instructions: + domain_zsheet["domainSystemInstructions"] = instructions + if knowledge: + domain_zsheet["knowledge"] = knowledge + + connections = [ + {"connection_id": f"et-connection-{dialect}", "dialect": dialect, "name": dialect} + for dialect in sorted({_WISDOM_DIALECT[d] for d in dataset_dialects.values()}) + ] + + export = { + "version": "1.0", + "export_metadata": { + "exported_at": exported_at or datetime.now(timezone.utc).isoformat(), + "source_domain_id": domain_uuid, + "domain_name": model.name, + }, + "domain": {"zsheet_json": domain_zsheet}, + "tables": tables, + "connections": connections, + "reviewed_queries": {"ref": None, "items_json": "{}"}, + "synonym_sets": {"ref": None, "items_json": "{}"}, + "table_metadata": table_metadata, + "recommended_questions": [], + } + return ConverterResult(output=export, issues=issues) + + def _report_custom_extensions(self, model: OSISemanticModel, issues: List[ConverterIssue]) -> None: + elements = [(model.name, model.custom_extensions)] + for dataset in model.datasets: + elements.append((dataset.name, dataset.custom_extensions)) + for field in dataset.fields or []: + elements.append((f"{dataset.name}.{field.name}", field.custom_extensions)) + for relationship in model.relationships or []: + elements.append((relationship.name, relationship.custom_extensions)) + for metric in model.metrics or []: + elements.append((metric.name, metric.custom_extensions)) + for name, extensions in elements: + if extensions: + issues.append( + ConverterIssue(issue_type=ConverterIssueType.CUSTOM_EXTENSION_DROPPED, element_name=name) + ) + + def _infer_dataset_dialect(self, dataset: OSIDataset) -> OSIDialect: + for field in dataset.fields or []: + for entry in field.expression.dialects: + if entry.dialect in (OSIDialect.SNOWFLAKE, OSIDialect.DATABRICKS): + return entry.dialect + return OSIDialect.ANSI_SQL + + def _split_ai_context(self, model: OSISemanticModel, issues: List[ConverterIssue]) -> Tuple[str, List[dict]]: + ai_context = model.ai_context + if ai_context is None: + return "", [] + if isinstance(ai_context, OSIAIContextObject): + if ai_context.synonyms or ai_context.examples: + issues.append(ConverterIssue(issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=model.name)) + text = ai_context.instructions or "" + else: + text = ai_context + + instruction_lines: List[str] = [] + contents: List[str] = [] + current: Optional[str] = None + for line in text.split("\n"): + if line.startswith("- "): + if current is not None: + contents.append(current) + current = line[2:] + elif current is not None: + current += "\n" + line + else: + instruction_lines.append(line) + if current is not None: + contents.append(current) + + knowledge = [ + {"name": content, "content": content, "id": _stable_id("ET_UNSTRUCTURED_KNOWLEDGE", content)} + for content in contents + if content.strip() + ] + return "\n".join(instruction_lines).strip(), knowledge + + def _convert_dataset( + self, + dataset: OSIDataset, + zsheet_refs: Dict[str, dict], + dialect: OSIDialect, + measures: List[dict], + issues: List[ConverterIssue], + ) -> dict: + ref = zsheet_refs[dataset.name] + ref_lite = {"uuid": ref["uuid"], "name": ref["name"]} + database, schema, table = self._split_source(dataset.source) + + columns: List[dict] = [] + formulas: List[dict] = [] + for field in dataset.fields or []: + expression = self._pick_expression(field.expression, dialect, f"{dataset.name}.{field.name}", issues) + if field.ai_context is not None: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=f"{dataset.name}.{field.name}" + ) + ) + properties: dict = {} + if field.label: + properties["displayName"] = field.label + if field.dimension and field.dimension.is_time: + properties["dataType"] = "TIMESTAMP" + if self._is_bare_column(expression, field.name): + column: dict = {"name": field.name} + if field.description: + column["description"] = field.description + if properties: + column["properties"] = properties + column["location"] = {"name": field.name, "zsheetRef": ref_lite} + columns.append(column) + else: + formula: dict = {"name": field.name, "expression": expression} + if field.description: + formula["description"] = field.description + if properties: + formula["properties"] = properties + formula["location"] = {"name": field.name, "zsheetRef": ref_lite} + formula["id"] = _stable_id("FORMULA", f"{dataset.name}.{field.name}") + formulas.append(formula) + + zsheet: dict = { + "ref": ref, + "location": { + "database": database, + "schema": schema, + "dbTable": table, + "connectionId": f"et-connection-{_WISDOM_DIALECT[dialect]}", + }, + "source": {"alias": dataset.name, "zsheet": ref_lite}, + "columns": columns, + } + if dataset.description: + zsheet["description"] = dataset.description + if formulas: + zsheet["formulas"] = formulas + if measures: + zsheet["measures"] = measures + if dataset.primary_key: + zsheet["primaryKey"] = {"columns": list(dataset.primary_key)} + if dataset.unique_keys: + issues.append(ConverterIssue(issue_type=ConverterIssueType.UNIQUE_KEYS_DROPPED, element_name=dataset.name)) + if dataset.ai_context is not None: + content = self._ai_context_text(dataset.ai_context, dataset.name, issues) + if content: + zsheet["knowledge"] = [ + {"name": content, "content": content, "id": _stable_id("ET_UNSTRUCTURED_KNOWLEDGE", content)} + ] + return zsheet + + def _ai_context_text(self, ai_context, element_name: str, issues: List[ConverterIssue]) -> str: + if isinstance(ai_context, OSIAIContextObject): + if ai_context.synonyms or ai_context.examples: + issues.append( + ConverterIssue(issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=element_name) + ) + return ai_context.instructions or "" + return ai_context + + def _split_source(self, source: str) -> Tuple[str, str, str]: + parts = source.split(".") + if len(parts) >= 3: + return parts[0], parts[1], ".".join(parts[2:]) + if len(parts) == 2: + return "", parts[0], parts[1] + return "", "", source + + def _pick_expression( + self, expression: OSIExpression, dialect: OSIDialect, element_name: str, issues: List[ConverterIssue] + ) -> str: + by_dialect = {entry.dialect: entry.expression for entry in expression.dialects} + if dialect in by_dialect: + return by_dialect[dialect] + if OSIDialect.ANSI_SQL in by_dialect: + return by_dialect[OSIDialect.ANSI_SQL] + issues.append( + ConverterIssue(issue_type=ConverterIssueType.MISSING_DIALECT_EXPRESSION, element_name=element_name) + ) + return expression.dialects[0].expression + + def _is_bare_column(self, expression: str, name: str) -> bool: + return expression in (name, f'"{name}"', f"`{name}`") + + def _assign_metrics( + self, model: OSISemanticModel, dataset_dialects: Dict[str, OSIDialect], issues: List[ConverterIssue] + ) -> Dict[str, List[dict]]: + measures: Dict[str, List[dict]] = {} + dataset_names = [dataset.name for dataset in model.datasets] + for metric in model.metrics or []: + if metric.ai_context is not None: + issues.append( + ConverterIssue(issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=metric.name) + ) + # Attach to a dataset first so the expression can be picked in that dataset's dialect. + reference_text = " ".join(entry.expression for entry in metric.expression.dialects) + dataset_name = self._find_referenced_dataset(reference_text, dataset_names) + if dataset_name is None: + dataset_name = dataset_names[0] + issues.append( + ConverterIssue(issue_type=ConverterIssueType.METRIC_TABLE_UNRESOLVED, element_name=metric.name) + ) + expression = self._pick_expression( + metric.expression, dataset_dialects[dataset_name], metric.name, issues + ) + ref = {"uuid": _stable_id("ET_ZSHEET", dataset_name), "name": dataset_name} + measure: dict = {"name": metric.name, "expression": expression} + if metric.description: + measure["description"] = metric.description + measure["location"] = {"name": metric.name, "zsheetRef": ref} + measure["id"] = _stable_id("MEASURE", f"{dataset_name}.{metric.name}") + measures.setdefault(dataset_name, []).append(measure) + return measures + + def _find_referenced_dataset(self, expression: str, dataset_names: List[str]) -> Optional[str]: + best: Optional[str] = None + best_position = len(expression) + 1 + for name in dataset_names: + pattern = re.compile(r'(? List[dict]: + edges = [] + for relationship in model.relationships or []: + left = relationship.from_dataset + right = relationship.to + if left not in zsheet_refs or right not in zsheet_refs: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.RELATIONSHIP_DROPPED, element_name=relationship.name + ) + ) + continue + relationship_type = "MANY_TO_ONE" + if isinstance(relationship.ai_context, str): + if relationship.ai_context.startswith("one-to-one"): + relationship_type = "ONE_TO_ONE" + elif relationship.ai_context.startswith("many-to-many"): + relationship_type = "MANY_TO_MANY" + else: + issues.append( + ConverterIssue( + issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=relationship.name + ) + ) + elif relationship.ai_context is not None: + issues.append( + ConverterIssue(issue_type=ConverterIssueType.AI_CONTEXT_DROPPED, element_name=relationship.name) + ) + + left_ref = {"uuid": zsheet_refs[left]["uuid"], "name": left} + right_ref = {"uuid": zsheet_refs[right]["uuid"], "name": right} + conditions = [ + { + "leftColumn": {"name": from_column, "zsheetRef": left_ref}, + "rightColumn": {"name": to_column, "zsheetRef": right_ref}, + } + for from_column, to_column in zip(relationship.from_columns, relationship.to_columns) + ] + properties: dict = {"relationshipType": relationship_type} + if len(conditions) == 1: + properties["joinCondition"] = conditions[0] + else: + properties["compoundJoinCondition"] = { + "nestedCondition": { + "logicalOperator": "AND", + "conditions": [{"simpleCondition": condition} for condition in conditions], + } + } + edges.append( + { + "properties": properties, + "leftDataSource": {"zsheet": left_ref}, + "rightDataSource": {"zsheet": right_ref}, + } + ) + return edges diff --git a/converters/wisdom/tests/test_osi_to_wisdom.py b/converters/wisdom/tests/test_osi_to_wisdom.py new file mode 100644 index 0000000..1ee1717 --- /dev/null +++ b/converters/wisdom/tests/test_osi_to_wisdom.py @@ -0,0 +1,229 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from pathlib import Path + +import pytest + +from ossie import ( + OSIDataset, + OSIDialect, + OSIDialectExpression, + OSIDocument, + OSIExpression, + OSIField, + OSIRelationship, + OSISemanticModel, +) +from ossie_wisdom import ConverterIssueType, OSIToWisdomConverter, WisdomToOSIConverter + +FIXTURE = Path(__file__).parent / "fixtures" / "sample_export.json" + + +def _snowflake(expression): + return OSIExpression(dialects=[OSIDialectExpression(dialect=OSIDialect.SNOWFLAKE, expression=expression)]) + + +@pytest.fixture(scope="module") +def osi_document(): + export = json.loads(FIXTURE.read_text()) + return WisdomToOSIConverter().convert(export).output + + +@pytest.fixture(scope="module") +def result(osi_document): + return OSIToWisdomConverter().convert(osi_document, exported_at="2026-07-10T00:00:00+00:00") + + +@pytest.fixture(scope="module") +def export(result): + return result.output + + +def _issues_of(result, issue_type): + return [issue for issue in result.issues if issue.issue_type is issue_type] + + +def _table(export, name): + return next( + table["zsheet_json"] for table in export["tables"] if table["zsheet_json"]["ref"]["name"] == name + ) + + +def test_export_envelope(export): + assert export["version"] == "1.0" + assert export["export_metadata"]["domain_name"] == "Sample Sales" + assert export["export_metadata"]["source_domain_id"] == export["domain"]["zsheet_json"]["ref"]["uuid"] + domain = export["domain"]["zsheet_json"] + assert domain["zsheetType"] == "DOMAIN" + assert domain["description"] == "Synthetic sales domain used for converter tests" + + +def test_ai_context_splits_into_instructions_and_knowledge(export): + domain = export["domain"]["zsheet_json"] + assert domain["domainSystemInstructions"] == "Only answer questions about sales data." + assert [knowledge["content"] for knowledge in domain["knowledge"]] == [ + "The fiscal year starts in February.", + "Pipeline refers to open orders expected to close this quarter.", + ] + + +def test_tables_and_locations(export): + orders = _table(export, "orders") + assert orders["location"]["database"] == "analytics" + assert orders["location"]["schema"] == "sales" + assert orders["location"]["dbTable"] == "orders" + assert orders["primaryKey"] == {"columns": ["order_id"]} + assert _table(export, "customers")["primaryKey"] == {"columns": ["customer_id"]} + metadata = {entry["zsheet_uuid"]: entry for entry in export["table_metadata"]} + assert metadata[orders["ref"]["uuid"]]["table_name"] == "orders" + + +def test_fields_split_into_columns_and_formulas(export): + orders = _table(export, "orders") + column_names = [column["name"] for column in orders["columns"]] + assert "order_id" in column_names + # A quoted bare-name expression is recognized as a plain column. + assert "Discount - Percent" in column_names + formulas = {formula["name"]: formula for formula in orders["formulas"]} + assert set(formulas) == {"is_large"} + assert formulas["is_large"]["expression"] == 'CASE WHEN "orders"."amount" > 100 THEN TRUE ELSE FALSE END' + assert formulas["is_large"]["properties"]["displayName"] == "Is Large" + status = next(column for column in orders["columns"] if column["name"] == "status") + assert status["description"] == "Current order status" + assert status["properties"]["displayName"] == "Order Status" + + +def test_metrics_attach_to_referenced_tables(export): + orders_measures = {measure["name"] for measure in _table(export, "orders")["measures"]} + customers_measures = {measure["name"] for measure in _table(export, "customers")["measures"]} + assert orders_measures == {"total_amount"} + assert customers_measures == {"customers_total_amount"} + + +def test_relationship_types_restored(export): + edges = export["domain"]["zsheet_json"]["relationshipGraph"]["relationships"] + types = [edge["properties"]["relationshipType"] for edge in edges] + assert types == ["MANY_TO_ONE", "MANY_TO_ONE", "MANY_TO_MANY", "MANY_TO_ONE"] + compound = edges[3]["properties"]["compoundJoinCondition"]["nestedCondition"] + assert compound["logicalOperator"] == "AND" + assert len(compound["conditions"]) == 2 + + +def test_connections_are_per_dialect(export): + dialects = {connection["dialect"] for connection in export["connections"]} + assert dialects == {"snowflake", "ansi"} + assert _table(export, "orders")["location"]["connectionId"] == "et-connection-snowflake" + assert _table(export, "tags")["location"]["connectionId"] == "et-connection-ansi" + + +def test_round_trip_preserves_osi_document(osi_document, export): + round_tripped = WisdomToOSIConverter().convert(export).output + assert round_tripped == osi_document + + +def test_deterministic_output(osi_document, export): + again = OSIToWisdomConverter().convert(osi_document, exported_at="2026-07-10T00:00:00+00:00").output + assert again == export + + +def test_extra_models_and_unrepresentable_elements_are_reported(): + dataset = OSIDataset( + name="orders", + source="analytics.sales.orders", + unique_keys=[["order_id"]], + fields=[ + OSIField(name="order_id", expression=_snowflake("order_id"), ai_context="the identifier"), + ], + ) + second = OSISemanticModel(name="second", datasets=[OSIDataset(name="d", source="a.b.c")]) + document = OSIDocument( + semantic_model=[ + OSISemanticModel( + name="first", + datasets=[dataset], + relationships=[ + OSIRelationship( + name="orders_to_missing", + from_dataset="orders", + to="missing", + from_columns=["x"], + to_columns=["y"], + ) + ], + ), + second, + ] + ) + result = OSIToWisdomConverter().convert(document, exported_at="2026-07-10T00:00:00+00:00") + assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.EXTRA_MODEL_DROPPED)] == ["second"] + assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.UNIQUE_KEYS_DROPPED)] == ["orders"] + assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.AI_CONTEXT_DROPPED)] == [ + "orders.order_id" + ] + assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.RELATIONSHIP_DROPPED)] == [ + "orders_to_missing" + ] + assert result.output["domain"]["zsheet_json"]["relationshipGraph"]["relationships"] == [] + + +def test_one_to_one_note_restores_relationship_type(): + document = OSIDocument( + semantic_model=[ + OSISemanticModel( + name="m", + datasets=[ + OSIDataset(name="a", source="db.s.a"), + OSIDataset(name="b", source="db.s.b"), + ], + relationships=[ + OSIRelationship( + name="a_to_b", + from_dataset="a", + to="b", + from_columns=["id"], + to_columns=["id"], + ai_context="one-to-one relationship", + ) + ], + ) + ] + ) + export = OSIToWisdomConverter().convert(document, exported_at="2026-07-10T00:00:00+00:00").output + edges = export["domain"]["zsheet_json"]["relationshipGraph"]["relationships"] + assert edges[0]["properties"]["relationshipType"] == "ONE_TO_ONE" + + +def test_unresolved_metric_attaches_to_first_dataset(): + from ossie import OSIMetric + + document = OSIDocument( + semantic_model=[ + OSISemanticModel( + name="m", + datasets=[OSIDataset(name="a", source="db.s.a"), OSIDataset(name="b", source="db.s.b")], + metrics=[OSIMetric(name="row_count", expression=_snowflake("COUNT(*)"))], + ) + ] + ) + result = OSIToWisdomConverter().convert(document, exported_at="2026-07-10T00:00:00+00:00") + export = result.output + assert [measure["name"] for measure in _table(export, "a")["measures"]] == ["row_count"] + assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.METRIC_TABLE_UNRESOLVED)] == [ + "row_count" + ]