From a5ad5ec4c3c431f3e74e72765472af8d434b72ce Mon Sep 17 00:00:00 2001 From: Cara Baestlein Date: Thu, 2 Jul 2026 11:55:39 +0100 Subject: [PATCH 1/4] feat(converters): add Ataccama ONE -> OSI catalog importer Import selected Ataccama ONE catalog items into an OSI semantic model via the public Catalog REST API (OAuth2 client-credentials auth, cursor pagination). - CatalogItem -> dataset; CatalogAttribute -> field (quoted ANSI_SQL identifier) - business terms -> ai_context; governance/DQ/source URNs -> ATACCAMA custom_extensions - rich-text descriptions flattened to plain text; is_time inferred from DATE/DATETIME - CLI `ataccama-to-osi`, scoped by explicit catalog-item URNs - import-only: the public Catalog API cannot create catalog items/attributes, so OSI -> Ataccama export is out of scope (documented in the README) - tests run offline against a sanitized fixture and validate output against the OSI schema vendor_name is a free-form string per the spec, so no core-spec changes are required. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 8 + converters/ataccama/README.md | 108 ++++++ converters/ataccama/pyproject.toml | 44 +++ .../ataccama/src/ataccama_osi/__init__.py | 14 + .../src/ataccama_osi/ataccama_to_osi.py | 226 ++++++++++++ converters/ataccama/src/ataccama_osi/cli.py | 112 ++++++ .../ataccama/src/ataccama_osi/client.py | 126 +++++++ .../ataccama/src/ataccama_osi/models.py | 191 ++++++++++ converters/ataccama/tests/__init__.py | 0 .../tests/fixtures/ataccama_bundles.json | 334 ++++++++++++++++++ .../ataccama/tests/test_ataccama_to_osi.py | 133 +++++++ 11 files changed, 1296 insertions(+) create mode 100644 converters/ataccama/README.md create mode 100644 converters/ataccama/pyproject.toml create mode 100644 converters/ataccama/src/ataccama_osi/__init__.py create mode 100644 converters/ataccama/src/ataccama_osi/ataccama_to_osi.py create mode 100644 converters/ataccama/src/ataccama_osi/cli.py create mode 100644 converters/ataccama/src/ataccama_osi/client.py create mode 100644 converters/ataccama/src/ataccama_osi/models.py create mode 100644 converters/ataccama/tests/__init__.py create mode 100644 converters/ataccama/tests/fixtures/ataccama_bundles.json create mode 100644 converters/ataccama/tests/test_ataccama_to_osi.py diff --git a/.gitignore b/.gitignore index 63e36ff..764a1be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,10 @@ **/__pycache__/ **/.venv/ +*.egg-info/ + +# Local secrets / credentials — never commit +.env +.env.local +*.env +.secrets* +.ataccama.env diff --git a/converters/ataccama/README.md b/converters/ataccama/README.md new file mode 100644 index 0000000..0bf5c5a --- /dev/null +++ b/converters/ataccama/README.md @@ -0,0 +1,108 @@ +# Ataccama OSI Converter + +Importer from [Ataccama ONE](https://docs.ataccama.com/ataccama-one-agentic/latest/rest-api/rest-api-overview.html) +catalog metadata to the [OSI (Open Semantic Interchange)](https://github.com/open-semantic-interchange/OSI) +semantic model specification. + +Given one or more Ataccama **catalog items** (selected by URN), the converter produces +a single OSI semantic model: each catalog item becomes a dataset, each catalog attribute +becomes a field, and business terms / governance metadata are carried across as +`ai_context` and `ATACCAMA` custom extensions. + +## Scope & direction + +- **Direction:** Ataccama → OSI (import only). See [Limitations](#limitations) for why + export (OSI → Ataccama) is not provided. +- **Input:** explicit catalog-item URNs. Ataccama catalogs are large (tens of thousands + of items, many of which are BI artifacts rather than tables), so the converter is + intentionally scoped to the items you name rather than crawling everything. +- **Data access:** a live REST client that authenticates via OAuth2 client-credentials + (tokens are short-lived and refreshed automatically) and follows the catalog API's + cursor pagination. + +## Configuration + +The client reads connection settings from environment variables (or a `--env-file`): + +| Variable | Description | +|---|---| +| `ATACCAMA_BASE_URL` | API root, e.g. `https:///api` | +| `ATACCAMA_TOKEN_URL` | OAuth2 token endpoint (Keycloak `.../protocol/openid-connect/token`) | +| `ATACCAMA_CLIENT_ID` | service-account client id | +| `ATACCAMA_CLIENT_SECRET` | service-account client secret | + +> Never commit credentials. Keep them in a git-ignored env file. + +## Usage + +```bash +# CLI +ataccama-to-osi --env-file .ataccama.env \ + --urn urn:ata::catalog:catalog-item: \ + --urn urn:ata::catalog:catalog-item: \ + --model-name my_model \ + --output my_model.osi.yaml + +# or supply a file of URNs (one per line) +ataccama-to-osi --env-file .ataccama.env --urns-file items.txt -o my_model.osi.yaml +``` + +```python +# Library +from ataccama_osi import AtaccamaClient, ataccama_to_osi +import yaml + +client = AtaccamaClient(base_url="https:///api", token_url="...", + client_id="...", client_secret="...") +bundles = [client.fetch_bundle(urn) for urn in urns] +document = ataccama_to_osi(bundles, model_name="my_model") +print(yaml.dump(document, sort_keys=False)) +``` + +## Development + +```bash +uv sync --group dev # or: python -m venv .venv && pip install -e . pytest jsonschema +uv run pytest +``` + +Tests run fully offline against a recorded catalog fixture +(`tests/fixtures/ataccama_bundles.json`) and validate the output against the OSI schema. + +## Concept mapping + +| Ataccama ONE (Catalog API) | OSI Semantic Model | +|---|---| +| `CatalogItem` | `dataset` | +| `CatalogItem.name` | `dataset.name` (de-duplicated if repeated) | +| `CatalogItem.locations` + name | `dataset.source` (best-effort dotted namespace) | +| `CatalogItem.description` (rich text) | `dataset.description` (flattened to plain text) | +| assigned `Term`s | `dataset.ai_context` (`synonyms` + `instructions`) | +| `CatalogAttribute` | `field` | +| `CatalogAttribute.name` | `field.name` + quoted `ANSI_SQL` identifier expression | +| `CatalogAttribute.dataType` ∈ {DATE, DATETIME, TIMESTAMP, TIME} | `field.dimension.is_time = true` | +| `CatalogAttribute.description` / `comment` | `field.description` | +| attribute `Term`s | `field.ai_context` | +| URNs, `dataType`, `columnType`, connection/source/stewardship/DQ | `custom_extensions` (`vendor_name: ATACCAMA`) | + +All Ataccama-specific metadata with no OSI-core home is preserved in `ATACCAMA` +custom extensions (as a JSON string) so the model round-trips without data loss. + +## Limitations + +The Ataccama Catalog API is a **catalog / governance / data-quality** surface, while OSI +is an **analytics semantic model**. Some OSI constructs therefore have no source today: + +- **Metrics** — Ataccama has no analytics metrics; none are emitted. +- **Relationships / `primary_key` / `unique_keys`** — the Catalog API does not expose + foreign or primary keys on catalog items, so these are omitted. +- **`source` string** — the API exposes only a folder hierarchy (`locations`) and + `originPath`, not `database.schema.table`. `source` is a best-effort dotted namespace; + the authoritative connection/source URNs are preserved in `custom_extensions`. +- **Expressions are physical** — attributes map to plain quoted column identifiers + (`ANSI_SQL` only); there are no computed/multi-dialect expressions. +- **Export (OSI → Ataccama)** is not provided: the public Catalog API can only `PATCH` + `description`/stewardship/aliases on existing items and cannot create catalog items, + attributes, or relationships. +- Items with no catalogued attributes (e.g. some dashboards/reports) produce a dataset + with no `fields`. diff --git a/converters/ataccama/pyproject.toml b/converters/ataccama/pyproject.toml new file mode 100644 index 0000000..2e0b59a --- /dev/null +++ b/converters/ataccama/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "ataccama-osi" +version = "0.2.0.dev0" +description = "Importer from Ataccama ONE catalog metadata to the OSI semantic model" +authors = [{name = "Ataccama", email = "support@ataccama.com"}] +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "pyyaml>=6.0.1", + "requests>=2.31.0", +] + +[project.scripts] +ataccama-to-osi = "ataccama_osi.cli:main" + +[dependency-groups] +dev = [ + "ruff>=0.11.5", + "pytest>=8.3.5", + "pytest-cov>=6.0.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +include = ["src/ataccama_osi/**/*"] +ignore-vcs = true + +[tool.hatch.build.targets.wheel] +packages = ["src/ataccama_osi"] + +[tool.ruff] +line-length = 120 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "UP", "W", "C4", "PIE", "SIM"] +ignore = ["E501"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/converters/ataccama/src/ataccama_osi/__init__.py b/converters/ataccama/src/ataccama_osi/__init__.py new file mode 100644 index 0000000..82aee37 --- /dev/null +++ b/converters/ataccama/src/ataccama_osi/__init__.py @@ -0,0 +1,14 @@ +"""Ataccama ONE -> OSI semantic model importer.""" + +from ataccama_osi.ataccama_to_osi import ataccama_to_osi +from ataccama_osi.client import AtaccamaClient +from ataccama_osi.models import CatalogAttribute, CatalogItem, CatalogItemBundle, Term + +__all__ = [ + "AtaccamaClient", + "CatalogAttribute", + "CatalogItem", + "CatalogItemBundle", + "Term", + "ataccama_to_osi", +] diff --git a/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py b/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py new file mode 100644 index 0000000..49e8132 --- /dev/null +++ b/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py @@ -0,0 +1,226 @@ +"""Convert Ataccama ONE catalog metadata to an OSI semantic model. + +Scope: the caller supplies one or more catalog items (as :class:`CatalogItemBundle`s); +each becomes an OSI dataset, and the collection becomes a single OSI semantic model. + +Mapping summary (see README for the full table and known limitations): + CatalogItem -> dataset (name, source, description, ai_context, custom_extensions) + CatalogAttribute -> field (name, ANSI_SQL identifier expression, dimension.is_time, + description, ai_context, custom_extensions) + Term / termAssignments -> ai_context.synonyms + ATACCAMA custom_extension +""" + +from __future__ import annotations + +import json +from typing import Any + +from ataccama_osi.models import CatalogAttribute, CatalogItem, CatalogItemBundle, Term + +OSI_VERSION = "0.2.0.dev0" +VENDOR = "ATACCAMA" + +# Ataccama semantic data types that represent points in time. +TIME_DATA_TYPES = {"DATE", "DATETIME", "TIMESTAMP", "TIME"} + + +# --- helpers ------------------------------------------------------------- + + +def flatten_richtext(value: Any) -> str | None: + """Flatten an Ataccama description into plain text. + + Descriptions arrive either as a plain string or as Slate rich-text JSON, e.g. + ``[{"type": "paragraph", "children": [{"text": "..."}]}]`` (sometimes as a JSON + string). Returns ``None`` when there is no usable text. + """ + if value is None: + return None + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return None + # Rich text is occasionally delivered as a JSON-encoded string. + if stripped[0] in "[{": + try: + return flatten_richtext(json.loads(stripped)) + except (ValueError, TypeError): + return stripped + return stripped + + def collect(node: Any, out: list[str]) -> None: + if isinstance(node, dict): + text = node.get("text") + if isinstance(text, str): + out.append(text) + for child in node.get("children", []) or []: + collect(child, out) + elif isinstance(node, list): + for child in node: + collect(child, out) + + parts: list[str] = [] + collect(value, parts) + text = "".join(parts).strip() + return text or None + + +def _unique_name(desired: str, used: set[str], *, fallback: str) -> str: + """Return a name unique within ``used`` (OSI requires unique dataset/field names).""" + base = desired.strip() or fallback + candidate = base + n = 2 + while candidate in used: + candidate = f"{base}_{n}" + n += 1 + used.add(candidate) + return candidate + + +def _sql_identifier(name: str) -> str: + """Quote a column name as an ANSI SQL identifier (names may contain spaces/punctuation).""" + escaped = name.replace('"', '""') + return f'"{escaped}"' + + +def build_source(item: CatalogItem) -> str: + """Best-effort physical source reference. + + The Catalog API does not expose ``database.schema.table``; it only gives a + folder-name hierarchy (``locations``, leaf-first) plus the item name. We build a + dotted namespace from the reversed locations followed by the item name. The + authoritative connection/source URNs are preserved in custom_extensions. + """ + parts = [loc.name for loc in reversed(item.locations) if loc.name] + parts.append(item.name) + return ".".join(parts) if parts else item.name + + +def _terms_ai_context(term_urns: list[str], terms: dict[str, Term]) -> dict[str, Any] | None: + """Build an ai_context object from assigned business terms.""" + synonyms: list[str] = [] + instructions: list[str] = [] + for urn in term_urns: + term = terms.get(urn) + if not term: + continue + if term.name and term.name not in synonyms: + synonyms.append(term.name) + desc = flatten_richtext(term.description) + if desc: + instructions.append(f"{term.name}: {desc}" if term.name else desc) + ctx: dict[str, Any] = {} + if instructions: + ctx["instructions"] = " ".join(instructions) + if synonyms: + ctx["synonyms"] = synonyms + return ctx or None + + +def _ataccama_extension(data: dict[str, Any]) -> dict[str, str]: + """Wrap Ataccama-specific metadata as an OSI custom_extension entry.""" + return {"vendor_name": VENDOR, "data": json.dumps(data, sort_keys=True)} + + +# --- attribute -> field -------------------------------------------------- + + +def attribute_to_field(attr: CatalogAttribute, terms: dict[str, Term], used: set[str]) -> dict[str, Any]: + name = _unique_name(attr.name, used, fallback="column") + field_obj: dict[str, Any] = { + "name": name, + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": _sql_identifier(attr.name or name)}]}, + } + + if attr.data_type and attr.data_type.upper() in TIME_DATA_TYPES: + field_obj["dimension"] = {"is_time": True} + + description = flatten_richtext(attr.description) or flatten_richtext(attr.comment) + if description: + field_obj["description"] = description + + ai_context = _terms_ai_context([ta.term_urn for ta in attr.term_assignments], terms) + if ai_context: + field_obj["ai_context"] = ai_context + + # Preserve source-system typing and the attribute URN for round-tripping. + ext_data: dict[str, Any] = {"attribute_urn": attr.urn} + if attr.data_type: + ext_data["data_type"] = attr.data_type + if attr.column_type: + ext_data["column_type"] = attr.column_type + if attr.term_assignments: + ext_data["term_urns"] = [ta.term_urn for ta in attr.term_assignments] + field_obj["custom_extensions"] = [_ataccama_extension(ext_data)] + + return field_obj + + +# --- catalog item -> dataset --------------------------------------------- + + +def bundle_to_dataset(bundle: CatalogItemBundle, used_dataset_names: set[str]) -> dict[str, Any]: + item = bundle.item + name = _unique_name(item.name, used_dataset_names, fallback="dataset") + + dataset: dict[str, Any] = {"name": name, "source": build_source(item)} + + description = flatten_richtext(item.description) + if description: + dataset["description"] = description + + ai_context = _terms_ai_context([ta.term_urn for ta in item.term_assignments], bundle.terms) + if ai_context: + dataset["ai_context"] = ai_context + + used_field_names: set[str] = set() + fields = [attribute_to_field(attr, bundle.terms, used_field_names) for attr in bundle.attributes] + if fields: + dataset["fields"] = fields + + # Preserve everything with no OSI-core home so the model round-trips. + ext_data: dict[str, Any] = {"catalog_item_urn": item.urn} + if item.connection_urn: + ext_data["connection_urn"] = item.connection_urn + if item.source_urn: + ext_data["source_urn"] = item.source_urn + if item.origin_path: + ext_data["origin_path"] = item.origin_path + if item.locations: + ext_data["locations"] = [loc.name for loc in item.locations] + if item.stewardship_group_urn: + ext_data["stewardship_group_urn"] = item.stewardship_group_urn + if item.primary_dq_monitor_urn: + ext_data["primary_dq_monitor_urn"] = item.primary_dq_monitor_urn + if item.term_assignments: + ext_data["term_urns"] = [ta.term_urn for ta in item.term_assignments] + if item.aliases: + ext_data["aliases"] = item.aliases + dataset["custom_extensions"] = [_ataccama_extension(ext_data)] + + return dataset + + +# --- top level ----------------------------------------------------------- + + +def ataccama_to_osi( + bundles: list[CatalogItemBundle], + model_name: str = "ataccama_model", + model_description: str | None = None, + tenant: str | None = None, +) -> dict[str, Any]: + """Convert a list of catalog-item bundles into an OSI document dict (ready for YAML).""" + used_dataset_names: set[str] = set() + datasets = [bundle_to_dataset(b, used_dataset_names) for b in bundles] + + semantic_model: dict[str, Any] = {"name": model_name, "datasets": datasets} + if model_description: + semantic_model["description"] = model_description + + model_ext: dict[str, Any] = {"source": "ataccama-one-catalog"} + if tenant: + model_ext["tenant"] = tenant + semantic_model["custom_extensions"] = [_ataccama_extension(model_ext)] + + return {"version": OSI_VERSION, "semantic_model": [semantic_model]} diff --git a/converters/ataccama/src/ataccama_osi/cli.py b/converters/ataccama/src/ataccama_osi/cli.py new file mode 100644 index 0000000..57c805d --- /dev/null +++ b/converters/ataccama/src/ataccama_osi/cli.py @@ -0,0 +1,112 @@ +"""CLI: import selected Ataccama catalog items into an OSI semantic model. + +Scope is defined by explicit catalog-item URNs (``--urn`` repeated, or ``--urns-file``). + +Configuration is read from environment variables (optionally loaded from an env file +via ``--env-file``): + + ATACCAMA_BASE_URL API root, e.g. https:///api + ATACCAMA_TOKEN_URL OAuth2 token endpoint (Keycloak openid-connect/token) + ATACCAMA_CLIENT_ID service-account client id + ATACCAMA_CLIENT_SECRET service-account client secret + +Example: + ataccama-to-osi --env-file .ataccama.env \\ + --urn urn:ata:tenant:catalog:catalog-item:... \\ + --model-name my_model --output my_model.osi.yaml +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +import yaml + +from ataccama_osi.ataccama_to_osi import ataccama_to_osi +from ataccama_osi.client import AtaccamaClient + +REQUIRED_VARS = ("ATACCAMA_BASE_URL", "ATACCAMA_TOKEN_URL", "ATACCAMA_CLIENT_ID", "ATACCAMA_CLIENT_SECRET") + + +def _load_env_file(path: Path) -> None: + """Load simple KEY=VALUE lines into os.environ (does not overwrite existing vars).""" + for raw in path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + os.environ.setdefault(key.strip(), value.strip()) + + +def _tenant_from_urn(urn: str) -> str | None: + # urn:ata:::: + parts = urn.split(":") + return parts[2] if len(parts) > 2 else None + + +def _collect_urns(args: argparse.Namespace) -> list[str]: + urns: list[str] = list(args.urn or []) + if args.urns_file: + for raw in Path(args.urns_file).read_text().splitlines(): + line = raw.strip() + if line and not line.startswith("#"): + urns.append(line) + # de-duplicate, preserve order + seen: set[str] = set() + return [u for u in urns if not (u in seen or seen.add(u))] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Import Ataccama ONE catalog items into an OSI semantic model.") + parser.add_argument("--urn", action="append", help="Catalog-item URN to convert (repeatable).") + parser.add_argument("--urns-file", help="File with one catalog-item URN per line.") + parser.add_argument("--model-name", default="ataccama_model", help="OSI semantic model name.") + parser.add_argument("--model-description", default=None, help="OSI semantic model description.") + parser.add_argument("--output", "-o", default="-", help="Output YAML path ('-' for stdout).") + parser.add_argument("--env-file", help="Optional KEY=VALUE file with Ataccama connection config.") + args = parser.parse_args(argv) + + if args.env_file: + _load_env_file(Path(args.env_file)) + + urns = _collect_urns(args) + if not urns: + parser.error("provide at least one catalog-item URN via --urn or --urns-file") + + missing = [v for v in REQUIRED_VARS if not os.environ.get(v)] + if missing: + parser.error(f"missing required configuration: {', '.join(missing)}") + + client = AtaccamaClient( + base_url=os.environ["ATACCAMA_BASE_URL"], + token_url=os.environ["ATACCAMA_TOKEN_URL"], + client_id=os.environ["ATACCAMA_CLIENT_ID"], + client_secret=os.environ["ATACCAMA_CLIENT_SECRET"], + ) + + bundles = [] + for urn in urns: + print(f"Fetching {urn} ...", file=sys.stderr) + bundles.append(client.fetch_bundle(urn)) + + document = ataccama_to_osi( + bundles, + model_name=args.model_name, + model_description=args.model_description, + tenant=_tenant_from_urn(urns[0]), + ) + + text = yaml.dump(document, default_flow_style=False, sort_keys=False, allow_unicode=True) + if args.output == "-": + sys.stdout.write(text) + else: + Path(args.output).write_text(text) + print(f"Wrote {args.output}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/converters/ataccama/src/ataccama_osi/client.py b/converters/ataccama/src/ataccama_osi/client.py new file mode 100644 index 0000000..644dc66 --- /dev/null +++ b/converters/ataccama/src/ataccama_osi/client.py @@ -0,0 +1,126 @@ +"""Live REST client for the Ataccama ONE public API. + +Handles OAuth2 client-credentials auth (tokens are short-lived, ~5 min, so we mint +and refresh them automatically) and cursor-based pagination (``after``/``size``). +""" + +from __future__ import annotations + +import time +from typing import Any, Iterator + +import requests + +from ataccama_osi.models import CatalogAttribute, CatalogItem, CatalogItemBundle, Term + +DEFAULT_PAGE_SIZE = 200 +# Refresh the token this many seconds before it actually expires. +TOKEN_EXPIRY_SKEW_S = 30 + + +class AtaccamaClient: + """Minimal client for the endpoints the OSI importer needs.""" + + def __init__( + self, + base_url: str, + token_url: str, + client_id: str, + client_secret: str, + *, + timeout: float = 30.0, + session: requests.Session | None = None, + ) -> None: + # base_url is the API root, e.g. https:///api + self.base_url = base_url.rstrip("/") + self.token_url = token_url + self.client_id = client_id + self.client_secret = client_secret + self.timeout = timeout + self._session = session or requests.Session() + self._token: str | None = None + self._token_expiry: float = 0.0 + + # --- auth --- + + def _access_token(self) -> str: + if self._token and time.time() < self._token_expiry - TOKEN_EXPIRY_SKEW_S: + return self._token + resp = self._session.post( + self.token_url, + data={ + "grant_type": "client_credentials", + "client_id": self.client_id, + "client_secret": self.client_secret, + }, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=self.timeout, + ) + resp.raise_for_status() + payload = resp.json() + self._token = payload["access_token"] + self._token_expiry = time.time() + float(payload.get("expires_in", 300)) + return self._token + + # --- low-level HTTP --- + + def _get(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + resp = self._session.get( + f"{self.base_url}{path}", + params=params, + headers={"Authorization": f"Bearer {self._access_token()}"}, + timeout=self.timeout, + ) + resp.raise_for_status() + return resp.json() + + def _paged(self, path: str, params: dict[str, Any] | None = None) -> Iterator[dict[str, Any]]: + """Iterate every record across pages, following the ``meta.next`` cursor.""" + params = dict(params or {}) + params.setdefault("size", DEFAULT_PAGE_SIZE) + after: str | None = None + while True: + if after is not None: + params["after"] = after + body = self._get(path, params) + yield from body.get("data", []) + after = (body.get("meta") or {}).get("next") + if not after: + break + + # --- catalog endpoints --- + + def get_catalog_item(self, urn: str) -> CatalogItem: + return CatalogItem.from_dict(self._get(f"/catalog/v1/catalog-items/{urn}")) + + def get_attributes(self, catalog_item_urn: str) -> list[CatalogAttribute]: + return [ + CatalogAttribute.from_dict(a) + for a in self._paged("/catalog/v1/attributes", {"catalogItemUrn": catalog_item_urn}) + ] + + def get_term(self, urn: str) -> Term: + return Term.from_dict(self._get(f"/catalog/v1/terms/{urn}")) + + # --- composite --- + + def fetch_bundle(self, catalog_item_urn: str) -> CatalogItemBundle: + """Fetch a catalog item, its attributes, and every term it (or its columns) reference.""" + item = self.get_catalog_item(catalog_item_urn) + attributes = self.get_attributes(catalog_item_urn) + + term_urns: set[str] = {ta.term_urn for ta in item.term_assignments} + for attr in attributes: + term_urns.update(ta.term_urn for ta in attr.term_assignments) + + terms: dict[str, Term] = {} + for urn in sorted(term_urns): + if not urn: + continue + try: + terms[urn] = self.get_term(urn) + except requests.HTTPError: + # A referenced term may be inaccessible; skip it rather than fail the whole run. + continue + + return CatalogItemBundle(item=item, attributes=attributes, terms=terms) diff --git a/converters/ataccama/src/ataccama_osi/models.py b/converters/ataccama/src/ataccama_osi/models.py new file mode 100644 index 0000000..0292f8d --- /dev/null +++ b/converters/ataccama/src/ataccama_osi/models.py @@ -0,0 +1,191 @@ +"""Data models for Ataccama ONE catalog metadata and the pieces of OSI we emit. + +Ataccama Catalog API structure (public REST API, /catalog/v1): + CatalogItem + ├── urn, name, description (Slate rich-text JSON), originPath + ├── connection {urn} + ├── source {urn} + ├── locations[] {name, description} # folder-name hierarchy (leaf-first) + ├── termAssignments[] {termUrn} + ├── effectiveStewardship {groupUrn} + ├── primaryDqMonitor {urn} + └── aliases[] {type, value} + CatalogAttribute + ├── urn, catalogItemUrn, name + ├── description (Slate rich-text JSON) + ├── dataType (STRING, LONG, INTEGER, FLOAT, DOUBLE, BOOLEAN, DATE, DATETIME, ...) + ├── columnType (raw source-system type, e.g. VARCHAR, R8) + ├── comment + └── termAssignments[] {termUrn} + Term + ├── urn, name, type (businessTerm, securityTerm) + └── description (Slate rich-text JSON) + +OSI semantic model structure (what we produce): + version, semantic_model[] + ├── name, description, ai_context, custom_extensions[] + └── datasets[] {name, source, description, ai_context, fields[], custom_extensions[]} + └── fields[] {name, expression{dialects[]}, dimension{is_time}, description, + ai_context, custom_extensions[]} +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +# --- Ataccama Catalog types --- + + +@dataclass +class TermAssignment: + term_urn: str + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> TermAssignment: + return cls(term_urn=d.get("termUrn", "")) + + +@dataclass +class CatalogLocation: + name: str + description: str | None = None + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> CatalogLocation: + return cls(name=d.get("name", ""), description=d.get("description")) + + +@dataclass +class CatalogAttribute: + urn: str + name: str + catalog_item_urn: str = "" + description: Any = None + data_type: str | None = None + column_type: str | None = None + comment: str | None = None + term_assignments: list[TermAssignment] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> CatalogAttribute: + return cls( + urn=d.get("urn", ""), + name=d.get("name", ""), + catalog_item_urn=d.get("catalogItemUrn", ""), + description=d.get("description"), + data_type=d.get("dataType"), + column_type=d.get("columnType"), + comment=d.get("comment"), + term_assignments=[TermAssignment.from_dict(t) for t in d.get("termAssignments", [])], + ) + + +@dataclass +class CatalogItem: + urn: str + name: str + description: Any = None + origin_path: str | None = None + connection_urn: str | None = None + source_urn: str | None = None + locations: list[CatalogLocation] = field(default_factory=list) + term_assignments: list[TermAssignment] = field(default_factory=list) + stewardship_group_urn: str | None = None + primary_dq_monitor_urn: str | None = None + aliases: list[dict[str, Any]] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> CatalogItem: + connection = d.get("connection") or {} + source = d.get("source") or {} + stewardship = d.get("effectiveStewardship") or {} + monitor = d.get("primaryDqMonitor") or {} + return cls( + urn=d.get("urn", ""), + name=d.get("name", ""), + description=d.get("description"), + origin_path=d.get("originPath"), + connection_urn=connection.get("urn"), + source_urn=source.get("urn"), + locations=[CatalogLocation.from_dict(loc) for loc in d.get("locations", [])], + term_assignments=[TermAssignment.from_dict(t) for t in d.get("termAssignments", [])], + stewardship_group_urn=stewardship.get("groupUrn"), + primary_dq_monitor_urn=monitor.get("urn"), + aliases=list(d.get("aliases", [])), + ) + + +@dataclass +class Term: + urn: str + name: str + type: str | None = None + description: Any = None + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> Term: + return cls( + urn=d.get("urn", ""), + name=d.get("name", ""), + type=d.get("type"), + description=d.get("description"), + ) + + +@dataclass +class CatalogItemBundle: + """A catalog item together with everything needed to convert it to a dataset.""" + + item: CatalogItem + attributes: list[CatalogAttribute] = field(default_factory=list) + terms: dict[str, Term] = field(default_factory=dict) + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> CatalogItemBundle: + return cls( + item=CatalogItem.from_dict(d["item"]), + attributes=[CatalogAttribute.from_dict(a) for a in d.get("attributes", [])], + terms={urn: Term.from_dict(t) for urn, t in d.get("terms", {}).items()}, + ) + + def to_dict(self) -> dict[str, Any]: + """Serialize back to the raw-JSON shape (used for recording test fixtures).""" + return { + "item": _dataclass_to_raw_item(self.item), + "attributes": [_dataclass_to_raw_attr(a) for a in self.attributes], + "terms": {urn: _dataclass_to_raw_term(t) for urn, t in self.terms.items()}, + } + + +def _dataclass_to_raw_item(item: CatalogItem) -> dict[str, Any]: + return { + "urn": item.urn, + "name": item.name, + "description": item.description, + "originPath": item.origin_path, + "connection": {"urn": item.connection_urn} if item.connection_urn else None, + "source": {"urn": item.source_urn} if item.source_urn else None, + "locations": [{"name": loc.name, "description": loc.description} for loc in item.locations], + "termAssignments": [{"termUrn": t.term_urn} for t in item.term_assignments], + "effectiveStewardship": ({"groupUrn": item.stewardship_group_urn} if item.stewardship_group_urn else None), + "primaryDqMonitor": ({"urn": item.primary_dq_monitor_urn} if item.primary_dq_monitor_urn else None), + "aliases": item.aliases, + } + + +def _dataclass_to_raw_attr(attr: CatalogAttribute) -> dict[str, Any]: + return { + "urn": attr.urn, + "catalogItemUrn": attr.catalog_item_urn, + "name": attr.name, + "description": attr.description, + "dataType": attr.data_type, + "columnType": attr.column_type, + "comment": attr.comment, + "termAssignments": [{"termUrn": t.term_urn} for t in attr.term_assignments], + } + + +def _dataclass_to_raw_term(term: Term) -> dict[str, Any]: + return {"urn": term.urn, "name": term.name, "type": term.type, "description": term.description} diff --git a/converters/ataccama/tests/__init__.py b/converters/ataccama/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/converters/ataccama/tests/fixtures/ataccama_bundles.json b/converters/ataccama/tests/fixtures/ataccama_bundles.json new file mode 100644 index 0000000..4b9f52d --- /dev/null +++ b/converters/ataccama/tests/fixtures/ataccama_bundles.json @@ -0,0 +1,334 @@ +[ + { + "item": { + "urn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Financial KPIs", + "description": null, + "originPath": "Executive Analytics (demo data)/Executive KPIs Accelerator/11111111-1111-1111-1111-000000000001", + "connection": { + "urn": "urn:ata:example-tenant:processing-center:connection:00000000-0000-0000-0000-000000000001" + }, + "source": { + "urn": "urn:ata:example-tenant:catalog:source:00000000-0000-0000-0000-000000000001" + }, + "locations": [ + { + "name": "Executive KPIs Accelerator", + "description": null + }, + { + "name": "Executive Analytics (demo data)", + "description": null + }, + { + "name": "Projects", + "description": null + } + ], + "termAssignments": [], + "effectiveStewardship": null, + "primaryDqMonitor": { + "urn": "urn:ata:example-tenant:data-quality:dq-monitor:00000000-0000-0000-0000-000000000001" + }, + "aliases": [] + }, + "attributes": [ + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000001", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Prior Yr. SM", + "description": null, + "dataType": "STRING", + "columnType": "", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000002", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Actual v. Prior %", + "description": null, + "dataType": "STRING", + "columnType": "", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000003", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "KPI Filter - Allocated Expenses", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000004", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "KPI Filter - Expense Driver", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000005", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "KPI Filter - Resources", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000006", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Arrow", + "description": null, + "dataType": "STRING", + "columnType": "", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000007", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Year", + "description": null, + "dataType": "FLOAT", + "columnType": "R8", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000008", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Time Filter", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000009", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Value", + "description": null, + "dataType": "FLOAT", + "columnType": "R8", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000010", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Cat Filter", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000011", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Actual v. Prior", + "description": null, + "dataType": "STRING", + "columnType": "", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000012", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "vs. Budget", + "description": null, + "dataType": "FLOAT", + "columnType": "R8", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000013", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "KPI Filter - Business Initiatives", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000014", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "KPI Filter - Financials", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000015", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Category", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000016", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Display", + "description": null, + "dataType": "STRING", + "columnType": "", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000017", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Extract", + "description": null, + "dataType": "STRING", + "columnType": "", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000018", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "X", + "description": null, + "dataType": "STRING", + "columnType": "", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000019", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Name", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000020", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Actual", + "description": null, + "dataType": "STRING", + "columnType": "", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000021", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Y", + "description": null, + "dataType": "STRING", + "columnType": "", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000022", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "1", + "description": null, + "dataType": "STRING", + "columnType": "", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000023", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "KPI", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000024", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "Month", + "description": null, + "dataType": "DATE", + "columnType": "DBDATE", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000025", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "KPI Filter - Revenue Drivers", + "description": null, + "dataType": "STRING", + "columnType": "STR", + "comment": null, + "termAssignments": [] + } + ], + "terms": {} + }, + { + "item": { + "urn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000002", + "name": "Analysis Year to Date Summary", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"This report provides a comprehensive overview of marketing performance metrics for the current year, offering valuable insights into the effectiveness of various marketing strategies and campaigns. It serves as a crucial tool for marketing teams to assess progress, identify trends, and make informed decisions to optimize future initiatives. By consolidating key performance indicators and data points, it enables stakeholders to quickly evaluate the impact of marketing efforts on business objectives, ensuring alignment with overall company goals. This summary is instrumental in facilitating strategic planning and resource allocation within the marketing domain.\"}]}]", + "originPath": "Operational Reporting Library (demo data)/11111111-1111-1111-1111-000000000002", + "connection": { + "urn": "urn:ata:example-tenant:processing-center:connection:00000000-0000-0000-0000-000000000002" + }, + "source": { + "urn": "urn:ata:example-tenant:catalog:source:00000000-0000-0000-0000-000000000002" + }, + "locations": [ + { + "name": "Operational Reporting Library (demo data)", + "description": null + }, + { + "name": "Workspaces", + "description": null + } + ], + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001" + } + ], + "effectiveStewardship": { + "groupUrn": "urn:ata:example-tenant:iam:group:00000000-0000-0000-0000-000000000001" + }, + "primaryDqMonitor": { + "urn": "urn:ata:example-tenant:data-quality:dq-monitor:00000000-0000-0000-0000-000000000002" + }, + "aliases": [] + }, + "attributes": [], + "terms": { + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001", + "name": "Marketing", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"Marketing refers to activities a company undertakes to promote the buying or selling a product or service. Marketing includes advertising, selling, and delivering products to consumers or other businesses. In this instance, Marketing is a Data Domain comprising several business terms under itself.\"}]}]" + } + } + } +] \ No newline at end of file diff --git a/converters/ataccama/tests/test_ataccama_to_osi.py b/converters/ataccama/tests/test_ataccama_to_osi.py new file mode 100644 index 0000000..06e3cc0 --- /dev/null +++ b/converters/ataccama/tests/test_ataccama_to_osi.py @@ -0,0 +1,133 @@ +"""Tests for the Ataccama -> OSI conversion (offline, using a recorded fixture).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml +from jsonschema import Draft202012Validator + +from ataccama_osi.ataccama_to_osi import ( + OSI_VERSION, + attribute_to_field, + ataccama_to_osi, + build_source, + flatten_richtext, +) +from ataccama_osi.models import CatalogAttribute, CatalogItem, CatalogItemBundle, CatalogLocation + +FIXTURE = Path(__file__).parent / "fixtures" / "ataccama_bundles.json" +SCHEMA = Path(__file__).parents[3] / "core-spec" / "osi-schema.json" + + +@pytest.fixture +def bundles() -> list[CatalogItemBundle]: + raw = json.loads(FIXTURE.read_text()) + return [CatalogItemBundle.from_dict(b) for b in raw] + + +@pytest.fixture +def document(bundles: list[CatalogItemBundle]) -> dict: + return ataccama_to_osi(bundles, model_name="test_model", tenant="example-tenant") + + +def _find_dataset(document: dict, name: str) -> dict: + return next(d for d in document["semantic_model"][0]["datasets"] if d["name"] == name) + + +def test_document_is_schema_valid(document: dict) -> None: + schema = json.loads(SCHEMA.read_text()) + errors = list(Draft202012Validator(schema).iter_errors(document)) + assert not errors, [e.message for e in errors] + assert document["version"] == OSI_VERSION + + +def test_model_structure(document: dict) -> None: + model = document["semantic_model"][0] + assert model["name"] == "test_model" + assert len(model["datasets"]) == 2 + # tenant preserved at model level + assert any(e["vendor_name"] == "ATACCAMA" for e in model["custom_extensions"]) + + +def test_attribute_becomes_quoted_ansi_field(document: dict) -> None: + ds = _find_dataset(document, "Financial KPIs") + assert len(ds["fields"]) == 25 + field = ds["fields"][0] + assert field["name"] == "Prior Yr. SM" + dialects = field["expression"]["dialects"] + assert dialects == [{"dialect": "ANSI_SQL", "expression": '"Prior Yr. SM"'}] + # attribute urn + typing preserved for round-tripping + ext = json.loads(field["custom_extensions"][0]["data"]) + assert ext["data_type"] == "STRING" + assert ext["attribute_urn"].startswith("urn:ata:") + + +def test_richtext_description_is_flattened(document: dict) -> None: + ds = _find_dataset(document, "Analysis Year to Date Summary") + assert ds["description"].startswith("This report provides") + # no Slate JSON leaked through + assert "{" not in ds["description"] and "children" not in ds["description"] + + +def test_assigned_term_becomes_ai_context(document: dict) -> None: + ds = _find_dataset(document, "Analysis Year to Date Summary") + assert ds["ai_context"]["synonyms"] == ["Marketing"] + assert "Marketing" in ds["ai_context"]["instructions"] + + +def test_governance_metadata_preserved(document: dict) -> None: + ds = _find_dataset(document, "Analysis Year to Date Summary") + ext = json.loads(ds["custom_extensions"][0]["data"]) + assert ext["catalog_item_urn"].startswith("urn:ata:") + assert "connection_urn" in ext and "stewardship_group_urn" in ext + + +# --- unit tests for helpers --- + + +@pytest.mark.parametrize( + "value,expected", + [ + (None, None), + ("", None), + ("plain text", "plain text"), + ('[{"type":"paragraph","children":[{"text":"hi"}]}]', "hi"), + ([{"type": "paragraph", "children": [{"text": "a "}, {"text": "b"}]}], "a b"), + ], +) +def test_flatten_richtext(value, expected) -> None: + assert flatten_richtext(value) == expected + + +def test_is_time_inferred_from_datatype() -> None: + attr = CatalogAttribute(urn="urn:ata:t:catalog:catalog-attribute:1", name="created_at", data_type="DATETIME") + field = attribute_to_field(attr, terms={}, used=set()) + assert field["dimension"] == {"is_time": True} + + +def test_non_time_datatype_has_no_dimension() -> None: + attr = CatalogAttribute(urn="urn:ata:t:catalog:catalog-attribute:1", name="amount", data_type="DOUBLE") + field = attribute_to_field(attr, terms={}, used=set()) + assert "dimension" not in field + + +def test_duplicate_dataset_names_are_disambiguated() -> None: + items = [ + CatalogItemBundle(item=CatalogItem(urn=f"urn:ata:t:catalog:catalog-item:{i}", name="Orders")) + for i in range(2) + ] + doc = ataccama_to_osi(items) + names = [d["name"] for d in doc["semantic_model"][0]["datasets"]] + assert names == ["Orders", "Orders_2"] + + +def test_build_source_uses_reversed_locations_then_name() -> None: + item = CatalogItem( + urn="urn:ata:t:catalog:catalog-item:1", + name="customers", + locations=[CatalogLocation(name="sales"), CatalogLocation(name="Workspaces")], + ) + assert build_source(item) == "Workspaces.sales.customers" From 141f80b927d392bbf7be9fd543528f1382342e7e Mon Sep 17 00:00:00 2001 From: Cara Baestlein Date: Thu, 2 Jul 2026 12:29:53 +0100 Subject: [PATCH 2/4] feat(converters): enrich Ataccama import with data-quality scores Fetch the latest DQ results (each item's primary monitor) and attach them to the ATACCAMA custom extensions: - dataset level: overall + per-dimension quality (passed/failed + derived pass_rate_pct) plus the DQ results link - field level: per-attribute quality DQ is fetched by default; `--no-dq` skips the calls. `pass_rate_pct` is derived from the raw passed/failed counts and is not presented as Ataccama's official score (the Data Trust Score/Index is not exposed by the public APIs). Rebuilds the test fixture around BANK_TRANSACTIONS + aggregation (both carry DQ results), with all tenant, host, URN, and personal identifiers sanitized to synthetic values. Co-Authored-By: Claude Opus 4.8 --- converters/ataccama/README.md | 31 +- .../src/ataccama_osi/ataccama_to_osi.py | 77 +- converters/ataccama/src/ataccama_osi/cli.py | 7 +- .../ataccama/src/ataccama_osi/client.py | 27 +- .../ataccama/src/ataccama_osi/models.py | 4 + .../tests/fixtures/ataccama_bundles.json | 3605 ++++++++++++++++- .../ataccama/tests/test_ataccama_to_osi.py | 132 +- 7 files changed, 3639 insertions(+), 244 deletions(-) diff --git a/converters/ataccama/README.md b/converters/ataccama/README.md index 0bf5c5a..1eff1fd 100644 --- a/converters/ataccama/README.md +++ b/converters/ataccama/README.md @@ -6,8 +6,9 @@ semantic model specification. Given one or more Ataccama **catalog items** (selected by URN), the converter produces a single OSI semantic model: each catalog item becomes a dataset, each catalog attribute -becomes a field, and business terms / governance metadata are carried across as -`ai_context` and `ATACCAMA` custom extensions. +becomes a field, business terms are carried across as `ai_context`, and +**data-quality scores** plus other governance metadata are attached as `ATACCAMA` +custom extensions. ## Scope & direction @@ -43,6 +44,9 @@ ataccama-to-osi --env-file .ataccama.env \ --model-name my_model \ --output my_model.osi.yaml +# data-quality results are fetched by default; use --no-dq to skip them +ataccama-to-osi --env-file .ataccama.env --urn --no-dq -o my_model.osi.yaml + # or supply a file of URNs (one per line) ataccama-to-osi --env-file .ataccama.env --urns-file items.txt -o my_model.osi.yaml ``` @@ -83,17 +87,36 @@ Tests run fully offline against a recorded catalog fixture | `CatalogAttribute.dataType` ∈ {DATE, DATETIME, TIMESTAMP, TIME} | `field.dimension.is_time = true` | | `CatalogAttribute.description` / `comment` | `field.description` | | attribute `Term`s | `field.ai_context` | -| URNs, `dataType`, `columnType`, connection/source/stewardship/DQ | `custom_extensions` (`vendor_name: ATACCAMA`) | +| DQ `overallQuality` + `dimensionResults` (latest processing) | dataset `custom_extensions` → `dq` (`passed`, `failed`, `pass_rate_pct`, `dimensions[]`, `results_link`) | +| DQ per-attribute `overallQuality` | field `custom_extensions` → `dq` (`passed`, `failed`, `pass_rate_pct`) | +| URNs, `dataType`, `columnType`, connection/source/stewardship/monitor | `custom_extensions` (`vendor_name: ATACCAMA`) | All Ataccama-specific metadata with no OSI-core home is preserved in `ATACCAMA` custom extensions (as a JSON string) so the model round-trips without data loss. +### Data quality + +Data-quality results (from the DQ API's latest processing on each item's primary +monitor) are fetched by default and attached to the `ATACCAMA` extension — overall +and per-dimension quality on the dataset, and per-column quality on each field. Pass +`--no-dq` to skip the DQ calls. + +Notes: +- `pass_rate_pct` is **derived** (`passed / (passed + failed)`); the raw `passed`/`failed` + counts are authoritative. Ataccama's own UI score may weight dimensions/rules + differently, so this is not presented as an official Ataccama "Data Quality score". +- Dimensions with no evaluated records in the processing are omitted. +- Items without a published monitor/results simply carry no `dq` block (best-effort). + ## Limitations The Ataccama Catalog API is a **catalog / governance / data-quality** surface, while OSI is an **analytics semantic model**. Some OSI constructs therefore have no source today: -- **Metrics** — Ataccama has no analytics metrics; none are emitted. +- **Metrics** — Ataccama has no analytics metrics; none are emitted. (Data-quality + scores are attached as `ATACCAMA` extensions, not as OSI metrics.) +- **Data Trust Score / Index** — not exposed by the public APIs, so it is not + emitted. Only rule-level DQ results (which the converter does carry) are available. - **Relationships / `primary_key` / `unique_keys`** — the Catalog API does not expose foreign or primary keys on catalog items, so these are omitted. - **`source` string** — the API exposes only a folder hierarchy (`locations`) and diff --git a/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py b/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py index 49e8132..09d4c1d 100644 --- a/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py +++ b/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py @@ -122,10 +122,77 @@ def _ataccama_extension(data: dict[str, Any]) -> dict[str, str]: return {"vendor_name": VENDOR, "data": json.dumps(data, sort_keys=True)} +# --- data quality --- + + +def _quality_summary(overall_quality: dict[str, Any] | None) -> dict[str, Any] | None: + """Turn an OverallQuality {passedCount, failedCount} into a compact summary. + + ``pass_rate_pct`` is DERIVED (passed / (passed + failed)); the raw counts are the + authoritative signal. Ataccama's own UI score may weight dimensions/rules differently. + """ + if not overall_quality: + return None + passed = overall_quality.get("passedCount") or 0 + failed = overall_quality.get("failedCount") or 0 + total = passed + failed + summary: dict[str, Any] = {"passed": passed, "failed": failed} + if total: + summary["pass_rate_pct"] = round(passed / total * 100, 1) + return summary + + +def _dataset_dq(dq_results: dict[str, Any] | None) -> dict[str, Any] | None: + """Build the dataset-level DQ block from a raw DqResults payload.""" + if not dq_results: + return None + dq = _quality_summary(dq_results.get("overallQuality")) or {} + + dimensions = [] + for dim in dq_results.get("dimensionResults", []) or []: + passed = dim.get("passedCount") or 0 + failed = dim.get("failedCount") or 0 + if passed + failed == 0: # dimension not evaluated in this processing + continue + entry = {"name": (dim.get("dimension") or {}).get("name"), "passed": passed, "failed": failed} + entry["pass_rate_pct"] = round(passed / (passed + failed) * 100, 1) + dimensions.append(entry) + if dimensions: + dq["dimensions"] = dimensions + + active_findings = sum(1 for f in dq_results.get("overallDqFindings", []) or [] if f.get("status") == "ACTIVE") + if active_findings: + dq["active_findings"] = active_findings + if dq_results.get("processingUrn"): + dq["processing_urn"] = dq_results["processingUrn"] + if dq_results.get("dqResultsLink"): + dq["results_link"] = dq_results["dqResultsLink"] + + return dq or None + + +def _attribute_dq_index(dq_results: dict[str, Any] | None) -> dict[str, dict[str, Any]]: + """Map attributeUrn -> quality summary from a raw DqResults payload.""" + index: dict[str, dict[str, Any]] = {} + if not dq_results: + return index + for attr_result in dq_results.get("dqAttributeResults", []) or []: + urn = attr_result.get("attributeUrn") + summary = _quality_summary(attr_result.get("overallQuality")) + if urn and summary: + index[urn] = summary + return index + + # --- attribute -> field -------------------------------------------------- -def attribute_to_field(attr: CatalogAttribute, terms: dict[str, Term], used: set[str]) -> dict[str, Any]: +def attribute_to_field( + attr: CatalogAttribute, + terms: dict[str, Term], + used: set[str], + dq_by_attr: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any]: name = _unique_name(attr.name, used, fallback="column") field_obj: dict[str, Any] = { "name": name, @@ -151,6 +218,8 @@ def attribute_to_field(attr: CatalogAttribute, terms: dict[str, Term], used: set ext_data["column_type"] = attr.column_type if attr.term_assignments: ext_data["term_urns"] = [ta.term_urn for ta in attr.term_assignments] + if dq_by_attr and attr.urn in dq_by_attr: + ext_data["dq"] = dq_by_attr[attr.urn] field_obj["custom_extensions"] = [_ataccama_extension(ext_data)] return field_obj @@ -173,8 +242,9 @@ def bundle_to_dataset(bundle: CatalogItemBundle, used_dataset_names: set[str]) - if ai_context: dataset["ai_context"] = ai_context + dq_by_attr = _attribute_dq_index(bundle.dq_results) used_field_names: set[str] = set() - fields = [attribute_to_field(attr, bundle.terms, used_field_names) for attr in bundle.attributes] + fields = [attribute_to_field(attr, bundle.terms, used_field_names, dq_by_attr) for attr in bundle.attributes] if fields: dataset["fields"] = fields @@ -196,6 +266,9 @@ def bundle_to_dataset(bundle: CatalogItemBundle, used_dataset_names: set[str]) - ext_data["term_urns"] = [ta.term_urn for ta in item.term_assignments] if item.aliases: ext_data["aliases"] = item.aliases + dataset_dq = _dataset_dq(bundle.dq_results) + if dataset_dq: + ext_data["dq"] = dataset_dq dataset["custom_extensions"] = [_ataccama_extension(ext_data)] return dataset diff --git a/converters/ataccama/src/ataccama_osi/cli.py b/converters/ataccama/src/ataccama_osi/cli.py index 57c805d..88492da 100644 --- a/converters/ataccama/src/ataccama_osi/cli.py +++ b/converters/ataccama/src/ataccama_osi/cli.py @@ -67,6 +67,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--model-description", default=None, help="OSI semantic model description.") parser.add_argument("--output", "-o", default="-", help="Output YAML path ('-' for stdout).") parser.add_argument("--env-file", help="Optional KEY=VALUE file with Ataccama connection config.") + parser.add_argument( + "--no-dq", + action="store_true", + help="Skip fetching data-quality results (DQ enrichment is included by default).", + ) args = parser.parse_args(argv) if args.env_file: @@ -90,7 +95,7 @@ def main(argv: list[str] | None = None) -> int: bundles = [] for urn in urns: print(f"Fetching {urn} ...", file=sys.stderr) - bundles.append(client.fetch_bundle(urn)) + bundles.append(client.fetch_bundle(urn, with_dq=not args.no_dq)) document = ataccama_to_osi( bundles, diff --git a/converters/ataccama/src/ataccama_osi/client.py b/converters/ataccama/src/ataccama_osi/client.py index 644dc66..9a83f7e 100644 --- a/converters/ataccama/src/ataccama_osi/client.py +++ b/converters/ataccama/src/ataccama_osi/client.py @@ -102,10 +102,29 @@ def get_attributes(self, catalog_item_urn: str) -> list[CatalogAttribute]: def get_term(self, urn: str) -> Term: return Term.from_dict(self._get(f"/catalog/v1/terms/{urn}")) + # --- data-quality endpoint --- + + def get_dq_results( + self, catalog_item_urn: str, monitor: str = "primary", processing: str = "latest" + ) -> dict | None: + """Latest DQ results for a catalog item's primary monitor. + + Returns the raw ``DqResults`` payload (overall + per-dimension + per-attribute + quality), or ``None`` if the item has no monitor / published results. + """ + path = ( + f"/data-quality/v1/catalog-items/{catalog_item_urn}" + f"/dq-monitors/{monitor}/processings/{processing}/dq-results" + ) + try: + return self._get(path) + except requests.HTTPError: + return None + # --- composite --- - def fetch_bundle(self, catalog_item_urn: str) -> CatalogItemBundle: - """Fetch a catalog item, its attributes, and every term it (or its columns) reference.""" + def fetch_bundle(self, catalog_item_urn: str, *, with_dq: bool = True) -> CatalogItemBundle: + """Fetch a catalog item, its attributes, referenced terms, and (by default) DQ results.""" item = self.get_catalog_item(catalog_item_urn) attributes = self.get_attributes(catalog_item_urn) @@ -123,4 +142,6 @@ def fetch_bundle(self, catalog_item_urn: str) -> CatalogItemBundle: # A referenced term may be inaccessible; skip it rather than fail the whole run. continue - return CatalogItemBundle(item=item, attributes=attributes, terms=terms) + dq_results = self.get_dq_results(catalog_item_urn) if with_dq else None + + return CatalogItemBundle(item=item, attributes=attributes, terms=terms, dq_results=dq_results) diff --git a/converters/ataccama/src/ataccama_osi/models.py b/converters/ataccama/src/ataccama_osi/models.py index 0292f8d..4085b08 100644 --- a/converters/ataccama/src/ataccama_osi/models.py +++ b/converters/ataccama/src/ataccama_osi/models.py @@ -140,6 +140,8 @@ class CatalogItemBundle: item: CatalogItem attributes: list[CatalogAttribute] = field(default_factory=list) terms: dict[str, Term] = field(default_factory=dict) + # Raw DqResults payload (overall + per-dimension + per-attribute quality), or None. + dq_results: dict[str, Any] | None = None @classmethod def from_dict(cls, d: dict[str, Any]) -> CatalogItemBundle: @@ -147,6 +149,7 @@ def from_dict(cls, d: dict[str, Any]) -> CatalogItemBundle: item=CatalogItem.from_dict(d["item"]), attributes=[CatalogAttribute.from_dict(a) for a in d.get("attributes", [])], terms={urn: Term.from_dict(t) for urn, t in d.get("terms", {}).items()}, + dq_results=d.get("dq_results"), ) def to_dict(self) -> dict[str, Any]: @@ -155,6 +158,7 @@ def to_dict(self) -> dict[str, Any]: "item": _dataclass_to_raw_item(self.item), "attributes": [_dataclass_to_raw_attr(a) for a in self.attributes], "terms": {urn: _dataclass_to_raw_term(t) for urn, t in self.terms.items()}, + "dq_results": self.dq_results, } diff --git a/converters/ataccama/tests/fixtures/ataccama_bundles.json b/converters/ataccama/tests/fixtures/ataccama_bundles.json index 4b9f52d..e418716 100644 --- a/converters/ataccama/tests/fixtures/ataccama_bundles.json +++ b/converters/ataccama/tests/fixtures/ataccama_bundles.json @@ -2,9 +2,9 @@ { "item": { "urn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Financial KPIs", - "description": null, - "originPath": "Executive Analytics (demo data)/Executive KPIs Accelerator/11111111-1111-1111-1111-000000000001", + "name": "BANK_TRANSACTIONS", + "description": "[{\"type\": \"paragraph\", \"children\": [{\"text\": \"\"}, {\"type\": \"a\", \"url\": \"https://ataccama-partner.atlan.com/assets/11111111-1111-1111-1111-000000000001/overview\", \"children\": [{\"text\": \"View in Atlan\"}]}, {\"text\": \"\"}]}]", + "originPath": "CLOUD_DEMO/BANK_TRANSACTIONS", "connection": { "urn": "urn:ata:example-tenant:processing-center:connection:00000000-0000-0000-0000-000000000001" }, @@ -13,20 +13,25 @@ }, "locations": [ { - "name": "Executive KPIs Accelerator", + "name": "CLOUD_DEMO", "description": null }, { - "name": "Executive Analytics (demo data)", + "name": "DEMO_ENV", "description": null + } + ], + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001" }, { - "name": "Projects", - "description": null + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000002" } ], - "termAssignments": [], - "effectiveStewardship": null, + "effectiveStewardship": { + "groupUrn": "urn:ata:example-tenant:iam:group:00000000-0000-0000-0000-000000000001" + }, "primaryDqMonitor": { "urn": "urn:ata:example-tenant:data-quality:dq-monitor:00000000-0000-0000-0000-000000000001" }, @@ -36,299 +41,3507 @@ { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000001", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Prior Yr. SM", + "name": "TRANSACTION_ID", "description": null, "dataType": "STRING", - "columnType": "", - "comment": null, + "columnType": "VARCHAR", + "comment": "", "termAssignments": [] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000002", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Actual v. Prior %", + "name": "CUSTOMER_ID", "description": null, "dataType": "STRING", - "columnType": "", - "comment": null, - "termAssignments": [] + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000003" + }, + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000004" + }, + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000005" + } + ] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000003", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "KPI Filter - Allocated Expenses", + "name": "CUSTOMER_NAME", "description": null, "dataType": "STRING", - "columnType": "STR", - "comment": null, - "termAssignments": [] + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000006" + }, + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000003" + } + ] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000004", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "KPI Filter - Expense Driver", + "name": "EMAIL", "description": null, "dataType": "STRING", - "columnType": "STR", - "comment": null, - "termAssignments": [] + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000007" + } + ] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000005", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "KPI Filter - Resources", + "name": "BIRTH_DATE", "description": null, - "dataType": "STRING", - "columnType": "STR", - "comment": null, - "termAssignments": [] + "dataType": "DATE", + "columnType": "DATE", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000008" + } + ] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000006", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Arrow", + "name": "COUNTRY_CODE", "description": null, "dataType": "STRING", - "columnType": "", - "comment": null, - "termAssignments": [] + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000009" + } + ] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000007", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Year", + "name": "ACCOUNT_BALANCE", "description": null, "dataType": "FLOAT", - "columnType": "R8", - "comment": null, + "columnType": "NUMBER", + "comment": "", "termAssignments": [] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000008", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Time Filter", + "name": "ORDER_AMOUNT", "description": null, - "dataType": "STRING", - "columnType": "STR", - "comment": null, + "dataType": "FLOAT", + "columnType": "NUMBER", + "comment": "", "termAssignments": [] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000009", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Value", + "name": "APPROVAL_STATUS", "description": null, - "dataType": "FLOAT", - "columnType": "R8", - "comment": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", "termAssignments": [] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000010", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Cat Filter", + "name": "PAYMENT_METHOD", "description": null, "dataType": "STRING", - "columnType": "STR", - "comment": null, + "columnType": "VARCHAR", + "comment": "", "termAssignments": [] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000011", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Actual v. Prior", + "name": "TRANSACTION_DATE", "description": null, - "dataType": "STRING", - "columnType": "", - "comment": null, + "dataType": "DATE", + "columnType": "DATE", + "comment": "", "termAssignments": [] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000012", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "vs. Budget", + "name": "CREATED_BY", "description": null, - "dataType": "FLOAT", - "columnType": "R8", - "comment": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", "termAssignments": [] }, { "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000013", "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "KPI Filter - Business Initiatives", + "name": "RISK_RATING", "description": null, "dataType": "STRING", - "columnType": "STR", - "comment": null, - "termAssignments": [] + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000010" + } + ] + } + ], + "terms": { + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000002": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000002", + "name": "Finance", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"Finance is a term for matters regarding the management, creation, and study of money and investments. It involves the use of credit and debt, securities, and investment to finance current projects using future income flows. Because of this temporal aspect, finance is closely linked to the time value of money, interest rates, and other related topics. In this instance, Finance is effectively a Data Domain comprising several business terms under itself.\"}]}]" }, - { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000014", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "KPI Filter - Financials", - "description": null, - "dataType": "STRING", - "columnType": "STR", - "comment": null, - "termAssignments": [] + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000008": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000008", + "name": "Date of Birth", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"The date on which the person was born.\"}]}]" }, - { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000015", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Category", - "description": null, - "dataType": "STRING", - "columnType": "STR", - "comment": null, - "termAssignments": [] + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000005": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000005", + "name": "Customer ID", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"Customer ID\"}]}]" }, - { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000016", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Display", - "description": null, - "dataType": "STRING", - "columnType": "", - "comment": null, - "termAssignments": [] + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001", + "name": "Personal Data", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"Personal information might be subject to GDPR, CCPA or other privacy and data protection regulations.\"}]}]" }, - { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000017", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Extract", - "description": null, - "dataType": "STRING", - "columnType": "", - "comment": null, - "termAssignments": [] + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000010": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000010", + "name": "Surname", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"A person's name in joint with other family members as distinguished from a first or given name.\"}]}]" }, - { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000018", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "X", - "description": null, - "dataType": "STRING", - "columnType": "", - "comment": null, - "termAssignments": [] + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000009": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000009", + "name": "ISO-2 Country Code", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"It was first published in 1998. ISO 3166-2 aims to establish an international standard of short and unique alphanumeric codes to represent the relevant administrative divisions and dependent territories of all countries in a more convenient and less ambiguous form than their full names.\"}]}]" }, - { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000019", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Name", - "description": null, - "dataType": "STRING", - "columnType": "STR", - "comment": null, - "termAssignments": [] + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000007": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000007", + "name": "E-mail", + "type": "businessTerm", + "description": "[{\"type\":\"p\",\"children\":[{\"text\":\"An email address serves as a unique identifier for sending and receiving electronic communications within the organization. It ensures accurate message delivery and supports efficient internal and external communication. As it constitutes \"},{\"type\":\"entityReference\",\"gid\":\"11111111-1111-1111-1111-000000000002\",\"entityName\":\"term\",\"children\":[{\"text\":\"Personal Data\"}]},{\"text\":\" under the \"},{\"type\":\"entityReference\",\"gid\":\"11111111-1111-1111-1111-000000000003\",\"entityName\":\"term\",\"children\":[{\"text\":\"Data Protection Policy\"}]},{\"text\":\", it \"},{\"text\":\"must be used lawfully and only for legitimate business purposes\",\"bold\":true},{\"text\":\".\"}]},{\"type\":\"p\",\"children\":[{\"text\":\"In line with the \"},{\"type\":\"entityReference\",\"gid\":\"11111111-1111-1111-1111-000000000004\",\"entityName\":\"term\",\"children\":[{\"text\":\"Data Access & Control Policy\"}]},{\"text\":\", email data should be accessed solely by authorized personnel and handled securely.\"}]},{\"type\":\"p\",\"children\":[{\"text\":\"According to the \"},{\"type\":\"entityReference\",\"gid\":\"11111111-1111-1111-1111-000000000005\",\"entityName\":\"term\",\"children\":[{\"text\":\"Information Security Policy\"}]},{\"text\":\", all email accounts must be protected through strong passwords and multi-factor authentication to prevent unauthorized access or misuse.\"}]},{\"type\":\"p\",\"children\":[{\"text\":\"Email records should also adhere to the \"},{\"type\":\"entityReference\",\"gid\":\"11111111-1111-1111-1111-000000000006\",\"entityName\":\"term\",\"children\":[{\"text\":\"Data Retention Policy\"}]},{\"text\":\", being stored only for as long as necessary for operational or legal purposes.\"}]},{\"type\":\"p\",\"children\":[{\"text\":\"Proper management of \"},{\"text\":\"email addresses ensures compliance, data integrity, and secure communication\",\"bold\":true},{\"text\":\" across the organization.\"}]},{\"type\":\"paragraph\",\"children\":[{\"text\":\"\"}]}]" }, - { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000020", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Actual", - "description": null, - "dataType": "STRING", - "columnType": "", - "comment": null, - "termAssignments": [] + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000003": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000003", + "name": "TB Test Term", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"Detect \\\"CUSTOMER\\\" in column name\"}]}]" }, - { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000021", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Y", - "description": null, - "dataType": "STRING", - "columnType": "", - "comment": null, - "termAssignments": [] + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000004": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000004", + "name": "Reference Term", + "type": "policyTerm", + "description": null }, - { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000022", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "1", - "description": null, - "dataType": "STRING", - "columnType": "", - "comment": null, - "termAssignments": [] + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000006": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000006", + "name": "Hide data", + "type": "policyTerm", + "description": null + } + }, + "dq_results": { + "processingUrn": "urn:ata:example-tenant:data-quality:processing:00000000-0000-0000-0000-000000000001", + "overallQuality": { + "passedCount": 1784, + "failedCount": 472 }, - { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000023", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "KPI", - "description": null, - "dataType": "STRING", - "columnType": "STR", - "comment": null, - "termAssignments": [] + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 2230, + "failedCount": 26, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 1784, + "failedCount": 472, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 2256, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "overallDqFindings": [], + "catalogItemLink": "https://one.example.com/catalog/catalog-items/11111111-1111-1111-1111-000000000007", + "dqResultsLink": "https://one.example.com/entities/catalog-item-dq-configurations/11111111-1111-1111-1111-000000000008/results", + "invalidSamplesLink": "https://one.example.com/entities/catalog-item-dq-configurations/11111111-1111-1111-1111-000000000008/invalid-records", + "invalidRecordsExportResults": [ + { + "success": false, + "message": "Export definition or datasource context not found for exportId: 11111111-1111-1111-1111-000000000009", + "location": null, + "link": null + } + ], + "dqAttributeResults": [ + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000001", + "attributeName": "TRANSACTION_ID", + "overallQuality": { + "passedCount": 2251, + "failedCount": 5 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 2251, + "failedCount": 5, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000001", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000001", + "ruleInstanceName": "r_TRANSACTION_ID", + "ruleName": "r_TRANSACTION_ID", + "results": [ + { + "count": 2251, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 5, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2251 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000002", + "name": "not unique", + "count": 5 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000002", + "attributeName": "CUSTOMER_ID", + "overallQuality": { + "passedCount": 2256, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 2256, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000002", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000002", + "ruleInstanceName": "AI Rule Valid Customer ID", + "ruleName": "AI Rule Valid Customer ID", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000003", + "name": "Missing CUSTOMER_ID", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000003", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000003", + "ruleInstanceName": "AI Rule Consistent Account-Customer Mapping", + "ruleName": "AI Rule Consistent Account-Customer Mapping", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000004", + "name": "CustomerID is missing", + "count": 0 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000003", + "attributeName": "CUSTOMER_NAME", + "overallQuality": { + "passedCount": 2256, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 2256, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000004", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000004", + "ruleInstanceName": "AI Rule Valid Customer Name", + "ruleName": "AI Rule Valid Customer Name", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000005", + "name": "Customer name is missing or null", + "count": 0 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000004", + "attributeName": "EMAIL", + "overallQuality": { + "passedCount": 0, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000005", + "attributeName": "BIRTH_DATE", + "overallQuality": { + "passedCount": 2216, + "failedCount": 40 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 2216, + "failedCount": 40, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000005", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000005", + "ruleInstanceName": "AI Rule Valid Birth Date Rule", + "ruleName": "AI Rule Valid Birth Date Rule", + "results": [ + { + "count": 2216, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 40, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2216 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000006", + "name": "Birth date is in the future or missing", + "count": 40 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000006", + "attributeName": "COUNTRY_CODE", + "overallQuality": { + "passedCount": 2183, + "failedCount": 73 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 2230, + "failedCount": 26, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 2183, + "failedCount": 73, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000006", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000006", + "ruleInstanceName": "RDM - Country code length check", + "ruleName": "RDM - Country code length check", + "results": [ + { + "count": 2183, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 73, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 73 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000007", + "name": "All good", + "count": 2183 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000007", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000007", + "ruleInstanceName": "AI Rule Country Code Not Null Check", + "ruleName": "AI Rule Country Code Not Null Check", + "results": [ + { + "count": 26, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2230, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2230 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000008", + "name": "Country code is missing", + "count": 26 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000008", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000008", + "ruleInstanceName": "AI Rule Rule for COUNTRY", + "ruleName": "AI Rule Rule for COUNTRY", + "results": [ + { + "count": 26, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2230, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2230 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000009", + "name": "Country is missing", + "count": 26 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000007", + "attributeName": "ACCOUNT_BALANCE", + "overallQuality": { + "passedCount": 0, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000008", + "attributeName": "ORDER_AMOUNT", + "overallQuality": { + "passedCount": 1898, + "failedCount": 358 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 1898, + "failedCount": 358, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000009", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000009", + "ruleInstanceName": "AI Rule Order Amount and Risk Rating Check", + "ruleName": "AI Rule Order Amount and Risk Rating Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000010", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000010", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000010", + "ruleInstanceName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "ruleName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000011", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000009", + "attributeName": "APPROVAL_STATUS", + "overallQuality": { + "passedCount": 0, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000010", + "attributeName": "PAYMENT_METHOD", + "overallQuality": { + "passedCount": 1898, + "failedCount": 358 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 1898, + "failedCount": 358, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000009", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000009", + "ruleInstanceName": "AI Rule Order Amount and Risk Rating Check", + "ruleName": "AI Rule Order Amount and Risk Rating Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000010", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000010", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000010", + "ruleInstanceName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "ruleName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000011", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000011", + "attributeName": "TRANSACTION_DATE", + "overallQuality": { + "passedCount": 0, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 2256, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000011", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000011", + "ruleInstanceName": "AI Rule No Future Dated Transactions", + "ruleName": "AI Rule No Future Dated Transactions", + "results": [ + { + "count": 0, + "result": { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + }, + { + "count": 0, + "result": { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + } + }, + { + "count": 0, + "result": { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + } + }, + { + "count": 2256, + "result": { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000012", + "name": "Transaction date is in the future", + "count": 0 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000012", + "attributeName": "CREATED_BY", + "overallQuality": { + "passedCount": 2256, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 2256, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000012", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000012", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000013", + "name": "Value is null", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000013", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000013", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000014", + "name": "Missing value", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000014", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000014", + "ruleInstanceName": "AI Rule Valid Created By", + "ruleName": "AI Rule Valid Created By", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000015", + "name": "CREATED_BY is missing", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000015", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000015", + "ruleInstanceName": "AI Rule Created By Completeness", + "ruleName": "AI Rule Created By Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000016", + "name": "Created by field is null or empty", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000016", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000016", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000017", + "name": "Missing value", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000017", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000017", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000018", + "name": "Missing value", + "count": 0 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000018", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000018", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000019", + "name": "Missing value", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000019", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000019", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000020", + "name": "Value is null", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000020", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000014", + "ruleInstanceName": "AI Rule Valid Created By", + "ruleName": "AI Rule Valid Created By", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000015", + "name": "CREATED_BY is missing", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000021", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000020", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000021", + "name": "Missing value", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000022", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000015", + "ruleInstanceName": "AI Rule Created By Completeness", + "ruleName": "AI Rule Created By Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000016", + "name": "Created by field is null or empty", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000023", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000021", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000022", + "name": "Missing value", + "count": 0 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000013", + "attributeName": "RISK_RATING", + "overallQuality": { + "passedCount": 1898, + "failedCount": 358 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 1898, + "failedCount": 358, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000009", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000009", + "ruleInstanceName": "AI Rule Order Amount and Risk Rating Check", + "ruleName": "AI Rule Order Amount and Risk Rating Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000010", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000010", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000010", + "ruleInstanceName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "ruleName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000011", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + } + ] + } + ] + } + }, + { + "item": { + "urn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000002", + "name": "aggregation", + "description": null, + "originPath": null, + "connection": { + "urn": "urn:ata:example-tenant:processing-center:connection:00000000-0000-0000-0000-000000000002" + }, + "source": { + "urn": "urn:ata:example-tenant:catalog:source:00000000-0000-0000-0000-000000000002" }, + "locations": [], + "termAssignments": [], + "effectiveStewardship": null, + "primaryDqMonitor": { + "urn": "urn:ata:example-tenant:data-quality:dq-monitor:00000000-0000-0000-0000-000000000002" + }, + "aliases": [] + }, + "attributes": [ { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000024", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "Month", + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000014", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000002", + "name": "PAYMENT_METHOD", "description": null, - "dataType": "DATE", - "columnType": "DBDATE", + "dataType": "STRING", + "columnType": "VARCHAR", "comment": null, "termAssignments": [] }, { - "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000025", - "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", - "name": "KPI Filter - Revenue Drivers", + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000015", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000002", + "name": "NUM_OF_TRANSACTIONS", "description": null, - "dataType": "STRING", - "columnType": "STR", + "dataType": "LONG", + "columnType": "NUMBER", "comment": null, "termAssignments": [] } ], - "terms": {} - }, - { - "item": { - "urn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000002", - "name": "Analysis Year to Date Summary", - "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"This report provides a comprehensive overview of marketing performance metrics for the current year, offering valuable insights into the effectiveness of various marketing strategies and campaigns. It serves as a crucial tool for marketing teams to assess progress, identify trends, and make informed decisions to optimize future initiatives. By consolidating key performance indicators and data points, it enables stakeholders to quickly evaluate the impact of marketing efforts on business objectives, ensuring alignment with overall company goals. This summary is instrumental in facilitating strategic planning and resource allocation within the marketing domain.\"}]}]", - "originPath": "Operational Reporting Library (demo data)/11111111-1111-1111-1111-000000000002", - "connection": { - "urn": "urn:ata:example-tenant:processing-center:connection:00000000-0000-0000-0000-000000000002" + "terms": {}, + "dq_results": { + "processingUrn": "urn:ata:example-tenant:data-quality:processing:00000000-0000-0000-0000-000000000002", + "overallQuality": { + "passedCount": 0, + "failedCount": 4 }, - "source": { - "urn": "urn:ata:example-tenant:catalog:source:00000000-0000-0000-0000-000000000002" - }, - "locations": [ + "dimensionResults": [ { - "name": "Operational Reporting Library (demo data)", - "description": null + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 4, + "failedCount": 0, + "ruleInstanceCount": 2 }, { - "name": "Workspaces", - "description": null + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 4, + "ruleInstanceCount": 2 } ], - "termAssignments": [ + "overallDqFindings": [], + "catalogItemLink": "https://one.example.com/catalog/catalog-items/11111111-1111-1111-1111-000000000010", + "dqResultsLink": "https://one.example.com/entities/catalog-item-dq-configurations/11111111-1111-1111-1111-000000000011/results", + "invalidSamplesLink": "https://one.example.com/entities/catalog-item-dq-configurations/11111111-1111-1111-1111-000000000011/invalid-records", + "invalidRecordsExportResults": [], + "dqAttributeResults": [ { - "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001" + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000014", + "attributeName": "PAYMENT_METHOD", + "overallQuality": { + "passedCount": 0, + "failedCount": 4 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 4, + "ruleInstanceCount": 3 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 4, + "ruleInstanceCount": 3 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000024", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000022", + "ruleInstanceName": "AI Rule SSN Validation Rule", + "ruleName": "AI Rule SSN Validation Rule", + "results": [ + { + "count": 0, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 4, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000023", + "name": "Does not match the SSN format", + "count": 4 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000025", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000023", + "ruleInstanceName": "AI Rule Completeness Rule for CUSTOMER", + "ruleName": "AI Rule Completeness Rule for CUSTOMER", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 4, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000024", + "name": "Customer attribute is null or empty", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000026", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000024", + "ruleInstanceName": "AI Rule Insurance Completeness Rule", + "ruleName": "AI Rule Insurance Completeness Rule", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 4, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000025", + "name": "Critical field is empty", + "count": 0 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000015", + "attributeName": "NUM_OF_TRANSACTIONS", + "overallQuality": { + "passedCount": 4, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 4, + "failedCount": 0, + "ruleInstanceCount": 1 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 4, + "failedCount": 0, + "ruleInstanceCount": 1 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000027", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000025", + "ruleInstanceName": "AI Rule Accuracy Rule for INVOICED_USD", + "ruleName": "AI Rule Accuracy Rule for INVOICED_USD", + "results": [ + { + "count": 4, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 0, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000026", + "name": "Negative value", + "count": 0 + } + ], + "dqFindings": [] + } + ] } - ], - "effectiveStewardship": { - "groupUrn": "urn:ata:example-tenant:iam:group:00000000-0000-0000-0000-000000000001" - }, - "primaryDqMonitor": { - "urn": "urn:ata:example-tenant:data-quality:dq-monitor:00000000-0000-0000-0000-000000000002" - }, - "aliases": [] - }, - "attributes": [], - "terms": { - "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001": { - "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001", - "name": "Marketing", - "type": "businessTerm", - "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"Marketing refers to activities a company undertakes to promote the buying or selling a product or service. Marketing includes advertising, selling, and delivering products to consumers or other businesses. In this instance, Marketing is a Data Domain comprising several business terms under itself.\"}]}]" - } + ] } } ] \ No newline at end of file diff --git a/converters/ataccama/tests/test_ataccama_to_osi.py b/converters/ataccama/tests/test_ataccama_to_osi.py index 06e3cc0..d2d64a5 100644 --- a/converters/ataccama/tests/test_ataccama_to_osi.py +++ b/converters/ataccama/tests/test_ataccama_to_osi.py @@ -1,4 +1,8 @@ -"""Tests for the Ataccama -> OSI conversion (offline, using a recorded fixture).""" +"""Tests for the Ataccama -> OSI conversion (offline, using a recorded fixture). + +Fixture items: BANK_TRANSACTIONS (a warehouse table with terms, DATE columns, and +rich DQ results) and aggregation (a derived output with no locations and a 0% DQ case). +""" from __future__ import annotations @@ -11,8 +15,9 @@ from ataccama_osi.ataccama_to_osi import ( OSI_VERSION, - attribute_to_field, + _quality_summary, ataccama_to_osi, + attribute_to_field, build_source, flatten_richtext, ) @@ -33,10 +38,21 @@ def document(bundles: list[CatalogItemBundle]) -> dict: return ataccama_to_osi(bundles, model_name="test_model", tenant="example-tenant") -def _find_dataset(document: dict, name: str) -> dict: +def _dataset(document: dict, name: str) -> dict: return next(d for d in document["semantic_model"][0]["datasets"] if d["name"] == name) +def _field(dataset: dict, name: str) -> dict: + return next(f for f in dataset["fields"] if f["name"] == name) + + +def _ext(obj: dict) -> dict: + return json.loads(obj["custom_extensions"][0]["data"]) + + +# --- structure & core mapping --- + + def test_document_is_schema_valid(document: dict) -> None: schema = json.loads(SCHEMA.read_text()) errors = list(Draft202012Validator(schema).iter_errors(document)) @@ -47,45 +63,81 @@ def test_document_is_schema_valid(document: dict) -> None: def test_model_structure(document: dict) -> None: model = document["semantic_model"][0] assert model["name"] == "test_model" - assert len(model["datasets"]) == 2 - # tenant preserved at model level - assert any(e["vendor_name"] == "ATACCAMA" for e in model["custom_extensions"]) + assert {d["name"] for d in model["datasets"]} == {"BANK_TRANSACTIONS", "aggregation"} def test_attribute_becomes_quoted_ansi_field(document: dict) -> None: - ds = _find_dataset(document, "Financial KPIs") - assert len(ds["fields"]) == 25 - field = ds["fields"][0] - assert field["name"] == "Prior Yr. SM" - dialects = field["expression"]["dialects"] - assert dialects == [{"dialect": "ANSI_SQL", "expression": '"Prior Yr. SM"'}] - # attribute urn + typing preserved for round-tripping - ext = json.loads(field["custom_extensions"][0]["data"]) - assert ext["data_type"] == "STRING" - assert ext["attribute_urn"].startswith("urn:ata:") + ds = _dataset(document, "BANK_TRANSACTIONS") + assert len(ds["fields"]) == 13 + field = _field(ds, "TRANSACTION_ID") + assert field["expression"]["dialects"] == [{"dialect": "ANSI_SQL", "expression": '"TRANSACTION_ID"'}] + assert _ext(field)["data_type"] == "STRING" + +def test_date_columns_are_time_dimensions(document: dict) -> None: + ds = _dataset(document, "BANK_TRANSACTIONS") + assert _field(ds, "BIRTH_DATE")["dimension"] == {"is_time": True} + assert _field(ds, "TRANSACTION_DATE")["dimension"] == {"is_time": True} + assert "dimension" not in _field(ds, "ACCOUNT_BALANCE") -def test_richtext_description_is_flattened(document: dict) -> None: - ds = _find_dataset(document, "Analysis Year to Date Summary") - assert ds["description"].startswith("This report provides") - # no Slate JSON leaked through - assert "{" not in ds["description"] and "children" not in ds["description"] +def test_source_from_locations_and_limitation(document: dict) -> None: + # DB-backed item: reversed locations + name form a qualified path + assert _dataset(document, "BANK_TRANSACTIONS")["source"] == "DEMO_ENV.CLOUD_DEMO.BANK_TRANSACTIONS" + # derived item with no locations: source degrades to the bare name + assert _dataset(document, "aggregation")["source"] == "aggregation" -def test_assigned_term_becomes_ai_context(document: dict) -> None: - ds = _find_dataset(document, "Analysis Year to Date Summary") - assert ds["ai_context"]["synonyms"] == ["Marketing"] - assert "Marketing" in ds["ai_context"]["instructions"] + +def test_terms_become_ai_context(document: dict) -> None: + ds = _dataset(document, "BANK_TRANSACTIONS") + assert set(ds["ai_context"]["synonyms"]) >= {"Personal Data", "Finance"} + email = _field(ds, "EMAIL") + assert email["ai_context"]["synonyms"] == ["E-mail"] + assert "email" in email["ai_context"]["instructions"].lower() def test_governance_metadata_preserved(document: dict) -> None: - ds = _find_dataset(document, "Analysis Year to Date Summary") - ext = json.loads(ds["custom_extensions"][0]["data"]) + ext = _ext(_dataset(document, "BANK_TRANSACTIONS")) assert ext["catalog_item_urn"].startswith("urn:ata:") assert "connection_urn" in ext and "stewardship_group_urn" in ext -# --- unit tests for helpers --- +# --- data quality enrichment --- + + +def test_dataset_dq_overall_and_dimensions(document: dict) -> None: + dq = _ext(_dataset(document, "BANK_TRANSACTIONS"))["dq"] + assert dq["passed"] == 1784 and dq["failed"] == 472 + assert dq["pass_rate_pct"] == 79.1 + dims = {d["name"]: d for d in dq["dimensions"]} + assert dims["Completeness"]["pass_rate_pct"] == 98.8 + assert dims["Validity"]["pass_rate_pct"] == 79.1 + # dimensions with no evaluated records (Uniqueness/Accuracy = 0/0) are dropped + assert "Uniqueness" not in dims and "Accuracy" not in dims + assert dq["results_link"].startswith("https://") + + +def test_field_level_dq(document: dict) -> None: + ds = _dataset(document, "BANK_TRANSACTIONS") + dq = _ext(_field(ds, "TRANSACTION_ID"))["dq"] + assert dq == {"passed": 2251, "failed": 5, "pass_rate_pct": 99.8} + + +def test_zero_pass_rate_case(document: dict) -> None: + dq = _ext(_dataset(document, "aggregation"))["dq"] + assert dq["passed"] == 0 and dq["failed"] == 4 + assert dq["pass_rate_pct"] == 0.0 + + +def test_no_dq_keys_when_dq_absent(bundles: list[CatalogItemBundle]) -> None: + b = bundles[0] + b.dq_results = None # simulate --no-dq / item without a monitor + ds = ataccama_to_osi([b])["semantic_model"][0]["datasets"][0] + assert "dq" not in _ext(ds) + assert all("dq" not in _ext(f) for f in ds["fields"]) + + +# --- helper unit tests --- @pytest.mark.parametrize( @@ -102,16 +154,21 @@ def test_flatten_richtext(value, expected) -> None: assert flatten_richtext(value) == expected -def test_is_time_inferred_from_datatype() -> None: - attr = CatalogAttribute(urn="urn:ata:t:catalog:catalog-attribute:1", name="created_at", data_type="DATETIME") - field = attribute_to_field(attr, terms={}, used=set()) - assert field["dimension"] == {"is_time": True} +@pytest.mark.parametrize( + "oq,expected", + [ + (None, None), + ({"passedCount": 3, "failedCount": 1}, {"passed": 3, "failed": 1, "pass_rate_pct": 75.0}), + ({"passedCount": 0, "failedCount": 0}, {"passed": 0, "failed": 0}), # not evaluated -> no rate + ], +) +def test_quality_summary(oq, expected) -> None: + assert _quality_summary(oq) == expected -def test_non_time_datatype_has_no_dimension() -> None: - attr = CatalogAttribute(urn="urn:ata:t:catalog:catalog-attribute:1", name="amount", data_type="DOUBLE") - field = attribute_to_field(attr, terms={}, used=set()) - assert "dimension" not in field +def test_is_time_inferred_from_datatype() -> None: + attr = CatalogAttribute(urn="urn:ata:t:catalog:catalog-attribute:1", name="created_at", data_type="DATETIME") + assert attribute_to_field(attr, terms={}, used=set())["dimension"] == {"is_time": True} def test_duplicate_dataset_names_are_disambiguated() -> None: @@ -119,8 +176,7 @@ def test_duplicate_dataset_names_are_disambiguated() -> None: CatalogItemBundle(item=CatalogItem(urn=f"urn:ata:t:catalog:catalog-item:{i}", name="Orders")) for i in range(2) ] - doc = ataccama_to_osi(items) - names = [d["name"] for d in doc["semantic_model"][0]["datasets"]] + names = [d["name"] for d in ataccama_to_osi(items)["semantic_model"][0]["datasets"]] assert names == ["Orders", "Orders_2"] From fb13c7df97fe88b1a82211969913a7c0f6d69507 Mon Sep 17 00:00:00 2001 From: Cara Baestlein Date: Thu, 2 Jul 2026 12:40:58 +0100 Subject: [PATCH 3/4] Update README --- converters/ataccama/README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/converters/ataccama/README.md b/converters/ataccama/README.md index 1eff1fd..b759783 100644 --- a/converters/ataccama/README.md +++ b/converters/ataccama/README.md @@ -87,12 +87,9 @@ Tests run fully offline against a recorded catalog fixture | `CatalogAttribute.dataType` ∈ {DATE, DATETIME, TIMESTAMP, TIME} | `field.dimension.is_time = true` | | `CatalogAttribute.description` / `comment` | `field.description` | | attribute `Term`s | `field.ai_context` | -| DQ `overallQuality` + `dimensionResults` (latest processing) | dataset `custom_extensions` → `dq` (`passed`, `failed`, `pass_rate_pct`, `dimensions[]`, `results_link`) | -| DQ per-attribute `overallQuality` | field `custom_extensions` → `dq` (`passed`, `failed`, `pass_rate_pct`) | -| URNs, `dataType`, `columnType`, connection/source/stewardship/monitor | `custom_extensions` (`vendor_name: ATACCAMA`) | - -All Ataccama-specific metadata with no OSI-core home is preserved in `ATACCAMA` -custom extensions (as a JSON string) so the model round-trips without data loss. +| DQ `overallQuality` + `dimensionResults` (latest processing) | dataset `custom_extensions` (`vendor_name: ATACCAMA`) → `data.dq` (`passed`, `failed`, `pass_rate_pct`, `dimensions[]`, `results_link`) | +| DQ per-attribute `overallQuality` | field `custom_extensions` (`vendor_name: ATACCAMA`) → `data.dq` (`passed`, `failed`, `pass_rate_pct`) | +| URNs, `dataType`, `columnType`, connection/source/stewardship/monitor | `custom_extensions` (`vendor_name: ATACCAMA`) → `data` | ### Data quality From 714c0343bada32fbc04732b43d1cb2b998a2fee9 Mon Sep 17 00:00:00 2001 From: Cara Baestlein Date: Thu, 2 Jul 2026 13:27:13 +0100 Subject: [PATCH 4/4] feat(converters): enrich Ataccama DQ with threshold, findings, and opt-in AI warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add the monitor's overall DQ threshold to the dataset dq block as `threshold_pct` + `below_threshold` (the pass/fail bar shown in Ataccama), fetched from the monitor-config endpoint; omitted when no threshold is configured. - Always report `active_findings` (0 = no open findings) so the clean state is explicit. - Opt-in `--dq-ai-warnings`: append a plain-language DQ caveat to `ai_context.instructions` for datasets Ataccama flags as below quality — below its configured threshold or with active findings. Ataccama is the sole source of truth: the converter never applies a threshold of its own, so items without a configured threshold (and columns) are never flagged. - Reword `pass_rate_pct`: it is Ataccama's `overallQuality` as a percentage (the API returns counts, not a preformatted score), not a custom "derived" metric. - README: clarify DQ lives inside the single ATACCAMA extension's `data`, document the threshold/findings fields and the opt-in warnings. Co-Authored-By: Claude Opus 4.8 --- converters/ataccama/README.md | 38 ++++++-- .../src/ataccama_osi/ataccama_to_osi.py | 93 ++++++++++++++++--- converters/ataccama/src/ataccama_osi/cli.py | 8 ++ .../ataccama/src/ataccama_osi/client.py | 30 +++++- .../ataccama/src/ataccama_osi/models.py | 4 + .../tests/fixtures/ataccama_bundles.json | 6 +- .../ataccama/tests/test_ataccama_to_osi.py | 70 ++++++++++++++ 7 files changed, 225 insertions(+), 24 deletions(-) diff --git a/converters/ataccama/README.md b/converters/ataccama/README.md index b759783..a0606de 100644 --- a/converters/ataccama/README.md +++ b/converters/ataccama/README.md @@ -47,6 +47,9 @@ ataccama-to-osi --env-file .ataccama.env \ # data-quality results are fetched by default; use --no-dq to skip them ataccama-to-osi --env-file .ataccama.env --urn --no-dq -o my_model.osi.yaml +# opt-in: append DQ warnings to ai_context for datasets Ataccama flags as below quality +ataccama-to-osi --env-file .ataccama.env --urn --dq-ai-warnings -o my_model.osi.yaml + # or supply a file of URNs (one per line) ataccama-to-osi --env-file .ataccama.env --urns-file items.txt -o my_model.osi.yaml ``` @@ -87,7 +90,7 @@ Tests run fully offline against a recorded catalog fixture | `CatalogAttribute.dataType` ∈ {DATE, DATETIME, TIMESTAMP, TIME} | `field.dimension.is_time = true` | | `CatalogAttribute.description` / `comment` | `field.description` | | attribute `Term`s | `field.ai_context` | -| DQ `overallQuality` + `dimensionResults` (latest processing) | dataset `custom_extensions` (`vendor_name: ATACCAMA`) → `data.dq` (`passed`, `failed`, `pass_rate_pct`, `dimensions[]`, `results_link`) | +| DQ `overallQuality` + `dimensionResults` + findings + monitor threshold | dataset `custom_extensions` (`vendor_name: ATACCAMA`) → `data.dq` (`passed`, `failed`, `pass_rate_pct`, `threshold_pct`, `below_threshold`, `active_findings`, `dimensions[]`, `results_link`) | | DQ per-attribute `overallQuality` | field `custom_extensions` (`vendor_name: ATACCAMA`) → `data.dq` (`passed`, `failed`, `pass_rate_pct`) | | URNs, `dataType`, `columnType`, connection/source/stewardship/monitor | `custom_extensions` (`vendor_name: ATACCAMA`) → `data` | @@ -99,12 +102,34 @@ and per-dimension quality on the dataset, and per-column quality on each field. `--no-dq` to skip the DQ calls. Notes: -- `pass_rate_pct` is **derived** (`passed / (passed + failed)`); the raw `passed`/`failed` - counts are authoritative. Ataccama's own UI score may weight dimensions/rules - differently, so this is not presented as an official Ataccama "Data Quality score". +- `pass_rate_pct` is Ataccama's overall quality (`overallQuality`) expressed as a + percentage: `passed / (passed + failed)`. The DQ API returns only the pass/fail + **counts**, not a preformatted percentage, so the converter formats it — the counts + themselves are Ataccama's own, consistent with what the platform reports. +- `threshold_pct` / `below_threshold` come from the monitor's configured overall + threshold (`overallDqThresholds`, fetched from the monitor-config endpoint) — i.e. + the same pass/fail bar shown in Ataccama. They are omitted when the monitor has no + overall threshold configured. +- `active_findings` is always reported (`0` = no open data-quality findings), so the + clean state is an explicit signal rather than an omission. - Dimensions with no evaluated records in the processing are omitted. - Items without a published monitor/results simply carry no `dq` block (best-effort). +#### AI warnings (opt-in) + +By default the DQ block is descriptive data only. With `--dq-ai-warnings`, a plain-language +warning is appended to `ai_context.instructions` for any dataset Ataccama flags as having +data quality issues (data quality below configured threshold or active findings), so a +downstream AI/BI tool reading the OSI model (not just the vendor extension) is told to +treat the data with caution. Example: + +> *Data-quality warning: 60.0% of quality checks passed on the latest run (below the +> configured 75% quality threshold). Verify this data before using it for analysis or +> automated decisions.* + +Warnings are appended to any existing (term-derived) instructions, never replacing them, +and require DQ (so not usable with `--no-dq`). + ## Limitations The Ataccama Catalog API is a **catalog / governance / data-quality** surface, while OSI @@ -112,8 +137,9 @@ is an **analytics semantic model**. Some OSI constructs therefore have no source - **Metrics** — Ataccama has no analytics metrics; none are emitted. (Data-quality scores are attached as `ATACCAMA` extensions, not as OSI metrics.) -- **Data Trust Score / Index** — not exposed by the public APIs, so it is not - emitted. Only rule-level DQ results (which the converter does carry) are available. +- **Data Trust Score / Index** — not exposed by the public APIs (no precomputed + score field exists), so it is not emitted. The converter instead carries the DQ + results and monitor threshold that the APIs do provide. - **Relationships / `primary_key` / `unique_keys`** — the Catalog API does not expose foreign or primary keys on catalog items, so these are omitted. - **`source` string** — the API exposes only a folder hierarchy (`locations`) and diff --git a/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py b/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py index 09d4c1d..a8e9056 100644 --- a/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py +++ b/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py @@ -128,8 +128,9 @@ def _ataccama_extension(data: dict[str, Any]) -> dict[str, str]: def _quality_summary(overall_quality: dict[str, Any] | None) -> dict[str, Any] | None: """Turn an OverallQuality {passedCount, failedCount} into a compact summary. - ``pass_rate_pct`` is DERIVED (passed / (passed + failed)); the raw counts are the - authoritative signal. Ataccama's own UI score may weight dimensions/rules differently. + ``pass_rate_pct`` is Ataccama's overall quality expressed as a percentage + (passed / (passed + failed)). The DQ API returns only counts, not a preformatted + percentage, so the converter formats it; the underlying counts are Ataccama's own. """ if not overall_quality: return None @@ -142,12 +143,22 @@ def _quality_summary(overall_quality: dict[str, Any] | None) -> dict[str, Any] | return summary -def _dataset_dq(dq_results: dict[str, Any] | None) -> dict[str, Any] | None: - """Build the dataset-level DQ block from a raw DqResults payload.""" +def _dataset_dq(dq_results: dict[str, Any] | None, threshold_pct: float | None = None) -> dict[str, Any] | None: + """Build the dataset-level DQ block from a raw DqResults payload. + + ``threshold_pct`` is the monitor's configured overall pass/fail bar (from the + monitor config). When present alongside a pass rate, ``below_threshold`` records + whether the data is under that bar — the same pass/fail state shown in Ataccama. + """ if not dq_results: return None dq = _quality_summary(dq_results.get("overallQuality")) or {} + if threshold_pct is not None: + dq["threshold_pct"] = threshold_pct + if "pass_rate_pct" in dq: + dq["below_threshold"] = dq["pass_rate_pct"] < threshold_pct + dimensions = [] for dim in dq_results.get("dimensionResults", []) or []: passed = dim.get("passedCount") or 0 @@ -160,9 +171,11 @@ def _dataset_dq(dq_results: dict[str, Any] | None) -> dict[str, Any] | None: if dimensions: dq["dimensions"] = dimensions - active_findings = sum(1 for f in dq_results.get("overallDqFindings", []) or [] if f.get("status") == "ACTIVE") - if active_findings: - dq["active_findings"] = active_findings + # Always report the active-finding count (0 = no open data-quality issues) so the + # "clean" state is an explicit signal, not just an omission. + dq["active_findings"] = sum( + 1 for f in dq_results.get("overallDqFindings", []) or [] if f.get("status") == "ACTIVE" + ) if dq_results.get("processingUrn"): dq["processing_urn"] = dq_results["processingUrn"] if dq_results.get("dqResultsLink"): @@ -184,6 +197,45 @@ def _attribute_dq_index(dq_results: dict[str, Any] | None) -> dict[str, dict[str return index +# --- optional AI warnings driven by DQ --- + + +def _dataset_dq_warning(dq: dict[str, Any] | None) -> str | None: + """A natural-language warning for AI consumers when a dataset is below quality. + + Fires only on Ataccama's own signals — below the monitor's configured threshold, or + an active finding — so Ataccama remains the sole source of truth. If neither applies + (including when no threshold is configured), no warning is produced. + """ + if not dq: + return None + reasons: list[str] = [] + if dq.get("below_threshold") is True: + reasons.append(f"below the configured {dq.get('threshold_pct')}% quality threshold") + if dq.get("active_findings"): + reasons.append(f"{dq['active_findings']} active data-quality finding(s)") + if not reasons: + return None + pass_rate = dq.get("pass_rate_pct") + prefix = f"{pass_rate}% of quality checks passed on the latest run" if pass_rate is not None else "quality checks failed" + return ( + f"Data-quality warning: {prefix} ({'; '.join(reasons)}). " + "Verify this data before using it for analysis or automated decisions." + ) + + +def _inject_warning(ai_context: dict[str, Any] | None, warning: str | None) -> dict[str, Any] | None: + """Append a warning to an ai_context object's instructions (creating one if needed).""" + if not warning: + return ai_context + if ai_context is None: + return {"instructions": warning} + ctx = dict(ai_context) if isinstance(ai_context, dict) else {"instructions": str(ai_context)} + existing = ctx.get("instructions") + ctx["instructions"] = f"{existing} {warning}" if existing else warning + return ctx + + # --- attribute -> field -------------------------------------------------- @@ -206,6 +258,8 @@ def attribute_to_field( if description: field_obj["description"] = description + field_dq = dq_by_attr.get(attr.urn) if dq_by_attr else None + ai_context = _terms_ai_context([ta.term_urn for ta in attr.term_assignments], terms) if ai_context: field_obj["ai_context"] = ai_context @@ -218,8 +272,8 @@ def attribute_to_field( ext_data["column_type"] = attr.column_type if attr.term_assignments: ext_data["term_urns"] = [ta.term_urn for ta in attr.term_assignments] - if dq_by_attr and attr.urn in dq_by_attr: - ext_data["dq"] = dq_by_attr[attr.urn] + if field_dq: + ext_data["dq"] = field_dq field_obj["custom_extensions"] = [_ataccama_extension(ext_data)] return field_obj @@ -228,7 +282,11 @@ def attribute_to_field( # --- catalog item -> dataset --------------------------------------------- -def bundle_to_dataset(bundle: CatalogItemBundle, used_dataset_names: set[str]) -> dict[str, Any]: +def bundle_to_dataset( + bundle: CatalogItemBundle, + used_dataset_names: set[str], + dq_ai_warnings: bool = False, +) -> dict[str, Any]: item = bundle.item name = _unique_name(item.name, used_dataset_names, fallback="dataset") @@ -238,7 +296,11 @@ def bundle_to_dataset(bundle: CatalogItemBundle, used_dataset_names: set[str]) - if description: dataset["description"] = description + dataset_dq = _dataset_dq(bundle.dq_results, bundle.dq_threshold_pct) + ai_context = _terms_ai_context([ta.term_urn for ta in item.term_assignments], bundle.terms) + if dq_ai_warnings: + ai_context = _inject_warning(ai_context, _dataset_dq_warning(dataset_dq)) if ai_context: dataset["ai_context"] = ai_context @@ -266,7 +328,6 @@ def bundle_to_dataset(bundle: CatalogItemBundle, used_dataset_names: set[str]) - ext_data["term_urns"] = [ta.term_urn for ta in item.term_assignments] if item.aliases: ext_data["aliases"] = item.aliases - dataset_dq = _dataset_dq(bundle.dq_results) if dataset_dq: ext_data["dq"] = dataset_dq dataset["custom_extensions"] = [_ataccama_extension(ext_data)] @@ -282,10 +343,16 @@ def ataccama_to_osi( model_name: str = "ataccama_model", model_description: str | None = None, tenant: str | None = None, + dq_ai_warnings: bool = False, ) -> dict[str, Any]: - """Convert a list of catalog-item bundles into an OSI document dict (ready for YAML).""" + """Convert a list of catalog-item bundles into an OSI document dict (ready for YAML). + + When ``dq_ai_warnings`` is set, datasets that Ataccama flags as below quality (below + the configured threshold, or with active findings) get a natural-language warning + appended to their ``ai_context.instructions`` (see ``_dataset_dq_warning``). + """ used_dataset_names: set[str] = set() - datasets = [bundle_to_dataset(b, used_dataset_names) for b in bundles] + datasets = [bundle_to_dataset(b, used_dataset_names, dq_ai_warnings) for b in bundles] semantic_model: dict[str, Any] = {"name": model_name, "datasets": datasets} if model_description: diff --git a/converters/ataccama/src/ataccama_osi/cli.py b/converters/ataccama/src/ataccama_osi/cli.py index 88492da..5899429 100644 --- a/converters/ataccama/src/ataccama_osi/cli.py +++ b/converters/ataccama/src/ataccama_osi/cli.py @@ -72,6 +72,13 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Skip fetching data-quality results (DQ enrichment is included by default).", ) + parser.add_argument( + "--dq-ai-warnings", + action="store_true", + help="Append a data-quality warning to ai_context.instructions for datasets that " + "Ataccama flags as below quality — below its configured threshold, or with active " + "findings (opt-in; requires DQ, so not with --no-dq).", + ) args = parser.parse_args(argv) if args.env_file: @@ -102,6 +109,7 @@ def main(argv: list[str] | None = None) -> int: model_name=args.model_name, model_description=args.model_description, tenant=_tenant_from_urn(urns[0]), + dq_ai_warnings=args.dq_ai_warnings, ) text = yaml.dump(document, default_flow_style=False, sort_keys=False, allow_unicode=True) diff --git a/converters/ataccama/src/ataccama_osi/client.py b/converters/ataccama/src/ataccama_osi/client.py index 9a83f7e..0b9ea0d 100644 --- a/converters/ataccama/src/ataccama_osi/client.py +++ b/converters/ataccama/src/ataccama_osi/client.py @@ -121,6 +121,19 @@ def get_dq_results( except requests.HTTPError: return None + def get_dq_overall_threshold(self, catalog_item_urn: str, monitor: str = "primary") -> float | None: + """The monitor's configured overall DQ threshold (0-100), or None if unset. + + This is the pass/fail bar shown in Ataccama, and is only available from the + monitor-config endpoint (not the DQ results payload). + """ + try: + mon = self._get(f"/data-quality/v1/catalog-items/{catalog_item_urn}/dq-monitors/{monitor}") + except requests.HTTPError: + return None + thresholds = mon.get("overallDqThresholds") or [] + return thresholds[0].get("value") if thresholds else None + # --- composite --- def fetch_bundle(self, catalog_item_urn: str, *, with_dq: bool = True) -> CatalogItemBundle: @@ -142,6 +155,17 @@ def fetch_bundle(self, catalog_item_urn: str, *, with_dq: bool = True) -> Catalo # A referenced term may be inaccessible; skip it rather than fail the whole run. continue - dq_results = self.get_dq_results(catalog_item_urn) if with_dq else None - - return CatalogItemBundle(item=item, attributes=attributes, terms=terms, dq_results=dq_results) + dq_results = None + dq_threshold_pct = None + if with_dq: + dq_results = self.get_dq_results(catalog_item_urn) + if dq_results is not None: + dq_threshold_pct = self.get_dq_overall_threshold(catalog_item_urn) + + return CatalogItemBundle( + item=item, + attributes=attributes, + terms=terms, + dq_results=dq_results, + dq_threshold_pct=dq_threshold_pct, + ) diff --git a/converters/ataccama/src/ataccama_osi/models.py b/converters/ataccama/src/ataccama_osi/models.py index 4085b08..0f48c19 100644 --- a/converters/ataccama/src/ataccama_osi/models.py +++ b/converters/ataccama/src/ataccama_osi/models.py @@ -142,6 +142,8 @@ class CatalogItemBundle: terms: dict[str, Term] = field(default_factory=dict) # Raw DqResults payload (overall + per-dimension + per-attribute quality), or None. dq_results: dict[str, Any] | None = None + # The monitor's configured overall DQ threshold (0-100), or None if not set. + dq_threshold_pct: float | None = None @classmethod def from_dict(cls, d: dict[str, Any]) -> CatalogItemBundle: @@ -150,6 +152,7 @@ def from_dict(cls, d: dict[str, Any]) -> CatalogItemBundle: attributes=[CatalogAttribute.from_dict(a) for a in d.get("attributes", [])], terms={urn: Term.from_dict(t) for urn, t in d.get("terms", {}).items()}, dq_results=d.get("dq_results"), + dq_threshold_pct=d.get("dq_threshold_pct"), ) def to_dict(self) -> dict[str, Any]: @@ -159,6 +162,7 @@ def to_dict(self) -> dict[str, Any]: "attributes": [_dataclass_to_raw_attr(a) for a in self.attributes], "terms": {urn: _dataclass_to_raw_term(t) for urn, t in self.terms.items()}, "dq_results": self.dq_results, + "dq_threshold_pct": self.dq_threshold_pct, } diff --git a/converters/ataccama/tests/fixtures/ataccama_bundles.json b/converters/ataccama/tests/fixtures/ataccama_bundles.json index e418716..38514c2 100644 --- a/converters/ataccama/tests/fixtures/ataccama_bundles.json +++ b/converters/ataccama/tests/fixtures/ataccama_bundles.json @@ -3204,7 +3204,8 @@ ] } ] - } + }, + "dq_threshold_pct": 75 }, { "item": { @@ -3542,6 +3543,7 @@ ] } ] - } + }, + "dq_threshold_pct": null } ] \ No newline at end of file diff --git a/converters/ataccama/tests/test_ataccama_to_osi.py b/converters/ataccama/tests/test_ataccama_to_osi.py index d2d64a5..fec2e15 100644 --- a/converters/ataccama/tests/test_ataccama_to_osi.py +++ b/converters/ataccama/tests/test_ataccama_to_osi.py @@ -15,6 +15,8 @@ from ataccama_osi.ataccama_to_osi import ( OSI_VERSION, + _dataset_dq, + _dataset_dq_warning, _quality_summary, ataccama_to_osi, attribute_to_field, @@ -115,6 +117,21 @@ def test_dataset_dq_overall_and_dimensions(document: dict) -> None: # dimensions with no evaluated records (Uniqueness/Accuracy = 0/0) are dropped assert "Uniqueness" not in dims and "Accuracy" not in dims assert dq["results_link"].startswith("https://") + # active-finding count is always reported (0 = no open issues) + assert dq["active_findings"] == 0 + + +def test_dataset_dq_threshold(document: dict) -> None: + # BANK_TRANSACTIONS has a configured overall bar of 75%; 79.1% is above it. + dq = _ext(_dataset(document, "BANK_TRANSACTIONS"))["dq"] + assert dq["threshold_pct"] == 75 + assert dq["below_threshold"] is False + + +def test_no_threshold_when_monitor_has_none(document: dict) -> None: + # aggregation's monitor has no overall threshold configured. + dq = _ext(_dataset(document, "aggregation"))["dq"] + assert "threshold_pct" not in dq and "below_threshold" not in dq def test_field_level_dq(document: dict) -> None: @@ -137,6 +154,52 @@ def test_no_dq_keys_when_dq_absent(bundles: list[CatalogItemBundle]) -> None: assert all("dq" not in _ext(f) for f in ds["fields"]) +# --- opt-in DQ AI warnings --- + + +def test_ai_warnings_off_by_default(document: dict) -> None: + # Default conversion must not inject warnings into ai_context. + ds = _dataset(document, "BANK_TRANSACTIONS") + assert "Data-quality warning" not in (ds.get("ai_context", {}).get("instructions") or "") + email = _field(ds, "EMAIL") + assert "Data-quality warning" not in email["ai_context"]["instructions"] + + +def test_ai_warning_appended_when_below_ataccama_threshold() -> None: + # A dataset Ataccama flags as below its own configured threshold gets a warning. + bundle = CatalogItemBundle( + item=CatalogItem(urn="urn:ata:t:catalog:catalog-item:1", name="risky"), + dq_results={"overallQuality": {"passedCount": 50, "failedCount": 50}, "overallDqFindings": []}, + dq_threshold_pct=75, + ) + ds = ataccama_to_osi([bundle], dq_ai_warnings=True)["semantic_model"][0]["datasets"][0] + instr = ds["ai_context"]["instructions"] + assert "Data-quality warning" in instr and "configured 75% quality threshold" in instr + + +def test_no_ai_warning_without_ataccama_signal(bundles: list[CatalogItemBundle]) -> None: + # BANK_TRANSACTIONS is above its 75% bar with no active findings; aggregation has no + # threshold configured. Neither should be flagged — Ataccama is the source of truth. + doc = ataccama_to_osi(bundles, dq_ai_warnings=True) + bank = _dataset(doc, "BANK_TRANSACTIONS")["ai_context"]["instructions"] + assert "Data-quality warning" not in bank + agg = (_dataset(doc, "aggregation").get("ai_context") or {}).get("instructions", "") + assert "Data-quality warning" not in agg + + +def test_dataset_dq_warning_triggers() -> None: + # below Ataccama's own configured threshold + below = _dataset_dq_warning({"pass_rate_pct": 60.0, "threshold_pct": 75, "below_threshold": True}) + assert below and "configured 75% quality threshold" in below + # passing, no active findings -> no warning (no converter-side fallback exists) + assert _dataset_dq_warning({"pass_rate_pct": 99.0, "active_findings": 0}) is None + # a dataset with no configured threshold is never flagged on pass rate alone + assert _dataset_dq_warning({"pass_rate_pct": 10.0}) is None + # active findings alone + af = _dataset_dq_warning({"pass_rate_pct": 99.0, "active_findings": 2}) + assert af and "2 active data-quality finding(s)" in af + + # --- helper unit tests --- @@ -166,6 +229,13 @@ def test_quality_summary(oq, expected) -> None: assert _quality_summary(oq) == expected +def test_below_threshold_flag_true() -> None: + dq = _dataset_dq({"overallQuality": {"passedCount": 60, "failedCount": 40}}, threshold_pct=75) + assert dq["pass_rate_pct"] == 60.0 + assert dq["threshold_pct"] == 75 + assert dq["below_threshold"] is True + + def test_is_time_inferred_from_datatype() -> None: attr = CatalogAttribute(urn="urn:ata:t:catalog:catalog-attribute:1", name="created_at", data_type="DATETIME") assert attribute_to_field(attr, terms={}, used=set())["dimension"] == {"is_time": True}