diff --git a/.gitignore b/.gitignore index 63e36ff..764a1be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,10 @@ **/__pycache__/ **/.venv/ +*.egg-info/ + +# Local secrets / credentials — never commit +.env +.env.local +*.env +.secrets* +.ataccama.env diff --git a/converters/ataccama/README.md b/converters/ataccama/README.md new file mode 100644 index 0000000..a0606de --- /dev/null +++ b/converters/ataccama/README.md @@ -0,0 +1,154 @@ +# Ataccama OSI Converter + +Importer from [Ataccama ONE](https://docs.ataccama.com/ataccama-one-agentic/latest/rest-api/rest-api-overview.html) +catalog metadata to the [OSI (Open Semantic Interchange)](https://github.com/open-semantic-interchange/OSI) +semantic model specification. + +Given one or more Ataccama **catalog items** (selected by URN), the converter produces +a single OSI semantic model: each catalog item becomes a dataset, each catalog attribute +becomes a field, business terms are carried across as `ai_context`, and +**data-quality scores** plus other governance metadata are attached as `ATACCAMA` +custom extensions. + +## Scope & direction + +- **Direction:** Ataccama → OSI (import only). See [Limitations](#limitations) for why + export (OSI → Ataccama) is not provided. +- **Input:** explicit catalog-item URNs. Ataccama catalogs are large (tens of thousands + of items, many of which are BI artifacts rather than tables), so the converter is + intentionally scoped to the items you name rather than crawling everything. +- **Data access:** a live REST client that authenticates via OAuth2 client-credentials + (tokens are short-lived and refreshed automatically) and follows the catalog API's + cursor pagination. + +## Configuration + +The client reads connection settings from environment variables (or a `--env-file`): + +| Variable | Description | +|---|---| +| `ATACCAMA_BASE_URL` | API root, e.g. `https:///api` | +| `ATACCAMA_TOKEN_URL` | OAuth2 token endpoint (Keycloak `.../protocol/openid-connect/token`) | +| `ATACCAMA_CLIENT_ID` | service-account client id | +| `ATACCAMA_CLIENT_SECRET` | service-account client secret | + +> Never commit credentials. Keep them in a git-ignored env file. + +## Usage + +```bash +# CLI +ataccama-to-osi --env-file .ataccama.env \ + --urn urn:ata::catalog:catalog-item: \ + --urn urn:ata::catalog:catalog-item: \ + --model-name my_model \ + --output my_model.osi.yaml + +# data-quality results are fetched by default; use --no-dq to skip them +ataccama-to-osi --env-file .ataccama.env --urn --no-dq -o my_model.osi.yaml + +# opt-in: append DQ warnings to ai_context for datasets Ataccama flags as below quality +ataccama-to-osi --env-file .ataccama.env --urn --dq-ai-warnings -o my_model.osi.yaml + +# or supply a file of URNs (one per line) +ataccama-to-osi --env-file .ataccama.env --urns-file items.txt -o my_model.osi.yaml +``` + +```python +# Library +from ataccama_osi import AtaccamaClient, ataccama_to_osi +import yaml + +client = AtaccamaClient(base_url="https:///api", token_url="...", + client_id="...", client_secret="...") +bundles = [client.fetch_bundle(urn) for urn in urns] +document = ataccama_to_osi(bundles, model_name="my_model") +print(yaml.dump(document, sort_keys=False)) +``` + +## Development + +```bash +uv sync --group dev # or: python -m venv .venv && pip install -e . pytest jsonschema +uv run pytest +``` + +Tests run fully offline against a recorded catalog fixture +(`tests/fixtures/ataccama_bundles.json`) and validate the output against the OSI schema. + +## Concept mapping + +| Ataccama ONE (Catalog API) | OSI Semantic Model | +|---|---| +| `CatalogItem` | `dataset` | +| `CatalogItem.name` | `dataset.name` (de-duplicated if repeated) | +| `CatalogItem.locations` + name | `dataset.source` (best-effort dotted namespace) | +| `CatalogItem.description` (rich text) | `dataset.description` (flattened to plain text) | +| assigned `Term`s | `dataset.ai_context` (`synonyms` + `instructions`) | +| `CatalogAttribute` | `field` | +| `CatalogAttribute.name` | `field.name` + quoted `ANSI_SQL` identifier expression | +| `CatalogAttribute.dataType` ∈ {DATE, DATETIME, TIMESTAMP, TIME} | `field.dimension.is_time = true` | +| `CatalogAttribute.description` / `comment` | `field.description` | +| attribute `Term`s | `field.ai_context` | +| DQ `overallQuality` + `dimensionResults` + findings + monitor threshold | dataset `custom_extensions` (`vendor_name: ATACCAMA`) → `data.dq` (`passed`, `failed`, `pass_rate_pct`, `threshold_pct`, `below_threshold`, `active_findings`, `dimensions[]`, `results_link`) | +| DQ per-attribute `overallQuality` | field `custom_extensions` (`vendor_name: ATACCAMA`) → `data.dq` (`passed`, `failed`, `pass_rate_pct`) | +| URNs, `dataType`, `columnType`, connection/source/stewardship/monitor | `custom_extensions` (`vendor_name: ATACCAMA`) → `data` | + +### Data quality + +Data-quality results (from the DQ API's latest processing on each item's primary +monitor) are fetched by default and attached to the `ATACCAMA` extension — overall +and per-dimension quality on the dataset, and per-column quality on each field. Pass +`--no-dq` to skip the DQ calls. + +Notes: +- `pass_rate_pct` is Ataccama's overall quality (`overallQuality`) expressed as a + percentage: `passed / (passed + failed)`. The DQ API returns only the pass/fail + **counts**, not a preformatted percentage, so the converter formats it — the counts + themselves are Ataccama's own, consistent with what the platform reports. +- `threshold_pct` / `below_threshold` come from the monitor's configured overall + threshold (`overallDqThresholds`, fetched from the monitor-config endpoint) — i.e. + the same pass/fail bar shown in Ataccama. They are omitted when the monitor has no + overall threshold configured. +- `active_findings` is always reported (`0` = no open data-quality findings), so the + clean state is an explicit signal rather than an omission. +- Dimensions with no evaluated records in the processing are omitted. +- Items without a published monitor/results simply carry no `dq` block (best-effort). + +#### AI warnings (opt-in) + +By default the DQ block is descriptive data only. With `--dq-ai-warnings`, a plain-language +warning is appended to `ai_context.instructions` for any dataset Ataccama flags as having +data quality issues (data quality below configured threshold or active findings), so a +downstream AI/BI tool reading the OSI model (not just the vendor extension) is told to +treat the data with caution. Example: + +> *Data-quality warning: 60.0% of quality checks passed on the latest run (below the +> configured 75% quality threshold). Verify this data before using it for analysis or +> automated decisions.* + +Warnings are appended to any existing (term-derived) instructions, never replacing them, +and require DQ (so not usable with `--no-dq`). + +## Limitations + +The Ataccama Catalog API is a **catalog / governance / data-quality** surface, while OSI +is an **analytics semantic model**. Some OSI constructs therefore have no source today: + +- **Metrics** — Ataccama has no analytics metrics; none are emitted. (Data-quality + scores are attached as `ATACCAMA` extensions, not as OSI metrics.) +- **Data Trust Score / Index** — not exposed by the public APIs (no precomputed + score field exists), so it is not emitted. The converter instead carries the DQ + results and monitor threshold that the APIs do provide. +- **Relationships / `primary_key` / `unique_keys`** — the Catalog API does not expose + foreign or primary keys on catalog items, so these are omitted. +- **`source` string** — the API exposes only a folder hierarchy (`locations`) and + `originPath`, not `database.schema.table`. `source` is a best-effort dotted namespace; + the authoritative connection/source URNs are preserved in `custom_extensions`. +- **Expressions are physical** — attributes map to plain quoted column identifiers + (`ANSI_SQL` only); there are no computed/multi-dialect expressions. +- **Export (OSI → Ataccama)** is not provided: the public Catalog API can only `PATCH` + `description`/stewardship/aliases on existing items and cannot create catalog items, + attributes, or relationships. +- Items with no catalogued attributes (e.g. some dashboards/reports) produce a dataset + with no `fields`. diff --git a/converters/ataccama/pyproject.toml b/converters/ataccama/pyproject.toml new file mode 100644 index 0000000..2e0b59a --- /dev/null +++ b/converters/ataccama/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "ataccama-osi" +version = "0.2.0.dev0" +description = "Importer from Ataccama ONE catalog metadata to the OSI semantic model" +authors = [{name = "Ataccama", email = "support@ataccama.com"}] +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "pyyaml>=6.0.1", + "requests>=2.31.0", +] + +[project.scripts] +ataccama-to-osi = "ataccama_osi.cli:main" + +[dependency-groups] +dev = [ + "ruff>=0.11.5", + "pytest>=8.3.5", + "pytest-cov>=6.0.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +include = ["src/ataccama_osi/**/*"] +ignore-vcs = true + +[tool.hatch.build.targets.wheel] +packages = ["src/ataccama_osi"] + +[tool.ruff] +line-length = 120 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "UP", "W", "C4", "PIE", "SIM"] +ignore = ["E501"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/converters/ataccama/src/ataccama_osi/__init__.py b/converters/ataccama/src/ataccama_osi/__init__.py new file mode 100644 index 0000000..82aee37 --- /dev/null +++ b/converters/ataccama/src/ataccama_osi/__init__.py @@ -0,0 +1,14 @@ +"""Ataccama ONE -> OSI semantic model importer.""" + +from ataccama_osi.ataccama_to_osi import ataccama_to_osi +from ataccama_osi.client import AtaccamaClient +from ataccama_osi.models import CatalogAttribute, CatalogItem, CatalogItemBundle, Term + +__all__ = [ + "AtaccamaClient", + "CatalogAttribute", + "CatalogItem", + "CatalogItemBundle", + "Term", + "ataccama_to_osi", +] diff --git a/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py b/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py new file mode 100644 index 0000000..a8e9056 --- /dev/null +++ b/converters/ataccama/src/ataccama_osi/ataccama_to_osi.py @@ -0,0 +1,366 @@ +"""Convert Ataccama ONE catalog metadata to an OSI semantic model. + +Scope: the caller supplies one or more catalog items (as :class:`CatalogItemBundle`s); +each becomes an OSI dataset, and the collection becomes a single OSI semantic model. + +Mapping summary (see README for the full table and known limitations): + CatalogItem -> dataset (name, source, description, ai_context, custom_extensions) + CatalogAttribute -> field (name, ANSI_SQL identifier expression, dimension.is_time, + description, ai_context, custom_extensions) + Term / termAssignments -> ai_context.synonyms + ATACCAMA custom_extension +""" + +from __future__ import annotations + +import json +from typing import Any + +from ataccama_osi.models import CatalogAttribute, CatalogItem, CatalogItemBundle, Term + +OSI_VERSION = "0.2.0.dev0" +VENDOR = "ATACCAMA" + +# Ataccama semantic data types that represent points in time. +TIME_DATA_TYPES = {"DATE", "DATETIME", "TIMESTAMP", "TIME"} + + +# --- helpers ------------------------------------------------------------- + + +def flatten_richtext(value: Any) -> str | None: + """Flatten an Ataccama description into plain text. + + Descriptions arrive either as a plain string or as Slate rich-text JSON, e.g. + ``[{"type": "paragraph", "children": [{"text": "..."}]}]`` (sometimes as a JSON + string). Returns ``None`` when there is no usable text. + """ + if value is None: + return None + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return None + # Rich text is occasionally delivered as a JSON-encoded string. + if stripped[0] in "[{": + try: + return flatten_richtext(json.loads(stripped)) + except (ValueError, TypeError): + return stripped + return stripped + + def collect(node: Any, out: list[str]) -> None: + if isinstance(node, dict): + text = node.get("text") + if isinstance(text, str): + out.append(text) + for child in node.get("children", []) or []: + collect(child, out) + elif isinstance(node, list): + for child in node: + collect(child, out) + + parts: list[str] = [] + collect(value, parts) + text = "".join(parts).strip() + return text or None + + +def _unique_name(desired: str, used: set[str], *, fallback: str) -> str: + """Return a name unique within ``used`` (OSI requires unique dataset/field names).""" + base = desired.strip() or fallback + candidate = base + n = 2 + while candidate in used: + candidate = f"{base}_{n}" + n += 1 + used.add(candidate) + return candidate + + +def _sql_identifier(name: str) -> str: + """Quote a column name as an ANSI SQL identifier (names may contain spaces/punctuation).""" + escaped = name.replace('"', '""') + return f'"{escaped}"' + + +def build_source(item: CatalogItem) -> str: + """Best-effort physical source reference. + + The Catalog API does not expose ``database.schema.table``; it only gives a + folder-name hierarchy (``locations``, leaf-first) plus the item name. We build a + dotted namespace from the reversed locations followed by the item name. The + authoritative connection/source URNs are preserved in custom_extensions. + """ + parts = [loc.name for loc in reversed(item.locations) if loc.name] + parts.append(item.name) + return ".".join(parts) if parts else item.name + + +def _terms_ai_context(term_urns: list[str], terms: dict[str, Term]) -> dict[str, Any] | None: + """Build an ai_context object from assigned business terms.""" + synonyms: list[str] = [] + instructions: list[str] = [] + for urn in term_urns: + term = terms.get(urn) + if not term: + continue + if term.name and term.name not in synonyms: + synonyms.append(term.name) + desc = flatten_richtext(term.description) + if desc: + instructions.append(f"{term.name}: {desc}" if term.name else desc) + ctx: dict[str, Any] = {} + if instructions: + ctx["instructions"] = " ".join(instructions) + if synonyms: + ctx["synonyms"] = synonyms + return ctx or None + + +def _ataccama_extension(data: dict[str, Any]) -> dict[str, str]: + """Wrap Ataccama-specific metadata as an OSI custom_extension entry.""" + return {"vendor_name": VENDOR, "data": json.dumps(data, sort_keys=True)} + + +# --- data quality --- + + +def _quality_summary(overall_quality: dict[str, Any] | None) -> dict[str, Any] | None: + """Turn an OverallQuality {passedCount, failedCount} into a compact summary. + + ``pass_rate_pct`` is Ataccama's overall quality expressed as a percentage + (passed / (passed + failed)). The DQ API returns only counts, not a preformatted + percentage, so the converter formats it; the underlying counts are Ataccama's own. + """ + if not overall_quality: + return None + passed = overall_quality.get("passedCount") or 0 + failed = overall_quality.get("failedCount") or 0 + total = passed + failed + summary: dict[str, Any] = {"passed": passed, "failed": failed} + if total: + summary["pass_rate_pct"] = round(passed / total * 100, 1) + return summary + + +def _dataset_dq(dq_results: dict[str, Any] | None, threshold_pct: float | None = None) -> dict[str, Any] | None: + """Build the dataset-level DQ block from a raw DqResults payload. + + ``threshold_pct`` is the monitor's configured overall pass/fail bar (from the + monitor config). When present alongside a pass rate, ``below_threshold`` records + whether the data is under that bar — the same pass/fail state shown in Ataccama. + """ + if not dq_results: + return None + dq = _quality_summary(dq_results.get("overallQuality")) or {} + + if threshold_pct is not None: + dq["threshold_pct"] = threshold_pct + if "pass_rate_pct" in dq: + dq["below_threshold"] = dq["pass_rate_pct"] < threshold_pct + + dimensions = [] + for dim in dq_results.get("dimensionResults", []) or []: + passed = dim.get("passedCount") or 0 + failed = dim.get("failedCount") or 0 + if passed + failed == 0: # dimension not evaluated in this processing + continue + entry = {"name": (dim.get("dimension") or {}).get("name"), "passed": passed, "failed": failed} + entry["pass_rate_pct"] = round(passed / (passed + failed) * 100, 1) + dimensions.append(entry) + if dimensions: + dq["dimensions"] = dimensions + + # Always report the active-finding count (0 = no open data-quality issues) so the + # "clean" state is an explicit signal, not just an omission. + dq["active_findings"] = sum( + 1 for f in dq_results.get("overallDqFindings", []) or [] if f.get("status") == "ACTIVE" + ) + if dq_results.get("processingUrn"): + dq["processing_urn"] = dq_results["processingUrn"] + if dq_results.get("dqResultsLink"): + dq["results_link"] = dq_results["dqResultsLink"] + + return dq or None + + +def _attribute_dq_index(dq_results: dict[str, Any] | None) -> dict[str, dict[str, Any]]: + """Map attributeUrn -> quality summary from a raw DqResults payload.""" + index: dict[str, dict[str, Any]] = {} + if not dq_results: + return index + for attr_result in dq_results.get("dqAttributeResults", []) or []: + urn = attr_result.get("attributeUrn") + summary = _quality_summary(attr_result.get("overallQuality")) + if urn and summary: + index[urn] = summary + return index + + +# --- optional AI warnings driven by DQ --- + + +def _dataset_dq_warning(dq: dict[str, Any] | None) -> str | None: + """A natural-language warning for AI consumers when a dataset is below quality. + + Fires only on Ataccama's own signals — below the monitor's configured threshold, or + an active finding — so Ataccama remains the sole source of truth. If neither applies + (including when no threshold is configured), no warning is produced. + """ + if not dq: + return None + reasons: list[str] = [] + if dq.get("below_threshold") is True: + reasons.append(f"below the configured {dq.get('threshold_pct')}% quality threshold") + if dq.get("active_findings"): + reasons.append(f"{dq['active_findings']} active data-quality finding(s)") + if not reasons: + return None + pass_rate = dq.get("pass_rate_pct") + prefix = f"{pass_rate}% of quality checks passed on the latest run" if pass_rate is not None else "quality checks failed" + return ( + f"Data-quality warning: {prefix} ({'; '.join(reasons)}). " + "Verify this data before using it for analysis or automated decisions." + ) + + +def _inject_warning(ai_context: dict[str, Any] | None, warning: str | None) -> dict[str, Any] | None: + """Append a warning to an ai_context object's instructions (creating one if needed).""" + if not warning: + return ai_context + if ai_context is None: + return {"instructions": warning} + ctx = dict(ai_context) if isinstance(ai_context, dict) else {"instructions": str(ai_context)} + existing = ctx.get("instructions") + ctx["instructions"] = f"{existing} {warning}" if existing else warning + return ctx + + +# --- attribute -> field -------------------------------------------------- + + +def attribute_to_field( + attr: CatalogAttribute, + terms: dict[str, Term], + used: set[str], + dq_by_attr: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any]: + name = _unique_name(attr.name, used, fallback="column") + field_obj: dict[str, Any] = { + "name": name, + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": _sql_identifier(attr.name or name)}]}, + } + + if attr.data_type and attr.data_type.upper() in TIME_DATA_TYPES: + field_obj["dimension"] = {"is_time": True} + + description = flatten_richtext(attr.description) or flatten_richtext(attr.comment) + if description: + field_obj["description"] = description + + field_dq = dq_by_attr.get(attr.urn) if dq_by_attr else None + + ai_context = _terms_ai_context([ta.term_urn for ta in attr.term_assignments], terms) + if ai_context: + field_obj["ai_context"] = ai_context + + # Preserve source-system typing and the attribute URN for round-tripping. + ext_data: dict[str, Any] = {"attribute_urn": attr.urn} + if attr.data_type: + ext_data["data_type"] = attr.data_type + if attr.column_type: + ext_data["column_type"] = attr.column_type + if attr.term_assignments: + ext_data["term_urns"] = [ta.term_urn for ta in attr.term_assignments] + if field_dq: + ext_data["dq"] = field_dq + field_obj["custom_extensions"] = [_ataccama_extension(ext_data)] + + return field_obj + + +# --- catalog item -> dataset --------------------------------------------- + + +def bundle_to_dataset( + bundle: CatalogItemBundle, + used_dataset_names: set[str], + dq_ai_warnings: bool = False, +) -> dict[str, Any]: + item = bundle.item + name = _unique_name(item.name, used_dataset_names, fallback="dataset") + + dataset: dict[str, Any] = {"name": name, "source": build_source(item)} + + description = flatten_richtext(item.description) + if description: + dataset["description"] = description + + dataset_dq = _dataset_dq(bundle.dq_results, bundle.dq_threshold_pct) + + ai_context = _terms_ai_context([ta.term_urn for ta in item.term_assignments], bundle.terms) + if dq_ai_warnings: + ai_context = _inject_warning(ai_context, _dataset_dq_warning(dataset_dq)) + if ai_context: + dataset["ai_context"] = ai_context + + dq_by_attr = _attribute_dq_index(bundle.dq_results) + used_field_names: set[str] = set() + fields = [attribute_to_field(attr, bundle.terms, used_field_names, dq_by_attr) for attr in bundle.attributes] + if fields: + dataset["fields"] = fields + + # Preserve everything with no OSI-core home so the model round-trips. + ext_data: dict[str, Any] = {"catalog_item_urn": item.urn} + if item.connection_urn: + ext_data["connection_urn"] = item.connection_urn + if item.source_urn: + ext_data["source_urn"] = item.source_urn + if item.origin_path: + ext_data["origin_path"] = item.origin_path + if item.locations: + ext_data["locations"] = [loc.name for loc in item.locations] + if item.stewardship_group_urn: + ext_data["stewardship_group_urn"] = item.stewardship_group_urn + if item.primary_dq_monitor_urn: + ext_data["primary_dq_monitor_urn"] = item.primary_dq_monitor_urn + if item.term_assignments: + ext_data["term_urns"] = [ta.term_urn for ta in item.term_assignments] + if item.aliases: + ext_data["aliases"] = item.aliases + if dataset_dq: + ext_data["dq"] = dataset_dq + dataset["custom_extensions"] = [_ataccama_extension(ext_data)] + + return dataset + + +# --- top level ----------------------------------------------------------- + + +def ataccama_to_osi( + bundles: list[CatalogItemBundle], + model_name: str = "ataccama_model", + model_description: str | None = None, + tenant: str | None = None, + dq_ai_warnings: bool = False, +) -> dict[str, Any]: + """Convert a list of catalog-item bundles into an OSI document dict (ready for YAML). + + When ``dq_ai_warnings`` is set, datasets that Ataccama flags as below quality (below + the configured threshold, or with active findings) get a natural-language warning + appended to their ``ai_context.instructions`` (see ``_dataset_dq_warning``). + """ + used_dataset_names: set[str] = set() + datasets = [bundle_to_dataset(b, used_dataset_names, dq_ai_warnings) for b in bundles] + + semantic_model: dict[str, Any] = {"name": model_name, "datasets": datasets} + if model_description: + semantic_model["description"] = model_description + + model_ext: dict[str, Any] = {"source": "ataccama-one-catalog"} + if tenant: + model_ext["tenant"] = tenant + semantic_model["custom_extensions"] = [_ataccama_extension(model_ext)] + + return {"version": OSI_VERSION, "semantic_model": [semantic_model]} diff --git a/converters/ataccama/src/ataccama_osi/cli.py b/converters/ataccama/src/ataccama_osi/cli.py new file mode 100644 index 0000000..5899429 --- /dev/null +++ b/converters/ataccama/src/ataccama_osi/cli.py @@ -0,0 +1,125 @@ +"""CLI: import selected Ataccama catalog items into an OSI semantic model. + +Scope is defined by explicit catalog-item URNs (``--urn`` repeated, or ``--urns-file``). + +Configuration is read from environment variables (optionally loaded from an env file +via ``--env-file``): + + ATACCAMA_BASE_URL API root, e.g. https:///api + ATACCAMA_TOKEN_URL OAuth2 token endpoint (Keycloak openid-connect/token) + ATACCAMA_CLIENT_ID service-account client id + ATACCAMA_CLIENT_SECRET service-account client secret + +Example: + ataccama-to-osi --env-file .ataccama.env \\ + --urn urn:ata:tenant:catalog:catalog-item:... \\ + --model-name my_model --output my_model.osi.yaml +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +import yaml + +from ataccama_osi.ataccama_to_osi import ataccama_to_osi +from ataccama_osi.client import AtaccamaClient + +REQUIRED_VARS = ("ATACCAMA_BASE_URL", "ATACCAMA_TOKEN_URL", "ATACCAMA_CLIENT_ID", "ATACCAMA_CLIENT_SECRET") + + +def _load_env_file(path: Path) -> None: + """Load simple KEY=VALUE lines into os.environ (does not overwrite existing vars).""" + for raw in path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + os.environ.setdefault(key.strip(), value.strip()) + + +def _tenant_from_urn(urn: str) -> str | None: + # urn:ata:::: + parts = urn.split(":") + return parts[2] if len(parts) > 2 else None + + +def _collect_urns(args: argparse.Namespace) -> list[str]: + urns: list[str] = list(args.urn or []) + if args.urns_file: + for raw in Path(args.urns_file).read_text().splitlines(): + line = raw.strip() + if line and not line.startswith("#"): + urns.append(line) + # de-duplicate, preserve order + seen: set[str] = set() + return [u for u in urns if not (u in seen or seen.add(u))] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Import Ataccama ONE catalog items into an OSI semantic model.") + parser.add_argument("--urn", action="append", help="Catalog-item URN to convert (repeatable).") + parser.add_argument("--urns-file", help="File with one catalog-item URN per line.") + parser.add_argument("--model-name", default="ataccama_model", help="OSI semantic model name.") + parser.add_argument("--model-description", default=None, help="OSI semantic model description.") + parser.add_argument("--output", "-o", default="-", help="Output YAML path ('-' for stdout).") + parser.add_argument("--env-file", help="Optional KEY=VALUE file with Ataccama connection config.") + parser.add_argument( + "--no-dq", + action="store_true", + help="Skip fetching data-quality results (DQ enrichment is included by default).", + ) + parser.add_argument( + "--dq-ai-warnings", + action="store_true", + help="Append a data-quality warning to ai_context.instructions for datasets that " + "Ataccama flags as below quality — below its configured threshold, or with active " + "findings (opt-in; requires DQ, so not with --no-dq).", + ) + args = parser.parse_args(argv) + + if args.env_file: + _load_env_file(Path(args.env_file)) + + urns = _collect_urns(args) + if not urns: + parser.error("provide at least one catalog-item URN via --urn or --urns-file") + + missing = [v for v in REQUIRED_VARS if not os.environ.get(v)] + if missing: + parser.error(f"missing required configuration: {', '.join(missing)}") + + client = AtaccamaClient( + base_url=os.environ["ATACCAMA_BASE_URL"], + token_url=os.environ["ATACCAMA_TOKEN_URL"], + client_id=os.environ["ATACCAMA_CLIENT_ID"], + client_secret=os.environ["ATACCAMA_CLIENT_SECRET"], + ) + + bundles = [] + for urn in urns: + print(f"Fetching {urn} ...", file=sys.stderr) + bundles.append(client.fetch_bundle(urn, with_dq=not args.no_dq)) + + document = ataccama_to_osi( + bundles, + model_name=args.model_name, + model_description=args.model_description, + tenant=_tenant_from_urn(urns[0]), + dq_ai_warnings=args.dq_ai_warnings, + ) + + text = yaml.dump(document, default_flow_style=False, sort_keys=False, allow_unicode=True) + if args.output == "-": + sys.stdout.write(text) + else: + Path(args.output).write_text(text) + print(f"Wrote {args.output}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/converters/ataccama/src/ataccama_osi/client.py b/converters/ataccama/src/ataccama_osi/client.py new file mode 100644 index 0000000..0b9ea0d --- /dev/null +++ b/converters/ataccama/src/ataccama_osi/client.py @@ -0,0 +1,171 @@ +"""Live REST client for the Ataccama ONE public API. + +Handles OAuth2 client-credentials auth (tokens are short-lived, ~5 min, so we mint +and refresh them automatically) and cursor-based pagination (``after``/``size``). +""" + +from __future__ import annotations + +import time +from typing import Any, Iterator + +import requests + +from ataccama_osi.models import CatalogAttribute, CatalogItem, CatalogItemBundle, Term + +DEFAULT_PAGE_SIZE = 200 +# Refresh the token this many seconds before it actually expires. +TOKEN_EXPIRY_SKEW_S = 30 + + +class AtaccamaClient: + """Minimal client for the endpoints the OSI importer needs.""" + + def __init__( + self, + base_url: str, + token_url: str, + client_id: str, + client_secret: str, + *, + timeout: float = 30.0, + session: requests.Session | None = None, + ) -> None: + # base_url is the API root, e.g. https:///api + self.base_url = base_url.rstrip("/") + self.token_url = token_url + self.client_id = client_id + self.client_secret = client_secret + self.timeout = timeout + self._session = session or requests.Session() + self._token: str | None = None + self._token_expiry: float = 0.0 + + # --- auth --- + + def _access_token(self) -> str: + if self._token and time.time() < self._token_expiry - TOKEN_EXPIRY_SKEW_S: + return self._token + resp = self._session.post( + self.token_url, + data={ + "grant_type": "client_credentials", + "client_id": self.client_id, + "client_secret": self.client_secret, + }, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=self.timeout, + ) + resp.raise_for_status() + payload = resp.json() + self._token = payload["access_token"] + self._token_expiry = time.time() + float(payload.get("expires_in", 300)) + return self._token + + # --- low-level HTTP --- + + def _get(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + resp = self._session.get( + f"{self.base_url}{path}", + params=params, + headers={"Authorization": f"Bearer {self._access_token()}"}, + timeout=self.timeout, + ) + resp.raise_for_status() + return resp.json() + + def _paged(self, path: str, params: dict[str, Any] | None = None) -> Iterator[dict[str, Any]]: + """Iterate every record across pages, following the ``meta.next`` cursor.""" + params = dict(params or {}) + params.setdefault("size", DEFAULT_PAGE_SIZE) + after: str | None = None + while True: + if after is not None: + params["after"] = after + body = self._get(path, params) + yield from body.get("data", []) + after = (body.get("meta") or {}).get("next") + if not after: + break + + # --- catalog endpoints --- + + def get_catalog_item(self, urn: str) -> CatalogItem: + return CatalogItem.from_dict(self._get(f"/catalog/v1/catalog-items/{urn}")) + + def get_attributes(self, catalog_item_urn: str) -> list[CatalogAttribute]: + return [ + CatalogAttribute.from_dict(a) + for a in self._paged("/catalog/v1/attributes", {"catalogItemUrn": catalog_item_urn}) + ] + + def get_term(self, urn: str) -> Term: + return Term.from_dict(self._get(f"/catalog/v1/terms/{urn}")) + + # --- data-quality endpoint --- + + def get_dq_results( + self, catalog_item_urn: str, monitor: str = "primary", processing: str = "latest" + ) -> dict | None: + """Latest DQ results for a catalog item's primary monitor. + + Returns the raw ``DqResults`` payload (overall + per-dimension + per-attribute + quality), or ``None`` if the item has no monitor / published results. + """ + path = ( + f"/data-quality/v1/catalog-items/{catalog_item_urn}" + f"/dq-monitors/{monitor}/processings/{processing}/dq-results" + ) + try: + return self._get(path) + except requests.HTTPError: + return None + + def get_dq_overall_threshold(self, catalog_item_urn: str, monitor: str = "primary") -> float | None: + """The monitor's configured overall DQ threshold (0-100), or None if unset. + + This is the pass/fail bar shown in Ataccama, and is only available from the + monitor-config endpoint (not the DQ results payload). + """ + try: + mon = self._get(f"/data-quality/v1/catalog-items/{catalog_item_urn}/dq-monitors/{monitor}") + except requests.HTTPError: + return None + thresholds = mon.get("overallDqThresholds") or [] + return thresholds[0].get("value") if thresholds else None + + # --- composite --- + + def fetch_bundle(self, catalog_item_urn: str, *, with_dq: bool = True) -> CatalogItemBundle: + """Fetch a catalog item, its attributes, referenced terms, and (by default) DQ results.""" + item = self.get_catalog_item(catalog_item_urn) + attributes = self.get_attributes(catalog_item_urn) + + term_urns: set[str] = {ta.term_urn for ta in item.term_assignments} + for attr in attributes: + term_urns.update(ta.term_urn for ta in attr.term_assignments) + + terms: dict[str, Term] = {} + for urn in sorted(term_urns): + if not urn: + continue + try: + terms[urn] = self.get_term(urn) + except requests.HTTPError: + # A referenced term may be inaccessible; skip it rather than fail the whole run. + continue + + dq_results = None + dq_threshold_pct = None + if with_dq: + dq_results = self.get_dq_results(catalog_item_urn) + if dq_results is not None: + dq_threshold_pct = self.get_dq_overall_threshold(catalog_item_urn) + + return CatalogItemBundle( + item=item, + attributes=attributes, + terms=terms, + dq_results=dq_results, + dq_threshold_pct=dq_threshold_pct, + ) diff --git a/converters/ataccama/src/ataccama_osi/models.py b/converters/ataccama/src/ataccama_osi/models.py new file mode 100644 index 0000000..0f48c19 --- /dev/null +++ b/converters/ataccama/src/ataccama_osi/models.py @@ -0,0 +1,199 @@ +"""Data models for Ataccama ONE catalog metadata and the pieces of OSI we emit. + +Ataccama Catalog API structure (public REST API, /catalog/v1): + CatalogItem + ├── urn, name, description (Slate rich-text JSON), originPath + ├── connection {urn} + ├── source {urn} + ├── locations[] {name, description} # folder-name hierarchy (leaf-first) + ├── termAssignments[] {termUrn} + ├── effectiveStewardship {groupUrn} + ├── primaryDqMonitor {urn} + └── aliases[] {type, value} + CatalogAttribute + ├── urn, catalogItemUrn, name + ├── description (Slate rich-text JSON) + ├── dataType (STRING, LONG, INTEGER, FLOAT, DOUBLE, BOOLEAN, DATE, DATETIME, ...) + ├── columnType (raw source-system type, e.g. VARCHAR, R8) + ├── comment + └── termAssignments[] {termUrn} + Term + ├── urn, name, type (businessTerm, securityTerm) + └── description (Slate rich-text JSON) + +OSI semantic model structure (what we produce): + version, semantic_model[] + ├── name, description, ai_context, custom_extensions[] + └── datasets[] {name, source, description, ai_context, fields[], custom_extensions[]} + └── fields[] {name, expression{dialects[]}, dimension{is_time}, description, + ai_context, custom_extensions[]} +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +# --- Ataccama Catalog types --- + + +@dataclass +class TermAssignment: + term_urn: str + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> TermAssignment: + return cls(term_urn=d.get("termUrn", "")) + + +@dataclass +class CatalogLocation: + name: str + description: str | None = None + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> CatalogLocation: + return cls(name=d.get("name", ""), description=d.get("description")) + + +@dataclass +class CatalogAttribute: + urn: str + name: str + catalog_item_urn: str = "" + description: Any = None + data_type: str | None = None + column_type: str | None = None + comment: str | None = None + term_assignments: list[TermAssignment] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> CatalogAttribute: + return cls( + urn=d.get("urn", ""), + name=d.get("name", ""), + catalog_item_urn=d.get("catalogItemUrn", ""), + description=d.get("description"), + data_type=d.get("dataType"), + column_type=d.get("columnType"), + comment=d.get("comment"), + term_assignments=[TermAssignment.from_dict(t) for t in d.get("termAssignments", [])], + ) + + +@dataclass +class CatalogItem: + urn: str + name: str + description: Any = None + origin_path: str | None = None + connection_urn: str | None = None + source_urn: str | None = None + locations: list[CatalogLocation] = field(default_factory=list) + term_assignments: list[TermAssignment] = field(default_factory=list) + stewardship_group_urn: str | None = None + primary_dq_monitor_urn: str | None = None + aliases: list[dict[str, Any]] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> CatalogItem: + connection = d.get("connection") or {} + source = d.get("source") or {} + stewardship = d.get("effectiveStewardship") or {} + monitor = d.get("primaryDqMonitor") or {} + return cls( + urn=d.get("urn", ""), + name=d.get("name", ""), + description=d.get("description"), + origin_path=d.get("originPath"), + connection_urn=connection.get("urn"), + source_urn=source.get("urn"), + locations=[CatalogLocation.from_dict(loc) for loc in d.get("locations", [])], + term_assignments=[TermAssignment.from_dict(t) for t in d.get("termAssignments", [])], + stewardship_group_urn=stewardship.get("groupUrn"), + primary_dq_monitor_urn=monitor.get("urn"), + aliases=list(d.get("aliases", [])), + ) + + +@dataclass +class Term: + urn: str + name: str + type: str | None = None + description: Any = None + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> Term: + return cls( + urn=d.get("urn", ""), + name=d.get("name", ""), + type=d.get("type"), + description=d.get("description"), + ) + + +@dataclass +class CatalogItemBundle: + """A catalog item together with everything needed to convert it to a dataset.""" + + item: CatalogItem + attributes: list[CatalogAttribute] = field(default_factory=list) + terms: dict[str, Term] = field(default_factory=dict) + # Raw DqResults payload (overall + per-dimension + per-attribute quality), or None. + dq_results: dict[str, Any] | None = None + # The monitor's configured overall DQ threshold (0-100), or None if not set. + dq_threshold_pct: float | None = None + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> CatalogItemBundle: + return cls( + item=CatalogItem.from_dict(d["item"]), + attributes=[CatalogAttribute.from_dict(a) for a in d.get("attributes", [])], + terms={urn: Term.from_dict(t) for urn, t in d.get("terms", {}).items()}, + dq_results=d.get("dq_results"), + dq_threshold_pct=d.get("dq_threshold_pct"), + ) + + def to_dict(self) -> dict[str, Any]: + """Serialize back to the raw-JSON shape (used for recording test fixtures).""" + return { + "item": _dataclass_to_raw_item(self.item), + "attributes": [_dataclass_to_raw_attr(a) for a in self.attributes], + "terms": {urn: _dataclass_to_raw_term(t) for urn, t in self.terms.items()}, + "dq_results": self.dq_results, + "dq_threshold_pct": self.dq_threshold_pct, + } + + +def _dataclass_to_raw_item(item: CatalogItem) -> dict[str, Any]: + return { + "urn": item.urn, + "name": item.name, + "description": item.description, + "originPath": item.origin_path, + "connection": {"urn": item.connection_urn} if item.connection_urn else None, + "source": {"urn": item.source_urn} if item.source_urn else None, + "locations": [{"name": loc.name, "description": loc.description} for loc in item.locations], + "termAssignments": [{"termUrn": t.term_urn} for t in item.term_assignments], + "effectiveStewardship": ({"groupUrn": item.stewardship_group_urn} if item.stewardship_group_urn else None), + "primaryDqMonitor": ({"urn": item.primary_dq_monitor_urn} if item.primary_dq_monitor_urn else None), + "aliases": item.aliases, + } + + +def _dataclass_to_raw_attr(attr: CatalogAttribute) -> dict[str, Any]: + return { + "urn": attr.urn, + "catalogItemUrn": attr.catalog_item_urn, + "name": attr.name, + "description": attr.description, + "dataType": attr.data_type, + "columnType": attr.column_type, + "comment": attr.comment, + "termAssignments": [{"termUrn": t.term_urn} for t in attr.term_assignments], + } + + +def _dataclass_to_raw_term(term: Term) -> dict[str, Any]: + return {"urn": term.urn, "name": term.name, "type": term.type, "description": term.description} diff --git a/converters/ataccama/tests/__init__.py b/converters/ataccama/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/converters/ataccama/tests/fixtures/ataccama_bundles.json b/converters/ataccama/tests/fixtures/ataccama_bundles.json new file mode 100644 index 0000000..38514c2 --- /dev/null +++ b/converters/ataccama/tests/fixtures/ataccama_bundles.json @@ -0,0 +1,3549 @@ +[ + { + "item": { + "urn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "BANK_TRANSACTIONS", + "description": "[{\"type\": \"paragraph\", \"children\": [{\"text\": \"\"}, {\"type\": \"a\", \"url\": \"https://ataccama-partner.atlan.com/assets/11111111-1111-1111-1111-000000000001/overview\", \"children\": [{\"text\": \"View in Atlan\"}]}, {\"text\": \"\"}]}]", + "originPath": "CLOUD_DEMO/BANK_TRANSACTIONS", + "connection": { + "urn": "urn:ata:example-tenant:processing-center:connection:00000000-0000-0000-0000-000000000001" + }, + "source": { + "urn": "urn:ata:example-tenant:catalog:source:00000000-0000-0000-0000-000000000001" + }, + "locations": [ + { + "name": "CLOUD_DEMO", + "description": null + }, + { + "name": "DEMO_ENV", + "description": null + } + ], + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001" + }, + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000002" + } + ], + "effectiveStewardship": { + "groupUrn": "urn:ata:example-tenant:iam:group:00000000-0000-0000-0000-000000000001" + }, + "primaryDqMonitor": { + "urn": "urn:ata:example-tenant:data-quality:dq-monitor:00000000-0000-0000-0000-000000000001" + }, + "aliases": [] + }, + "attributes": [ + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000001", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "TRANSACTION_ID", + "description": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000002", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "CUSTOMER_ID", + "description": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000003" + }, + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000004" + }, + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000005" + } + ] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000003", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "CUSTOMER_NAME", + "description": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000006" + }, + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000003" + } + ] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000004", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "EMAIL", + "description": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000007" + } + ] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000005", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "BIRTH_DATE", + "description": null, + "dataType": "DATE", + "columnType": "DATE", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000008" + } + ] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000006", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "COUNTRY_CODE", + "description": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000009" + } + ] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000007", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "ACCOUNT_BALANCE", + "description": null, + "dataType": "FLOAT", + "columnType": "NUMBER", + "comment": "", + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000008", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "ORDER_AMOUNT", + "description": null, + "dataType": "FLOAT", + "columnType": "NUMBER", + "comment": "", + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000009", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "APPROVAL_STATUS", + "description": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000010", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "PAYMENT_METHOD", + "description": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000011", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "TRANSACTION_DATE", + "description": null, + "dataType": "DATE", + "columnType": "DATE", + "comment": "", + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000012", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "CREATED_BY", + "description": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000013", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000001", + "name": "RISK_RATING", + "description": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": "", + "termAssignments": [ + { + "termUrn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000010" + } + ] + } + ], + "terms": { + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000002": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000002", + "name": "Finance", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"Finance is a term for matters regarding the management, creation, and study of money and investments. It involves the use of credit and debt, securities, and investment to finance current projects using future income flows. Because of this temporal aspect, finance is closely linked to the time value of money, interest rates, and other related topics. In this instance, Finance is effectively a Data Domain comprising several business terms under itself.\"}]}]" + }, + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000008": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000008", + "name": "Date of Birth", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"The date on which the person was born.\"}]}]" + }, + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000005": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000005", + "name": "Customer ID", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"Customer ID\"}]}]" + }, + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000001", + "name": "Personal Data", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"Personal information might be subject to GDPR, CCPA or other privacy and data protection regulations.\"}]}]" + }, + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000010": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000010", + "name": "Surname", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"A person's name in joint with other family members as distinguished from a first or given name.\"}]}]" + }, + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000009": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000009", + "name": "ISO-2 Country Code", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"It was first published in 1998. ISO 3166-2 aims to establish an international standard of short and unique alphanumeric codes to represent the relevant administrative divisions and dependent territories of all countries in a more convenient and less ambiguous form than their full names.\"}]}]" + }, + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000007": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000007", + "name": "E-mail", + "type": "businessTerm", + "description": "[{\"type\":\"p\",\"children\":[{\"text\":\"An email address serves as a unique identifier for sending and receiving electronic communications within the organization. It ensures accurate message delivery and supports efficient internal and external communication. As it constitutes \"},{\"type\":\"entityReference\",\"gid\":\"11111111-1111-1111-1111-000000000002\",\"entityName\":\"term\",\"children\":[{\"text\":\"Personal Data\"}]},{\"text\":\" under the \"},{\"type\":\"entityReference\",\"gid\":\"11111111-1111-1111-1111-000000000003\",\"entityName\":\"term\",\"children\":[{\"text\":\"Data Protection Policy\"}]},{\"text\":\", it \"},{\"text\":\"must be used lawfully and only for legitimate business purposes\",\"bold\":true},{\"text\":\".\"}]},{\"type\":\"p\",\"children\":[{\"text\":\"In line with the \"},{\"type\":\"entityReference\",\"gid\":\"11111111-1111-1111-1111-000000000004\",\"entityName\":\"term\",\"children\":[{\"text\":\"Data Access & Control Policy\"}]},{\"text\":\", email data should be accessed solely by authorized personnel and handled securely.\"}]},{\"type\":\"p\",\"children\":[{\"text\":\"According to the \"},{\"type\":\"entityReference\",\"gid\":\"11111111-1111-1111-1111-000000000005\",\"entityName\":\"term\",\"children\":[{\"text\":\"Information Security Policy\"}]},{\"text\":\", all email accounts must be protected through strong passwords and multi-factor authentication to prevent unauthorized access or misuse.\"}]},{\"type\":\"p\",\"children\":[{\"text\":\"Email records should also adhere to the \"},{\"type\":\"entityReference\",\"gid\":\"11111111-1111-1111-1111-000000000006\",\"entityName\":\"term\",\"children\":[{\"text\":\"Data Retention Policy\"}]},{\"text\":\", being stored only for as long as necessary for operational or legal purposes.\"}]},{\"type\":\"p\",\"children\":[{\"text\":\"Proper management of \"},{\"text\":\"email addresses ensures compliance, data integrity, and secure communication\",\"bold\":true},{\"text\":\" across the organization.\"}]},{\"type\":\"paragraph\",\"children\":[{\"text\":\"\"}]}]" + }, + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000003": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000003", + "name": "TB Test Term", + "type": "businessTerm", + "description": "[{\"type\":\"paragraph\",\"children\":[{\"text\":\"Detect \\\"CUSTOMER\\\" in column name\"}]}]" + }, + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000004": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000004", + "name": "Reference Term", + "type": "policyTerm", + "description": null + }, + "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000006": { + "urn": "urn:ata:example-tenant:catalog:term:00000000-0000-0000-0000-000000000006", + "name": "Hide data", + "type": "policyTerm", + "description": null + } + }, + "dq_results": { + "processingUrn": "urn:ata:example-tenant:data-quality:processing:00000000-0000-0000-0000-000000000001", + "overallQuality": { + "passedCount": 1784, + "failedCount": 472 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 2230, + "failedCount": 26, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 1784, + "failedCount": 472, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 2256, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "overallDqFindings": [], + "catalogItemLink": "https://one.example.com/catalog/catalog-items/11111111-1111-1111-1111-000000000007", + "dqResultsLink": "https://one.example.com/entities/catalog-item-dq-configurations/11111111-1111-1111-1111-000000000008/results", + "invalidSamplesLink": "https://one.example.com/entities/catalog-item-dq-configurations/11111111-1111-1111-1111-000000000008/invalid-records", + "invalidRecordsExportResults": [ + { + "success": false, + "message": "Export definition or datasource context not found for exportId: 11111111-1111-1111-1111-000000000009", + "location": null, + "link": null + } + ], + "dqAttributeResults": [ + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000001", + "attributeName": "TRANSACTION_ID", + "overallQuality": { + "passedCount": 2251, + "failedCount": 5 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 2251, + "failedCount": 5, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000001", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000001", + "ruleInstanceName": "r_TRANSACTION_ID", + "ruleName": "r_TRANSACTION_ID", + "results": [ + { + "count": 2251, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 5, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2251 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000002", + "name": "not unique", + "count": 5 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000002", + "attributeName": "CUSTOMER_ID", + "overallQuality": { + "passedCount": 2256, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 2256, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000002", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000002", + "ruleInstanceName": "AI Rule Valid Customer ID", + "ruleName": "AI Rule Valid Customer ID", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000003", + "name": "Missing CUSTOMER_ID", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000003", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000003", + "ruleInstanceName": "AI Rule Consistent Account-Customer Mapping", + "ruleName": "AI Rule Consistent Account-Customer Mapping", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000004", + "name": "CustomerID is missing", + "count": 0 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000003", + "attributeName": "CUSTOMER_NAME", + "overallQuality": { + "passedCount": 2256, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 2256, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000004", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000004", + "ruleInstanceName": "AI Rule Valid Customer Name", + "ruleName": "AI Rule Valid Customer Name", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000005", + "name": "Customer name is missing or null", + "count": 0 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000004", + "attributeName": "EMAIL", + "overallQuality": { + "passedCount": 0, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000005", + "attributeName": "BIRTH_DATE", + "overallQuality": { + "passedCount": 2216, + "failedCount": 40 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 2216, + "failedCount": 40, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000005", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000005", + "ruleInstanceName": "AI Rule Valid Birth Date Rule", + "ruleName": "AI Rule Valid Birth Date Rule", + "results": [ + { + "count": 2216, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 40, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2216 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000006", + "name": "Birth date is in the future or missing", + "count": 40 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000006", + "attributeName": "COUNTRY_CODE", + "overallQuality": { + "passedCount": 2183, + "failedCount": 73 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 2230, + "failedCount": 26, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 2183, + "failedCount": 73, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000006", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000006", + "ruleInstanceName": "RDM - Country code length check", + "ruleName": "RDM - Country code length check", + "results": [ + { + "count": 2183, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 73, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 73 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000007", + "name": "All good", + "count": 2183 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000007", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000007", + "ruleInstanceName": "AI Rule Country Code Not Null Check", + "ruleName": "AI Rule Country Code Not Null Check", + "results": [ + { + "count": 26, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2230, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2230 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000008", + "name": "Country code is missing", + "count": 26 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000008", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000008", + "ruleInstanceName": "AI Rule Rule for COUNTRY", + "ruleName": "AI Rule Rule for COUNTRY", + "results": [ + { + "count": 26, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2230, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2230 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000009", + "name": "Country is missing", + "count": 26 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000007", + "attributeName": "ACCOUNT_BALANCE", + "overallQuality": { + "passedCount": 0, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000008", + "attributeName": "ORDER_AMOUNT", + "overallQuality": { + "passedCount": 1898, + "failedCount": 358 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 1898, + "failedCount": 358, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000009", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000009", + "ruleInstanceName": "AI Rule Order Amount and Risk Rating Check", + "ruleName": "AI Rule Order Amount and Risk Rating Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000010", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000010", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000010", + "ruleInstanceName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "ruleName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000011", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000009", + "attributeName": "APPROVAL_STATUS", + "overallQuality": { + "passedCount": 0, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000010", + "attributeName": "PAYMENT_METHOD", + "overallQuality": { + "passedCount": 1898, + "failedCount": 358 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 1898, + "failedCount": 358, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000009", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000009", + "ruleInstanceName": "AI Rule Order Amount and Risk Rating Check", + "ruleName": "AI Rule Order Amount and Risk Rating Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000010", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000010", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000010", + "ruleInstanceName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "ruleName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000011", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000011", + "attributeName": "TRANSACTION_DATE", + "overallQuality": { + "passedCount": 0, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 2256, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000011", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000011", + "ruleInstanceName": "AI Rule No Future Dated Transactions", + "ruleName": "AI Rule No Future Dated Transactions", + "results": [ + { + "count": 0, + "result": { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + }, + { + "count": 0, + "result": { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + } + }, + { + "count": 0, + "result": { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + } + }, + { + "count": 2256, + "result": { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000012", + "name": "Transaction date is in the future", + "count": 0 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000012", + "attributeName": "CREATED_BY", + "overallQuality": { + "passedCount": 2256, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 2256, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000012", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000012", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000013", + "name": "Value is null", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000013", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000013", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000014", + "name": "Missing value", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000014", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000014", + "ruleInstanceName": "AI Rule Valid Created By", + "ruleName": "AI Rule Valid Created By", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000015", + "name": "CREATED_BY is missing", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000015", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000015", + "ruleInstanceName": "AI Rule Created By Completeness", + "ruleName": "AI Rule Created By Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000016", + "name": "Created by field is null or empty", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000016", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000016", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000017", + "name": "Missing value", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000017", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000017", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000018", + "name": "Missing value", + "count": 0 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000018", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000018", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000019", + "name": "Missing value", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000019", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000019", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000020", + "name": "Value is null", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000020", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000014", + "ruleInstanceName": "AI Rule Valid Created By", + "ruleName": "AI Rule Valid Created By", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000015", + "name": "CREATED_BY is missing", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000021", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000020", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000021", + "name": "Missing value", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000022", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000015", + "ruleInstanceName": "AI Rule Created By Completeness", + "ruleName": "AI Rule Created By Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000016", + "name": "Created by field is null or empty", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000023", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000021", + "ruleInstanceName": "AI Rule Demo Rule for Completeness", + "ruleName": "AI Rule Demo Rule for Completeness", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 2256, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 2256 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000022", + "name": "Missing value", + "count": 0 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000013", + "attributeName": "RISK_RATING", + "overallQuality": { + "passedCount": 1898, + "failedCount": 358 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 1898, + "failedCount": 358, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Accuracy", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000005", + "results": [ + { + "name": "Accurate", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000012" + }, + { + "name": "Not Accurate", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000013" + }, + { + "name": "No reference available", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000014" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Uniqueness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000004", + "results": [ + { + "name": "Unique", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000009" + }, + { + "name": "Not Unique", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000010" + }, + { + "name": "Not Populated", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000011" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + }, + { + "dimension": { + "name": "Timeliness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000003", + "results": [ + { + "name": "Timeliness ok", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000005" + }, + { + "name": "Minor delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000006" + }, + { + "name": "Major delay", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000007" + }, + { + "name": "Future", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000008" + } + ] + }, + "passedCount": 0, + "failedCount": 0, + "ruleInstanceCount": 0 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000009", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000009", + "ruleInstanceName": "AI Rule Order Amount and Risk Rating Check", + "ruleName": "AI Rule Order Amount and Risk Rating Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000010", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000010", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000010", + "ruleInstanceName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "ruleName": "AI Rule Order Amount, Risk Rating, and Payment Method Check", + "results": [ + { + "count": 1898, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 358, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000001", + "name": "OTHER", + "count": 1898 + }, + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000011", + "name": "Payment method is not 'Credit Card' for high risk and high order amount", + "count": 358 + } + ], + "dqFindings": [] + } + ] + } + ] + }, + "dq_threshold_pct": 75 + }, + { + "item": { + "urn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000002", + "name": "aggregation", + "description": null, + "originPath": null, + "connection": { + "urn": "urn:ata:example-tenant:processing-center:connection:00000000-0000-0000-0000-000000000002" + }, + "source": { + "urn": "urn:ata:example-tenant:catalog:source:00000000-0000-0000-0000-000000000002" + }, + "locations": [], + "termAssignments": [], + "effectiveStewardship": null, + "primaryDqMonitor": { + "urn": "urn:ata:example-tenant:data-quality:dq-monitor:00000000-0000-0000-0000-000000000002" + }, + "aliases": [] + }, + "attributes": [ + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000014", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000002", + "name": "PAYMENT_METHOD", + "description": null, + "dataType": "STRING", + "columnType": "VARCHAR", + "comment": null, + "termAssignments": [] + }, + { + "urn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000015", + "catalogItemUrn": "urn:ata:example-tenant:catalog:catalog-item:00000000-0000-0000-0000-000000000002", + "name": "NUM_OF_TRANSACTIONS", + "description": null, + "dataType": "LONG", + "columnType": "NUMBER", + "comment": null, + "termAssignments": [] + } + ], + "terms": {}, + "dq_results": { + "processingUrn": "urn:ata:example-tenant:data-quality:processing:00000000-0000-0000-0000-000000000002", + "overallQuality": { + "passedCount": 0, + "failedCount": 4 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 4, + "failedCount": 0, + "ruleInstanceCount": 2 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 4, + "ruleInstanceCount": 2 + } + ], + "overallDqFindings": [], + "catalogItemLink": "https://one.example.com/catalog/catalog-items/11111111-1111-1111-1111-000000000010", + "dqResultsLink": "https://one.example.com/entities/catalog-item-dq-configurations/11111111-1111-1111-1111-000000000011/results", + "invalidSamplesLink": "https://one.example.com/entities/catalog-item-dq-configurations/11111111-1111-1111-1111-000000000011/invalid-records", + "invalidRecordsExportResults": [], + "dqAttributeResults": [ + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000014", + "attributeName": "PAYMENT_METHOD", + "overallQuality": { + "passedCount": 0, + "failedCount": 4 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 0, + "failedCount": 4, + "ruleInstanceCount": 3 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 0, + "failedCount": 4, + "ruleInstanceCount": 3 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000024", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000022", + "ruleInstanceName": "AI Rule SSN Validation Rule", + "ruleName": "AI Rule SSN Validation Rule", + "results": [ + { + "count": 0, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 4, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000023", + "name": "Does not match the SSN format", + "count": 4 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000025", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000023", + "ruleInstanceName": "AI Rule Completeness Rule for CUSTOMER", + "ruleName": "AI Rule Completeness Rule for CUSTOMER", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 4, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000024", + "name": "Customer attribute is null or empty", + "count": 0 + } + ], + "dqFindings": [] + }, + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000026", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000024", + "ruleInstanceName": "AI Rule Insurance Completeness Rule", + "ruleName": "AI Rule Insurance Completeness Rule", + "results": [ + { + "count": 0, + "result": { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + }, + { + "count": 4, + "result": { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000025", + "name": "Critical field is empty", + "count": 0 + } + ], + "dqFindings": [] + } + ] + }, + { + "attributeUrn": "urn:ata:example-tenant:catalog:catalog-attribute:00000000-0000-0000-0000-000000000015", + "attributeName": "NUM_OF_TRANSACTIONS", + "overallQuality": { + "passedCount": 4, + "failedCount": 0 + }, + "dimensionResults": [ + { + "dimension": { + "name": "Completeness", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000001", + "results": [ + { + "name": "Complete", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000001" + }, + { + "name": "Not Complete", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000002" + } + ] + }, + "passedCount": 4, + "failedCount": 0, + "ruleInstanceCount": 1 + }, + { + "dimension": { + "name": "Validity", + "urn": "urn:ata:example-tenant:data-quality:dimension:00000000-0000-0000-0000-000000000002", + "results": [ + { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + }, + { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + ] + }, + "passedCount": 4, + "failedCount": 0, + "ruleInstanceCount": 1 + } + ], + "ruleInstanceResults": [ + { + "ruleInstanceUrn": "urn:ata:example-tenant:data-quality:rule-instance:00000000-0000-0000-0000-000000000027", + "ruleUrn": "urn:ata:example-tenant:data-quality:rule:00000000-0000-0000-0000-000000000025", + "ruleInstanceName": "AI Rule Accuracy Rule for INVOICED_USD", + "ruleName": "AI Rule Accuracy Rule for INVOICED_USD", + "results": [ + { + "count": 4, + "result": { + "name": "Valid", + "passedOverallResult": true, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000003" + } + }, + { + "count": 0, + "result": { + "name": "Invalid", + "passedOverallResult": false, + "urn": "urn:ata:example-tenant:data-quality:dimension-result:00000000-0000-0000-0000-000000000004" + } + } + ], + "explanations": [ + { + "urn": "urn:ata:example-tenant:data-quality:explanation:00000000-0000-0000-0000-000000000026", + "name": "Negative value", + "count": 0 + } + ], + "dqFindings": [] + } + ] + } + ] + }, + "dq_threshold_pct": null + } +] \ No newline at end of file diff --git a/converters/ataccama/tests/test_ataccama_to_osi.py b/converters/ataccama/tests/test_ataccama_to_osi.py new file mode 100644 index 0000000..fec2e15 --- /dev/null +++ b/converters/ataccama/tests/test_ataccama_to_osi.py @@ -0,0 +1,259 @@ +"""Tests for the Ataccama -> OSI conversion (offline, using a recorded fixture). + +Fixture items: BANK_TRANSACTIONS (a warehouse table with terms, DATE columns, and +rich DQ results) and aggregation (a derived output with no locations and a 0% DQ case). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml +from jsonschema import Draft202012Validator + +from ataccama_osi.ataccama_to_osi import ( + OSI_VERSION, + _dataset_dq, + _dataset_dq_warning, + _quality_summary, + ataccama_to_osi, + attribute_to_field, + build_source, + flatten_richtext, +) +from ataccama_osi.models import CatalogAttribute, CatalogItem, CatalogItemBundle, CatalogLocation + +FIXTURE = Path(__file__).parent / "fixtures" / "ataccama_bundles.json" +SCHEMA = Path(__file__).parents[3] / "core-spec" / "osi-schema.json" + + +@pytest.fixture +def bundles() -> list[CatalogItemBundle]: + raw = json.loads(FIXTURE.read_text()) + return [CatalogItemBundle.from_dict(b) for b in raw] + + +@pytest.fixture +def document(bundles: list[CatalogItemBundle]) -> dict: + return ataccama_to_osi(bundles, model_name="test_model", tenant="example-tenant") + + +def _dataset(document: dict, name: str) -> dict: + return next(d for d in document["semantic_model"][0]["datasets"] if d["name"] == name) + + +def _field(dataset: dict, name: str) -> dict: + return next(f for f in dataset["fields"] if f["name"] == name) + + +def _ext(obj: dict) -> dict: + return json.loads(obj["custom_extensions"][0]["data"]) + + +# --- structure & core mapping --- + + +def test_document_is_schema_valid(document: dict) -> None: + schema = json.loads(SCHEMA.read_text()) + errors = list(Draft202012Validator(schema).iter_errors(document)) + assert not errors, [e.message for e in errors] + assert document["version"] == OSI_VERSION + + +def test_model_structure(document: dict) -> None: + model = document["semantic_model"][0] + assert model["name"] == "test_model" + assert {d["name"] for d in model["datasets"]} == {"BANK_TRANSACTIONS", "aggregation"} + + +def test_attribute_becomes_quoted_ansi_field(document: dict) -> None: + ds = _dataset(document, "BANK_TRANSACTIONS") + assert len(ds["fields"]) == 13 + field = _field(ds, "TRANSACTION_ID") + assert field["expression"]["dialects"] == [{"dialect": "ANSI_SQL", "expression": '"TRANSACTION_ID"'}] + assert _ext(field)["data_type"] == "STRING" + + +def test_date_columns_are_time_dimensions(document: dict) -> None: + ds = _dataset(document, "BANK_TRANSACTIONS") + assert _field(ds, "BIRTH_DATE")["dimension"] == {"is_time": True} + assert _field(ds, "TRANSACTION_DATE")["dimension"] == {"is_time": True} + assert "dimension" not in _field(ds, "ACCOUNT_BALANCE") + + +def test_source_from_locations_and_limitation(document: dict) -> None: + # DB-backed item: reversed locations + name form a qualified path + assert _dataset(document, "BANK_TRANSACTIONS")["source"] == "DEMO_ENV.CLOUD_DEMO.BANK_TRANSACTIONS" + # derived item with no locations: source degrades to the bare name + assert _dataset(document, "aggregation")["source"] == "aggregation" + + +def test_terms_become_ai_context(document: dict) -> None: + ds = _dataset(document, "BANK_TRANSACTIONS") + assert set(ds["ai_context"]["synonyms"]) >= {"Personal Data", "Finance"} + email = _field(ds, "EMAIL") + assert email["ai_context"]["synonyms"] == ["E-mail"] + assert "email" in email["ai_context"]["instructions"].lower() + + +def test_governance_metadata_preserved(document: dict) -> None: + ext = _ext(_dataset(document, "BANK_TRANSACTIONS")) + assert ext["catalog_item_urn"].startswith("urn:ata:") + assert "connection_urn" in ext and "stewardship_group_urn" in ext + + +# --- data quality enrichment --- + + +def test_dataset_dq_overall_and_dimensions(document: dict) -> None: + dq = _ext(_dataset(document, "BANK_TRANSACTIONS"))["dq"] + assert dq["passed"] == 1784 and dq["failed"] == 472 + assert dq["pass_rate_pct"] == 79.1 + dims = {d["name"]: d for d in dq["dimensions"]} + assert dims["Completeness"]["pass_rate_pct"] == 98.8 + assert dims["Validity"]["pass_rate_pct"] == 79.1 + # dimensions with no evaluated records (Uniqueness/Accuracy = 0/0) are dropped + assert "Uniqueness" not in dims and "Accuracy" not in dims + assert dq["results_link"].startswith("https://") + # active-finding count is always reported (0 = no open issues) + assert dq["active_findings"] == 0 + + +def test_dataset_dq_threshold(document: dict) -> None: + # BANK_TRANSACTIONS has a configured overall bar of 75%; 79.1% is above it. + dq = _ext(_dataset(document, "BANK_TRANSACTIONS"))["dq"] + assert dq["threshold_pct"] == 75 + assert dq["below_threshold"] is False + + +def test_no_threshold_when_monitor_has_none(document: dict) -> None: + # aggregation's monitor has no overall threshold configured. + dq = _ext(_dataset(document, "aggregation"))["dq"] + assert "threshold_pct" not in dq and "below_threshold" not in dq + + +def test_field_level_dq(document: dict) -> None: + ds = _dataset(document, "BANK_TRANSACTIONS") + dq = _ext(_field(ds, "TRANSACTION_ID"))["dq"] + assert dq == {"passed": 2251, "failed": 5, "pass_rate_pct": 99.8} + + +def test_zero_pass_rate_case(document: dict) -> None: + dq = _ext(_dataset(document, "aggregation"))["dq"] + assert dq["passed"] == 0 and dq["failed"] == 4 + assert dq["pass_rate_pct"] == 0.0 + + +def test_no_dq_keys_when_dq_absent(bundles: list[CatalogItemBundle]) -> None: + b = bundles[0] + b.dq_results = None # simulate --no-dq / item without a monitor + ds = ataccama_to_osi([b])["semantic_model"][0]["datasets"][0] + assert "dq" not in _ext(ds) + assert all("dq" not in _ext(f) for f in ds["fields"]) + + +# --- opt-in DQ AI warnings --- + + +def test_ai_warnings_off_by_default(document: dict) -> None: + # Default conversion must not inject warnings into ai_context. + ds = _dataset(document, "BANK_TRANSACTIONS") + assert "Data-quality warning" not in (ds.get("ai_context", {}).get("instructions") or "") + email = _field(ds, "EMAIL") + assert "Data-quality warning" not in email["ai_context"]["instructions"] + + +def test_ai_warning_appended_when_below_ataccama_threshold() -> None: + # A dataset Ataccama flags as below its own configured threshold gets a warning. + bundle = CatalogItemBundle( + item=CatalogItem(urn="urn:ata:t:catalog:catalog-item:1", name="risky"), + dq_results={"overallQuality": {"passedCount": 50, "failedCount": 50}, "overallDqFindings": []}, + dq_threshold_pct=75, + ) + ds = ataccama_to_osi([bundle], dq_ai_warnings=True)["semantic_model"][0]["datasets"][0] + instr = ds["ai_context"]["instructions"] + assert "Data-quality warning" in instr and "configured 75% quality threshold" in instr + + +def test_no_ai_warning_without_ataccama_signal(bundles: list[CatalogItemBundle]) -> None: + # BANK_TRANSACTIONS is above its 75% bar with no active findings; aggregation has no + # threshold configured. Neither should be flagged — Ataccama is the source of truth. + doc = ataccama_to_osi(bundles, dq_ai_warnings=True) + bank = _dataset(doc, "BANK_TRANSACTIONS")["ai_context"]["instructions"] + assert "Data-quality warning" not in bank + agg = (_dataset(doc, "aggregation").get("ai_context") or {}).get("instructions", "") + assert "Data-quality warning" not in agg + + +def test_dataset_dq_warning_triggers() -> None: + # below Ataccama's own configured threshold + below = _dataset_dq_warning({"pass_rate_pct": 60.0, "threshold_pct": 75, "below_threshold": True}) + assert below and "configured 75% quality threshold" in below + # passing, no active findings -> no warning (no converter-side fallback exists) + assert _dataset_dq_warning({"pass_rate_pct": 99.0, "active_findings": 0}) is None + # a dataset with no configured threshold is never flagged on pass rate alone + assert _dataset_dq_warning({"pass_rate_pct": 10.0}) is None + # active findings alone + af = _dataset_dq_warning({"pass_rate_pct": 99.0, "active_findings": 2}) + assert af and "2 active data-quality finding(s)" in af + + +# --- helper unit tests --- + + +@pytest.mark.parametrize( + "value,expected", + [ + (None, None), + ("", None), + ("plain text", "plain text"), + ('[{"type":"paragraph","children":[{"text":"hi"}]}]', "hi"), + ([{"type": "paragraph", "children": [{"text": "a "}, {"text": "b"}]}], "a b"), + ], +) +def test_flatten_richtext(value, expected) -> None: + assert flatten_richtext(value) == expected + + +@pytest.mark.parametrize( + "oq,expected", + [ + (None, None), + ({"passedCount": 3, "failedCount": 1}, {"passed": 3, "failed": 1, "pass_rate_pct": 75.0}), + ({"passedCount": 0, "failedCount": 0}, {"passed": 0, "failed": 0}), # not evaluated -> no rate + ], +) +def test_quality_summary(oq, expected) -> None: + assert _quality_summary(oq) == expected + + +def test_below_threshold_flag_true() -> None: + dq = _dataset_dq({"overallQuality": {"passedCount": 60, "failedCount": 40}}, threshold_pct=75) + assert dq["pass_rate_pct"] == 60.0 + assert dq["threshold_pct"] == 75 + assert dq["below_threshold"] is True + + +def test_is_time_inferred_from_datatype() -> None: + attr = CatalogAttribute(urn="urn:ata:t:catalog:catalog-attribute:1", name="created_at", data_type="DATETIME") + assert attribute_to_field(attr, terms={}, used=set())["dimension"] == {"is_time": True} + + +def test_duplicate_dataset_names_are_disambiguated() -> None: + items = [ + CatalogItemBundle(item=CatalogItem(urn=f"urn:ata:t:catalog:catalog-item:{i}", name="Orders")) + for i in range(2) + ] + names = [d["name"] for d in ataccama_to_osi(items)["semantic_model"][0]["datasets"]] + assert names == ["Orders", "Orders_2"] + + +def test_build_source_uses_reversed_locations_then_name() -> None: + item = CatalogItem( + urn="urn:ata:t:catalog:catalog-item:1", + name="customers", + locations=[CatalogLocation(name="sales"), CatalogLocation(name="Workspaces")], + ) + assert build_source(item) == "Workspaces.sales.customers"