diff --git a/core-spec/osi-schema.json b/core-spec/osi-schema.json index 72cb164..4258fd0 100644 --- a/core-spec/osi-schema.json +++ b/core-spec/osi-schema.json @@ -108,6 +108,54 @@ "required": ["dialects"], "additionalProperties": false }, + "Source": { + "description": "Dataset source. Legacy string sources are supported for backward compatibility.", + "oneOf": [ + { + "type": "string", + "description": "Legacy source string for a physical table/view reference or query.", + "deprecated": true + }, + { + "$ref": "#/$defs/TableSource" + }, + { + "$ref": "#/$defs/QuerySource" + } + ] + }, + "TableSource": { + "type": "object", + "description": "Object-backed dataset source.", + "properties": { + "kind": { + "const": "table", + "description": "Identifies this source as an object-backed dataset." + }, + "object": { + "type": "string", + "description": "Reference to the underlying physical table/view, e.g. database.schema.table." + } + }, + "required": ["kind", "object"], + "additionalProperties": false + }, + "QuerySource": { + "type": "object", + "description": "Query-backed dataset source.", + "properties": { + "kind": { + "const": "query", + "description": "Identifies this source as a query-backed dataset." + }, + "object": { + "$ref": "#/$defs/Expression", + "description": "Query expression with dialect support." + } + }, + "required": ["kind", "object"], + "additionalProperties": false + }, "Dimension": { "type": "object", "description": "Dimension metadata", @@ -163,8 +211,7 @@ "description": "Unique identifier for the dataset" }, "source": { - "type": "string", - "description": "Reference to underlying physical table/view (database.schema.table) or query" + "$ref": "#/$defs/Source" }, "primary_key": { "type": "array", diff --git a/core-spec/spec.md b/core-spec/spec.md index ea7e775..f87dbd2 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -82,7 +82,7 @@ Logical datasets represent business entities or concepts (fact and dimension tab | Field | Type | Required | Description | |-------|------|----------|-------------| | `name` | string | Yes | Unique identifier for the dataset | -| `source` | string | Yes | Reference to underlying physical table/view (e.g., `database.schema.table`) or query | +| `source` | string/object | Yes | Structured source definition. Legacy string sources are still supported for backward compatibility. | | `primary_key` | array | No | Primary key columns that uniquely identify rows (single or composite) | | `unique_keys` | array of arrays | No | Array of unique key definitions (each can be single or composite) | | `description` | string | No | Human-readable description | @@ -109,12 +109,37 @@ unique_keys: - [first_name, last_name] # Composite unique key ``` +### Source Examples + +```yaml +# Legacy string source, deprecated but still supported during migration +source: sales.public.orders + +# Object-backed source +source: + kind: table + object: sales.public.orders + +# Query-backed source +source: + kind: query + object: + dialects: + - dialect: ANSI_SQL + expression: | + select order_id, customer_id, revenue + from sales.public.orders + where revenue > 10000 +``` + ### Example ```yaml datasets: - name: orders - source: sales.public.orders + source: + kind: table + object: sales.public.orders primary_key: [order_id] unique_keys: - [order_id] @@ -430,7 +455,9 @@ semantic_model: datasets: - name: orders - source: sales.public.orders + source: + kind: table + object: sales.public.orders primary_key: [order_id] description: Customer orders fields: @@ -465,7 +492,9 @@ semantic_model: description: Order amount - name: customers - source: sales.public.customers + source: + kind: table + object: sales.public.customers primary_key: [id] description: Customer information fields: diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index ed46203..f4d5c91 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -66,8 +66,18 @@ datasets: - name: string # Required: Reference to the underlying physical table/view or query - # Format should be either database_name.schema_name.table_name or query - source: string + # Legacy string source remains supported for backward compatibility. + source: + kind: table + object: string + + # Query-backed source: + # source: + # kind: query + # object: + # dialects: + # - dialect: string + # expression: string # Optional: Primary key definition that uniquely identifies rows in this dataset # Can be a single column or a composite of multiple columns diff --git a/python/src/osi/__init__.py b/python/src/osi/__init__.py index 00970fd..f2aba75 100644 --- a/python/src/osi/__init__.py +++ b/python/src/osi/__init__.py @@ -10,8 +10,11 @@ OSIExpression, OSIField, OSIMetric, + OSIQuerySource, OSIRelationship, OSISemanticModel, + OSISource, + OSITableSource, OSIVendor, ) @@ -27,7 +30,10 @@ "OSIExpression", "OSIField", "OSIMetric", + "OSIQuerySource", "OSIRelationship", "OSISemanticModel", + "OSISource", + "OSITableSource", "OSIVendor", ] diff --git a/python/src/osi/models.py b/python/src/osi/models.py index 766b6e0..442f6bd 100644 --- a/python/src/osi/models.py +++ b/python/src/osi/models.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Optional, Union +from typing import Any, Literal, Optional, Union import yaml from pydantic import BaseModel, ConfigDict, Field @@ -66,6 +66,27 @@ class OSIExpression(BaseModel): dialects: list[OSIDialectExpression] +class OSITableSource(BaseModel): + """Object-backed dataset source.""" + + model_config = ConfigDict(frozen=True) + + kind: Literal["table"] + object: str + + +class OSIQuerySource(BaseModel): + """Query-backed dataset source.""" + + model_config = ConfigDict(frozen=True) + + kind: Literal["query"] + object: OSIExpression + + +OSISource = Union[str, OSITableSource, OSIQuerySource] + + class OSIDimension(BaseModel): """Dimension metadata on a field.""" @@ -94,7 +115,7 @@ class OSIDataset(BaseModel): model_config = ConfigDict(frozen=True) name: str - source: str + source: OSISource primary_key: Optional[list[str]] = None unique_keys: Optional[list[list[str]]] = None description: Optional[str] = None