From 5bf3927e767068910cffecf1825d3d90690a15bd Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Mon, 5 Jan 2026 19:57:28 -0500 Subject: [PATCH] Complete ReadOnlyCatalogInterface implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add CatalogReadOnlyError exception and implement ReadOnlyCatalogInterface that raises this error for all DDL operations (create, drop, rename, modify). Read-only catalogs only support read operations: - catalogs() - list catalogs - catalog_attach/detach - attach to/detach from catalogs - schemas() - list schemas - schema_get() - get schema info - schema_contents() - list schema contents - table_get(), view_get() - get table/view info - table_scan_function_get() - get scan function for tables 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .beads/issues.jsonl | 6 +- vgi/catalog/catalog_interface.py | 350 +++++++++++++++++++++++++++++++ vgi/exceptions.py | 19 ++ 3 files changed, 372 insertions(+), 3 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 13ec848..9950e1a 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -19,7 +19,7 @@ {"id":"vgi-python-79e","title":"Unify ProtocolInput classes with shared base","description":"ProtocolInput classes in scalar_function.py:151-166 and table_in_out_function.py:109-142 have similar structure with batch and metadata fields. The table_in_out version adds is_finalize logic. Create shared base ProtocolInput in protocol_types.py with table_in_out extending it.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-04T20:06:41.31917-05:00","created_by":"rusty","updated_at":"2026-01-04T21:53:26.965345-05:00","closed_at":"2026-01-04T21:53:26.965345-05:00","close_reason":"PR #9 created - unified ProtocolInput with shared base in protocol_types.py"} {"id":"vgi-python-8gz","title":"VGI Catalog Interface Implementation","description":"Complete the VGI Catalog Interface implementation to enable DuckDB ATTACH support.\n\nThe CatalogInterface ABC is already implemented in vgi/catalog/catalog_interface.py.\n\nRemaining work:\n- Add serialize/deserialize methods to dataclasses\n- Add InvocationType.CATALOG to protocol\n- Worker integration for catalog dispatch \n- CatalogClient class (new worker per call pattern)\n- Optional SQLite-based catalog storage\n- Example InMemoryCatalog\n- Tests\n\nSee: catalog-plan.md","status":"open","priority":1,"issue_type":"feature","created_at":"2026-01-05T19:26:27.348627-05:00","created_by":"rusty","updated_at":"2026-01-05T19:26:27.348627-05:00"} {"id":"vgi-python-8ra","title":"Implement Arrow-based argument specification serialization","description":"## Overview\n\nImplement serialization and deserialization of function argument specifications using Apache Arrow schemas. This enables functions to describe their argument signatures (types, positions, special markers) in a format that can be transmitted over IPC and understood by DuckDB for function registration.\n\n## Design\n\nUses a **single Arrow schema** where:\n- Positional arguments come first (field order = position index)\n- Named arguments follow (marked with `vgi_arg=named` metadata)\n- Special types (TableInput, AnyArrow, varargs) use field metadata markers\n\n## Key Components\n\n1. `ArgumentSpec` dataclass - represents one argument's specification\n2. `argument_specs_to_schema()` - convert specs to Arrow schema\n3. `schema_to_argument_specs()` - convert schema back to specs\n4. `extract_argument_specs()` - extract specs from function class Arg descriptors\n\n## Metadata Keys\n\n| Key | Value | Meaning |\n|-----|-------|---------|\n| `vgi_arg` | `named` | Named argument (not positional) |\n| `vgi_type` | `table` | Receives table input (Arg[TableInput]) |\n| `vgi_type` | `any` | Accepts any Arrow type (Arg[AnyArrow]) |\n| `vgi_varargs` | `true` | Collects remaining positional args |\n\n## References\n\n- Plan file: `.claude/plans/purrfect-foraging-nygaard.md`\n- Arguments module: `vgi/arguments.py`","status":"closed","priority":2,"issue_type":"feature","created_at":"2026-01-05T11:18:01.05631-05:00","created_by":"rusty","updated_at":"2026-01-05T11:34:12.712096-05:00","closed_at":"2026-01-05T11:34:12.712096-05:00","close_reason":"Implemented Arrow-based argument specification serialization with tests and documentation"} -{"id":"vgi-python-9j7","title":"Add catalog dispatch to Worker class","description":"Integrate CatalogInterface handling into Worker class.\n\nFile: vgi/worker.py\n\nChanges:\n1. Add catalog_interface class attribute: type[CatalogInterface] | None = None\n\n2. In run() method, detect InvocationType.CATALOG and dispatch to _handle_catalog_invocation()\n\n3. Implement _handle_catalog_invocation(invocation: Invocation):\n - Check catalog_interface is not None (raise ValueError if missing)\n - Instantiate catalog_interface class\n - Get method from function_name field (e.g., 'catalog_attach')\n - Deserialize arguments from input batch (column names → kwargs)\n - Call method with keyword arguments\n - Serialize and stream result back\n\n4. Key protocol difference: No bind→init→stream phases, just invoke→stream\n\n5. Handle different return types:\n - None → 0-row/0-column batch\n - Dataclass → serialize to single-row batch\n - Iterable → stream multiple batches\n\n6. Error handling: Return exceptions as EXCEPTION log messages (same as functions)","status":"in_progress","priority":1,"issue_type":"task","created_at":"2026-01-05T19:26:57.845071-05:00","created_by":"rusty","updated_at":"2026-01-05T19:41:11.77842-05:00","dependencies":[{"issue_id":"vgi-python-9j7","depends_on_id":"vgi-python-085","type":"blocks","created_at":"2026-01-05T19:27:50.589219-05:00","created_by":"rusty"},{"issue_id":"vgi-python-9j7","depends_on_id":"vgi-python-po3","type":"blocks","created_at":"2026-01-05T19:27:50.620681-05:00","created_by":"rusty"}]} +{"id":"vgi-python-9j7","title":"Add catalog dispatch to Worker class","description":"Integrate CatalogInterface handling into Worker class.\n\nFile: vgi/worker.py\n\nChanges:\n1. Add catalog_interface class attribute: type[CatalogInterface] | None = None\n\n2. In run() method, detect InvocationType.CATALOG and dispatch to _handle_catalog_invocation()\n\n3. Implement _handle_catalog_invocation(invocation: Invocation):\n - Check catalog_interface is not None (raise ValueError if missing)\n - Instantiate catalog_interface class\n - Get method from function_name field (e.g., 'catalog_attach')\n - Deserialize arguments from input batch (column names → kwargs)\n - Call method with keyword arguments\n - Serialize and stream result back\n\n4. Key protocol difference: No bind→init→stream phases, just invoke→stream\n\n5. Handle different return types:\n - None → 0-row/0-column batch\n - Dataclass → serialize to single-row batch\n - Iterable → stream multiple batches\n\n6. Error handling: Return exceptions as EXCEPTION log messages (same as functions)","status":"closed","priority":1,"issue_type":"task","created_at":"2026-01-05T19:26:57.845071-05:00","created_by":"rusty","updated_at":"2026-01-05T19:44:05.99412-05:00","closed_at":"2026-01-05T19:44:05.99412-05:00","close_reason":"PR #26 created with Worker catalog dispatch","dependencies":[{"issue_id":"vgi-python-9j7","depends_on_id":"vgi-python-085","type":"blocks","created_at":"2026-01-05T19:27:50.589219-05:00","created_by":"rusty"},{"issue_id":"vgi-python-9j7","depends_on_id":"vgi-python-po3","type":"blocks","created_at":"2026-01-05T19:27:50.620681-05:00","created_by":"rusty"}]} {"id":"vgi-python-9ql","title":"VGI Catalog Interface Implementation","description":"Add CatalogInterface ABC that lets VGI workers expose catalogs (databases), schemas, tables, views, and functions. Enables DuckDB ATTACH command support via VGI workers.\n\nKey components:\n- Type aliases and dataclasses (AttachId, TransactionId, SchemaInfo, TableInfo, etc.)\n- CatalogInterface abstract base class with ~40 methods\n- InvocationType.CATALOG for protocol dispatch\n- CatalogClient for client-side operations\n- Worker integration for catalog invocation handling\n\nSee: catalog-plan.md and ~/.claude/plans/iterative-waddling-adleman.md","status":"closed","priority":1,"issue_type":"feature","created_at":"2026-01-05T19:16:41.100846-05:00","created_by":"rusty","updated_at":"2026-01-05T19:21:50.071596-05:00","closed_at":"2026-01-05T19:21:50.071596-05:00","close_reason":"User requested closure"} {"id":"vgi-python-a99","title":"Add settings accessor to function base classes","description":"Add a property to access DuckDB settings values in function implementations.\n\nChanges needed:\n- Add 'settings: dict[str, str]' property to Function base class\n- Property should return self.invocation.duckdb_settings or empty dict\n- Add convenience method like 'get_setting(name, default=None)'\n- Update ScalarFunction, TableFunctionGenerator, TableInOutFunction\n\nExample usage in function:\ndef compute(self, batch):\n tz = self.get_setting('timezone', 'UTC')\n # or\n tz = self.settings.get('timezone', 'UTC')","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-04T13:05:48.221602-05:00","created_by":"rusty","updated_at":"2026-01-04T13:20:41.171991-05:00","closed_at":"2026-01-04T13:20:41.171991-05:00","close_reason":"Implementation complete, all tests pass","dependencies":[{"issue_id":"vgi-python-a99","depends_on_id":"vgi-python-aad","type":"blocks","created_at":"2026-01-04T13:06:13.738212-05:00","created_by":"rusty"}]} {"id":"vgi-python-a9i","title":"Add test coverage for worker error paths and edge cases","notes":"Coverage: 88% in vgi/worker.py. Missing tests for:\n- Lines 161, 310, 320, 322, 325, 333: Function lookup edge cases\n- Lines 394, 408: Validation error paths\n- Lines 442-448: Log message handling loop\n- Lines 618-621, 652, 660, 681: Table function edge cases\n\nThese are error handling and edge case paths in the worker protocol.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-04T22:15:33.888277-05:00","created_by":"rusty","updated_at":"2026-01-04T22:34:34.795304-05:00","closed_at":"2026-01-04T22:34:34.795304-05:00","close_reason":"Added tests for registry caching and _suggest_similar_names. Coverage improved from 88% to 92%. Remaining uncovered lines are deep protocol paths in subprocess handling."} @@ -38,11 +38,11 @@ {"id":"vgi-python-cvj","title":"Add PYTHON_TO_ARROW type mapping to vgi/arguments.py","description":"Add the Python→Arrow type mapping dict after imports:\n```python\nPYTHON_TO_ARROW: dict[type, pa.DataType] = {\n int: pa.int64(),\n str: pa.utf8(),\n float: pa.float64(),\n bool: pa.bool_(),\n bytes: pa.binary(),\n}\n```\nExport in __all__.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:37.900421-05:00","created_by":"rusty","updated_at":"2026-01-05T15:48:42.422086-05:00","closed_at":"2026-01-05T15:48:42.422086-05:00","close_reason":"PR #18 created"} {"id":"vgi-python-d73","title":"Create docs/argument-serialization.md","description":"## Overview\n\nCreate LLM-friendly documentation explaining the argument specification serialization format. This document should enable future implementors (human or AI) to understand how function argument signatures are serialized to Arrow schemas.\n\n## File Location\n\n`docs/argument-serialization.md`\n\n## Document Structure\n\n### Title and Purpose\n\nExplain that this document describes how VGI function argument specifications are serialized to Apache Arrow schemas for IPC transmission and DuckDB function registration.\n\n### Quick Reference\n\nA concise summary table showing:\n- Metadata keys and their meanings\n- Special type representations\n\n### Schema Format\n\nExplain the single-schema design:\n1. All arguments are fields in one Arrow schema\n2. Positional arguments come first, in order (field index = position index)\n3. Named arguments follow, marked with metadata\n4. Field name = Python attribute name (or argument key for named)\n5. Field type = exact Arrow type\n\n### Metadata Keys Reference\n\nComplete table of all metadata keys:\n\n| Key | Value | Description |\n|-----|-------|-------------|\n| `vgi_arg` | `named` | Field is a named argument, not positional. The field name is the argument key. |\n| `vgi_type` | `table` | Argument receives streaming table input (Arg[TableInput]). Arrow type is pa.null(). |\n| `vgi_type` | `any` | Argument accepts any Arrow type (Arg[AnyArrow]). Arrow type is pa.null(). |\n| `vgi_varargs` | `true` | Argument collects all remaining positional args. Arrow type is the element type. |\n\n### Special Type Handling\n\nExplain how special argument types are represented:\n\n#### TableInput\n- Arrow type: `pa.null()`\n- Metadata: `{b\"vgi_type\": b\"table\"}`\n- Meaning: This position receives streaming RecordBatches, not a scalar value\n\n#### AnyArrow\n- Arrow type: `pa.null()`\n- Metadata: `{b\"vgi_type\": b\"any\"}`\n- Meaning: Accepts any valid Arrow scalar type at runtime\n\n#### Varargs\n- Arrow type: The element type (e.g., `pa.int64()` for `Arg[int](..., varargs=True)`)\n- Metadata: `{b\"vgi_varargs\": b\"true\"}`\n- Meaning: Collects all remaining positional arguments from this position onwards\n\n### Examples\n\n#### Example 1: Simple Function\n\n```python\nclass MyFunction(TableInOutFunction):\n count = Arg[int](0) # Positional 0\n name = Arg[str](1) # Positional 1\n verbose = Arg[bool](\"verbose\") # Named\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"count\", pa.int64()),\n pa.field(\"name\", pa.utf8()),\n pa.field(\"verbose\", pa.bool_(), metadata={b\"vgi_arg\": b\"named\"}),\n])\n```\n\n#### Example 2: Function with Table Input\n\n```python\nclass TransformFunction(TableInOutFunction):\n multiplier = Arg[float](0)\n data = Arg[TableInput](1)\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"multiplier\", pa.float64()),\n pa.field(\"data\", pa.null(), metadata={b\"vgi_type\": b\"table\"}),\n])\n```\n\n#### Example 3: Function with Varargs\n\n```python\nclass SumFunction(TableInOutFunction):\n columns = Arg[str](0, varargs=True)\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"columns\", pa.utf8(), metadata={b\"vgi_varargs\": b\"true\"}),\n])\n```\n\n#### Example 4: Complex Function\n\n```python\nclass ComplexFunction(TableInOutFunction):\n count = Arg[int](0)\n data = Arg[TableInput](1)\n extra = Arg[float](2, varargs=True)\n format = Arg[str](\"format\")\n threshold = Arg[AnyArrow](\"threshold\")\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"count\", pa.int64()),\n pa.field(\"data\", pa.null(), metadata={b\"vgi_type\": b\"table\"}),\n pa.field(\"extra\", pa.float64(), metadata={b\"vgi_varargs\": b\"true\"}),\n pa.field(\"format\", pa.utf8(), metadata={b\"vgi_arg\": b\"named\"}),\n pa.field(\"threshold\", pa.null(), metadata={b\"vgi_arg\": b\"named\", b\"vgi_type\": b\"any\"}),\n])\n```\n\n### Serialization Code\n\nShow how to serialize and deserialize:\n\n```python\n# Serialize to bytes\nschema_bytes = schema.serialize().to_pybytes()\n\n# Deserialize from bytes\nschema = pa.ipc.read_schema(pa.py_buffer(schema_bytes))\n```\n\n### Parsing Algorithm\n\nExplain how to parse a schema back to argument specs:\n\n1. Initialize position_index = 0\n2. For each field in schema:\n a. Check if field has `vgi_arg=named` metadata\n b. If named: position = field.name (string)\n c. If positional: position = position_index, then increment position_index\n d. Check for `vgi_type` metadata (table or any)\n e. Check for `vgi_varargs` metadata\n f. Create ArgumentSpec with extracted info\n\n### Not Included\n\nExplicitly state what is NOT serialized:\n- Default values\n- Validation constraints (ge, le, choices, pattern)\n- Documentation strings\n\nThese are Python-side concerns handled by the Arg descriptor at runtime.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T11:19:17.488877-05:00","created_by":"rusty","updated_at":"2026-01-05T11:33:29.168007-05:00","closed_at":"2026-01-05T11:33:29.168007-05:00","close_reason":"Created comprehensive LLM-friendly documentation","dependencies":[{"issue_id":"vgi-python-d73","depends_on_id":"vgi-python-8ra","type":"blocks","created_at":"2026-01-05T11:19:30.820384-05:00","created_by":"rusty"}]} {"id":"vgi-python-dv0","title":"Add arrow_type parameter to Arg class","description":"In vgi/arguments.py:\n1. Add 'arrow_type' to __slots__\n2. Add parameter: arrow_type: pa.DataType | None = None\n3. Store: self.arrow_type = arrow_type\n4. Update __repr__ to include arrow_type if set","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:38.020395-05:00","created_by":"rusty","updated_at":"2026-01-05T15:50:51.513273-05:00","closed_at":"2026-01-05T15:50:51.513273-05:00","close_reason":"PR #19 created","dependencies":[{"issue_id":"vgi-python-dv0","depends_on_id":"vgi-python-cvj","type":"blocks","created_at":"2026-01-05T15:45:13.696822-05:00","created_by":"rusty"}]} -{"id":"vgi-python-dvd","title":"Complete ReadOnlyCatalogInterface implementation","description":"Complete ReadOnlyCatalogInterface that prevents all DDL operations.\n\nFile: vgi/catalog/catalog_interface.py (already exists)\n\nCurrent state: Only has class attributes, no method overrides.\n\nChanges needed:\n1. Override all DDL methods to raise CatalogReadOnlyError:\n - catalog_create, catalog_drop\n - schema_create, schema_drop\n - table_create, table_drop, table_rename, table_comment_set\n - All table_column_* methods\n - table_not_null_set, table_not_null_drop\n - view_create, view_drop, view_rename, view_comment_set\n - Transaction methods (begin/commit/rollback)\n\n2. Create CatalogReadOnlyError exception in vgi/exceptions.py\n\n3. Ensure catalog_attach returns supports_transactions=False\n\nInclude tests verifying all DDL operations raise CatalogReadOnlyError.","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-05T19:27:15.199453-05:00","created_by":"rusty","updated_at":"2026-01-05T19:27:15.199453-05:00"} +{"id":"vgi-python-dvd","title":"Complete ReadOnlyCatalogInterface implementation","description":"Complete ReadOnlyCatalogInterface that prevents all DDL operations.\n\nFile: vgi/catalog/catalog_interface.py (already exists)\n\nCurrent state: Only has class attributes, no method overrides.\n\nChanges needed:\n1. Override all DDL methods to raise CatalogReadOnlyError:\n - catalog_create, catalog_drop\n - schema_create, schema_drop\n - table_create, table_drop, table_rename, table_comment_set\n - All table_column_* methods\n - table_not_null_set, table_not_null_drop\n - view_create, view_drop, view_rename, view_comment_set\n - Transaction methods (begin/commit/rollback)\n\n2. Create CatalogReadOnlyError exception in vgi/exceptions.py\n\n3. Ensure catalog_attach returns supports_transactions=False\n\nInclude tests verifying all DDL operations raise CatalogReadOnlyError.","status":"in_progress","priority":2,"issue_type":"task","created_at":"2026-01-05T19:27:15.199453-05:00","created_by":"rusty","updated_at":"2026-01-05T19:54:28.722579-05:00"} {"id":"vgi-python-dvo","title":"Export AnyValue in vgi/__init__.py","description":"Import and add AnyValue to __all__ exports","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T10:41:41.65732-05:00","created_by":"rusty","updated_at":"2026-01-05T11:07:09.187969-05:00","closed_at":"2026-01-05T11:07:09.187969-05:00","close_reason":"Exported AnyArrow in vgi/__init__.py","dependencies":[{"issue_id":"vgi-python-dvo","depends_on_id":"vgi-python-ckg","type":"blocks","created_at":"2026-01-05T10:41:48.715634-05:00","created_by":"rusty"}]} {"id":"vgi-python-e37","title":"move Invocation from function.py out to own file","description":"The Invocation clas is kind of seperate from functions, so it should be in its own file. Move it and all of its other associated classes like InvocationType to its own file","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-04T09:18:46.605941-05:00","created_by":"rusty","updated_at":"2026-01-04T09:24:37.922675-05:00","closed_at":"2026-01-04T09:24:37.922675-05:00","close_reason":"Closed"} {"id":"vgi-python-e46","title":"Create vgi/catalog/client.py - CatalogClient class","description":"Create CatalogClient for client-side catalog operations.\n\nFiles to create:\n- vgi/catalog/client.py\n\nCatalogClient class:\n- __init__(worker_command: str)\n- Context manager support (__enter__, __exit__)\n- start() / stop() methods\n\nCore methods (mirroring CatalogInterface):\n- catalogs() -\u003e list[str]\n- attach(name, options) -\u003e CatalogAttachResult\n- detach(attach_id) -\u003e None\n- schemas(attach_id, transaction_id) -\u003e Iterator[SchemaInfo]\n- schema_get(attach_id, transaction_id, name) -\u003e SchemaInfo | None\n- schema_contents(attach_id, transaction_id, name) -\u003e Iterator[TableInfo | ViewInfo | FunctionInfo]\n- table_get(...) -\u003e TableInfo | None\n- view_get(...) -\u003e ViewInfo | None\n- function_get(...) -\u003e FunctionInfo | None\n- table_scan_function_get(...) -\u003e ScanFunctionResult\n\nDDL methods (optional, may raise NotImplementedError from worker):\n- catalog_create, catalog_drop\n- schema_create, schema_drop\n- table_create, table_drop, table_rename, etc.\n- view_create, view_drop, view_rename, etc.\n\nTransaction methods:\n- transaction_begin(attach_id) -\u003e TransactionId | None\n- transaction_commit(attach_id, transaction_id)\n- transaction_rollback(attach_id, transaction_id)\n\nInternal methods:\n- _invoke(method_name, **kwargs) -\u003e pa.RecordBatch | Iterator[pa.RecordBatch]\n- _create_invocation(method_name, kwargs) -\u003e Invocation\n- _deserialize_result(batch, return_type) -\u003e Any\n\nHandle:\n- Streaming responses for Iterable returns\n- Exception propagation from worker\n- None returns (0-row/0-column batches)","status":"closed","priority":1,"issue_type":"task","created_at":"2026-01-05T19:18:04.65125-05:00","created_by":"rusty","updated_at":"2026-01-05T19:21:50.055794-05:00","closed_at":"2026-01-05T19:21:50.055794-05:00","close_reason":"User requested closure","dependencies":[{"issue_id":"vgi-python-e46","depends_on_id":"vgi-python-tw7","type":"blocks","created_at":"2026-01-05T19:18:44.316642-05:00","created_by":"rusty"},{"issue_id":"vgi-python-e46","depends_on_id":"vgi-python-fd2","type":"blocks","created_at":"2026-01-05T19:18:44.440065-05:00","created_by":"rusty"},{"issue_id":"vgi-python-e46","depends_on_id":"vgi-python-4mg","type":"blocks","created_at":"2026-01-05T19:18:44.559963-05:00","created_by":"rusty"}]} -{"id":"vgi-python-e6o","title":"Implement CatalogClient class","description":"Create CatalogClient for client-side catalog operations.\n\nFile: vgi/client/catalog_client.py\n\nCatalogClient class:\n- __init__(worker_command: str)\n- Each method call spawns new worker (matches VGI short-lived pattern)\n\nCore methods mirroring CatalogInterface:\n- catalogs() -\u003e list[str]\n- catalog_attach(name, options) -\u003e CatalogAttachResult\n- catalog_detach(attach_id) -\u003e None\n- schemas(attach_id, transaction_id) -\u003e Iterator[SchemaInfo]\n- schema_get(...) -\u003e SchemaInfo | None\n- schema_contents(...) -\u003e Iterator[TableInfo | ViewInfo | FunctionInfo]\n- table_get(...) -\u003e TableInfo | None\n- view_get(...) -\u003e ViewInfo | None\n- table_scan_function_get(...) -\u003e ScanFunctionResult\n\nDDL methods (may raise NotImplementedError from worker):\n- catalog_create, catalog_drop\n- schema_create, schema_drop\n- table_* methods, view_* methods\n\nTransaction methods:\n- catalog_transaction_begin/commit/rollback\n\nInternal:\n- _invoke(method_name, **kwargs) -\u003e pa.RecordBatch | Iterator[pa.RecordBatch]\n- _create_invocation(method_name, kwargs) -\u003e Invocation (with InvocationType.CATALOG)\n- Uses existing IPC utilities for communication","status":"in_progress","priority":1,"issue_type":"task","created_at":"2026-01-05T19:26:57.975309-05:00","created_by":"rusty","updated_at":"2026-01-05T19:44:28.358853-05:00","dependencies":[{"issue_id":"vgi-python-e6o","depends_on_id":"vgi-python-085","type":"blocks","created_at":"2026-01-05T19:27:50.730122-05:00","created_by":"rusty"},{"issue_id":"vgi-python-e6o","depends_on_id":"vgi-python-po3","type":"blocks","created_at":"2026-01-05T19:27:50.762036-05:00","created_by":"rusty"}]} +{"id":"vgi-python-e6o","title":"Implement CatalogClient class","description":"Create CatalogClient for client-side catalog operations.\n\nFile: vgi/client/catalog_client.py\n\nCatalogClient class:\n- __init__(worker_command: str)\n- Each method call spawns new worker (matches VGI short-lived pattern)\n\nCore methods mirroring CatalogInterface:\n- catalogs() -\u003e list[str]\n- catalog_attach(name, options) -\u003e CatalogAttachResult\n- catalog_detach(attach_id) -\u003e None\n- schemas(attach_id, transaction_id) -\u003e Iterator[SchemaInfo]\n- schema_get(...) -\u003e SchemaInfo | None\n- schema_contents(...) -\u003e Iterator[TableInfo | ViewInfo | FunctionInfo]\n- table_get(...) -\u003e TableInfo | None\n- view_get(...) -\u003e ViewInfo | None\n- table_scan_function_get(...) -\u003e ScanFunctionResult\n\nDDL methods (may raise NotImplementedError from worker):\n- catalog_create, catalog_drop\n- schema_create, schema_drop\n- table_* methods, view_* methods\n\nTransaction methods:\n- catalog_transaction_begin/commit/rollback\n\nInternal:\n- _invoke(method_name, **kwargs) -\u003e pa.RecordBatch | Iterator[pa.RecordBatch]\n- _create_invocation(method_name, kwargs) -\u003e Invocation (with InvocationType.CATALOG)\n- Uses existing IPC utilities for communication","status":"closed","priority":1,"issue_type":"task","created_at":"2026-01-05T19:26:57.975309-05:00","created_by":"rusty","updated_at":"2026-01-05T19:48:33.366915-05:00","closed_at":"2026-01-05T19:48:33.366915-05:00","close_reason":"PR #27 created with CatalogClient implementation","dependencies":[{"issue_id":"vgi-python-e6o","depends_on_id":"vgi-python-085","type":"blocks","created_at":"2026-01-05T19:27:50.730122-05:00","created_by":"rusty"},{"issue_id":"vgi-python-e6o","depends_on_id":"vgi-python-po3","type":"blocks","created_at":"2026-01-05T19:27:50.762036-05:00","created_by":"rusty"}]} {"id":"vgi-python-e9q","title":"Unify ProtocolOutput classes with shared base","description":"ProtocolOutput classes in table_function.py:177-224 and table_in_out_function.py:144-207 share similar metadata() method and from_process_result() classmethod. The table_in_out version adds status field. Create shared base with table_in_out extending it for status support.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-04T20:06:41.45014-05:00","created_by":"rusty","updated_at":"2026-01-04T21:54:55.871986-05:00","closed_at":"2026-01-04T21:54:55.871986-05:00","close_reason":"Not warranted - dataclass inheritance with slots=True doesn't allow adding required field (status) between inherited fields. The classes have different semantics (table_in_out requires status for generator state tracking) making inheritance impractical."} {"id":"vgi-python-eg7","title":"Create InMemoryCatalog example implementation","description":"Create an in-memory catalog implementation for testing and as an example.\n\nFile: vgi/examples/catalog.py\n\nInMemoryCatalog(CatalogInterface):\n- In-memory storage using dicts\n- Implements all required abstract methods\n- Implements common optional methods (schema_create, table_create, etc.)\n- Generates attach_id as random UUID bytes\n- Does NOT support transactions (returns None)\n\nData structures:\n- _catalogs: dict[str, CatalogData]\n- _attachments: dict[AttachId, str] # attach_id -\u003e catalog_name\n\nCreate example worker:\n```python\nclass InMemoryCatalogWorker(Worker):\n catalog_interface = InMemoryCatalog\n```\n\nAdd entry point: vgi-example-catalog-worker","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-05T19:27:27.604912-05:00","created_by":"rusty","updated_at":"2026-01-05T19:27:27.604912-05:00","dependencies":[{"issue_id":"vgi-python-eg7","depends_on_id":"vgi-python-085","type":"blocks","created_at":"2026-01-05T19:27:50.87322-05:00","created_by":"rusty"}]} {"id":"vgi-python-f5z","title":"Create vgi/catalog/storage.py - Catalog persistence","description":"Create storage layer for catalog attach_id and transaction_id persistence.\n\nFiles to create:\n- vgi/catalog/storage.py\n\nCatalogStorage protocol (similar to FunctionStorage):\n- attach_put(attach_id, catalog_name, options) -\u003e None\n- attach_get(attach_id) -\u003e tuple[str, dict] | None\n- attach_delete(attach_id) -\u003e None\n- attach_list() -\u003e list[AttachId]\n\n- transaction_put(transaction_id, attach_id, state) -\u003e None\n- transaction_get(transaction_id) -\u003e tuple[AttachId, bytes] | None\n- transaction_delete(transaction_id) -\u003e None\n\nCatalogStorageSqlite implementation:\n- Default location: ~/.state/vgi/vgi_catalog.db\n- WAL mode for concurrent access\n- Schema:\n CREATE TABLE catalog_attachments (\n attach_id BLOB PRIMARY KEY,\n catalog_name TEXT NOT NULL,\n options TEXT, -- JSON\n created_at REAL DEFAULT (julianday('now'))\n )\n CREATE TABLE catalog_transactions (\n transaction_id BLOB PRIMARY KEY,\n attach_id BLOB NOT NULL,\n state BLOB,\n created_at REAL DEFAULT (julianday('now'))\n )\n\nInclude cleanup strategies for stale attachments/transactions.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T19:18:04.531387-05:00","created_by":"rusty","updated_at":"2026-01-05T19:21:50.073983-05:00","closed_at":"2026-01-05T19:21:50.073983-05:00","close_reason":"User requested closure","dependencies":[{"issue_id":"vgi-python-f5z","depends_on_id":"vgi-python-tw7","type":"blocks","created_at":"2026-01-05T19:18:44.194468-05:00","created_by":"rusty"}]} diff --git a/vgi/catalog/catalog_interface.py b/vgi/catalog/catalog_interface.py index 6e8d673..b791058 100644 --- a/vgi/catalog/catalog_interface.py +++ b/vgi/catalog/catalog_interface.py @@ -896,7 +896,357 @@ class ReadOnlyCatalogInterface(CatalogInterface): This is a convenience base class for catalogs that only support reading metadata and data, not creating or modifying objects. + + Subclasses must implement: + - catalogs() - List available catalogs + - catalog_attach() - Attach to a catalog + - schema_get() - Get schema information + - table_get() - Get table information + - view_get() - Get view information + + Optional methods that can be overridden: + - catalog_detach() - Custom detach logic + - schemas() - Custom schema listing (default returns 'main') + - schema_contents() - List schema contents + - table_scan_function_get() - Get scan function for tables + + All DDL operations (create, drop, rename, modify) will raise + CatalogReadOnlyError. """ supports_transactions = False catalog_version_frozen = True + + # ========== Catalog DDL (not supported) ========== + + def catalog_create( + self, *, name: str, on_conflict: OnConflict, options: dict[str, Any] + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot create catalog: catalog is read-only") + + def catalog_drop(self, *, name: str) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot drop catalog: catalog is read-only") + + # ========== Transaction methods (not supported) ========== + + def catalog_transaction_begin(self, *, attach_id: AttachId) -> TransactionId | None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError( + "Cannot begin transaction: catalog is read-only and does not support " + "transactions" + ) + + def catalog_transaction_commit( + self, *, attach_id: AttachId, transaction_id: TransactionId + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError( + "Cannot commit transaction: catalog is read-only and does not support " + "transactions" + ) + + def catalog_transaction_rollback( + self, *, attach_id: AttachId, transaction_id: TransactionId + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError( + "Cannot rollback transaction: catalog is read-only and does not support " + "transactions" + ) + + # ========== Schema DDL (not supported) ========== + + def schema_create( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + name: str, + comment: str | None, + tags: dict[str, str], + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot create schema: catalog is read-only") + + def schema_drop( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + name: str, + ignore_not_found: bool, + cascade: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot drop schema: catalog is read-only") + + # ========== Table DDL (not supported) ========== + + def table_create( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + columns: SerializedSchema, + on_conflict: OnConflict, + not_null_constraints: list[int], + unique_constraints: list[list[int]], + check_constraints: list[str], + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot create table: catalog is read-only") + + def table_drop( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot drop table: catalog is read-only") + + def table_comment_set( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + comment: str | None, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot set table comment: catalog is read-only") + + def table_rename( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + new_name: str, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot rename table: catalog is read-only") + + def table_column_add( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + column_definition: SerializedSchema, + ignore_not_found: bool, + if_column_not_exists: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot add column: catalog is read-only") + + def table_column_drop( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + column_name: str, + ignore_not_found: bool, + if_column_exists: bool, + cascade: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot drop column: catalog is read-only") + + def table_column_rename( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + column_name: str, + new_column_name: str, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot rename column: catalog is read-only") + + def table_column_default_set( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + column_name: str, + expression: SqlExpression, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot set column default: catalog is read-only") + + def table_column_default_drop( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + column_name: str, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot drop column default: catalog is read-only") + + def table_column_type_change( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + column_definition: SerializedSchema, + expression: SqlExpression | None, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot change column type: catalog is read-only") + + def table_not_null_drop( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + column_name: str, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError( + "Cannot drop NOT NULL constraint: catalog is read-only" + ) + + def table_not_null_set( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + column_name: str, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError( + "Cannot set NOT NULL constraint: catalog is read-only" + ) + + # ========== View DDL (not supported) ========== + + def view_create( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + definition: str, + on_conflict: OnConflict, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot create view: catalog is read-only") + + def view_drop( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot drop view: catalog is read-only") + + def view_rename( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + new_name: str, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot rename view: catalog is read-only") + + def view_comment_set( + self, + *, + attach_id: AttachId, + transaction_id: TransactionId | None, + schema_name: str, + name: str, + comment: str | None, + ignore_not_found: bool, + ) -> None: + """Not supported - raises CatalogReadOnlyError.""" + from vgi.exceptions import CatalogReadOnlyError + + raise CatalogReadOnlyError("Cannot set view comment: catalog is read-only") diff --git a/vgi/exceptions.py b/vgi/exceptions.py index 4eaba63..58eae89 100644 --- a/vgi/exceptions.py +++ b/vgi/exceptions.py @@ -16,11 +16,30 @@ import pyarrow as pa __all__ = [ + "CatalogReadOnlyError", "ExecutionIdentifierError", "SchemaValidationError", ] +class CatalogReadOnlyError(Exception): + """Raised when a DDL operation is attempted on a read-only catalog. + + This exception is raised by ReadOnlyCatalogInterface when any + create, drop, rename, or modify operation is attempted. + + Read-only catalogs only support: + - catalogs() - list catalogs + - catalog_attach/detach - attach to/detach from catalogs + - schemas() - list schemas + - schema_get() - get schema info + - schema_contents() - list schema contents + - table_get(), view_get() - get table/view info + - table_scan_function_get() - get scan function for tables + + """ + + class ExecutionIdentifierError(ValueError): """Raised when an operation requires an execution_identifier that hasn't been set.