From 0ccf669bc32dca0fce07a20d3774f2b83c31e36e Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Mon, 5 Jan 2026 15:48:17 -0500 Subject: [PATCH 1/2] feat: Add PYTHON_TO_ARROW type mapping to arguments.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a dictionary mapping Python types to PyArrow types for use in argument type inference. This mapping is exported in __all__. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .beads/issues.jsonl | 11 +++++++++-- vgi/arguments.py | 10 ++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 2975647..c74d70c 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,6 +1,7 @@ {"id":"vgi-python-0fe","title":"Add is_varargs to ParameterInfo and metadata extraction","description":"In vgi/metadata.py:\n- Add is_varargs: bool = False to ParameterInfo\n- Update to_dict() and from_dict()\n- Add is_varargs field to _PARAMETER_STRUCT for Arrow serialization\n- Extract varargs flag in extract_parameters()\n- Add _validate_varargs() with rules:\n - Only one varargs parameter allowed\n - Must be positional (not named)\n - Must be last positional (before TableInput if present)\n - Cannot have default value","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T10:49:20.141375-05:00","created_by":"rusty","updated_at":"2026-01-05T10:58:21.242603-05:00","closed_at":"2026-01-05T10:58:21.242603-05:00","close_reason":"Added is_varargs to ParameterInfo, _PARAMETER_STRUCT, extract_parameters(), and _validate_varargs()","dependencies":[{"issue_id":"vgi-python-0fe","depends_on_id":"vgi-python-jrf","type":"blocks","created_at":"2026-01-05T10:49:26.421664-05:00","created_by":"rusty"}]} {"id":"vgi-python-0hr","title":"Remove redundant InitInputType class attribute","description":"InitInputType class attribute duplicates the generic type parameter: 'class ScalarFunctionGenerator(Function[FunctionInitInput])' already specifies the type, but 'InitInputType = FunctionInitInput' repeats it. Investigate using get_type_hints or __orig_bases__ to infer the type and remove the redundant attribute.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-04T20:06:53.780529-05:00","created_by":"rusty","updated_at":"2026-01-04T22:00:40.221423-05:00","closed_at":"2026-01-04T22:00:40.221423-05:00","close_reason":"PR #10 created - uses _get_init_input_type() to infer type from generic parameter"} {"id":"vgi-python-1s5","title":"Move distributed state management to optional mixin","description":"The Function base class in function.py includes ~200 lines for distributed state management (store_state, collect_states, enqueue_work, dequeue_work, work queue storage). Not all functions need this. Extract to DistributedStateMixin that functions can opt into, keeping Function base class simpler for basic use cases.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-04T20:06:53.606614-05:00","created_by":"rusty","updated_at":"2026-01-04T21:22:09.772825-05:00","closed_at":"2026-01-04T21:22:09.772825-05:00","close_reason":"Analysis complete: extraction not recommended. The distributed state methods are tightly coupled with execution_identifier and storage, which are used by core initialization methods. Extraction would require moving initialize_global_state/load_global_state to the mixin, breaking the protocol and requiring multiple inheritance. Current API is already opt-in (just don't call the methods) and well-documented."} +{"id":"vgi-python-229","title":"Add tests for example function Arrow serialization","description":"Add to tests/test_argument_spec.py:\n\nclass TestExampleFunctionsSerialization:\n- test_scalar_functions: DoubleColumnFunction, AddColumnsFunction, UpperCaseFunction\n- test_table_functions: SequenceFunction, RangeFunction, ConstantTableFunction\n- test_table_in_out_functions: EchoFunction, BufferInputFunction, RepeatInputsFunction, SumAllColumnsFunction\n\nEach test should:\n1. Call extract_argument_specs(cls)\n2. Verify all specs have non-null arrow_type\n3. Verify schema roundtrip works","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:57.064847-05:00","created_by":"rusty","updated_at":"2026-01-05T15:44:57.064847-05:00","dependencies":[{"issue_id":"vgi-python-229","depends_on_id":"vgi-python-coi","type":"blocks","created_at":"2026-01-05T15:45:14.210202-05:00","created_by":"rusty"}]} {"id":"vgi-python-29x","title":"Add tests for varargs metadata validation","description":"In tests/test_metadata.py:\n- Varargs must be positional - error if named\n- Only one varargs allowed - error if multiple\n- Varargs must be last positional - error if regular arg after\n- Varargs + TableInput ordering works correctly\n- Arrow serialization round-trip preserves is_varargs\n- ParameterInfo.is_varargs correctly extracted","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T10:49:20.406998-05:00","created_by":"rusty","updated_at":"2026-01-05T11:02:41.043316-05:00","closed_at":"2026-01-05T11:02:41.043316-05:00","close_reason":"Added tests for varargs metadata extraction and validation","dependencies":[{"issue_id":"vgi-python-29x","depends_on_id":"vgi-python-jrf","type":"blocks","created_at":"2026-01-05T10:49:26.482596-05:00","created_by":"rusty"},{"issue_id":"vgi-python-29x","depends_on_id":"vgi-python-0fe","type":"blocks","created_at":"2026-01-05T10:49:26.512138-05:00","created_by":"rusty"}]} {"id":"vgi-python-2ln","title":"Add varargs support for trailing positional arguments","description":"Allow a single trailing positional argument to accept one or more values of the same type using Arg[T](position, varargs=True). Returns tuple[T, ...]. Requires at least 1 value. Must be last positional arg (before TableInput if present).","status":"closed","priority":2,"issue_type":"feature","created_at":"2026-01-05T10:49:07.327391-05:00","created_by":"rusty","updated_at":"2026-01-05T11:02:41.151847-05:00","closed_at":"2026-01-05T11:02:41.151847-05:00","close_reason":"Completed varargs support for trailing positional arguments"} {"id":"vgi-python-35i","title":"Test SchemaValidationError detailed message paths","notes":"Coverage: 67% in vgi/exceptions.py. Missing tests for:\n- Lines 116-123: Type mismatch detection in _build_detailed_message\n- Lines 128-131: Field order difference detection \n- Lines 149-151: Type mismatch reporting\n- Lines 155-157: Field order difference reporting\n\nTest scenarios needed:\n1. Schema with same fields but different types\n2. Schema with nullable vs non-nullable mismatch\n3. Schema with same fields in different order","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-04T22:15:25.858704-05:00","created_by":"rusty","updated_at":"2026-01-04T22:25:58.697852-05:00","closed_at":"2026-01-04T22:25:58.697852-05:00","close_reason":"Added comprehensive tests for SchemaValidationError. Coverage improved from 67% to 99%."} @@ -18,6 +19,7 @@ {"id":"vgi-python-aad","title":"Design: DuckDB settings/pragmas access for VGI functions","description":"Design how VGI functions can declare required DuckDB settings/pragmas in their Meta class, and how these settings values should be passed during the bind phase.\n\nKey design decisions:\n1. How to declare required settings in function Meta (e.g., required_settings = ['timezone', 'threads'])\n2. How to add settings to Invocation dataclass\n3. How settings values should be accessed in function code\n4. Serialization format for settings in Arrow IPC\n\nRecommendation: Add 'duckdb_settings: dict[str, str] | None' to Invocation and 'required_settings: list[str]' to Meta class.","status":"closed","priority":1,"issue_type":"task","created_at":"2026-01-04T13:05:47.619105-05:00","created_by":"rusty","updated_at":"2026-01-04T13:11:13.197139-05:00","closed_at":"2026-01-04T13:11:13.197139-05:00","close_reason":"Design document created at docs/design-duckdb-settings.md"} {"id":"vgi-python-awm","title":"Add AnyValue sentinel type for arguments accepting any Arrow type","status":"closed","priority":2,"issue_type":"feature","created_at":"2026-01-05T10:41:33.898226-05:00","created_by":"rusty","updated_at":"2026-01-05T11:10:17.138416-05:00","closed_at":"2026-01-05T11:10:17.138416-05:00","close_reason":"Closed"} {"id":"vgi-python-bi8","title":"Extract common _process_with_exception_handling into mixin","description":"The _process_with_exception_handling and _process_and_validate methods are duplicated across scalar_function.py:296-346, table_function.py:386-438, and table_in_out_function.py:586-642. All follow same pattern: try _process_and_validate, catch exceptions, return OutputComplete with error message. Extract to ProcessingMixin that all function types inherit from.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-04T20:06:41.02111-05:00","created_by":"rusty","updated_at":"2026-01-04T21:49:46.320532-05:00","closed_at":"2026-01-04T21:49:46.320532-05:00","close_reason":"PR #8 created - extracted _should_terminate and added _create_error_output to Function base class","dependencies":[{"issue_id":"vgi-python-bi8","depends_on_id":"vgi-python-6o0","type":"blocks","created_at":"2026-01-04T20:07:49.181408-05:00","created_by":"rusty"}]} +{"id":"vgi-python-bjl","title":"Add tests for Arg.arrow_type functionality","description":"Add to tests/test_argument_spec.py:\n\nclass TestArgArrowType:\n- test_explicit_arrow_type_stored: Verify arrow_type is stored\n- test_default_arrow_type_is_none: Verify default is None\n\nclass TestExtractArgumentSpecsAutoInference:\n- test_int_infers_int64\n- test_str_infers_utf8\n- test_float_infers_float64\n- test_explicit_overrides_inference","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:56.945537-05:00","created_by":"rusty","updated_at":"2026-01-05T15:44:56.945537-05:00","dependencies":[{"issue_id":"vgi-python-bjl","depends_on_id":"vgi-python-coi","type":"blocks","created_at":"2026-01-05T15:45:14.093359-05:00","created_by":"rusty"}]} {"id":"vgi-python-bku","title":"Change cardinality() method to property for consistency with output_schema","description":"Inconsistent access patterns: output_schema is a property but cardinality() is a method. Both return immutable data. Change cardinality() to a property for API consistency. Located in table_function.py:304-314.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-04T20:06:53.211782-05:00","created_by":"rusty","updated_at":"2026-01-04T21:33:10.617152-05:00","closed_at":"2026-01-04T21:33:10.617152-05:00","close_reason":"PR #6 created: https://github.com/Query-farm/vgi-python/pull/6"} {"id":"vgi-python-bkz","title":"Update metadata.py to detect AnyValue type","description":"Update _get_arg_type_info() to handle AnyValue like TableInput","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T10:41:41.534263-05:00","created_by":"rusty","updated_at":"2026-01-05T11:06:27.291747-05:00","closed_at":"2026-01-05T11:06:27.291747-05:00","close_reason":"Updated _get_arg_type_info() to detect AnyArrow type","dependencies":[{"issue_id":"vgi-python-bkz","depends_on_id":"vgi-python-ckg","type":"blocks","created_at":"2026-01-05T10:41:48.678841-05:00","created_by":"rusty"}]} {"id":"vgi-python-bq4","title":"Update vgi/__init__.py with argument_spec exports","description":"## Overview\n\nUpdate the main package `__init__.py` to export the new argument specification classes and functions.\n\n## File Location\n\n`vgi/__init__.py`\n\n## Changes Required\n\n### Add Import Statement\n\nAdd import from the new module:\n\n```python\nfrom vgi.argument_spec import (\n ArgumentSpec,\n argument_specs_to_schema,\n schema_to_argument_specs,\n)\n```\n\n### Update __all__ List\n\nAdd the new exports to `__all__`:\n\n```python\n__all__ = [\n # ... existing exports ...\n \"ArgumentSpec\",\n \"argument_specs_to_schema\",\n \"schema_to_argument_specs\",\n]\n```\n\n### Placement\n\n- Import should be grouped with other argument-related imports (near `Arg`, `Arguments`)\n- Exports in `__all__` should be alphabetically sorted\n\n## Verification\n\nAfter changes, verify:\n1. `from vgi import ArgumentSpec` works\n2. `from vgi import argument_specs_to_schema` works\n3. `from vgi import schema_to_argument_specs` works","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T11:19:25.519702-05:00","created_by":"rusty","updated_at":"2026-01-05T11:34:07.676358-05:00","closed_at":"2026-01-05T11:34:07.676358-05:00","close_reason":"Added ArgumentSpec, argument_specs_to_schema, schema_to_argument_specs exports","dependencies":[{"issue_id":"vgi-python-bq4","depends_on_id":"vgi-python-cd0","type":"blocks","created_at":"2026-01-05T11:19:30.860732-05:00","created_by":"rusty"}]} @@ -25,7 +27,10 @@ {"id":"vgi-python-c2b","title":"Add duckdb_settings field to Invocation class","description":"Update vgi/invocation.py to add a duckdb_settings field to the Invocation dataclass.\n\nChanges needed:\n- Add 'duckdb_settings: dict[str, str] | None = None' field to Invocation\n- Update serialize() to include settings in Arrow IPC batch\n- Update deserialize() to read settings from Arrow IPC batch\n- Handle None case (no settings requested)\n\nSerialization: Use a struct field with string key-value pairs or a map type.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-04T13:05:47.765077-05:00","created_by":"rusty","updated_at":"2026-01-04T13:20:41.167817-05:00","closed_at":"2026-01-04T13:20:41.167817-05:00","close_reason":"Implementation complete, all tests pass","dependencies":[{"issue_id":"vgi-python-c2b","depends_on_id":"vgi-python-aad","type":"blocks","created_at":"2026-01-04T13:06:13.664038-05:00","created_by":"rusty"}]} {"id":"vgi-python-cd0","title":"Create vgi/argument_spec.py module","description":"## Overview\n\nCreate the core module implementing Arrow-based argument specification serialization.\n\n## File Location\n\n`vgi/argument_spec.py`\n\n## Constants to Define\n\n```python\n# Metadata keys (all bytes for Arrow compatibility)\nVGI_ARG_KEY = b\"vgi_arg\"\nVGI_ARG_NAMED = b\"named\"\n\nVGI_TYPE_KEY = b\"vgi_type\"\nVGI_TYPE_TABLE = b\"table\"\nVGI_TYPE_ANY = b\"any\"\n\nVGI_VARARGS_KEY = b\"vgi_varargs\"\nVGI_VARARGS_TRUE = b\"true\"\n```\n\n## ArgumentSpec Dataclass\n\n```python\n@dataclass(frozen=True)\nclass ArgumentSpec:\n \"\"\"Specification for a single function argument.\"\"\"\n name: str # Python attribute name\n position: int | str # int for positional index, str for named key\n arrow_type: pa.DataType # Arrow type (pa.null() for special types)\n is_table_input: bool = False # Arg[TableInput]\n is_any_type: bool = False # Arg[AnyArrow]\n is_varargs: bool = False # varargs=True\n```\n\n## Functions to Implement\n\n### argument_specs_to_schema(specs: Sequence[ArgumentSpec]) -\u003e pa.Schema\n\nConvert ArgumentSpecs to a single Arrow schema:\n1. Sort specs: positional first (by index), then named\n2. For each spec, create a pa.field with:\n - name = spec.name\n - type = spec.arrow_type (or pa.null() for table/any)\n - metadata = appropriate markers based on flags\n3. Return pa.schema(fields)\n\n### schema_to_argument_specs(schema: pa.Schema) -\u003e list[ArgumentSpec]\n\nConvert schema back to ArgumentSpecs:\n1. Iterate through schema fields in order\n2. Track position index (increments for non-named args)\n3. Check field metadata for markers:\n - `vgi_arg=named` -\u003e position is field name string\n - `vgi_type=table` -\u003e is_table_input=True\n - `vgi_type=any` -\u003e is_any_type=True\n - `vgi_varargs=true` -\u003e is_varargs=True\n4. Return list of ArgumentSpec\n\n### extract_argument_specs(cls: type, arg_types: dict[str, pa.DataType]) -\u003e list[ArgumentSpec]\n\nExtract specs from a function class with Arg descriptors:\n1. Walk class MRO to find all Arg descriptors (like extract_parameters in metadata.py)\n2. For each Arg descriptor:\n - Get name from attribute name\n - Get position from arg.position\n - Get arrow_type from arg_types dict\n - Check type hints for TableInput/AnyArrow\n - Check arg.varargs flag\n3. Sort and return list\n\n## Dependencies\n\n- Import `Arg`, `TableInput`, `AnyArrow` from `vgi.arguments`\n- Reference `extract_parameters()` pattern in `vgi/metadata.py`","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T11:18:32.777241-05:00","created_by":"rusty","updated_at":"2026-01-05T11:28:07.227452-05:00","closed_at":"2026-01-05T11:28:07.227452-05:00","close_reason":"Created vgi/argument_spec.py with ArgumentSpec dataclass and serialization functions","dependencies":[{"issue_id":"vgi-python-cd0","depends_on_id":"vgi-python-8ra","type":"blocks","created_at":"2026-01-05T11:19:30.743936-05:00","created_by":"rusty"}]} {"id":"vgi-python-ckg","title":"Add AnyValue sentinel class to vgi/arguments.py","description":"Add AnyValue class similar to TableInput, export in __all__","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T10:41:41.392694-05:00","created_by":"rusty","updated_at":"2026-01-05T11:05:38.37392-05:00","closed_at":"2026-01-05T11:05:38.37392-05:00","close_reason":"Added AnyArrow sentinel class to arguments.py","dependencies":[{"issue_id":"vgi-python-ckg","depends_on_id":"vgi-python-awm","type":"blocks","created_at":"2026-01-05T10:41:52.658405-05:00","created_by":"rusty"}]} +{"id":"vgi-python-coi","title":"Update extract_argument_specs() to remove arg_types parameter","description":"In vgi/argument_spec.py:\n1. Remove arg_types parameter from function signature\n2. Update arrow_type resolution logic:\n - Use arg.arrow_type if explicitly set\n - Infer from Python type hint using PYTHON_TO_ARROW\n - Handle TableInput/AnyArrow → pa.null()\n - Warn and default to pa.null() for unknown types\n3. Import PYTHON_TO_ARROW from vgi.arguments","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:38.141157-05:00","created_by":"rusty","updated_at":"2026-01-05T15:44:38.141157-05:00","dependencies":[{"issue_id":"vgi-python-coi","depends_on_id":"vgi-python-cvj","type":"blocks","created_at":"2026-01-05T15:45:13.831745-05:00","created_by":"rusty"},{"issue_id":"vgi-python-coi","depends_on_id":"vgi-python-dv0","type":"blocks","created_at":"2026-01-05T15:45:13.864608-05:00","created_by":"rusty"}]} +{"id":"vgi-python-cvj","title":"Add PYTHON_TO_ARROW type mapping to vgi/arguments.py","description":"Add the Python→Arrow type mapping dict after imports:\n```python\nPYTHON_TO_ARROW: dict[type, pa.DataType] = {\n int: pa.int64(),\n str: pa.utf8(),\n float: pa.float64(),\n bool: pa.bool_(),\n bytes: pa.binary(),\n}\n```\nExport in __all__.","status":"in_progress","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:37.900421-05:00","created_by":"rusty","updated_at":"2026-01-05T15:46:01.292126-05:00"} {"id":"vgi-python-d73","title":"Create docs/argument-serialization.md","description":"## Overview\n\nCreate LLM-friendly documentation explaining the argument specification serialization format. This document should enable future implementors (human or AI) to understand how function argument signatures are serialized to Arrow schemas.\n\n## File Location\n\n`docs/argument-serialization.md`\n\n## Document Structure\n\n### Title and Purpose\n\nExplain that this document describes how VGI function argument specifications are serialized to Apache Arrow schemas for IPC transmission and DuckDB function registration.\n\n### Quick Reference\n\nA concise summary table showing:\n- Metadata keys and their meanings\n- Special type representations\n\n### Schema Format\n\nExplain the single-schema design:\n1. All arguments are fields in one Arrow schema\n2. Positional arguments come first, in order (field index = position index)\n3. Named arguments follow, marked with metadata\n4. Field name = Python attribute name (or argument key for named)\n5. Field type = exact Arrow type\n\n### Metadata Keys Reference\n\nComplete table of all metadata keys:\n\n| Key | Value | Description |\n|-----|-------|-------------|\n| `vgi_arg` | `named` | Field is a named argument, not positional. The field name is the argument key. |\n| `vgi_type` | `table` | Argument receives streaming table input (Arg[TableInput]). Arrow type is pa.null(). |\n| `vgi_type` | `any` | Argument accepts any Arrow type (Arg[AnyArrow]). Arrow type is pa.null(). |\n| `vgi_varargs` | `true` | Argument collects all remaining positional args. Arrow type is the element type. |\n\n### Special Type Handling\n\nExplain how special argument types are represented:\n\n#### TableInput\n- Arrow type: `pa.null()`\n- Metadata: `{b\"vgi_type\": b\"table\"}`\n- Meaning: This position receives streaming RecordBatches, not a scalar value\n\n#### AnyArrow\n- Arrow type: `pa.null()`\n- Metadata: `{b\"vgi_type\": b\"any\"}`\n- Meaning: Accepts any valid Arrow scalar type at runtime\n\n#### Varargs\n- Arrow type: The element type (e.g., `pa.int64()` for `Arg[int](..., varargs=True)`)\n- Metadata: `{b\"vgi_varargs\": b\"true\"}`\n- Meaning: Collects all remaining positional arguments from this position onwards\n\n### Examples\n\n#### Example 1: Simple Function\n\n```python\nclass MyFunction(TableInOutFunction):\n count = Arg[int](0) # Positional 0\n name = Arg[str](1) # Positional 1\n verbose = Arg[bool](\"verbose\") # Named\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"count\", pa.int64()),\n pa.field(\"name\", pa.utf8()),\n pa.field(\"verbose\", pa.bool_(), metadata={b\"vgi_arg\": b\"named\"}),\n])\n```\n\n#### Example 2: Function with Table Input\n\n```python\nclass TransformFunction(TableInOutFunction):\n multiplier = Arg[float](0)\n data = Arg[TableInput](1)\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"multiplier\", pa.float64()),\n pa.field(\"data\", pa.null(), metadata={b\"vgi_type\": b\"table\"}),\n])\n```\n\n#### Example 3: Function with Varargs\n\n```python\nclass SumFunction(TableInOutFunction):\n columns = Arg[str](0, varargs=True)\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"columns\", pa.utf8(), metadata={b\"vgi_varargs\": b\"true\"}),\n])\n```\n\n#### Example 4: Complex Function\n\n```python\nclass ComplexFunction(TableInOutFunction):\n count = Arg[int](0)\n data = Arg[TableInput](1)\n extra = Arg[float](2, varargs=True)\n format = Arg[str](\"format\")\n threshold = Arg[AnyArrow](\"threshold\")\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"count\", pa.int64()),\n pa.field(\"data\", pa.null(), metadata={b\"vgi_type\": b\"table\"}),\n pa.field(\"extra\", pa.float64(), metadata={b\"vgi_varargs\": b\"true\"}),\n pa.field(\"format\", pa.utf8(), metadata={b\"vgi_arg\": b\"named\"}),\n pa.field(\"threshold\", pa.null(), metadata={b\"vgi_arg\": b\"named\", b\"vgi_type\": b\"any\"}),\n])\n```\n\n### Serialization Code\n\nShow how to serialize and deserialize:\n\n```python\n# Serialize to bytes\nschema_bytes = schema.serialize().to_pybytes()\n\n# Deserialize from bytes\nschema = pa.ipc.read_schema(pa.py_buffer(schema_bytes))\n```\n\n### Parsing Algorithm\n\nExplain how to parse a schema back to argument specs:\n\n1. Initialize position_index = 0\n2. For each field in schema:\n a. Check if field has `vgi_arg=named` metadata\n b. If named: position = field.name (string)\n c. If positional: position = position_index, then increment position_index\n d. Check for `vgi_type` metadata (table or any)\n e. Check for `vgi_varargs` metadata\n f. Create ArgumentSpec with extracted info\n\n### Not Included\n\nExplicitly state what is NOT serialized:\n- Default values\n- Validation constraints (ge, le, choices, pattern)\n- Documentation strings\n\nThese are Python-side concerns handled by the Arg descriptor at runtime.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T11:19:17.488877-05:00","created_by":"rusty","updated_at":"2026-01-05T11:33:29.168007-05:00","closed_at":"2026-01-05T11:33:29.168007-05:00","close_reason":"Created comprehensive LLM-friendly documentation","dependencies":[{"issue_id":"vgi-python-d73","depends_on_id":"vgi-python-8ra","type":"blocks","created_at":"2026-01-05T11:19:30.820384-05:00","created_by":"rusty"}]} +{"id":"vgi-python-dv0","title":"Add arrow_type parameter to Arg class","description":"In vgi/arguments.py:\n1. Add 'arrow_type' to __slots__\n2. Add parameter: arrow_type: pa.DataType | None = None\n3. Store: self.arrow_type = arrow_type\n4. Update __repr__ to include arrow_type if set","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:38.020395-05:00","created_by":"rusty","updated_at":"2026-01-05T15:44:38.020395-05:00","dependencies":[{"issue_id":"vgi-python-dv0","depends_on_id":"vgi-python-cvj","type":"blocks","created_at":"2026-01-05T15:45:13.696822-05:00","created_by":"rusty"}]} {"id":"vgi-python-dvo","title":"Export AnyValue in vgi/__init__.py","description":"Import and add AnyValue to __all__ exports","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T10:41:41.65732-05:00","created_by":"rusty","updated_at":"2026-01-05T11:07:09.187969-05:00","closed_at":"2026-01-05T11:07:09.187969-05:00","close_reason":"Exported AnyArrow in vgi/__init__.py","dependencies":[{"issue_id":"vgi-python-dvo","depends_on_id":"vgi-python-ckg","type":"blocks","created_at":"2026-01-05T10:41:48.715634-05:00","created_by":"rusty"}]} {"id":"vgi-python-e37","title":"move Invocation from function.py out to own file","description":"The Invocation clas is kind of seperate from functions, so it should be in its own file. Move it and all of its other associated classes like InvocationType to its own file","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-04T09:18:46.605941-05:00","created_by":"rusty","updated_at":"2026-01-04T09:24:37.922675-05:00","closed_at":"2026-01-04T09:24:37.922675-05:00","close_reason":"Closed"} {"id":"vgi-python-e9q","title":"Unify ProtocolOutput classes with shared base","description":"ProtocolOutput classes in table_function.py:177-224 and table_in_out_function.py:144-207 share similar metadata() method and from_process_result() classmethod. The table_in_out version adds status field. Create shared base with table_in_out extending it for status support.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-04T20:06:41.45014-05:00","created_by":"rusty","updated_at":"2026-01-04T21:54:55.871986-05:00","closed_at":"2026-01-04T21:54:55.871986-05:00","close_reason":"Not warranted - dataclass inheritance with slots=True doesn't allow adding required field (status) between inherited fields. The classes have different semantics (table_in_out requires status for generator state tracking) making inheritance impractical."} @@ -33,15 +38,17 @@ {"id":"vgi-python-g7i","title":"Add validation for contiguous positional indices","description":"Neither argument_specs_to_schema() nor schema_to_argument_specs() validates that positional argument indices are contiguous (0, 1, 2...). Gaps like (0, 2, 3) would serialize fine but might indicate a bug. Consider adding validation that positional indices form a contiguous sequence starting from 0.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-05T11:51:19.868862-05:00","created_by":"rusty","updated_at":"2026-01-05T11:56:01.878179-05:00","closed_at":"2026-01-05T11:56:01.878179-05:00","close_reason":"Closed"} {"id":"vgi-python-ivf","title":"Add required_settings to function Meta class","description":"Update function metadata to support declaring required DuckDB settings.\n\nChanges needed:\n- Add 'required_settings: list[str]' to FunctionMeta in vgi/metadata.py\n- Update Meta class resolution in vgi/function.py\n- Add validation that required_settings is a list of strings\n- Make it available via get_metadata() for introspection\n\nExample usage:\nclass MyFunction(TableInOutFunction):\n class Meta:\n required_settings = ['timezone', 'threads']","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-04T13:05:47.903747-05:00","created_by":"rusty","updated_at":"2026-01-04T13:20:41.169516-05:00","closed_at":"2026-01-04T13:20:41.169516-05:00","close_reason":"Implementation complete, all tests pass","dependencies":[{"issue_id":"vgi-python-ivf","depends_on_id":"vgi-python-aad","type":"blocks","created_at":"2026-01-04T13:06:13.690253-05:00","created_by":"rusty"}]} {"id":"vgi-python-j4t","title":"Update client to pass DuckDB settings in Invocation","description":"Update vgi/client/client.py to support passing DuckDB settings.\n\nChanges needed:\n- Add 'duckdb_settings: dict[str, str] | None = None' parameter to relevant methods\n- Include settings in Invocation creation\n- Add helper to query function's required_settings from metadata\n\nThe client needs to know what settings to pass. Options:\n1. Client queries worker for function metadata first\n2. Settings passed explicitly by caller\n3. Client introspects function class if available locally","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-04T13:05:48.358656-05:00","created_by":"rusty","updated_at":"2026-01-04T13:20:41.173178-05:00","closed_at":"2026-01-04T13:20:41.173178-05:00","close_reason":"Implementation complete, all tests pass","dependencies":[{"issue_id":"vgi-python-j4t","depends_on_id":"vgi-python-aad","type":"blocks","created_at":"2026-01-04T13:06:13.761572-05:00","created_by":"rusty"}]} -{"id":"vgi-python-j8a","title":"Investigate named argument field name ambiguity","description":"For named arguments, position is set to field.name (the SQL key). But there's potential ambiguity between the Python attribute name and the named argument key if they could differ. Currently they're the same, but if ArgumentSpec.name (attribute) ever differs from ArgumentSpec.position (key), the schema only preserves one. Investigate whether this is a real concern or document the assumption that they're always equal.","status":"open","priority":4,"issue_type":"task","created_at":"2026-01-05T11:51:20.257539-05:00","created_by":"rusty","updated_at":"2026-01-05T11:51:20.257539-05:00"} +{"id":"vgi-python-j8a","title":"Investigate named argument field name ambiguity","description":"For named arguments, position is set to field.name (the SQL key). But there's potential ambiguity between the Python attribute name and the named argument key if they could differ. Currently they're the same, but if ArgumentSpec.name (attribute) ever differs from ArgumentSpec.position (key), the schema only preserves one. Investigate whether this is a real concern or document the assumption that they're always equal.","status":"closed","priority":4,"issue_type":"task","created_at":"2026-01-05T11:51:20.257539-05:00","created_by":"rusty","updated_at":"2026-01-05T12:16:59.737914-05:00","closed_at":"2026-01-05T12:16:59.737914-05:00","close_reason":"Closed"} {"id":"vgi-python-j9k","title":"Add protocol types for IPC stream writers in cli.py","notes":"Line 53: self._writer: Any = None\n\nCould define a Protocol type for the IPC stream writer interface:\n```python\nclass IPCWriter(Protocol):\n def write_batch(self, batch: pa.RecordBatch) -\u003e None: ...\n def close(self) -\u003e None: ...\n```\n\nPart of 14.17% imprecision in cli.py (34 Anys total).","status":"closed","priority":4,"issue_type":"task","created_at":"2026-01-04T22:19:50.31711-05:00","created_by":"rusty","updated_at":"2026-01-04T22:37:01.488788-05:00","closed_at":"2026-01-04T22:37:01.488788-05:00","close_reason":"Replaced _writer: Any with _writer: pq.ParquetWriter | None. Removes 1 Any and provides proper type information."} {"id":"vgi-python-jrf","title":"Add varargs parameter to Arg descriptor","description":"In vgi/arguments.py:\n- Add varargs: bool = False to Arg.__init__ and __slots__\n- Update _resolve() to collect positional[position:] when varargs=True\n- Validate at least 1 value provided\n- Update _validate() to validate each element in tuple\n- Add Arguments.get_varargs(start, type=None) method\n- Update __repr__ to show varargs flag","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T10:49:20.012964-05:00","created_by":"rusty","updated_at":"2026-01-05T10:55:22.479344-05:00","closed_at":"2026-01-05T10:55:22.479344-05:00","close_reason":"Implemented varargs parameter in Arg descriptor with get_varargs() method and _validate_single()"} {"id":"vgi-python-k7x","title":"Use Mapping instead of dict in extract_argument_specs signature","description":"The arg_types parameter in extract_argument_specs() is typed as dict[str, pa.DataType]. Using Mapping[str, pa.DataType] from collections.abc would be more flexible, accepting any mapping type.","status":"closed","priority":4,"issue_type":"task","created_at":"2026-01-05T11:51:21.021496-05:00","created_by":"rusty","updated_at":"2026-01-05T12:03:51.771301-05:00","closed_at":"2026-01-05T12:03:51.771301-05:00","close_reason":"Closed"} {"id":"vgi-python-kz4","title":"Rename TableInOutGeneratorFunction to TableInOutGenerator for consistency","description":"Naming inconsistency: TableFunctionGenerator uses *Generator suffix, but TableInOutGeneratorFunction uses *GeneratorFunction suffix. Rename TableInOutGeneratorFunction to TableInOutGenerator for consistency. Also consider renaming ScalarFunctionGenerator if needed.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-04T20:06:41.581028-05:00","created_by":"rusty","updated_at":"2026-01-04T21:43:58.141038-05:00","closed_at":"2026-01-04T21:43:58.141038-05:00","close_reason":"PR #7 created: https://github.com/Query-farm/vgi-python/pull/7"} -{"id":"vgi-python-l1u","title":"Consider custom __repr__ for ArgumentSpec","description":"The default dataclass __repr__ includes the full Arrow type repr which can be verbose. Consider a custom __repr__ that's more concise for debugging, e.g., 'ArgumentSpec(name=\"count\", pos=0, type=int64)' instead of showing the full pa.DataType object.","status":"open","priority":4,"issue_type":"task","created_at":"2026-01-05T11:51:21.415976-05:00","created_by":"rusty","updated_at":"2026-01-05T11:51:21.415976-05:00"} +{"id":"vgi-python-l1u","title":"Consider custom __repr__ for ArgumentSpec","description":"The default dataclass __repr__ includes the full Arrow type repr which can be verbose. Consider a custom __repr__ that's more concise for debugging, e.g., 'ArgumentSpec(name=\"count\", pos=0, type=int64)' instead of showing the full pa.DataType object.","status":"closed","priority":4,"issue_type":"task","created_at":"2026-01-05T11:51:21.415976-05:00","created_by":"rusty","updated_at":"2026-01-05T12:15:02.029743-05:00","closed_at":"2026-01-05T12:15:02.029743-05:00","close_reason":"Closed"} +{"id":"vgi-python-l5z","title":"Update existing tests that use arg_types parameter","description":"In tests/test_argument_spec.py:\n- Update all calls to extract_argument_specs() that pass arg_types\n- Remove the arg_types parameter from test function calls\n- Ensure tests still pass with auto-inference","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:56.81929-05:00","created_by":"rusty","updated_at":"2026-01-05T15:44:56.81929-05:00","dependencies":[{"issue_id":"vgi-python-l5z","depends_on_id":"vgi-python-coi","type":"blocks","created_at":"2026-01-05T15:45:13.980985-05:00","created_by":"rusty"}]} {"id":"vgi-python-lec","title":"Add test coverage for testing.py helper edge cases","notes":"Coverage: 89% in vgi/testing.py. Missing tests for:\n- Lines 421-422, 450-451: StopIteration handling in _process_batch\n- Lines 468-472: FINISHED status during data phase\n- Lines 485-486, 502-503: _finalize edge cases\n\nLow priority since these are test helpers.","status":"closed","priority":4,"issue_type":"task","created_at":"2026-01-04T22:15:34.006563-05:00","created_by":"rusty","updated_at":"2026-01-05T12:07:48.026786-05:00","closed_at":"2026-01-05T12:07:48.026786-05:00","close_reason":"Closed"} {"id":"vgi-python-lzc","title":"Extract duplicated sort_key function in argument_spec","description":"The sort_key function is duplicated at lines 139-142 and 309-312 in argument_spec.py. Extract it to a module-level function to follow DRY principles.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-05T11:51:19.141041-05:00","created_by":"rusty","updated_at":"2026-01-05T11:55:22.535816-05:00","closed_at":"2026-01-05T11:55:22.535816-05:00","close_reason":"Closed"} {"id":"vgi-python-m45","title":"Create tests/test_argument_spec.py","description":"## Overview\n\nCreate comprehensive tests for the argument specification serialization module.\n\n## File Location\n\n`tests/test_argument_spec.py`\n\n## Test Classes and Cases\n\n### TestArgumentSpecToSchema\n\nTest converting ArgumentSpec objects to Arrow schema.\n\n#### test_positional_arguments_preserve_order\n- Create specs with positions 0, 1, 2\n- Convert to schema\n- Verify field order matches position order\n- Verify field types are preserved\n\n#### test_named_arguments_have_metadata\n- Create spec with position='key' (named)\n- Convert to schema\n- Verify field has `vgi_arg=named` metadata\n\n#### test_mixed_positional_and_named\n- Create mix of positional (0, 1) and named ('format', 'verbose') specs\n- Convert to schema\n- Verify positional come first, then named\n- Verify named have correct metadata\n\n#### test_table_input_uses_null_type\n- Create spec with is_table_input=True\n- Convert to schema\n- Verify field type is pa.null()\n- Verify field has `vgi_type=table` metadata\n\n#### test_any_type_uses_null_type\n- Create spec with is_any_type=True\n- Convert to schema\n- Verify field type is pa.null()\n- Verify field has `vgi_type=any` metadata\n\n#### test_varargs_has_metadata\n- Create spec with is_varargs=True and arrow_type=pa.int64()\n- Convert to schema\n- Verify field type is pa.int64() (element type preserved)\n- Verify field has `vgi_varargs=true` metadata\n\n### TestSchemaToArgumentSpecs\n\nTest converting Arrow schema back to ArgumentSpec objects.\n\n#### test_positional_arguments_from_schema\n- Create schema with 3 fields (no metadata)\n- Convert to specs\n- Verify positions are 0, 1, 2\n\n#### test_named_arguments_from_metadata\n- Create schema with `vgi_arg=named` metadata on fields\n- Convert to specs\n- Verify position is field name string\n\n#### test_table_input_detected\n- Create schema with `vgi_type=table` metadata\n- Convert to specs\n- Verify is_table_input=True\n\n#### test_any_type_detected\n- Create schema with `vgi_type=any` metadata\n- Convert to specs\n- Verify is_any_type=True\n\n#### test_varargs_detected\n- Create schema with `vgi_varargs=true` metadata\n- Convert to specs\n- Verify is_varargs=True\n\n### TestRoundTrip\n\nTest that specs survive serialization round-trip.\n\n#### test_complex_arrow_types_preserved\nTest each of these types round-trips correctly:\n- pa.int64(), pa.float32(), pa.utf8()\n- pa.list_(pa.float64())\n- pa.struct([pa.field('a', pa.int32()), pa.field('b', pa.string())])\n- pa.map_(pa.string(), pa.int64())\n- pa.decimal128(10, 2)\n- pa.timestamp('us', tz='UTC')\n\n#### test_full_function_signature_roundtrip\n- Create specs matching a realistic function:\n - count: int, position 0\n - data: TableInput, position 1\n - extra: float varargs, position 2\n - format: str, named 'format'\n- Convert to schema, serialize to bytes, deserialize, convert back to specs\n- Verify all specs match original\n\n### TestExtractArgumentSpecs\n\nTest extracting specs from function classes.\n\n#### test_extract_from_simple_function\n- Define function class with Arg descriptors\n- Call extract_argument_specs with arg_types dict\n- Verify specs match descriptors\n\n#### test_extract_table_input\n- Define function with Arg[TableInput]\n- Extract specs\n- Verify is_table_input=True\n\n#### test_extract_any_arrow\n- Define function with Arg[AnyArrow]\n- Extract specs\n- Verify is_any_type=True\n\n#### test_extract_varargs\n- Define function with Arg[int](2, varargs=True)\n- Extract specs\n- Verify is_varargs=True\n\n### TestEdgeCases\n\n#### test_empty_schema\n- Convert empty list of specs to schema\n- Verify empty schema works\n- Convert back, verify empty list\n\n#### test_only_named_arguments\n- Create specs with only named arguments (no positional)\n- Round-trip and verify\n\n#### test_only_positional_arguments\n- Create specs with only positional arguments (no named)\n- Round-trip and verify\n\n## Test Utilities\n\nConsider creating fixtures for common patterns:\n- `make_spec()` helper for creating ArgumentSpec\n- Sample function classes for extraction tests","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T11:18:53.312911-05:00","created_by":"rusty","updated_at":"2026-01-05T11:32:35.580879-05:00","closed_at":"2026-01-05T11:32:35.580879-05:00","close_reason":"Created comprehensive tests with 43 passing test cases","dependencies":[{"issue_id":"vgi-python-m45","depends_on_id":"vgi-python-cd0","type":"blocks","created_at":"2026-01-05T11:19:30.779207-05:00","created_by":"rusty"}]} +{"id":"vgi-python-n2b","title":"Update example functions with explicit arrow_type","description":"Update a subset of example functions to demonstrate explicit arrow_type:\n\n- vgi/examples/table.py: RangeFunction.step → arrow_type=pa.int32()\n- vgi/examples/scalar.py: One function → explicit arrow_type\n\nMost examples keep Arg[int](0) to test auto-inference.","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:57.188629-05:00","created_by":"rusty","updated_at":"2026-01-05T15:44:57.188629-05:00","dependencies":[{"issue_id":"vgi-python-n2b","depends_on_id":"vgi-python-dv0","type":"blocks","created_at":"2026-01-05T15:45:14.330881-05:00","created_by":"rusty"}]} {"id":"vgi-python-odi","title":"Change max_processes from method to property in Function hierarchy","description":"Refactor max_processes from a method to a property across the Function class hierarchy (Function, ScalarFunction, TableFunctionGenerator, TableInOutFunction, etc.). This makes the API more consistent since max_processes is effectively a constant per function class and properties are more idiomatic for such values.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-04T11:25:29.750648-05:00","created_by":"rusty","updated_at":"2026-01-04T11:50:57.566545-05:00","closed_at":"2026-01-04T11:50:57.566545-05:00","close_reason":"Closed"} {"id":"vgi-python-p91","title":"Move exception classes from function.py to own file","description":"Move InitIdentifierError and SchemaValidationError from vgi/function.py to a new vgi/exceptions.py file. Update imports in function.py and any other files that reference these exceptions.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-04T09:12:28.058227-05:00","created_by":"rusty","updated_at":"2026-01-04T09:17:52.477661-05:00","closed_at":"2026-01-04T09:17:52.477661-05:00","close_reason":"Closed"} {"id":"vgi-python-qud","title":"Test FunctionStorageSqlite: global_delete, global_exists, queue_clear","notes":"Coverage: 83% in vgi/function_storage.py. Missing tests for:\n- Line 266: KeyError path in global_get (key not found)\n- Lines 273-278: global_delete method\n- Lines 282-290: global_exists method \n- Line 337: queue_push with empty list\n- Lines 376-385: queue_clear method\n\nThese storage operations need direct unit tests to ensure correctness.","status":"closed","priority":3,"issue_type":"task","created_at":"2026-01-04T22:15:25.982124-05:00","created_by":"rusty","updated_at":"2026-01-04T22:30:05.625934-05:00","closed_at":"2026-01-04T22:30:05.625934-05:00","close_reason":"Added comprehensive tests for FunctionStorageSqlite. Coverage improved from 83% to 98%."} diff --git a/vgi/arguments.py b/vgi/arguments.py index 2fe3e12..ac14f6a 100644 --- a/vgi/arguments.py +++ b/vgi/arguments.py @@ -29,6 +29,15 @@ class MyFunction(TableInOutFunction): if TYPE_CHECKING: from pyarrow import Scalar +# Python type to Arrow type mapping for Arg type hints +PYTHON_TO_ARROW: dict[type, pa.DataType] = { + int: pa.int64(), + str: pa.utf8(), + float: pa.float64(), + bool: pa.bool_(), + bytes: pa.binary(), +} + # Sentinel for missing default value - proper type pattern class _MissingType: @@ -53,6 +62,7 @@ def __bool__(self) -> bool: "Arg", "ArgumentValidationError", "Arguments", + "PYTHON_TO_ARROW", "TableInput", ] From d60c8704cb5d85ada58eaa6b54a204d246ded374 Mon Sep 17 00:00:00 2001 From: Rusty Conover Date: Mon, 5 Jan 2026 15:48:45 -0500 Subject: [PATCH 2/2] bd sync: 2026-01-05 15:48:45 --- .beads/issues.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index c74d70c..b6b8fe5 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -28,7 +28,7 @@ {"id":"vgi-python-cd0","title":"Create vgi/argument_spec.py module","description":"## Overview\n\nCreate the core module implementing Arrow-based argument specification serialization.\n\n## File Location\n\n`vgi/argument_spec.py`\n\n## Constants to Define\n\n```python\n# Metadata keys (all bytes for Arrow compatibility)\nVGI_ARG_KEY = b\"vgi_arg\"\nVGI_ARG_NAMED = b\"named\"\n\nVGI_TYPE_KEY = b\"vgi_type\"\nVGI_TYPE_TABLE = b\"table\"\nVGI_TYPE_ANY = b\"any\"\n\nVGI_VARARGS_KEY = b\"vgi_varargs\"\nVGI_VARARGS_TRUE = b\"true\"\n```\n\n## ArgumentSpec Dataclass\n\n```python\n@dataclass(frozen=True)\nclass ArgumentSpec:\n \"\"\"Specification for a single function argument.\"\"\"\n name: str # Python attribute name\n position: int | str # int for positional index, str for named key\n arrow_type: pa.DataType # Arrow type (pa.null() for special types)\n is_table_input: bool = False # Arg[TableInput]\n is_any_type: bool = False # Arg[AnyArrow]\n is_varargs: bool = False # varargs=True\n```\n\n## Functions to Implement\n\n### argument_specs_to_schema(specs: Sequence[ArgumentSpec]) -\u003e pa.Schema\n\nConvert ArgumentSpecs to a single Arrow schema:\n1. Sort specs: positional first (by index), then named\n2. For each spec, create a pa.field with:\n - name = spec.name\n - type = spec.arrow_type (or pa.null() for table/any)\n - metadata = appropriate markers based on flags\n3. Return pa.schema(fields)\n\n### schema_to_argument_specs(schema: pa.Schema) -\u003e list[ArgumentSpec]\n\nConvert schema back to ArgumentSpecs:\n1. Iterate through schema fields in order\n2. Track position index (increments for non-named args)\n3. Check field metadata for markers:\n - `vgi_arg=named` -\u003e position is field name string\n - `vgi_type=table` -\u003e is_table_input=True\n - `vgi_type=any` -\u003e is_any_type=True\n - `vgi_varargs=true` -\u003e is_varargs=True\n4. Return list of ArgumentSpec\n\n### extract_argument_specs(cls: type, arg_types: dict[str, pa.DataType]) -\u003e list[ArgumentSpec]\n\nExtract specs from a function class with Arg descriptors:\n1. Walk class MRO to find all Arg descriptors (like extract_parameters in metadata.py)\n2. For each Arg descriptor:\n - Get name from attribute name\n - Get position from arg.position\n - Get arrow_type from arg_types dict\n - Check type hints for TableInput/AnyArrow\n - Check arg.varargs flag\n3. Sort and return list\n\n## Dependencies\n\n- Import `Arg`, `TableInput`, `AnyArrow` from `vgi.arguments`\n- Reference `extract_parameters()` pattern in `vgi/metadata.py`","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T11:18:32.777241-05:00","created_by":"rusty","updated_at":"2026-01-05T11:28:07.227452-05:00","closed_at":"2026-01-05T11:28:07.227452-05:00","close_reason":"Created vgi/argument_spec.py with ArgumentSpec dataclass and serialization functions","dependencies":[{"issue_id":"vgi-python-cd0","depends_on_id":"vgi-python-8ra","type":"blocks","created_at":"2026-01-05T11:19:30.743936-05:00","created_by":"rusty"}]} {"id":"vgi-python-ckg","title":"Add AnyValue sentinel class to vgi/arguments.py","description":"Add AnyValue class similar to TableInput, export in __all__","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T10:41:41.392694-05:00","created_by":"rusty","updated_at":"2026-01-05T11:05:38.37392-05:00","closed_at":"2026-01-05T11:05:38.37392-05:00","close_reason":"Added AnyArrow sentinel class to arguments.py","dependencies":[{"issue_id":"vgi-python-ckg","depends_on_id":"vgi-python-awm","type":"blocks","created_at":"2026-01-05T10:41:52.658405-05:00","created_by":"rusty"}]} {"id":"vgi-python-coi","title":"Update extract_argument_specs() to remove arg_types parameter","description":"In vgi/argument_spec.py:\n1. Remove arg_types parameter from function signature\n2. Update arrow_type resolution logic:\n - Use arg.arrow_type if explicitly set\n - Infer from Python type hint using PYTHON_TO_ARROW\n - Handle TableInput/AnyArrow → pa.null()\n - Warn and default to pa.null() for unknown types\n3. Import PYTHON_TO_ARROW from vgi.arguments","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:38.141157-05:00","created_by":"rusty","updated_at":"2026-01-05T15:44:38.141157-05:00","dependencies":[{"issue_id":"vgi-python-coi","depends_on_id":"vgi-python-cvj","type":"blocks","created_at":"2026-01-05T15:45:13.831745-05:00","created_by":"rusty"},{"issue_id":"vgi-python-coi","depends_on_id":"vgi-python-dv0","type":"blocks","created_at":"2026-01-05T15:45:13.864608-05:00","created_by":"rusty"}]} -{"id":"vgi-python-cvj","title":"Add PYTHON_TO_ARROW type mapping to vgi/arguments.py","description":"Add the Python→Arrow type mapping dict after imports:\n```python\nPYTHON_TO_ARROW: dict[type, pa.DataType] = {\n int: pa.int64(),\n str: pa.utf8(),\n float: pa.float64(),\n bool: pa.bool_(),\n bytes: pa.binary(),\n}\n```\nExport in __all__.","status":"in_progress","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:37.900421-05:00","created_by":"rusty","updated_at":"2026-01-05T15:46:01.292126-05:00"} +{"id":"vgi-python-cvj","title":"Add PYTHON_TO_ARROW type mapping to vgi/arguments.py","description":"Add the Python→Arrow type mapping dict after imports:\n```python\nPYTHON_TO_ARROW: dict[type, pa.DataType] = {\n int: pa.int64(),\n str: pa.utf8(),\n float: pa.float64(),\n bool: pa.bool_(),\n bytes: pa.binary(),\n}\n```\nExport in __all__.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:37.900421-05:00","created_by":"rusty","updated_at":"2026-01-05T15:48:42.422086-05:00","closed_at":"2026-01-05T15:48:42.422086-05:00","close_reason":"PR #18 created"} {"id":"vgi-python-d73","title":"Create docs/argument-serialization.md","description":"## Overview\n\nCreate LLM-friendly documentation explaining the argument specification serialization format. This document should enable future implementors (human or AI) to understand how function argument signatures are serialized to Arrow schemas.\n\n## File Location\n\n`docs/argument-serialization.md`\n\n## Document Structure\n\n### Title and Purpose\n\nExplain that this document describes how VGI function argument specifications are serialized to Apache Arrow schemas for IPC transmission and DuckDB function registration.\n\n### Quick Reference\n\nA concise summary table showing:\n- Metadata keys and their meanings\n- Special type representations\n\n### Schema Format\n\nExplain the single-schema design:\n1. All arguments are fields in one Arrow schema\n2. Positional arguments come first, in order (field index = position index)\n3. Named arguments follow, marked with metadata\n4. Field name = Python attribute name (or argument key for named)\n5. Field type = exact Arrow type\n\n### Metadata Keys Reference\n\nComplete table of all metadata keys:\n\n| Key | Value | Description |\n|-----|-------|-------------|\n| `vgi_arg` | `named` | Field is a named argument, not positional. The field name is the argument key. |\n| `vgi_type` | `table` | Argument receives streaming table input (Arg[TableInput]). Arrow type is pa.null(). |\n| `vgi_type` | `any` | Argument accepts any Arrow type (Arg[AnyArrow]). Arrow type is pa.null(). |\n| `vgi_varargs` | `true` | Argument collects all remaining positional args. Arrow type is the element type. |\n\n### Special Type Handling\n\nExplain how special argument types are represented:\n\n#### TableInput\n- Arrow type: `pa.null()`\n- Metadata: `{b\"vgi_type\": b\"table\"}`\n- Meaning: This position receives streaming RecordBatches, not a scalar value\n\n#### AnyArrow\n- Arrow type: `pa.null()`\n- Metadata: `{b\"vgi_type\": b\"any\"}`\n- Meaning: Accepts any valid Arrow scalar type at runtime\n\n#### Varargs\n- Arrow type: The element type (e.g., `pa.int64()` for `Arg[int](..., varargs=True)`)\n- Metadata: `{b\"vgi_varargs\": b\"true\"}`\n- Meaning: Collects all remaining positional arguments from this position onwards\n\n### Examples\n\n#### Example 1: Simple Function\n\n```python\nclass MyFunction(TableInOutFunction):\n count = Arg[int](0) # Positional 0\n name = Arg[str](1) # Positional 1\n verbose = Arg[bool](\"verbose\") # Named\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"count\", pa.int64()),\n pa.field(\"name\", pa.utf8()),\n pa.field(\"verbose\", pa.bool_(), metadata={b\"vgi_arg\": b\"named\"}),\n])\n```\n\n#### Example 2: Function with Table Input\n\n```python\nclass TransformFunction(TableInOutFunction):\n multiplier = Arg[float](0)\n data = Arg[TableInput](1)\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"multiplier\", pa.float64()),\n pa.field(\"data\", pa.null(), metadata={b\"vgi_type\": b\"table\"}),\n])\n```\n\n#### Example 3: Function with Varargs\n\n```python\nclass SumFunction(TableInOutFunction):\n columns = Arg[str](0, varargs=True)\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"columns\", pa.utf8(), metadata={b\"vgi_varargs\": b\"true\"}),\n])\n```\n\n#### Example 4: Complex Function\n\n```python\nclass ComplexFunction(TableInOutFunction):\n count = Arg[int](0)\n data = Arg[TableInput](1)\n extra = Arg[float](2, varargs=True)\n format = Arg[str](\"format\")\n threshold = Arg[AnyArrow](\"threshold\")\n\n# Serializes to:\nschema = pa.schema([\n pa.field(\"count\", pa.int64()),\n pa.field(\"data\", pa.null(), metadata={b\"vgi_type\": b\"table\"}),\n pa.field(\"extra\", pa.float64(), metadata={b\"vgi_varargs\": b\"true\"}),\n pa.field(\"format\", pa.utf8(), metadata={b\"vgi_arg\": b\"named\"}),\n pa.field(\"threshold\", pa.null(), metadata={b\"vgi_arg\": b\"named\", b\"vgi_type\": b\"any\"}),\n])\n```\n\n### Serialization Code\n\nShow how to serialize and deserialize:\n\n```python\n# Serialize to bytes\nschema_bytes = schema.serialize().to_pybytes()\n\n# Deserialize from bytes\nschema = pa.ipc.read_schema(pa.py_buffer(schema_bytes))\n```\n\n### Parsing Algorithm\n\nExplain how to parse a schema back to argument specs:\n\n1. Initialize position_index = 0\n2. For each field in schema:\n a. Check if field has `vgi_arg=named` metadata\n b. If named: position = field.name (string)\n c. If positional: position = position_index, then increment position_index\n d. Check for `vgi_type` metadata (table or any)\n e. Check for `vgi_varargs` metadata\n f. Create ArgumentSpec with extracted info\n\n### Not Included\n\nExplicitly state what is NOT serialized:\n- Default values\n- Validation constraints (ge, le, choices, pattern)\n- Documentation strings\n\nThese are Python-side concerns handled by the Arg descriptor at runtime.","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T11:19:17.488877-05:00","created_by":"rusty","updated_at":"2026-01-05T11:33:29.168007-05:00","closed_at":"2026-01-05T11:33:29.168007-05:00","close_reason":"Created comprehensive LLM-friendly documentation","dependencies":[{"issue_id":"vgi-python-d73","depends_on_id":"vgi-python-8ra","type":"blocks","created_at":"2026-01-05T11:19:30.820384-05:00","created_by":"rusty"}]} {"id":"vgi-python-dv0","title":"Add arrow_type parameter to Arg class","description":"In vgi/arguments.py:\n1. Add 'arrow_type' to __slots__\n2. Add parameter: arrow_type: pa.DataType | None = None\n3. Store: self.arrow_type = arrow_type\n4. Update __repr__ to include arrow_type if set","status":"open","priority":2,"issue_type":"task","created_at":"2026-01-05T15:44:38.020395-05:00","created_by":"rusty","updated_at":"2026-01-05T15:44:38.020395-05:00","dependencies":[{"issue_id":"vgi-python-dv0","depends_on_id":"vgi-python-cvj","type":"blocks","created_at":"2026-01-05T15:45:13.696822-05:00","created_by":"rusty"}]} {"id":"vgi-python-dvo","title":"Export AnyValue in vgi/__init__.py","description":"Import and add AnyValue to __all__ exports","status":"closed","priority":2,"issue_type":"task","created_at":"2026-01-05T10:41:41.65732-05:00","created_by":"rusty","updated_at":"2026-01-05T11:07:09.187969-05:00","closed_at":"2026-01-05T11:07:09.187969-05:00","close_reason":"Exported AnyArrow in vgi/__init__.py","dependencies":[{"issue_id":"vgi-python-dvo","depends_on_id":"vgi-python-ckg","type":"blocks","created_at":"2026-01-05T10:41:48.715634-05:00","created_by":"rusty"}]}