From 44631e0f23fb3dc0c1be1bd9d2b75ef1973cee97 Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Wed, 8 Jul 2026 17:44:55 +0900 Subject: [PATCH 1/4] Support upgrading tables to format version 3 Allow upgrade_table_version(3) and initialize next-row-id to 0 on upgrade, per the spec's Row Lineage for Upgraded Tables. Persisting the upgraded metadata still requires V3 metadata serialization (#3551). Closes #3622 --- pyiceberg/table/__init__.py | 4 ++-- pyiceberg/table/metadata.py | 5 ++++- pyiceberg/table/update/__init__.py | 7 +++++-- tests/integration/test_reads.py | 10 ++++++++-- tests/table/test_init.py | 15 +++++++++++++++ 5 files changed, 34 insertions(+), 7 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 63b87d290e..fa18d38070 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -48,7 +48,7 @@ from pyiceberg.table.inspect import InspectTable from pyiceberg.table.locations import LocationProvider, load_location_provider from pyiceberg.table.maintenance import MaintenanceTable -from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadata +from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, SUPPORTED_TABLE_FORMAT_VERSION, TableMetadata from pyiceberg.table.name_mapping import NameMapping from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef from pyiceberg.table.snapshots import ( @@ -300,7 +300,7 @@ def upgrade_table_version(self, format_version: TableVersion) -> Transaction: Returns: The alter table builder. """ - if format_version not in {1, 2}: + if not 1 <= format_version <= SUPPORTED_TABLE_FORMAT_VERSION: raise ValueError(f"Unsupported table format version: {format_version}") if format_version < self.table_metadata.format_version: diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 26b6e3d3ad..fde0a54e64 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -66,7 +66,10 @@ INITIAL_SPEC_ID = 0 DEFAULT_SCHEMA_ID = 0 -SUPPORTED_TABLE_FORMAT_VERSION = 2 +SUPPORTED_TABLE_FORMAT_VERSION = 3 + +INITIAL_ROW_ID = 0 +"""The initial value of `next-row-id` for newly upgraded V3 tables.""" def cleanup_snapshot_id(data: dict[str, Any]) -> dict[str, Any]: diff --git a/pyiceberg/table/update/__init__.py b/pyiceberg/table/update/__init__.py index 64838b0bd6..9f1a4cb2b4 100644 --- a/pyiceberg/table/update/__init__.py +++ b/pyiceberg/table/update/__init__.py @@ -29,7 +29,7 @@ from pyiceberg.exceptions import CommitFailedException from pyiceberg.partitioning import PARTITION_FIELD_ID_START, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table.metadata import SUPPORTED_TABLE_FORMAT_VERSION, TableMetadata, TableMetadataUtil +from pyiceberg.table.metadata import INITIAL_ROW_ID, SUPPORTED_TABLE_FORMAT_VERSION, TableMetadata, TableMetadataUtil from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import ( MetadataLogEntry, @@ -321,7 +321,10 @@ def _( elif update.format_version == base_metadata.format_version: return base_metadata - updated_metadata = base_metadata.model_copy(update={"format_version": update.format_version}) + updates: dict[str, Any] = {"format_version": update.format_version} + if update.format_version >= 3 and getattr(base_metadata, "next_row_id", None) is None: + updates["next_row_id"] = INITIAL_ROW_ID + updated_metadata = base_metadata.model_copy(update=updates) context.add_update(update) return TableMetadataUtil._construct_without_validation(updated_metadata) diff --git a/tests/integration/test_reads.py b/tests/integration/test_reads.py index 60aefa283f..d98fa21daf 100644 --- a/tests/integration/test_reads.py +++ b/tests/integration/test_reads.py @@ -969,10 +969,16 @@ def test_upgrade_table_version(catalog: Catalog) -> None: transaction.upgrade_table_version(format_version=1) assert "Cannot downgrade v2 table to v1" in str(e.value) + with table_test_table_version.transaction() as transaction: + transaction.upgrade_table_version(format_version=3) + + assert table_test_table_version.format_version == 3 + assert table_test_table_version.metadata.next_row_id == 0 + with pytest.raises(ValueError) as e: with table_test_table_version.transaction() as transaction: - transaction.upgrade_table_version(format_version=3) - assert "Unsupported table format version: 3" in str(e.value) + transaction.upgrade_table_version(format_version=4) # type: ignore[arg-type] + assert "Unsupported table format version: 4" in str(e.value) @pytest.mark.integration diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 1670e62587..3da2db1340 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -83,6 +83,7 @@ SetPropertiesUpdate, SetSnapshotRefUpdate, SetStatisticsUpdate, + UpgradeFormatVersionUpdate, _apply_table_update, _TableMetadataUpdateContext, update_table_metadata, @@ -1240,6 +1241,20 @@ def test_update_metadata_with_multiple_updates(table_v1: Table) -> None: assert new_metadata.properties == {"owner": "test", "test_a": "test_a1"} +def test_update_metadata_upgrade_to_v3(table_v2: Table) -> None: + from pyiceberg.table.metadata import TableMetadataV3 + + new_metadata = update_table_metadata(table_v2.metadata, (UpgradeFormatVersionUpdate(format_version=3),)) + assert isinstance(new_metadata, TableMetadataV3) + assert new_metadata.format_version == 3 + assert new_metadata.next_row_id == 0 + + # rebuild the metadata to trigger validation + validated = TableMetadataUtil.parse_obj(copy(new_metadata.model_dump())) + assert validated.format_version == 3 + assert validated.next_row_id == 0 + + def test_update_metadata_schema_immutability( table_v2_with_fixed_and_decimal_types: TableMetadataV2, ) -> None: From 95fd03eb00973e8685fd099dcb2632ff5f315c93 Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Wed, 8 Jul 2026 21:27:36 +0900 Subject: [PATCH 2/4] Align upgrade path with the Java implementation Support upgrading via the format-version table property in set_properties, matching TableMetadata.replaceProperties in Java: the property triggers UpgradeFormatVersionUpdate instead of being stored. Add tests for parity with TestTableMetadata and TestRowLineageMetadata: direct v1-to-v3 upgrade, same-version no-op, and property-based upgrade. --- pyiceberg/table/__init__.py | 13 +++++++++--- tests/table/test_init.py | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index fa18d38070..feec535d05 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -26,7 +26,7 @@ from functools import cached_property from itertools import chain from types import TracebackType -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar, cast from pydantic import Field @@ -316,6 +316,9 @@ def set_properties(self, properties: Properties = EMPTY_DICT, **kwargs: Any) -> When a property is already set, it will be overwritten. + Setting the `format-version` property upgrades the table to that format + version instead of storing it as a property. + Args: properties: The properties set on the table. kwargs: properties can also be pass as kwargs. @@ -325,8 +328,12 @@ def set_properties(self, properties: Properties = EMPTY_DICT, **kwargs: Any) -> """ if properties and kwargs: raise ValueError("Cannot pass both properties and kwargs") - updates = properties or kwargs - return self._apply((SetPropertiesUpdate(updates=updates),)) + updates = dict(properties or kwargs) + if (new_format_version := updates.pop(TableProperties.FORMAT_VERSION, None)) is not None: + self.upgrade_table_version(format_version=cast(TableVersion, int(new_format_version))) + if updates: + return self._apply((SetPropertiesUpdate(updates=updates),)) + return self def _set_ref_snapshot( self, diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 3da2db1340..3c62f1d140 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -1255,6 +1255,46 @@ def test_update_metadata_upgrade_to_v3(table_v2: Table) -> None: assert validated.next_row_id == 0 +def test_update_metadata_upgrade_v1_to_v3(table_v1: Table) -> None: + from pyiceberg.table.metadata import TableMetadataV3 + + new_metadata = update_table_metadata(table_v1.metadata, (UpgradeFormatVersionUpdate(format_version=3),)) + assert isinstance(new_metadata, TableMetadataV3) + assert new_metadata.format_version == 3 + assert new_metadata.next_row_id == 0 + + # rebuild the metadata to trigger validation + validated = TableMetadataUtil.parse_obj(copy(new_metadata.model_dump())) + assert validated.format_version == 3 + assert validated.next_row_id == 0 + + +def test_update_metadata_upgrade_same_version_is_noop(table_v2: Table) -> None: + new_metadata = update_table_metadata(table_v2.metadata, (UpgradeFormatVersionUpdate(format_version=2),)) + assert new_metadata == table_v2.metadata + assert new_metadata.format_version == 2 + + +def test_set_properties_format_version_upgrades(table_v2: Table) -> None: + transaction = table_v2.transaction() + transaction.set_properties({"format-version": "3", "owner": "test"}) + + assert transaction.table_metadata.format_version == 3 + assert transaction.table_metadata.next_row_id == 0 + assert transaction.table_metadata.properties["owner"] == "test" + assert "format-version" not in transaction.table_metadata.properties + assert UpgradeFormatVersionUpdate(format_version=3) in transaction._updates + + +def test_set_properties_same_format_version_is_noop(table_v2: Table) -> None: + transaction = table_v2.transaction() + transaction.set_properties({"format-version": "2"}) + + assert transaction.table_metadata.format_version == 2 + assert "format-version" not in transaction.table_metadata.properties + assert transaction._updates == () + + def test_update_metadata_schema_immutability( table_v2_with_fixed_and_decimal_types: TableMetadataV2, ) -> None: From e6b2f340bb70e9f5ad1d1a782cc84d5f82e747e7 Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Thu, 9 Jul 2026 09:05:04 +0900 Subject: [PATCH 3/4] Address review: validate format-version property strictly Reject non-integer format-version values in set_properties with a clear error instead of truncating floats or failing with a confusing cast error, and verify in the integration test that the upgraded metadata persists and reloads from the catalog. --- pyiceberg/table/__init__.py | 2 ++ tests/integration/test_reads.py | 5 +++++ tests/table/test_init.py | 7 +++++++ 3 files changed, 14 insertions(+) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index feec535d05..46169cbae0 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -330,6 +330,8 @@ def set_properties(self, properties: Properties = EMPTY_DICT, **kwargs: Any) -> raise ValueError("Cannot pass both properties and kwargs") updates = dict(properties or kwargs) if (new_format_version := updates.pop(TableProperties.FORMAT_VERSION, None)) is not None: + if isinstance(new_format_version, bool) or not str(new_format_version).isdigit(): + raise ValueError(f"Invalid format-version: {new_format_version}") self.upgrade_table_version(format_version=cast(TableVersion, int(new_format_version))) if updates: return self._apply((SetPropertiesUpdate(updates=updates),)) diff --git a/tests/integration/test_reads.py b/tests/integration/test_reads.py index d98fa21daf..d09faa2f06 100644 --- a/tests/integration/test_reads.py +++ b/tests/integration/test_reads.py @@ -975,6 +975,11 @@ def test_upgrade_table_version(catalog: Catalog) -> None: assert table_test_table_version.format_version == 3 assert table_test_table_version.metadata.next_row_id == 0 + # the upgraded metadata must persist and reload from the catalog + reloaded = catalog.load_table("default.test_table_version") + assert reloaded.format_version == 3 + assert reloaded.metadata.next_row_id == 0 + with pytest.raises(ValueError) as e: with table_test_table_version.transaction() as transaction: transaction.upgrade_table_version(format_version=4) # type: ignore[arg-type] diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 3c62f1d140..d8f4f89330 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -2044,3 +2044,10 @@ def test_build_large_partition_predicate(table_v2: Table) -> None: ) bind(table_v2.metadata.schema(), expr, case_sensitive=True) + + +def test_set_properties_invalid_format_version_rejected(table_v2: Table) -> None: + transaction = table_v2.transaction() + for invalid in ("3.0", "three", "-1"): + with pytest.raises(ValueError, match="Invalid format-version"): + transaction.set_properties({"format-version": invalid}) From 17b47be58172b2744a789e91538cb2163c62afab Mon Sep 17 00:00:00 2001 From: Noritaka Sekiyama Date: Thu, 9 Jul 2026 09:55:47 +0900 Subject: [PATCH 4/4] Fix mypy unused-ignore in the upgrade integration test --- tests/integration/test_reads.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_reads.py b/tests/integration/test_reads.py index d09faa2f06..dd60b73232 100644 --- a/tests/integration/test_reads.py +++ b/tests/integration/test_reads.py @@ -21,7 +21,7 @@ import uuid from datetime import datetime, timedelta from pathlib import PosixPath -from typing import Any +from typing import Any, cast from urllib.parse import urlparse import pyarrow as pa @@ -52,6 +52,7 @@ from pyiceberg.schema import Schema from pyiceberg.table import Table from pyiceberg.table.snapshots import Operation +from pyiceberg.typedef import TableVersion from pyiceberg.types import ( BinaryType, BooleanType, @@ -982,7 +983,7 @@ def test_upgrade_table_version(catalog: Catalog) -> None: with pytest.raises(ValueError) as e: with table_test_table_version.transaction() as transaction: - transaction.upgrade_table_version(format_version=4) # type: ignore[arg-type] + transaction.upgrade_table_version(format_version=cast(TableVersion, 4)) assert "Unsupported table format version: 4" in str(e.value)