From b4290e0eb1503eeba83cb194187bd0869c121253 Mon Sep 17 00:00:00 2001 From: Maarten Sebregts Date: Thu, 13 Nov 2025 09:26:38 +0100 Subject: [PATCH 1/2] Disable implicit conversion between major versions of the DD Implicit conversion between major versions of the DD is almost always giving incorrect results, so it's better to disallow this instead of emitting warnings (which people usually don't read). This commit changes the warnings that were previously emitted into runtime errors. The errors refer to the (updated) documentation, which provides an example that does work correctly. --- docs/source/multi-dd.rst | 64 ++++++++++++++++++++++++++ imas/backends/imas_core/db_entry_al.py | 15 +++--- imas/db_entry.py | 16 +++---- 3 files changed, 78 insertions(+), 17 deletions(-) diff --git a/docs/source/multi-dd.rst b/docs/source/multi-dd.rst index a115ce27..ae1175fd 100644 --- a/docs/source/multi-dd.rst +++ b/docs/source/multi-dd.rst @@ -213,6 +213,70 @@ explicit conversion mechanisms. ``boundary_secondary_separatrix`` structures from DD3. See also: https://github.com/iterorganization/IMAS-Python/issues/60. + +.. _`Loading IDSs from a different major version`: + +Loading IDSs from a different major version +------------------------------------------- + +If you try to load an IDS that was stored in a different major version of the DD than +you are using, IMAS-Python will raise a runtime error, for example: + +.. code-block:: text + + On-disk data is stored in DD 3.39.1 which has a different major version than the + requested DD version (4.0.0). IMAS-Python will not automatically convert this + data for you. + +You need to explicitly convert the data, which you can do as follows: + +.. code-block:: python + + # Opened data entry + entry = imas.DBEntry(...) + + # A plain get, or get_slice will raise a RuntimeError when the data is stored in + # a different major version of the DD: + # entry.get("equilibrium") + + # So instead, we'll load the IDS in the DD version it is stored on disk + tmp_eq = entry.get("equilibrium", autoconvert=False) + # And explicitly convert it to the target version + equilibrium = imas.convert_ids(tmp_eq, entry.dd_version) + + +.. _`Storing IDSs with a different major version`: + +Storing IDSs with a different major version +------------------------------------------- + +If you try to put an IDS that was created for a different major version of the DD than +the Data Entry you want to store it in, IMAS-Python raise a runtime error, for example: + +.. code-block:: text + + Provided IDS uses DD 3.42.2 which has a different major version than the Data + Entry (4.0.0). IMAS-Python will not automatically convert this data for you. + +You need to explicitly convert the data, which you can do as follows: + +.. code-block:: python + + # IDS with data, in DD 3.42.2 + equilibrium = imas.IDSFactory("3.42.2").equilibrium() + ... + + # Data Entry uses DD 4.0.0 + with imas.DBEntry(uri, "w", dd_version="4.0.0") as entry: + # This put would raise a runtime error, because the major version of the IDS + # and the DBEntry don't match: + # entry.put(equilibrium) + + # So instead, we'll explicitly convert the IDS and put that one + entry.put(imas.convert_ids(equilibrium, entry.dd_version)) + + + .. _`DD background`: Background information diff --git a/imas/backends/imas_core/db_entry_al.py b/imas/backends/imas_core/db_entry_al.py index 0336179f..dad5019b 100644 --- a/imas/backends/imas_core/db_entry_al.py +++ b/imas/backends/imas_core/db_entry_al.py @@ -282,14 +282,13 @@ def put(self, ids: IDSToplevel, occurrence: int, is_slice: bool) -> None: nbc_map = None if ids._version != self._ids_factory._version: if ids._version.split(".")[0] != self._ids_factory._version.split(".")[0]: - logger.warning( - "Provided IDS uses DD %s which has a different major version than " - "the Data Entry (%s). IMAS-Python will convert the data " - "automatically, but this does not cover all changes. " - "See %s/multi-dd.html#conversion-of-idss-between-dd-versions", - ids._version, - self._ids_factory._version, - imas.PUBLISHED_DOCUMENTATION_ROOT, + raise RuntimeError( + f"Provided IDS uses DD {ids._version} which has a different major " + f"version than the Data Entry ({self._ids_factory._version}). " + "IMAS-Python will not automatically convert this data for you." + "See the documentation for more details and fixes: " + f"{imas.PUBLISHED_DOCUMENTATION_ROOT}" + "/multi-dd.html#storing-idss-with-a-different-major-version" ) ddmap, source_is_older = dd_version_map_from_factories( ids_name, ids._parent, self._ids_factory diff --git a/imas/db_entry.py b/imas/db_entry.py index d9ad7f3e..471a50ad 100644 --- a/imas/db_entry.py +++ b/imas/db_entry.py @@ -605,15 +605,13 @@ def _get( nbc_map = None if dd_version and dd_version != destination._dd_version: if dd_version.split(".")[0] != destination._dd_version.split(".")[0]: - logger.warning( - "On-disk data is stored in DD %s which has a different major " - "version than the requested DD version (%s). IMAS-Python will " - "convert the data automatically, but this does not cover all " - "changes. " - "See %s/multi-dd.html#conversion-of-idss-between-dd-versions", - dd_version, - destination._dd_version, - imas.PUBLISHED_DOCUMENTATION_ROOT, + raise RuntimeError( + f"On-disk data is stored in DD {dd_version} which has a different " + "major version than the requested DD version " + f"({destination._dd_version}). IMAS-Python will not automatically " + "convert this data for you. See the documentation for more " + f"details and fixes: {imas.PUBLISHED_DOCUMENTATION_ROOT}" + "/multi-dd.html#loading-idss-from-a-different-major-version" ) ddmap, source_is_older = dd_version_map_from_factories( ids_name, IDSFactory(version=dd_version), self._ids_factory From a3b7a4a1cf318a8304a4d9c203f69e9e1a7e56f6 Mon Sep 17 00:00:00 2001 From: Maarten Sebregts Date: Thu, 13 Nov 2025 09:55:10 +0100 Subject: [PATCH 2/2] Fix failing test cases --- imas/test/test_lazy_loading.py | 6 ++++-- imas/training.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/imas/test/test_lazy_loading.py b/imas/test/test_lazy_loading.py index ff241016..4a7c65ca 100644 --- a/imas/test/test_lazy_loading.py +++ b/imas/test/test_lazy_loading.py @@ -224,10 +224,12 @@ def test_lazy_load_with_new_structure(requires_imas): eq.time_slice.resize(1) dbentry.put(eq) - entry2 = DBEntry(MEMORY_BACKEND, "ITER", 1, 1, data_version="3", dd_version="4.0.0") + entry2 = DBEntry( + MEMORY_BACKEND, "ITER", 1, 1, data_version="3", dd_version="3.39.0" + ) entry2.open() lazy_eq = entry2.get("equilibrium", lazy=True) - assert not lazy_eq.time_slice[0].boundary.dr_dz_zero_point.r.has_value + assert not lazy_eq.time_slice[0].boundary_separatrix.dr_dz_zero_point.r.has_value def test_lazy_load_multiple_ids(backend, worker_id, tmp_path): diff --git a/imas/training.py b/imas/training.py index 93e0c006..6effcc5b 100644 --- a/imas/training.py +++ b/imas/training.py @@ -19,8 +19,8 @@ def get_training_db_entry() -> imas.DBEntry: output_entry = imas.DBEntry("imas:memory?path=/", "w") for ids_name in ["core_profiles", "equilibrium"]: - ids = entry.get(ids_name) + ids = entry.get(ids_name, autoconvert=False) with patch.dict("os.environ", {"IMAS_AL_DISABLE_VALIDATE": "1"}): - output_entry.put(ids) + output_entry.put(imas.convert_ids(ids, output_entry.dd_version)) entry.close() return output_entry