diff --git a/benchmarks/core_profiles.py b/benchmarks/core_profiles.py index d7ab54cd..0bdeef8b 100644 --- a/benchmarks/core_profiles.py +++ b/benchmarks/core_profiles.py @@ -10,8 +10,6 @@ available_serializers, available_slicing_backends, create_dbentry, - factory, - hlis, ) N_SLICES = 32 @@ -26,13 +24,13 @@ def fill_slices(core_profiles, times): times: time values to fill a slice for """ core_profiles.ids_properties.homogeneous_time = 1 # HOMOGENEOUS - core_profiles.ids_properties.comment = "Generated for the IMAS-Python benchmark suite" + core_profiles.ids_properties.comment = ( + "Generated for the IMAS-Python benchmark suite" + ) core_profiles.ids_properties.creation_date = datetime.date.today().isoformat() core_profiles.code.name = "IMAS-Python ASV benchmark" core_profiles.code.version = imas.__version__ - core_profiles.code.repository = ( - "https://github.com/iterorganization/IMAS-Python" - ) + core_profiles.code.repository = "https://github.com/iterorganization/IMAS-Python" core_profiles.time = np.array(times) core_profiles.profiles_1d.resize(len(times)) @@ -50,13 +48,13 @@ def fill_slices(core_profiles, times): profiles_1d.ion.resize(len(ions)) profiles_1d.neutral.resize(len(ions)) for i, ion in enumerate(ions): - if hasattr(profiles_1d.ion[i], 'label'): + if hasattr(profiles_1d.ion[i], "label"): profiles_1d.ion[i].label = ion profiles_1d.neutral[i].label = ion - if hasattr(profiles_1d.ion[i], 'name'): + if hasattr(profiles_1d.ion[i], "name"): profiles_1d.ion[i].name = ion profiles_1d.neutral[i].name = ion - + # profiles_1d.ion[i].label = profiles_1d.neutral[i].label = ion profiles_1d.ion[i].z_ion = 1.0 profiles_1d.ion[i].neutral_index = profiles_1d.neutral[i].ion_index = i + 1 @@ -70,31 +68,31 @@ def fill_slices(core_profiles, times): class GetSlice: - params = [hlis, available_slicing_backends] - param_names = ["hli", "backend"] + params = [available_slicing_backends] + param_names = ["backend"] - def setup(self, hli, backend): - self.dbentry = create_dbentry(hli, backend) - core_profiles = factory[hli].core_profiles() + def setup(self, backend): + self.dbentry = create_dbentry(backend) + core_profiles = imas.IDSFactory().core_profiles() fill_slices(core_profiles, TIME) self.dbentry.put(core_profiles) - def time_get_slice(self, hli, backend): + def time_get_slice(self, backend): for t in TIME: self.dbentry.get_slice("core_profiles", t, imas.ids_defs.CLOSEST_INTERP) - def teardown(self, hli, backend): + def teardown(self, backend): if hasattr(self, "dbentry"): # imas + netCDF has no dbentry self.dbentry.close() class Get: - params = [hlis, available_backends] - param_names = ["hli", "backend"] + params = [available_backends] + param_names = ["backend"] setup = GetSlice.setup teardown = GetSlice.teardown - def time_get(self, hli, backend): + def time_get(self, backend): self.dbentry.get("core_profiles") @@ -103,8 +101,8 @@ class LazyGet: param_names = ["lazy", "backend"] def setup(self, lazy, backend): - self.dbentry = create_dbentry("imas", backend) - core_profiles = factory["imas"].core_profiles() + self.dbentry = create_dbentry(backend) + core_profiles = imas.IDSFactory().core_profiles() fill_slices(core_profiles, TIME) self.dbentry.put(core_profiles) @@ -118,75 +116,72 @@ def teardown(self, lazy, backend): class Generate: - params = [hlis] - param_names = ["hli"] + def setup(self): + self.core_profiles = imas.IDSFactory().core_profiles() - def setup(self, hli): - self.core_profiles = factory[hli].core_profiles() - - def time_generate(self, hli): + def time_generate(self): fill_slices(self.core_profiles, TIME) - def time_generate_slices(self, hli): + def time_generate_slices(self): for t in TIME: fill_slices(self.core_profiles, [t]) - def time_create_core_profiles(self, hli): - factory[hli].core_profiles() + def time_create_core_profiles(self): + imas.IDSFactory().core_profiles() class Put: - params = [["0", "1"], hlis, available_backends] + params = [["0", "1"], available_backends] param_names = ["disable_validate", "hli", "backend"] - def setup(self, disable_validate, hli, backend): - create_dbentry(hli, backend).close() # catch unsupported combinations - self.core_profiles = factory[hli].core_profiles() + def setup(self, disable_validate, backend): + create_dbentry(backend).close() # catch unsupported combinations + self.core_profiles = imas.IDSFactory().core_profiles() fill_slices(self.core_profiles, TIME) os.environ["IMAS_AL_DISABLE_VALIDATE"] = disable_validate - def time_put(self, disable_validate, hli, backend): - with create_dbentry(hli, backend) as dbentry: + def time_put(self, disable_validate, backend): + with create_dbentry(backend) as dbentry: dbentry.put(self.core_profiles) class PutSlice: - params = [["0", "1"], hlis, available_slicing_backends] - param_names = ["disable_validate", "hli", "backend"] + params = [["0", "1"], available_slicing_backends] + param_names = ["disable_validate", "backend"] - def setup(self, disable_validate, hli, backend): - create_dbentry(hli, backend).close() # catch unsupported combinations - self.core_profiles = factory[hli].core_profiles() + def setup(self, disable_validate, backend): + create_dbentry(backend).close() # catch unsupported combinations + self.core_profiles = imas.IDSFactory().core_profiles() os.environ["IMAS_AL_DISABLE_VALIDATE"] = disable_validate - def time_put_slice(self, disable_validate, hli, backend): - with create_dbentry(hli, backend) as dbentry: + def time_put_slice(self, disable_validate, backend): + with create_dbentry(backend) as dbentry: for t in TIME: fill_slices(self.core_profiles, [t]) dbentry.put_slice(self.core_profiles) class Serialize: - params = [hlis, available_serializers] - param_names = ["hli", "serializer"] + params = [available_serializers] + param_names = ["serializer"] - def setup(self, hli, serializer): - self.core_profiles = factory[hli].core_profiles() + def setup(self, serializer): + self.core_profiles = imas.IDSFactory().core_profiles() fill_slices(self.core_profiles, TIME) - def time_serialize(self, hli, serializer): + def time_serialize(self, serializer): self.core_profiles.serialize(serializer) class Deserialize: - params = [hlis, available_serializers] - param_names = ["hli", "serializer"] + params = [available_serializers] + param_names = ["serializer"] - def setup(self, hli, serializer): - self.core_profiles = factory[hli].core_profiles() + def setup(self, serializer): + self.core_profiles = imas.IDSFactory().core_profiles() fill_slices(self.core_profiles, TIME) self.data = self.core_profiles.serialize(serializer) - self.core_profiles = factory[hli].core_profiles() + self.core_profiles = imas.IDSFactory().core_profiles() - def time_deserialize(self, hli, serializer): + def time_deserialize(self, serializer): self.core_profiles.deserialize(self.data) diff --git a/benchmarks/edge_profiles.py b/benchmarks/edge_profiles.py index cb78629f..f1ec7fd7 100644 --- a/benchmarks/edge_profiles.py +++ b/benchmarks/edge_profiles.py @@ -5,7 +5,7 @@ import imas -from .utils import available_backends, create_dbentry, factory, hlis +from .utils import available_backends, create_dbentry N_POINTS = 600 # number of random R,Z points N_LINES = 1200 # number of random lines in R,Z plane @@ -27,9 +27,7 @@ def fill_ggd(edge_profiles, times): edge_profiles.ids_properties.creation_date = datetime.date.today().isoformat() edge_profiles.code.name = "IMAS-Python ASV benchmark" edge_profiles.code.version = imas.__version__ - edge_profiles.code.repository = ( - "https://github.com/iterorganization/IMAS-Python" - ) + edge_profiles.code.repository = "https://github.com/iterorganization/IMAS-Python" # This GGD grid is not a valid description, but it's a good stress test for the # typical access patterns that exist in GGD grids @@ -124,45 +122,42 @@ def fill_ggd(edge_profiles, times): class Get: - params = [hlis, available_backends] - param_names = ["hli", "backend"] + params = [available_backends] + param_names = ["backend"] - def setup(self, hli, backend): - self.dbentry = create_dbentry(hli, backend) - edge_profiles = factory[hli].edge_profiles() + def setup(self, backend): + self.dbentry = create_dbentry(backend) + edge_profiles = imas.IDSFactory().edge_profiles() fill_ggd(edge_profiles, TIME) self.dbentry.put(edge_profiles) - def time_get(self, hli, backend): + def time_get(self, backend): self.dbentry.get("edge_profiles") - def teardown(self, hli, backend): + def teardown(self, backend): if hasattr(self, "dbentry"): # imas + netCDF has no dbentry self.dbentry.close() class Generate: - params = [hlis] - param_names = ["hli"] - - def time_generate(self, hli): - edge_profiles = factory[hli].edge_profiles() + def time_generate(self): + edge_profiles = imas.IDSFactory().edge_profiles() fill_ggd(edge_profiles, TIME) - def time_create_edge_profiles(self, hli): - factory[hli].edge_profiles() + def time_create_edge_profiles(self): + imas.IDSFactory().edge_profiles() class Put: - params = [["0", "1"], hlis, available_backends] - param_names = ["disable_validate", "hli", "backend"] + params = [["0", "1"], available_backends] + param_names = ["disable_validate", "backend"] - def setup(self, disable_validate, hli, backend): - create_dbentry(hli, backend).close() # catch unsupported combinations - self.edge_profiles = factory[hli].edge_profiles() + def setup(self, disable_validate, backend): + create_dbentry(backend).close() # catch unsupported combinations + self.edge_profiles = imas.IDSFactory().edge_profiles() fill_ggd(self.edge_profiles, TIME) os.environ["IMAS_AL_DISABLE_VALIDATE"] = disable_validate - def time_put(self, disable_validate, hli, backend): - with create_dbentry(hli, backend) as dbentry: + def time_put(self, disable_validate, backend): + with create_dbentry(backend) as dbentry: dbentry.put(self.edge_profiles) diff --git a/benchmarks/utils.py b/benchmarks/utils.py index 47ae2576..3ff30fd3 100644 --- a/benchmarks/utils.py +++ b/benchmarks/utils.py @@ -1,10 +1,10 @@ -import importlib import logging import uuid from pathlib import Path import imas import imas.exception +import imas.ids_defs # Backend constants HDF5 = "HDF5" @@ -56,28 +56,12 @@ def backend_exists(backend): backend for backend in available_backends if backend not in [ASCII, NETCDF] ] -hlis = ["imas"] -DBEntry = { - "imas": imas.DBEntry, -} -factory = { - "imas": imas.IDSFactory(), -} -available_serializers = [imas.ids_defs.ASCII_SERIALIZER_PROTOCOL] - +available_serializers = [ + imas.ids_defs.ASCII_SERIALIZER_PROTOCOL, + imas.ids_defs.FLEXBUFFERS_SERIALIZER_PROTOCOL, +] -def create_dbentry(hli, backend): - if backend == NETCDF: - if hli == "imas": # check if netcdf backend is available - try: - assert ( - imas.DBEntry._select_implementation("x.nc").__name__ - == "NCDBEntryImpl" - ) - except (AttributeError, AssertionError): - raise NotImplementedError( - "This version of IMAS-Python doesn't implement netCDF." - ) from None - path = Path.cwd() / f"DB-{hli}-{backend}" - return DBEntry[hli](create_uri(backend, path), "w") +def create_dbentry(backend): + path = Path.cwd() / f"DB-{backend}" + return imas.DBEntry(create_uri(backend, path), "w") diff --git a/docs/source/courses/basic/analyze.rst b/docs/source/courses/basic/analyze.rst index 21a7c68b..97dd1ea3 100644 --- a/docs/source/courses/basic/analyze.rst +++ b/docs/source/courses/basic/analyze.rst @@ -25,15 +25,10 @@ can use the data. .. hint:: Use the ASCII data supplied with IMAS-Python for all exercises. It contains two IDSs (``equilibrium`` and ``core_profiles``) filled with data from three - time slices of ITER reference data. Two convenience methods are available in the - :mod:`imas.training` module to open the DBEntry for this training data. - - 1. :meth:`imas.training.get_training_db_entry()` returns an opened - ``imas.DBEntry`` object. Use this method if you want to use the IMAS-Python - interface. - 2. :meth:`imas.training.get_training_imas_db_entry()` returns an opened - ``imas.DBEntry`` object. Use this method if you want to use the Python Access - Layer interface. + time slices of ITER reference data. A convenience method is available in the + :mod:`imas.training` module to open the DBEntry for this training data: + :meth:`imas.training.get_training_db_entry()` returns an opened + ``imas.DBEntry`` object. Exercise 1 '''''''''' diff --git a/docs/source/lazy_loading.rst b/docs/source/lazy_loading.rst index 9dda19e0..745df066 100644 --- a/docs/source/lazy_loading.rst +++ b/docs/source/lazy_loading.rst @@ -90,4 +90,13 @@ Lazy loading of data may speed up your programs, but also comes with some limita more efficient to do a full :code:`get()` or :code:`get_slice()` when you intend to use most of the data stored in an IDS. 5. When using IMAS-Python with remote data access (i.e. the UDA backend), a full - :code:`get()` or :code:`get_slice()` is more efficient than lazy loading. + :code:`get()` or :code:`get_slice()` may be more efficient than using lazy loading. + + It is recommended to add the parameter ``;cache_mode=none`` [#cache_mode_none]_ to + the end of a UDA IMAS URI when using lazy loading: otherwise the UDA backend will + still load the full IDS from the remote server. + + +.. [#cache_mode_none] The option ``cache_mode=none`` requires IMAS Core version 5.5.1 or + newer, and a remote UDA server with `IMAS UDA-Plugins + `__ version 1.7.0 or newer. diff --git a/imas/backends/imas_core/al_context.py b/imas/backends/imas_core/al_context.py index 1685e384..ede33bac 100644 --- a/imas/backends/imas_core/al_context.py +++ b/imas/backends/imas_core/al_context.py @@ -71,10 +71,7 @@ def global_action(self, path: str, rwmode: int, datapath: str = "") -> "ALContex Returns: The created context. """ - args = [self.ctx, path, rwmode] - if datapath: # AL4 compatibility: datapath arg was added in AL5 - args.append(datapath) - status, ctx = ll_interface.begin_global_action(*args) + status, ctx = ll_interface.begin_global_action(self.ctx, path, rwmode, datapath) if status != 0: raise LowlevelError("global_action", status) return ALContext(ctx) diff --git a/imas/backends/imas_core/db_entry_al.py b/imas/backends/imas_core/db_entry_al.py index b3240ebd..2500bd3c 100644 --- a/imas/backends/imas_core/db_entry_al.py +++ b/imas/backends/imas_core/db_entry_al.py @@ -5,30 +5,28 @@ import logging import os from collections import deque +import re from typing import Any, Deque, List, Optional, Union from urllib.parse import urlparse +from packaging.version import Version + from imas.backends.db_entry_impl import GetSampleParameters, GetSliceParameters from imas.db_entry import DBEntryImpl from imas.exception import DataEntryException, LowlevelError from imas.ids_convert import NBCPathMap, dd_version_map_from_factories from imas.ids_defs import ( - ASCII_BACKEND, CHAR_DATA, CLOSE_PULSE, CREATE_PULSE, ERASE_PULSE, FORCE_CREATE_PULSE, FORCE_OPEN_PULSE, - HDF5_BACKEND, IDS_TIME_MODE_UNKNOWN, IDS_TIME_MODES, INTEGER_DATA, - MDSPLUS_BACKEND, - MEMORY_BACKEND, OPEN_PULSE, READ_OP, - UDA_BACKEND, UNDEFINED_INTERP, UNDEFINED_TIME, WRITE_OP, @@ -40,16 +38,9 @@ from .al_context import ALContext, LazyALContext from .db_entry_helpers import delete_children, get_children, put_children from .imas_interface import LLInterfaceError, has_imas, ll_interface -from .mdsplus_model import ensure_data_dir, mdsplus_model_dir +from .mdsplus_model import mdsplus_model_dir from .uda_support import extract_idsdef, get_dd_version_from_idsdef_xml -_BACKEND_NAME = { - ASCII_BACKEND: "ascii", - HDF5_BACKEND: "hdf5", - MEMORY_BACKEND: "memory", - MDSPLUS_BACKEND: "mdsplus", - UDA_BACKEND: "uda", -} _OPEN_MODES = { "r": OPEN_PULSE, "a": FORCE_OPEN_PULSE, @@ -71,13 +62,26 @@ def require_imas_available(): class ALDBEntryImpl(DBEntryImpl): """DBEntry implementation using imas_core as a backend.""" - """Map to the expected open_pulse (AL4) / begin_dataentry_action (AL5) argument.""" + def __init__(self, uri: str, mode: int, factory: IDSFactory): + # Setup backend and lowlevel Access Layer: + backend = urlparse(uri).path.lower().lstrip("/") + self._setup_backend(backend, mode, factory) + status, ctx = ll_interface.begin_dataentry_action(uri, mode) + if status != 0: + raise LowlevelError("opening/creating data entry", status) - def __init__(self, backend: str, ctx: ALContext, factory: IDSFactory): self.backend = backend - self._db_ctx = ctx + self._db_ctx = ALContext(ctx) self._ids_factory = factory self._lazy_ctx_cache: Deque[ALContext] = deque() + self._uri = uri + + # Parse query options, mimic logic in AL-Core instead of using + # urllib.parse.parse_qs(..). See https://github.com/jholloc/simple-uri-parser + self._querydict = {} + for option in re.split("[&;?]", urlparse(self._uri).query): + name, _, value = option.partition("=") + self._querydict[name] = value @classmethod def from_uri(cls, uri: str, mode: str, factory: IDSFactory) -> "ALDBEntryImpl": @@ -85,7 +89,7 @@ def from_uri(cls, uri: str, mode: str, factory: IDSFactory) -> "ALDBEntryImpl": if mode not in _OPEN_MODES: modes = list(_OPEN_MODES) raise ValueError(f"Unknown mode {mode!r}, was expecting any of {modes}") - return cls._from_uri(uri, _OPEN_MODES[mode], factory) + return cls(uri, _OPEN_MODES[mode], factory) @classmethod def from_pulse_run( @@ -108,60 +112,18 @@ def from_pulse_run( data_version = data_version or factory.dd_version options = options if options else "" - if ll_interface._al_version.major >= 5: - # We need a URI for AL 5 or later, construct from legacy parameters - status, uri = ll_interface.build_uri_from_legacy_parameters( - backend_id, pulse, run, user_name, db_name, data_version, options - ) - if status != 0: - raise LowlevelError("build URI from legacy parameters", status) - - return cls._from_uri(uri, mode, factory) - - else: - # AL4 legacy support: - backend = _BACKEND_NAME.get(backend_id, "") - cls._setup_backend(backend, mode, factory, user_name, db_name, run) - - status, ctx = ll_interface.begin_pulse_action( - backend_id, pulse, run, user_name, db_name, data_version - ) - if status != 0: - raise LowlevelError("begin pulse action", status) - - status = ll_interface.open_pulse(ctx, mode, options) - if status != 0: - raise LowlevelError("opening/creating data entry", status) - - return cls(backend, ALContext(ctx), factory) - - @classmethod - def _from_uri(cls, uri: str, mode: int, factory: IDSFactory) -> "ALDBEntryImpl": - """Helper method to actually open/create the dataentry.""" - backend = urlparse(uri).path.lower().lstrip("/") - cls._setup_backend(backend, mode, factory) - - status, ctx = ll_interface.begin_dataentry_action(uri, mode) + # Construct URI from legacy parameters + status, uri = ll_interface.build_uri_from_legacy_parameters( + backend_id, pulse, run, user_name, db_name, data_version, options + ) if status != 0: - raise LowlevelError("opening/creating data entry", status) + raise LowlevelError("build URI from legacy parameters", status) - return cls(backend, ALContext(ctx), factory) + return cls(uri, mode, factory) @classmethod - def _setup_backend( - cls, - backend: str, - mode: int, - factory: IDSFactory, - user_name: str = "", - db_name="", - run=1, - ) -> None: - """Custom logic for preparing some backends. - - Note: user_name, db_name and run are only used for AL 4.x, they can be - omitted when using AL 5 or later. - """ + def _setup_backend(cls, backend: str, mode: int, factory: IDSFactory) -> None: + """Custom logic for preparing some backends.""" if backend == "mdsplus": # MDSplus models: if mode != OPEN_PULSE: @@ -170,22 +132,6 @@ def _setup_backend( if ids_path: os.environ["ids_path"] = ids_path - if ll_interface._al_version.major == 4: - # Ensure the data directory exists - # Note: MDSPLUS model directory only uses the major version component of - # IMAS_VERSION, so we'll take the first character of IMAS_VERSION: - version = factory.version[0] - ensure_data_dir(user_name, db_name, version, run) - - elif backend == "hdf5": - pass # nothing to set up - - elif backend == "memory": - pass # nothing to set up - - elif backend == "ascii": - pass # nothing to set up - elif backend == "uda": # Set IDSDEF_PATH to point the UDA backend to the selected DD version idsdef_path = None @@ -193,7 +139,6 @@ def _setup_backend( if factory._xml_path is not None: # Factory was constructed with an explicit XML path, point UDA to that: idsdef_path = factory._xml_path - elif "IMAS_PREFIX" in os.environ: # Check if UDA can use the IDSDef.xml stored in $IMAS_PREFIX/include/ idsdef_path = os.environ["IMAS_PREFIX"] + "/include/IDSDef.xml" @@ -203,15 +148,9 @@ def _setup_backend( if idsdef_path is None: # Extract XML from the DD zip and point UDA to it idsdef_path = extract_idsdef(factory.version) - os.environ["IDSDEF_PATH"] = idsdef_path - logger.warning( - "The UDA backend is not tested with " - "IMAS-Python and may not work properly. " - "Please raise any issues you find." - ) - elif backend == "flexbuffers": + elif backend in ["hdf5", "memory", "ascii", "flexbuffers"]: pass # nothing to set up else: @@ -249,6 +188,8 @@ def get( raise RuntimeError("Database entry is not open.") if lazy and self.backend == "ascii": raise RuntimeError("Lazy loading is not supported by the ASCII backend.") + if self.backend == "uda": + self._check_uda_warnings(lazy) # Mixing contexts can be problematic, ensure all lazy contexts are closed: self._clear_lazy_ctx_cache() @@ -405,3 +346,28 @@ def list_all_occurrences(self, ids_name: str) -> List[int]: "Access Layer 5.1 or newer is required." ) from None return occurrence_list + + def _check_uda_warnings(self, lazy: bool) -> None: + """Various checks / warnings for the UDA backend.""" + cache_mode = self._querydict.get("cache_mode") + if lazy and cache_mode != "none": + # cache_mode=none requires imas core 5.5.1 or newer, and a recent enough UDA + # server plugin (which we cannot check...) + cache_mode_hint = "" + if ll_interface._al_version >= Version("5.5.1"): + cache_mode_hint = ( + "\nYou may add the parameter ';cache_mode=none' to the IMAS URI " + "to avoid loading all of the data from the remote server." + ) + logger.warning( + "The UDA backend will load all IDS data from the remote server. " + "Lazy loading with the UDA backend may therefore still be slow.%s", + cache_mode_hint, + ) + + if cache_mode == "none" and ll_interface._al_version < Version("5.5.1"): + logger.warning( + "UDA option 'cache_mode=None' may not work correctly with " + "IMAS Core version %s.", + ll_interface._al_version, + ) diff --git a/imas/backends/imas_core/imas_interface.py b/imas/backends/imas_core/imas_interface.py index 064508d7..8fa3963b 100644 --- a/imas/backends/imas_core/imas_interface.py +++ b/imas/backends/imas_core/imas_interface.py @@ -6,6 +6,7 @@ This module tries to abstract away most API incompatibilities between the supported Access Layer versions (for example the rename of _ual_lowlevel to _al_lowlevel). """ + import inspect import logging @@ -61,9 +62,6 @@ class LowlevelInterface: - If the lowlevel drops methods, we need to update the implementation fo the method to provide a proper error message or a workaround. - - Renamed methods (if this will ever happen) are perhaps best handled in the - ``__init__`` by providing a mapping of new to old name, so far this was only - relevant for the ``ual_`` to ``al_`` rename. """ def __init__(self, lowlevel): @@ -84,23 +82,13 @@ def __init__(self, lowlevel): # Introduced after 5.0.0 self._al_version_str = self._lowlevel.get_al_version() self._al_version = Version(self._al_version_str) - elif hasattr(lowlevel, "al_read_data"): - # In AL 5.0.0, all `ual_` methods were renamed to `al_` + else: self._al_version_str = "5.0.0" self._al_version = Version(self._al_version_str) - else: - # AL 4, don't try to determine in more detail - self._al_version_str = "4.?.?" - self._al_version = Version("4") - public_methods.remove("close_pulse") - if self._al_version < Version("5"): - method_prefix = "ual_" - else: - method_prefix = "al_" # Overwrite all of our methods that are implemented in the lowlevel for method in public_methods: - ll_method = getattr(lowlevel, method_prefix + method, None) + ll_method = getattr(lowlevel, f"al_{method}", None) if ll_method is not None: setattr(self, method, ll_method) @@ -115,24 +103,10 @@ def _minimal_version(self, minversion): f"but the current version is {self._al_version_str}" ) - # AL 4 lowlevel API - - def begin_pulse_action(self, backendID, shot, run, user, tokamak, version): - # Removed in AL5, compatibility handled in DBEntry - raise LLInterfaceError(f"{__name__} is not implemented") - - def open_pulse(self, pulseCtx, mode, options): - # Removed in AL5, compatibility handled in DBEntry - raise LLInterfaceError(f"{__name__} is not implemented") - def close_pulse(self, pulseCtx, mode): - # options argument (mandatory in AL4) was removed in AL5 - # This method is overwritten in AL5, but for AL4 we need to do this: - return lowlevel.ual_close_pulse(pulseCtx, mode, None) + raise LLInterfaceError(f"{__name__} is not implemented") - def begin_global_action(self, pulseCtx, dataobjectname, rwmode, datapath=""): - # datapath was added in AL5 to support more efficient partial_get in the - # UDA backend. TODO: figure out if this is useful for lazy loading. + def begin_global_action(self, pulseCtx, dataobjectname, rwmode, datapath): raise LLInterfaceError(f"{__name__} is not implemented") def begin_slice_action(self, pulseCtx, dataobjectname, rwmode, time, interpmode): diff --git a/imas/backends/imas_core/mdsplus_model.py b/imas/backends/imas_core/mdsplus_model.py index 3c91cefb..c5f09e29 100644 --- a/imas/backends/imas_core/mdsplus_model.py +++ b/imas/backends/imas_core/mdsplus_model.py @@ -364,25 +364,3 @@ def jTraverser_jar() -> Path: return jar_path else: raise MDSPlusModelError("jTraverser.jar not found. Is MDSplus-Java available?") - - -def ensure_data_dir(user: str, tokamak: str, version: str, run: int) -> None: - """Ensure that a data dir exists with a similar algorithm that - the MDSplus backend uses to set the data path. - See also mdsplus_backend.cpp:751 (setDataEnv)""" - if user == "public": - if "IMAS_HOME" not in os.environ: - raise RuntimeError( - "Environment variable IMAS_HOME must be set to access " - "the public database." - ) - dbdir = Path(os.environ["IMAS_HOME"]) / "shared" / "imasdb" / tokamak / version - elif user[0] == "/": - dbdir = Path(user) / tokamak / version - else: - dbdir = Path.home() / "public" / "imasdb" / tokamak / version - - # Check subfolder based on run - assert 0 <= run <= 99_999 - index = run // 10_000 - (dbdir / str(index)).mkdir(parents=True, exist_ok=True) diff --git a/imas/ids_toplevel.py b/imas/ids_toplevel.py index 15ae0970..947bf72f 100644 --- a/imas/ids_toplevel.py +++ b/imas/ids_toplevel.py @@ -1,7 +1,6 @@ # This file is part of IMAS-Python. # You should have received the IMAS-Python LICENSE file with this project. -"""Represents a Top-level IDS (like ``core_profiles``, ``equilibrium``, etc) -""" +"""Represents a Top-level IDS (like ``core_profiles``, ``equilibrium``, etc)""" import logging import os @@ -12,11 +11,10 @@ import numpy import imas -from imas.backends.imas_core.imas_interface import ll_interface, lowlevel +from imas.backends.imas_core.imas_interface import lowlevel from imas.exception import ValidationError from imas.ids_base import IDSDoc from imas.ids_defs import ( - ASCII_BACKEND, ASCII_SERIALIZER_PROTOCOL, CHAR_DATA, DEFAULT_SERIALIZER_PROTOCOL, @@ -47,19 +45,12 @@ def _serializer_tmpdir() -> str: def _create_serialization_dbentry(filepath: str, dd_version: str) -> "DBEntry": """Create a temporary DBEntry for use in the ASCII serialization protocol.""" - if ll_interface._al_version.major == 4: # AL4 compatibility - dbentry = imas.DBEntry( - ASCII_BACKEND, "serialize", 1, 1, "serialize", dd_version=dd_version - ) - dbentry.create(options=f"-fullpath {filepath}") - return dbentry - else: # AL5 - path = Path(filepath) - return imas.DBEntry( - f"imas:ascii?path={path.parent};filename={path.name}", - "w", - dd_version=dd_version, - ) + path = Path(filepath) + return imas.DBEntry( + f"imas:ascii?path={path.parent};filename={path.name}", + "w", + dd_version=dd_version, + ) class IDSToplevel(IDSStructure): diff --git a/imas/test/test_cli.py b/imas/test/test_cli.py index 6ff09c23..c6ddbc0e 100644 --- a/imas/test/test_cli.py +++ b/imas/test/test_cli.py @@ -2,10 +2,7 @@ import pytest from click.testing import CliRunner -from packaging.version import Version -from imas.backends.imas_core.imas_interface import has_imas -from imas.backends.imas_core.imas_interface import ll_interface from imas.command.cli import print_version from imas.command.db_analysis import analyze_db, process_db_analysis from imas.db_entry import DBEntry @@ -20,13 +17,7 @@ def test_imas_version(): @pytest.mark.cli -@pytest.mark.skipif( - not has_imas or ll_interface._al_version < Version("5.0"), - reason="Needs AL >= 5 AND Requires IMAS Core.", -) -def test_db_analysis( - tmp_path, -): +def test_db_analysis(tmp_path, requires_imas): # This only tests the happy flow, error handling is not tested db_path = tmp_path / "test_db_analysis" with DBEntry(f"imas:hdf5?path={db_path}", "w") as entry: diff --git a/imas/test/test_dbentry.py b/imas/test/test_dbentry.py index a1380101..e13d82a4 100644 --- a/imas/test/test_dbentry.py +++ b/imas/test/test_dbentry.py @@ -2,7 +2,6 @@ import imas import imas.ids_defs -from imas.backends.imas_core.imas_interface import has_imas, ll_interface from imas.exception import UnknownDDVersion from imas.test.test_helpers import compare_children, open_dbentry @@ -23,11 +22,7 @@ def test_dbentry_contextmanager(requires_imas): assert entry2._dbe_impl is None -@pytest.mark.skipif( - not has_imas or ll_interface._al_version.major < 5, - reason="URI API not available", -) -def test_dbentry_contextmanager_uri(tmp_path): +def test_dbentry_contextmanager_uri(tmp_path, requires_imas): entry = imas.DBEntry(f"imas:ascii?path={tmp_path}/testdb", "w") ids = entry.factory.core_profiles() ids.ids_properties.homogeneous_time = 0 diff --git a/imas/test/test_time_slicing.py b/imas/test/test_time_slicing.py index 60788689..21c689f4 100644 --- a/imas/test/test_time_slicing.py +++ b/imas/test/test_time_slicing.py @@ -3,18 +3,15 @@ """ import logging -import os import numpy as np import pytest -from imas.backends.imas_core.mdsplus_model import ensure_data_dir, mdsplus_model_dir from imas.ids_defs import ( ASCII_BACKEND, CLOSEST_INTERP, IDS_TIME_MODE_HETEROGENEOUS, IDS_TIME_MODE_HOMOGENEOUS, - MDSPLUS_BACKEND, ) from imas.ids_factory import IDSFactory from imas.test.test_helpers import open_dbentry @@ -97,10 +94,6 @@ def test_hli_time_slicing_put(backend, worker_id, tmp_path, time_mode): else: pulse = int(worker_id[2:]) + 1 - # ensure presence of mdsplus model dir - if backend == MDSPLUS_BACKEND: - os.environ["ids_path"] = mdsplus_model_dir(IDSFactory()) - ensure_data_dir(str(tmp_path), "test", "3", 9999) db_entry = imas.DBEntry(backend, "test", pulse, 9999, user_name=str(tmp_path)) status, ctx = db_entry.create() if status != 0: diff --git a/imas/training.py b/imas/training.py index 9c4df602..93e0c006 100644 --- a/imas/training.py +++ b/imas/training.py @@ -1,9 +1,7 @@ # This file is part of IMAS-Python. # You should have received the IMAS-Python LICENSE file with this project. -"""Functions that are useful for the IMAS-Python training courses. -""" +"""Functions that are useful for the IMAS-Python training courses.""" -import importlib from unittest.mock import patch try: @@ -12,34 +10,17 @@ from importlib_resources import files import imas -from imas.backends.imas_core.imas_interface import ll_interface -def _initialize_training_db(DBEntry_cls): +def get_training_db_entry() -> imas.DBEntry: + """Open and return an ``imas.DBEntry`` pointing to the training data.""" assets_path = files(imas) / "assets/" - pulse, run, user, database = 134173, 106, "public", "ITER" - if ll_interface._al_version.major == 4: - entry = DBEntry_cls(imas.ids_defs.ASCII_BACKEND, database, pulse, run, user) - entry.open(options=f"-prefix {assets_path}/") - else: - entry = DBEntry_cls(f"imas:ascii?path={assets_path}", "r") + entry = imas.DBEntry(f"imas:ascii?path={assets_path}", "r") - output_entry = DBEntry_cls(imas.ids_defs.MEMORY_BACKEND, database, pulse, run) - output_entry.create() + output_entry = imas.DBEntry("imas:memory?path=/", "w") for ids_name in ["core_profiles", "equilibrium"]: ids = entry.get(ids_name) with patch.dict("os.environ", {"IMAS_AL_DISABLE_VALIDATE": "1"}): output_entry.put(ids) entry.close() return output_entry - - -def get_training_db_entry() -> imas.DBEntry: - """Open and return an ``imas.DBEntry`` pointing to the training data.""" - return _initialize_training_db(imas.DBEntry) - - -def get_training_imas_db_entry(): - """Open and return an ``imas.DBEntry`` pointing to the training data.""" - imas = importlib.import_module("imas") - return _initialize_training_db(imas.DBEntry)