From c41e4cff1b70546047ada4ad573c95249d0f1282 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 3 Feb 2026 13:02:33 +0100 Subject: [PATCH 1/5] Data resampling (init) --- backend/ibex/core/ibex_service.py | 62 +------------------ backend/ibex/core/utils.py | 58 +++++++++++++++++ .../ibex/data_source/imas_python_source.py | 31 +++++++++- .../data_source/imas_python_source_utils.py | 27 ++++++++ backend/ibex/endpoints/data.py | 6 +- 5 files changed, 121 insertions(+), 63 deletions(-) create mode 100644 backend/ibex/data_source/imas_python_source_utils.py diff --git a/backend/ibex/core/ibex_service.py b/backend/ibex/core/ibex_service.py index 1a4e337f..9832c9d0 100644 --- a/backend/ibex/core/ibex_service.py +++ b/backend/ibex/core/ibex_service.py @@ -1,70 +1,13 @@ """Logic between endpoint and data sources""" import time -import re from pathlib import Path from functools import wraps # for measure_execution_time() from typing import Any, Callable, Optional, Sequence, List from ibex.data_source.imas_python_source import IMASPythonSource from ibex.data_source.exception import CannotGenerateUriException -from dataclasses import dataclass - - -@dataclass -class IMAS_URI: - """ - Helper class to extract arguments from imas uri - """ - - #: Full URI containing pulse file identifier, ids name and path to node - full_uri: str = "" - - #: pulse file identifier extracted from full URI - uri_entry_identifiers: str = "" - #: fragment part from full URI containing ids name and path to node - uri_fragment: str = "" - #: ids name extracted from full URI - ids_name: str = "" - #: path to node extracted from full URI - node_path: str = "" - #: ids occurrence number extracted from full URI - occurrence: int = 0 - - def __init__(self, full_uri): - """ - IMAS_URI constructor - :param full_uri: pulsefile uri along with #fragment part - """ - - self.full_uri = full_uri - - if "#" not in self.full_uri: - self.uri_entry_identifiers = self.full_uri - return - - self.uri_entry_identifiers, self.uri_fragment = self.full_uri.split("#", 1) - - pattern = r"^(?P[^:/]+)(?::(?P[^/]*))?(?:/(?P.*))?$" - - match = re.match(pattern, self.uri_fragment) - - if not match: - return - - self.ids_name = match.group("idsname") if match.group("idsname") else "" - self.occurrence = match.group("occurrence") if match.group("occurrence") else 0 - self.node_path = match.group("node_path") if match.group("node_path") else "" - - def __str__(self): - return ( - f"FULL URI : {self.full_uri}\n" - f"URI : {self.uri_entry_identifiers}\n" - f"FRAGMENT : {self.uri_fragment}\n" - f"IDS : {self.ids_name}\n" - f"OCCURRENCE : {self.occurrence}\n" - f"NODE_PATH : {self.node_path}\n" - ) +from ibex.core.utils import IMAS_URI # helper decorator used during development @@ -172,13 +115,14 @@ def get_multiple_node_data(uri: str) -> dict: ) -def get_plot_data(uri: str, downsampling_method: str | None, downsampled_size: int) -> dict: +def get_plot_data(uri: str, interpolate_over: List[str] | None, downsampling_method: str | None, downsampled_size: int) -> dict: uri_obj = IMAS_URI(uri) return data_source.get_plot_data( uri=uri_obj.uri_entry_identifiers, ids=uri_obj.ids_name, node_path=uri_obj.node_path, occurrence=uri_obj.occurrence, + interpolate_over=interpolate_over, downsampling_method=downsampling_method, downsampled_size=downsampled_size, ) diff --git a/backend/ibex/core/utils.py b/backend/ibex/core/utils.py index f07404e6..149836d9 100644 --- a/backend/ibex/core/utils.py +++ b/backend/ibex/core/utils.py @@ -6,6 +6,8 @@ from ibex.data_source.exception import NotAnArrayException import numpy as np # type: ignore +from dataclasses import dataclass +import re def find_first_value_in_list(data: list): @@ -202,3 +204,59 @@ def downsample_data(data: List, target_size: int, method: str | None = None, x=N return x, data[s_ds] return x, data[s_ds] + +@dataclass +class IMAS_URI: + """ + Helper class to extract arguments from imas uri + """ + + #: Full URI containing pulse file identifier, ids name and path to node + full_uri: str = "" + + #: pulse file identifier extracted from full URI + uri_entry_identifiers: str = "" + #: fragment part from full URI containing ids name and path to node + uri_fragment: str = "" + #: ids name extracted from full URI + ids_name: str = "" + #: path to node extracted from full URI + node_path: str = "" + #: ids occurrence number extracted from full URI + occurrence: int = 0 + + def __init__(self, full_uri): + """ + IMAS_URI constructor + :param full_uri: pulsefile uri along with #fragment part + """ + + self.full_uri = full_uri + + if "#" not in self.full_uri: + self.uri_entry_identifiers = self.full_uri + return + + self.uri_entry_identifiers, self.uri_fragment = self.full_uri.split("#", 1) + + pattern = r"^(?P[^:/]+)(?::(?P[^/]*))?(?:/(?P.*))?$" + + match = re.match(pattern, self.uri_fragment) + + if not match: + return + + self.ids_name = match.group("idsname") if match.group("idsname") else "" + self.occurrence = match.group("occurrence") if match.group("occurrence") else 0 + self.node_path = match.group("node_path") if match.group("node_path") else "" + + def __str__(self): + return ( + f"FULL URI : {self.full_uri}\n" + f"URI : {self.uri_entry_identifiers}\n" + f"FRAGMENT : {self.uri_fragment}\n" + f"IDS : {self.ids_name}\n" + f"OCCURRENCE : {self.occurrence}\n" + f"NODE_PATH : {self.node_path}\n" + ) + diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 3a822dea..63767b18 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -37,6 +37,8 @@ InvalidParametersException, ) from ibex.core.utils import downsample_data, transform_2D_data, find_first_value_in_list +from ibex.core.utils import IMAS_URI +from ibex.data_source.imas_python_source_utils import convert_ids_data_into_numpy_array, resample_data, union_arrays class IMASPythonSource(DataSourceInterface): @@ -575,6 +577,7 @@ def get_plot_data( ids: str, node_path: str, occurrence: int = 0, + interpolate_over: List[str] | None = None, downsampling_method: str | None = None, downsampled_size: int = 1000, ): @@ -764,7 +767,33 @@ def is_empty(seq): } coordinates_to_be_returned.append(c) first_value = find_first_value_in_list(ids_data) - data_to_be_returned = ids_data + data_to_be_returned = convert_ids_data_into_numpy_array(ids_data) + + # ============= BEGIN resample data onto new time vector ============= + + if interpolate_over: + if first_value.metadata.ndim == 1 and coordinates_to_be_returned[0]["name"] == "time": + #1 collect all time vectors + time_vectors = [] + + time_vectors.append(coordinates_to_be_returned[0]["value"]) + for interpolation_uri in interpolate_over: + uri = IMAS_URI(interpolation_uri) + time_vectors.append(self.get_data(uri=uri.uri_entry_identifiers, ids=uri.ids_name, node_path="time", occurrence=uri.occurrence)["value"]) + + #2 calculate common time vector + common_tv = union_arrays(time_vectors) + + #3 interpolate data_to_be_returned + data_to_be_returned = resample_data(data=data_to_be_returned,original_x=coordinates_to_be_returned[0]["value"], target_x=common_tv) + + #4 replace `time` coordinate with new time vector + coordinates_to_be_returned[0]["value"] = common_tv + + if not coordinates_to_be_returned[0]["shape"] == "irregular": + coordinates_to_be_returned[0]["shape"] = np.asarray(coordinates_to_be_returned[0]["value"]).shape + + # ============= END resample data onto new time vector ============= if first_value.metadata.ndim == 2: # Transform 2D arrays. diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py new file mode 100644 index 00000000..a51287a0 --- /dev/null +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -0,0 +1,27 @@ +import numpy as np +from functools import reduce +from imas.ids_primitive import IDSNumericArray + +def union_arrays(data : list): + return reduce(np.union1d, data) + +def resample_data(data, original_x, target_x): + + if isinstance(data, list): + return [resample_data(x, original_x, target_x) for x in data] + elif isinstance(data, IDSNumericArray): + return np.interp(target_x, original_x, data.value, left=np.nan, right=np.nan) + elif isinstance(data, np.ndarray): + return np.interp(target_x, original_x, data, left=np.nan, right=np.nan) + else: + return data + + +def convert_ids_data_into_numpy_array(data: list): + + if isinstance(data, list): + return [convert_ids_data_into_numpy_array(x) for x in data] + elif isinstance(data, IDSNumericArray): + return data.value + else: + return data \ No newline at end of file diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index 0cc0f8c4..67f74d31 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -1,7 +1,7 @@ """Endpoints extracting data from data source""" import orjson -from typing import List, Any +from typing import List, Any, Optional from fastapi import APIRouter, Query # type: ignore from fastapi.responses import ORJSONResponse # type: ignore @@ -74,7 +74,7 @@ def field_value( description="Returns single (or tensorized) data node value with detailed parameters used to plot the data", ) @ibex_service.measure_execution_time -def plot_data(uri: str, downsampling_method: str | None = Query(None), downsampled_size: int = 1000) -> Any: +def plot_data(uri: str, interpolate_over : Optional[List[str]] = Query(None), downsampling_method: str | None = Query(None), downsampled_size: int = 1000) -> Any: """ IBEX endpoint. Prepares and returns full information about data node and it's coordinates. @@ -114,4 +114,4 @@ def plot_data(uri: str, downsampling_method: str | None = Query(None), downsampl :rtype: dict (automatically converted to JSON by FastAPI) :return: JSON response """ - return CustomORJSONResponse(ibex_service.get_plot_data(uri.strip(), downsampling_method, downsampled_size)) + return CustomORJSONResponse(ibex_service.get_plot_data(uri.strip(), interpolate_over, downsampling_method, downsampled_size)) From 0b38152348eb6e5f8cdc331a40ef58c198138e46 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 24 Feb 2026 12:27:03 +0100 Subject: [PATCH 2/5] Cooridnates joining (WIP) --- .../ibex/data_source/imas_python_source.py | 50 ++++++++++++++++- .../data_source/imas_python_source_utils.py | 53 ++++++++++++++++++- 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index b06e422f..b4e500fc 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -38,7 +38,7 @@ ) from ibex.core.utils import downsample_data, transform_2D_data, find_first_value_in_list from ibex.core.utils import IMAS_URI -from ibex.data_source.imas_python_source_utils import convert_ids_data_into_numpy_array, resample_data, union_arrays +from ibex.data_source.imas_python_source_utils import convert_ids_data_into_numpy_array, resample_data, union_arrays, resample_data2, join_coordinates class IMASPythonSource(DataSourceInterface): @@ -767,7 +767,47 @@ def get_plot_data( # ============= BEGIN resample data onto new time vector ============= + def convert_to_lists(data): + """TODO: delete after development""" + if isinstance(data, list): + return [convert_to_lists(d) for d in data] + elif isinstance(data, (np.ndarray, IDSNumericArray)): + return data.tolist() + else: + return data + if interpolate_over: + # =================== GATHER ALL COORDINATES =================== + coords = {} + for c in coordinates_to_be_returned: + coords[c["name"]] = c["value"] + + for _uri in interpolate_over: + _uri_obj = IMAS_URI(_uri) + + if _uri_obj.ids_name != ids or _uri_obj.node_path != node_path: + raise InvalidParametersException("IDS name and node path should be the same for source and target URI when interpolating data") + + coord = self.get_plot_data(uri=_uri_obj.uri_entry_identifiers, + ids=_uri_obj.ids_name, + node_path=_uri_obj.node_path, + occurrence=_uri_obj.occurrence, + downsampling_method=downsampling_method, + downsampled_size=downsampled_size)["data"]["coordinates"] + + _coord_dict = {} + for c in coord: + _coord_dict[c["name"]] = c["value"] + + # merge gathered coordinate into common coordinate list + for key in coords.keys(): + coords[key] = join_coordinates(coords[key] , _coord_dict[key]) + + # =================== INTERPOLATE =================== + + + + """ if first_value.metadata.ndim == 1 and coordinates_to_be_returned[0]["name"] == "time": #1 collect all time vectors time_vectors = [] @@ -781,13 +821,19 @@ def get_plot_data( common_tv = union_arrays(time_vectors) #3 interpolate data_to_be_returned - data_to_be_returned = resample_data(data=data_to_be_returned,original_x=coordinates_to_be_returned[0]["value"], target_x=common_tv) + if False: + data_to_be_returned = resample_data(data=data_to_be_returned,original_x=coordinates_to_be_returned[0]["value"], target_x=common_tv) + else: + print(f" START RESAMPLING") + data_to_be_returned = resample_data2(data=data_to_be_returned,original_x=coordinates_to_be_returned[0]["value"],target_x=common_tv) #4 replace `time` coordinate with new time vector coordinates_to_be_returned[0]["value"] = common_tv if not coordinates_to_be_returned[0]["shape"] == "irregular": coordinates_to_be_returned[0]["shape"] = np.asarray(coordinates_to_be_returned[0]["value"]).shape + """ + # ============= END resample data onto new time vector ============= diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index a51287a0..d5fbf555 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -1,10 +1,47 @@ -import numpy as np from functools import reduce + +import numpy as np from imas.ids_primitive import IDSNumericArray +from scipy.interpolate import LinearNDInterpolator + def union_arrays(data : list): return reduce(np.union1d, data) +def join_coordinates(a, b): + """ + Joins N-dimensional coordinate arrays together. Assumes data is np.ndarray or IDSNumericArray + :param a: left side of join - list or np.ndarray + :param b: right side of join - list or np.ndarray + :return: joined list + """ + + # merge np arrays + if isinstance(a, (IDSNumericArray,np.ndarray)) and isinstance(b, (IDSNumericArray,np.ndarray)): + return np.unique(np.concatenate((a, b), axis=0)) + + # if a and b are lists → join them and + if isinstance(a, list) and isinstance(b, list): + max_len = max(len(a), len(b)) + result = [] + + for i in range(max_len): + if i < len(a) and i < len(b): + result.append(join_coordinates(a[i], b[i])) + elif i < len(a): + result.append(a[i]) + else: + result.append(b[i]) + + return result + + if a is None: + return b + if b is None: + return a + + raise TypeError(f"Different data types when merging coordinates during data interpolation: {type(a)} vs {type(b)}") + def resample_data(data, original_x, target_x): if isinstance(data, list): @@ -16,6 +53,20 @@ def resample_data(data, original_x, target_x): else: return data +def resample_data2(data1, original_coords, target_coords): + + # time_slice(itime)/profiles_2d(i1)/psi + # 1- time_slice(itime)/profiles_2d(i1)/grid/dim1 + # 2- time_slice(itime)/profiles_2d(i1)/grid/dim2 + + # class LinearNDInterpolator(points, values, fill_value=np.nan, rescale=False) + # interp = LinearNDInterpolator(list(zip(x, y)), z) + + interp = LinearNDInterpolator(original_coords, data1) + result = interp(target_coords, data1) + + print(f"==== RESULT {result}") + return result def convert_ids_data_into_numpy_array(data: list): From f74ad2bbe87b2d0bfd792a5eea0cd7356d37986a Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 2 Mar 2026 14:09:07 +0100 Subject: [PATCH 3/5] Data resampling (WIP) --- .../ibex/data_source/imas_python_source.py | 47 ++++++++++++++---- .../data_source/imas_python_source_utils.py | 49 +++++++++++++++++-- 2 files changed, 82 insertions(+), 14 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index a1ddc5c4..2a67cc1d 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -38,7 +38,7 @@ ) from ibex.core.utils import downsample_data, transform_2D_data, find_first_value_in_list from ibex.core.utils import IMAS_URI -from ibex.data_source.imas_python_source_utils import convert_ids_data_into_numpy_array, resample_data, union_arrays, resample_data2, join_coordinates +from ibex.data_source.imas_python_source_utils import convert_ids_data_into_numpy_array, resample_data, union_arrays, resample_data2, join_coordinates, pad_to_rectangular, flatten class IMASPythonSource(DataSourceInterface): @@ -782,9 +782,13 @@ def convert_to_lists(data): if interpolate_over: # =================== GATHER ALL COORDINATES =================== - coords = {} - for c in coordinates_to_be_returned: - coords[c["name"]] = c["value"] + original_coord_values = [] + new_common_coords = coordinates_to_be_returned + for c in new_common_coords: + c["value"] = convert_to_lists(c["value"]) + original_coord_values.append(sorted(set(flatten(c["value"])))) + original_coord_values.reverse() + for _uri in interpolate_over: _uri_obj = IMAS_URI(_uri) @@ -792,24 +796,47 @@ def convert_to_lists(data): if _uri_obj.ids_name != ids or _uri_obj.node_path != node_path: raise InvalidParametersException("IDS name and node path should be the same for source and target URI when interpolating data") - coord = self.get_plot_data(uri=_uri_obj.uri_entry_identifiers, + interpolate_to_coordinates = self.get_plot_data(uri=_uri_obj.uri_entry_identifiers, ids=_uri_obj.ids_name, node_path=_uri_obj.node_path, occurrence=_uri_obj.occurrence, downsampling_method=downsampling_method, downsampled_size=downsampled_size)["data"]["coordinates"] - _coord_dict = {} - for c in coord: - _coord_dict[c["name"]] = c["value"] + #_coord_dict = {} + #for c in new_coord: + # _coord_dict[c["name"]] = c["value"] # merge gathered coordinate into common coordinate list - for key in coords.keys(): - coords[key] = join_coordinates(coords[key] , _coord_dict[key]) + #for key in new_common_coords.keys(): + # new_common_coords[key] = join_coordinates(new_common_coords[key] , _coord_dict[key]) + + if len(interpolate_to_coordinates) != len(coordinates_to_be_returned): + # TODO: return proper exception + raise Exception("") + + for x,y in zip(coordinates_to_be_returned, interpolate_to_coordinates): + if x["name"] != y["name"]: + # TODO: return proper exception + # coordinates between quantities doesn't match + raise Exception("") + + x["value"] = sorted(set(flatten(x["value"])+flatten(convert_to_lists(y["value"])))) + # join values + # reverse coordinates list so it matches data dimensions + new_common_coords.reverse() + common_coords_values = [c["value"] for c in new_common_coords] # =================== INTERPOLATE =================== + #print(f"=== ORIGINAL COORDS: {original_coord_values}") + print(f"=== NEW COORDS: {common_coords_values}") + # === make data vector rectangular === + data_to_be_returned = pad_to_rectangular(data_to_be_returned) + print(f"=== OLD DATA VALUES: {data_to_be_returned}") + new_data_values = resample_data2(tuple(original_coord_values),data_to_be_returned, tuple(common_coords_values)) + print(f"=== NEW DATA VALUES: {new_data_values}") """ if first_value.metadata.ndim == 1 and coordinates_to_be_returned[0]["name"] == "time": diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index d5fbf555..d4a10eaf 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -2,12 +2,50 @@ import numpy as np from imas.ids_primitive import IDSNumericArray -from scipy.interpolate import LinearNDInterpolator +from scipy.interpolate import LinearNDInterpolator, RegularGridInterpolator def union_arrays(data : list): return reduce(np.union1d, data) +def flatten(lst): + result = [] + for item in lst: + if isinstance(item, list): + result.extend(flatten(item)) + else: + result.append(item) + return result + +def get_max_shape(lst, level=0, shape=None): + if shape is None: + shape = [] + + if isinstance(lst, (list,np.ndarray)): + if len(shape) <= level: + shape.append(0) + shape[level] = max(shape[level], len(lst)) + + for item in lst: + get_max_shape(item, level + 1, shape) + + return shape + + +def fill_array(arr, lst, index=()): + if isinstance(lst, list): + for i, item in enumerate(lst): + fill_array(arr, item, index + (i,)) + else: + arr[index] = lst + + +def pad_to_rectangular(lst): + shape = tuple(get_max_shape(lst)) + arr = np.full(shape, np.nan) + fill_array(arr, lst) + return arr + def join_coordinates(a, b): """ Joins N-dimensional coordinate arrays together. Assumes data is np.ndarray or IDSNumericArray @@ -53,7 +91,7 @@ def resample_data(data, original_x, target_x): else: return data -def resample_data2(data1, original_coords, target_coords): +def resample_data2(original_coords, data, target_coords): # time_slice(itime)/profiles_2d(i1)/psi # 1- time_slice(itime)/profiles_2d(i1)/grid/dim1 @@ -62,8 +100,11 @@ def resample_data2(data1, original_coords, target_coords): # class LinearNDInterpolator(points, values, fill_value=np.nan, rescale=False) # interp = LinearNDInterpolator(list(zip(x, y)), z) - interp = LinearNDInterpolator(original_coords, data1) - result = interp(target_coords, data1) + print(f"=== RESAMPLE ORIGINAL COORDS: {original_coords}") + print(f"=== RESAMPLE TARGET COORDS: {target_coords}") + print(f"=== RESAMPLE INPUT DATA: {data}") + interpolator = RegularGridInterpolator(original_coords, data) + result = interpolator(target_coords) print(f"==== RESULT {result}") return result From 312426b5c5b712f1829e2d0dcf879c8fdc89c7db Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 9 Mar 2026 13:56:03 +0100 Subject: [PATCH 4/5] Data resampling + apply formatting --- backend/ibex/core/ibex_service.py | 4 +- backend/ibex/core/utils.py | 2 +- .../ibex/data_source/imas_python_source.py | 84 +++++---------- .../data_source/imas_python_source_utils.py | 102 ++++++++---------- backend/ibex/endpoints/data.py | 11 +- 5 files changed, 81 insertions(+), 122 deletions(-) diff --git a/backend/ibex/core/ibex_service.py b/backend/ibex/core/ibex_service.py index 46da9300..4d99c5fc 100644 --- a/backend/ibex/core/ibex_service.py +++ b/backend/ibex/core/ibex_service.py @@ -115,7 +115,9 @@ def get_multiple_node_data(uri: str) -> dict: ) -def get_plot_data(uri: str, interpolate_over: List[str] | None, downsampling_method: str | None, downsampled_size: int) -> dict: +def get_plot_data( + uri: str, interpolate_over: List[str] | None, downsampling_method: str | None, downsampled_size: int +) -> dict: uri_obj = IMAS_URI(uri) return data_source.get_plot_data( uri=uri_obj.uri_entry_identifiers, diff --git a/backend/ibex/core/utils.py b/backend/ibex/core/utils.py index 5454c1dd..a5f9df52 100644 --- a/backend/ibex/core/utils.py +++ b/backend/ibex/core/utils.py @@ -215,6 +215,7 @@ def downsample_data(data: List, target_size: int, method: str | None = None, x=N return x, data[s_ds] + @dataclass class IMAS_URI: """ @@ -269,4 +270,3 @@ def __str__(self): f"OCCURRENCE : {self.occurrence}\n" f"NODE_PATH : {self.node_path}\n" ) - diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 2a67cc1d..ad0125db 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -38,7 +38,12 @@ ) from ibex.core.utils import downsample_data, transform_2D_data, find_first_value_in_list from ibex.core.utils import IMAS_URI -from ibex.data_source.imas_python_source_utils import convert_ids_data_into_numpy_array, resample_data, union_arrays, resample_data2, join_coordinates, pad_to_rectangular, flatten +from ibex.data_source.imas_python_source_utils import ( + convert_ids_data_into_numpy_array, + resample_data, + pad_to_rectangular, + flatten, +) class IMASPythonSource(DataSourceInterface): @@ -789,82 +794,45 @@ def convert_to_lists(data): original_coord_values.append(sorted(set(flatten(c["value"])))) original_coord_values.reverse() - for _uri in interpolate_over: _uri_obj = IMAS_URI(_uri) if _uri_obj.ids_name != ids or _uri_obj.node_path != node_path: - raise InvalidParametersException("IDS name and node path should be the same for source and target URI when interpolating data") - - interpolate_to_coordinates = self.get_plot_data(uri=_uri_obj.uri_entry_identifiers, - ids=_uri_obj.ids_name, - node_path=_uri_obj.node_path, - occurrence=_uri_obj.occurrence, - downsampling_method=downsampling_method, - downsampled_size=downsampled_size)["data"]["coordinates"] - - #_coord_dict = {} - #for c in new_coord: - # _coord_dict[c["name"]] = c["value"] + raise InvalidParametersException( + "IDS name and node path should be the same for source and target URI when interpolating data" + ) - # merge gathered coordinate into common coordinate list - #for key in new_common_coords.keys(): - # new_common_coords[key] = join_coordinates(new_common_coords[key] , _coord_dict[key]) + interpolate_to_coordinates = self.get_plot_data( + uri=_uri_obj.uri_entry_identifiers, + ids=_uri_obj.ids_name, + node_path=_uri_obj.node_path, + occurrence=_uri_obj.occurrence, + downsampling_method=downsampling_method, + downsampled_size=downsampled_size, + )["data"]["coordinates"] if len(interpolate_to_coordinates) != len(coordinates_to_be_returned): - # TODO: return proper exception - raise Exception("") + message = "Interpolation error. Source and target nodes have different number of coordinates." + raise InvalidParametersException(message) - for x,y in zip(coordinates_to_be_returned, interpolate_to_coordinates): + for x, y in zip(coordinates_to_be_returned, interpolate_to_coordinates): if x["name"] != y["name"]: - # TODO: return proper exception # coordinates between quantities doesn't match - raise Exception("") + message = f"Interpolation error. Coordinates names does not match between target and source nodes ({x['name']} vs. {y['name']})." + raise InvalidParametersException(message) - x["value"] = sorted(set(flatten(x["value"])+flatten(convert_to_lists(y["value"])))) - # join values + x["value"] = sorted(set(flatten(x["value"]) + flatten(convert_to_lists(y["value"])))) # reverse coordinates list so it matches data dimensions new_common_coords.reverse() common_coords_values = [c["value"] for c in new_common_coords] # =================== INTERPOLATE =================== - #print(f"=== ORIGINAL COORDS: {original_coord_values}") - print(f"=== NEW COORDS: {common_coords_values}") # === make data vector rectangular === data_to_be_returned = pad_to_rectangular(data_to_be_returned) - print(f"=== OLD DATA VALUES: {data_to_be_returned}") - new_data_values = resample_data2(tuple(original_coord_values),data_to_be_returned, tuple(common_coords_values)) - - print(f"=== NEW DATA VALUES: {new_data_values}") - - """ - if first_value.metadata.ndim == 1 and coordinates_to_be_returned[0]["name"] == "time": - #1 collect all time vectors - time_vectors = [] - - time_vectors.append(coordinates_to_be_returned[0]["value"]) - for interpolation_uri in interpolate_over: - uri = IMAS_URI(interpolation_uri) - time_vectors.append(self.get_data(uri=uri.uri_entry_identifiers, ids=uri.ids_name, node_path="time", occurrence=uri.occurrence)["value"]) - - #2 calculate common time vector - common_tv = union_arrays(time_vectors) - - #3 interpolate data_to_be_returned - if False: - data_to_be_returned = resample_data(data=data_to_be_returned,original_x=coordinates_to_be_returned[0]["value"], target_x=common_tv) - else: - print(f" START RESAMPLING") - data_to_be_returned = resample_data2(data=data_to_be_returned,original_x=coordinates_to_be_returned[0]["value"],target_x=common_tv) - - #4 replace `time` coordinate with new time vector - coordinates_to_be_returned[0]["value"] = common_tv - - if not coordinates_to_be_returned[0]["shape"] == "irregular": - coordinates_to_be_returned[0]["shape"] = np.asarray(coordinates_to_be_returned[0]["value"]).shape - """ - + data_to_be_returned = resample_data( + tuple(original_coord_values), data_to_be_returned, tuple(common_coords_values) + ) # ============= END resample data onto new time vector ============= diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index d4a10eaf..55fd2f17 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -2,11 +2,12 @@ import numpy as np from imas.ids_primitive import IDSNumericArray -from scipy.interpolate import LinearNDInterpolator, RegularGridInterpolator +from scipy.interpolate import RegularGridInterpolator -def union_arrays(data : list): - return reduce(np.union1d, data) +def union_arrays(data: list): + return reduce(np.union1d, data) + def flatten(lst): result = [] @@ -17,11 +18,17 @@ def flatten(lst): result.append(item) return result + def get_max_shape(lst, level=0, shape=None): + """ + Returns shape of irregular array. Result contains maximum array length in every dimension. + :param lst: input array + :return: + """ if shape is None: shape = [] - if isinstance(lst, (list,np.ndarray)): + if isinstance(lst, (list, np.ndarray)): if len(shape) <= level: shape.append(0) shape[level] = max(shape[level], len(lst)) @@ -33,6 +40,14 @@ def get_max_shape(lst, level=0, shape=None): def fill_array(arr, lst, index=()): + """ + Recursively fills an array with values from a nested list. + + :param arr: Array-like object supporting tuple indexing. + :param lst: Nested list with values to insert into the array. + :param index: Current index used during recursion. + :return: None (modifies arr in place). + """ if isinstance(lst, list): for i, item in enumerate(lst): fill_array(arr, item, index + (i,)) @@ -41,74 +56,41 @@ def fill_array(arr, lst, index=()): def pad_to_rectangular(lst): + """ + Converts a nested list into a rectangular NumPy array by padding + missing values with NaN. + + :param lst: Nested list with uneven lengths. + :return: NumPy array with NaN padding. + """ shape = tuple(get_max_shape(lst)) arr = np.full(shape, np.nan) fill_array(arr, lst) return arr -def join_coordinates(a, b): + +def resample_data(original_coords: list, data: list, target_coords: list): """ - Joins N-dimensional coordinate arrays together. Assumes data is np.ndarray or IDSNumericArray - :param a: left side of join - list or np.ndarray - :param b: right side of join - list or np.ndarray - :return: joined list + Resamples data onto new set of coordinates. + :param original_coords: List of original data coordinates. + :param data: Nested n-dimensional data array. + :param target_coords: List of target coordinates. + :return: Resampled data array. """ + interpolator = RegularGridInterpolator(original_coords, data, bounds_error=False) - # merge np arrays - if isinstance(a, (IDSNumericArray,np.ndarray)) and isinstance(b, (IDSNumericArray,np.ndarray)): - return np.unique(np.concatenate((a, b), axis=0)) - - # if a and b are lists → join them and - if isinstance(a, list) and isinstance(b, list): - max_len = max(len(a), len(b)) - result = [] - - for i in range(max_len): - if i < len(a) and i < len(b): - result.append(join_coordinates(a[i], b[i])) - elif i < len(a): - result.append(a[i]) - else: - result.append(b[i]) + # build mesh grid (manipulate coordinates to be list of coordinates e.g. [[x1,y1,z1,h1...], [x2,y2,z2,h3...]]) + mesh = np.meshgrid(*target_coords, indexing="ij") + points = np.stack(mesh, axis=-1).reshape(-1, len(target_coords)) - return result + result = interpolator(points) - if a is None: - return b - if b is None: - return a + # revert mesh shape + result = result.reshape([len(c) for c in target_coords]) - raise TypeError(f"Different data types when merging coordinates during data interpolation: {type(a)} vs {type(b)}") - -def resample_data(data, original_x, target_x): - - if isinstance(data, list): - return [resample_data(x, original_x, target_x) for x in data] - elif isinstance(data, IDSNumericArray): - return np.interp(target_x, original_x, data.value, left=np.nan, right=np.nan) - elif isinstance(data, np.ndarray): - return np.interp(target_x, original_x, data, left=np.nan, right=np.nan) - else: - return data - -def resample_data2(original_coords, data, target_coords): - - # time_slice(itime)/profiles_2d(i1)/psi - # 1- time_slice(itime)/profiles_2d(i1)/grid/dim1 - # 2- time_slice(itime)/profiles_2d(i1)/grid/dim2 - - # class LinearNDInterpolator(points, values, fill_value=np.nan, rescale=False) - # interp = LinearNDInterpolator(list(zip(x, y)), z) - - print(f"=== RESAMPLE ORIGINAL COORDS: {original_coords}") - print(f"=== RESAMPLE TARGET COORDS: {target_coords}") - print(f"=== RESAMPLE INPUT DATA: {data}") - interpolator = RegularGridInterpolator(original_coords, data) - result = interpolator(target_coords) - - print(f"==== RESULT {result}") return result + def convert_ids_data_into_numpy_array(data: list): if isinstance(data, list): @@ -116,4 +98,4 @@ def convert_ids_data_into_numpy_array(data: list): elif isinstance(data, IDSNumericArray): return data.value else: - return data \ No newline at end of file + return data diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index 67f74d31..2336c0ff 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -74,7 +74,12 @@ def field_value( description="Returns single (or tensorized) data node value with detailed parameters used to plot the data", ) @ibex_service.measure_execution_time -def plot_data(uri: str, interpolate_over : Optional[List[str]] = Query(None), downsampling_method: str | None = Query(None), downsampled_size: int = 1000) -> Any: +def plot_data( + uri: str, + interpolate_over: Optional[List[str]] = Query(None), + downsampling_method: str | None = Query(None), + downsampled_size: int = 1000, +) -> Any: """ IBEX endpoint. Prepares and returns full information about data node and it's coordinates. @@ -114,4 +119,6 @@ def plot_data(uri: str, interpolate_over : Optional[List[str]] = Query(None), d :rtype: dict (automatically converted to JSON by FastAPI) :return: JSON response """ - return CustomORJSONResponse(ibex_service.get_plot_data(uri.strip(), interpolate_over, downsampling_method, downsampled_size)) + return CustomORJSONResponse( + ibex_service.get_plot_data(uri.strip(), interpolate_over, downsampling_method, downsampled_size) + ) From 423048887c497b0b6b0d8d85436ca00fe490210b Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 9 Mar 2026 13:59:47 +0100 Subject: [PATCH 5/5] Delete TODO comment --- backend/ibex/data_source/imas_python_source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index ad0125db..2030516c 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -777,7 +777,6 @@ def get_plot_data( # ============= BEGIN resample data onto new time vector ============= def convert_to_lists(data): - """TODO: delete after development""" if isinstance(data, list): return [convert_to_lists(d) for d in data] elif isinstance(data, (np.ndarray, IDSNumericArray)):