Develop improve normalization

ledsa/analysis/ConfigDataAnalysis.py

@@ -7,7 +7,7 @@ class ConfigDataAnalysis(cp.ConfigParser):
 
     """
     def __init__(self, load_config_file=True, camera_position=None, num_layers=20, domain_bounds=None,
-                 led_array_indices=None, num_ref_images=10, camera_channels=0, num_cores=1,
+                 led_array_indices=None, num_ref_images=10, ref_img_indices=None, camera_channels=0, num_cores=1,
                  reference_property='sum_col_val',
                  average_images=False, solver='linear', weighting_preference=-6e-3, weighting_curvature=1e-6,
                  num_iterations=200, lambda_reg=1e-3):
@@ -24,6 +24,8 @@ def __init__(self, load_config_file=True, camera_position=None, num_layers=20, d
         :type led_array_indices: list[int] or None
         :param num_ref_images: Number of images used to compute normalize LED intensities. Defaults to 10.
         :type num_ref_images: int
+        :param ref_img_indices: Indices of reference images to use. If None, use num_ref_imgs.
+        :type ref_img_indices: list[int] or None
         :param camera_channels: Camera channels to be considered in the analysis. Defaults to 0.
         :type camera_channels: List[int]
         :param num_cores: Number of CPU cores for (multicore) processing. If greater than 1, multicore processing is applied. Defaults to 1.
@@ -54,6 +56,9 @@ def __init__(self, load_config_file=True, camera_position=None, num_layers=20, d
             self['DEFAULT']['   reference_property'] = str(reference_property)
             self.set('DEFAULT', '   # Number images used to compute normalize LED intensities')
             self['DEFAULT']['   num_ref_images'] = str(num_ref_images)
+            self.set('DEFAULT', '   # Indices of reference images to use')
+            self['DEFAULT']['   ref_img_indices'] = str(ref_img_indices)
+            self.set('DEFAULT', '   # Camera channels to be considered in the analysis')
             self['DEFAULT']['   camera_channels'] = str(camera_channels)
             self.set('DEFAULT', '   # Intensities are computed as average from two consecutive images if set to True ')
             self['DEFAULT']['   average_images'] = str(average_images)
@@ -106,23 +111,40 @@ def save(self) -> None:
             self.write(configfile)
         print('config_analysis.ini saved')
 
-    def get_list_of_values(self, section:str, option:str, dtype=int) -> None:
+    def get_list_of_values(self, section: str, option: str, dtype=int, fallback=None):
         """
-        Returns a list of values of a specified dtype from a given section and option.
-
-        :param section: Section in the configuration file.
-        :type section: str
-        :param option: Option under the specified section.
-        :type option: str
-        :param dtype: Data type of the values to be returned. Defaults to int.
-        :type dtype: type
-        :return: List of values or None if the option's value is 'None'.
-        :rtype: list or None
+        Return a list[dtype] for 'section'/'option'.
+        - Missing section/option → warn and return fallback
+        - Value 'None' or empty → return None
+        - Values are split on whitespace
         """
-        if self[section][option] == 'None':
+        if not self.has_option(section, option):
+            print(
+                "Config option missing: [%s].%s — using fallback=%r",
+                section, option, fallback
+            )
+            return fallback
+
+        raw = self.get(section, option, fallback=None)
+        if raw is None:
+            print(
+                "Config option unreadable: [%s].%s — using fallback=%r",
+                section, option, fallback
+            )
+            return fallback
+
+        raw = raw.strip()
+        if raw.lower() == 'none' or raw == '':
             return None
-        values = [dtype(i) for i in self[section][option].split()]
-        return values
+
+        try:
+            return [dtype(item) for item in raw.split()]
+        except Exception as e:
+            print(
+                "Config parse error for [%s].%s=%r (%s) — using fallback=%r",
+                section, option, raw, e, fallback
+            )
+            return fallback
 
     def in_camera_channels(self) -> None:
         """

ledsa/analysis/ExperimentData.py

@@ -34,6 +34,8 @@ class ExperimentData:
     :type num_iterations: int
     :ivar num_ref_images: Number of reference images.
     :type num_ref_images: int
+    :ivar ref_img_indices: Indices of reference images to use. If None, use num_ref_imgs.
+    :type ref_img_indices: list[int] or None
     :ivar reference_property: Reference property to be analysed.
     :type reference_property: str
     :ivar merge_led_arrays: Merge LED arrays option.
@@ -53,6 +55,7 @@ def __init__(self, load_config_file=True):
         self.weighting_curvature = None
         self.num_iterations = None
         self.num_ref_images = None
+        self.ref_img_indices = None
         self.lambda_reg = None
         self.reference_property = None
         self.merge_led_arrays = None
@@ -67,7 +70,9 @@ def load_config_parameters(self) -> None:
         config_analysis = self.config_analysis
         num_layers = int(config_analysis['model_parameters']['num_layers'])
         self.channels = config_analysis.get_list_of_values('DEFAULT', 'camera_channels')
-        self.num_ref_images = int(config_analysis['DEFAULT']['num_ref_images'])
+        self.ref_img_indices = config_analysis.get_list_of_values('DEFAULT', 'ref_img_indices')
+        if self.ref_img_indices is None:
+            self.num_ref_images = int(config_analysis['DEFAULT']['num_ref_images'])
         self.solver = config_analysis['DEFAULT']['solver']
         if self.solver == 'nonlinear':
             self.weighting_preference = float(config_analysis['DEFAULT']['weighting_preference'])
@@ -76,7 +81,6 @@ def load_config_parameters(self) -> None:
         elif self.solver == 'linear':
             self.lambda_reg = float(config_analysis['DEFAULT']['lambda_reg'])
         self.reference_property = config_analysis['DEFAULT']['reference_property']
-        self.solver = config_analysis['DEFAULT']['solver']
 
         self.led_arrays = config_analysis.get_list_of_values('model_parameters', 'led_array_indices')
         if self.led_arrays is None:

ledsa/analysis/ExtinctionCoefficients.py

@@ -6,6 +6,7 @@
 
 from ledsa.analysis.Experiment import Experiment, Layers, Camera
 from ledsa.core.file_handling import read_hdf, read_hdf_avg, extend_hdf, create_analysis_infos_avg
+from ledsa.data_extraction.data_integrity import check_intensity_normalization
 
 from importlib.metadata import version
 
@@ -19,8 +20,10 @@ class ExtinctionCoefficients(ABC):
     :vartype experiment: Experiment
     :ivar reference_property: Reference property to be analysed.
     :vartype reference_property: str
-    :ivar num_ref_imgs: Number of reference images.
+    :ivar num_ref_imgs: Number of reference images. # TODO: create test for this
     :vartype num_ref_imgs: int
+    :ivar ref_img_indices: Indices of reference images to use. If None, use num_ref_imgs.
+    :vartype ref_img_indices: list[int] or None
     :ivar calculated_img_data: DataFrame containing calculated image data.
     :vartype calculated_img_data: pd.DataFrame
     :ivar distances_per_led_and_layer: Array of distances traversed between camera and LEDs in each layer.
@@ -34,21 +37,26 @@ class ExtinctionCoefficients(ABC):
     :ivar solver: Indication whether the calculation is to be carried out numerically or analytically.
     :vartype type: str
     """
-    def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, average_images=False):
+
+    def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, ref_img_indices=None,
+                 average_images=False):
         """
         :param experiment: Object representing the experimental setup.
         :type experiment: Experiment
         :param reference_property: Reference property to be analysed.
         :type reference_property: str
         :param num_ref_imgs: Number of reference images.
         :type num_ref_imgs: int
+        :param ref_img_indices: Indices of reference images to use. If None, use num_ref_imgs.
+        :type ref_img_indices: list[int] or None
         :param average_images: Flag to determine if intensities are computed as an average from two consecutive images.
         :type average_images: bool
         """
         self.coefficients_per_image_and_layer = []
         self.experiment = experiment
         self.reference_property = reference_property
         self.num_ref_imgs = num_ref_imgs
+        self.ref_img_indices = ref_img_indices
         self.calculated_img_data = pd.DataFrame()
         self.distances_per_led_and_layer = np.array([])
         self.ref_intensities = np.array([])
@@ -164,10 +172,16 @@ def calc_and_set_ref_intensities(self) -> None:
          Calculate and set the reference intensities for all LEDs based on the reference images.
 
          """
-        ref_img_data = self.calculated_img_data.query(f'img_id <= {self.num_ref_imgs}')
-        ref_intensities = ref_img_data.groupby(level='led_id').mean()
+        if self.ref_img_indices is not None:
+            ref_img_data = self.calculated_img_data.query(f'img_id == {self.ref_img_indices}')
+        else:
+            ref_img_data = self.calculated_img_data.query(f'img_id <= {self.num_ref_imgs}')
+        print(
+            f"Images with indices {ref_img_data.index.get_level_values('img_id').unique().values} were used for calculating reference intensities.")
 
-        self.ref_intensities = ref_intensities[self.reference_property].to_numpy()
+        ref_intensities = ref_img_data.groupby(level='led_id')[self.reference_property].mean()
+        check_intensity_normalization(ref_img_data, ref_intensities, self.reference_property)
+        self.ref_intensities = ref_intensities.to_numpy()
 
     def apply_color_correction(self, cc_matrix, on='sum_col_val',
                                nchannels=3) -> None:  # TODO: remove hardcoding of nchannels

ledsa/analysis/ExtinctionCoefficientsLinear.py

@@ -19,7 +19,7 @@ class ExtinctionCoefficientsLinear(ExtinctionCoefficients):
     :vartype solver: str
     """
 
-    def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, average_images=False, lambda_reg=1e-3,):
+    def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, average_images=False, ref_img_indices=None, lambda_reg=1e-3,):
         """
         Initialize the ExtinctionCoefficientsLinear object.
 
@@ -29,12 +29,14 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10
         :type reference_property: str
         :param num_ref_imgs: Number of reference images.
         :type num_ref_imgs: int
+        :param ref_img_indices: Indices of reference images to use. If None, use num_ref_imgs.
+        :type ref_img_indices: list[int] or None
         :param average_images: Flag to determine if intensities are computed as an average from consecutive images.
         :type average_images: bool
         :param lambda_reg: Regularization parameter for Tikhonov regularization.
         :type lambda_reg: float
         """
-        super().__init__(experiment, reference_property, num_ref_imgs, average_images)
+        super().__init__(experiment, reference_property, num_ref_imgs, ref_img_indices, average_images)
         self.lambda_reg = lambda_reg
         self.solver = 'linear'
 

ledsa/analysis/ExtinctionCoefficientsNonLinear.py

@@ -27,7 +27,7 @@ class ExtinctionCoefficientsNonLinear(ExtinctionCoefficients):
     :ivar solver: Type of solver (linear or nonlinear).
     :vartype type: str
     """
-    def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, average_images=False, weighting_curvature=1e-6,
+    def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, ref_img_indices=None, average_images=False, weighting_curvature=1e-6,
                  weighting_preference=-6e-3, num_iterations=200):
         """
         :param experiment: Object representing the experimental setup.
@@ -36,6 +36,8 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10
         :type reference_property: str
         :param num_ref_imgs: Number of reference images.
         :type num_ref_imgs: int
+        :param ref_img_indices: Indices of reference images to use. If None, use num_ref_imgs.
+        :type ref_img_indices: list[int] or None
         :param average_images: Flag to determine if intensities are computed as an average from two consecutive images.
         :type average_images: bool
         :param weighting_curvature: Weighting factor for the smoothness of the solution.
@@ -46,7 +48,7 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10
         :type num_iterations: int
         """
 
-        super().__init__(experiment, reference_property, num_ref_imgs, average_images)
+        super().__init__(experiment, reference_property, num_ref_imgs, ref_img_indices, average_images)
         self.bounds = [(0, 10) for _ in range(self.experiment.layers.amount)]
         self.weighting_preference = weighting_preference
         self.weighting_curvature = weighting_curvature

ledsa/core/parser_arguments_run.py

@@ -226,12 +226,14 @@ def extionction_coefficient_calculation(args) -> None:
                 if solver == 'nonlinear':
                     eca = ECN.ExtinctionCoefficientsNonLinear(ex, reference_property=ex_data.reference_property,
                                                               num_ref_imgs=ex_data.num_ref_images,
+                                                              ref_img_indices=ex_data.ref_img_indices,
                                                               weighting_curvature=ex_data.weighting_curvature,
                                                               weighting_preference=ex_data.weighting_preference,
                                                               num_iterations=ex_data.num_iterations)
                 elif solver == 'linear':
                     eca = ECA.ExtinctionCoefficientsLinear(ex, reference_property=ex_data.reference_property,
                                                            num_ref_imgs=ex_data.num_ref_images,
+                                                           ref_img_indices=ex_data.ref_img_indices,
                                                            lambda_reg=ex_data.lambda_reg)
                 else:
                     raise ValueError(f"Invalid solver type '{solver}'. Must be 'linear' or 'nonlinear'.")

ledsa/data_extraction/data_integrity.py

@@ -0,0 +1,48 @@
+import warnings
+
+import pandas as pd
+
+
+def check_intensity_normalization(
+        ref_img_data: pd.DataFrame,
+        ref_intensities: pd.Series,
+        reference_property: str,
+        tolerance: float = 0.075 # TODO: this should be an user adjustable parameter
+) -> None:
+    """
+    Checks if the intensity normalization is within the specified tolerance.
+
+    :param ref_img_data: DataFrame containing reference image data.
+    :type ref_img_data: pd.DataFrame
+    :param ref_intensities: Series containing reference intensities.
+    :type ref_intensities: pd.Series
+    :param reference_property: The property to check for normalization.
+    :type reference_property: str
+    :param tolerance: The accepted tolerance for relative deviation.
+    :type tolerance: float
+    """
+    rel_devs = (ref_img_data[reference_property] - ref_intensities).abs() / ref_intensities.abs()
+    hits = rel_devs[rel_devs > tolerance]
+
+    if hits.empty:
+        return
+
+    lines = [
+        f"  img_id={img_id}, led_id={led_id}, rel. deviation = {val:.2%}"
+        for (img_id, led_id), val in hits.items()
+    ]
+
+    msg = (
+            f"In the process of normalisation {len(hits)} value(s) exceed(s) {tolerance:.0%} tolerance of relative deviation"
+            f" against mean intensities! You might check the reference images.\n"
+            + "\n".join(lines)
+    )
+
+    warnings.warn(msg, category=UserWarning, stacklevel=2)
+
+
+def check_led_positions():
+    pass
+
+def check_led_saturations():
+    pass