feat: Param Hanlder Benchmark

.github/workflows/CIValidations.yml

@@ -22,24 +22,24 @@ jobs:
       matrix:
         include:
           - name: Reweight Validations
-            test_1: MaCh3CLI --ReweightValidations
-            test_2: empty
-            test_3: empty
+            test: MaCh3CLI --ReweightValidations
+            compilation_flags:
 
           - name: Covariance Validations
-            test_1: MaCh3CLI --CovarianceValidations
-            test_2: empty
-            test_3: empty
+            test: MaCh3CLI --CovarianceValidations
+            compilation_flags:
 
           - name: Fitter Validations
-            test_1: MaCh3CLI --FitterValidations
-            test_2: empty
-            test_3: empty
+            test: MaCh3CLI --FitterValidations
+            compilation_flags:
 
           - name: NuMCMC Tools Validations
-            test_1: MaCh3CLI --NuMCMCToolsValidations
-            test_2: empty
-            test_3: empty
+            test: MaCh3CLI --NuMCMCToolsValidations
+            compilation_flags:
+
+          - name: Fitter Validations Low Memory
+            test: MaCh3CLI --FitterValidations
+            compilation_flags: -DMaCh3Tutorial_LOW_MEMORY_STRUCTS_ENABLED=ON
 
     container:
       image: ghcr.io/mach3-software/mach3:alma9v1.3.0
@@ -55,7 +55,7 @@ jobs:
         cd MaCh3Validations
         mkdir build
         cd build
-        cmake ../ -DMaCh3Tutorial_UNITTESTS_ENABLED=TRUE
+        cmake ../ -DMaCh3Tutorial_UNITTESTS_ENABLED=TRUE ${{ matrix.compilation_flags }}
 
     - name: Build MaCh3 Validations
       run: |
@@ -68,20 +68,4 @@ jobs:
         source /opt/MaCh3Validations/build/bin/setup.MaCh3Tutorial.sh
         cd /opt/MaCh3Validations/build/CIValidations
 
-        ${{ matrix.test_1 }}
-
-        #KS: GHA is stupid when it comes to arrays. I tried something fancy but it was crashing terribly
-        #If you know how to write this better please let me know
-
-        # Run the second test if it is defined
-        if [[ "${{ matrix.test_2 }}" != "empty" ]]; then
-            echo " "
-            echo "Performing Second test"
-            ${{ matrix.test_2 }}
-        fi
-        # Run the third test if it is defined
-        if [[ "${{ matrix.test_3 }}" != "empty" ]]; then
-            echo " "
-            echo "Performing Third test"
-            ${{ matrix.test_3 }}
-        fi
+        ${{ matrix.test }}

CIValidations/Benchmark/MaCh3Benchmark.cpp

@@ -5,6 +5,75 @@
 #include "catch2/catch_test_macros.hpp"
 #include "catch2/benchmark/catch_benchmark.hpp"
 
+void CreateChungusYaml(const std::string& filename = "ChungusSystematics.yaml",
+                       size_t nParams = 5000)
+{
+  std::ofstream out(filename);
+  if (!out.is_open()) {
+    throw std::runtime_error("Failed to open YAML file for writing");
+  }
+  out << "Systematics:\n\n";
+
+  for (size_t i = 0; i < nParams; ++i) {
+    double error   = 0.10;
+    double step_scale = 0.2;
+    out << "- Systematic:\n";
+    out << "    Names:\n";
+    out << "      FancyName: Norm_Param_" << i << "\n";
+    out << "    Error: " << error << "\n";
+    out << "    FlatPrior: false\n";
+    out << "    FixParam: false\n";
+    out << "    Mode: [ 1 ]\n";
+    out << "    ParameterBounds: [0, 999.]\n";
+    out << "    ParameterGroup: Xsec\n";
+    out << "    TargetNuclei: [12, 16]\n";
+    out << "    KinematicCuts:\n";
+    out << "    ParameterValues:\n";
+    out << "      Generated: 1.\n";
+    out << "      PreFitValue: 1.\n";
+    out << "    SampleNames: [\"Tutorial_*\", \"ND2137\"]\n";
+    out << "    StepScale:\n";
+    out << "      MCMC: "<< step_scale <<"\n";
+    out << "    Type: Norm\n\n";
+  }
+}
+
+std::string FormatMB(double value, int precision)
+{
+  std::ostringstream ss;
+  ss << std::fixed << std::setprecision(precision) << value << " MB";
+  return ss.str();
+}
+
+/// @brief This is super hacky but Catch2 doesn't support RAM printing only time...
+void PrintRAMBenchmark(const std::string& name)
+{
+  double ram = MaCh3Utils::getValue("VmRSS") / 1024.0;
+
+  std::cout << "\n";
+
+  // Main row
+  std::cout << std::left  << std::setw(40) << name
+            << std::right << std::setw(10) << 100
+            << std::right << std::setw(14) << 1
+            << std::right << std::setw(11) << std::fixed << std::setprecision(3)
+            << ram << " MB\n";
+
+  // Mean row
+  std::cout << std::left << std::setw(40) << " "
+            << std::right << std::setw(10) << FormatMB(ram, 3)
+            << std::right << std::setw(14) << FormatMB(ram, 3)
+            << std::right << std::setw(11) << std::fixed << std::setprecision(3)
+            << ram << " MB\n";
+
+  // Std dev row
+  std::cout << std::left << std::setw(40) << " "
+            << std::right << std::setw(10) << FormatMB(0.0, 5)
+            << std::right << std::setw(14) << FormatMB(0.0, 5)
+            << std::right << std::setw(11) << std::fixed << std::setprecision(5)
+            << 0.00000 << " MB\n";
+}
+
 TEST_CASE("Benchmark MaCh3") {
   // Initialise manger responsible for config handling
   auto FitManager = std::make_unique<Manager>("TutorialConfigs/FitterConfig.yaml");
@@ -63,6 +132,18 @@ TEST_CASE("Benchmark MaCh3") {
     }
   };
 
+  // Get RAM table BEFORE starting chungus
+  PrintRAMBenchmark("RAM tracker");
+
+  CreateChungusYaml("ChungusSystematics.yaml", 2000);
+  auto Chungus = std::make_unique<ParameterHandlerGeneric>(std::vector<std::string>{"ChungusSystematics.yaml"}, "xsec_cov");
+
+  BENCHMARK("ParameterHandler Operations") {
+    Chungus->ProposeStep();
+    Chungus->GetLikelihood();
+    Chungus->AcceptStep();
+  };
+
   for (size_t i = 0; i < NDSamples.size(); ++i) {
     delete NDSamples[i];
   }

CMakeLists.txt

@@ -43,6 +43,8 @@ include(CIValidations)
 option(MaCh3_CPM "Whether MaCh3 was obtained using CPM or find_package" OFF)
 option(MaCh3Tutorial_PROFILING_ENABLED "Will run with gperftools" OFF)
 option(Use_External_MaCh3 "Use an externally installed MaCh3 instead of CPM" OFF)
+option(MaCh3Tutorial_DEBUG_ENABLED "Use debug options" OFF)
+option(MaCh3Tutorial_LOW_MEMORY_STRUCTS_ENABLED "Use floats" OFF)
 
 # KS: Here we try to find tag matching tutorial version. If we can't find one then use develop
 # This will allow to grab tutorial for tagged MaCh3 version without a need of manually changing version
@@ -93,6 +95,14 @@ if(NOT MaCh3_FOUND)
     list(APPEND MaCh3_OPTIONS "MaCh3_NATIVE_ENABLED ON")
   endif()
 
+  if (MaCh3Tutorial_DEBUG_ENABLED)
+    list(APPEND MaCh3_OPTIONS "MaCh3_DEBUG_ENABLED ON")
+  endif()
+
+  if (MaCh3Tutorial_LOW_MEMORY_STRUCTS_ENABLED)
+    list(APPEND MaCh3_OPTIONS "MaCh3_LOW_MEMORY_STRUCTS_ENABLED ON")
+  endif()
+
   CPMAddPackage(
     NAME MaCh3
     GIT_TAG ${MaCh3_Branch}