Test2 python

.github/workflows/key4hep_build.yml

@@ -22,6 +22,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+    - name: Enable verbose CTest output
+      run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV
     - uses: key4hep/key4hep-actions/key4hep-build@main
       with:
         build_type: ${{ matrix.build_type }}
@@ -31,7 +33,8 @@ jobs:
       with:
         name: k4GeneratorsConfig_report_${{ matrix.build_type }}_${{ matrix.image }}
         path: |
-          test/output/*
+          test/Run-Cards/*.dat
+          test/Run-Cards/*.root
     - name: Show full CTest log on failure
       if: failure()
       run: |

CMakeLists.txt

@@ -57,14 +57,14 @@ install(DIRECTORY ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/ DESTINATION ${C
 
 install(DIRECTORY ${PROJECT_SOURCE_DIR}/python
   DESTINATION ${CMAKE_INSTALL_PREFIX}
-  REGEX test_.*\\.py|main.py$ EXCLUDE  # Don't install python unittests
+  REGEX test_.*\\.py|k4GeneratorsConfig.py$ EXCLUDE  # Don't install python unittests
   PATTERN __pycache__ EXCLUDE  # Or pythons caches
 )
 # Install main as an executable
-install(PROGRAMS ${PROJECT_SOURCE_DIR}/python/main.py DESTINATION ${CMAKE_INSTALL_PREFIX}/python)
+install(PROGRAMS ${PROJECT_SOURCE_DIR}/python/k4GeneratorsConfig.py DESTINATION ${CMAKE_INSTALL_PREFIX}/python)
 
 install(CODE "execute_process(COMMAND bash -c \"set -e
-    link_target=${CMAKE_INSTALL_PREFIX}/python/main.py
+    link_target=${CMAKE_INSTALL_PREFIX}/python/k4GeneratorsConfig.py
     link_name=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/k4GeneratorsConfig
     rm -f $link_name
     ln -s $link_target $link_name

k4GeneratorsConfig/src/eventGenerationCollections.cxx

@@ -5,7 +5,7 @@
 #include <iostream>
 #include <sys/stat.h>
 
-k4GeneratorsConfig::eventGenerationCollections::eventGenerationCollections() : m_validCounter(0), m_invalidCounter(0) {}
+k4GeneratorsConfig::eventGenerationCollections::eventGenerationCollections() {}
 k4GeneratorsConfig::eventGenerationCollections::eventGenerationCollections(
     const eventGenerationCollections& theOriginal) {
   if (this != &theOriginal) {
@@ -29,29 +29,29 @@ k4GeneratorsConfig::eventGenerationCollections::operator=(const eventGenerationC
   return *this;
 }
 k4GeneratorsConfig::eventGenerationCollections::~eventGenerationCollections() {}
-void k4GeneratorsConfig::eventGenerationCollections::Execute() {
+void k4GeneratorsConfig::eventGenerationCollections::Execute(std::string topDir) {
 
   // first make the collection
-  makeCollections();
+  makeCollections(topDir);
 
   // second order the collection according to the process
   orderCollections();
 }
-void k4GeneratorsConfig::eventGenerationCollections::makeCollections() {
+void k4GeneratorsConfig::eventGenerationCollections::makeCollections(std::string topDir) {
 
-  for (const auto& generators : std::filesystem::directory_iterator("Run-Cards")) {
-    std::filesystem::path generatorsPath = generators.path();
-    if (!std::filesystem::is_directory(generatorsPath))
+  for (const auto& generator : std::filesystem::directory_iterator(topDir)) {
+    std::filesystem::path generatorPath = generator.path();
+    if (!std::filesystem::is_directory(generatorPath))
       continue;
 
-    for (const auto& procs : std::filesystem::directory_iterator(generatorsPath.string())) {
-      std::filesystem::path processPath = procs.path();
+    for (const auto& process : std::filesystem::directory_iterator(generatorPath.string())) {
+      std::filesystem::path processPath = process.path();
       if (!std::filesystem::is_directory(processPath))
         continue;
 
       k4GeneratorsConfig::xsection* xsec = new k4GeneratorsConfig::xsection();
-      for (const auto& files : std::filesystem::directory_iterator(processPath.string())) {
-        std::filesystem::path filenamePath = files.path();
+      for (const auto& filename : std::filesystem::directory_iterator(processPath.string())) {
+        std::filesystem::path filenamePath = filename.path();
         if (!std::filesystem::is_regular_file(filenamePath))
           continue;
         // take care of the total cross section extracted from the EDM4HEP file
@@ -61,13 +61,13 @@ void k4GeneratorsConfig::eventGenerationCollections::makeCollections() {
           xsec->setProcess(processPath.filename().string());
           xsec->setFile(filenamePath.string());
           // in some cases the generator name is not available, therefore derive from the filename
-          xsec->setGenerator(generatorsPath.filename().string());
+          xsec->setGenerator(generatorPath.filename().string());
           std::cout << "Generator " << xsec->Generator() << " has been processed" << std::endl;
           m_xsectionCollection.push_back(*xsec);
           if (xsec->isValid())
-            m_validCounter++;
+            addSuccess(xsec->Generator());
           if (!xsec->isValid())
-            m_invalidCounter++;
+            addFailure(xsec->Generator());
         }
       }
       // we need to keep xsec alive for the analysisHistos distributions
@@ -84,7 +84,7 @@ void k4GeneratorsConfig::eventGenerationCollections::makeCollections() {
           diffDist->setFile(filenamePath.string());
           diffDist->setSQRTS(xsec->SQRTS());
           // in some cases the generator name is not available, therefore derive from the filename
-          diffDist->setGenerator(generatorsPath.filename().string());
+          diffDist->setGenerator(generatorPath.filename().string());
           std::cout << "Generator " << diffDist->Generator() << " has been processed for analysisHistos distributions"
                     << std::endl;
           m_analysisHistosCollection.push_back(*diffDist);
@@ -147,8 +147,36 @@ bool k4GeneratorsConfig::eventGenerationCollections::compareLexical(analysisHist
 
   return false;
 }
-unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfSuccesses() { return m_validCounter; }
-unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfFailures() { return m_invalidCounter; }
+void k4GeneratorsConfig::eventGenerationCollections::addSuccess(std::string generator) {
+  if (m_validCounter.find(generator) != m_validCounter.end()) {
+    m_validCounter[generator]++;
+  } else {
+    m_validCounter[generator] = 1;
+  }
+}
+void k4GeneratorsConfig::eventGenerationCollections::addFailure(std::string generator) {
+  if (m_invalidCounter.find(generator) != m_invalidCounter.end()) {
+    m_invalidCounter[generator]++;
+  } else {
+    m_invalidCounter[generator] = 1;
+  }
+}
+unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfSuccesses() const {
+  unsigned int validTotal = 0;
+  std::map<std::string, unsigned int>::const_iterator imap;
+  for (imap = m_validCounter.begin(); imap != m_validCounter.end(); imap++) {
+    validTotal += imap->second;
+  }
+  return validTotal;
+}
+unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfFailures() const {
+  unsigned int invalidTotal = 0;
+  std::map<std::string, unsigned int>::const_iterator imap;
+  for (imap = m_invalidCounter.begin(); imap != m_invalidCounter.end(); imap++) {
+    invalidTotal += imap->second;
+  }
+  return invalidTotal;
+}
 void k4GeneratorsConfig::eventGenerationCollections::Write2Root(std::string dirname, std::string filename) {
 
   eventGenerationCollections2Root out(dirname, filename);
@@ -225,7 +253,20 @@ void k4GeneratorsConfig::eventGenerationCollections::PrintSummary(std::ostream&
   }
   output << std::endl;
   // last thing the invalids
-  output << "Number of runs           : " << m_invalidCounter + m_validCounter << std::endl;
-  output << "Number of failed runs    : " << m_invalidCounter << std::endl;
-  output << "Number of successful runs: " << m_validCounter << std::endl;
+  output << "Number of runs           : " << NbOfFailures() + NbOfSuccesses() << std::endl;
+  output << "Number of failed runs    : " << NbOfFailures() << std::endl;
+  output << "Number of successful runs: " << NbOfSuccesses() << std::endl;
+  // details only for failures:
+  if (NbOfFailures() > 0) {
+    output << std::endl << "Detail of Failures:" << std::endl;
+    std::map<std::string, unsigned int>::const_iterator failure, success;
+    for (failure = m_invalidCounter.begin(); failure != m_invalidCounter.end(); failure++) {
+      output << failure->first << " : " << failure->second << " Failures ";
+      unsigned int successCount = 0;
+      if ((success = m_validCounter.find(failure->first)) != m_validCounter.end()) {
+        successCount = success->second;
+      }
+      output << " / " << successCount + failure->second << " Runs" << std::endl;
+    }
+  }
 }

k4GeneratorsConfig/src/eventGenerationCollections.h

@@ -1,6 +1,7 @@
 #ifndef K4GENERATORSCONFIG_EVENTGENERATIONCOLLECTIONS_H
 #define K4GENERATORSCONFIG_EVENTGENERATIONCOLLECTIONS_H
 
+#include <map>
 #include <vector>
 
 #include "analysisHistos.h"
@@ -14,15 +15,18 @@ class eventGenerationCollections {
   eventGenerationCollections& operator=(const eventGenerationCollections&);
   ~eventGenerationCollections();
 
-  void Execute();
-  void makeCollections();
+  void Execute(std::string);
+  void makeCollections(std::string);
   void orderCollections();
   bool compareLength(xsection, xsection);
   bool compareLexical(xsection, xsection);
   bool compareLexical(analysisHistos, analysisHistos);
 
-  unsigned int NbOfSuccesses();
-  unsigned int NbOfFailures();
+  void addSuccess(std::string);
+  void addFailure(std::string);
+
+  unsigned int NbOfSuccesses() const;
+  unsigned int NbOfFailures() const;
 
   void Write2Root(std::string, std::string);
 
@@ -35,8 +39,8 @@ class eventGenerationCollections {
 private:
   std::vector<k4GeneratorsConfig::xsection> m_xsectionCollection;
   std::vector<k4GeneratorsConfig::analysisHistos> m_analysisHistosCollection;
-  unsigned int m_validCounter;
-  unsigned int m_invalidCounter;
+  std::map<std::string, unsigned int> m_validCounter;
+  std::map<std::string, unsigned int> m_invalidCounter;
 };
 } // namespace k4GeneratorsConfig
 

k4GeneratorsConfig/src/eventGenerationSummary.cxx

@@ -7,12 +7,16 @@
 //
 int main(int argc, char** argv) {
 
+  std::string topDir = "Run-Cards";
   std::string filename = "GenerationSummary.dat";
   std::string dirRoot = ".";
   std::string fileRoot = "eventGenerationSummary.root";
   int c;
-  while ((c = getopt(argc, argv, "hf:d:r:")) != -1)
+  while ((c = getopt(argc, argv, "hw:f:d:r:")) != -1)
     switch (c) {
+    case 'w':
+      topDir = optarg;
+      break;
     case 'f':
       filename = optarg;
       break;
@@ -23,11 +27,14 @@ int main(int argc, char** argv) {
       fileRoot = optarg;
       break;
     case 'h':
-      std::cout << "Usage: xsectionSummary -h -f filename" << std::endl;
+      std::cout << "Usage: xsectionSummary -h -w workdir -f filename -d rootdir -r rootfile" << std::endl;
       std::cout << "-h: print this help" << std::endl;
-      std::cout << "-f filename: print the summary information to this file" << std::endl;
-      std::cout << "-d dirname: write the RootTree and figures to this directory" << std::endl;
-      std::cout << "-r filename: write the RootTree to this file" << std::endl;
+      std::cout << "-w dirname: top directory to start the search for generators and processes (default: Run-Cards)"
+                << std::endl;
+      std::cout << "-f filename: print the summary information to this file (default: GenerationSummary.dat)"
+                << std::endl;
+      std::cout << "-d rootdir: write the RootTree and figures to this directory (default: .)" << std::endl;
+      std::cout << "-r footfile: write the RootTree to this file (default: eventGenerationSummary.root)" << std::endl;
       exit(0);
     default:
       exit(0);
@@ -36,7 +43,7 @@ int main(int argc, char** argv) {
   // instantiate the collection as pointer
   k4GeneratorsConfig::eventGenerationCollections* evgenColls = new k4GeneratorsConfig::eventGenerationCollections();
   // execute the gathering of information including detailed output
-  evgenColls->Execute();
+  evgenColls->Execute(topDir);
   // do the root analysis
   evgenColls->Write2Root(dirRoot, fileRoot);
   // print the summary on screen

python/Generators/Babayaga.py

@@ -3,8 +3,8 @@
 class Babayaga(GeneratorBase):
     """Babayaga class"""
 
-    def __init__(self, procinfo, settings):
-        super().__init__(procinfo, settings, "Babayaga", "dat")
+    def __init__(self, procinfo):
+        super().__init__(procinfo, "Babayaga", "dat")
 
         self.version = "x.y.z"
 
@@ -49,7 +49,7 @@ def write_process(self):
             self.addOption2GeneratorDatacard("ord", "alpha")
             self.addOption2GeneratorDatacard("EWKc", "on")
 
-        self.addOption2GeneratorDatacard("nev", self.procinfo.get("events"))
+        self.addOption2GeneratorDatacard("nev", self.procinfo.settings.get_nevents())
         self.addOption2GeneratorDatacard("ecms", self.procinfo.get("sqrts"))
 
         # output format only hepm2 or hepmc3, the actual version is detected by the linked library, so strip the number
@@ -60,12 +60,12 @@ def write_process(self):
         for key in self.procDB.getDict():
             self.addOption2GeneratorDatacard(key,self.procDB.getDict()[key])
 
-        if self.procinfo.eventmode == "unweighted":
+        if self.procinfo.get("eventmode") == "unweighted":
             self.addOption2GeneratorDatacard("mode", "unweighted")
         else:
             self.addOption2GeneratorDatacard("mode", "weighted")
 
-        if self.settings.get_block("selectors"):
+        if self.procinfo.settings.get_block("selectors"):
             self.writeAllSelectors()
 
     def add_decay(self):

python/Generators/GeneratorBase.py

@@ -11,11 +11,10 @@
 class GeneratorBase(ABC):
     """GeneratorBase class"""
 
-    def __init__(self, procinfo, settings, name, inputFileExtension):
+    def __init__(self, procinfo, name, inputFileExtension):
 
         # general settings of the class
         self.procinfo = procinfo
-        self.settings = settings
         self.name = name
         self.procDBName = f"{name}ProcDB"
         self.inputFileExtension    = inputFileExtension
@@ -32,9 +31,9 @@ def __init__(self, procinfo, settings, name, inputFileExtension):
         # check consistency: note that the ParameterSets have been defined, check with the particle masses
         self.checkModelParameters()
 
-        # define the output directory as function of the OutDir spec + generator name + process name
+        # define OutDir as generator name + process name, we assume that we are in the working directory
         self.outdir = (
-            f"{procinfo.get('OutDir')}/{self.name}/{self.procinfo.get('procname')}"
+            f"{self.name}/{self.procinfo.get('procname')}"
         )
 
         # configure the filenames
@@ -80,7 +79,7 @@ def __init__(self, procinfo, settings, name, inputFileExtension):
         self.prepareAnalysisContent()
 
         # the generator settings are stored in a public member:
-        self.gen_settings = settings.get_block(self.name.lower())
+        self.gen_settings = self.procinfo.settings.get_block(self.name.lower())
         if self.gen_settings is not None:
             self.gen_settings = {k.lower(): v for k, v in self.gen_settings.items()}
 
@@ -110,7 +109,7 @@ def __init__(self, procinfo, settings, name, inputFileExtension):
 
         self.procDBparameters  = dict()
         self.procDBparticles  = dict()
-        if self.settings.get("usedefaults", True):
+        if self.procinfo.settings.get("usedefaults", True):
             self.procDB.execute()
             self.procDBparameters  = self.procDB.getDictParameters()
             self.procDBparticles   = self.procDB.getDictParticles()
@@ -136,9 +135,9 @@ def validateSelectorsDict(self):
                 raise ValueError(f"{self.name} {key} not found in SelectorKeys list")
 
     def writeAllSelectors(self):
-        selectors = getattr(self.settings, "selectors")
+        selectors = getattr(self.procinfo.settings, "selectors")
         try:
-            procselectors = getattr(self.settings, "procselectors")
+            procselectors = getattr(self.procinfo.settings, "procselectors")
             for proc, sel in procselectors.items():
                 if proc != self.procinfo.get("procname"):
                     continue
@@ -231,8 +230,8 @@ def checkModelParameters(self):
     def isCompatible(self, target, prediction):
         # maximum relative deviation
         RelDiffThreshold = 0.001
-        if self.settings.get("EWParamDevThreshold".lower()) is not None:
-            RelDiffThreshold = self.settings.get("EWParamDevThreshold".lower())
+        if self.procinfo.settings.get("EWParamDevThreshold".lower()) is not None:
+            RelDiffThreshold = self.procinfo.settings.get("EWParamDevThreshold".lower())
         # do the safe math
         relDelta = 1.
         # if the target is zero, then take the absolute deviation, if not the relative deviation
@@ -521,10 +520,10 @@ def prepareKey4hepScript(self):
 
     def prepareAnalysisContent(self):
         # analysis is conditioned on the output format
-        outformat = self.settings.get_output_format()
+        outformat = self.procinfo.settings.get_output_format()
         # write the EDM4HEP analysis part based on the final state
         analysis = "\n"
-        if outformat == "edm4hep" and self.settings.key4HEPAnalysisON():
+        if outformat == "edm4hep" and self.procinfo.settings.key4HEPAnalysisON():
             analysis += f"key4HEPAnalysis -i {self.GeneratorDatacardBase}.edm4hep -o {self.GeneratorDatacardBase}.root -p "
 
             for pdg in self.procinfo.get_finalstate_pdgList():
@@ -533,10 +532,10 @@ def prepareAnalysisContent(self):
             analysis +="\n"
 
         # write the RIVET analysis
-        if (outformat == "edm4hep" or outformat == "hepmc3") and self.settings.rivetON():
-            yodaout = self.settings.yodaoutput + f"/{self.procinfo.get('procname')}.yoda"
+        if (outformat == "edm4hep" or outformat == "hepmc3") and self.procinfo.settings.rivetON():
+            yodaout = self.procinfo.settings.yodaoutput + f"/{self.procinfo.get('procname')}.yoda"
             analysis += f"rivet"
-            for ana in self.settings.analysisname:
+            for ana in self.procinfo.settings.analysisname:
                 analysis += f" -a {ana}"
             analysis+=f" -o {yodaout} {self.procinfo.get('procname')}.{self.procinfo.get_output_format()}\n"