From cafd35421dc71c4c92cd5b453c68a23035bf169e Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Thu, 26 Mar 2026 16:26:55 +0100 Subject: [PATCH 01/88] remove generatorDBLabel and replace with DBTag in process --- python/Generators/MadgraphProcDB.py | 11 +++++---- python/Generators/PythiaProcDB.py | 36 ++++++++++++++--------------- python/Generators/SherpaProcDB.py | 12 +++++----- python/Generators/WhizardProcDB.py | 6 ++--- python/Process.py | 25 +++++++------------- 5 files changed, 42 insertions(+), 48 deletions(-) diff --git a/python/Generators/MadgraphProcDB.py b/python/Generators/MadgraphProcDB.py index 92aa9ef0..06b71089 100644 --- a/python/Generators/MadgraphProcDB.py +++ b/python/Generators/MadgraphProcDB.py @@ -8,11 +8,14 @@ def __init__(self, process): self.process = process def execute(self): - # choose as function of generatorDBLabel - if self.process.get_generatorDBLabel().startswith("11_11") and len(self.process.final) == 2 : - if abs(self.process.final[0]) <= 16: + # choose as function of DBTag + tag = self.process.get_DBTag() + initialState = tag[0] + finalState = tag[1] + if initialState == [-11,11] and len(finalState) == 2: + isFermionPair = all( abs(pdg)<=16 for pdg in finalState) + if isFermionPair: self.write_Difermion() - label = self.process.get_generatorDBLabel() def write_Difermion(self): self.rundict['set pt_min_pdg'] = f"{{{self.process.final[0]}: 0 }}" diff --git a/python/Generators/PythiaProcDB.py b/python/Generators/PythiaProcDB.py index 4d5ae83d..20fc8950 100644 --- a/python/Generators/PythiaProcDB.py +++ b/python/Generators/PythiaProcDB.py @@ -23,39 +23,39 @@ def execute(self): self.rundict['321:mayDecay'] = "true" self.rundict['130:mayDecay'] = "true" - # choose as function of generatorDBLabel - label = self.process.get_generatorDBLabel() - if label == "11_11_1_1": + # choose as function of DBTag + tag = self.process.get_DBTag() + if tag == [[-11,11],[-1,1]]: self.write_Difermion(1) - elif label == "11_11_2_2": + elif tag == [[-11,11],[-2,2]]: self.write_Difermion(2) - elif label == "11_11_3_3": + elif tag == [[-11,11],[-3,3]]: self.write_Difermion(3) - elif label == "11_11_4_4": + elif tag == [[-11,11],[-4,4]]: self.write_Difermion(4) - elif label == "11_11_5_5": + elif tag == [[-11,11],[-5,5]]: self.write_Difermion(5) - elif label == "11_11_6_6": + elif tag == [[-11,11],[-6,6]]: self.write_Ditop() - elif label == "11_11_12_12": + elif tag == [[-11,11],[-12,12]]: self.write_Difermion(12) - elif label == "11_11_13_13": + elif tag == [[-11,11],[-13,13]]: self.write_Difermion(13) - elif label == "11_11_14_14": + elif tag == [[-11,11],[-14,14]]: self.write_Difermion(14) - elif label == "11_11_15_15": + elif tag == [[-11,11],[-15,15]]: self.write_Difermion(15) - elif label == "11_11_16_16": + elif tag == [[-11,11],[-16,16]]: self.write_Difermion(16) - elif label == "11_11_22_22": + elif tag == [[-11,11],[22,22]]: self.write_Diphoton() - elif label == "11_11_23_23": + elif tag == [[-11,11],[23,23]]: self.write_ZZ() - elif label == "11_11_24_24": + elif tag == [[-11,11],[24,24]]: self.write_WW() - elif label == "11_11_23_25": + elif tag == [[-11,11],[23,25]]: self.write_run_ZH() - elif label == "11_11_12_12_25": + elif tag == [[-11,11],[-12,12,25]]: self.write_run_Hnunu() def write_Difermion(self, pdg): diff --git a/python/Generators/SherpaProcDB.py b/python/Generators/SherpaProcDB.py index e63455ad..eeb1b21b 100644 --- a/python/Generators/SherpaProcDB.py +++ b/python/Generators/SherpaProcDB.py @@ -12,15 +12,15 @@ def execute(self): self.rundict['EW_SCHEME'] = 3 # the procdict: self.procdict['Order'] = "{QCD: 0, EW: 2}" - # choose as function of generatorDBLabel - label = self.process.get_generatorDBLabel() - if label == "11_11_5_5": + # choose as function of DBTag + tag = self.process.get_DBTag() + if tag == [[-11,11],[-5,5]]: self.particlesdict['5'] = {'Massive' : 1} - if label == "11_11_6_6": + if tag == [[-11,11],[-6,6]]: self.particlesdict['6'] = {'Massive' : 1} - if label == "11_11_15_15": + if tag == [[-11,11],[-15,15]]: self.particlesdict['15'] = {'Massive' : 1} - if label == "11_11_23_25": + if tag == [[-11,11],[23,25]]: self.particlesdict['23'] = {'Width' : 0} self.particlesdict['25'] = {'Width' : 0} # b quark has to be made massive diff --git a/python/Generators/WhizardProcDB.py b/python/Generators/WhizardProcDB.py index 7b9bf33e..7abe6e28 100644 --- a/python/Generators/WhizardProcDB.py +++ b/python/Generators/WhizardProcDB.py @@ -8,9 +8,9 @@ def __init__(self, process): self.process = process def execute(self): - # choose as function of generatorDBLabel - label = self.process.get_generatorDBLabel() - if label == "11_11_23_25": + # choose as function of DBTag + tag = self.process.get_DBTag() + if tag == [ [-11,11], [23,25]]: self.write_ZH() def write_ZH(self): diff --git a/python/Process.py b/python/Process.py index b348ad9f..cbd2b691 100644 --- a/python/Process.py +++ b/python/Process.py @@ -28,6 +28,7 @@ def __init__(self, args, procname, params, particleData, **options): self._particlesOfProcessList = [] # label to be used in the generatorDB self.generatorDBLabel = "" + self.generatorDBTag = [] self.procname = procname for arg in self._required_args: @@ -59,25 +60,12 @@ def prepareProcess(self): self._finalStatePDGList.append(str(p)) self._proclabel += f"{self._finalStateParticleDict[p].name} " self._particlesOfProcessList.append(self._finalStateParticleDict[p]) - # generate the label for the generatorDB - # first the initial state - initialstate = [abs(self.initial[0]), abs(self.initial[1])] - #sort ascending + # now the new DBTag: + initialstate = [self.initial[0], self.initial[1]] initialstate.sort() - # add to label - for pdg in initialstate: - self.generatorDBLabel += f"_{str(abs(pdg))}" - # remove leading "_" - self.generatorDBLabel = self.generatorDBLabel[ - (self.generatorDBLabel.index("_") + 1) : - ] - # now the final state - finalstate = [abs(part) for part in self.final] - # sort ascending + finalstate = self.final finalstate.sort() - # add to label - for pdg in finalstate: - self.generatorDBLabel += f"_{str(abs(pdg))}" + self._DBTag = [initialstate, finalstate] def get_beam_flavour(self, beam): if beam not in {1, 2}: @@ -143,6 +131,9 @@ def get_BeamstrahlungFile(self): def get_generatorDBLabel(self): return self.generatorDBLabel + def get_DBTag(self): + return self._DBTag + def print_info(self): print(f"Creating Runcards for {self._proclabel} at {self.sqrts} GeV") print("Particles are defined with the following parameters") From d4c0ad0fe1ccf15dba3dc3e023896b79c70e580a Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Thu, 26 Mar 2026 16:49:27 +0100 Subject: [PATCH 02/88] correct and simplify MADGRAPH ProcDB --- python/Generators/MadgraphProcDB.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/Generators/MadgraphProcDB.py b/python/Generators/MadgraphProcDB.py index 06b71089..a577dca2 100644 --- a/python/Generators/MadgraphProcDB.py +++ b/python/Generators/MadgraphProcDB.py @@ -14,9 +14,10 @@ def execute(self): finalState = tag[1] if initialState == [-11,11] and len(finalState) == 2: isFermionPair = all( abs(pdg)<=16 for pdg in finalState) + pdg = abs(finalState[0]) if isFermionPair: - self.write_Difermion() + self.write_Difermion(pdg) - def write_Difermion(self): - self.rundict['set pt_min_pdg'] = f"{{{self.process.final[0]}: 0 }}" + def write_Difermion(self,pdg): + self.rundict['set pt_min_pdg'] = f"{{{pdg}: 0 }}" From 85042413f91d42fed20af99a9cf94c61aa3b6ff7 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 08:46:36 +0100 Subject: [PATCH 03/88] missing protection --- k4GeneratorsConfig/src/WriterEDM4HEP.cxx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/k4GeneratorsConfig/src/WriterEDM4HEP.cxx b/k4GeneratorsConfig/src/WriterEDM4HEP.cxx index f51769cc..4e37c3f6 100644 --- a/k4GeneratorsConfig/src/WriterEDM4HEP.cxx +++ b/k4GeneratorsConfig/src/WriterEDM4HEP.cxx @@ -138,7 +138,6 @@ void WriterEDM4HEP::write_event(const GenEvent& evt) { if (evt.weights().size() > 0) { evtHeader.setWeight(evt.weights()[0]); } - // push to collection evtHeaderCollection.push_back(evtHeader); @@ -186,7 +185,9 @@ void WriterEDM4HEP::write_event(const GenEvent& evt) { // retrieve the pdf information HepMC3::ConstGenPdfInfoPtr pdfinfo = evt.pdf_info(); - generatorParameters.setPartonIds({pdfinfo->parton_id[0], pdfinfo->parton_id[1]}); + if ( pdfinfo ){ + generatorParameters.setPartonIds({pdfinfo->parton_id[0], pdfinfo->parton_id[1]}); + } // add the object to the collection: generatorParametersCollection.push_back(generatorParameters); From 8f029c10b3c8c8109161c738425a611d548fdea7 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 08:50:42 +0100 Subject: [PATCH 04/88] clang mod --- k4GeneratorsConfig/src/WriterEDM4HEP.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4GeneratorsConfig/src/WriterEDM4HEP.cxx b/k4GeneratorsConfig/src/WriterEDM4HEP.cxx index 4e37c3f6..33a61b97 100644 --- a/k4GeneratorsConfig/src/WriterEDM4HEP.cxx +++ b/k4GeneratorsConfig/src/WriterEDM4HEP.cxx @@ -185,7 +185,7 @@ void WriterEDM4HEP::write_event(const GenEvent& evt) { // retrieve the pdf information HepMC3::ConstGenPdfInfoPtr pdfinfo = evt.pdf_info(); - if ( pdfinfo ){ + if (pdfinfo){ generatorParameters.setPartonIds({pdfinfo->parton_id[0], pdfinfo->parton_id[1]}); } From 2c45c1c7c44d3d9c66391e8c98646f3e30378425 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 08:53:48 +0100 Subject: [PATCH 05/88] clang mod --- k4GeneratorsConfig/src/WriterEDM4HEP.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4GeneratorsConfig/src/WriterEDM4HEP.cxx b/k4GeneratorsConfig/src/WriterEDM4HEP.cxx index 33a61b97..cc4a9d04 100644 --- a/k4GeneratorsConfig/src/WriterEDM4HEP.cxx +++ b/k4GeneratorsConfig/src/WriterEDM4HEP.cxx @@ -185,7 +185,7 @@ void WriterEDM4HEP::write_event(const GenEvent& evt) { // retrieve the pdf information HepMC3::ConstGenPdfInfoPtr pdfinfo = evt.pdf_info(); - if (pdfinfo){ + if (pdfinfo) { generatorParameters.setPartonIds({pdfinfo->parton_id[0], pdfinfo->parton_id[1]}); } From b6c70235ee6f548bad0149894d06cd5c680bbf94 Mon Sep 17 00:00:00 2001 From: Alan Price Date: Fri, 27 Mar 2026 11:39:09 +0100 Subject: [PATCH 06/88] Enable verbose output on test fail --- .github/workflows/key4hep_build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/key4hep_build.yml b/.github/workflows/key4hep_build.yml index 4b19412c..d248b962 100644 --- a/.github/workflows/key4hep_build.yml +++ b/.github/workflows/key4hep_build.yml @@ -22,10 +22,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Enable verbose CTest output + run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV - uses: key4hep/key4hep-actions/key4hep-build@main with: build_type: ${{ matrix.build_type }} image: ${{ matrix.image }} + ctest_args: "-VV --output-on-failure" - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: From e560f8d46aee02b17cc0174cbe0fa9d3f524765d Mon Sep 17 00:00:00 2001 From: Alan Price Date: Fri, 27 Mar 2026 11:52:35 +0100 Subject: [PATCH 07/88] Enable verbose output on test fail --- .github/workflows/key4hep_build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/key4hep_build.yml b/.github/workflows/key4hep_build.yml index d248b962..a63c6fbf 100644 --- a/.github/workflows/key4hep_build.yml +++ b/.github/workflows/key4hep_build.yml @@ -28,10 +28,15 @@ jobs: with: build_type: ${{ matrix.build_type }} image: ${{ matrix.image }} - ctest_args: "-VV --output-on-failure" - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: name: k4GeneratorsConfig_report_${{ matrix.build_type }}_${{ matrix.image }} path: | test/output/* + - name: Show full CTest log on failure + if: failure() + run: | + echo "===== LastTest.log =====" + cat build/Testing/Temporary/LastTest.log || true + From f8a873991958fa4143d9ce58f2c88099e0524d6c Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 16:17:00 +0100 Subject: [PATCH 08/88] cosmetics on setup.py --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 11e203f1..d3f5d553 100644 --- a/setup.py +++ b/setup.py @@ -8,14 +8,14 @@ setup( - name="k4Generators", # Required + name="k4GeneratorsConfig", # Required version="alpha", # Required - description="A python module for creating Monte-Carlo generator cards", # Optional + description="A package for Monte-Carlo generator comparisons", # Optional # long_description=long_description, # Optional # long_description_content_type="text/markdown", # Optional (see note above) # url="https://github.com/pypa/sampleproject", # Optional - author="Alan Price", # Optional - author_email="alan.price[at]cern.ch", # Optional + author="Alan Price, Dirk Zerwas", # Optional + author_email="alan.price[at]uj.edu.pl, dirk.zerwas[at]in2p3.fr", # Optional classifiers=[ # Optional # How mature is this project? Common values are # 3 - Alpha @@ -50,4 +50,4 @@ # "Say Thanks!": "http://saythanks.io/to/example", # "Source": "https://github.com/pypa/sampleproject/", }, -) \ No newline at end of file +) From ee3104569536bb811753a610a0257735f24a5715 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 21:29:27 +0100 Subject: [PATCH 09/88] python variant for DC generation --- python/createGeneratorDatacards.py | 71 +++++++++++++++++ python/main.py | 4 +- python/run.py | 99 ++++++++++++++++++++++++ setup.csh | 2 +- setup.sh | 1 + setup.zsh | 1 + test/CMakeLists.txt | 9 +++ test/createGeneratorDatacardsFromYaml.sh | 1 + 8 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 python/createGeneratorDatacards.py create mode 100644 python/run.py diff --git a/python/createGeneratorDatacards.py b/python/createGeneratorDatacards.py new file mode 100644 index 00000000..b3b60f14 --- /dev/null +++ b/python/createGeneratorDatacards.py @@ -0,0 +1,71 @@ +import os +import sys +import shutil +from pathlib import Path + +from main import main + +class createGeneratorDatacards: + """Generator Generator Datacards""" + + def __init__(self, yamlDirectory, yamlFile, workDirectory, outputDirectory): + # useful for the class + self._yamlDir = yamlDirectory + self._yamlFiles = [] + self._workDir = workDirectory + self._outDir = outputDirectory + + # make the directory for the work + self.makeDirectory(workDirectory) + # make the output directory for the output + self.makeDirectory(outputDirectory) + + # prepare yamls: + self.prepareYamls(yamlFile); + + # prepare the Sqrts files: + self.prepareECMS(); + + # process all yamls: + self.process(); + + def makeDirectory(self, dirname): + # Overwrite directory if it exists + if not os.path.exists(dirname): + os.makedirs(dirname) + else: + shutil.rmtree(dirname) + os.makedirs(dirname) + + def prepareYamls(self, yamlFile): + # move the yamls to the work directory + # the yaml file is not specified copy all yaml files to the work directory: + if yamlFile == "None": + self._yamlFiles = [filename for filename in os.listdir(self._yamlDir) if filename.endswith('.yaml')] + else: + self._yamlFiles =[yamlFile] + + # copy the yaml files + for filename in self._yamlFiles: + shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) + + def prepareECMS(self): + # move the ECMS files to the work directory + ecmsFiles = [filename for filename in os.listdir(self._yamlDir) if filename.endswith('.dat') and filename.startswith('ecms')] + + # copy the yaml files + for filename in ecmsFiles: + shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) + + def process(self): + # go to the working directory + os.chdir(self._workDir) + # loop over all files + for filename in self._yamlFiles: + processName = Path(filename).stem + ecmsName = "ecms"+processName+".dat" + print("Processing : "+processName) + if os.path.isfile(ecmsName): + main([filename,'--ecmsFile',ecmsName]) + else: + main([filename]) diff --git a/python/main.py b/python/main.py index 87971530..ec1688d2 100755 --- a/python/main.py +++ b/python/main.py @@ -22,7 +22,7 @@ def make_output_directory(generators, output_directory, procname): os.makedirs(generator_directory) -def main(): +def main(args=None): # parser = argparse.ArgumentParser(prog='k4gen',description='Process input YAML files.') parser = argparse.ArgumentParser( prog="k4GeneratorsConfig", @@ -106,7 +106,7 @@ def main(): default=None, help="force the use of the version in default is latest, format: YYYY-MM-DD", ) - args = parser.parse_args() + args = parser.parse_args(args) files = args.inputfiles energies = args.ecms ecmsfiles = args.ecmsFiles diff --git a/python/run.py b/python/run.py new file mode 100644 index 00000000..fa4aa781 --- /dev/null +++ b/python/run.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +import os +import shutil +import sys +import argparse +import textwrap + +from createGeneratorDatacards import createGeneratorDatacards + +def run(): + parser = argparse.ArgumentParser( + prog="k4GeneratorsConfig", + formatter_class=argparse.RawDescriptionHelpFormatter, + description=textwrap.dedent( + """\ +The following options are available: +------------------------------------ + """ + ), + ) + parser.add_argument( + "--create", + action='store_true', + help="create the generator datacards from the yaml files", + ) + parser.add_argument( + "--yamlDir", + type=str, + default="../examples", + help="directory where the yamlFiles are located default: k4GeneratorsConfig/examples", + ) + parser.add_argument( + "--yamlFile", + type=str, + default="None", + help="name of file to be processed default: all found in the yamlDir", + ) + parser.add_argument( + "--check", + action='store_true', + help="check the generator datacards with respect to the reference in k4GeneratorsConfig/test/ref-results", + ) + parser.add_argument( + "--run", + action='store_true', + help="run the event generation", + ) + parser.add_argument( + "--summary", + action='store_true', + help="compare the results of the event generation process by process", + ) + parser.add_argument( + "--workDir", + type=str, + default="../test/work", + help="directory where the Work is performed default: k4GeneratorsConfig/test/work", + ) + parser.add_argument( + "--outputDir", + type=str, + default="../test/output", + help="directory where the output is stored default: k4GeneratorsConfig/test/output", + ) + + args = parser.parse_args() + + yamlDirectory = os.path.dirname(os.path.realpath(__file__))+"/"+args.yamlDir + if args.yamlDir.startswith("/"): + yamlDirectory = args.yamlDir + + yamlFile = "None" + if args.yamlFile != "None": + yamlFile = yamlDirectory+"/"+args.yamlFile + + workDirectory = os.path.dirname(os.path.realpath(__file__))+"/"+args.workDir + if args.workDir.startswith("/"): + workDirectory = args.workDir + + outputDirectory = os.path.dirname(os.path.realpath(__file__))+"/"+args.outputDir + if args.outputDir.startswith("/"): + outputDirectory = args.outputDir + + if args.create: + createGeneratorDatacards(yamlDirectory,yamlFile,workDirectory,outputDirectory) + + #if args.check: + #checkGeneratorDatacards() + + #if args.run: + #runEventGeneration() + + #if args.summary: + #runSummary() + +if __name__ == "__main__": + run() + diff --git a/setup.csh b/setup.csh index 0fa8d0d9..efb61d56 100755 --- a/setup.csh +++ b/setup.csh @@ -42,4 +42,4 @@ if (! -d "${K4GeneratorsConfigDir}/install/") then endif # Set executable -alias k4GeneratorsConfig "python3 ${K4GeneratorsConfigDir}/python/main.py" +alias k4GeneratorsConfigRun "python3 ${K4GeneratorsConfigDir}/python/run.py" diff --git a/setup.sh b/setup.sh index 66ad3368..bd05615f 100755 --- a/setup.sh +++ b/setup.sh @@ -30,3 +30,4 @@ fi # Set executable alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/main.py" +alias k4GeneratorsConfigRun="python3 ${K4GeneratorsConfigDir}/python/run.py" diff --git a/setup.zsh b/setup.zsh index d0805ef5..fe5d9f9b 100755 --- a/setup.zsh +++ b/setup.zsh @@ -30,3 +30,4 @@ fi # Set executable alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/main.py" +alias k4GeneratorsConfigRun="python3 ${K4GeneratorsConfigDir}/python/run.py" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 60402c6b..194f80b2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,6 +3,7 @@ set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}") set(TEST_TIMEOUT 3500) # we split the execution into four steps: +set(pyRUN_YAMLS "k4GeneratorsConfigRun --create") set(RUN_YAMLS "${CMAKE_CURRENT_SOURCE_DIR}/createGeneratorDatacardsFromYaml.sh") set(CHECK_DATACARDS "${CMAKE_CURRENT_SOURCE_DIR}/checkGeneratorDatacards.sh") set(RUN_GENERATORS "${CMAKE_CURRENT_SOURCE_DIR}/runEventGeneration.sh") @@ -17,6 +18,14 @@ foreach(generator ${generatorList}) message(${generator} " appended to the runlist ") endforeach() +# add the first test: setting up the yaml files +add_test(NAME pycreateGeneratorDatacards COMMAND $py{RUN_YAMLS} ) +set_tests_properties("pycreateGeneratorDatacards" + PROPERTIES + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ENVIRONMENT ${TEST_ENVIRONMENT} +) + # add the first test: setting up the yaml files add_test(NAME createGeneratorDatacards COMMAND ${RUN_YAMLS} ) set_tests_properties("createGeneratorDatacards" diff --git a/test/createGeneratorDatacardsFromYaml.sh b/test/createGeneratorDatacardsFromYaml.sh index a71e89ac..e29e5de4 100755 --- a/test/createGeneratorDatacardsFromYaml.sh +++ b/test/createGeneratorDatacardsFromYaml.sh @@ -23,6 +23,7 @@ while getopts ${OPTSTRING} opt; do echo "Arguments are:" echo "-h for help" echo "-d YAMLDIRECTORY " + echo "-f YAMLFILE " exit 0 ;; ?) From 8f484a857c4ad452e76d21828c106744b84c20cb Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 21:34:35 +0100 Subject: [PATCH 10/88] fix syntax error --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 194f80b2..db615973 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -19,7 +19,7 @@ foreach(generator ${generatorList}) endforeach() # add the first test: setting up the yaml files -add_test(NAME pycreateGeneratorDatacards COMMAND $py{RUN_YAMLS} ) +add_test(NAME pycreateGeneratorDatacards COMMAND ${pyRUN_YAMLS} ) set_tests_properties("pycreateGeneratorDatacards" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} From eb476bd4df1ec86f071bd22ff2390c65e6dd6c89 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 21:44:11 +0100 Subject: [PATCH 11/88] invoke python --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index db615973..a1e5aa8a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,7 +3,7 @@ set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}") set(TEST_TIMEOUT 3500) # we split the execution into four steps: -set(pyRUN_YAMLS "k4GeneratorsConfigRun --create") +set(pyRUN_YAMLS "python3 run.py --create") set(RUN_YAMLS "${CMAKE_CURRENT_SOURCE_DIR}/createGeneratorDatacardsFromYaml.sh") set(CHECK_DATACARDS "${CMAKE_CURRENT_SOURCE_DIR}/checkGeneratorDatacards.sh") set(RUN_GENERATORS "${CMAKE_CURRENT_SOURCE_DIR}/runEventGeneration.sh") From dbc141603b96ebbe924ba8a643965b00ffdce71f Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 21:48:28 +0100 Subject: [PATCH 12/88] invoke python --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a1e5aa8a..cefa9d93 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -22,7 +22,7 @@ endforeach() add_test(NAME pycreateGeneratorDatacards COMMAND ${pyRUN_YAMLS} ) set_tests_properties("pycreateGeneratorDatacards" PROPERTIES - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python ENVIRONMENT ${TEST_ENVIRONMENT} ) From 33a019fb02b337a4b39442c80bd811bf6a971ed0 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 21:54:20 +0100 Subject: [PATCH 13/88] invoke python --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cefa9d93..33a92668 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -22,7 +22,7 @@ endforeach() add_test(NAME pycreateGeneratorDatacards COMMAND ${pyRUN_YAMLS} ) set_tests_properties("pycreateGeneratorDatacards" PROPERTIES - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../python ENVIRONMENT ${TEST_ENVIRONMENT} ) From f7010079ef938554459a9f88452ded432a8a91a5 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 22:11:04 +0100 Subject: [PATCH 14/88] invoke python --- test/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 33a92668..550f37c9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,5 +1,5 @@ # generl settings: -set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}") +set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}" "PYTHONPATH=${PROJECT_SOURCE_DIR}/python:$ENV{PYTHONPATH}") set(TEST_TIMEOUT 3500) # we split the execution into four steps: @@ -22,8 +22,8 @@ endforeach() add_test(NAME pycreateGeneratorDatacards COMMAND ${pyRUN_YAMLS} ) set_tests_properties("pycreateGeneratorDatacards" PROPERTIES - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../python - ENVIRONMENT ${TEST_ENVIRONMENT} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ENVIRONMENT ${TEST_ENVIRONMENT} ) # add the first test: setting up the yaml files From cb9cf1022b9fc337916913dd81513a08d6a1eef8 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 22:21:24 +0100 Subject: [PATCH 15/88] invoke python --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 550f37c9..072ce156 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,7 +3,7 @@ set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}" "PYTHONPATH=${P set(TEST_TIMEOUT 3500) # we split the execution into four steps: -set(pyRUN_YAMLS "python3 run.py --create") +set(pyRUN_YAMLS "${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../python/run.py --create") set(RUN_YAMLS "${CMAKE_CURRENT_SOURCE_DIR}/createGeneratorDatacardsFromYaml.sh") set(CHECK_DATACARDS "${CMAKE_CURRENT_SOURCE_DIR}/checkGeneratorDatacards.sh") set(RUN_GENERATORS "${CMAKE_CURRENT_SOURCE_DIR}/runEventGeneration.sh") From 681baa8ec30a43605532673e156e0ed0c69876f8 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 22:24:01 +0100 Subject: [PATCH 16/88] invoke python --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 072ce156..87e4c8a9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,5 +1,5 @@ # generl settings: -set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}" "PYTHONPATH=${PROJECT_SOURCE_DIR}/python:$ENV{PYTHONPATH}") +set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}") set(TEST_TIMEOUT 3500) # we split the execution into four steps: From 14d321fbf8788e32040958b8c4c12ed342bdb474 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Fri, 27 Mar 2026 22:28:07 +0100 Subject: [PATCH 17/88] invoke python --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 87e4c8a9..b99f4957 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,7 +3,7 @@ set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}") set(TEST_TIMEOUT 3500) # we split the execution into four steps: -set(pyRUN_YAMLS "${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../python/run.py --create") +set(pyRUN_YAMLS "${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/python/run.py --create") set(RUN_YAMLS "${CMAKE_CURRENT_SOURCE_DIR}/createGeneratorDatacardsFromYaml.sh") set(CHECK_DATACARDS "${CMAKE_CURRENT_SOURCE_DIR}/checkGeneratorDatacards.sh") set(RUN_GENERATORS "${CMAKE_CURRENT_SOURCE_DIR}/runEventGeneration.sh") From 3e9576d39beea03aecac1801ea76fbc9938163b9 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 09:48:21 +0100 Subject: [PATCH 18/88] consistenty in Particle Instantiation and move to python --- python/Particles.py | 375 ++++++++++++++++++++------------------------ python/main.py | 3 + test/CMakeLists.txt | 13 +- 3 files changed, 179 insertions(+), 212 deletions(-) diff --git a/python/Particles.py b/python/Particles.py index b0728bbb..8d94c90b 100644 --- a/python/Particles.py +++ b/python/Particles.py @@ -9,15 +9,13 @@ class Particle: "name", "antiname", "mass", - "width", - "texname", - "antitexname", + "width" ] def __init__( - self, pdg_code, name, antiname, mass, width, texname, antitexname, **options + self, pdg_code, name, antiname, mass, width, **options ): - args = (pdg_code, name, antiname, mass, width, texname, antitexname) + args = (pdg_code, name, antiname, mass, width) assert len(self._required_args) == len(args) for i, name in enumerate(self._required_args): @@ -44,8 +42,6 @@ def anti(self): self.name, self.mass, self.width, - self.antitexname, - self.texname, ) @staticmethod @@ -78,200 +74,177 @@ def name_from_pdg(pdg): print(f"{pdg} code not found") -Photon = Particle( - pdg_code=22, name="a", antiname="a", mass=0, width=0, texname="a", antitexname="a" -) +class ParticleCollection: + """The default Particles are instantiated""" -Z = Particle( - pdg_code=23, - name="Z", - antiname="Z", - mass=Param.MZ.value, - width=Param.WZ.value, - texname="Z", - antitexname="Z", -) + def __init__(self): -W__plus__ = Particle( - pdg_code=24, - name="W+", - antiname="W-", - spin=3, - color=1, - mass=Param.MW.value, - width=Param.WW.value, - texname="W+", - antitexname="W-", -) -W__minus__ = W__plus__.anti() - - -g = Particle( - pdg_code=21, - name="g", - antiname="g", - mass=0, - width=0, - texname="g", - antitexname="g", -) - -ve = Particle( - pdg_code=12, - name="ve", - antiname="ve~", - spin=2, - color=1, - mass=0, - width=0, - texname="ve", - antitexname="ve~", -) - -ve__tilde__ = ve.anti() - -vm = Particle( - pdg_code=14, - name="vm", - antiname="vm~", - mass=0, - width=0, - texname="vm", - antitexname="vm~", -) - -vm__tilde__ = vm.anti() - -vt = Particle( - pdg_code=16, - name="vt", - antiname="vt~", - mass=0, - width=0, - texname="vt", - antitexname="vt~", -) - -vt__tilde__ = vt.anti() - -e__minus__ = Particle( - pdg_code=11, - name="e-", - antiname="e+", - mass=0.0, - width=0, - texname="e-", - antitexname="e+", -) - -e__plus__ = e__minus__.anti() - -mu__minus__ = Particle( - pdg_code=13, - name="mu-", - antiname="mu+", - mass=0, - width=0, - texname="mu-", - antitexname="mu+", -) - -mu__plus__ = mu__minus__.anti() - -ta__minus__ = Particle( - pdg_code=15, - name="ta-", - antiname="ta+", - mass=0, - width=0, - texname="ta-", - antitexname="ta+", -) - -ta__plus__ = ta__minus__.anti() - -u = Particle( - pdg_code=2, - name="u", - antiname="u~", - mass=0, - width=0, - texname="u", - antitexname="u~", -) - -u__tilde__ = u.anti() - -c = Particle( - pdg_code=4, - name="c", - antiname="c~", - mass=0, - width=0, - texname="c", - antitexname="c~", -) - -c__tilde__ = c.anti() - -t = Particle( - pdg_code=6, - name="t", - antiname="t~", - mass=Param.MT.value, - width=Param.WT.value, - texname="t", - antitexname="t~", -) - -t__tilde__ = t.anti() - -d = Particle( - pdg_code=1, - name="d", - antiname="d~", - mass=0, - width=0, - texname="d", - antitexname="d~", -) - -d__tilde__ = d.anti() - -s = Particle( - pdg_code=3, - name="s", - antiname="s~", - spin=2, - color=3, - mass=0, - width=0, - texname="s", - antitexname="s~", -) - -s__tilde__ = s.anti() - -b = Particle( - pdg_code=5, - name="b", - antiname="b~", - spin=2, - color=3, - mass=Param.MB.value, - width=0, - texname="b", - antitexname="b~", -) - -b__tilde__ = b.anti() + globals()['Photon'] = Particle( + pdg_code=22, + name="a", + antiname="a", + mass=0, + width=0 + ) -H = Particle( - pdg_code=25, - name="H", - antiname="H", - mass=Param.MH.value, - width=Param.WH.value, - texname="H", - antitexname="H", -) + globals()['Z'] = Particle( + pdg_code=23, + name="Z", + antiname="Z", + mass=Param.MZ.value, + width=Param.WZ.value, + ) + + globals()['W__plus__'] = Particle( + pdg_code=24, + name="W+", + antiname="W-", + spin=3, + color=1, + mass=Param.MW.value, + width=Param.WW.value, + ) + globals()['W__minus__'] = W__plus__.anti() + + + globals()['g'] = Particle( + pdg_code=21, + name="g", + antiname="g", + mass=0, + width=0, + ) + + globals()['ve'] = Particle( + pdg_code=12, + name="ve", + antiname="ve~", + spin=2, + color=1, + mass=0, + width=0, + ) + + globals()['ve__tilde__'] = ve.anti() + + globals()['vm'] = Particle( + pdg_code=14, + name="vm", + antiname="vm~", + mass=0, + width=0, + ) + + globals()['vm__tilde__'] = vm.anti() + + globals()['vt'] = Particle( + pdg_code=16, + name="vt", + antiname="vt~", + mass=0, + width=0, + ) + + globals()['vt__tilde__'] = vt.anti() + + globals()['e__minus__'] = Particle( + pdg_code=11, + name="e-", + antiname="e+", + mass=0.0, + width=0, + ) + + globals()['e__plus__'] = e__minus__.anti() + + globals()['mu__minus__'] = Particle( + pdg_code=13, + name="mu-", + antiname="mu+", + mass=0, + width=0, + ) + + globals()['mu__plus__'] = mu__minus__.anti() + + globals()['ta__minus__'] = Particle( + pdg_code=15, + name="ta-", + antiname="ta+", + mass=0, + width=0, + ) + + globals()['ta__plus__'] = ta__minus__.anti() + + globals()['u'] = Particle( + pdg_code=2, + name="u", + antiname="u~", + mass=0, + width=0, + ) + + globals()['u__tilde__'] = u.anti() + + globals()['c'] = Particle( + pdg_code=4, + name="c", + antiname="c~", + mass=0, + width=0, + ) + + globals()['c__tilde__'] = c.anti() + + globals()['t'] = Particle( + pdg_code=6, + name="t", + antiname="t~", + mass=Param.MT.value, + width=Param.WT.value, + ) + + globals()['t__tilde__'] = t.anti() + + globals()['d'] = Particle( + pdg_code=1, + name="d", + antiname="d~", + mass=0, + width=0, + ) + + globals()['d__tilde__'] = d.anti() + + globals()['s'] = Particle( + pdg_code=3, + name="s", + antiname="s~", + spin=2, + color=3, + mass=0, + width=0, + ) + + globals()['s__tilde__'] = s.anti() + + globals()['b'] = Particle( + pdg_code=5, + name="b", + antiname="b~", + spin=2, + color=3, + mass=Param.MB.value, + width=0, + ) + + globals()['b__tilde__'] = b.anti() + + globals()['H'] = Particle( + pdg_code=25, + name="H", + antiname="H", + mass=Param.MH.value, + width=Param.WH.value, + ) diff --git a/python/main.py b/python/main.py index ec1688d2..aa7cdfea 100755 --- a/python/main.py +++ b/python/main.py @@ -11,6 +11,7 @@ import Input as Settings import Process as process_module import Generators as generators_module +from Particles import ParticleCollection def make_output_directory(generators, output_directory, procname): # Overwrite directory if it exists @@ -164,6 +165,8 @@ def main(args=None): def executeFiles(files, sqrts, rndmSeedFallback=4711, events=-1): + # first step reset all particles: + ParticleCollection() if sqrts == 0: print("Generating and writing configuration files") else: diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b99f4957..7589f794 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,8 +3,7 @@ set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}") set(TEST_TIMEOUT 3500) # we split the execution into four steps: -set(pyRUN_YAMLS "${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/python/run.py --create") -set(RUN_YAMLS "${CMAKE_CURRENT_SOURCE_DIR}/createGeneratorDatacardsFromYaml.sh") +set(RUN_YAMLS "${PROJECT_SOURCE_DIR}/python/run.py") set(CHECK_DATACARDS "${CMAKE_CURRENT_SOURCE_DIR}/checkGeneratorDatacards.sh") set(RUN_GENERATORS "${CMAKE_CURRENT_SOURCE_DIR}/runEventGeneration.sh") set(RUN_SUMMARY "${CMAKE_CURRENT_SOURCE_DIR}/runSummary.sh") @@ -19,19 +18,11 @@ foreach(generator ${generatorList}) endforeach() # add the first test: setting up the yaml files -add_test(NAME pycreateGeneratorDatacards COMMAND ${pyRUN_YAMLS} ) +add_test(NAME pycreateGeneratorDatacards COMMAND python3 ${RUN_YAMLS} --create --workDir ../test/ci) set_tests_properties("pycreateGeneratorDatacards" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ENVIRONMENT ${TEST_ENVIRONMENT} -) - -# add the first test: setting up the yaml files -add_test(NAME createGeneratorDatacards COMMAND ${RUN_YAMLS} ) -set_tests_properties("createGeneratorDatacards" - PROPERTIES - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ENVIRONMENT ${TEST_ENVIRONMENT} FIXTURES_SETUP "processYamls" ) From 69c56e0367ee9a73d5f8f251b4e05c5815e41bd1 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 11:43:23 +0100 Subject: [PATCH 19/88] make comparisons work --- python/CIUtils.py | 158 +++++++++++++++++++++++++++++ python/createGeneratorDatacards.py | 71 ------------- python/run.py | 39 +++---- test/CMakeLists.txt | 6 +- 4 files changed, 176 insertions(+), 98 deletions(-) create mode 100644 python/CIUtils.py delete mode 100644 python/createGeneratorDatacards.py diff --git a/python/CIUtils.py b/python/CIUtils.py new file mode 100644 index 00000000..de0dc09b --- /dev/null +++ b/python/CIUtils.py @@ -0,0 +1,158 @@ +from abc import ABC,abstractmethod +import os +import sys +import shutil +from pathlib import Path +import filecmp + +from main import main + +class CIUtilsBase(ABC): + """Generator Generator Datacards""" + def __init__(self, workDirectory, outputDirectory): + + # consistent processing of names + self._workDir = os.path.dirname(os.path.realpath(__file__))+"/"+workDirectory + if workDirectory.startswith("/"): + self._workDir = args.workDir + + self.outDir = os.path.dirname(os.path.realpath(__file__))+"/"+outputDirectory + if outputDirectory.startswith("/"): + self.outdir = outputDirectory + + self._referenceDir = os.path.dirname(os.path.realpath(__file__))+"/../test/ref-results" + + +class createGeneratorDatacards(CIUtilsBase): + """Generator Generator Datacards""" + + def __init__(self, yamlDirectory, yamlFile, workDirectory, outputDirectory): + super().__init__(workDirectory, outputDirectory) + + # specific members for DC creation + self._yamlDir = os.path.dirname(os.path.realpath(__file__))+"/"+yamlDirectory + if yamlDirectory.startswith("/"): + self._yamlDir = yamlDirectory + + self._yamlFiles = [] + + # make the directory for the work + self.makeDirectory(workDirectory) + # make the output directory for the output + self.makeDirectory(outputDirectory) + + # prepare yamls: + self.prepareYamls(yamlFile); + + # prepare the Sqrts files: + self.prepareECMS(); + + # process all yamls: + self.process(); + + def makeDirectory(self, dirname): + # Overwrite directory if it exists + if not os.path.exists(dirname): + os.makedirs(dirname) + else: + shutil.rmtree(dirname) + os.makedirs(dirname) + + def prepareYamls(self, yamlFile): + # move the yamls to the work directory + # the yaml file is not specified copy all yaml files to the work directory: + if yamlFile == "None": + self._yamlFiles = [filename for filename in os.listdir(self._yamlDir) if filename.endswith('.yaml')] + else: + self._yamlFiles =[self.yamlDir+"/"+yamlFile] + + # copy the yaml files + for filename in self._yamlFiles: + shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) + + def prepareECMS(self): + # move the ECMS files to the work directory + ecmsFiles = [filename for filename in os.listdir(self._yamlDir) if filename.endswith('.dat') and filename.startswith('ecms')] + + # copy the yaml files + for filename in ecmsFiles: + shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) + + def process(self): + # go to the working directory + os.chdir(self._workDir) + # loop over all files + for filename in self._yamlFiles: + processName = Path(filename).stem + ecmsName = "ecms"+processName+".dat" + print("Processing : "+processName) + if os.path.isfile(ecmsName): + main([filename,'--ecmsFile',ecmsName]) + else: + main([filename]) + +class checkGeneratorDatacards(CIUtilsBase): + """Check Generator Datacards""" + + def __init__(self, generator, workDirectory, outputDirectory): + super().__init__(workDirectory, outputDirectory) + + # specific stuff: + self.generatorDir = self._workDir+"/Run-Cards" + + # retrieve all generators in the workdirector: + generators = self.getGenerators(generator) + + # now compare to reference + self.process(generators) + + def getGenerators(self, generator): + if generator == "All": + return os.listdir(self.generatorDir) + else: + return [generator] + + def getProcesses(self, generator): + return os.listdir(self.generatorDir+"/"+generator) + + def getFileNames(self, directory): + filenames = os.listdir(directory) + filenames =[name for name in filenames if os.path.isfile(os.path.join(directory,name))] + return filenames + + def process(self, generators): + failure = False + + for generator in generators: + processes = self.getProcesses(generator) + + for proc in processes: + genProc = "/"+generator+"/"+proc + newDir = self.generatorDir+genProc + refDir = self._referenceDir+genProc + fileNames = self.getFileNames(refDir) + + for name in fileNames: + newFile = newDir+"/"+name + refFile = refDir+"/"+name + message = "Generator "+generator+" Process "+proc+" File "+name + if filecmp.cmp(refFile,newFile, shallow=False): + print(message+" identical") + else: + print(message+" differ") + failure = True + if failure: + sys.exit("Failed comparison") + + +class runEventGeneration(CIUtilsBase): + """Check Generator Datacards""" + + def __init__(self, workDirectory, outputDirectory): + super().__init__(workDirectory, outputDirectory) + +class runSummary(CIUtilsBase): + """Run summary of all processes""" + + def __init__(self, workDirectory, outputDirectory): + super().__init__(workDirectory, outputDirectory) diff --git a/python/createGeneratorDatacards.py b/python/createGeneratorDatacards.py deleted file mode 100644 index b3b60f14..00000000 --- a/python/createGeneratorDatacards.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -import sys -import shutil -from pathlib import Path - -from main import main - -class createGeneratorDatacards: - """Generator Generator Datacards""" - - def __init__(self, yamlDirectory, yamlFile, workDirectory, outputDirectory): - # useful for the class - self._yamlDir = yamlDirectory - self._yamlFiles = [] - self._workDir = workDirectory - self._outDir = outputDirectory - - # make the directory for the work - self.makeDirectory(workDirectory) - # make the output directory for the output - self.makeDirectory(outputDirectory) - - # prepare yamls: - self.prepareYamls(yamlFile); - - # prepare the Sqrts files: - self.prepareECMS(); - - # process all yamls: - self.process(); - - def makeDirectory(self, dirname): - # Overwrite directory if it exists - if not os.path.exists(dirname): - os.makedirs(dirname) - else: - shutil.rmtree(dirname) - os.makedirs(dirname) - - def prepareYamls(self, yamlFile): - # move the yamls to the work directory - # the yaml file is not specified copy all yaml files to the work directory: - if yamlFile == "None": - self._yamlFiles = [filename for filename in os.listdir(self._yamlDir) if filename.endswith('.yaml')] - else: - self._yamlFiles =[yamlFile] - - # copy the yaml files - for filename in self._yamlFiles: - shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) - - def prepareECMS(self): - # move the ECMS files to the work directory - ecmsFiles = [filename for filename in os.listdir(self._yamlDir) if filename.endswith('.dat') and filename.startswith('ecms')] - - # copy the yaml files - for filename in ecmsFiles: - shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) - - def process(self): - # go to the working directory - os.chdir(self._workDir) - # loop over all files - for filename in self._yamlFiles: - processName = Path(filename).stem - ecmsName = "ecms"+processName+".dat" - print("Processing : "+processName) - if os.path.isfile(ecmsName): - main([filename,'--ecmsFile',ecmsName]) - else: - main([filename]) diff --git a/python/run.py b/python/run.py index fa4aa781..d37c1f03 100644 --- a/python/run.py +++ b/python/run.py @@ -6,7 +6,8 @@ import argparse import textwrap -from createGeneratorDatacards import createGeneratorDatacards +from CIUtils import createGeneratorDatacards +from CIUtils import checkGeneratorDatacards def run(): parser = argparse.ArgumentParser( @@ -41,6 +42,12 @@ def run(): action='store_true', help="check the generator datacards with respect to the reference in k4GeneratorsConfig/test/ref-results", ) + parser.add_argument( + "--generator", + type=str, + default="All", + help="generator to be run, default: All", + ) parser.add_argument( "--run", action='store_true', @@ -66,33 +73,17 @@ def run(): args = parser.parse_args() - yamlDirectory = os.path.dirname(os.path.realpath(__file__))+"/"+args.yamlDir - if args.yamlDir.startswith("/"): - yamlDirectory = args.yamlDir - - yamlFile = "None" - if args.yamlFile != "None": - yamlFile = yamlDirectory+"/"+args.yamlFile - - workDirectory = os.path.dirname(os.path.realpath(__file__))+"/"+args.workDir - if args.workDir.startswith("/"): - workDirectory = args.workDir - - outputDirectory = os.path.dirname(os.path.realpath(__file__))+"/"+args.outputDir - if args.outputDir.startswith("/"): - outputDirectory = args.outputDir - if args.create: - createGeneratorDatacards(yamlDirectory,yamlFile,workDirectory,outputDirectory) + createGeneratorDatacards(args.yamlDir,args.yamlFile,args.workDir,args.outputDir) - #if args.check: - #checkGeneratorDatacards() + if args.check: + checkGeneratorDatacards(args.generator,args.workDir,args.outputDir) - #if args.run: - #runEventGeneration() + if args.run: + runEventGeneration(args.workDir,args.outputDir) - #if args.summary: - #runSummary() + if args.summary: + runSummary(args.workDir,args.outputDir) if __name__ == "__main__": run() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7589f794..781c8d18 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,7 +3,7 @@ set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}") set(TEST_TIMEOUT 3500) # we split the execution into four steps: -set(RUN_YAMLS "${PROJECT_SOURCE_DIR}/python/run.py") +set(EXECUTABLE "${PROJECT_SOURCE_DIR}/python/run.py") set(CHECK_DATACARDS "${CMAKE_CURRENT_SOURCE_DIR}/checkGeneratorDatacards.sh") set(RUN_GENERATORS "${CMAKE_CURRENT_SOURCE_DIR}/runEventGeneration.sh") set(RUN_SUMMARY "${CMAKE_CURRENT_SOURCE_DIR}/runSummary.sh") @@ -18,7 +18,7 @@ foreach(generator ${generatorList}) endforeach() # add the first test: setting up the yaml files -add_test(NAME pycreateGeneratorDatacards COMMAND python3 ${RUN_YAMLS} --create --workDir ../test/ci) +add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --create --workDir ../test/ci) set_tests_properties("pycreateGeneratorDatacards" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -28,7 +28,7 @@ set_tests_properties("pycreateGeneratorDatacards" # add the second test: comparing the yaml files to the reference files function(check_generator_datacards generator) - add_test(NAME checkGeneratorDatacards_${generator} COMMAND ${CHECK_DATACARDS} -g ${generator}) + add_test(NAME checkGeneratorDatacards_${generator} COMMAND ${CHECK_DATACARDS} --check --workDir ../test/ci --generator ${generator}) set_tests_properties("checkGeneratorDatacards_${generator}" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} From 0f0e7839bf7fad60564215aa6365bdcc6cc092b3 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 11:53:20 +0100 Subject: [PATCH 20/88] move check to python --- test/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 781c8d18..3213a926 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -19,7 +19,7 @@ endforeach() # add the first test: setting up the yaml files add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --create --workDir ../test/ci) -set_tests_properties("pycreateGeneratorDatacards" +set_tests_properties("createGeneratorDatacards" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ENVIRONMENT ${TEST_ENVIRONMENT} @@ -28,7 +28,7 @@ set_tests_properties("pycreateGeneratorDatacards" # add the second test: comparing the yaml files to the reference files function(check_generator_datacards generator) - add_test(NAME checkGeneratorDatacards_${generator} COMMAND ${CHECK_DATACARDS} --check --workDir ../test/ci --generator ${generator}) + add_test(NAME checkGeneratorDatacards_${generator} COMMAND python3 ${EXECUTABLE} --check --workDir ../test/ci --generator ${generator}) set_tests_properties("checkGeneratorDatacards_${generator}" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} From 543a3198ea5d7ba5e89419838d6487946ff9f8f3 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 12:05:54 +0100 Subject: [PATCH 21/88] precommit changes --- python/CIUtils.py | 9 ++++---- python/Particles.py | 53 ++++++++++++++++++++++----------------------- test/CMakeLists.txt | 2 +- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/python/CIUtils.py b/python/CIUtils.py index de0dc09b..8ecbb72e 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -99,7 +99,7 @@ def __init__(self, generator, workDirectory, outputDirectory): # specific stuff: self.generatorDir = self._workDir+"/Run-Cards" - + # retrieve all generators in the workdirector: generators = self.getGenerators(generator) @@ -111,10 +111,10 @@ def getGenerators(self, generator): return os.listdir(self.generatorDir) else: return [generator] - + def getProcesses(self, generator): return os.listdir(self.generatorDir+"/"+generator) - + def getFileNames(self, directory): filenames = os.listdir(directory) filenames =[name for name in filenames if os.path.isfile(os.path.join(directory,name))] @@ -143,10 +143,9 @@ def process(self, generators): failure = True if failure: sys.exit("Failed comparison") - class runEventGeneration(CIUtilsBase): - """Check Generator Datacards""" + """Run Event Generation""" def __init__(self, workDirectory, outputDirectory): super().__init__(workDirectory, outputDirectory) diff --git a/python/Particles.py b/python/Particles.py index 8d94c90b..53fd47da 100644 --- a/python/Particles.py +++ b/python/Particles.py @@ -94,7 +94,7 @@ def __init__(self): mass=Param.MZ.value, width=Param.WZ.value, ) - + globals()['W__plus__'] = Particle( pdg_code=24, name="W+", @@ -105,8 +105,7 @@ def __init__(self): width=Param.WW.value, ) globals()['W__minus__'] = W__plus__.anti() - - + globals()['g'] = Particle( pdg_code=21, name="g", @@ -114,7 +113,7 @@ def __init__(self): mass=0, width=0, ) - + globals()['ve'] = Particle( pdg_code=12, name="ve", @@ -124,7 +123,7 @@ def __init__(self): mass=0, width=0, ) - + globals()['ve__tilde__'] = ve.anti() globals()['vm'] = Particle( @@ -134,9 +133,9 @@ def __init__(self): mass=0, width=0, ) - + globals()['vm__tilde__'] = vm.anti() - + globals()['vt'] = Particle( pdg_code=16, name="vt", @@ -144,9 +143,9 @@ def __init__(self): mass=0, width=0, ) - + globals()['vt__tilde__'] = vt.anti() - + globals()['e__minus__'] = Particle( pdg_code=11, name="e-", @@ -154,9 +153,9 @@ def __init__(self): mass=0.0, width=0, ) - + globals()['e__plus__'] = e__minus__.anti() - + globals()['mu__minus__'] = Particle( pdg_code=13, name="mu-", @@ -164,9 +163,9 @@ def __init__(self): mass=0, width=0, ) - + globals()['mu__plus__'] = mu__minus__.anti() - + globals()['ta__minus__'] = Particle( pdg_code=15, name="ta-", @@ -174,9 +173,9 @@ def __init__(self): mass=0, width=0, ) - + globals()['ta__plus__'] = ta__minus__.anti() - + globals()['u'] = Particle( pdg_code=2, name="u", @@ -184,9 +183,9 @@ def __init__(self): mass=0, width=0, ) - + globals()['u__tilde__'] = u.anti() - + globals()['c'] = Particle( pdg_code=4, name="c", @@ -194,9 +193,9 @@ def __init__(self): mass=0, width=0, ) - + globals()['c__tilde__'] = c.anti() - + globals()['t'] = Particle( pdg_code=6, name="t", @@ -204,9 +203,9 @@ def __init__(self): mass=Param.MT.value, width=Param.WT.value, ) - + globals()['t__tilde__'] = t.anti() - + globals()['d'] = Particle( pdg_code=1, name="d", @@ -214,9 +213,9 @@ def __init__(self): mass=0, width=0, ) - + globals()['d__tilde__'] = d.anti() - + globals()['s'] = Particle( pdg_code=3, name="s", @@ -226,9 +225,9 @@ def __init__(self): mass=0, width=0, ) - + globals()['s__tilde__'] = s.anti() - + globals()['b'] = Particle( pdg_code=5, name="b", @@ -238,9 +237,9 @@ def __init__(self): mass=Param.MB.value, width=0, ) - + globals()['b__tilde__'] = b.anti() - + globals()['H'] = Particle( pdg_code=25, name="H", diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3213a926..990e8f19 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -22,7 +22,7 @@ add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --create -- set_tests_properties("createGeneratorDatacards" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ENVIRONMENT ${TEST_ENVIRONMENT} + ENVIRONMENT ${TEST_ENVIRONMENT} FIXTURES_SETUP "processYamls" ) From 283b00ca267397b9fd6fd4c3cf8af53ebed34489 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 12:44:00 +0100 Subject: [PATCH 22/88] precommit corrections --- python/Particles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/Particles.py b/python/Particles.py index 53fd47da..e3625315 100644 --- a/python/Particles.py +++ b/python/Particles.py @@ -125,7 +125,7 @@ def __init__(self): ) globals()['ve__tilde__'] = ve.anti() - + globals()['vm'] = Particle( pdg_code=14, name="vm", From e45e669c7c09bdffaef09919333b25a123a6084d Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 15:05:12 +0100 Subject: [PATCH 23/88] move ci eventGeneration runs to python --- python/CIUtils.py | 126 +++++++++++++++++++++++++++++++------------- python/run.py | 13 ++--- test/CMakeLists.txt | 4 +- 3 files changed, 96 insertions(+), 47 deletions(-) diff --git a/python/CIUtils.py b/python/CIUtils.py index 8ecbb72e..f9101288 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -1,6 +1,7 @@ from abc import ABC,abstractmethod import os import sys +import subprocess import shutil from pathlib import Path import filecmp @@ -22,6 +23,37 @@ def __init__(self, workDirectory, outputDirectory): self._referenceDir = os.path.dirname(os.path.realpath(__file__))+"/../test/ref-results" + self._generatorDir = self._workDir+"/Run-Cards" + + def getGenerators(self, generator): + if generator == "All": + return os.listdir(self._generatorDir) + else: + return [generator] + + def getProcesses(self, generator): + return os.listdir(self._generatorDir+"/"+generator) + + def process(self, generators): + # where do we start from + cwd = os.getcwd() + + failure = False + for generator in generators: + processes = self.getProcesses(generator) + + for process in processes: + if not self.execute(generator, process): + failure = True + + if failure: + sys.exit("Failed") + # return to the starting point + os.chdir(cwd) + + @abstractmethod + def execute(self, generator, process): + pass class createGeneratorDatacards(CIUtilsBase): """Generator Generator Datacards""" @@ -47,8 +79,11 @@ def __init__(self, yamlDirectory, yamlFile, workDirectory, outputDirectory): # prepare the Sqrts files: self.prepareECMS(); - # process all yamls: - self.process(); + # run all yamls: + self.run(); + + def execute(self, generator, process): + pass def makeDirectory(self, dirname): # Overwrite directory if it exists @@ -78,7 +113,9 @@ def prepareECMS(self): for filename in ecmsFiles: shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) - def process(self): + def run(self): + # remember where we start from + cwd = os.getcwd() # go to the working directory os.chdir(self._workDir) # loop over all files @@ -90,6 +127,8 @@ def process(self): main([filename,'--ecmsFile',ecmsName]) else: main([filename]) + # return tu the starting point + os.chdir(cwd) class checkGeneratorDatacards(CIUtilsBase): """Check Generator Datacards""" @@ -97,59 +136,70 @@ class checkGeneratorDatacards(CIUtilsBase): def __init__(self, generator, workDirectory, outputDirectory): super().__init__(workDirectory, outputDirectory) - # specific stuff: - self.generatorDir = self._workDir+"/Run-Cards" - # retrieve all generators in the workdirector: generators = self.getGenerators(generator) # now compare to reference self.process(generators) - def getGenerators(self, generator): - if generator == "All": - return os.listdir(self.generatorDir) - else: - return [generator] - - def getProcesses(self, generator): - return os.listdir(self.generatorDir+"/"+generator) - def getFileNames(self, directory): filenames = os.listdir(directory) filenames =[name for name in filenames if os.path.isfile(os.path.join(directory,name))] return filenames - def process(self, generators): - failure = False - - for generator in generators: - processes = self.getProcesses(generator) + def execute(self, generator, process): + genProc = "/"+generator+"/"+process + newDir = self._generatorDir+genProc + refDir = self._referenceDir+genProc + fileNames = self.getFileNames(refDir) + + success = True + for name in fileNames: + newFile = newDir+"/"+name + refFile = refDir+"/"+name + message = "Generator "+generator+" Process "+process+" File "+name + if filecmp.cmp(refFile,newFile, shallow=False): + print(message+" identical") + else: + print(message+" differ") + success = False - for proc in processes: - genProc = "/"+generator+"/"+proc - newDir = self.generatorDir+genProc - refDir = self._referenceDir+genProc - fileNames = self.getFileNames(refDir) - - for name in fileNames: - newFile = newDir+"/"+name - refFile = refDir+"/"+name - message = "Generator "+generator+" Process "+proc+" File "+name - if filecmp.cmp(refFile,newFile, shallow=False): - print(message+" identical") - else: - print(message+" differ") - failure = True - if failure: - sys.exit("Failed comparison") + return success class runEventGeneration(CIUtilsBase): """Run Event Generation""" - def __init__(self, workDirectory, outputDirectory): + def __init__(self, generator, workDirectory, outputDirectory): super().__init__(workDirectory, outputDirectory) + # retrieve all generators in the workdirector: + generators = self.getGenerators(generator) + + # now compare to reference + self.process(generators) + + def execute(self, generator, process): + # go to the directory + genProcDir = self._generatorDir+"/"+generator+"/"+process + os.chdir(genProcDir) + # retrieve the script + scripts = [script for script in os.listdir(genProcDir) if script.startswith("Run_") and script.endswith(".sh")] + if len(scripts) != 1: + print(f"Found more than one script for generator {generator} with process {process}") + return False + # execute + success = True + for script in scripts: + try: + result = subprocess.run("./"+script, capture_output=True, check=True) + except CalledProcessError: + print(f"Execution error for {generator} in process {process}") + print(result.stdout) + print(result.stderr) + success = False + + return success + class runSummary(CIUtilsBase): """Run summary of all processes""" diff --git a/python/run.py b/python/run.py index d37c1f03..b0402e38 100644 --- a/python/run.py +++ b/python/run.py @@ -8,8 +8,9 @@ from CIUtils import createGeneratorDatacards from CIUtils import checkGeneratorDatacards +from CIUtils import runEventGeneration -def run(): +def run(arguments=None): parser = argparse.ArgumentParser( prog="k4GeneratorsConfig", formatter_class=argparse.RawDescriptionHelpFormatter, @@ -71,19 +72,19 @@ def run(): help="directory where the output is stored default: k4GeneratorsConfig/test/output", ) - args = parser.parse_args() + args = parser.parse_args(arguments) if args.create: - createGeneratorDatacards(args.yamlDir,args.yamlFile,args.workDir,args.outputDir) + createGeneratorDatacards(args.yamlDir, args.yamlFile, args.workDir, args.outputDir) if args.check: - checkGeneratorDatacards(args.generator,args.workDir,args.outputDir) + checkGeneratorDatacards(args.generator, args.workDir, args.outputDir) if args.run: - runEventGeneration(args.workDir,args.outputDir) + runEventGeneration(args.generator, args.workDir, args.outputDir) if args.summary: - runSummary(args.workDir,args.outputDir) + runSummary(args.workDir, args.outputDir) if __name__ == "__main__": run() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 990e8f19..fca69770 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,8 +4,6 @@ set(TEST_TIMEOUT 3500) # we split the execution into four steps: set(EXECUTABLE "${PROJECT_SOURCE_DIR}/python/run.py") -set(CHECK_DATACARDS "${CMAKE_CURRENT_SOURCE_DIR}/checkGeneratorDatacards.sh") -set(RUN_GENERATORS "${CMAKE_CURRENT_SOURCE_DIR}/runEventGeneration.sh") set(RUN_SUMMARY "${CMAKE_CURRENT_SOURCE_DIR}/runSummary.sh") # the generators to be run @@ -45,7 +43,7 @@ endforeach() # third test running the generators (define the function first then call it function(add_generator_run name generator) - add_test(NAME ${name}_${generator} COMMAND ${RUN_GENERATORS} -g ${generator} ) + add_test(NAME ${name}_${generator} COMMAND python3 ${EXECUTABLE} --run --workDir ../test/ci --generator ${generator} ) set_tests_properties(${name}_${generator} PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} From 0684dc179dedb5264988d3e1f717bce6c54041df Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 15:09:12 +0100 Subject: [PATCH 24/88] precommit correction --- python/CIUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/CIUtils.py b/python/CIUtils.py index f9101288..5b651d69 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -38,7 +38,7 @@ def process(self, generators): # where do we start from cwd = os.getcwd() - failure = False + failure = False for generator in generators: processes = self.getProcesses(generator) From d68d5a8f3b94c3d83940448bf8c082c74b3b1a2e Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 15:59:57 +0100 Subject: [PATCH 25/88] last step --- python/CIUtils.py | 52 +++++++++++++++++++++++++++++++++++++++------ python/run.py | 1 + test/CMakeLists.txt | 3 +-- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/python/CIUtils.py b/python/CIUtils.py index 5b651d69..a9e557ad 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -17,7 +17,7 @@ def __init__(self, workDirectory, outputDirectory): if workDirectory.startswith("/"): self._workDir = args.workDir - self.outDir = os.path.dirname(os.path.realpath(__file__))+"/"+outputDirectory + self._outDir = os.path.dirname(os.path.realpath(__file__))+"/"+outputDirectory if outputDirectory.startswith("/"): self.outdir = outputDirectory @@ -34,6 +34,11 @@ def getGenerators(self, generator): def getProcesses(self, generator): return os.listdir(self._generatorDir+"/"+generator) + def getFileNames(self, directory): + filenames = os.listdir(directory) + filenames =[name for name in filenames if os.path.isfile(os.path.join(directory,name))] + return filenames + def process(self, generators): # where do we start from cwd = os.getcwd() @@ -82,8 +87,34 @@ def __init__(self, yamlDirectory, yamlFile, workDirectory, outputDirectory): # run all yamls: self.run(); + # move the output to storage + generators = self.getGenerators("All") + # now compare to reference + self.process(generators) + + def execute(self, generator, process): - pass + + if generator not in os.listdir(self._outDir): + self.makeDirectory(f"{self._outDir}/{generator}") + if process not in os.listdir(f"{self._outDir}/{generator}"): + self.makeDirectory(f"{self._outDir}/{generator}/{process}") + outDir = f"{self._outDir}/{generator}/{process}" + + genProc = "/"+generator+"/"+process + genProcDir = self._generatorDir+genProc + fileNames = self.getFileNames(genProcDir) + + success = True + for name in fileNames: + theFile = genProcDir+"/"+name + try: + shutil.copy(theFile, outDir) + except: + print(f"{theFile} could not be copied to {outDir}") + success= False + + return success def makeDirectory(self, dirname): # Overwrite directory if it exists @@ -142,11 +173,6 @@ def __init__(self, generator, workDirectory, outputDirectory): # now compare to reference self.process(generators) - def getFileNames(self, directory): - filenames = os.listdir(directory) - filenames =[name for name in filenames if os.path.isfile(os.path.join(directory,name))] - return filenames - def execute(self, generator, process): genProc = "/"+generator+"/"+process newDir = self._generatorDir+genProc @@ -205,3 +231,15 @@ class runSummary(CIUtilsBase): def __init__(self, workDirectory, outputDirectory): super().__init__(workDirectory, outputDirectory) + + print("Extracting the cross sections by reading EDM4HEP files and superposing the differential distributions") + + try: + result = subprocess.run(["eventGenerationSummary", + "-f", f"{self._outDir}}/GenerationSummary.dat", + "-d", "../output", + capture_output=True, check=True) + except CalledProcessError: + print(f"Execution error eventGenerationSummary") + print(result.stdout) + print(result.stderr) diff --git a/python/run.py b/python/run.py index b0402e38..eb02f878 100644 --- a/python/run.py +++ b/python/run.py @@ -9,6 +9,7 @@ from CIUtils import createGeneratorDatacards from CIUtils import checkGeneratorDatacards from CIUtils import runEventGeneration +from CIUtils import runSummary def run(arguments=None): parser = argparse.ArgumentParser( diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fca69770..9d455ef4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,7 +4,6 @@ set(TEST_TIMEOUT 3500) # we split the execution into four steps: set(EXECUTABLE "${PROJECT_SOURCE_DIR}/python/run.py") -set(RUN_SUMMARY "${CMAKE_CURRENT_SOURCE_DIR}/runSummary.sh") # the generators to be run set(generatorList Babayaga KKMC Madgraph Pythia Sherpa Whizard ) @@ -58,7 +57,7 @@ foreach(generator ${generatorList}) endforeach() # third test (after all others have been run: gather the xsections -add_test(NAME xsectionRuns COMMAND ${RUN_SUMMARY} ) +add_test(NAME xsectionRuns COMMAND python3 ${EXECUTABLE} --summary --workDir ../test/ci ) set_tests_properties("xsectionRuns" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} From 70fdde4f575ae5b540db12828ae198be50075607 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 18:28:45 +0100 Subject: [PATCH 26/88] working on last step --- python/CIUtils.py | 93 ++++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/python/CIUtils.py b/python/CIUtils.py index a9e557ad..b86dd98f 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -23,7 +23,7 @@ def __init__(self, workDirectory, outputDirectory): self._referenceDir = os.path.dirname(os.path.realpath(__file__))+"/../test/ref-results" - self._generatorDir = self._workDir+"/Run-Cards" + self._generatorDir = f"{self._workDir}/Run-Cards" def getGenerators(self, generator): if generator == "All": @@ -32,13 +32,27 @@ def getGenerators(self, generator): return [generator] def getProcesses(self, generator): - return os.listdir(self._generatorDir+"/"+generator) + return os.listdir(f"{self._generatorDir}/{generator}") def getFileNames(self, directory): filenames = os.listdir(directory) filenames =[name for name in filenames if os.path.isfile(os.path.join(directory,name))] return filenames + def makeDirectory(self, dirname): + # Overwrite directory if it exists + if not os.path.exists(dirname): + os.makedirs(dirname) + else: + shutil.rmtree(dirname) + os.makedirs(dirname) + + def makeOutputDirectory(self, generator, process): + if generator not in os.listdir(self._outDir): + self.makeDirectory(f"{self._outDir}/{generator}") + if process not in os.listdir(f"{self._outDir}/{generator}"): + self.makeDirectory(f"{self._outDir}/{generator}/{process}") + def process(self, generators): # where do we start from cwd = os.getcwd() @@ -91,23 +105,18 @@ def __init__(self, yamlDirectory, yamlFile, workDirectory, outputDirectory): generators = self.getGenerators("All") # now compare to reference self.process(generators) - def execute(self, generator, process): - if generator not in os.listdir(self._outDir): - self.makeDirectory(f"{self._outDir}/{generator}") - if process not in os.listdir(f"{self._outDir}/{generator}"): - self.makeDirectory(f"{self._outDir}/{generator}/{process}") + self.makeOutputDirectory(generator, process) outDir = f"{self._outDir}/{generator}/{process}" - genProc = "/"+generator+"/"+process - genProcDir = self._generatorDir+genProc + genProcDir = f"{self._generatorDir}/{generator}/{process}" fileNames = self.getFileNames(genProcDir) success = True for name in fileNames: - theFile = genProcDir+"/"+name + theFile = f"{genProcDir}/{name}" try: shutil.copy(theFile, outDir) except: @@ -116,14 +125,6 @@ def execute(self, generator, process): return success - def makeDirectory(self, dirname): - # Overwrite directory if it exists - if not os.path.exists(dirname): - os.makedirs(dirname) - else: - shutil.rmtree(dirname) - os.makedirs(dirname) - def prepareYamls(self, yamlFile): # move the yamls to the work directory # the yaml file is not specified copy all yaml files to the work directory: @@ -174,20 +175,20 @@ def __init__(self, generator, workDirectory, outputDirectory): self.process(generators) def execute(self, generator, process): - genProc = "/"+generator+"/"+process - newDir = self._generatorDir+genProc - refDir = self._referenceDir+genProc + genProc = f"{generator}/{process}" + newDir = f"{self._generatorDir}/{genProc}" + refDir = f"{self._referenceDir}/{genProc}" fileNames = self.getFileNames(refDir) success = True for name in fileNames: - newFile = newDir+"/"+name - refFile = refDir+"/"+name - message = "Generator "+generator+" Process "+process+" File "+name + newFile = f"{newDir}/{name}" + refFile = f"{refDir}/{name}" + message = f"Generator {generator} Process {process} File {name}" if filecmp.cmp(refFile,newFile, shallow=False): - print(message+" identical") + print(f"{message} identical") else: - print(message+" differ") + print(f"{message} differ") success = False return success @@ -206,7 +207,7 @@ def __init__(self, generator, workDirectory, outputDirectory): def execute(self, generator, process): # go to the directory - genProcDir = self._generatorDir+"/"+generator+"/"+process + genProcDir = f"{self._generatorDir}/{generator}/{process}" os.chdir(genProcDir) # retrieve the script scripts = [script for script in os.listdir(genProcDir) if script.startswith("Run_") and script.endswith(".sh")] @@ -217,13 +218,24 @@ def execute(self, generator, process): success = True for script in scripts: try: - result = subprocess.run("./"+script, capture_output=True, check=True) - except CalledProcessError: + result = subprocess.run(f"./{script}", capture_output=True, check=True) + except subprocess.CalledProcessError as e: print(f"Execution error for {generator} in process {process}") - print(result.stdout) - print(result.stderr) + print(e.returncode) + print(e.output) success = False + self.makeOutputDirectory(generator, process) + outDir = f"{self._outDir}/{generator}/{process}" + fileNames = [filename for filename in self.getFileNames(genProcDir) if filename.endswith(".edm4hep")] + for name in fileNames: + theFile = f"{genProcDir}/{name}" + try: + shutil.copy(theFile, outDir) + except: + print(f"{theFile} could not be copied to {outDir}") + success= False + return success class runSummary(CIUtilsBase): @@ -233,13 +245,20 @@ def __init__(self, workDirectory, outputDirectory): super().__init__(workDirectory, outputDirectory) print("Extracting the cross sections by reading EDM4HEP files and superposing the differential distributions") - + # remember where we start from + cwd = os.getcwd() + os.chdir(self._workDir) try: result = subprocess.run(["eventGenerationSummary", - "-f", f"{self._outDir}}/GenerationSummary.dat", - "-d", "../output", + "-f", f"{self._outDir}/GenerationSummary.dat", + "-d", f"{self._outDir}"], capture_output=True, check=True) - except CalledProcessError: + except subprocess.CalledProcessError as e: print(f"Execution error eventGenerationSummary") - print(result.stdout) - print(result.stderr) + print(e.returncode) + print(e.output) + # return tu the starting point + os.chdir(cwd) + + def execute(self, generator, process): + pass From 949fabed6d07a08d84a210dfa1ccd2887af40cb3 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 19:18:22 +0100 Subject: [PATCH 27/88] add option to specify the path to dir with generator directories --- .../src/eventGenerationCollections.cxx | 24 +++++++++---------- .../src/eventGenerationCollections.h | 4 ++-- .../src/eventGenerationSummary.cxx | 17 ++++++++----- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.cxx b/k4GeneratorsConfig/src/eventGenerationCollections.cxx index b6509919..289f1174 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.cxx +++ b/k4GeneratorsConfig/src/eventGenerationCollections.cxx @@ -29,29 +29,29 @@ k4GeneratorsConfig::eventGenerationCollections::operator=(const eventGenerationC return *this; } k4GeneratorsConfig::eventGenerationCollections::~eventGenerationCollections() {} -void k4GeneratorsConfig::eventGenerationCollections::Execute() { +void k4GeneratorsConfig::eventGenerationCollections::Execute(std::string topDir) { // first make the collection - makeCollections(); + makeCollections(topDir); // second order the collection according to the process orderCollections(); } -void k4GeneratorsConfig::eventGenerationCollections::makeCollections() { +void k4GeneratorsConfig::eventGenerationCollections::makeCollections(std::string topDir) { - for (const auto& generators : std::filesystem::directory_iterator("Run-Cards")) { - std::filesystem::path generatorsPath = generators.path(); - if (!std::filesystem::is_directory(generatorsPath)) + for (const auto& generator : std::filesystem::directory_iterator(topDir)) { + std::filesystem::path generatorPath = generator.path(); + if (!std::filesystem::is_directory(generatorPath)) continue; - for (const auto& procs : std::filesystem::directory_iterator(generatorsPath.string())) { - std::filesystem::path processPath = procs.path(); + for (const auto& process : std::filesystem::directory_iterator(generatorPath.string())) { + std::filesystem::path processPath = process.path(); if (!std::filesystem::is_directory(processPath)) continue; k4GeneratorsConfig::xsection* xsec = new k4GeneratorsConfig::xsection(); - for (const auto& files : std::filesystem::directory_iterator(processPath.string())) { - std::filesystem::path filenamePath = files.path(); + for (const auto& filename : std::filesystem::directory_iterator(processPath.string())) { + std::filesystem::path filenamePath = filename.path(); if (!std::filesystem::is_regular_file(filenamePath)) continue; // take care of the total cross section extracted from the EDM4HEP file @@ -61,7 +61,7 @@ void k4GeneratorsConfig::eventGenerationCollections::makeCollections() { xsec->setProcess(processPath.filename().string()); xsec->setFile(filenamePath.string()); // in some cases the generator name is not available, therefore derive from the filename - xsec->setGenerator(generatorsPath.filename().string()); + xsec->setGenerator(generatorPath.filename().string()); std::cout << "Generator " << xsec->Generator() << " has been processed" << std::endl; m_xsectionCollection.push_back(*xsec); if (xsec->isValid()) @@ -84,7 +84,7 @@ void k4GeneratorsConfig::eventGenerationCollections::makeCollections() { diffDist->setFile(filenamePath.string()); diffDist->setSQRTS(xsec->SQRTS()); // in some cases the generator name is not available, therefore derive from the filename - diffDist->setGenerator(generatorsPath.filename().string()); + diffDist->setGenerator(generatorPath.filename().string()); std::cout << "Generator " << diffDist->Generator() << " has been processed for analysisHistos distributions" << std::endl; m_analysisHistosCollection.push_back(*diffDist); diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.h b/k4GeneratorsConfig/src/eventGenerationCollections.h index 897d48c1..7349861f 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.h +++ b/k4GeneratorsConfig/src/eventGenerationCollections.h @@ -14,8 +14,8 @@ class eventGenerationCollections { eventGenerationCollections& operator=(const eventGenerationCollections&); ~eventGenerationCollections(); - void Execute(); - void makeCollections(); + void Execute(std::string); + void makeCollections(std::string); void orderCollections(); bool compareLength(xsection, xsection); bool compareLexical(xsection, xsection); diff --git a/k4GeneratorsConfig/src/eventGenerationSummary.cxx b/k4GeneratorsConfig/src/eventGenerationSummary.cxx index 99fce0a4..ddf84111 100644 --- a/k4GeneratorsConfig/src/eventGenerationSummary.cxx +++ b/k4GeneratorsConfig/src/eventGenerationSummary.cxx @@ -7,12 +7,16 @@ // int main(int argc, char** argv) { + std::string topDir = "Run-Cards"; std::string filename = "GenerationSummary.dat"; std::string dirRoot = "."; std::string fileRoot = "eventGenerationSummary.root"; int c; - while ((c = getopt(argc, argv, "hf:d:r:")) != -1) + while ((c = getopt(argc, argv, "hw:f:d:r:")) != -1) switch (c) { + case 'w': + topDir = optarg; + break; case 'f': filename = optarg; break; @@ -23,11 +27,12 @@ int main(int argc, char** argv) { fileRoot = optarg; break; case 'h': - std::cout << "Usage: xsectionSummary -h -f filename" << std::endl; + std::cout << "Usage: xsectionSummary -h -w workdir -f filename -d rootdir -r rootfile" << std::endl; std::cout << "-h: print this help" << std::endl; - std::cout << "-f filename: print the summary information to this file" << std::endl; - std::cout << "-d dirname: write the RootTree and figures to this directory" << std::endl; - std::cout << "-r filename: write the RootTree to this file" << std::endl; + std::cout << "-w dirname: top directory to start the search for generators and processes (default: Run-Cards)" << std::endl; + std::cout << "-f filename: print the summary information to this file (default: GenerationSummary.dat)" << std::endl; + std::cout << "-d rootdir: write the RootTree and figures to this directory (default: .)" << std::endl; + std::cout << "-r footfile: write the RootTree to this file (default: eventGenerationSummary.root)" << std::endl; exit(0); default: exit(0); @@ -36,7 +41,7 @@ int main(int argc, char** argv) { // instantiate the collection as pointer k4GeneratorsConfig::eventGenerationCollections* evgenColls = new k4GeneratorsConfig::eventGenerationCollections(); // execute the gathering of information including detailed output - evgenColls->Execute(); + evgenColls->Execute(topDir); // do the root analysis evgenColls->Write2Root(dirRoot, fileRoot); // print the summary on screen From cdcf895c00b5dc88b73e76b5068d6595a73dc13b Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 19:21:36 +0100 Subject: [PATCH 28/88] make evgenSummary safer --- python/CIUtils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/CIUtils.py b/python/CIUtils.py index b86dd98f..d2ca74a5 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -250,6 +250,7 @@ def __init__(self, workDirectory, outputDirectory): os.chdir(self._workDir) try: result = subprocess.run(["eventGenerationSummary", + "-w", f"{self._generatorDir", "-f", f"{self._outDir}/GenerationSummary.dat", "-d", f"{self._outDir}"], capture_output=True, check=True) From cbfd32718fa5a194bc0e4722dc55d32659ae2acd Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 19:50:55 +0100 Subject: [PATCH 29/88] cleanup for final test --- python/CIUtils.py | 50 +++---- python/run.py | 38 ++++-- test/CMakeLists.txt | 8 +- test/check.sh | 162 ----------------------- test/checkGeneratorDatacards.sh | 71 ---------- test/createGeneratorDatacardsFromYaml.sh | 93 ------------- test/runEventGeneration.sh | 62 --------- test/runSummary.sh | 16 --- 8 files changed, 54 insertions(+), 446 deletions(-) delete mode 100755 test/check.sh delete mode 100755 test/checkGeneratorDatacards.sh delete mode 100755 test/createGeneratorDatacardsFromYaml.sh delete mode 100755 test/runEventGeneration.sh delete mode 100755 test/runSummary.sh diff --git a/python/CIUtils.py b/python/CIUtils.py index d2ca74a5..d2e0994d 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -10,20 +10,20 @@ class CIUtilsBase(ABC): """Generator Generator Datacards""" - def __init__(self, workDirectory, outputDirectory): + def __init__(self, args): # consistent processing of names - self._workDir = os.path.dirname(os.path.realpath(__file__))+"/"+workDirectory - if workDirectory.startswith("/"): + self._workDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.workDir + if args.workDir.startswith("/"): self._workDir = args.workDir - self._outDir = os.path.dirname(os.path.realpath(__file__))+"/"+outputDirectory - if outputDirectory.startswith("/"): - self.outdir = outputDirectory + self._outDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.outputDir + if args.outputDir.startswith("/"): + self.outdir = args.outputDir - self._referenceDir = os.path.dirname(os.path.realpath(__file__))+"/../test/ref-results" + self._referenceDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.refDir - self._generatorDir = f"{self._workDir}/Run-Cards" + self._generatorDir = f"{self._workDir}/{args.generatorDirName}" def getGenerators(self, generator): if generator == "All": @@ -77,23 +77,23 @@ def execute(self, generator, process): class createGeneratorDatacards(CIUtilsBase): """Generator Generator Datacards""" - def __init__(self, yamlDirectory, yamlFile, workDirectory, outputDirectory): - super().__init__(workDirectory, outputDirectory) + def __init__(self, args): + super().__init__(args) # specific members for DC creation - self._yamlDir = os.path.dirname(os.path.realpath(__file__))+"/"+yamlDirectory - if yamlDirectory.startswith("/"): - self._yamlDir = yamlDirectory + self._yamlDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.yamlDir + if args.yamlDir.startswith("/"): + self._yamlDir = args.yamlDir self._yamlFiles = [] # make the directory for the work - self.makeDirectory(workDirectory) + self.makeDirectory(args.workDir) # make the output directory for the output - self.makeDirectory(outputDirectory) + self.makeDirectory(args.outputDir) # prepare yamls: - self.prepareYamls(yamlFile); + self.prepareYamls(args.yamlFile); # prepare the Sqrts files: self.prepareECMS(); @@ -165,11 +165,11 @@ def run(self): class checkGeneratorDatacards(CIUtilsBase): """Check Generator Datacards""" - def __init__(self, generator, workDirectory, outputDirectory): - super().__init__(workDirectory, outputDirectory) + def __init__(self, args): + super().__init__(args) # retrieve all generators in the workdirector: - generators = self.getGenerators(generator) + generators = self.getGenerators(args.generator) # now compare to reference self.process(generators) @@ -196,11 +196,11 @@ def execute(self, generator, process): class runEventGeneration(CIUtilsBase): """Run Event Generation""" - def __init__(self, generator, workDirectory, outputDirectory): - super().__init__(workDirectory, outputDirectory) + def __init__(self, args): + super().__init__(args) # retrieve all generators in the workdirector: - generators = self.getGenerators(generator) + generators = self.getGenerators(args.generator) # now compare to reference self.process(generators) @@ -241,8 +241,8 @@ def execute(self, generator, process): class runSummary(CIUtilsBase): """Run summary of all processes""" - def __init__(self, workDirectory, outputDirectory): - super().__init__(workDirectory, outputDirectory) + def __init__(self, args): + super().__init__(args) print("Extracting the cross sections by reading EDM4HEP files and superposing the differential distributions") # remember where we start from @@ -250,7 +250,7 @@ def __init__(self, workDirectory, outputDirectory): os.chdir(self._workDir) try: result = subprocess.run(["eventGenerationSummary", - "-w", f"{self._generatorDir", + "-w", f"{self._generatorDir}", "-f", f"{self._outDir}/GenerationSummary.dat", "-d", f"{self._outDir}"], capture_output=True, check=True) diff --git a/python/run.py b/python/run.py index eb02f878..aacd2985 100644 --- a/python/run.py +++ b/python/run.py @@ -25,67 +25,79 @@ def run(arguments=None): parser.add_argument( "--create", action='store_true', - help="create the generator datacards from the yaml files", + help="create the generator datacards from the yaml files" ) parser.add_argument( "--yamlDir", type=str, default="../examples", - help="directory where the yamlFiles are located default: k4GeneratorsConfig/examples", + help="path to the yamlFiles (default: k4GeneratorsConfig/examples)" ) parser.add_argument( "--yamlFile", type=str, default="None", - help="name of file to be processed default: all found in the yamlDir", + help="name of the ONLY file to be processed (default: all are processed)" ) parser.add_argument( "--check", action='store_true', - help="check the generator datacards with respect to the reference in k4GeneratorsConfig/test/ref-results", + help="check the generator datacards with respect to the reference" + ) + parser.add_argument( + "--refDir", + type=str, + default="../test/ref-results", + help="path to the reference files (default: k4GeneratorsConfig/test/ref-results)" ) parser.add_argument( "--generator", type=str, default="All", - help="generator to be run, default: All", + help="generator to be run (default: All processed)" ) parser.add_argument( "--run", action='store_true', - help="run the event generation", + help="run the event generation step" ) parser.add_argument( "--summary", action='store_true', - help="compare the results of the event generation process by process", + help="compare the results of the event generation process by process and produce summary output in outputDir" ) parser.add_argument( "--workDir", type=str, default="../test/work", - help="directory where the Work is performed default: k4GeneratorsConfig/test/work", + help="path to work directory (default: k4GeneratorsConfig/test/work)" ) parser.add_argument( "--outputDir", type=str, default="../test/output", - help="directory where the output is stored default: k4GeneratorsConfig/test/output", + help="path to the output (default: k4GeneratorsConfig/test/output)" + ) + parser.add_argument( + "--generatorDirName", + type=str, + default="Run-Cards", + help="relative path to the Generator directories (default: outputDir/generatorDirName)" ) args = parser.parse_args(arguments) if args.create: - createGeneratorDatacards(args.yamlDir, args.yamlFile, args.workDir, args.outputDir) + createGeneratorDatacards(args) if args.check: - checkGeneratorDatacards(args.generator, args.workDir, args.outputDir) + checkGeneratorDatacards(args) if args.run: - runEventGeneration(args.generator, args.workDir, args.outputDir) + runEventGeneration(args) if args.summary: - runSummary(args.workDir, args.outputDir) + runSummary(args) if __name__ == "__main__": run() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9d455ef4..a3fd4738 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,7 +15,7 @@ foreach(generator ${generatorList}) endforeach() # add the first test: setting up the yaml files -add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --create --workDir ../test/ci) +add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --create ) set_tests_properties("createGeneratorDatacards" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -25,7 +25,7 @@ set_tests_properties("createGeneratorDatacards" # add the second test: comparing the yaml files to the reference files function(check_generator_datacards generator) - add_test(NAME checkGeneratorDatacards_${generator} COMMAND python3 ${EXECUTABLE} --check --workDir ../test/ci --generator ${generator}) + add_test(NAME checkGeneratorDatacards_${generator} COMMAND python3 ${EXECUTABLE} --check --generator ${generator} ) set_tests_properties("checkGeneratorDatacards_${generator}" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -42,7 +42,7 @@ endforeach() # third test running the generators (define the function first then call it function(add_generator_run name generator) - add_test(NAME ${name}_${generator} COMMAND python3 ${EXECUTABLE} --run --workDir ../test/ci --generator ${generator} ) + add_test(NAME ${name}_${generator} COMMAND python3 ${EXECUTABLE} --run --generator ${generator} ) set_tests_properties(${name}_${generator} PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -57,7 +57,7 @@ foreach(generator ${generatorList}) endforeach() # third test (after all others have been run: gather the xsections -add_test(NAME xsectionRuns COMMAND python3 ${EXECUTABLE} --summary --workDir ../test/ci ) +add_test(NAME xsectionRuns COMMAND python3 ${EXECUTABLE} --summary ) set_tests_properties("xsectionRuns" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/test/check.sh b/test/check.sh deleted file mode 100755 index 211eb580..00000000 --- a/test/check.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env bash - -set -e - -shopt -s expand_aliases -source ../setup.sh - -# decode command line options - -OPTSTRING=":bhr" - -runEvgen="true" -runReducedEvgen="false" -while getopts ${OPTSTRING} opt; do - case ${opt} in -# x) -# echo "Option -x was triggered, Argument: ${OPTARG}" -# ;; - b) - echo "Option -b was triggered, event generation step will not be run" - runEvgen="false" - ;; - r) - echo "Option -r was triggered, event generation step will be run only for one process per yaml file" - runReducedEvgen="true" - ;; - h) - echo "Arguments are:" - echo "-h for help" - echo "-b to block the event generation step fully" - echo "-r to reduce the number of event generation steps to 1 per yaml file" - exit 0 - ;; - ?) - echo "Invalid option: -${OPTARG}." - exit 1 - ;; - esac -done - -# STEP 0 prepare the input - -mkdir -p ci-setups - -CWD=${PWD} -REFDIR="${PWD}/ref-results" -EXAMPLEDIR="${PWD}/../examples" - -cp "$EXAMPLEDIR"/*yaml ci-setups -cd ci-setups - -function checkFile() { - local generator="$1" - local refgenerator="$(basename "$generator")" - local procname="$2" - local outFile="$3" - if [[ -e "$REFDIR/$refgenerator/$procname/$outFile" ]]; then - if diff "$REFDIR/$refgenerator/$procname/$outFile" "$PWD/$generator/$procname/$outFile" &> /dev/null; then - echo "Process " $procname : "Files are identical for file" $outFile - else - echo "Process " $procname "Files are different for file" $outFile - diff "$REFDIR/$refgenerator/$procname/$outFile" "$PWD/$generator/$procname/$outFile" - exit 1 - fi - else - echo "Did not find $outFile. Not checking!" - fi -} - -function checkOutputs() { - for generator in */*; do - [[ -d "$generator" ]] || continue - echo "Checking $generator" - for outFile in "$PWD/$generator"/*/*; do - [[ -f "$outFile" ]] || continue - local fullpath="$(dirname "$outFile")" - local procname="$(basename "$fullpath")" - checkFile "$generator" "$procname" "$(basename "$outFile")" - done - done -} - -function processYAML() { - local yamlFile="$1" - local filename="${yamlFile%.yaml}" - - mkdir -p "test-$filename" - cd "test-$filename" - echo "Processing file: $yamlFile" - k4GeneratorsConfig "../$yamlFile" - checkOutputs - cd .. -} - -# STEP 1: check the input - -for yamlFile in *.yaml; do - processYAML "$yamlFile" -done - -# STEP 2: run the generators - -function processRun() { - isOK=0 - topDir=${PWD} - thepath="$(dirname "$1")" - runfile="$(basename "$1")" - echo Running $runfile in $thepath - # move to the directory where the script is located - cd $thepath - # run the script - k4ConfigRanGen=0 - ./$runfile - if [[ $? -eq 0 ]]; then - echo k4GeneratorsConfig::Event generation succssful for $runfile in directory $thepath - k4ConfigRanGen=1 - else - k4ConfigRanGen=0 - fi - cd $topDir -} - - -# STEP 3 now we can go through the .sh and run them -if [[ $runEvgen = "true" ]]; then - counter=0 - counterRan=0 - for yamlDir in test-*; do - cd $yamlDir - echo $PWD is the current directory - firstProcessRead="false" - lastGenerator="murks" - for aRunScript in */*/*/*.sh; do - proc="$(dirname "$aRunScript")" - generatorWithPath="$(dirname "$proc")" - if [[ $runReducedEvgen = "true" && $firstProcessRead = "true" && $generatorWithPath = "$lastGenerator" ]]; then - continue - fi - firstProcessRead="true" - lastGenerator=$generatorWithPath - if [[ $k4ConfigRanGen -eq 1 ]]; then - counterRan=$((counterRan+1)) - fi - counter=$((counter+1)) - processRun "$aRunScript" - done - cd .. - done - echo k4GeneratorsConfig::EvGen Summary - echo tried $counter generator runs - echo with $counterRan successful executions - - # STEP 4 - # since we have run the generators we can also do the summary now: - echo Extracting the cross sections by reading EDM4HEP files - ${K4GENERATORSCONFIG}/xsectionSummary -f ${CWD}/xsectionSummary.dat -fi - -# STEP 4: clean up -rm -r "${CWD}/ci-setups" - -exit 0 diff --git a/test/checkGeneratorDatacards.sh b/test/checkGeneratorDatacards.sh deleted file mode 100755 index 126dd6cb..00000000 --- a/test/checkGeneratorDatacards.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -set -e -shopt -s expand_aliases -source ../setup.sh - -EXAMPLEDIR="${PWD}/../examples" -REFDIR="${PWD}/ref-results" - -OPTSTRING="g:h" -while getopts ${OPTSTRING} opt; do - case ${opt} in - g) - echo "Option -g was triggered, Argument: ${OPTARG}" - GENERATOR="${OPTARG}" - echo $0 will only check $GENERATOR - ;; - h) - echo "Arguments are:" - echo "-h for help" - echo "-g GENERATOR " - exit 0 - ;; - ?) - echo "Invalid option: -${OPTARG}." - exit 1 - ;; - esac -done - -# if the generator is not requested explicitely, we define a wildcard (=all) -if [ -z $GENERATOR ]; then - GENERATOR=* -fi - -CWD=${PWD} -cd ci - -function checkFile() { - local generator="$1" - local refgenerator="$(basename "$generator")" - local procname="$2" - local outFile="$3" - if [[ -e "$REFDIR/$refgenerator/$procname/$outFile" ]]; then - if diff "$REFDIR/$refgenerator/$procname/$outFile" "$PWD/$generator/$procname/$outFile" &> /dev/null; then - echo "Process " $procname : "Files are identical for file" $outFile - else - echo "Process " $procname "Files are different for file" $outFile - diff "$REFDIR/$refgenerator/$procname/$outFile" "$PWD/$generator/$procname/$outFile" - exit 1 - fi - else - echo "Did not find $outFile. Not checking!" - fi -} - -# loop over the generators requested: -for generator in */$GENERATOR; do - [[ -d "$generator" ]] || continue - for outFile in "$PWD/$generator"/*/*; do - [[ -f "$outFile" ]] || continue - fullpath="$(dirname "$outFile")" - procname="$(basename "$fullpath")" - checkFile "$generator" "$procname" "$(basename "$outFile")" - # copy the files to the output - mkdir -p "${CWD}"/output/"$generator"/"$procname"/ - cp "$outFile" "${CWD}"/output/"$generator"/"$procname"/"$(basename "$outFile")" - done -done - -exit 0 diff --git a/test/createGeneratorDatacardsFromYaml.sh b/test/createGeneratorDatacardsFromYaml.sh deleted file mode 100755 index e29e5de4..00000000 --- a/test/createGeneratorDatacardsFromYaml.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash - -set -e -shopt -s expand_aliases -source ../setup.sh - -YAMLDIR="${PWD}/../examples" - -OPTSTRING="d:h" -while getopts ${OPTSTRING} opt; do - case ${opt} in - d) - echo "Option -d was triggered, Argument: ${OPTARG}" - YAMLDIR="${OPTARG}" - echo Searching for yaml files in directory $YAMLDIR - ;; - f) - echo "Option -f was triggered, Argument: ${OPTARG}" - YAMLFILE="${OPTARG}" - echo only $YAMLFILE will be processed - ;; - h) - echo "Arguments are:" - echo "-h for help" - echo "-d YAMLDIRECTORY " - echo "-f YAMLFILE " - exit 0 - ;; - ?) - echo "Invalid option: -${OPTARG}." - exit 1 - ;; - esac -done - -CWD=${PWD} -# only create the directory if it does not exist yet -if [[ ! -d ${CWD}/ci ]]; then - mkdir -p ${CWD}/ci -else - # clean up the content if necessary - rm -Rf ci/* -fi -# only create the directory if it does not exist yet -if [[ ! -d ${CWD}/output ]]; then - mkdir -p ${CWD}/output -else - # clean up the content if necessary - rm -Rf output/* -fi - - -# only copy if the file does not exist yet: -for yamlFileWithPath in "$YAMLDIR"/*.yaml; do - yamlFile="$(basename "$yamlFileWithPath")" - if [[ ! -f ci/"$yamlFile" ]]; then - echo copying $yamlFileWithPath to ci - cp -f "$yamlFileWithPath" ci - fi -done - -# check whether ecms.dat files are available: -for ecmsFileWithPath in "$YAMLDIR"/"ecms"*.dat; do - ecmsFile="$(basename "$ecmsFileWithPath")" - if [[ -f "$ecmsFileWithPath" && ! -f ci/"$ecmsFile" && "$ecmsFile" != "ecms.dat" ]]; then - echo copying $ecmsFileWithPath to ci - cp -f "$ecmsFileWithPath" ci - fi -done - -cd ci - -# STEP 1: check the input - -function processYAML() { - local yamlFile="$1" - local filename="${yamlFile%.yaml}" - - echo "Processing file: $yamlFile" - if [[ ! -f ecms"$filename".dat ]]; then - k4GeneratorsConfig "$yamlFile" - else - k4GeneratorsConfig "$yamlFile" --ecmsFile ecms"$filename".dat - fi -} - - -for yamlFile in *.yaml; do - processYAML "$yamlFile" -done - - -exit 0 diff --git a/test/runEventGeneration.sh b/test/runEventGeneration.sh deleted file mode 100755 index 21013140..00000000 --- a/test/runEventGeneration.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -set -e - -shopt -s expand_aliases -echo the KEY4HEPSTACK is $KEY4HEP_STACK - -# decode command line options - -OPTSTRING="g:h" -GENERATOR="" -while getopts ${OPTSTRING} opt; do - case ${opt} in - g) - echo "Option -g was triggered, event generation step will be run only for ${OPTARG}" - GENERATOR="$OPTARG" - ;; - h) - echo "Arguments are:" - echo "-h for help" - echo "-g GENERATORNAME only run this generator" - exit 0 - ;; - ?) - echo "Invalid option: -${OPTARG}." - exit 1 - ;; - esac -done - -# if the generator is not requested explicitely, we define a wildcard (=all) -if [ -z $GENERATOR ]; then - GENERATOR=* -fi - -CWD=${PWD} -cd ci - -# run a generator: argument is a pathname -function processRun() { - topDir=${PWD} - thepath="$(dirname "$1")" - runfile="$(basename "$1")" - echo Running $runfile in $thepath - # move to the directory where the script is located - cd $thepath - # run the script - ./$runfile - if [[ $? -eq 0 ]]; then - echo k4GeneratorsConfig::Event generation successful for $runfile in directory $thepath - fi - cd $topDir -} - -file_pattern="*/${GENERATOR}/*/*.sh" -if ls $file_pattern 1> /dev/null 2>&1; then - for aRunScript in ${file_pattern}; do - processRun "$aRunScript" - done -fi - -exit 0 diff --git a/test/runSummary.sh b/test/runSummary.sh deleted file mode 100755 index be75a914..00000000 --- a/test/runSummary.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -e - -shopt -s expand_aliases -source ../setup.sh - -CWD=${PWD} -cd ci - -#STEP 4 -#since we have run the generators we can also do the summary now: -echo "Extracting the cross sections by reading EDM4HEP files and superposing the differential distributions" -eventGenerationSummary -f ${CWD}/output/GenerationSummary.dat -d ../output - -exit 0 From 592f7f554ae4eee030c4f2230ccae9096ac0ce67 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 20:47:15 +0100 Subject: [PATCH 30/88] clang --- k4GeneratorsConfig/src/eventGenerationSummary.cxx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/k4GeneratorsConfig/src/eventGenerationSummary.cxx b/k4GeneratorsConfig/src/eventGenerationSummary.cxx index ddf84111..eb68c032 100644 --- a/k4GeneratorsConfig/src/eventGenerationSummary.cxx +++ b/k4GeneratorsConfig/src/eventGenerationSummary.cxx @@ -29,8 +29,10 @@ int main(int argc, char** argv) { case 'h': std::cout << "Usage: xsectionSummary -h -w workdir -f filename -d rootdir -r rootfile" << std::endl; std::cout << "-h: print this help" << std::endl; - std::cout << "-w dirname: top directory to start the search for generators and processes (default: Run-Cards)" << std::endl; - std::cout << "-f filename: print the summary information to this file (default: GenerationSummary.dat)" << std::endl; + std::cout << "-w dirname: top directory to start the search for generators and processes (default: Run-Cards)" + << std::endl; + std::cout << "-f filename: print the summary information to this file (default: GenerationSummary.dat)" + << std::endl; std::cout << "-d rootdir: write the RootTree and figures to this directory (default: .)" << std::endl; std::cout << "-r footfile: write the RootTree to this file (default: eventGenerationSummary.root)" << std::endl; exit(0); From 7045b04affab7dcca4578738c363eff121438329 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 20:58:05 +0100 Subject: [PATCH 31/88] test return code --- python/CIUtils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/CIUtils.py b/python/CIUtils.py index d2e0994d..005e1162 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -254,6 +254,8 @@ def __init__(self, args): "-f", f"{self._outDir}/GenerationSummary.dat", "-d", f"{self._outDir}"], capture_output=True, check=True) + if result.returncode != 0: + sys.exit("At least one process failed") except subprocess.CalledProcessError as e: print(f"Execution error eventGenerationSummary") print(e.returncode) From 05376384880be6ab6abb96e67f1dff90c1863015 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 20:59:24 +0100 Subject: [PATCH 32/88] test return code --- python/CIUtils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/CIUtils.py b/python/CIUtils.py index 005e1162..081ec200 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -260,6 +260,8 @@ def __init__(self, args): print(f"Execution error eventGenerationSummary") print(e.returncode) print(e.output) + sys.exit("Exception thrown by eventGenerationSummary") + # return tu the starting point os.chdir(cwd) From 164defd936383a7d137dca573211fba71afe7ee5 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 21:24:58 +0100 Subject: [PATCH 33/88] output formatting --- python/CIUtils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/CIUtils.py b/python/CIUtils.py index 081ec200..4f8e0250 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -254,12 +254,10 @@ def __init__(self, args): "-f", f"{self._outDir}/GenerationSummary.dat", "-d", f"{self._outDir}"], capture_output=True, check=True) - if result.returncode != 0: - sys.exit("At least one process failed") except subprocess.CalledProcessError as e: print(f"Execution error eventGenerationSummary") - print(e.returncode) - print(e.output) + print(e.returncode.decode("utf-8")) + print(e.output.decode("utf-8")) sys.exit("Exception thrown by eventGenerationSummary") # return tu the starting point From e7af1182f22ac526975f3e4f960ad4d28ad401b6 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sat, 28 Mar 2026 21:41:13 +0100 Subject: [PATCH 34/88] output formatting --- python/CIUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/CIUtils.py b/python/CIUtils.py index 4f8e0250..eddc3498 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -256,7 +256,7 @@ def __init__(self, args): capture_output=True, check=True) except subprocess.CalledProcessError as e: print(f"Execution error eventGenerationSummary") - print(e.returncode.decode("utf-8")) + print(e.returncode) print(e.output.decode("utf-8")) sys.exit("Exception thrown by eventGenerationSummary") From 7a190fd5f8123813466b549609ca3962aae337bb Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sun, 29 Mar 2026 18:18:28 +0200 Subject: [PATCH 35/88] allow multiple yamlfiles on input --- python/CIUtils.py | 28 +++++++++++++++++----------- python/run.py | 7 ++++--- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/python/CIUtils.py b/python/CIUtils.py index eddc3498..c2b7c503 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -9,16 +9,16 @@ from main import main class CIUtilsBase(ABC): - """Generator Generator Datacards""" + """Base class for all k4GeneratorConfig operations""" def __init__(self, args): # consistent processing of names self._workDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.workDir - if args.workDir.startswith("/"): + if args.workDir.startswith('/'): self._workDir = args.workDir self._outDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.outputDir - if args.outputDir.startswith("/"): + if args.outputDir.startswith('/'): self.outdir = args.outputDir self._referenceDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.refDir @@ -82,7 +82,7 @@ def __init__(self, args): # specific members for DC creation self._yamlDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.yamlDir - if args.yamlDir.startswith("/"): + if args.yamlDir.startswith('/'): self._yamlDir = args.yamlDir self._yamlFiles = [] @@ -93,7 +93,7 @@ def __init__(self, args): self.makeDirectory(args.outputDir) # prepare yamls: - self.prepareYamls(args.yamlFile); + self.prepareYamls(args.yamlFiles); # prepare the Sqrts files: self.prepareECMS(); @@ -125,17 +125,23 @@ def execute(self, generator, process): return success - def prepareYamls(self, yamlFile): + def prepareYamls(self, yamlFiles): # move the yamls to the work directory # the yaml file is not specified copy all yaml files to the work directory: - if yamlFile == "None": + if len(yamlFiles) == 0: self._yamlFiles = [filename for filename in os.listdir(self._yamlDir) if filename.endswith('.yaml')] else: - self._yamlFiles =[self.yamlDir+"/"+yamlFile] + # check that all files end with .yaml + if not all([yamlfile.endswith('.yaml') for yamlfile in yamlFiles]): + sys.exit(f"prepareYamls::unknown file extension in input list: {yamlFiles}\nExpect .yaml") + self._yamlFiles =[self._yamlDir+"/"+yamlFile for yamlFile in yamlFiles] # copy the yaml files for filename in self._yamlFiles: - shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) + try: + shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) + except FileNotFoundError as e: + sys.exit(f"prepareYamls::file not found: {e.filename}") def prepareECMS(self): # move the ECMS files to the work directory @@ -210,7 +216,7 @@ def execute(self, generator, process): genProcDir = f"{self._generatorDir}/{generator}/{process}" os.chdir(genProcDir) # retrieve the script - scripts = [script for script in os.listdir(genProcDir) if script.startswith("Run_") and script.endswith(".sh")] + scripts = [script for script in os.listdir(genProcDir) if script.startswith('Run_') and script.endswith('.sh')] if len(scripts) != 1: print(f"Found more than one script for generator {generator} with process {process}") return False @@ -227,7 +233,7 @@ def execute(self, generator, process): self.makeOutputDirectory(generator, process) outDir = f"{self._outDir}/{generator}/{process}" - fileNames = [filename for filename in self.getFileNames(genProcDir) if filename.endswith(".edm4hep")] + fileNames = [filename for filename in self.getFileNames(genProcDir) if filename.endswith('.edm4hep')] for name in fileNames: theFile = f"{genProcDir}/{name}" try: diff --git a/python/run.py b/python/run.py index aacd2985..67208f63 100644 --- a/python/run.py +++ b/python/run.py @@ -34,10 +34,11 @@ def run(arguments=None): help="path to the yamlFiles (default: k4GeneratorsConfig/examples)" ) parser.add_argument( - "--yamlFile", + "--yamlFiles", type=str, - default="None", - help="name of the ONLY file to be processed (default: all are processed)" + nargs="+", + default="", + help="yamlfiles to be processed (default: all are processed)" ) parser.add_argument( "--check", From ac97ea857f859ed82336b198d8dca7850ed76ea2 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sun, 29 Mar 2026 18:30:26 +0200 Subject: [PATCH 36/88] simpler directory logic --- python/CIUtils.py | 4 ++-- python/run.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/CIUtils.py b/python/CIUtils.py index c2b7c503..e095bc8e 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -13,11 +13,11 @@ class CIUtilsBase(ABC): def __init__(self, args): # consistent processing of names - self._workDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.workDir + self._workDir = os.getcwd()+"/"+args.workDir if args.workDir.startswith('/'): self._workDir = args.workDir - self._outDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.outputDir + self._outDir = os.getcwd()+"/"+args.outputDir if args.outputDir.startswith('/'): self.outdir = args.outputDir diff --git a/python/run.py b/python/run.py index 67208f63..0d72bbef 100644 --- a/python/run.py +++ b/python/run.py @@ -70,20 +70,20 @@ def run(arguments=None): parser.add_argument( "--workDir", type=str, - default="../test/work", - help="path to work directory (default: k4GeneratorsConfig/test/work)" + default="work", + help="path to work directory (default: ./work)" ) parser.add_argument( "--outputDir", type=str, - default="../test/output", - help="path to the output (default: k4GeneratorsConfig/test/output)" + default="output", + help="path to the output (default: ./output)" ) parser.add_argument( "--generatorDirName", type=str, default="Run-Cards", - help="relative path to the Generator directories (default: outputDir/generatorDirName)" + help="relative path to the Generator directories in outputDir (default: outputDir/Run-Cards)" ) args = parser.parse_args(arguments) From fc5f30f79eb1ba6b8e3b438a95ecca2f44adf5f1 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sun, 29 Mar 2026 20:12:14 +0200 Subject: [PATCH 37/88] better naming for classes and args --- python/CIUtils.py | 8 ++++---- python/run.py | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/python/CIUtils.py b/python/CIUtils.py index e095bc8e..d24d81e3 100644 --- a/python/CIUtils.py +++ b/python/CIUtils.py @@ -74,7 +74,7 @@ def process(self, generators): def execute(self, generator, process): pass -class createGeneratorDatacards(CIUtilsBase): +class makeGeneratorDatacards(CIUtilsBase): """Generator Generator Datacards""" def __init__(self, args): @@ -199,7 +199,7 @@ def execute(self, generator, process): return success -class runEventGeneration(CIUtilsBase): +class generate(CIUtilsBase): """Run Event Generation""" def __init__(self, args): @@ -244,8 +244,8 @@ def execute(self, generator, process): return success -class runSummary(CIUtilsBase): - """Run summary of all processes""" +class summary(CIUtilsBase): + """Make a summary of all processes""" def __init__(self, args): super().__init__(args) diff --git a/python/run.py b/python/run.py index 0d72bbef..ae000fe5 100644 --- a/python/run.py +++ b/python/run.py @@ -6,10 +6,10 @@ import argparse import textwrap -from CIUtils import createGeneratorDatacards +from CIUtils import makeGeneratorDatacards from CIUtils import checkGeneratorDatacards -from CIUtils import runEventGeneration -from CIUtils import runSummary +from CIUtils import generate +from CIUtils import summary def run(arguments=None): parser = argparse.ArgumentParser( @@ -23,9 +23,9 @@ def run(arguments=None): ), ) parser.add_argument( - "--create", + "--make", action='store_true', - help="create the generator datacards from the yaml files" + help="make the generator datacards from the yaml files" ) parser.add_argument( "--yamlDir", @@ -58,9 +58,9 @@ def run(arguments=None): help="generator to be run (default: All processed)" ) parser.add_argument( - "--run", + "--generate", action='store_true', - help="run the event generation step" + help="run the event generation" ) parser.add_argument( "--summary", @@ -88,17 +88,17 @@ def run(arguments=None): args = parser.parse_args(arguments) - if args.create: - createGeneratorDatacards(args) + if args.make: + makeGeneratorDatacards(args) if args.check: checkGeneratorDatacards(args) - if args.run: - runEventGeneration(args) + if args.generate: + generate(args) if args.summary: - runSummary(args) + summary(args) if __name__ == "__main__": run() From 8e03128232a3cd787c556ba48fe4939832630500 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sun, 29 Mar 2026 20:26:34 +0200 Subject: [PATCH 38/88] naming improvements --- python/{run.py => k4GeneratorsConfig.py} | 4 ++-- setup.csh | 2 +- setup.sh | 2 +- setup.zsh | 2 +- test/CMakeLists.txt | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) rename python/{run.py => k4GeneratorsConfig.py} (97%) diff --git a/python/run.py b/python/k4GeneratorsConfig.py similarity index 97% rename from python/run.py rename to python/k4GeneratorsConfig.py index ae000fe5..0bece642 100644 --- a/python/run.py +++ b/python/k4GeneratorsConfig.py @@ -11,7 +11,7 @@ from CIUtils import generate from CIUtils import summary -def run(arguments=None): +def k4GeneratorsConfig(arguments=None): parser = argparse.ArgumentParser( prog="k4GeneratorsConfig", formatter_class=argparse.RawDescriptionHelpFormatter, @@ -101,5 +101,5 @@ def run(arguments=None): summary(args) if __name__ == "__main__": - run() + k4GeneratorsConfig() diff --git a/setup.csh b/setup.csh index efb61d56..f11a5bc5 100755 --- a/setup.csh +++ b/setup.csh @@ -42,4 +42,4 @@ if (! -d "${K4GeneratorsConfigDir}/install/") then endif # Set executable -alias k4GeneratorsConfigRun "python3 ${K4GeneratorsConfigDir}/python/run.py" +alias k4GeneratorsConfigRun "python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" diff --git a/setup.sh b/setup.sh index bd05615f..599ffac6 100755 --- a/setup.sh +++ b/setup.sh @@ -30,4 +30,4 @@ fi # Set executable alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/main.py" -alias k4GeneratorsConfigRun="python3 ${K4GeneratorsConfigDir}/python/run.py" +alias k4GeneratorsConfigRun="python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" diff --git a/setup.zsh b/setup.zsh index fe5d9f9b..eb2df7ae 100755 --- a/setup.zsh +++ b/setup.zsh @@ -30,4 +30,4 @@ fi # Set executable alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/main.py" -alias k4GeneratorsConfigRun="python3 ${K4GeneratorsConfigDir}/python/run.py" +alias k4GeneratorsConfigRun="python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a3fd4738..dafcb7fe 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,7 +3,7 @@ set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}") set(TEST_TIMEOUT 3500) # we split the execution into four steps: -set(EXECUTABLE "${PROJECT_SOURCE_DIR}/python/run.py") +set(EXECUTABLE "${PROJECT_SOURCE_DIR}/python/k4GeneratorsConfig.py") # the generators to be run set(generatorList Babayaga KKMC Madgraph Pythia Sherpa Whizard ) @@ -15,7 +15,7 @@ foreach(generator ${generatorList}) endforeach() # add the first test: setting up the yaml files -add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --create ) +add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --make ) set_tests_properties("createGeneratorDatacards" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -42,7 +42,7 @@ endforeach() # third test running the generators (define the function first then call it function(add_generator_run name generator) - add_test(NAME ${name}_${generator} COMMAND python3 ${EXECUTABLE} --run --generator ${generator} ) + add_test(NAME ${name}_${generator} COMMAND python3 ${EXECUTABLE} --generate --generator ${generator} ) set_tests_properties(${name}_${generator} PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} From c40b1794d468eb3154ac3f2c9639e27f2325af70 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sun, 29 Mar 2026 21:18:28 +0200 Subject: [PATCH 39/88] improved readability and rigorous comparison check --- python/{CIUtils.py => Production.py} | 30 ++++++++++++++++++++++------ python/k4GeneratorsConfig.py | 8 ++++---- 2 files changed, 28 insertions(+), 10 deletions(-) rename python/{CIUtils.py => Production.py} (89%) diff --git a/python/CIUtils.py b/python/Production.py similarity index 89% rename from python/CIUtils.py rename to python/Production.py index d24d81e3..2f0763c3 100644 --- a/python/CIUtils.py +++ b/python/Production.py @@ -8,7 +8,7 @@ from main import main -class CIUtilsBase(ABC): +class ProductionBase(ABC): """Base class for all k4GeneratorConfig operations""" def __init__(self, args): @@ -74,7 +74,7 @@ def process(self, generators): def execute(self, generator, process): pass -class makeGeneratorDatacards(CIUtilsBase): +class makeGeneratorDatacards(ProductionBase): """Generator Generator Datacards""" def __init__(self, args): @@ -168,7 +168,7 @@ def run(self): # return tu the starting point os.chdir(cwd) -class checkGeneratorDatacards(CIUtilsBase): +class checkGeneratorDatacards(ProductionBase): """Check Generator Datacards""" def __init__(self, args): @@ -184,12 +184,30 @@ def execute(self, generator, process): genProc = f"{generator}/{process}" newDir = f"{self._generatorDir}/{genProc}" refDir = f"{self._referenceDir}/{genProc}" - fileNames = self.getFileNames(refDir) + fileNames = self.getFileNames(newDir) success = True + # check that all files were created (new) and are available for comparison (ref) + if len(fileNames) != len(self.getFileNames(refDir)): + print(f"Number of files for Generator {generator} process {process} differ between reference {len(fileNames)} and creation {len(self.getFileNames(refDir))}") + success = False + for name in fileNames: newFile = f"{newDir}/{name}" + # new file must exist + if not os.path.isfile(newFile): + print(f"File {newFile} not found") + success = False + continue + refFile = f"{refDir}/{name}" + # reference file must exist + if not os.path.isfile(refFile): + print(f"File {refFile} not found") + continue + success = False + + # both files exist, so we can compare message = f"Generator {generator} Process {process} File {name}" if filecmp.cmp(refFile,newFile, shallow=False): print(f"{message} identical") @@ -199,7 +217,7 @@ def execute(self, generator, process): return success -class generate(CIUtilsBase): +class generate(ProductionBase): """Run Event Generation""" def __init__(self, args): @@ -244,7 +262,7 @@ def execute(self, generator, process): return success -class summary(CIUtilsBase): +class summary(ProductionBase): """Make a summary of all processes""" def __init__(self, args): diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index 0bece642..32f7e3af 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -6,10 +6,10 @@ import argparse import textwrap -from CIUtils import makeGeneratorDatacards -from CIUtils import checkGeneratorDatacards -from CIUtils import generate -from CIUtils import summary +from Production import makeGeneratorDatacards +from Production import checkGeneratorDatacards +from Production import generate +from Production import summary def k4GeneratorsConfig(arguments=None): parser = argparse.ArgumentParser( From 6ed24c63267c8a409442b55df11bda6a51f0b37a Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Sun, 29 Mar 2026 21:25:09 +0200 Subject: [PATCH 40/88] clang bug fix --- python/Production.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/Production.py b/python/Production.py index 2f0763c3..5e4163bb 100644 --- a/python/Production.py +++ b/python/Production.py @@ -187,11 +187,11 @@ def execute(self, generator, process): fileNames = self.getFileNames(newDir) success = True - # check that all files were created (new) and are available for comparison (ref) + # check that all files were created (new) and are available for comparison (ref) if len(fileNames) != len(self.getFileNames(refDir)): print(f"Number of files for Generator {generator} process {process} differ between reference {len(fileNames)} and creation {len(self.getFileNames(refDir))}") success = False - + for name in fileNames: newFile = f"{newDir}/{name}" # new file must exist From c16338b800db54bfd47846bb297bf175422181c1 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Mon, 30 Mar 2026 17:01:05 +0200 Subject: [PATCH 41/88] extended commandlines and simplification --- python/Production.py | 124 ++++++++++++++++++++++------------- python/k4GeneratorsConfig.py | 63 +++++++++++++++--- 2 files changed, 132 insertions(+), 55 deletions(-) diff --git a/python/Production.py b/python/Production.py index 5e4163bb..8da0292c 100644 --- a/python/Production.py +++ b/python/Production.py @@ -39,13 +39,14 @@ def getFileNames(self, directory): filenames =[name for name in filenames if os.path.isfile(os.path.join(directory,name))] return filenames - def makeDirectory(self, dirname): + def makeDirectory(self, dirname, overwrite=True): # Overwrite directory if it exists if not os.path.exists(dirname): os.makedirs(dirname) else: - shutil.rmtree(dirname) - os.makedirs(dirname) + if overwrite: + shutil.rmtree(dirname) + os.makedirs(dirname) def makeOutputDirectory(self, generator, process): if generator not in os.listdir(self._outDir): @@ -80,26 +81,46 @@ class makeGeneratorDatacards(ProductionBase): def __init__(self, args): super().__init__(args) - # specific members for DC creation - self._yamlDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.yamlDir - if args.yamlDir.startswith('/'): - self._yamlDir = args.yamlDir + # the sqrts argument can be a list of sqrts or a list of strings + sqrtsGlobal = [] + try: + all(float(val) for val in args.sqrts) + # write the values to a file + try: + sqrtsFileName = f"{self._workDir}/sqrts.yaml" + sqrtsFile = open(sqrtsFileName,"x") + sqrtsList = "ecms:" + for sqrts in args.sqrts: + sqrtsList += f" {sqrts}," + sqrtsList = sqrtsList[:-1] + sqrtsFile.write(sqrtsList) + sqrtsFile.close() + except FileExistsError as e: + sys.exit(f"Command line specification --sqrts with a {args.sqrts} triggers writing of {self._workDir}/sqrts.yaml, but file exists") + sqrtsGlobal = [sqrtsFileName] + except ValueError as e: + if len(args.sqrts) == 1: + if Path(args.sqrts[0]).stem+Path(args.sqrts[0]).suffix == "sqrts.yaml" and os.path.isfile(args.sqrts[0]): + sqrtsGlobal = [os.path.abspath(args.sqrts[0])] - self._yamlFiles = [] + # specific members for DC creation + self._yamlFiles = [] + self._sqrtsFiles = [] - # make the directory for the work - self.makeDirectory(args.workDir) + # make the directory for the work, protect against "./" + self.makeDirectory(args.workDir, not (os.path.abspath(args.workDir) == os.getcwd() )) # make the output directory for the output - self.makeDirectory(args.outputDir) + self.makeDirectory(args.outputDir, not (os.path.abspath(args.outputDir) == os.getcwd() )) # prepare yamls: - self.prepareYamls(args.yamlFiles); + self.prepareYamls(args.yaml); - # prepare the Sqrts files: - self.prepareECMS(); + # prepare the Sqrts files if global is not set + if len(sqrtsGlobal) == 0: + self.prepareSQRTS(args.sqrts); # run all yamls: - self.run(); + self.run(sqrtsGlobal); # move the output to storage generators = self.getGenerators("All") @@ -125,33 +146,35 @@ def execute(self, generator, process): return success - def prepareYamls(self, yamlFiles): - # move the yamls to the work directory - # the yaml file is not specified copy all yaml files to the work directory: - if len(yamlFiles) == 0: - self._yamlFiles = [filename for filename in os.listdir(self._yamlDir) if filename.endswith('.yaml')] - else: - # check that all files end with .yaml - if not all([yamlfile.endswith('.yaml') for yamlfile in yamlFiles]): - sys.exit(f"prepareYamls::unknown file extension in input list: {yamlFiles}\nExpect .yaml") - self._yamlFiles =[self._yamlDir+"/"+yamlFile for yamlFile in yamlFiles] - - # copy the yaml files - for filename in self._yamlFiles: - try: - shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) - except FileNotFoundError as e: - sys.exit(f"prepareYamls::file not found: {e.filename}") - - def prepareECMS(self): - # move the ECMS files to the work directory - ecmsFiles = [filename for filename in os.listdir(self._yamlDir) if filename.endswith('.dat') and filename.startswith('ecms')] - - # copy the yaml files - for filename in ecmsFiles: - shutil.copy(os.path.join(self._yamlDir,filename),self._workDir) - - def run(self): + def prepareYamls(self, yamlList): + # check whether this is a list of directories or mixed or files + yamlDirs = [] + for yaml in yamlList: + if os.path.isfile(yaml) and yaml.endswith('.yaml') and not yaml.startswith('sqrts'): + self._yamlFiles.append(os.path.abspath(yaml)) + elif os.path.isdir(yaml): + yamlDirs.append(os.path.abspath(yaml)) + + # now we have a list of directories and a list of yaml files, extend the list of yaml files in the directories + for yamlDir in yamlDirs: + self._yamlFiles.extend(f"{yamlDir}/{filename}" for filename in os.listdir(yamlDir) + if filename.endswith('.yaml') and not filename.startswith('sqrts')) + + def prepareSQRTS(self, sqrtsList): + # check whether this is a list of directories or mixed or files + sqrtsDirs = [] + for sqrts in sqrtsList: + if os.path.isfile(sqrts) and yaml.startswith('sqrts') and sqrts.endswith('.yaml'): + self._sqrtsFiles.extend(os.path.abspath(sqrts)) + elif os.path.isdir(sqrts): + sqrtsDirs.extend(os.path.abspath(sqrts)) + + # now we have a list of directories and a list of sqrts files, extend the list of sqrts files in the directories + for sqrtsDir in sqrtsDirs: + self._sqrtsFiles.extend(f"{sqrtsDir}/{filename}" for filename in os.listdir(sqrtsDir) + if filename.startswith('sqrts') and filename.endswith('.yaml')) + + def run(self, sqrtsGlobal): # remember where we start from cwd = os.getcwd() # go to the working directory @@ -159,13 +182,20 @@ def run(self): # loop over all files for filename in self._yamlFiles: processName = Path(filename).stem - ecmsName = "ecms"+processName+".dat" - print("Processing : "+processName) - if os.path.isfile(ecmsName): - main([filename,'--ecmsFile',ecmsName]) + sqrtsName = f"{Path(filename).parent}sqrts{processName}.yaml" + message = f"Processing : {processName} from file {filename}" + sqrtsSpecified = False + for name in self._sqrtsFiles: + if name == sqrtsName: + sqrtsSpecified = True + # everything is prepared, we can run now + if sqrtsName in self._sqrtsFiles and os.path.isfile(sqrtsName): + print(f"{message} with {sqrtsName}") + main([filename,'--ecmsFile',sqrtsName]) else: + print(f"{message}") main([filename]) - # return tu the starting point + # return to the starting point os.chdir(cwd) class checkGeneratorDatacards(ProductionBase): diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index 32f7e3af..06f919df 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -28,17 +28,53 @@ def k4GeneratorsConfig(arguments=None): help="make the generator datacards from the yaml files" ) parser.add_argument( - "--yamlDir", + "--yaml", + nargs="*", type=str, - default="../examples", - help="path to the yamlFiles (default: k4GeneratorsConfig/examples)" + default=[os.path.dirname(os.path.realpath(__file__))+'/../examples'], + help="yamlFiles and director(y/ies) with yaml files (default: k4GeneratorsConfig/examples)" ) parser.add_argument( - "--yamlFiles", + "--sqrts", + nargs="*", type=str, - nargs="+", - default="", - help="yamlfiles to be processed (default: all are processed)" + default=[], + help="either a space separated list of center of mass energies OR file(s) and director(y/ies) with sqrts lists in yaml format (name : sqrtsPROCESS.dat containing eg, sqrts:[91.,240.]), sqrts.yaml as single argument will be applied to all processes", + ) + parser.add_argument( + "--seed", + nargs=1, + type=int, + default=4711, + help="initial random number seed, increment for each file", + ) + parser.add_argument( + "--nevts", + type=int, + default=-1, + help="Number of events to be generated", + ) + parser.add_argument( + "--parameterTag", + type=str, + default="latest", + help="parameter tag in Parameters.yaml (default: latest)", + ) + parser.add_argument( + "--parameterTagFile", + type=str, + default=os.path.dirname(os.path.realpath(__file__))+'ParameterSets.yaml', + help="name of file containing the parameter sets of the requested parameterTag, default: ParameterSets.yaml in directory: k4GeneratorsConfig/python", + ) + parser.add_argument( + "--key4hepUseNightlies", + action='store_true', + help="configures the key4hepscripts to use nightlies instead of releases", + ) + parser.add_argument( + "--key4hepVersion", + default=None, + help="force the use of the version : YYYY-MM-DD (default: latest)", ) parser.add_argument( "--check", @@ -48,7 +84,7 @@ def k4GeneratorsConfig(arguments=None): parser.add_argument( "--refDir", type=str, - default="../test/ref-results", + default=os.path.dirname(os.path.realpath(__file__))+'../test/ref-results', help="path to the reference files (default: k4GeneratorsConfig/test/ref-results)" ) parser.add_argument( @@ -85,8 +121,19 @@ def k4GeneratorsConfig(arguments=None): default="Run-Cards", help="relative path to the Generator directories in outputDir (default: outputDir/Run-Cards)" ) + parser.add_argument( + "--all", + action='store_true', + help="activates --make --generate --summary" + ) args = parser.parse_args(arguments) + # --all overrides --make --generate --summary + if args.all: + args.make = True + args.generate = True + args.summary = True + print("k4GeneratorsConfig will make generator datacards, generate events, make a summary") if args.make: makeGeneratorDatacards(args) From 96eb0a9c8bb5b0f7d8e1950ff004e300b5c8f14f Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Mon, 30 Mar 2026 17:43:31 +0200 Subject: [PATCH 42/88] convert to new scheme with consistent commandline input --- python/Production.py | 45 ++++++++++--------- ...msGammaGamma.yaml => sqrtsGammaGamma.yaml} | 0 .../Hnunu/{ecmsHnunu.yaml => sqrtsHnunu.yaml} | 0 .../WW/{ecmsWW.yaml => sqrtsWW.yaml} | 0 .../ZH/{ecmsZH.yaml => sqrtsZH.yaml} | 0 .../ZHH/{ecmsZHH.yaml => sqrtsZHH.yaml} | 0 ...ZHPolarized.yaml => sqrtsZHPolarized.yaml} | 0 .../ZZ/{ecmsZZ.yaml => sqrtsZZ.yaml} | 0 .../ffbar/{ecmsffbar.yaml => sqrtsffbar.yaml} | 0 .../ttbar/{ecmsttbar.yaml => sqrtsttbar.yaml} | 0 10 files changed, 25 insertions(+), 20 deletions(-) rename share/ECFAHiggsFactories/GammaGamma/{ecmsGammaGamma.yaml => sqrtsGammaGamma.yaml} (100%) rename share/ECFAHiggsFactories/Hnunu/{ecmsHnunu.yaml => sqrtsHnunu.yaml} (100%) rename share/ECFAHiggsFactories/WW/{ecmsWW.yaml => sqrtsWW.yaml} (100%) rename share/ECFAHiggsFactories/ZH/{ecmsZH.yaml => sqrtsZH.yaml} (100%) rename share/ECFAHiggsFactories/ZHH/{ecmsZHH.yaml => sqrtsZHH.yaml} (100%) rename share/ECFAHiggsFactories/ZHPolarized/{ZHPolarized.yaml => sqrtsZHPolarized.yaml} (100%) rename share/ECFAHiggsFactories/ZZ/{ecmsZZ.yaml => sqrtsZZ.yaml} (100%) rename share/ECFAHiggsFactories/ffbar/{ecmsffbar.yaml => sqrtsffbar.yaml} (100%) rename share/ECFAHiggsFactories/ttbar/{ecmsttbar.yaml => sqrtsttbar.yaml} (100%) diff --git a/python/Production.py b/python/Production.py index 8da0292c..d1c5bb9d 100644 --- a/python/Production.py +++ b/python/Production.py @@ -81,46 +81,46 @@ class makeGeneratorDatacards(ProductionBase): def __init__(self, args): super().__init__(args) + # make the directory for the work, protect against "./" + self.makeDirectory(args.workDir, not (os.path.abspath(args.workDir) == os.getcwd() )) + # make the output directory for the output + self.makeDirectory(args.outputDir, not (os.path.abspath(args.outputDir) == os.getcwd() )) + # the sqrts argument can be a list of sqrts or a list of strings - sqrtsGlobal = [] + sqrtsGlobalFileName = str("") try: all(float(val) for val in args.sqrts) # write the values to a file try: - sqrtsFileName = f"{self._workDir}/sqrts.yaml" - sqrtsFile = open(sqrtsFileName,"x") - sqrtsList = "ecms:" + sqrtsGlobalFileName = f"{self._workDir}/sqrts.yaml" + sqrtsFile = open(sqrtsGlobalFileName,"x") + sqrtsList = "ecms: [ " for sqrts in args.sqrts: sqrtsList += f" {sqrts}," - sqrtsList = sqrtsList[:-1] + sqrtsList.rstrip(",") + sqrtsList += "]" sqrtsFile.write(sqrtsList) sqrtsFile.close() except FileExistsError as e: - sys.exit(f"Command line specification --sqrts with a {args.sqrts} triggers writing of {self._workDir}/sqrts.yaml, but file exists") - sqrtsGlobal = [sqrtsFileName] + sys.exit(f"Command line specification --sqrts with a {args.sqrts} triggers writing of {sqrtsGlobalFileName}, but file exists") except ValueError as e: if len(args.sqrts) == 1: if Path(args.sqrts[0]).stem+Path(args.sqrts[0]).suffix == "sqrts.yaml" and os.path.isfile(args.sqrts[0]): - sqrtsGlobal = [os.path.abspath(args.sqrts[0])] + sqrtsGlobalFileName = os.path.abspath(args.sqrts[0]) # specific members for DC creation self._yamlFiles = [] self._sqrtsFiles = [] - # make the directory for the work, protect against "./" - self.makeDirectory(args.workDir, not (os.path.abspath(args.workDir) == os.getcwd() )) - # make the output directory for the output - self.makeDirectory(args.outputDir, not (os.path.abspath(args.outputDir) == os.getcwd() )) - # prepare yamls: self.prepareYamls(args.yaml); # prepare the Sqrts files if global is not set - if len(sqrtsGlobal) == 0: + if not sqrtsGlobalFileName: self.prepareSQRTS(args.sqrts); # run all yamls: - self.run(sqrtsGlobal); + self.run(sqrtsGlobalFileName); # move the output to storage generators = self.getGenerators("All") @@ -181,15 +181,20 @@ def run(self, sqrtsGlobal): os.chdir(self._workDir) # loop over all files for filename in self._yamlFiles: + + # check SQRTS: priority: global then comparison with filenames for specific processes processName = Path(filename).stem - sqrtsName = f"{Path(filename).parent}sqrts{processName}.yaml" + sqrtsName = f"{Path(filename).parent}/sqrts{processName}.yaml" + if sqrtsGlobal: + sqrtsName = sqrtsGlobal + else: + if not any( name == sqrtsName for name in self._sqrtsFiles): + sqrtsName = "" + # the name is overidden if there is a global name specified: message = f"Processing : {processName} from file {filename}" sqrtsSpecified = False - for name in self._sqrtsFiles: - if name == sqrtsName: - sqrtsSpecified = True # everything is prepared, we can run now - if sqrtsName in self._sqrtsFiles and os.path.isfile(sqrtsName): + if sqrtsName and os.path.isfile(sqrtsName): print(f"{message} with {sqrtsName}") main([filename,'--ecmsFile',sqrtsName]) else: diff --git a/share/ECFAHiggsFactories/GammaGamma/ecmsGammaGamma.yaml b/share/ECFAHiggsFactories/GammaGamma/sqrtsGammaGamma.yaml similarity index 100% rename from share/ECFAHiggsFactories/GammaGamma/ecmsGammaGamma.yaml rename to share/ECFAHiggsFactories/GammaGamma/sqrtsGammaGamma.yaml diff --git a/share/ECFAHiggsFactories/Hnunu/ecmsHnunu.yaml b/share/ECFAHiggsFactories/Hnunu/sqrtsHnunu.yaml similarity index 100% rename from share/ECFAHiggsFactories/Hnunu/ecmsHnunu.yaml rename to share/ECFAHiggsFactories/Hnunu/sqrtsHnunu.yaml diff --git a/share/ECFAHiggsFactories/WW/ecmsWW.yaml b/share/ECFAHiggsFactories/WW/sqrtsWW.yaml similarity index 100% rename from share/ECFAHiggsFactories/WW/ecmsWW.yaml rename to share/ECFAHiggsFactories/WW/sqrtsWW.yaml diff --git a/share/ECFAHiggsFactories/ZH/ecmsZH.yaml b/share/ECFAHiggsFactories/ZH/sqrtsZH.yaml similarity index 100% rename from share/ECFAHiggsFactories/ZH/ecmsZH.yaml rename to share/ECFAHiggsFactories/ZH/sqrtsZH.yaml diff --git a/share/ECFAHiggsFactories/ZHH/ecmsZHH.yaml b/share/ECFAHiggsFactories/ZHH/sqrtsZHH.yaml similarity index 100% rename from share/ECFAHiggsFactories/ZHH/ecmsZHH.yaml rename to share/ECFAHiggsFactories/ZHH/sqrtsZHH.yaml diff --git a/share/ECFAHiggsFactories/ZHPolarized/ZHPolarized.yaml b/share/ECFAHiggsFactories/ZHPolarized/sqrtsZHPolarized.yaml similarity index 100% rename from share/ECFAHiggsFactories/ZHPolarized/ZHPolarized.yaml rename to share/ECFAHiggsFactories/ZHPolarized/sqrtsZHPolarized.yaml diff --git a/share/ECFAHiggsFactories/ZZ/ecmsZZ.yaml b/share/ECFAHiggsFactories/ZZ/sqrtsZZ.yaml similarity index 100% rename from share/ECFAHiggsFactories/ZZ/ecmsZZ.yaml rename to share/ECFAHiggsFactories/ZZ/sqrtsZZ.yaml diff --git a/share/ECFAHiggsFactories/ffbar/ecmsffbar.yaml b/share/ECFAHiggsFactories/ffbar/sqrtsffbar.yaml similarity index 100% rename from share/ECFAHiggsFactories/ffbar/ecmsffbar.yaml rename to share/ECFAHiggsFactories/ffbar/sqrtsffbar.yaml diff --git a/share/ECFAHiggsFactories/ttbar/ecmsttbar.yaml b/share/ECFAHiggsFactories/ttbar/sqrtsttbar.yaml similarity index 100% rename from share/ECFAHiggsFactories/ttbar/ecmsttbar.yaml rename to share/ECFAHiggsFactories/ttbar/sqrtsttbar.yaml From cf4037c837ae1de882bfae4e2120dccdf4947a00 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Mon, 30 Mar 2026 18:49:25 +0200 Subject: [PATCH 43/88] more debugging --- python/Production.py | 53 ++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/python/Production.py b/python/Production.py index d1c5bb9d..40383272 100644 --- a/python/Production.py +++ b/python/Production.py @@ -88,25 +88,24 @@ def __init__(self, args): # the sqrts argument can be a list of sqrts or a list of strings sqrtsGlobalFileName = str("") - try: - all(float(val) for val in args.sqrts) - # write the values to a file + if len(args.sqrts) > 0: try: - sqrtsGlobalFileName = f"{self._workDir}/sqrts.yaml" - sqrtsFile = open(sqrtsGlobalFileName,"x") - sqrtsList = "ecms: [ " - for sqrts in args.sqrts: - sqrtsList += f" {sqrts}," - sqrtsList.rstrip(",") - sqrtsList += "]" - sqrtsFile.write(sqrtsList) - sqrtsFile.close() - except FileExistsError as e: - sys.exit(f"Command line specification --sqrts with a {args.sqrts} triggers writing of {sqrtsGlobalFileName}, but file exists") - except ValueError as e: - if len(args.sqrts) == 1: - if Path(args.sqrts[0]).stem+Path(args.sqrts[0]).suffix == "sqrts.yaml" and os.path.isfile(args.sqrts[0]): - sqrtsGlobalFileName = os.path.abspath(args.sqrts[0]) + all(float(val) for val in args.sqrts) + # write the values to a file + try: + sqrtsGlobalFileName = f"{self._workDir}/sqrts.yaml" + sqrtsFile = open(sqrtsGlobalFileName,"x") + sqrtsList = "ecms: [ " + for sqrts in args.sqrts: + sqrtsList += f" {sqrts}," + sqrtsFile.write(sqrtsList.rstrip(",") + "]") + sqrtsFile.close() + except FileExistsError as e: + sys.exit(f"Command line specification --sqrts with a {args.sqrts} triggers writing of {sqrtsGlobalFileName}, but file exists") + except ValueError as e: + if len(args.sqrts) == 1: + if Path(args.sqrts[0]).stem+Path(args.sqrts[0]).suffix == "sqrts.yaml" and os.path.isfile(args.sqrts[0]): + sqrtsGlobalFileName = os.path.abspath(args.sqrts[0]) # specific members for DC creation self._yamlFiles = [] @@ -150,29 +149,31 @@ def prepareYamls(self, yamlList): # check whether this is a list of directories or mixed or files yamlDirs = [] for yaml in yamlList: - if os.path.isfile(yaml) and yaml.endswith('.yaml') and not yaml.startswith('sqrts'): + if os.path.isfile(yaml) and yaml.endswith('.yaml') and not Path(yaml).stem.startswith('sqrts'): self._yamlFiles.append(os.path.abspath(yaml)) elif os.path.isdir(yaml): yamlDirs.append(os.path.abspath(yaml)) # now we have a list of directories and a list of yaml files, extend the list of yaml files in the directories for yamlDir in yamlDirs: - self._yamlFiles.extend(f"{yamlDir}/{filename}" for filename in os.listdir(yamlDir) - if filename.endswith('.yaml') and not filename.startswith('sqrts')) + for filename in os.listdir(yamlDir): + if filename.endswith('.yaml') and not filename.startswith('sqrts'): + self._yamlFiles.append(f"{yamlDir}/{filename}") def prepareSQRTS(self, sqrtsList): # check whether this is a list of directories or mixed or files sqrtsDirs = [] for sqrts in sqrtsList: - if os.path.isfile(sqrts) and yaml.startswith('sqrts') and sqrts.endswith('.yaml'): - self._sqrtsFiles.extend(os.path.abspath(sqrts)) + if os.path.isfile(sqrts) and Path(sqrts).stem.startswith('sqrts') and sqrts.endswith('.yaml'): + self._sqrtsFiles.append(os.path.abspath(sqrts)) elif os.path.isdir(sqrts): - sqrtsDirs.extend(os.path.abspath(sqrts)) + sqrtsDirs.append(os.path.abspath(sqrts)) # now we have a list of directories and a list of sqrts files, extend the list of sqrts files in the directories for sqrtsDir in sqrtsDirs: - self._sqrtsFiles.extend(f"{sqrtsDir}/{filename}" for filename in os.listdir(sqrtsDir) - if filename.startswith('sqrts') and filename.endswith('.yaml')) + for filename in os.listdir(sqrtsDir): + if filename.startswith('sqrts') and filename.endswith('.yaml'): + self._sqrtsFiles.append(f"{sqrtsDir}/{filename}") def run(self, sqrtsGlobal): # remember where we start from From 7a0e9124eba1a782ad16ca27378135b282e0eaad Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Mon, 30 Mar 2026 19:43:15 +0200 Subject: [PATCH 44/88] full integration stitched --- python/Production.py | 25 ++++--- python/Yaml2Datacard.py | 125 +++++++++++++++++++++++++++++++++++ python/k4GeneratorsConfig.py | 4 +- 3 files changed, 141 insertions(+), 13 deletions(-) create mode 100644 python/Yaml2Datacard.py diff --git a/python/Production.py b/python/Production.py index 40383272..fe12bd19 100644 --- a/python/Production.py +++ b/python/Production.py @@ -6,7 +6,7 @@ from pathlib import Path import filecmp -from main import main +from Yaml2Datacard import Yaml2Datacard class ProductionBase(ABC): """Base class for all k4GeneratorConfig operations""" @@ -19,9 +19,11 @@ def __init__(self, args): self._outDir = os.getcwd()+"/"+args.outputDir if args.outputDir.startswith('/'): - self.outdir = args.outputDir + self._outdir = args.outputDir - self._referenceDir = os.path.dirname(os.path.realpath(__file__))+"/"+args.refDir + self._referenceDir = os.getcwd()+"/"+args.refDir + if args.refDir.startswith('/'): + self._referenceDir = args.refDir self._generatorDir = f"{self._workDir}/{args.generatorDirName}" @@ -118,6 +120,9 @@ def __init__(self, args): if not sqrtsGlobalFileName: self.prepareSQRTS(args.sqrts); + # args modifiable: + self.Yaml2DatacardArgs = args + # run all yamls: self.run(sqrtsGlobalFileName); @@ -191,16 +196,14 @@ def run(self, sqrtsGlobal): else: if not any( name == sqrtsName for name in self._sqrtsFiles): sqrtsName = "" - # the name is overidden if there is a global name specified: - message = f"Processing : {processName} from file {filename}" - sqrtsSpecified = False # everything is prepared, we can run now + self.Yaml2DatacardArgs.inputfiles = [filename] + message = f"Processing : {processName} from file {self.Yaml2DatacardArgs.inputfiles}" if sqrtsName and os.path.isfile(sqrtsName): - print(f"{message} with {sqrtsName}") - main([filename,'--ecmsFile',sqrtsName]) - else: - print(f"{message}") - main([filename]) + self.Yaml2DatacardArgs.sqrts = sqrtsName + message += f"{message} with {self.Yaml2DatacardArgs.sqrts}" + print(message) + Yaml2Datacard(self.Yaml2DatacardArgs) # return to the starting point os.chdir(cwd) diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py new file mode 100644 index 00000000..362a3bbf --- /dev/null +++ b/python/Yaml2Datacard.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse +import textwrap +from datetime import datetime + +import ReleaseSpecs +from ReleaseSpecs import ReleaseSpec +import Input as Settings +import Process as process_module +import Generators as generators_module +from Particles import ParticleCollection + +def make_output_directory(generators, output_directory, procname): + # Overwrite directory if it exists + if not os.path.exists(output_directory): + os.makedirs(output_directory) + for generator in generators: + generator_directory = os.path.join(output_directory, generator, procname) + if not os.path.exists(generator_directory): + os.makedirs(generator_directory) + + +def Yaml2Datacard(args=None): + + ReleaseSpec.set_info("key4hepUseNightlies",args.key4hepUseNightlies) + if ReleaseSpecs.key4hepUseNightlies.value: + print(f"key4HEP configuration: NIGHTLIES") + else: + print(f"key4HEP configuration: RELEASE") + + # make sure it's a valid date + if args.key4hepVersion is not None: + try: + relDate = datetime.strptime(args.key4hepVersion,'%Y-%m-%d') + if (datetime.today() - relDate).days < 0: + raise ValueError() + print(f"key4HEP configuration date: {args.key4hepVersion}") + except ValueError: + print(f"Invalid KEY4HEP release argument, YYYY-MM-DD expected, latest possible date {datetime.today().strftime('%Y-%m-%d')}") + print(f"Requested: {args.key4hepVersion}") + print("Cannot configure scripts correctly, exiting") + exit() + else: + print(f"key4HEP configuration date: latest") + # store for future use: + ReleaseSpec.set_info("key4hepReleaseDate",args.key4hepVersion) + + # so additionally we read the argument sqrtsFile + energies = [] + for ecmsfile in args.sqrts: + # open and read ecms file and append the energies to the command line arguments + ecmSettings = Settings.ECMSInput(ecmsfile) + energies.extend(ecmSettings.energies()) + + # now we read the global settings + try: + # make sure that we follow a symlink to the real location of the parametersets should replace that by share? + parameterSet = Settings.ParameterSets(args.parameterTagFile, args.parameterTag) + except FileNotFoundError as e: + print(f"ERROR: File {e} with parameters for tag {args.parameterTag} not found") + exit() + + # now execute file processes + rndmSeed = args.seed + if len(energies) == 0: + executeFiles(args.inputfiles, 0, rndmSeed, args.nevts) + else: + for sqrts in energies: + rndmIncrement = executeFiles(args.inputfiles, sqrts, rndmSeed, args.nevts) + # offset for next round by number of yaml files + rndmSeed = rndmSeed + rndmIncrement + + +def executeFiles(files, sqrts, rndmSeedFallback=4711, events=-1): + # first step reset all particles: + ParticleCollection() + if sqrts == 0: + print("Generating and writing configuration files") + else: + print("Generating and writing configuration files for ECM= ", sqrts) + + for yaml_file in files: + # read the input file + settings = Settings.Input(yaml_file, sqrts) + # set the number of events if present + if events != -1: + settings.set("events", events) + settings.gens() + processes = settings.get_processes(sqrts) + yamlParticleData = settings.get_particle_data() + generators = generators_module.Generators(settings) + try: + output_dir = getattr(settings, "outdir", "Run-Cards") + except KeyError: + # If no directory set in input, use default + output_dir = "Run-Cards" + + process_instances = {} + rndmIncrement = 0 + for key, value in processes.items(): + make_output_directory(settings.gens(), output_dir, key) + try: + randomseed = value["randomseed"] + except: + randomseed = rndmSeedFallback + rndmIncrement + value["randomseed"] = randomseed + rndmIncrement += 1 + param = process_module.ProcessParameters(settings) + # instantiate the class for each process + process_instances[key] = process_module.Process( + value, key, param, yamlParticleData, OutDir=output_dir + ) + # increment counter for randomseed + for process_instance in process_instances.values(): + process_instance.prepareProcess() + generators.runGeneratorConfiguration(process_instance) + + return rndmIncrement + + +if __name__ == "__main__": + Yaml2Datacard() diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index 06f919df..77ddc122 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -63,7 +63,7 @@ def k4GeneratorsConfig(arguments=None): parser.add_argument( "--parameterTagFile", type=str, - default=os.path.dirname(os.path.realpath(__file__))+'ParameterSets.yaml', + default=os.path.dirname(os.path.realpath(__file__))+'/ParameterSets.yaml', help="name of file containing the parameter sets of the requested parameterTag, default: ParameterSets.yaml in directory: k4GeneratorsConfig/python", ) parser.add_argument( @@ -84,7 +84,7 @@ def k4GeneratorsConfig(arguments=None): parser.add_argument( "--refDir", type=str, - default=os.path.dirname(os.path.realpath(__file__))+'../test/ref-results', + default=os.path.dirname(os.path.realpath(__file__))+'/../test/ref-results', help="path to the reference files (default: k4GeneratorsConfig/test/ref-results)" ) parser.add_argument( From 69598ba27c77863284b5e640b54dd21238cffa52 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Mon, 30 Mar 2026 19:49:55 +0200 Subject: [PATCH 45/88] clang --- python/Production.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/Production.py b/python/Production.py index fe12bd19..cf9d8466 100644 --- a/python/Production.py +++ b/python/Production.py @@ -122,7 +122,7 @@ def __init__(self, args): # args modifiable: self.Yaml2DatacardArgs = args - + # run all yamls: self.run(sqrtsGlobalFileName); @@ -158,7 +158,7 @@ def prepareYamls(self, yamlList): self._yamlFiles.append(os.path.abspath(yaml)) elif os.path.isdir(yaml): yamlDirs.append(os.path.abspath(yaml)) - + # now we have a list of directories and a list of yaml files, extend the list of yaml files in the directories for yamlDir in yamlDirs: for filename in os.listdir(yamlDir): @@ -188,7 +188,7 @@ def run(self, sqrtsGlobal): # loop over all files for filename in self._yamlFiles: - # check SQRTS: priority: global then comparison with filenames for specific processes + # check SQRTS: priority: global then comparison with filenames for specific processes processName = Path(filename).stem sqrtsName = f"{Path(filename).parent}/sqrts{processName}.yaml" if sqrtsGlobal: From 9043457dcfbf25c9eed092fb0f642a15d65b5c55 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 31 Mar 2026 10:29:27 +0200 Subject: [PATCH 46/88] more consistency --- CMakeLists.txt | 2 +- python/Production.py | 20 +++++---- python/Yaml2Datacard.py | 79 +++++++++++++++++------------------- python/k4GeneratorsConfig.py | 11 +---- setup.csh | 2 +- setup.sh | 3 +- setup.zsh | 3 +- 7 files changed, 54 insertions(+), 66 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a8e46cbb..8d90b1d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,7 +64,7 @@ install(DIRECTORY ${PROJECT_SOURCE_DIR}/python install(PROGRAMS ${PROJECT_SOURCE_DIR}/python/main.py DESTINATION ${CMAKE_INSTALL_PREFIX}/python) install(CODE "execute_process(COMMAND bash -c \"set -e - link_target=${CMAKE_INSTALL_PREFIX}/python/main.py + link_target=${CMAKE_INSTALL_PREFIX}/python/k4GeneratorsConfig.py link_name=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/k4GeneratorsConfig rm -f $link_name ln -s $link_target $link_name diff --git a/python/Production.py b/python/Production.py index cf9d8466..937aaf8a 100644 --- a/python/Production.py +++ b/python/Production.py @@ -161,9 +161,9 @@ def prepareYamls(self, yamlList): # now we have a list of directories and a list of yaml files, extend the list of yaml files in the directories for yamlDir in yamlDirs: - for filename in os.listdir(yamlDir): - if filename.endswith('.yaml') and not filename.startswith('sqrts'): - self._yamlFiles.append(f"{yamlDir}/{filename}") + self._yamlFiles.extend([f"{yamlDir}/{filename}" + for filename in os.listdir(yamlDir) + if filename.endswith('.yaml') and not filename.startswith('sqrts')]) def prepareSQRTS(self, sqrtsList): # check whether this is a list of directories or mixed or files @@ -176,9 +176,9 @@ def prepareSQRTS(self, sqrtsList): # now we have a list of directories and a list of sqrts files, extend the list of sqrts files in the directories for sqrtsDir in sqrtsDirs: - for filename in os.listdir(sqrtsDir): - if filename.startswith('sqrts') and filename.endswith('.yaml'): - self._sqrtsFiles.append(f"{sqrtsDir}/{filename}") + self._sqrtsFiles.extend([f"{sqrtsDir}/{filename}" + for filename in os.listdir(sqrtsDir) + if filename.startswith('sqrts') and filename.endswith('.yaml')]) def run(self, sqrtsGlobal): # remember where we start from @@ -197,11 +197,13 @@ def run(self, sqrtsGlobal): if not any( name == sqrtsName for name in self._sqrtsFiles): sqrtsName = "" # everything is prepared, we can run now - self.Yaml2DatacardArgs.inputfiles = [filename] - message = f"Processing : {processName} from file {self.Yaml2DatacardArgs.inputfiles}" + self.Yaml2DatacardArgs.yaml = filename + message = f"Processing : {processName} from file {self.Yaml2DatacardArgs.yaml}" if sqrtsName and os.path.isfile(sqrtsName): self.Yaml2DatacardArgs.sqrts = sqrtsName - message += f"{message} with {self.Yaml2DatacardArgs.sqrts}" + message += f" with {self.Yaml2DatacardArgs.sqrts}" + else: + self.Yaml2DatacardArgs.sqrts = "" print(message) Yaml2Datacard(self.Yaml2DatacardArgs) # return to the starting point diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 362a3bbf..86d01494 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -50,9 +50,8 @@ def Yaml2Datacard(args=None): # so additionally we read the argument sqrtsFile energies = [] - for ecmsfile in args.sqrts: - # open and read ecms file and append the energies to the command line arguments - ecmSettings = Settings.ECMSInput(ecmsfile) + if args.sqrts: + ecmSettings = Settings.ECMSInput(args.sqrts) energies.extend(ecmSettings.energies()) # now we read the global settings @@ -66,15 +65,14 @@ def Yaml2Datacard(args=None): # now execute file processes rndmSeed = args.seed if len(energies) == 0: - executeFiles(args.inputfiles, 0, rndmSeed, args.nevts) + executeFiles(args.yaml, 0, rndmSeed, args.nevts) else: for sqrts in energies: - rndmIncrement = executeFiles(args.inputfiles, sqrts, rndmSeed, args.nevts) + rndmIncrement = executeFiles(args.yaml, sqrts, rndmSeed, args.nevts) # offset for next round by number of yaml files rndmSeed = rndmSeed + rndmIncrement - -def executeFiles(files, sqrts, rndmSeedFallback=4711, events=-1): +def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): # first step reset all particles: ParticleCollection() if sqrts == 0: @@ -82,41 +80,40 @@ def executeFiles(files, sqrts, rndmSeedFallback=4711, events=-1): else: print("Generating and writing configuration files for ECM= ", sqrts) - for yaml_file in files: - # read the input file - settings = Settings.Input(yaml_file, sqrts) - # set the number of events if present - if events != -1: - settings.set("events", events) - settings.gens() - processes = settings.get_processes(sqrts) - yamlParticleData = settings.get_particle_data() - generators = generators_module.Generators(settings) + # read the input file + settings = Settings.Input(yaml, sqrts) + # set the number of events if present + if events != -1: + settings.set("events", events) + settings.gens() + processes = settings.get_processes(sqrts) + yamlParticleData = settings.get_particle_data() + generators = generators_module.Generators(settings) + try: + output_dir = getattr(settings, "outdir", "Run-Cards") + except KeyError: + # If no directory set in input, use default + output_dir = "Run-Cards" + + process_instances = {} + rndmIncrement = 0 + for key, value in processes.items(): + make_output_directory(settings.gens(), output_dir, key) try: - output_dir = getattr(settings, "outdir", "Run-Cards") - except KeyError: - # If no directory set in input, use default - output_dir = "Run-Cards" - - process_instances = {} - rndmIncrement = 0 - for key, value in processes.items(): - make_output_directory(settings.gens(), output_dir, key) - try: - randomseed = value["randomseed"] - except: - randomseed = rndmSeedFallback + rndmIncrement - value["randomseed"] = randomseed - rndmIncrement += 1 - param = process_module.ProcessParameters(settings) - # instantiate the class for each process - process_instances[key] = process_module.Process( - value, key, param, yamlParticleData, OutDir=output_dir - ) - # increment counter for randomseed - for process_instance in process_instances.values(): - process_instance.prepareProcess() - generators.runGeneratorConfiguration(process_instance) + randomseed = value["randomseed"] + except: + randomseed = rndmSeedFallback + rndmIncrement + value["randomseed"] = randomseed + rndmIncrement += 1 + param = process_module.ProcessParameters(settings) + # instantiate the class for each process + process_instances[key] = process_module.Process( + value, key, param, yamlParticleData, OutDir=output_dir + ) + # increment counter for randomseed + for process_instance in process_instances.values(): + process_instance.prepareProcess() + generators.runGeneratorConfiguration(process_instance) return rndmIncrement diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index 77ddc122..c3b24078 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -12,16 +12,7 @@ from Production import summary def k4GeneratorsConfig(arguments=None): - parser = argparse.ArgumentParser( - prog="k4GeneratorsConfig", - formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent( - """\ -The following options are available: ------------------------------------- - """ - ), - ) + parser = argparse.ArgumentParser(prog="k4GeneratorsConfig") parser.add_argument( "--make", action='store_true', diff --git a/setup.csh b/setup.csh index f11a5bc5..03769624 100755 --- a/setup.csh +++ b/setup.csh @@ -42,4 +42,4 @@ if (! -d "${K4GeneratorsConfigDir}/install/") then endif # Set executable -alias k4GeneratorsConfigRun "python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" +alias k4GeneratorsConfig "python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" diff --git a/setup.sh b/setup.sh index 599ffac6..eff874de 100755 --- a/setup.sh +++ b/setup.sh @@ -29,5 +29,4 @@ if [[ ! -d "${K4GeneratorsConfigDir}/install/" ]]; then fi # Set executable -alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/main.py" -alias k4GeneratorsConfigRun="python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" +alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" diff --git a/setup.zsh b/setup.zsh index eb2df7ae..f76fa8d5 100755 --- a/setup.zsh +++ b/setup.zsh @@ -29,5 +29,4 @@ if [[ ! -d "${K4GeneratorsConfigDir}/install/" ]]; then fi # Set executable -alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/main.py" -alias k4GeneratorsConfigRun="python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" +alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" From e07235b2f564e8d5ed1f376bf0bc8fc5d7dceea2 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 31 Mar 2026 11:14:12 +0200 Subject: [PATCH 47/88] renaming for simplicity --- python/Yaml2Datacard.py | 34 ++++++++++++------------- python/{Input.py => YamlInputReader.py} | 6 ++--- 2 files changed, 20 insertions(+), 20 deletions(-) rename python/{Input.py => YamlInputReader.py} (99%) diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 86d01494..f9edf6d5 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -8,9 +8,10 @@ import ReleaseSpecs from ReleaseSpecs import ReleaseSpec -import Input as Settings -import Process as process_module -import Generators as generators_module +import YamlInputReader as Reader +from Process import Process +from Process import ProcessParameters +from Generators import Generators from Particles import ParticleCollection def make_output_directory(generators, output_directory, procname): @@ -22,7 +23,6 @@ def make_output_directory(generators, output_directory, procname): if not os.path.exists(generator_directory): os.makedirs(generator_directory) - def Yaml2Datacard(args=None): ReleaseSpec.set_info("key4hepUseNightlies",args.key4hepUseNightlies) @@ -48,16 +48,16 @@ def Yaml2Datacard(args=None): # store for future use: ReleaseSpec.set_info("key4hepReleaseDate",args.key4hepVersion) - # so additionally we read the argument sqrtsFile + # sqrts choices from file energies = [] if args.sqrts: - ecmSettings = Settings.ECMSInput(args.sqrts) + ecmSettings = Reader.SQRTSReader(args.sqrts) energies.extend(ecmSettings.energies()) # now we read the global settings try: # make sure that we follow a symlink to the real location of the parametersets should replace that by share? - parameterSet = Settings.ParameterSets(args.parameterTagFile, args.parameterTag) + parameterSet = Reader.ParameterSetReader(args.parameterTagFile, args.parameterTag) except FileNotFoundError as e: print(f"ERROR: File {e} with parameters for tag {args.parameterTag} not found") exit() @@ -81,16 +81,16 @@ def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): print("Generating and writing configuration files for ECM= ", sqrts) # read the input file - settings = Settings.Input(yaml, sqrts) + reader = Reader.ProcessReader(yaml, sqrts) # set the number of events if present if events != -1: - settings.set("events", events) - settings.gens() - processes = settings.get_processes(sqrts) - yamlParticleData = settings.get_particle_data() - generators = generators_module.Generators(settings) + reader.set("events", events) + reader.gens() + processes = reader.get_processes(sqrts) + yamlParticleData = reader.get_particle_data() + generators = Generators(reader) try: - output_dir = getattr(settings, "outdir", "Run-Cards") + output_dir = getattr(reader, "outdir", "Run-Cards") except KeyError: # If no directory set in input, use default output_dir = "Run-Cards" @@ -98,16 +98,16 @@ def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): process_instances = {} rndmIncrement = 0 for key, value in processes.items(): - make_output_directory(settings.gens(), output_dir, key) + make_output_directory(reader.gens(), output_dir, key) try: randomseed = value["randomseed"] except: randomseed = rndmSeedFallback + rndmIncrement value["randomseed"] = randomseed rndmIncrement += 1 - param = process_module.ProcessParameters(settings) + param = ProcessParameters(reader) # instantiate the class for each process - process_instances[key] = process_module.Process( + process_instances[key] = Process( value, key, param, yamlParticleData, OutDir=output_dir ) # increment counter for randomseed diff --git a/python/Input.py b/python/YamlInputReader.py similarity index 99% rename from python/Input.py rename to python/YamlInputReader.py index 657a3bd4..1015391c 100644 --- a/python/Input.py +++ b/python/YamlInputReader.py @@ -4,7 +4,7 @@ import Parameters as ParameterModule from Parameters import Parameter as ParameterClass -class Input: +class ProcessReader: """Class for loading YAML files""" def __init__(self, file, sqrts): @@ -217,7 +217,7 @@ def key4HEPAnalysisON(self): if self.anatools is not None: return "key4hep" in self.anatools -class ECMSInput: +class SQRTSReader: """Class for loading YAML files with center of mass energies""" def __init__(self, file): @@ -238,7 +238,7 @@ def energies(self): ecmsList.extend(value) return ecmsList -class ParameterSets: +class ParameterSetReader: """Class for loading YAML files with the parameter settings""" def __init__(self, file, tag): From cbd4c369665dcbea63b3e6e2a798c2cd90dad1ec Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 31 Mar 2026 11:22:13 +0200 Subject: [PATCH 48/88] clean renaming --- python/Generators/Generators.py | 2 +- python/Yaml2Datacard.py | 18 +++++++++--------- python/YamlInputReader.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/python/Generators/Generators.py b/python/Generators/Generators.py index 310634fd..9dfff8a7 100644 --- a/python/Generators/Generators.py +++ b/python/Generators/Generators.py @@ -5,7 +5,7 @@ class Generators: def __init__(self, settings): self.settings = settings - self.generator_list = settings.gens() + self.generator_list = settings.get_generators() def set_process_info(self, proc_info): self.proc_info = proc_info diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index f9edf6d5..93f40ac8 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -81,16 +81,16 @@ def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): print("Generating and writing configuration files for ECM= ", sqrts) # read the input file - reader = Reader.ProcessReader(yaml, sqrts) + processReader = Reader.ProcessReader(yaml, sqrts) # set the number of events if present if events != -1: - reader.set("events", events) - reader.gens() - processes = reader.get_processes(sqrts) - yamlParticleData = reader.get_particle_data() - generators = Generators(reader) + processReader.set("events", events) + processReader.get_generators() + processes = processReader.get_processes(sqrts) + yamlParticleData = processReader.get_particle_data() + generators = Generators(processReader) try: - output_dir = getattr(reader, "outdir", "Run-Cards") + output_dir = getattr(processReader, "outdir", "Run-Cards") except KeyError: # If no directory set in input, use default output_dir = "Run-Cards" @@ -98,14 +98,14 @@ def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): process_instances = {} rndmIncrement = 0 for key, value in processes.items(): - make_output_directory(reader.gens(), output_dir, key) + make_output_directory(processReader.get_generators(), output_dir, key) try: randomseed = value["randomseed"] except: randomseed = rndmSeedFallback + rndmIncrement value["randomseed"] = randomseed rndmIncrement += 1 - param = ProcessParameters(reader) + param = ProcessParameters(processReader) # instantiate the class for each process process_instances[key] = Process( value, key, param, yamlParticleData, OutDir=output_dir diff --git a/python/YamlInputReader.py b/python/YamlInputReader.py index 1015391c..c1666b54 100644 --- a/python/YamlInputReader.py +++ b/python/YamlInputReader.py @@ -59,7 +59,7 @@ def get_subblock(self, k1, k2): except: return None - def gens(self): + def get_generators(self): if not self.is_set("generators"): raise ValueError("No Generators set!") return getattr(self, "generators") From 8514e8a13cc0fb517296d02c42b7720684abb3ba Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 31 Mar 2026 11:59:39 +0200 Subject: [PATCH 49/88] cosmetics --- python/YamlInputReader.py | 2 + python/main.py | 215 -------------------------------------- 2 files changed, 2 insertions(+), 215 deletions(-) delete mode 100755 python/main.py diff --git a/python/YamlInputReader.py b/python/YamlInputReader.py index c1666b54..fafe7626 100644 --- a/python/YamlInputReader.py +++ b/python/YamlInputReader.py @@ -17,6 +17,8 @@ def __init__(self, file, sqrts): self.load_file() for key, value in self.settings.items(): + print(key) + print(value) if key.lower() == "events": try: if "k" in value: diff --git a/python/main.py b/python/main.py deleted file mode 100755 index aa7cdfea..00000000 --- a/python/main.py +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import argparse -import textwrap -from datetime import datetime - -import ReleaseSpecs -from ReleaseSpecs import ReleaseSpec -import Input as Settings -import Process as process_module -import Generators as generators_module -from Particles import ParticleCollection - -def make_output_directory(generators, output_directory, procname): - # Overwrite directory if it exists - if not os.path.exists(output_directory): - os.makedirs(output_directory) - for generator in generators: - generator_directory = os.path.join(output_directory, generator, procname) - if not os.path.exists(generator_directory): - os.makedirs(generator_directory) - - -def main(args=None): - # parser = argparse.ArgumentParser(prog='k4gen',description='Process input YAML files.') - parser = argparse.ArgumentParser( - prog="k4GeneratorsConfig", - formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent( - """\ -Process input YAML files. -The following options are available: ------------------------------------- -SqrtS : float (center of mass energy) -ISRmode : int (0: off, 1: on) - OutputFormat : string (format output, available are hepmc2 and hepmc3) -OutDir : string (output directory, default=$PWD/Run-Cards) -Events : unsigned int (Number of Monte-Carlo events to be generated) -RandomSeed : unsigned int (specify a random seed, important when generating multiple files for the same process) -Processes : see README A list of processes which runcards should be generated. Each process should have its own unique name - Processes: - Muon: - Initial: [11, -11] - Final: [13, -13] -ParticleData : overwrite basic particle properties - ParticleData: - 25: - mass: 125 - width: 0 - -For MADGRAPH and Whizard only: -PolarisationDensity : int ([-1 or 1, 1 or -1]) default: [-1, 1] -PolarisationFraction : float ([0...1.,0....1.]), default [0,0] -Beamstrahlung : string (name of accelerator: ILC, FCC, CLIC, C3, HALFHF) - """ - ), - ) - parser.add_argument("inputfiles", nargs="+", type=str, help="Input YAML file") - parser.add_argument( - "--ecms", - nargs="*", - type=float, - default=[], - help="energies to be processed, overrides nominal yaml input file settings", - ) - parser.add_argument( - "--ecmsFiles", - nargs="*", - type=str, - default=[], - help="yaml files with energies format ecms: [energy1,....] ", - ) - parser.add_argument( - "--seed", - nargs=1, - type=int, - default=4711, - help="initial random number seed, increment for each file", - ) - parser.add_argument( - "--nevts", - type=int, - default=-1, - help="Number of events to be generated", - ) - parser.add_argument( - "--parameterTag", - type=str, - default="latest", - help="parameter tag in Parameters.yaml default is: latest", - ) - parser.add_argument( - "--parameterTagFile", - type=str, - default="ParameterSets.yaml", - help="name of file containing the parameter sets of the requested parameterTag, default: ParameterSets.yaml in directory: python", - ) - parser.add_argument( - "--key4hepUseNightlies", - action='store_true', - help="configures the key4hepscripts to use nightlies instead of releases", - ) - parser.add_argument( - "--key4hepVersion", - default=None, - help="force the use of the version in default is latest, format: YYYY-MM-DD", - ) - args = parser.parse_args(args) - files = args.inputfiles - energies = args.ecms - ecmsfiles = args.ecmsFiles - rndmSeed = args.seed - events = args.nevts - paramTag = args.parameterTag - paramFileName = os.path.dirname(os.path.realpath(__file__))+"/"+args.parameterTagFile - releaseDate = args.key4hepVersion - - ReleaseSpec.set_info("key4hepUseNightlies",args.key4hepUseNightlies) - if ReleaseSpecs.key4hepUseNightlies.value: - print(f"key4HEP configuration: NIGHTLIES") - else: - print(f"key4HEP configuration: RELEASE") - - # make sure it's a valid date - if releaseDate is not None: - try: - relDate = datetime.strptime(releaseDate,'%Y-%m-%d') - if (datetime.today() - relDate).days < 0: - raise ValueError() - print(f"key4HEP configuration date: {releaseDate}") - except ValueError: - print(f"Invalid KEY4HEP release argument, YYYY-MM-DD expected, latest possible date {datetime.today().strftime('%Y-%m-%d')}") - print(f"Requested: {releaseDate}") - print("Cannot configure scripts correctly, exiting") - exit() - else: - print(f"key4HEP configuration date: latest") - # store for future use: - ReleaseSpec.set_info("key4hepReleaseDate",releaseDate) - - # so additionally we read the argument ecmsFile - for ecmsfile in ecmsfiles: - # open and read ecms file and append the energies to the command line arguments - ecmSettings = Settings.ECMSInput(ecmsfile) - energies.extend(ecmSettings.energies()) - - # now we read the global settings - try: - # make sure that we follow a symlink to the real location of the parametersets should replace that by share? - parameterSet = Settings.ParameterSets(paramFileName, paramTag) - except FileNotFoundError as e: - print(f"ERROR: File {e} with parameters for tag {paramTag} not found") - exit() - - # now execute file processes - if len(energies) == 0: - executeFiles(files, 0, rndmSeed, events) - else: - for sqrts in energies: - rndmIncrement = executeFiles(files, sqrts, rndmSeed, events) - # offset for next round by number of yaml files - rndmSeed = rndmSeed + rndmIncrement - - -def executeFiles(files, sqrts, rndmSeedFallback=4711, events=-1): - # first step reset all particles: - ParticleCollection() - if sqrts == 0: - print("Generating and writing configuration files") - else: - print("Generating and writing configuration files for ECM= ", sqrts) - - for yaml_file in files: - # read the input file - settings = Settings.Input(yaml_file, sqrts) - # set the number of events if present - if events != -1: - settings.set("events", events) - settings.gens() - processes = settings.get_processes(sqrts) - yamlParticleData = settings.get_particle_data() - generators = generators_module.Generators(settings) - try: - output_dir = getattr(settings, "outdir", "Run-Cards") - except KeyError: - # If no directory set in input, use default - output_dir = "Run-Cards" - - process_instances = {} - rndmIncrement = 0 - for key, value in processes.items(): - make_output_directory(settings.gens(), output_dir, key) - try: - randomseed = value["randomseed"] - except: - randomseed = rndmSeedFallback + rndmIncrement - value["randomseed"] = randomseed - rndmIncrement += 1 - param = process_module.ProcessParameters(settings) - # instantiate the class for each process - process_instances[key] = process_module.Process( - value, key, param, yamlParticleData, OutDir=output_dir - ) - # increment counter for randomseed - for process_instance in process_instances.values(): - process_instance.prepareProcess() - generators.runGeneratorConfiguration(process_instance) - - return rndmIncrement - - -if __name__ == "__main__": - main() From 1701ce9c89e1d92063519b2a23d8768bac3fcb33 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 31 Mar 2026 12:02:35 +0200 Subject: [PATCH 50/88] cleanup --- python/Yaml2Datacard.py | 10 +++++----- python/YamlInputReader.py | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 93f40ac8..fd98b58e 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -95,7 +95,7 @@ def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): # If no directory set in input, use default output_dir = "Run-Cards" - process_instances = {} + processesDict = {} rndmIncrement = 0 for key, value in processes.items(): make_output_directory(processReader.get_generators(), output_dir, key) @@ -107,13 +107,13 @@ def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): rndmIncrement += 1 param = ProcessParameters(processReader) # instantiate the class for each process - process_instances[key] = Process( + processesDict[key] = Process( value, key, param, yamlParticleData, OutDir=output_dir ) # increment counter for randomseed - for process_instance in process_instances.values(): - process_instance.prepareProcess() - generators.runGeneratorConfiguration(process_instance) + for process in processesDict.values(): + process.prepareProcess() + generators.runGeneratorConfiguration(process) return rndmIncrement diff --git a/python/YamlInputReader.py b/python/YamlInputReader.py index fafe7626..c1666b54 100644 --- a/python/YamlInputReader.py +++ b/python/YamlInputReader.py @@ -17,8 +17,6 @@ def __init__(self, file, sqrts): self.load_file() for key, value in self.settings.items(): - print(key) - print(value) if key.lower() == "events": try: if "k" in value: From 04a7b215475960079e90e6e20d2f119fa2f9395c Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 31 Mar 2026 17:09:15 +0200 Subject: [PATCH 51/88] remove superfluous component --- python/Parameters.py | 78 ++++++++++++++--------------------------- python/Yaml2Datacard.py | 26 ++++++-------- 2 files changed, 38 insertions(+), 66 deletions(-) diff --git a/python/Parameters.py b/python/Parameters.py index b49da4fa..648ab112 100644 --- a/python/Parameters.py +++ b/python/Parameters.py @@ -2,10 +2,10 @@ class Parameter: - require_args = ["name", "value", "isParticleProperty", "texname"] + require_args = ["name", "value", "isParticleProperty"] - def __init__(self, name, value, isParticleProperty, texname): - args = (name, value, isParticleProperty, texname) + def __init__(self, name, value, isParticleProperty): + args = (name, value, isParticleProperty) for i, prop in enumerate(self.require_args): setattr(self, prop, args[i]) # keep a global list of parameters @@ -38,169 +38,145 @@ def updateList(name): alphaEMMZM1 = Parameter( name="alphaEMMZM1", value=127.9, - isParticleProperty=False, - texname="\\text{aEWM1}" + isParticleProperty=False ) alphaEMMZ = Parameter( name="alphaEMMZ", value=1/alphaEMMZM1.value, - isParticleProperty=False, - texname="\\alpha _{\\text{EW}}" + isParticleProperty=False ) alphaEMLOM1 = Parameter( name="alphaEMLOM1", value=1.32184e+02, - isParticleProperty=False, - texname="\\text{aEWLOM1}" + isParticleProperty=False ) alphaEMLO = Parameter( name="alphaEMLO", value=1/alphaEMLOM1.value, - isParticleProperty=False, - texname="\\alpha _{\\text{aEW LO}}" + isParticleProperty=False ) alphaEMM1 = Parameter( name="alphaEMM1", value=137.035999139, - isParticleProperty=False, - texname="\\text{aEW,Q=0,M1}" + isParticleProperty=False ) alphaEM = Parameter( name="alphaEM", value=1/alphaEMM1.value, - isParticleProperty=False, - texname="\\alpha _{\\text{EW,Q=0}}" + isParticleProperty=False ) GFermi = Parameter( name="GFermi", value=0.0000116637, - isParticleProperty=False, - texname="G_f" + isParticleProperty=False ) sin2thetaLO = Parameter( name="sin2thetaLO", value=0.223013, - isParticleProperty=False, - texname="sin ^{2}\\theta" + isParticleProperty=False ) sin2theta = Parameter( name="sin2theta", value=0.23155, - isParticleProperty=False, - texname="sin ^{2}\\theta" + isParticleProperty=False ) sin2thetaEff = Parameter( name="sin2thetaEff", value=0.23155, - isParticleProperty=False, - texname="sin ^{2}\\theta _{Eff}" + isParticleProperty=False ) alphaSMZ = Parameter( name="alphaSMZ", value=0.1184, - isParticleProperty=False, - texname="\\alpha _s" + isParticleProperty=False ) VEV = Parameter( name="VEV", value=246, - isParticleProperty=False, - texname="\\text{vev}" + isParticleProperty=False ) MZ = Parameter( name="MZ", value=91.1876, - isParticleProperty=True, - texname="\\text{MZ}" + isParticleProperty=True ) WZ = Parameter( name="WZ", value=2.4952, - isParticleProperty=True, - texname="\\text{WZ}" + isParticleProperty=True ) MW = Parameter( name="MW", value=80.379, - isParticleProperty=True, - texname="M_W" + isParticleProperty=True ) WW = Parameter( name="WW", value=2.085, - isParticleProperty=True, - texname="\\text{WW}" + isParticleProperty=True ) MB = Parameter( name="MB", value=4.7, - isParticleProperty=True, - texname="\\text{MB}" + isParticleProperty=True ) ymb = Parameter( name="ymb", value=MB.value/VEV.value, - isParticleProperty=True, - texname="\\text{ymb}" + isParticleProperty=True ) MT = Parameter( name="MT", value=172, - isParticleProperty=True, - texname="\\text{MT}" + isParticleProperty=True ) WT = Parameter( name="WT", value=1.50833649, - isParticleProperty=True, - texname="\\text{WT}" + isParticleProperty=True ) ymt = Parameter( name="ymt", value=MT.value/VEV.value, - isParticleProperty=True, - texname="\\text{ymt}" + isParticleProperty=True ) MH = Parameter( name="MH", value=125, - isParticleProperty=True, - texname="\\text{MH}" + isParticleProperty=True ) WH = Parameter( name="WH", value=0.00407, - isParticleProperty=True, - texname="\\text{WH}" + isParticleProperty=True ) MU_R = Parameter( name="MU_R", value=91.188, - isParticleProperty=False, - texname="\\text{\\mu_r}" + isParticleProperty=False ) diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index fd98b58e..90db916b 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -14,16 +14,16 @@ from Generators import Generators from Particles import ParticleCollection -def make_output_directory(generators, output_directory, procname): - # Overwrite directory if it exists - if not os.path.exists(output_directory): - os.makedirs(output_directory) +def makeDirectories4GeneratorsProcess(generatorDir, generators, procname): + # do not overwrite directory if it exists + if not os.path.exists(generatorDir): + os.makedirs(generatorDir) for generator in generators: - generator_directory = os.path.join(output_directory, generator, procname) - if not os.path.exists(generator_directory): - os.makedirs(generator_directory) + process_directory = os.path.join(generatorDir, generator, procname) + if not os.path.exists(process_directory): + os.makedirs(process_directory) -def Yaml2Datacard(args=None): +def Yaml2Datacard(args): ReleaseSpec.set_info("key4hepUseNightlies",args.key4hepUseNightlies) if ReleaseSpecs.key4hepUseNightlies.value: @@ -89,16 +89,12 @@ def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): processes = processReader.get_processes(sqrts) yamlParticleData = processReader.get_particle_data() generators = Generators(processReader) - try: - output_dir = getattr(processReader, "outdir", "Run-Cards") - except KeyError: - # If no directory set in input, use default - output_dir = "Run-Cards" + generatorDir = getattr(processReader, "outdir", "Run-Cards") processesDict = {} rndmIncrement = 0 for key, value in processes.items(): - make_output_directory(processReader.get_generators(), output_dir, key) + makeDirectories4GeneratorsProcess(generatorDir, processReader.get_generators(), key) try: randomseed = value["randomseed"] except: @@ -108,7 +104,7 @@ def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): param = ProcessParameters(processReader) # instantiate the class for each process processesDict[key] = Process( - value, key, param, yamlParticleData, OutDir=output_dir + value, key, param, yamlParticleData, OutDir=generatorDir ) # increment counter for randomseed for process in processesDict.values(): From 23666f072d5d7217f7c4ec8cb68b23ea692fc6eb Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 31 Mar 2026 17:14:27 +0200 Subject: [PATCH 52/88] bug corr --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d90b1d4..788300ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,7 +57,7 @@ install(DIRECTORY ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/ DESTINATION ${C install(DIRECTORY ${PROJECT_SOURCE_DIR}/python DESTINATION ${CMAKE_INSTALL_PREFIX} - REGEX test_.*\\.py|main.py$ EXCLUDE # Don't install python unittests + REGEX test_.*\\.py|k4GeneratorsConfig.py$ EXCLUDE # Don't install python unittests PATTERN __pycache__ EXCLUDE # Or pythons caches ) # Install main as an executable From 5ae173706a8243883215cec1d6dcf4ccb925ae30 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 31 Mar 2026 17:16:40 +0200 Subject: [PATCH 53/88] bug corr --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 788300ef..818bc537 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,7 +61,7 @@ install(DIRECTORY ${PROJECT_SOURCE_DIR}/python PATTERN __pycache__ EXCLUDE # Or pythons caches ) # Install main as an executable -install(PROGRAMS ${PROJECT_SOURCE_DIR}/python/main.py DESTINATION ${CMAKE_INSTALL_PREFIX}/python) +install(PROGRAMS ${PROJECT_SOURCE_DIR}/python/k4GeneratorsConfig.py DESTINATION ${CMAKE_INSTALL_PREFIX}/python) install(CODE "execute_process(COMMAND bash -c \"set -e link_target=${CMAKE_INSTALL_PREFIX}/python/k4GeneratorsConfig.py From 96ea732b4a38327ed687ea8a42ad49ec02f8be93 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 31 Mar 2026 21:22:38 +0200 Subject: [PATCH 54/88] consistent treatment of generatorDir as override --- python/Production.py | 2 +- python/Yaml2Datacard.py | 16 +++++++++------- python/k4GeneratorsConfig.py | 15 ++++++++++++--- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/python/Production.py b/python/Production.py index 937aaf8a..9baa196d 100644 --- a/python/Production.py +++ b/python/Production.py @@ -25,7 +25,7 @@ def __init__(self, args): if args.refDir.startswith('/'): self._referenceDir = args.refDir - self._generatorDir = f"{self._workDir}/{args.generatorDirName}" + self._generatorDir = f"{self._workDir}/{args.generatorDir}" def getGenerators(self, generator): if generator == "All": diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 90db916b..38f2db29 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -65,14 +65,14 @@ def Yaml2Datacard(args): # now execute file processes rndmSeed = args.seed if len(energies) == 0: - executeFiles(args.yaml, 0, rndmSeed, args.nevts) + executeFiles(args, 0, rndmSeed) else: for sqrts in energies: - rndmIncrement = executeFiles(args.yaml, sqrts, rndmSeed, args.nevts) + rndmIncrement = executeFiles(args, sqrts, rndmSeed) # offset for next round by number of yaml files rndmSeed = rndmSeed + rndmIncrement -def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): +def executeFiles(args, sqrts, rndmSeedFallback=4711): # first step reset all particles: ParticleCollection() if sqrts == 0: @@ -81,16 +81,18 @@ def executeFiles(yaml, sqrts, rndmSeedFallback=4711, events=-1): print("Generating and writing configuration files for ECM= ", sqrts) # read the input file - processReader = Reader.ProcessReader(yaml, sqrts) + processReader = Reader.ProcessReader(args.yaml, sqrts) # set the number of events if present - if events != -1: + if args.nevts != -1: processReader.set("events", events) processReader.get_generators() processes = processReader.get_processes(sqrts) yamlParticleData = processReader.get_particle_data() generators = Generators(processReader) - generatorDir = getattr(processReader, "outdir", "Run-Cards") - + # + generatorDir = getattr(processReader, "outdir", args.generatorDir) + if args.generatorDirOverwrite: + generatorDir = args.generatorDir processesDict = {} rndmIncrement = 0 for key, value in processes.items(): diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index c3b24078..e280ca09 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -106,11 +106,13 @@ def k4GeneratorsConfig(arguments=None): default="output", help="path to the output (default: ./output)" ) + generatorDirDefault = "Run-Cards" parser.add_argument( - "--generatorDirName", + "--generatorDir", type=str, - default="Run-Cards", - help="relative path to the Generator directories in outputDir (default: outputDir/Run-Cards)" + #default="Run-Cards", + default=argparse.SUPPRESS, + help=f"relative path to the Generator directories in outputDir (default: OUTPUTDIR/{generatorDirDefault})" ) parser.add_argument( "--all", @@ -119,6 +121,13 @@ def k4GeneratorsConfig(arguments=None): ) args = parser.parse_args(arguments) + try: + check = args.generatorDir + args.generatorDirOverwrite = True + except AttributeError: + # argument was not given, set the default and make it known: + args.generatorDir = generatorDirDefault + args.generatorDirOverwrite = False # --all overrides --make --generate --summary if args.all: args.make = True From d550c65272abf6223c0c0b3a6ff7311607535b7a Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 31 Mar 2026 22:33:35 +0200 Subject: [PATCH 55/88] further simplification --- python/Yaml2Datacard.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 38f2db29..cbd16382 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -49,10 +49,10 @@ def Yaml2Datacard(args): ReleaseSpec.set_info("key4hepReleaseDate",args.key4hepVersion) # sqrts choices from file - energies = [] + energies = [0.] if args.sqrts: - ecmSettings = Reader.SQRTSReader(args.sqrts) - energies.extend(ecmSettings.energies()) + sqrtsReader = Reader.SQRTSReader(args.sqrts) + energies = sqrtsReader.energies() # now we read the global settings try: @@ -62,30 +62,22 @@ def Yaml2Datacard(args): print(f"ERROR: File {e} with parameters for tag {args.parameterTag} not found") exit() - # now execute file processes + # execute file processes rndmSeed = args.seed - if len(energies) == 0: - executeFiles(args, 0, rndmSeed) - else: - for sqrts in energies: - rndmIncrement = executeFiles(args, sqrts, rndmSeed) - # offset for next round by number of yaml files - rndmSeed = rndmSeed + rndmIncrement + for sqrts in energies: + rndmIncrement = executeFiles(args, sqrts, rndmSeed) + # offset for next round by number of yaml files + rndmSeed = rndmSeed + rndmIncrement -def executeFiles(args, sqrts, rndmSeedFallback=4711): +def executeFiles(args, sqrts, rndmSeedFallback): # first step reset all particles: ParticleCollection() - if sqrts == 0: - print("Generating and writing configuration files") - else: - print("Generating and writing configuration files for ECM= ", sqrts) - # read the input file processReader = Reader.ProcessReader(args.yaml, sqrts) # set the number of events if present if args.nevts != -1: processReader.set("events", events) - processReader.get_generators() + processes = processReader.get_processes(sqrts) yamlParticleData = processReader.get_particle_data() generators = Generators(processReader) From 906926ab58e8d6a6e5aee07685fa80d0b2dc810a Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 00:00:54 +0200 Subject: [PATCH 56/88] #77 bugfix memory in prepareProcess and move Yaml2Datacard to object for future work --- python/Process.py | 3 +- python/Yaml2Datacard.py | 164 ++++++++++++++++++++-------------------- 2 files changed, 84 insertions(+), 83 deletions(-) diff --git a/python/Process.py b/python/Process.py index cbd2b691..deebd397 100644 --- a/python/Process.py +++ b/python/Process.py @@ -33,6 +33,7 @@ def __init__(self, args, procname, params, particleData, **options): for arg in self._required_args: setattr(self, arg, params.settings.get(arg)) + for setting in dir(params): if not setting.startswith("__"): setattr(self, setting, getattr(params, setting)) @@ -63,7 +64,7 @@ def prepareProcess(self): # now the new DBTag: initialstate = [self.initial[0], self.initial[1]] initialstate.sort() - finalstate = self.final + finalstate = [x for x in self.final] finalstate.sort() self._DBTag = [initialstate, finalstate] diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index cbd16382..1f3b6662 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -14,99 +14,99 @@ from Generators import Generators from Particles import ParticleCollection -def makeDirectories4GeneratorsProcess(generatorDir, generators, procname): - # do not overwrite directory if it exists - if not os.path.exists(generatorDir): - os.makedirs(generatorDir) - for generator in generators: - process_directory = os.path.join(generatorDir, generator, procname) - if not os.path.exists(process_directory): - os.makedirs(process_directory) +class Yaml2Datacard: + """Convert Input files into generator datacards""" + + def __init__(self,args): -def Yaml2Datacard(args): + ReleaseSpec.set_info("key4hepUseNightlies",args.key4hepUseNightlies) + if ReleaseSpecs.key4hepUseNightlies.value: + print(f"key4HEP configuration: NIGHTLIES") + else: + print(f"key4HEP configuration: RELEASE") - ReleaseSpec.set_info("key4hepUseNightlies",args.key4hepUseNightlies) - if ReleaseSpecs.key4hepUseNightlies.value: - print(f"key4HEP configuration: NIGHTLIES") - else: - print(f"key4HEP configuration: RELEASE") + # make sure it's a valid date + if args.key4hepVersion is not None: + try: + relDate = datetime.strptime(args.key4hepVersion,'%Y-%m-%d') + if (datetime.today() - relDate).days < 0: + raise ValueError() + print(f"key4HEP configuration date: {args.key4hepVersion}") + except ValueError: + print(f"Invalid KEY4HEP release argument, YYYY-MM-DD expected, latest possible date {datetime.today().strftime('%Y-%m-%d')}") + print(f"Requested: {args.key4hepVersion}") + print("Cannot configure scripts correctly, exiting") + exit() + else: + print(f"key4HEP configuration date: latest") + # store for future use: + ReleaseSpec.set_info("key4hepReleaseDate",args.key4hepVersion) - # make sure it's a valid date - if args.key4hepVersion is not None: + # sqrts choices from file + energies = [0.] + if args.sqrts: + sqrtsReader = Reader.SQRTSReader(args.sqrts) + energies = sqrtsReader.energies() + + # now we read the global settings try: - relDate = datetime.strptime(args.key4hepVersion,'%Y-%m-%d') - if (datetime.today() - relDate).days < 0: - raise ValueError() - print(f"key4HEP configuration date: {args.key4hepVersion}") - except ValueError: - print(f"Invalid KEY4HEP release argument, YYYY-MM-DD expected, latest possible date {datetime.today().strftime('%Y-%m-%d')}") - print(f"Requested: {args.key4hepVersion}") - print("Cannot configure scripts correctly, exiting") + # make sure that we follow a symlink to the real location of the parametersets should replace that by share? + parameterSet = Reader.ParameterSetReader(args.parameterTagFile, args.parameterTag) + except FileNotFoundError as e: + print(f"ERROR: File {e} with parameters for tag {args.parameterTag} not found") exit() - else: - print(f"key4HEP configuration date: latest") - # store for future use: - ReleaseSpec.set_info("key4hepReleaseDate",args.key4hepVersion) - - # sqrts choices from file - energies = [0.] - if args.sqrts: - sqrtsReader = Reader.SQRTSReader(args.sqrts) - energies = sqrtsReader.energies() - # now we read the global settings - try: - # make sure that we follow a symlink to the real location of the parametersets should replace that by share? - parameterSet = Reader.ParameterSetReader(args.parameterTagFile, args.parameterTag) - except FileNotFoundError as e: - print(f"ERROR: File {e} with parameters for tag {args.parameterTag} not found") - exit() + # execute file processes + rndmSeed = args.seed + for sqrts in energies: + rndmIncrement = self.executeFiles(args, sqrts, rndmSeed) + # offset for next round by number of yaml files + rndmSeed = rndmSeed + rndmIncrement - # execute file processes - rndmSeed = args.seed - for sqrts in energies: - rndmIncrement = executeFiles(args, sqrts, rndmSeed) - # offset for next round by number of yaml files - rndmSeed = rndmSeed + rndmIncrement + def executeFiles(self, args, sqrts, rndmSeedFallback): + # first step reset all particles: + ParticleCollection() + # read the input file + processReader = Reader.ProcessReader(args.yaml, sqrts) + # set the number of events if present + if args.nevts != -1: + processReader.set("events", events) -def executeFiles(args, sqrts, rndmSeedFallback): - # first step reset all particles: - ParticleCollection() - # read the input file - processReader = Reader.ProcessReader(args.yaml, sqrts) - # set the number of events if present - if args.nevts != -1: - processReader.set("events", events) + processes = processReader.get_processes(sqrts) + yamlParticleData = processReader.get_particle_data() + generators = Generators(processReader) + # + generatorDir = getattr(processReader, "outdir", args.generatorDir) + if args.generatorDirOverwrite: + generatorDir = args.generatorDir - processes = processReader.get_processes(sqrts) - yamlParticleData = processReader.get_particle_data() - generators = Generators(processReader) - # - generatorDir = getattr(processReader, "outdir", args.generatorDir) - if args.generatorDirOverwrite: - generatorDir = args.generatorDir - processesDict = {} - rndmIncrement = 0 - for key, value in processes.items(): - makeDirectories4GeneratorsProcess(generatorDir, processReader.get_generators(), key) - try: - randomseed = value["randomseed"] - except: - randomseed = rndmSeedFallback + rndmIncrement - value["randomseed"] = randomseed - rndmIncrement += 1 - param = ProcessParameters(processReader) - # instantiate the class for each process - processesDict[key] = Process( - value, key, param, yamlParticleData, OutDir=generatorDir - ) - # increment counter for randomseed - for process in processesDict.values(): - process.prepareProcess() - generators.runGeneratorConfiguration(process) + rndmIncrement = 0 + for key, value in processes.items(): + self.makeDirectories4GeneratorsProcess(generatorDir, processReader.get_generators(), key) + try: + randomseed = value["randomseed"] + except: + randomseed = rndmSeedFallback + rndmIncrement + value["randomseed"] = randomseed + rndmIncrement += 1 + param = ProcessParameters(processReader) + # instantiate the class for each process + process = Process( + value, key, param, yamlParticleData, OutDir=generatorDir + ) + process.prepareProcess() + generators.runGeneratorConfiguration(process) - return rndmIncrement + return rndmIncrement + def makeDirectories4GeneratorsProcess(self, generatorDir, generators, procname): + # do not overwrite directory if it exists + if not os.path.exists(generatorDir): + os.makedirs(generatorDir) + for generator in generators: + process_directory = os.path.join(generatorDir, generator, procname) + if not os.path.exists(process_directory): + os.makedirs(process_directory) if __name__ == "__main__": Yaml2Datacard() From 3e8d3511ab4813815fac43f07510bdfdfe6b7799 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 00:05:11 +0200 Subject: [PATCH 57/88] #77 clang --- python/Yaml2Datacard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 1f3b6662..3f8ce721 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -16,7 +16,7 @@ class Yaml2Datacard: """Convert Input files into generator datacards""" - + def __init__(self,args): ReleaseSpec.set_info("key4hepUseNightlies",args.key4hepUseNightlies) From 8f00b9f2fdf7e9d3869666f2904bd67b168a525c Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 11:07:12 +0200 Subject: [PATCH 58/88] consistent treatment of outputdirectories --- .github/workflows/key4hep_build.yml | 2 +- python/Generators/GeneratorBase.py | 4 +- python/Production.py | 82 +++++++---------------------- python/Yaml2Datacard.py | 48 +++++++++++------ python/k4GeneratorsConfig.py | 25 +++------ 5 files changed, 59 insertions(+), 102 deletions(-) diff --git a/.github/workflows/key4hep_build.yml b/.github/workflows/key4hep_build.yml index a63c6fbf..00663756 100644 --- a/.github/workflows/key4hep_build.yml +++ b/.github/workflows/key4hep_build.yml @@ -33,7 +33,7 @@ jobs: with: name: k4GeneratorsConfig_report_${{ matrix.build_type }}_${{ matrix.image }} path: | - test/output/* + test/Run-Cards/* - name: Show full CTest log on failure if: failure() run: | diff --git a/python/Generators/GeneratorBase.py b/python/Generators/GeneratorBase.py index 06c70fe4..c5050fa8 100644 --- a/python/Generators/GeneratorBase.py +++ b/python/Generators/GeneratorBase.py @@ -32,9 +32,9 @@ def __init__(self, procinfo, settings, name, inputFileExtension): # check consistency: note that the ParameterSets have been defined, check with the particle masses self.checkModelParameters() - # define the output directory as function of the OutDir spec + generator name + process name + # define OutDir as generator name + process name, we assume that we are in the working directory self.outdir = ( - f"{procinfo.get('OutDir')}/{self.name}/{self.procinfo.get('procname')}" + f"{self.name}/{self.procinfo.get('procname')}" ) # configure the filenames diff --git a/python/Production.py b/python/Production.py index 9baa196d..af2ef218 100644 --- a/python/Production.py +++ b/python/Production.py @@ -13,28 +13,22 @@ class ProductionBase(ABC): def __init__(self, args): # consistent processing of names - self._workDir = os.getcwd()+"/"+args.workDir - if args.workDir.startswith('/'): - self._workDir = args.workDir - - self._outDir = os.getcwd()+"/"+args.outputDir + self._outputDir = os.getcwd()+"/"+args.outputDir if args.outputDir.startswith('/'): - self._outdir = args.outputDir + self._outputDir = args.outputDir self._referenceDir = os.getcwd()+"/"+args.refDir if args.refDir.startswith('/'): self._referenceDir = args.refDir - self._generatorDir = f"{self._workDir}/{args.generatorDir}" - def getGenerators(self, generator): if generator == "All": - return os.listdir(self._generatorDir) + return os.listdir(self._outputDir) else: return [generator] def getProcesses(self, generator): - return os.listdir(f"{self._generatorDir}/{generator}") + return os.listdir(f"{self._outputDir}/{generator}") def getFileNames(self, directory): filenames = os.listdir(directory) @@ -50,12 +44,6 @@ def makeDirectory(self, dirname, overwrite=True): shutil.rmtree(dirname) os.makedirs(dirname) - def makeOutputDirectory(self, generator, process): - if generator not in os.listdir(self._outDir): - self.makeDirectory(f"{self._outDir}/{generator}") - if process not in os.listdir(f"{self._outDir}/{generator}"): - self.makeDirectory(f"{self._outDir}/{generator}/{process}") - def process(self, generators): # where do we start from cwd = os.getcwd() @@ -71,7 +59,8 @@ def process(self, generators): if failure: sys.exit("Failed") # return to the starting point - os.chdir(cwd) + if (cwd != os.getcwd()): + os.chdir(cwd) @abstractmethod def execute(self, generator, process): @@ -84,8 +73,6 @@ def __init__(self, args): super().__init__(args) # make the directory for the work, protect against "./" - self.makeDirectory(args.workDir, not (os.path.abspath(args.workDir) == os.getcwd() )) - # make the output directory for the output self.makeDirectory(args.outputDir, not (os.path.abspath(args.outputDir) == os.getcwd() )) # the sqrts argument can be a list of sqrts or a list of strings @@ -95,7 +82,7 @@ def __init__(self, args): all(float(val) for val in args.sqrts) # write the values to a file try: - sqrtsGlobalFileName = f"{self._workDir}/sqrts.yaml" + sqrtsGlobalFileName = f"{self._outputDir}/sqrts.yaml" sqrtsFile = open(sqrtsGlobalFileName,"x") sqrtsList = "ecms: [ " for sqrts in args.sqrts: @@ -120,35 +107,14 @@ def __init__(self, args): if not sqrtsGlobalFileName: self.prepareSQRTS(args.sqrts); - # args modifiable: + # args to transfer: self.Yaml2DatacardArgs = args # run all yamls: self.run(sqrtsGlobalFileName); - # move the output to storage - generators = self.getGenerators("All") - # now compare to reference - self.process(generators) - def execute(self, generator, process): - - self.makeOutputDirectory(generator, process) - outDir = f"{self._outDir}/{generator}/{process}" - - genProcDir = f"{self._generatorDir}/{generator}/{process}" - fileNames = self.getFileNames(genProcDir) - - success = True - for name in fileNames: - theFile = f"{genProcDir}/{name}" - try: - shutil.copy(theFile, outDir) - except: - print(f"{theFile} could not be copied to {outDir}") - success= False - - return success + pass def prepareYamls(self, yamlList): # check whether this is a list of directories or mixed or files @@ -184,7 +150,7 @@ def run(self, sqrtsGlobal): # remember where we start from cwd = os.getcwd() # go to the working directory - os.chdir(self._workDir) + os.chdir(self._outputDir) # loop over all files for filename in self._yamlFiles: @@ -223,8 +189,8 @@ def __init__(self, args): def execute(self, generator, process): genProc = f"{generator}/{process}" - newDir = f"{self._generatorDir}/{genProc}" - refDir = f"{self._referenceDir}/{genProc}" + newDir = f"{self._outputDir}/{genProc}" + refDir = f"{self._outputDir}/{genProc}" fileNames = self.getFileNames(newDir) success = True @@ -272,7 +238,7 @@ def __init__(self, args): def execute(self, generator, process): # go to the directory - genProcDir = f"{self._generatorDir}/{generator}/{process}" + genProcDir = f"{self._outputDir}/{generator}/{process}" os.chdir(genProcDir) # retrieve the script scripts = [script for script in os.listdir(genProcDir) if script.startswith('Run_') and script.endswith('.sh')] @@ -290,17 +256,6 @@ def execute(self, generator, process): print(e.output) success = False - self.makeOutputDirectory(generator, process) - outDir = f"{self._outDir}/{generator}/{process}" - fileNames = [filename for filename in self.getFileNames(genProcDir) if filename.endswith('.edm4hep')] - for name in fileNames: - theFile = f"{genProcDir}/{name}" - try: - shutil.copy(theFile, outDir) - except: - print(f"{theFile} could not be copied to {outDir}") - success= False - return success class summary(ProductionBase): @@ -312,12 +267,12 @@ def __init__(self, args): print("Extracting the cross sections by reading EDM4HEP files and superposing the differential distributions") # remember where we start from cwd = os.getcwd() - os.chdir(self._workDir) + os.chdir(self._outputDir) try: result = subprocess.run(["eventGenerationSummary", - "-w", f"{self._generatorDir}", - "-f", f"{self._outDir}/GenerationSummary.dat", - "-d", f"{self._outDir}"], + "-w", f"{self._outputDir}", + "-f", f"{self._outputDir}/GenerationSummary.dat", + "-d", f"{self._outputDir}"], capture_output=True, check=True) except subprocess.CalledProcessError as e: print(f"Execution error eventGenerationSummary") @@ -326,7 +281,8 @@ def __init__(self, args): sys.exit("Exception thrown by eventGenerationSummary") # return tu the starting point - os.chdir(cwd) + if (cwd != os.getcwd()): + os.chdir(cwd) def execute(self, generator, process): pass diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 3f8ce721..816908aa 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -64,47 +64,61 @@ def __init__(self,args): rndmSeed = rndmSeed + rndmIncrement def executeFiles(self, args, sqrts, rndmSeedFallback): + # remember where we started from: + cwd = os.getcwd() # first step reset all particles: ParticleCollection() # read the input file - processReader = Reader.ProcessReader(args.yaml, sqrts) + self.processReader = Reader.ProcessReader(args.yaml, sqrts) # set the number of events if present if args.nevts != -1: - processReader.set("events", events) - - processes = processReader.get_processes(sqrts) - yamlParticleData = processReader.get_particle_data() - generators = Generators(processReader) + self.processReader.set("events", events) + # the datacard outputDir may differ + self.processOutputDir(args) + # now extract information + processes = self.processReader.get_processes(sqrts) + yamlParticleData = self.processReader.get_particle_data() + generators = Generators(self.processReader) # - generatorDir = getattr(processReader, "outdir", args.generatorDir) - if args.generatorDirOverwrite: - generatorDir = args.generatorDir - rndmIncrement = 0 for key, value in processes.items(): - self.makeDirectories4GeneratorsProcess(generatorDir, processReader.get_generators(), key) + self.makeDirectories4GeneratorsProcess(self.processReader.get_generators(), key) try: randomseed = value["randomseed"] except: randomseed = rndmSeedFallback + rndmIncrement value["randomseed"] = randomseed rndmIncrement += 1 - param = ProcessParameters(processReader) + param = ProcessParameters(self.processReader) # instantiate the class for each process process = Process( - value, key, param, yamlParticleData, OutDir=generatorDir + value, key, param, yamlParticleData, OutDir=self.outputDir ) process.prepareProcess() generators.runGeneratorConfiguration(process) + # at the end back to the starting point dir for the next file + if (cwd != os.getcwd()): + os.chdir(cwd) return rndmIncrement - def makeDirectories4GeneratorsProcess(self, generatorDir, generators, procname): + def processOutputDir(self, args): + # the output directory + self.outputDir = getattr(self.processReader, "outdir", args.outputDir) + if args.outputDirOverride: + self.outputDir = args.outputDir + # the attribute always has to be reset to be sure.... + setattr(self.processReader, "outdir", args.outputDir) + # all the preparatory work has been done in args.outputDir + # create the new directory if it does not exist + if not args.outputDirOverride and not os.path.exists(self.outputDir): + os.makedirs(self.outputDir) + os.chdir(self.outputDir) + + def makeDirectories4GeneratorsProcess(self, generators, procname): # do not overwrite directory if it exists - if not os.path.exists(generatorDir): - os.makedirs(generatorDir) for generator in generators: - process_directory = os.path.join(generatorDir, generator, procname) + process_directory = os.path.join(generator, procname) if not os.path.exists(process_directory): os.makedirs(process_directory) diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index e280ca09..ae75c32c 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -94,25 +94,12 @@ def k4GeneratorsConfig(arguments=None): action='store_true', help="compare the results of the event generation process by process and produce summary output in outputDir" ) - parser.add_argument( - "--workDir", - type=str, - default="work", - help="path to work directory (default: ./work)" - ) + outputDirDefault = "Run-Cards" parser.add_argument( "--outputDir", type=str, - default="output", - help="path to the output (default: ./output)" - ) - generatorDirDefault = "Run-Cards" - parser.add_argument( - "--generatorDir", - type=str, - #default="Run-Cards", default=argparse.SUPPRESS, - help=f"relative path to the Generator directories in outputDir (default: OUTPUTDIR/{generatorDirDefault})" + help=f"path to output directory (default: ./{outputDirDefault}, if specified, overrides the outdir key in yaml)" ) parser.add_argument( "--all", @@ -122,12 +109,12 @@ def k4GeneratorsConfig(arguments=None): args = parser.parse_args(arguments) try: - check = args.generatorDir - args.generatorDirOverwrite = True + check = args.outputDir + args.outputDirOverride = True except AttributeError: # argument was not given, set the default and make it known: - args.generatorDir = generatorDirDefault - args.generatorDirOverwrite = False + args.outputDir = outputDirDefault + args.outputDirOverride = False # --all overrides --make --generate --summary if args.all: args.make = True From 2e87810c0a9c18ce8546838c9515817cc1fb030f Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 13:18:20 +0200 Subject: [PATCH 59/88] improved treatment of outdir --- python/Yaml2Datacard.py | 5 +++-- python/k4GeneratorsConfig.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 816908aa..1ee05399 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -111,8 +111,9 @@ def processOutputDir(self, args): setattr(self.processReader, "outdir", args.outputDir) # all the preparatory work has been done in args.outputDir # create the new directory if it does not exist - if not args.outputDirOverride and not os.path.exists(self.outputDir): - os.makedirs(self.outputDir) + if not args.outputDirOverride: + if not os.path.exists(self.outputDir): + os.makedirs(self.outputDir) os.chdir(self.outputDir) def makeDirectories4GeneratorsProcess(self, generators, procname): diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index ae75c32c..c5e5b248 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -94,7 +94,7 @@ def k4GeneratorsConfig(arguments=None): action='store_true', help="compare the results of the event generation process by process and produce summary output in outputDir" ) - outputDirDefault = "Run-Cards" + outputDirDefault = "work" parser.add_argument( "--outputDir", type=str, From 1995cac5d4177a546d8c35c5a94ce8bb8e64e133 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 13:44:29 +0200 Subject: [PATCH 60/88] add sanity check --- python/k4GeneratorsConfig.py | 251 ++++++++++++++++++----------------- 1 file changed, 131 insertions(+), 120 deletions(-) diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index c5e5b248..7e91007a 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -11,128 +11,139 @@ from Production import generate from Production import summary -def k4GeneratorsConfig(arguments=None): - parser = argparse.ArgumentParser(prog="k4GeneratorsConfig") - parser.add_argument( - "--make", - action='store_true', - help="make the generator datacards from the yaml files" - ) - parser.add_argument( - "--yaml", - nargs="*", - type=str, - default=[os.path.dirname(os.path.realpath(__file__))+'/../examples'], - help="yamlFiles and director(y/ies) with yaml files (default: k4GeneratorsConfig/examples)" - ) - parser.add_argument( - "--sqrts", - nargs="*", - type=str, - default=[], - help="either a space separated list of center of mass energies OR file(s) and director(y/ies) with sqrts lists in yaml format (name : sqrtsPROCESS.dat containing eg, sqrts:[91.,240.]), sqrts.yaml as single argument will be applied to all processes", - ) - parser.add_argument( - "--seed", - nargs=1, - type=int, - default=4711, - help="initial random number seed, increment for each file", - ) - parser.add_argument( - "--nevts", - type=int, - default=-1, - help="Number of events to be generated", - ) - parser.add_argument( - "--parameterTag", - type=str, - default="latest", - help="parameter tag in Parameters.yaml (default: latest)", - ) - parser.add_argument( - "--parameterTagFile", - type=str, - default=os.path.dirname(os.path.realpath(__file__))+'/ParameterSets.yaml', - help="name of file containing the parameter sets of the requested parameterTag, default: ParameterSets.yaml in directory: k4GeneratorsConfig/python", - ) - parser.add_argument( - "--key4hepUseNightlies", - action='store_true', - help="configures the key4hepscripts to use nightlies instead of releases", - ) - parser.add_argument( - "--key4hepVersion", - default=None, - help="force the use of the version : YYYY-MM-DD (default: latest)", - ) - parser.add_argument( - "--check", - action='store_true', - help="check the generator datacards with respect to the reference" - ) - parser.add_argument( - "--refDir", - type=str, - default=os.path.dirname(os.path.realpath(__file__))+'/../test/ref-results', - help="path to the reference files (default: k4GeneratorsConfig/test/ref-results)" - ) - parser.add_argument( - "--generator", - type=str, - default="All", - help="generator to be run (default: All processed)" - ) - parser.add_argument( - "--generate", - action='store_true', - help="run the event generation" - ) - parser.add_argument( - "--summary", - action='store_true', - help="compare the results of the event generation process by process and produce summary output in outputDir" - ) - outputDirDefault = "work" - parser.add_argument( - "--outputDir", - type=str, - default=argparse.SUPPRESS, - help=f"path to output directory (default: ./{outputDirDefault}, if specified, overrides the outdir key in yaml)" - ) - parser.add_argument( - "--all", - action='store_true', - help="activates --make --generate --summary" - ) +class k4GeneratorsConfig(): + def __init__(self,arguments=None): + parser = argparse.ArgumentParser(prog="k4GeneratorsConfig") + parser.add_argument( + "--make", + action='store_true', + help="make the generator datacards from the yaml files" + ) + parser.add_argument( + "--yaml", + nargs="*", + type=str, + default=[os.path.dirname(os.path.realpath(__file__))+'/../examples'], + help="yamlFiles and director(y/ies) with yaml files (default: k4GeneratorsConfig/examples)" + ) + parser.add_argument( + "--sqrts", + nargs="*", + type=str, + default=[], + help="either a space separated list of center of mass energies OR file(s) and director(y/ies) with sqrts lists in yaml format (name : sqrtsPROCESS.dat containing eg, sqrts:[91.,240.]), sqrts.yaml as single argument will be applied to all processes", + ) + parser.add_argument( + "--seed", + nargs=1, + type=int, + default=4711, + help="initial random number seed, increment for each file", + ) + parser.add_argument( + "--nevts", + type=int, + default=-1, + help="Number of events to be generated", + ) + parser.add_argument( + "--parameterTag", + type=str, + default="latest", + help="parameter tag in Parameters.yaml (default: latest)", + ) + parser.add_argument( + "--parameterTagFile", + type=str, + default=os.path.dirname(os.path.realpath(__file__))+'/ParameterSets.yaml', + help="name of file containing the parameter sets of the requested parameterTag, default: ParameterSets.yaml in directory: k4GeneratorsConfig/python", + ) + parser.add_argument( + "--key4hepUseNightlies", + action='store_true', + help="configures the key4hepscripts to use nightlies instead of releases", + ) + parser.add_argument( + "--key4hepVersion", + default=None, + help="force the use of the version : YYYY-MM-DD (default: latest)", + ) + parser.add_argument( + "--check", + action='store_true', + help="check the generator datacards with respect to the reference" + ) + parser.add_argument( + "--refDir", + type=str, + default=os.path.dirname(os.path.realpath(__file__))+'/../test/ref-results', + help="path to the reference files (default: k4GeneratorsConfig/test/ref-results)" + ) + parser.add_argument( + "--generator", + type=str, + default="All", + help="generator to be run (default: All processed)" + ) + parser.add_argument( + "--generate", + action='store_true', + help="run the event generation" + ) + parser.add_argument( + "--summary", + action='store_true', + help="compare the results of the event generation process by process and produce summary output in outputDir" + ) + self.outputDirDefault = "work" + parser.add_argument( + "--outputDir", + type=str, + default=argparse.SUPPRESS, + help=f"path to output directory (default: ./{self.outputDirDefault}, if specified, overrides the outdir key in yaml)" + ) + parser.add_argument( + "--all", + action='store_true', + help="activates --make --generate --summary" + ) - args = parser.parse_args(arguments) - try: - check = args.outputDir - args.outputDirOverride = True - except AttributeError: - # argument was not given, set the default and make it known: - args.outputDir = outputDirDefault - args.outputDirOverride = False - # --all overrides --make --generate --summary - if args.all: - args.make = True - args.generate = True - args.summary = True - print("k4GeneratorsConfig will make generator datacards, generate events, make a summary") + # decode the arguments + args = parser.parse_args(arguments) + # check the arguments + self.checkArguments(args) + # make the GeneratorDatacards + if args.make: + makeGeneratorDatacards(args) + # compare to the reference + if args.check: + checkGeneratorDatacards(args) + # run the event generation + if args.generate: + generate(args) + # produce the summary + if args.summary: + summary(args) - if args.make: - makeGeneratorDatacards(args) - - if args.check: - checkGeneratorDatacards(args) - - if args.generate: - generate(args) - - if args.summary: - summary(args) + def checkArguments(self, args): + # differentiate between option being given or not + try: + check = args.outputDir + args.outputDirOverride = True + except AttributeError: + # argument was not given, set the default and make it known: + args.outputDir = self.outputDirDefault + args.outputDirOverride = False + # --all overrides --make --generate --summary + if args.all: + args.make = True + args.generate = True + args.summary = True + print("k4GeneratorsConfig will make generator datacards, generate events, make a summary") + # outputDir should never be cwd + if ( os.path.abspath(args.outputDir) == os.path.abspath(os.getcwd()) ): + message = f"k4GeneratorsConfig::ERROR --outputDir {args.outputDir} not allowed \nPlease specify a directory other than the working directory" + sys.exit(message) if __name__ == "__main__": k4GeneratorsConfig() From bc8a66843ca2ccc7fb8ae89085dacdce03468953 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 15:03:47 +0200 Subject: [PATCH 61/88] add differ diff and make file creation safer --- python/Production.py | 40 ++++++++++++++++++++++-------------- python/Yaml2Datacard.py | 14 ++++++++++--- python/k4GeneratorsConfig.py | 2 ++ 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/python/Production.py b/python/Production.py index af2ef218..5bb6f838 100644 --- a/python/Production.py +++ b/python/Production.py @@ -5,6 +5,7 @@ import shutil from pathlib import Path import filecmp +import difflib from Yaml2Datacard import Yaml2Datacard @@ -37,12 +38,16 @@ def getFileNames(self, directory): def makeDirectory(self, dirname, overwrite=True): # Overwrite directory if it exists - if not os.path.exists(dirname): - os.makedirs(dirname) - else: - if overwrite: - shutil.rmtree(dirname) + try: + if not os.path.exists(dirname): os.makedirs(dirname) + else: + if overwrite: + shutil.rmtree(dirname) + os.makedirs(dirname) + except PermissionError: + message = f"k4GeneratorsConfig::ERROR:\n{dirname} cannot be created (full path: {os.path.abspath(dirname)})" + sys.exit(message) def process(self, generators): # where do we start from @@ -190,7 +195,7 @@ def __init__(self, args): def execute(self, generator, process): genProc = f"{generator}/{process}" newDir = f"{self._outputDir}/{genProc}" - refDir = f"{self._outputDir}/{genProc}" + refDir = f"{self._referenceDir}/{genProc}" fileNames = self.getFileNames(newDir) success = True @@ -200,26 +205,31 @@ def execute(self, generator, process): success = False for name in fileNames: - newFile = f"{newDir}/{name}" + filenameGenerated = f"{newDir}/{name}" # new file must exist - if not os.path.isfile(newFile): - print(f"File {newFile} not found") + if not os.path.isfile(filenameGenerated): + print(f"File {filenameGenerated} not found") success = False continue - refFile = f"{refDir}/{name}" + filenameRef = f"{refDir}/{name}" # reference file must exist - if not os.path.isfile(refFile): - print(f"File {refFile} not found") + if not os.path.isfile(filenameRef): + print(f"File {filenameRef} not found") continue success = False # both files exist, so we can compare message = f"Generator {generator} Process {process} File {name}" - if filecmp.cmp(refFile,newFile, shallow=False): - print(f"{message} identical") + if filecmp.cmp(filenameRef, filenameGenerated, shallow=False): + print(f"OK: {message}") else: - print(f"{message} differ") + print(f"NOTOK: {message}") + # if that's the case make a detailed comparison + fileRef = open(filenameGenerated, "r").readlines() + fileGenerated = open(filenameRef, "r").readlines() + print("".join(line for line in difflib.Differ().compare(fileRef,fileGenerated) + if line[0] == "-" or line[0] == "+")) success = False return success diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 1ee05399..bdef327f 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -112,8 +112,12 @@ def processOutputDir(self, args): # all the preparatory work has been done in args.outputDir # create the new directory if it does not exist if not args.outputDirOverride: - if not os.path.exists(self.outputDir): - os.makedirs(self.outputDir) + try: + if not os.path.exists(self.outputDir): + os.makedirs(self.outputDir) + except PermissionError: + message = f"Yaml2Datacard::processOutputDir::ERROR:\n{self.outputDir} cannot be created (full path: {os.path.abspath(self.outputDir)})" + sys.exit(message) os.chdir(self.outputDir) def makeDirectories4GeneratorsProcess(self, generators, procname): @@ -121,7 +125,11 @@ def makeDirectories4GeneratorsProcess(self, generators, procname): for generator in generators: process_directory = os.path.join(generator, procname) if not os.path.exists(process_directory): - os.makedirs(process_directory) + try: + os.makedirs(process_directory) + except PermissionError: + message = f"Yaml2Datacard::makeDirectories4GeneratorsProcess::ERROR:\n{process_directory} cannot be created (full path: {os.path.abspath(process_directory)})" + sys.exit(message) if __name__ == "__main__": Yaml2Datacard() diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index 7e91007a..d4ebd5d1 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -126,6 +126,8 @@ def __init__(self,arguments=None): summary(args) def checkArguments(self, args): + + # OPTION: outputDir # differentiate between option being given or not try: check = args.outputDir From 61371a5fa1da87d010de331f6d0c639053fb3769 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 15:06:22 +0200 Subject: [PATCH 62/88] add consistency --- test/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dafcb7fe..aa5ddca9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,7 +15,7 @@ foreach(generator ${generatorList}) endforeach() # add the first test: setting up the yaml files -add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --make ) +add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --make --workDir Run-Cards ) set_tests_properties("createGeneratorDatacards" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -25,7 +25,7 @@ set_tests_properties("createGeneratorDatacards" # add the second test: comparing the yaml files to the reference files function(check_generator_datacards generator) - add_test(NAME checkGeneratorDatacards_${generator} COMMAND python3 ${EXECUTABLE} --check --generator ${generator} ) + add_test(NAME checkGeneratorDatacards_${generator} COMMAND python3 ${EXECUTABLE} --check --generator ${generator} --workDir Run-Cards ) set_tests_properties("checkGeneratorDatacards_${generator}" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -42,7 +42,7 @@ endforeach() # third test running the generators (define the function first then call it function(add_generator_run name generator) - add_test(NAME ${name}_${generator} COMMAND python3 ${EXECUTABLE} --generate --generator ${generator} ) + add_test(NAME ${name}_${generator} COMMAND python3 ${EXECUTABLE} --generate --generator ${generator} --workDir Run-Cards ) set_tests_properties(${name}_${generator} PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -57,7 +57,7 @@ foreach(generator ${generatorList}) endforeach() # third test (after all others have been run: gather the xsections -add_test(NAME xsectionRuns COMMAND python3 ${EXECUTABLE} --summary ) +add_test(NAME xsectionRuns COMMAND python3 ${EXECUTABLE} --summary --workDir Run-Cards ) set_tests_properties("xsectionRuns" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} From 9f93c6562393c491af9c31788f5d76ae6ac74f81 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 15:19:56 +0200 Subject: [PATCH 63/88] further protection against unsupported combinations --- python/k4GeneratorsConfig.py | 12 ++++++++++++ test/CMakeLists.txt | 2 -- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index d4ebd5d1..b6632258 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -147,6 +147,18 @@ def checkArguments(self, args): message = f"k4GeneratorsConfig::ERROR --outputDir {args.outputDir} not allowed \nPlease specify a directory other than the working directory" sys.exit(message) + # OPTION all or make&&generate&&summary + if ( args.all or + (args.make and args.check) or + (args.make and args.generate) or + (args.make and args.summary) ): + if not args.outputDirOverride: + message = f"k4GeneratorsConfig::ERROR\n" + message += f"--make and (--check and/or --generate and/or --summary) requested\n" + message += f"yamlFiles may define multiple outputDirectory, functionality not foreseen\n" + message += f"BUT: --outputDir {args.outputDir} not defined \nPlease define a common output directory" + sys.exit(message) + if __name__ == "__main__": k4GeneratorsConfig() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index aa5ddca9..cc8aaa1d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -64,5 +64,3 @@ set_tests_properties("xsectionRuns" ENVIRONMENT ${TEST_ENVIRONMENT} DEPENDS "${RunList}" ) - -#add_test(NAME Finalize COMMAND ${CMAKE_BINARY_DIR}/bin/eventGenerationSummary -f ${CMAKE_SOURCE_DIR}/test/xsectionSummary.dat) From 994bcd7cf62cc96f292bbe0b122b1cf86aa1aeb1 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 15:26:36 +0200 Subject: [PATCH 64/88] clang --- python/k4GeneratorsConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index b6632258..c00f78fa 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -127,7 +127,7 @@ def __init__(self,arguments=None): def checkArguments(self, args): - # OPTION: outputDir + # OPTION: outputDir # differentiate between option being given or not try: check = args.outputDir From 7427b92bd49a593ba0b55f429575cb9ba645354a Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 15:36:52 +0200 Subject: [PATCH 65/88] correct option --- test/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cc8aaa1d..fc69d5cf 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,7 +15,7 @@ foreach(generator ${generatorList}) endforeach() # add the first test: setting up the yaml files -add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --make --workDir Run-Cards ) +add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --make --outputDir Run-Cards ) set_tests_properties("createGeneratorDatacards" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -25,7 +25,7 @@ set_tests_properties("createGeneratorDatacards" # add the second test: comparing the yaml files to the reference files function(check_generator_datacards generator) - add_test(NAME checkGeneratorDatacards_${generator} COMMAND python3 ${EXECUTABLE} --check --generator ${generator} --workDir Run-Cards ) + add_test(NAME checkGeneratorDatacards_${generator} COMMAND python3 ${EXECUTABLE} --check --generator ${generator} --outputDir Run-Cards ) set_tests_properties("checkGeneratorDatacards_${generator}" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -42,7 +42,7 @@ endforeach() # third test running the generators (define the function first then call it function(add_generator_run name generator) - add_test(NAME ${name}_${generator} COMMAND python3 ${EXECUTABLE} --generate --generator ${generator} --workDir Run-Cards ) + add_test(NAME ${name}_${generator} COMMAND python3 ${EXECUTABLE} --generate --generator ${generator} --outputDir Run-Cards ) set_tests_properties(${name}_${generator} PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -57,7 +57,7 @@ foreach(generator ${generatorList}) endforeach() # third test (after all others have been run: gather the xsections -add_test(NAME xsectionRuns COMMAND python3 ${EXECUTABLE} --summary --workDir Run-Cards ) +add_test(NAME xsectionRuns COMMAND python3 ${EXECUTABLE} --summary --outputDir Run-Cards ) set_tests_properties("xsectionRuns" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} From 7b0c750829e516bf862da3fa2a923d3db852956c Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 16:33:26 +0200 Subject: [PATCH 66/88] reduce filesize of artifact --- .github/workflows/key4hep_build.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/key4hep_build.yml b/.github/workflows/key4hep_build.yml index 00663756..a2e9ee6c 100644 --- a/.github/workflows/key4hep_build.yml +++ b/.github/workflows/key4hep_build.yml @@ -33,7 +33,14 @@ jobs: with: name: k4GeneratorsConfig_report_${{ matrix.build_type }}_${{ matrix.image }} path: | - test/Run-Cards/* + test/Run-Cards/*.dat + test/Run-Cards/*.root + test/Run-Cards/*/*/*.dat + test/Run-Cards/*/*/*.sin + test/Run-Cards/*/*/*.cmnd + test/Run-Cards/*/*/*.selectors + test/Run-Cards/*/*/Run*.sh + test/Run-Cards/*/*/*.edm4hep - name: Show full CTest log on failure if: failure() run: | From 6028a09e448401efbcf8e3abdd18c8462a9061e8 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 1 Apr 2026 17:00:50 +0200 Subject: [PATCH 67/88] reduce filesize of artifact --- .github/workflows/key4hep_build.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/key4hep_build.yml b/.github/workflows/key4hep_build.yml index a2e9ee6c..1cc492a4 100644 --- a/.github/workflows/key4hep_build.yml +++ b/.github/workflows/key4hep_build.yml @@ -35,12 +35,6 @@ jobs: path: | test/Run-Cards/*.dat test/Run-Cards/*.root - test/Run-Cards/*/*/*.dat - test/Run-Cards/*/*/*.sin - test/Run-Cards/*/*/*.cmnd - test/Run-Cards/*/*/*.selectors - test/Run-Cards/*/*/Run*.sh - test/Run-Cards/*/*/*.edm4hep - name: Show full CTest log on failure if: failure() run: | From 7ef99806f1d759c14df9bd48ecf16d9c6640b86f Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 10:55:31 +0200 Subject: [PATCH 68/88] bug corr indentation --- python/k4GeneratorsConfig.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index c00f78fa..1ff2a2ca 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -111,7 +111,7 @@ def __init__(self,arguments=None): # decode the arguments args = parser.parse_args(arguments) # check the arguments - self.checkArguments(args) + self.processArguments(args) # make the GeneratorDatacards if args.make: makeGeneratorDatacards(args) @@ -125,8 +125,7 @@ def __init__(self,arguments=None): if args.summary: summary(args) - def checkArguments(self, args): - + def processArguments(self, args): # OPTION: outputDir # differentiate between option being given or not try: @@ -136,12 +135,14 @@ def checkArguments(self, args): # argument was not given, set the default and make it known: args.outputDir = self.outputDirDefault args.outputDirOverride = False - # --all overrides --make --generate --summary - if args.all: - args.make = True - args.generate = True - args.summary = True - print("k4GeneratorsConfig will make generator datacards, generate events, make a summary") + + # --all overrides --make --generate --summary + if args.all: + args.make = True + args.generate = True + args.summary = True + print("k4GeneratorsConfig will make generator datacards, generate events, make a summary") + # outputDir should never be cwd if ( os.path.abspath(args.outputDir) == os.path.abspath(os.getcwd()) ): message = f"k4GeneratorsConfig::ERROR --outputDir {args.outputDir} not allowed \nPlease specify a directory other than the working directory" From d1b915b13c423cc05af74ef07502ba69b6c03982 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 11:50:02 +0200 Subject: [PATCH 69/88] details for success and failures --- .../src/eventGenerationCollections.cxx | 63 ++++++++++++++++--- .../src/eventGenerationCollections.h | 12 ++-- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.cxx b/k4GeneratorsConfig/src/eventGenerationCollections.cxx index 289f1174..a043133b 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.cxx +++ b/k4GeneratorsConfig/src/eventGenerationCollections.cxx @@ -5,7 +5,7 @@ #include #include -k4GeneratorsConfig::eventGenerationCollections::eventGenerationCollections() : m_validCounter(0), m_invalidCounter(0) {} +k4GeneratorsConfig::eventGenerationCollections::eventGenerationCollections() {} k4GeneratorsConfig::eventGenerationCollections::eventGenerationCollections( const eventGenerationCollections& theOriginal) { if (this != &theOriginal) { @@ -65,9 +65,9 @@ void k4GeneratorsConfig::eventGenerationCollections::makeCollections(std::string std::cout << "Generator " << xsec->Generator() << " has been processed" << std::endl; m_xsectionCollection.push_back(*xsec); if (xsec->isValid()) - m_validCounter++; + addSuccess(xsec->Generator()); if (!xsec->isValid()) - m_invalidCounter++; + addFailure(xsec->Generator()); } } // we need to keep xsec alive for the analysisHistos distributions @@ -147,8 +147,38 @@ bool k4GeneratorsConfig::eventGenerationCollections::compareLexical(analysisHist return false; } -unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfSuccesses() { return m_validCounter; } -unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfFailures() { return m_invalidCounter; } +void k4GeneratorsConfig::eventGenerationCollections::addSuccess(std::string generator) { + if ( m_validCounter.find(generator) != m_validCounter.end() ){ + m_validCounter[generator]++; + } + else { + m_validCounter[generator] = 1; + } +} +void k4GeneratorsConfig::eventGenerationCollections::addFailure(std::string generator) { + if ( m_invalidCounter.find(generator) != m_invalidCounter.end() ){ + m_invalidCounter[generator]++; + } + else { + m_invalidCounter[generator] = 1; + } +} +unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfSuccesses() const { + unsigned int validTotal = 0; + std::map::const_iterator imap; + for (imap = m_validCounter.begin(); imap != m_validCounter.end(); imap++){ + validTotal += imap->second; + } + return validTotal; +} +unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfFailures() const { + unsigned int invalidTotal = 0; + std::map::const_iterator imap; + for (imap = m_invalidCounter.begin(); imap != m_invalidCounter.end(); imap++){ + invalidTotal += imap->second; + } + return invalidTotal; +} void k4GeneratorsConfig::eventGenerationCollections::Write2Root(std::string dirname, std::string filename) { eventGenerationCollections2Root out(dirname, filename); @@ -225,7 +255,24 @@ void k4GeneratorsConfig::eventGenerationCollections::PrintSummary(std::ostream& } output << std::endl; // last thing the invalids - output << "Number of runs : " << m_invalidCounter + m_validCounter << std::endl; - output << "Number of failed runs : " << m_invalidCounter << std::endl; - output << "Number of successful runs: " << m_validCounter << std::endl; + output << "Number of runs : " << NbOfFailures() + NbOfSuccesses() << std::endl; + output << "Number of failed runs : " << NbOfFailures() << std::endl; + output << "Number of successful runs: " << NbOfSuccesses() << std::endl; + // details only for failures: + if ( NbOfFailures() > 0 ) { + output << std::endl + << "Details in Failures:" + << std::endl; + std::map::const_iterator failure, success; + for (failure = m_invalidCounter.begin(); failure != m_invalidCounter.end(); failure++){ + output << failure->first << " : " << failure->second << " Failures "; + if ( (success = m_validCounter.find(failure->first)) != m_validCounter.end() ){ + output << success->second; + } + else { + output << " 0 "; + } + output << "Successes" << std::endl; + } + } } diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.h b/k4GeneratorsConfig/src/eventGenerationCollections.h index 7349861f..f7ce9ccc 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.h +++ b/k4GeneratorsConfig/src/eventGenerationCollections.h @@ -2,6 +2,7 @@ #define K4GENERATORSCONFIG_EVENTGENERATIONCOLLECTIONS_H #include +#include #include "analysisHistos.h" #include "xsection.h" @@ -21,8 +22,11 @@ class eventGenerationCollections { bool compareLexical(xsection, xsection); bool compareLexical(analysisHistos, analysisHistos); - unsigned int NbOfSuccesses(); - unsigned int NbOfFailures(); + void addSuccess(std::string); + void addFailure(std::string); + + unsigned int NbOfSuccesses() const; + unsigned int NbOfFailures() const; void Write2Root(std::string, std::string); @@ -35,8 +39,8 @@ class eventGenerationCollections { private: std::vector m_xsectionCollection; std::vector m_analysisHistosCollection; - unsigned int m_validCounter; - unsigned int m_invalidCounter; + std::map m_validCounter; + std::map m_invalidCounter; }; } // namespace k4GeneratorsConfig From 7e459cbda3275a5430a5b48ee7961673346cacab Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 11:55:08 +0200 Subject: [PATCH 70/88] clang --- k4GeneratorsConfig/src/eventGenerationCollections.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.h b/k4GeneratorsConfig/src/eventGenerationCollections.h index f7ce9ccc..4b448242 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.h +++ b/k4GeneratorsConfig/src/eventGenerationCollections.h @@ -1,8 +1,8 @@ #ifndef K4GENERATORSCONFIG_EVENTGENERATIONCOLLECTIONS_H #define K4GENERATORSCONFIG_EVENTGENERATIONCOLLECTIONS_H -#include #include +#include #include "analysisHistos.h" #include "xsection.h" @@ -39,8 +39,8 @@ class eventGenerationCollections { private: std::vector m_xsectionCollection; std::vector m_analysisHistosCollection; - std::map m_validCounter; - std::map m_invalidCounter; + std::map m_validCounter; + std::map m_invalidCounter; }; } // namespace k4GeneratorsConfig From e2a3c91d921ff70e1602717de0507e04f6530d3d Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 12:01:21 +0200 Subject: [PATCH 71/88] clang --- .../src/eventGenerationCollections.cxx | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.cxx b/k4GeneratorsConfig/src/eventGenerationCollections.cxx index a043133b..af69ea2b 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.cxx +++ b/k4GeneratorsConfig/src/eventGenerationCollections.cxx @@ -148,33 +148,31 @@ bool k4GeneratorsConfig::eventGenerationCollections::compareLexical(analysisHist return false; } void k4GeneratorsConfig::eventGenerationCollections::addSuccess(std::string generator) { - if ( m_validCounter.find(generator) != m_validCounter.end() ){ + if (m_validCounter.find(generator) != m_validCounter.end()) { m_validCounter[generator]++; - } - else { + } else { m_validCounter[generator] = 1; } } void k4GeneratorsConfig::eventGenerationCollections::addFailure(std::string generator) { - if ( m_invalidCounter.find(generator) != m_invalidCounter.end() ){ + if (m_invalidCounter.find(generator) != m_invalidCounter.end()) { m_invalidCounter[generator]++; - } - else { + } else { m_invalidCounter[generator] = 1; } } unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfSuccesses() const { unsigned int validTotal = 0; - std::map::const_iterator imap; - for (imap = m_validCounter.begin(); imap != m_validCounter.end(); imap++){ + std::map::const_iterator imap; + for (imap = m_validCounter.begin(); imap != m_validCounter.end(); imap++) { validTotal += imap->second; } return validTotal; } unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfFailures() const { unsigned int invalidTotal = 0; - std::map::const_iterator imap; - for (imap = m_invalidCounter.begin(); imap != m_invalidCounter.end(); imap++){ + std::map::const_iterator imap; + for (imap = m_invalidCounter.begin(); imap != m_invalidCounter.end(); imap++) { invalidTotal += imap->second; } return invalidTotal; @@ -256,21 +254,18 @@ void k4GeneratorsConfig::eventGenerationCollections::PrintSummary(std::ostream& output << std::endl; // last thing the invalids output << "Number of runs : " << NbOfFailures() + NbOfSuccesses() << std::endl; - output << "Number of failed runs : " << NbOfFailures() << std::endl; + output << "Number of failed runs : " << NbOfFailures() << std::endl; output << "Number of successful runs: " << NbOfSuccesses() << std::endl; // details only for failures: - if ( NbOfFailures() > 0 ) { - output << std::endl - << "Details in Failures:" - << std::endl; - std::map::const_iterator failure, success; - for (failure = m_invalidCounter.begin(); failure != m_invalidCounter.end(); failure++){ + if (NbOfFailures() > 0) { + output << std::endl << "Details in Failures:" << std::endl; + std::map::const_iterator failure, success; + for (failure = m_invalidCounter.begin(); failure != m_invalidCounter.end(); failure++) { output << failure->first << " : " << failure->second << " Failures "; - if ( (success = m_validCounter.find(failure->first)) != m_validCounter.end() ){ - output << success->second; - } - else { - output << " 0 "; + if ((success = m_validCounter.find(failure->first)) != m_validCounter.end()) { + output << success->second; + } else { + output << " 0 "; } output << "Successes" << std::endl; } From f08649b7c4a29f79e3088cef97011ed3b80c382a Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 13:12:21 +0200 Subject: [PATCH 72/88] improve output --- k4GeneratorsConfig/src/eventGenerationCollections.cxx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.cxx b/k4GeneratorsConfig/src/eventGenerationCollections.cxx index af69ea2b..385c7e95 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.cxx +++ b/k4GeneratorsConfig/src/eventGenerationCollections.cxx @@ -258,16 +258,15 @@ void k4GeneratorsConfig::eventGenerationCollections::PrintSummary(std::ostream& output << "Number of successful runs: " << NbOfSuccesses() << std::endl; // details only for failures: if (NbOfFailures() > 0) { - output << std::endl << "Details in Failures:" << std::endl; + output << std::endl << "Detail of Failures:" << std::endl; std::map::const_iterator failure, success; for (failure = m_invalidCounter.begin(); failure != m_invalidCounter.end(); failure++) { output << failure->first << " : " << failure->second << " Failures "; + unsigned int successCount = 0; if ((success = m_validCounter.find(failure->first)) != m_validCounter.end()) { - output << success->second; - } else { - output << " 0 "; + successCount = success->second; } - output << "Successes" << std::endl; + output << " / " << successCount + failure->second << " Runs" << std::endl; } } } From 2560286de81318fd7bf1efa164c03c80e4b2e049 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 13:17:48 +0200 Subject: [PATCH 73/88] clang --- k4GeneratorsConfig/src/eventGenerationCollections.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.cxx b/k4GeneratorsConfig/src/eventGenerationCollections.cxx index 385c7e95..7bb07880 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.cxx +++ b/k4GeneratorsConfig/src/eventGenerationCollections.cxx @@ -264,7 +264,7 @@ void k4GeneratorsConfig::eventGenerationCollections::PrintSummary(std::ostream& output << failure->first << " : " << failure->second << " Failures "; unsigned int successCount = 0; if ((success = m_validCounter.find(failure->first)) != m_validCounter.end()) { - successCount = success->second; + successCount = success->second; } output << " / " << successCount + failure->second << " Runs" << std::endl; } From 6d9cacee9c04a8a51b4932b7199408f3c6bb6a5d Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 13:22:23 +0200 Subject: [PATCH 74/88] clang --- k4GeneratorsConfig/src/eventGenerationCollections.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.cxx b/k4GeneratorsConfig/src/eventGenerationCollections.cxx index 7bb07880..a39dceb5 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.cxx +++ b/k4GeneratorsConfig/src/eventGenerationCollections.cxx @@ -264,7 +264,7 @@ void k4GeneratorsConfig::eventGenerationCollections::PrintSummary(std::ostream& output << failure->first << " : " << failure->second << " Failures "; unsigned int successCount = 0; if ((success = m_validCounter.find(failure->first)) != m_validCounter.end()) { - successCount = success->second; + successCount = success->second; } output << " / " << successCount + failure->second << " Runs" << std::endl; } From 7a372c84311a58a2c03f60a4dc82939d0e6e2bae Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 14:59:02 +0200 Subject: [PATCH 75/88] protect against accidental overwrites --- python/Production.py | 5 ++-- python/k4GeneratorsConfig.py | 51 +++++++++++++++++++----------------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/python/Production.py b/python/Production.py index 5bb6f838..75d259ef 100644 --- a/python/Production.py +++ b/python/Production.py @@ -3,6 +3,7 @@ import sys import subprocess import shutil +import copy from pathlib import Path import filecmp import difflib @@ -112,8 +113,8 @@ def __init__(self, args): if not sqrtsGlobalFileName: self.prepareSQRTS(args.sqrts); - # args to transfer: - self.Yaml2DatacardArgs = args + # args to transfer: avoid overwrite + self.Yaml2DatacardArgs = copy.deepcopy(args) # run all yamls: self.run(sqrtsGlobalFileName); diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index 1ff2a2ca..c1f8dcf1 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -5,6 +5,7 @@ import sys import argparse import textwrap +import copy from Production import makeGeneratorDatacards from Production import checkGeneratorDatacards @@ -113,51 +114,53 @@ def __init__(self,arguments=None): # check the arguments self.processArguments(args) # make the GeneratorDatacards - if args.make: - makeGeneratorDatacards(args) + if self.args.make: + makeGeneratorDatacards(self.args) # compare to the reference - if args.check: - checkGeneratorDatacards(args) + if self.args.check: + checkGeneratorDatacards(self.args) # run the event generation - if args.generate: - generate(args) + if self.args.generate: + generate(self.args) # produce the summary - if args.summary: - summary(args) + if self.args.summary: + summary(self.args) def processArguments(self, args): + # make a deep copy of the arguments: + self.args = copy.deepcopy(args) # OPTION: outputDir # differentiate between option being given or not try: - check = args.outputDir - args.outputDirOverride = True + check = self.args.outputDir + self.args.outputDirOverride = True except AttributeError: # argument was not given, set the default and make it known: - args.outputDir = self.outputDirDefault - args.outputDirOverride = False + self.args.outputDir = self.outputDirDefault + self.args.outputDirOverride = False # --all overrides --make --generate --summary - if args.all: - args.make = True - args.generate = True - args.summary = True + if self.args.all: + self.args.make = True + self.args.generate = True + self.args.summary = True print("k4GeneratorsConfig will make generator datacards, generate events, make a summary") # outputDir should never be cwd - if ( os.path.abspath(args.outputDir) == os.path.abspath(os.getcwd()) ): - message = f"k4GeneratorsConfig::ERROR --outputDir {args.outputDir} not allowed \nPlease specify a directory other than the working directory" + if ( os.path.abspath(self.args.outputDir) == os.path.abspath(os.getcwd()) ): + message = f"k4GeneratorsConfig::ERROR --outputDir {self.args.outputDir} not allowed \nPlease specify a directory other than the working directory" sys.exit(message) # OPTION all or make&&generate&&summary - if ( args.all or - (args.make and args.check) or - (args.make and args.generate) or - (args.make and args.summary) ): - if not args.outputDirOverride: + if ( self.args.all or + (self.args.make and self.args.check) or + (self.args.make and self.args.generate) or + (self.args.make and self.args.summary) ): + if not self.args.outputDirOverride: message = f"k4GeneratorsConfig::ERROR\n" message += f"--make and (--check and/or --generate and/or --summary) requested\n" message += f"yamlFiles may define multiple outputDirectory, functionality not foreseen\n" - message += f"BUT: --outputDir {args.outputDir} not defined \nPlease define a common output directory" + message += f"BUT: --outputDir {self.args.outputDir} not defined \nPlease define a common output directory" sys.exit(message) if __name__ == "__main__": From 521db4e26e08a699e7f7d280b8679be56a4ff712 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 15:30:42 +0200 Subject: [PATCH 76/88] separate initialization and execution --- python/Production.py | 2 +- python/Yaml2Datacard.py | 125 ++++++++++++++++++++-------------------- 2 files changed, 63 insertions(+), 64 deletions(-) diff --git a/python/Production.py b/python/Production.py index 75d259ef..deae6f2e 100644 --- a/python/Production.py +++ b/python/Production.py @@ -177,7 +177,7 @@ def run(self, sqrtsGlobal): else: self.Yaml2DatacardArgs.sqrts = "" print(message) - Yaml2Datacard(self.Yaml2DatacardArgs) + Yaml2Datacard(self.Yaml2DatacardArgs).processFile() # return to the starting point os.chdir(cwd) diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index bdef327f..21f9b62f 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -4,6 +4,7 @@ import sys import argparse import textwrap +import copy from datetime import datetime import ReleaseSpecs @@ -18,100 +19,98 @@ class Yaml2Datacard: """Convert Input files into generator datacards""" def __init__(self,args): + # copy the arguments + self.args = copy.deepcopy(args) - ReleaseSpec.set_info("key4hepUseNightlies",args.key4hepUseNightlies) + ReleaseSpec.set_info("key4hepUseNightlies",self.args.key4hepUseNightlies) if ReleaseSpecs.key4hepUseNightlies.value: print(f"key4HEP configuration: NIGHTLIES") else: print(f"key4HEP configuration: RELEASE") # make sure it's a valid date - if args.key4hepVersion is not None: + if self.args.key4hepVersion is not None: try: - relDate = datetime.strptime(args.key4hepVersion,'%Y-%m-%d') + relDate = datetime.strptime(self.args.key4hepVersion,'%Y-%m-%d') if (datetime.today() - relDate).days < 0: raise ValueError() - print(f"key4HEP configuration date: {args.key4hepVersion}") + print(f"key4HEP configuration date: {self.args.key4hepVersion}") except ValueError: print(f"Invalid KEY4HEP release argument, YYYY-MM-DD expected, latest possible date {datetime.today().strftime('%Y-%m-%d')}") - print(f"Requested: {args.key4hepVersion}") + print(f"Requested: {self.args.key4hepVersion}") print("Cannot configure scripts correctly, exiting") exit() else: print(f"key4HEP configuration date: latest") # store for future use: - ReleaseSpec.set_info("key4hepReleaseDate",args.key4hepVersion) + ReleaseSpec.set_info("key4hepReleaseDate",self.args.key4hepVersion) # sqrts choices from file - energies = [0.] - if args.sqrts: - sqrtsReader = Reader.SQRTSReader(args.sqrts) - energies = sqrtsReader.energies() + self.energies = [0.] + if self.args.sqrts: + sqrtsReader = Reader.SQRTSReader(self.args.sqrts) + self.energies = sqrtsReader.energies() # now we read the global settings try: # make sure that we follow a symlink to the real location of the parametersets should replace that by share? - parameterSet = Reader.ParameterSetReader(args.parameterTagFile, args.parameterTag) + parameterSet = Reader.ParameterSetReader(self.args.parameterTagFile, self.args.parameterTag) except FileNotFoundError as e: - print(f"ERROR: File {e} with parameters for tag {args.parameterTag} not found") - exit() + exit(f"ERROR: File {e} with parameters for tag {self.args.parameterTag} not found"f"ERROR: File {e} with parameters for tag {self.args.parameterTag} not found") + def processFile(self): # execute file processes - rndmSeed = args.seed - for sqrts in energies: - rndmIncrement = self.executeFiles(args, sqrts, rndmSeed) - # offset for next round by number of yaml files - rndmSeed = rndmSeed + rndmIncrement - - def executeFiles(self, args, sqrts, rndmSeedFallback): - # remember where we started from: - cwd = os.getcwd() - # first step reset all particles: - ParticleCollection() - # read the input file - self.processReader = Reader.ProcessReader(args.yaml, sqrts) - # set the number of events if present - if args.nevts != -1: - self.processReader.set("events", events) - # the datacard outputDir may differ - self.processOutputDir(args) - # now extract information - processes = self.processReader.get_processes(sqrts) - yamlParticleData = self.processReader.get_particle_data() - generators = Generators(self.processReader) - # - rndmIncrement = 0 - for key, value in processes.items(): - self.makeDirectories4GeneratorsProcess(self.processReader.get_generators(), key) - try: - randomseed = value["randomseed"] - except: - randomseed = rndmSeedFallback + rndmIncrement - value["randomseed"] = randomseed - rndmIncrement += 1 - param = ProcessParameters(self.processReader) - # instantiate the class for each process - process = Process( - value, key, param, yamlParticleData, OutDir=self.outputDir - ) - process.prepareProcess() - generators.runGeneratorConfiguration(process) - # at the end back to the starting point dir for the next file - if (cwd != os.getcwd()): - os.chdir(cwd) + rndmSeed = self.args.seed + for sqrts in self.energies: + # remember where we started from: + cwd = os.getcwd() + # first step reset all particles: + ParticleCollection() + # read the input file + self.processReader = Reader.ProcessReader(self.args.yaml, sqrts) + # set the number of events if present + if self.args.nevts != -1: + self.processReader.set("events", events) + # the datacard outputDir may differ + self.processOutputDir() + # now extract information + processes = self.processReader.get_processes(sqrts) + yamlParticleData = self.processReader.get_particle_data() + generators = Generators(self.processReader) + # + for key, value in processes.items(): + self.makeDirectories4GeneratorsProcess(self.processReader.get_generators(), key) + try: + randomseed = value["randomseed"] + except: + # random seed not present, fall to external setting and increment for next round + value["randomseed"] = rndmseed + rndmseed += 1 + param = ProcessParameters(self.processReader) + # instantiate the class for each process + process = Process( + value, key, param, yamlParticleData, OutDir=self.outputDir + ) + process.prepareProcess() + generators.runGeneratorConfiguration(process) - return rndmIncrement + # increment for next sqrts + rndmseed += 1 + + # at the end back to the starting point dir for the next file + if (cwd != os.getcwd()): + os.chdir(cwd) - def processOutputDir(self, args): + def processOutputDir(self): # the output directory - self.outputDir = getattr(self.processReader, "outdir", args.outputDir) - if args.outputDirOverride: - self.outputDir = args.outputDir + self.outputDir = getattr(self.processReader, "outdir", self.args.outputDir) + if self.args.outputDirOverride: + self.outputDir = self.args.outputDir # the attribute always has to be reset to be sure.... - setattr(self.processReader, "outdir", args.outputDir) - # all the preparatory work has been done in args.outputDir + setattr(self.processReader, "outdir", self.args.outputDir) + # all the preparatory work has been done in self.args.outputDir # create the new directory if it does not exist - if not args.outputDirOverride: + if not self.args.outputDirOverride: try: if not os.path.exists(self.outputDir): os.makedirs(self.outputDir) From 563738b6c7a360f6b495a4d72b96b2d46426d4ba Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 15:35:06 +0200 Subject: [PATCH 77/88] typo fixed --- python/Yaml2Datacard.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 21f9b62f..08c2f8f6 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -84,8 +84,8 @@ def processFile(self): randomseed = value["randomseed"] except: # random seed not present, fall to external setting and increment for next round - value["randomseed"] = rndmseed - rndmseed += 1 + value["randomseed"] = rndmSeed + rndmSeed += 1 param = ProcessParameters(self.processReader) # instantiate the class for each process process = Process( @@ -95,7 +95,7 @@ def processFile(self): generators.runGeneratorConfiguration(process) # increment for next sqrts - rndmseed += 1 + rndmSeed += 1 # at the end back to the starting point dir for the next file if (cwd != os.getcwd()): From edff53bc821629d8500a0e80099ceeba036f85d7 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 15:50:36 +0200 Subject: [PATCH 78/88] clang --- python/Yaml2Datacard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index 08c2f8f6..d5889f84 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -96,7 +96,7 @@ def processFile(self): # increment for next sqrts rndmSeed += 1 - + # at the end back to the starting point dir for the next file if (cwd != os.getcwd()): os.chdir(cwd) From 6243e770ea14c7111170ab29a0bfb6ff07a4e45d Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Tue, 7 Apr 2026 16:49:51 +0200 Subject: [PATCH 79/88] protect against overwriting outputDirectory --- python/Production.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/Production.py b/python/Production.py index deae6f2e..cd9ea520 100644 --- a/python/Production.py +++ b/python/Production.py @@ -1,4 +1,5 @@ from abc import ABC,abstractmethod +from datetime import datetime import os import sys import subprocess @@ -37,15 +38,16 @@ def getFileNames(self, directory): filenames =[name for name in filenames if os.path.isfile(os.path.join(directory,name))] return filenames - def makeDirectory(self, dirname, overwrite=True): + def makeDirectory(self, dirname): # Overwrite directory if it exists try: if not os.path.exists(dirname): os.makedirs(dirname) else: - if overwrite: - shutil.rmtree(dirname) - os.makedirs(dirname) + # copy to a directory with the same name+date added to dir name + shutil.copytree(dirname, dirname+str(datetime.today()).replace(' ','_').replace(':','_').replace('.','_')) + shutil.rmtree(dirname) + os.makedirs(dirname) except PermissionError: message = f"k4GeneratorsConfig::ERROR:\n{dirname} cannot be created (full path: {os.path.abspath(dirname)})" sys.exit(message) @@ -79,7 +81,7 @@ def __init__(self, args): super().__init__(args) # make the directory for the work, protect against "./" - self.makeDirectory(args.outputDir, not (os.path.abspath(args.outputDir) == os.getcwd() )) + self.makeDirectory(args.outputDir) # the sqrts argument can be a list of sqrts or a list of strings sqrtsGlobalFileName = str("") From eef084dbb52b9d56761aa5e8e5fca073bcba1bb0 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 8 Apr 2026 14:02:04 +0200 Subject: [PATCH 80/88] warnings for incompatible options --- python/k4GeneratorsConfig.py | 104 ++++++++++++++++++++++++++++++----- 1 file changed, 89 insertions(+), 15 deletions(-) diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index c1f8dcf1..ffeb2c35 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -14,59 +14,76 @@ class k4GeneratorsConfig(): def __init__(self,arguments=None): + # define all command line arguments parser = argparse.ArgumentParser(prog="k4GeneratorsConfig") + parser.add_argument( "--make", action='store_true', help="make the generator datacards from the yaml files" ) + + self.yamlDefault = [os.path.dirname(os.path.realpath(__file__))+'/../examples'] parser.add_argument( "--yaml", nargs="*", type=str, - default=[os.path.dirname(os.path.realpath(__file__))+'/../examples'], + default=argparse.SUPPRESS, help="yamlFiles and director(y/ies) with yaml files (default: k4GeneratorsConfig/examples)" ) + + self.sqrtsDefault = [] parser.add_argument( "--sqrts", nargs="*", type=str, - default=[], + default=argparse.SUPPRESS, help="either a space separated list of center of mass energies OR file(s) and director(y/ies) with sqrts lists in yaml format (name : sqrtsPROCESS.dat containing eg, sqrts:[91.,240.]), sqrts.yaml as single argument will be applied to all processes", ) + + self.seedDefault = 4711 parser.add_argument( "--seed", nargs=1, type=int, - default=4711, - help="initial random number seed, increment for each file", + default=argparse.SUPPRESS, + help="If specified overrides the random number seed in the yamlFile, incremented for each process and each sqrts of a yaml file, default: yaml file", ) + + self.nevtsDefault = -1 parser.add_argument( "--nevts", type=int, - default=-1, - help="Number of events to be generated", + default=argparse.SUPPRESS, + help="If specified overrides the number of events to be generated in the yamlFile, default: use yaml", ) + + self.parameterTagDefault = "latest" parser.add_argument( "--parameterTag", type=str, - default="latest", + default=argparse.SUPPRESS, help="parameter tag in Parameters.yaml (default: latest)", ) + + self.parameterTagFileDefault = os.path.dirname(os.path.realpath(__file__))+'/ParameterSets.yaml' parser.add_argument( "--parameterTagFile", type=str, - default=os.path.dirname(os.path.realpath(__file__))+'/ParameterSets.yaml', + default=argparse.SUPPRESS, help="name of file containing the parameter sets of the requested parameterTag, default: ParameterSets.yaml in directory: k4GeneratorsConfig/python", ) + parser.add_argument( "--key4hepUseNightlies", action='store_true', help="configures the key4hepscripts to use nightlies instead of releases", ) + + self.key4hepVersionDefault = None parser.add_argument( "--key4hepVersion", - default=None, + default=argparse.SUPPRESS, help="force the use of the version : YYYY-MM-DD (default: latest)", ) parser.add_argument( @@ -96,6 +113,7 @@ def __init__(self,arguments=None): action='store_true', help="compare the results of the event generation process by process and produce summary output in outputDir" ) + self.outputDirDefault = "work" parser.add_argument( "--outputDir", @@ -132,7 +150,7 @@ def processArguments(self, args): # OPTION: outputDir # differentiate between option being given or not try: - check = self.args.outputDir + check = args.outputDir self.args.outputDirOverride = True except AttributeError: # argument was not given, set the default and make it known: @@ -140,7 +158,7 @@ def processArguments(self, args): self.args.outputDirOverride = False # --all overrides --make --generate --summary - if self.args.all: + if args.all: self.args.make = True self.args.generate = True self.args.summary = True @@ -152,10 +170,10 @@ def processArguments(self, args): sys.exit(message) # OPTION all or make&&generate&&summary - if ( self.args.all or - (self.args.make and self.args.check) or - (self.args.make and self.args.generate) or - (self.args.make and self.args.summary) ): + if ( args.all or + (args.make and args.check) or + (args.make and args.generate) or + (args.make and args.summary) ): if not self.args.outputDirOverride: message = f"k4GeneratorsConfig::ERROR\n" message += f"--make and (--check and/or --generate and/or --summary) requested\n" @@ -163,6 +181,62 @@ def processArguments(self, args): message += f"BUT: --outputDir {self.args.outputDir} not defined \nPlease define a common output directory" sys.exit(message) + # make sure that Warnings are issued when options specific to --make are invoked, make is false + message = "k4GeneratorsConfig::WARNING invoking option specific to --make or --all:" + try: + check = self.args.yaml + if not args.make and not args.all: + print(f"{message} --yaml {self.args.yaml} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.yaml = self.yamlDefault + try: + check = self.args.sqrts + if not args.make and not args.all: + print(f"{message} --sqrts {self.args.sqrts} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.sqrts = self.sqrtsDefault + try: + check = self.args.seed + if not args.make and not args.all: + print(f"{message} --seed {self.args.seed} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.seed = self.seedDefault + try: + check = self.args.nevts + if not args.make and not args.all: + print(f"{message} --nevts {self.args.nevts} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.nevts = self.nevtsDefault + try: + check = self.args.parameterTag + if not args.make and not args.all: + print(f"{message} --parameterTag {self.args.parameterTag} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.parameterTag = self.parameterTagDefault + try: + check = self.args.parameterTagFile + if not args.make and not args.all: + print(f"{message} --parameterTagFile {self.args.parameterTagFile} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.parameterTagFile = self.parameterTagFileDefault + try: + check = self.args.key4hepVersion + if not args.make and not args.all: + print(f"{message} --key4hepVersion {self.args.key4hepVersion} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.key4hepVersion = self.key4hepVersionDefault + + # special treatment for nightlies storetrue + if not args.make and not args.all and args.key4hepUseNightlies: + print(f"k4GeneratorsConfig::WARNING invoking option specific to --make or --all : --key4hepUseNightlies has no effect") + if __name__ == "__main__": k4GeneratorsConfig() From 051a813ea015eb6a729d953a19a56e9463b835ab Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 8 Apr 2026 14:09:04 +0200 Subject: [PATCH 81/88] clang --- python/k4GeneratorsConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index ffeb2c35..c87207fe 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -16,7 +16,7 @@ class k4GeneratorsConfig(): def __init__(self,arguments=None): # define all command line arguments parser = argparse.ArgumentParser(prog="k4GeneratorsConfig") - + parser.add_argument( "--make", action='store_true', From 4f243e2114911284f4f38bb30a7e7ca433b7a86a Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 8 Apr 2026 14:43:11 +0200 Subject: [PATCH 82/88] remove old element --- python/Process.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/Process.py b/python/Process.py index deebd397..4192fc23 100644 --- a/python/Process.py +++ b/python/Process.py @@ -27,7 +27,6 @@ def __init__(self, args, procname, params, particleData, **options): # all particles in process list self._particlesOfProcessList = [] # label to be used in the generatorDB - self.generatorDBLabel = "" self.generatorDBTag = [] self.procname = procname @@ -129,9 +128,6 @@ def get_BeamstrahlungFile(self): circe = CirceHelper(self.beamstrahlung, self.sqrts) return circe.getFile() - def get_generatorDBLabel(self): - return self.generatorDBLabel - def get_DBTag(self): return self._DBTag From 24c334c2860fbd1d59f925adf77de60551812c24 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 8 Apr 2026 17:04:36 +0200 Subject: [PATCH 83/88] remove ProcessParameters by access to settings --- python/Generators/Babayaga.py | 4 ++-- python/Generators/KKMC.py | 6 +++--- python/Generators/Madgraph.py | 6 +++--- python/Generators/Pythia.py | 2 +- python/Generators/Sherpa.py | 4 ++-- python/Generators/Whizard.py | 4 ++-- python/Process.py | 39 ++++++++++++++--------------------- python/Yaml2Datacard.py | 24 +++++++++++++-------- python/YamlInputReader.py | 2 +- python/k4GeneratorsConfig.py | 5 ++++- 10 files changed, 48 insertions(+), 48 deletions(-) diff --git a/python/Generators/Babayaga.py b/python/Generators/Babayaga.py index 7c75fcb6..9af6685a 100644 --- a/python/Generators/Babayaga.py +++ b/python/Generators/Babayaga.py @@ -49,7 +49,7 @@ def write_process(self): self.addOption2GeneratorDatacard("ord", "alpha") self.addOption2GeneratorDatacard("EWKc", "on") - self.addOption2GeneratorDatacard("nev", self.procinfo.get("events")) + self.addOption2GeneratorDatacard("nev", self.settings.get_nevents()) self.addOption2GeneratorDatacard("ecms", self.procinfo.get("sqrts")) # output format only hepm2 or hepmc3, the actual version is detected by the linked library, so strip the number @@ -60,7 +60,7 @@ def write_process(self): for key in self.procDB.getDict(): self.addOption2GeneratorDatacard(key,self.procDB.getDict()[key]) - if self.procinfo.eventmode == "unweighted": + if self.procinfo.get("eventmode") == "unweighted": self.addOption2GeneratorDatacard("mode", "unweighted") else: self.addOption2GeneratorDatacard("mode", "weighted") diff --git a/python/Generators/KKMC.py b/python/Generators/KKMC.py index 7db491f1..4908d897 100644 --- a/python/Generators/KKMC.py +++ b/python/Generators/KKMC.py @@ -57,13 +57,13 @@ def fill_datacard(self): self.replaceOptionInGeneratorDatacard("_besdelta", 0) # TODO add bes self.replaceOptionInGeneratorDatacard("_besmode", 0) - self.replaceOptionInGeneratorDatacard("_ewmode", self.procinfo.get("ewmode")) + self.replaceOptionInGeneratorDatacard("_ewmode", self.settings.get_ew_mode()) self.replaceOptionInGeneratorDatacard("_isrmode", self.procinfo.get("isrmode")) self.replaceOptionInGeneratorDatacard("_fsrmode", self.settings.get("fsrmode", 0)) self.replaceOptionInGeneratorDatacard("_FINALSTATES", f" {self.finalstate} 1") # output format only hepm2 or hepmc3, the actual version is detected by the linked library, so strip the number - if self.procinfo.eventmode == "unweighted": + if self.procinfo.get("eventmode") == "unweighted": self.replaceOptionInGeneratorDatacard("_wgtmode", "0") else: self.replaceOptionInGeneratorDatacard("_wgtmode", "1") @@ -88,7 +88,7 @@ def fill_key4hepScript(self): key4hepRun += "KKMCee -c {0} --nevts {2} -o {1}.hepmc3\n".format( self.GeneratorDatacardName, self.GeneratorDatacardBase, - self.procinfo.get("events"), + self.settings.get_nevents(), ) outformat = self.procinfo.get_output_format() if outformat == "edm4hep": diff --git a/python/Generators/Madgraph.py b/python/Generators/Madgraph.py index f44cd5c7..b26d38b7 100644 --- a/python/Generators/Madgraph.py +++ b/python/Generators/Madgraph.py @@ -73,7 +73,7 @@ def fill_datacard(self): # now add the particles checking for overlap with ProcDB self.prepareParticles() # temporary fix: increase LHE event size - self.addOption2GeneratorDatacard("set nevents", int(self.procinfo.get("events")*1.002)) + self.addOption2GeneratorDatacard("set nevents", int(self.settings.get_nevents()*1.002)) if self.procinfo.get("isrmode"): if self.procinfo.get("beamstrahlung") is not None: # if self.gen_settings is None: @@ -236,12 +236,12 @@ def fill_key4hepScript(self): def fill_PythiaCMND(self): # append the analysis to the content # for the errors allow 1 permil failures - allowedErrors = int(self.procinfo.get("events")*0.001) + allowedErrors = int(self.settings.get_nevents()*0.001) content = f"Main:timesAllowErrors = {allowedErrors}\n" content += "Check:epTolErr = 0.01\n" content += "Main:WriteHepMC = on\n" content += "Beams:frameType = 4\n" - content += "Main:numberOfEvents = {0}\n".format(self.procinfo.get("events")) + content += "Main:numberOfEvents = {0}\n".format(self.settings.get_nevents()) self.add2OptionalFile(content) def getModelName(self, model): diff --git a/python/Generators/Pythia.py b/python/Generators/Pythia.py index 7be78dea..efa50f9c 100644 --- a/python/Generators/Pythia.py +++ b/python/Generators/Pythia.py @@ -72,7 +72,7 @@ def fill_run(self): else: self.addOption2GeneratorDatacard("PartonLevel:FSR", "off") - self.addOption2GeneratorDatacard("Main:numberOfEvents", self.procinfo.get("events")) + self.addOption2GeneratorDatacard("Main:numberOfEvents", self.settings.get_nevents()) self.add2GeneratorDatacard("\n") # now add the model parameters diff --git a/python/Generators/Sherpa.py b/python/Generators/Sherpa.py index 6be3b42c..072507ab 100644 --- a/python/Generators/Sherpa.py +++ b/python/Generators/Sherpa.py @@ -53,7 +53,7 @@ def write_run(self): self.addOption2GeneratorDatacard("YFS_MODE", "FULL") else: self.addOption2GeneratorDatacard("YFS_MODE", "None") - self.addOption2GeneratorDatacard("EVENTS", self.procinfo.get("events")) + self.addOption2GeneratorDatacard("EVENTS", self.settings.get_nevents()) self.add2GeneratorDatacard("\n") # now add the model checking for overlap @@ -67,7 +67,7 @@ def write_run(self): for key in self.procDB.getDictRun(): self.addOption2GeneratorDatacard(key,self.procDB.getDictRun()[key]) - self.addOption2GeneratorDatacard("EVENT_GENERATION_MODE", self.procinfo.eventmode) + self.addOption2GeneratorDatacard("EVENT_GENERATION_MODE", self.procinfo.get("eventmode")) if self.gen_settings is not None: if "run" in self.gen_settings.keys(): for key, value in self.gen_settings["run"].items(): diff --git a/python/Generators/Whizard.py b/python/Generators/Whizard.py index 7c835012..f717992c 100644 --- a/python/Generators/Whizard.py +++ b/python/Generators/Whizard.py @@ -74,7 +74,7 @@ def write_process(self): self.add2GeneratorDatacard(f"process proc = {self.whiz_beam1}, {self.whiz_beam2} => {self.finalstate}\n") - self.addOption2GeneratorDatacard("n_events", self.procinfo.get("events")) + self.addOption2GeneratorDatacard("n_events", self.settings.get_nevents()) self.addOption2GeneratorDatacard("sqrts", self.procinfo.get("sqrts")) if self.procinfo.get("decay"): self.add_decay() @@ -95,7 +95,7 @@ def write_process(self): for key in self.procDB.getDict(): self.addOption2GeneratorDatacard(key,self.procDB.getDict()[key]) - if self.procinfo.eventmode == "unweighted": + if self.procinfo.get("eventmode") == "unweighted": self.addOption2GeneratorDatacard("?unweighted", "true") else: self.addOption2GeneratorDatacard("?unweighted", "false") diff --git a/python/Process.py b/python/Process.py index 4192fc23..e40226a1 100644 --- a/python/Process.py +++ b/python/Process.py @@ -17,7 +17,7 @@ class Process: "beamstrahlung", ] - def __init__(self, args, procname, params, particleData, **options): + def __init__(self, procname, args, inputFileRead, particleData, **options): # list of particles filled from the input yaml file self._inputParticlesList = [] if particleData is not None: @@ -28,14 +28,15 @@ def __init__(self, args, procname, params, particleData, **options): self._particlesOfProcessList = [] # label to be used in the generatorDB self.generatorDBTag = [] + + # process identifier self.procname = procname - for arg in self._required_args: - setattr(self, arg, params.settings.get(arg)) + # inputReader + self.settings = inputFileRead - for setting in dir(params): - if not setting.startswith("__"): - setattr(self, setting, getattr(params, setting)) + for arg in self._required_args: + setattr(self, arg, self.settings.get(arg)) for option, value in options.items(): setattr(self, option, value) @@ -88,7 +89,10 @@ def get(self, name): try: return getattr(self, name) except: - return None + try: + return self.settings.get(name) + except: + return None def get_args(self): return self._required_args @@ -109,16 +113,16 @@ def get_nlo(self): return self.get("nlo") def get_output_format(self): - return self.output_format + return self.settings.get_output_format() def get_PythiaTune(self): - return self.PythiaTune + return self.settings.get_PythiaTune() def get_PolarisationDensity(self): - return self.PolarisationDensity + return self.settings.get_PolarisationDensity() def get_PolarisationFraction(self): - return self.PolarisationFraction + return self.settings.get_PolarisationFraction() def get_rndmSeed(self): return self.get("randomseed") @@ -136,16 +140,3 @@ def print_info(self): print("Particles are defined with the following parameters") for part in self._particlesOfProcessList: part.print_info() - - -class ProcessParameters: - def __init__(self, settings): - self.settings = settings - self.model = settings.get_model() - self.events = settings.get_event_number() - self.output_format = settings.get_output_format() - self.PythiaTune = settings.get_PythiaTune() - self.PolarisationDensity = settings.get_PolarisationDensity() - self.PolarisationFraction = settings.get_PolarisationFraction() - self.eventmode = settings.get_weighted_mode() - self.ewmode = settings.get_ew_mode() diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py index d5889f84..134c5ef4 100644 --- a/python/Yaml2Datacard.py +++ b/python/Yaml2Datacard.py @@ -11,7 +11,6 @@ from ReleaseSpecs import ReleaseSpec import YamlInputReader as Reader from Process import Process -from Process import ProcessParameters from Generators import Generators from Particles import ParticleCollection @@ -73,23 +72,30 @@ def processFile(self): self.processReader.set("events", events) # the datacard outputDir may differ self.processOutputDir() - # now extract information + # extract information processes = self.processReader.get_processes(sqrts) yamlParticleData = self.processReader.get_particle_data() generators = Generators(self.processReader) # for key, value in processes.items(): + + # make the combination of generator directories and in each generator director the process directory self.makeDirectories4GeneratorsProcess(self.processReader.get_generators(), key) - try: - randomseed = value["randomseed"] - except: - # random seed not present, fall to external setting and increment for next round + + # the command line seed supersedes the seed read from file + if self.args.seedOverride: value["randomseed"] = rndmSeed - rndmSeed += 1 - param = ProcessParameters(self.processReader) + else: + # check for seed in the file/process + try: + randomseed = value["randomseed"] + except: + # random seed not present, fall to external default setting and increment for next round + value["randomseed"] = rndmSeed + rndmSeed += 1 # instantiate the class for each process process = Process( - value, key, param, yamlParticleData, OutDir=self.outputDir + key, value, self.processReader, yamlParticleData, OutDir=self.outputDir ) process.prepareProcess() generators.runGeneratorConfiguration(process) diff --git a/python/YamlInputReader.py b/python/YamlInputReader.py index c1666b54..555f1198 100644 --- a/python/YamlInputReader.py +++ b/python/YamlInputReader.py @@ -128,7 +128,7 @@ def get_particle_data(self): def get_model(self): return self.get("model", "SM") - def get_event_number(self): + def get_nevents(self): return self.get("events", 0) def get_isr_mode(self): diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index c87207fe..3df32c74 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -44,7 +44,6 @@ def __init__(self,arguments=None): self.seedDefault = 4711 parser.add_argument( "--seed", - nargs=1, type=int, default=argparse.SUPPRESS, help="If specified overrides the random number seed in the yamlFile, incremented for each process and each sqrts of a yaml file, default: yaml file", @@ -199,11 +198,15 @@ def processArguments(self, args): self.args.sqrts = self.sqrtsDefault try: check = self.args.seed + self.args.seedOverride = True + if check <= 0: + sys.exit(f"k4GeneratorsConfig::ERROR --seed {self.args.seed} specified, must be >0") if not args.make and not args.all: print(f"{message} --seed {self.args.seed} has no effect") except AttributeError as e: if args.make or args.all: self.args.seed = self.seedDefault + self.args.seedOverride = False try: check = self.args.nevts if not args.make and not args.all: From e4d37c5137f3f3ce37f08c4f0bf0e55e5969aca8 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 8 Apr 2026 18:04:16 +0200 Subject: [PATCH 84/88] #78 remove input file from settings of Generators --- python/Generators/Generators.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/Generators/Generators.py b/python/Generators/Generators.py index 9dfff8a7..21fd4498 100644 --- a/python/Generators/Generators.py +++ b/python/Generators/Generators.py @@ -4,7 +4,6 @@ class Generators: """Generator class""" def __init__(self, settings): - self.settings = settings self.generator_list = settings.get_generators() def set_process_info(self, proc_info): @@ -23,7 +22,7 @@ def runGeneratorConfiguration(self, proc_info): # get the ClassObject generatorClass = getattr(generator,generatorName) # execute the object - generatorObj = generatorClass(self.proc_info, self.settings) + generatorObj = generatorClass(self.proc_info, self.proc_info.settings) # execute the generator generatorObj.execute() # finalize the generator From 8e4aec864703ad0ef383506a4240e5f320a25a86 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 8 Apr 2026 18:15:22 +0200 Subject: [PATCH 85/88] #78 access basic info from procinfo for consistancy --- python/Generators/Babayaga.py | 8 ++++---- python/Generators/GeneratorBase.py | 25 ++++++++++++------------- python/Generators/Generators.py | 2 +- python/Generators/KKMC.py | 10 +++++----- python/Generators/Madgraph.py | 12 ++++++------ python/Generators/Pythia.py | 8 ++++---- python/Generators/Sherpa.py | 8 ++++---- python/Generators/Whizard.py | 8 ++++---- 8 files changed, 40 insertions(+), 41 deletions(-) diff --git a/python/Generators/Babayaga.py b/python/Generators/Babayaga.py index 9af6685a..7e2d84e4 100644 --- a/python/Generators/Babayaga.py +++ b/python/Generators/Babayaga.py @@ -3,8 +3,8 @@ class Babayaga(GeneratorBase): """Babayaga class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "Babayaga", "dat") + def __init__(self, procinfo): + super().__init__(procinfo, "Babayaga", "dat") self.version = "x.y.z" @@ -49,7 +49,7 @@ def write_process(self): self.addOption2GeneratorDatacard("ord", "alpha") self.addOption2GeneratorDatacard("EWKc", "on") - self.addOption2GeneratorDatacard("nev", self.settings.get_nevents()) + self.addOption2GeneratorDatacard("nev", self.procinfo.settings.get_nevents()) self.addOption2GeneratorDatacard("ecms", self.procinfo.get("sqrts")) # output format only hepm2 or hepmc3, the actual version is detected by the linked library, so strip the number @@ -65,7 +65,7 @@ def write_process(self): else: self.addOption2GeneratorDatacard("mode", "weighted") - if self.settings.get_block("selectors"): + if self.procinfo.settings.get_block("selectors"): self.writeAllSelectors() def add_decay(self): diff --git a/python/Generators/GeneratorBase.py b/python/Generators/GeneratorBase.py index c5050fa8..0513068f 100644 --- a/python/Generators/GeneratorBase.py +++ b/python/Generators/GeneratorBase.py @@ -11,11 +11,10 @@ class GeneratorBase(ABC): """GeneratorBase class""" - def __init__(self, procinfo, settings, name, inputFileExtension): + def __init__(self, procinfo, name, inputFileExtension): # general settings of the class self.procinfo = procinfo - self.settings = settings self.name = name self.procDBName = f"{name}ProcDB" self.inputFileExtension = inputFileExtension @@ -80,7 +79,7 @@ def __init__(self, procinfo, settings, name, inputFileExtension): self.prepareAnalysisContent() # the generator settings are stored in a public member: - self.gen_settings = settings.get_block(self.name.lower()) + self.gen_settings = self.procinfo.settings.get_block(self.name.lower()) if self.gen_settings is not None: self.gen_settings = {k.lower(): v for k, v in self.gen_settings.items()} @@ -110,7 +109,7 @@ def __init__(self, procinfo, settings, name, inputFileExtension): self.procDBparameters = dict() self.procDBparticles = dict() - if self.settings.get("usedefaults", True): + if self.procinfo.settings.get("usedefaults", True): self.procDB.execute() self.procDBparameters = self.procDB.getDictParameters() self.procDBparticles = self.procDB.getDictParticles() @@ -136,9 +135,9 @@ def validateSelectorsDict(self): raise ValueError(f"{self.name} {key} not found in SelectorKeys list") def writeAllSelectors(self): - selectors = getattr(self.settings, "selectors") + selectors = getattr(self.procinfo.settings, "selectors") try: - procselectors = getattr(self.settings, "procselectors") + procselectors = getattr(self.procinfo.settings, "procselectors") for proc, sel in procselectors.items(): if proc != self.procinfo.get("procname"): continue @@ -231,8 +230,8 @@ def checkModelParameters(self): def isCompatible(self, target, prediction): # maximum relative deviation RelDiffThreshold = 0.001 - if self.settings.get("EWParamDevThreshold".lower()) is not None: - RelDiffThreshold = self.settings.get("EWParamDevThreshold".lower()) + if self.procinfo.settings.get("EWParamDevThreshold".lower()) is not None: + RelDiffThreshold = self.procinfo.settings.get("EWParamDevThreshold".lower()) # do the safe math relDelta = 1. # if the target is zero, then take the absolute deviation, if not the relative deviation @@ -521,10 +520,10 @@ def prepareKey4hepScript(self): def prepareAnalysisContent(self): # analysis is conditioned on the output format - outformat = self.settings.get_output_format() + outformat = self.procinfo.settings.get_output_format() # write the EDM4HEP analysis part based on the final state analysis = "\n" - if outformat == "edm4hep" and self.settings.key4HEPAnalysisON(): + if outformat == "edm4hep" and self.procinfo.settings.key4HEPAnalysisON(): analysis += f"key4HEPAnalysis -i {self.GeneratorDatacardBase}.edm4hep -o {self.GeneratorDatacardBase}.root -p " for pdg in self.procinfo.get_finalstate_pdgList(): @@ -533,10 +532,10 @@ def prepareAnalysisContent(self): analysis +="\n" # write the RIVET analysis - if (outformat == "edm4hep" or outformat == "hepmc3") and self.settings.rivetON(): - yodaout = self.settings.yodaoutput + f"/{self.procinfo.get('procname')}.yoda" + if (outformat == "edm4hep" or outformat == "hepmc3") and self.procinfo.settings.rivetON(): + yodaout = self.procinfo.settings.yodaoutput + f"/{self.procinfo.get('procname')}.yoda" analysis += f"rivet" - for ana in self.settings.analysisname: + for ana in self.procinfo.settings.analysisname: analysis += f" -a {ana}" analysis+=f" -o {yodaout} {self.procinfo.get('procname')}.{self.procinfo.get_output_format()}\n" diff --git a/python/Generators/Generators.py b/python/Generators/Generators.py index 21fd4498..3cb3d572 100644 --- a/python/Generators/Generators.py +++ b/python/Generators/Generators.py @@ -22,7 +22,7 @@ def runGeneratorConfiguration(self, proc_info): # get the ClassObject generatorClass = getattr(generator,generatorName) # execute the object - generatorObj = generatorClass(self.proc_info, self.proc_info.settings) + generatorObj = generatorClass(self.proc_info) # execute the generator generatorObj.execute() # finalize the generator diff --git a/python/Generators/KKMC.py b/python/Generators/KKMC.py index 4908d897..0133580f 100644 --- a/python/Generators/KKMC.py +++ b/python/Generators/KKMC.py @@ -6,8 +6,8 @@ class KKMC(GeneratorBase): """KKMC class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "KKMC", "dat") + def __init__(self, procinfo): + super().__init__(procinfo, "KKMC", "dat") self.version = "x.y.z" self.executable = "KKMC-fcc.exe" @@ -57,9 +57,9 @@ def fill_datacard(self): self.replaceOptionInGeneratorDatacard("_besdelta", 0) # TODO add bes self.replaceOptionInGeneratorDatacard("_besmode", 0) - self.replaceOptionInGeneratorDatacard("_ewmode", self.settings.get_ew_mode()) + self.replaceOptionInGeneratorDatacard("_ewmode", self.procinfo.settings.get_ew_mode()) self.replaceOptionInGeneratorDatacard("_isrmode", self.procinfo.get("isrmode")) - self.replaceOptionInGeneratorDatacard("_fsrmode", self.settings.get("fsrmode", 0)) + self.replaceOptionInGeneratorDatacard("_fsrmode", self.procinfo.settings.get("fsrmode", 0)) self.replaceOptionInGeneratorDatacard("_FINALSTATES", f" {self.finalstate} 1") # output format only hepm2 or hepmc3, the actual version is detected by the linked library, so strip the number @@ -88,7 +88,7 @@ def fill_key4hepScript(self): key4hepRun += "KKMCee -c {0} --nevts {2} -o {1}.hepmc3\n".format( self.GeneratorDatacardName, self.GeneratorDatacardBase, - self.settings.get_nevents(), + self.procinfo.settings.get_nevents(), ) outformat = self.procinfo.get_output_format() if outformat == "edm4hep": diff --git a/python/Generators/Madgraph.py b/python/Generators/Madgraph.py index b26d38b7..d4db8136 100644 --- a/python/Generators/Madgraph.py +++ b/python/Generators/Madgraph.py @@ -5,8 +5,8 @@ class Madgraph(GeneratorBase): """Madgraph class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "Madgraph", "dat") + def __init__(self, procinfo): + super().__init__(procinfo, "Madgraph", "dat") self.version = "x.y.z" @@ -73,7 +73,7 @@ def fill_datacard(self): # now add the particles checking for overlap with ProcDB self.prepareParticles() # temporary fix: increase LHE event size - self.addOption2GeneratorDatacard("set nevents", int(self.settings.get_nevents()*1.002)) + self.addOption2GeneratorDatacard("set nevents", int(self.procinfo.settings.get_nevents()*1.002)) if self.procinfo.get("isrmode"): if self.procinfo.get("beamstrahlung") is not None: # if self.gen_settings is None: @@ -93,7 +93,7 @@ def fill_datacard(self): for key in self.procDB.getDict(): self.addOption2GeneratorDatacard(key, self.procDB.getDict()[key]) - # if self.settings.get_block("selectors"): + # if self.procinfo.settings.get_block("selectors"): self.writeAllSelectors() # else: # self.add_default_Selectors() @@ -236,12 +236,12 @@ def fill_key4hepScript(self): def fill_PythiaCMND(self): # append the analysis to the content # for the errors allow 1 permil failures - allowedErrors = int(self.settings.get_nevents()*0.001) + allowedErrors = int(self.procinfo.settings.get_nevents()*0.001) content = f"Main:timesAllowErrors = {allowedErrors}\n" content += "Check:epTolErr = 0.01\n" content += "Main:WriteHepMC = on\n" content += "Beams:frameType = 4\n" - content += "Main:numberOfEvents = {0}\n".format(self.settings.get_nevents()) + content += "Main:numberOfEvents = {0}\n".format(self.procinfo.settings.get_nevents()) self.add2OptionalFile(content) def getModelName(self, model): diff --git a/python/Generators/Pythia.py b/python/Generators/Pythia.py index efa50f9c..e8353c65 100644 --- a/python/Generators/Pythia.py +++ b/python/Generators/Pythia.py @@ -3,15 +3,15 @@ class Pythia(GeneratorBase): """Pythia class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "Pythia", "dat") + def __init__(self, procinfo): + super().__init__(procinfo, "Pythia", "dat") self.version = "x.y.z" self.executable = "pythiaRunner -f" self.setOptionalFileNameAndExtension(self.GeneratorDatacardBase,"selectors") - if settings.get_block("selectors"): + if self.procinfo.settings.get_block("selectors"): self.writeAllSelectors() def setSelectorsDict(self): @@ -72,7 +72,7 @@ def fill_run(self): else: self.addOption2GeneratorDatacard("PartonLevel:FSR", "off") - self.addOption2GeneratorDatacard("Main:numberOfEvents", self.settings.get_nevents()) + self.addOption2GeneratorDatacard("Main:numberOfEvents", self.procinfo.settings.get_nevents()) self.add2GeneratorDatacard("\n") # now add the model parameters diff --git a/python/Generators/Sherpa.py b/python/Generators/Sherpa.py index 072507ab..1bb9e58e 100644 --- a/python/Generators/Sherpa.py +++ b/python/Generators/Sherpa.py @@ -3,8 +3,8 @@ class Sherpa(GeneratorBase): """Sherpa class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "Sherpa", "dat") + def __init__(self, procinfo): + super().__init__(procinfo, "Sherpa", "dat") self.version = "3" self.executable = "Sherpa -f" @@ -53,7 +53,7 @@ def write_run(self): self.addOption2GeneratorDatacard("YFS_MODE", "FULL") else: self.addOption2GeneratorDatacard("YFS_MODE", "None") - self.addOption2GeneratorDatacard("EVENTS", self.settings.get_nevents()) + self.addOption2GeneratorDatacard("EVENTS", self.procinfo.settings.get_nevents()) self.add2GeneratorDatacard("\n") # now add the model checking for overlap @@ -159,7 +159,7 @@ def fill_datacard(self): if self.procinfo.get("decay"): self.write_decay() # writing selectors depends on the presence of the block - if self.settings.get_block("selectors"): + if self.procinfo.settings.get_block("selectors"): self.add2GeneratorDatacard("\nSELECTORS:\n") self.writeAllSelectors() diff --git a/python/Generators/Whizard.py b/python/Generators/Whizard.py index f717992c..d241af66 100644 --- a/python/Generators/Whizard.py +++ b/python/Generators/Whizard.py @@ -3,8 +3,8 @@ class Whizard(GeneratorBase): """Whizard class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "Whizard", "sin") + def __init__(self, procinfo): + super().__init__(procinfo, "Whizard", "sin") self.version = "x.y.z" @@ -74,7 +74,7 @@ def write_process(self): self.add2GeneratorDatacard(f"process proc = {self.whiz_beam1}, {self.whiz_beam2} => {self.finalstate}\n") - self.addOption2GeneratorDatacard("n_events", self.settings.get_nevents()) + self.addOption2GeneratorDatacard("n_events", self.procinfo.settings.get_nevents()) self.addOption2GeneratorDatacard("sqrts", self.procinfo.get("sqrts")) if self.procinfo.get("decay"): self.add_decay() @@ -100,7 +100,7 @@ def write_process(self): else: self.addOption2GeneratorDatacard("?unweighted", "false") - if self.settings.get_block("selectors"): + if self.procinfo.settings.get_block("selectors"): self.CutKeyWdPresent = False self.aCutIsPresent = False self.writeAllSelectors() From 1dba14eaea5932767167bc77dca827127c398a71 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Wed, 8 Apr 2026 18:18:02 +0200 Subject: [PATCH 86/88] #78 remove superfluous method --- python/Generators/Generators.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/Generators/Generators.py b/python/Generators/Generators.py index 3cb3d572..1c3e834d 100644 --- a/python/Generators/Generators.py +++ b/python/Generators/Generators.py @@ -6,13 +6,10 @@ class Generators: def __init__(self, settings): self.generator_list = settings.get_generators() - def set_process_info(self, proc_info): - self.proc_info = proc_info - def runGeneratorConfiguration(self, proc_info): # first set process info - self.set_process_info(proc_info) + self.proc_info = proc_info # second import and run the configuration of the generators for generatorName in self.generator_list: From ec011ce56194691ad2a7d16ea83ddbc26282dbd2 Mon Sep 17 00:00:00 2001 From: Dirk Zerwas Date: Thu, 9 Apr 2026 11:17:42 +0200 Subject: [PATCH 87/88] Process: deep copy of input and removal of double info --- python/Process.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/python/Process.py b/python/Process.py index 41f40a7c..e1d42b91 100644 --- a/python/Process.py +++ b/python/Process.py @@ -1,3 +1,4 @@ +import copy from Particles import Particle from Generators.CirceHelper import CirceHelper @@ -17,7 +18,7 @@ class Process: "beamstrahlung", ] - def __init__(self, procname, args, inputFileRead, particleData, **options): + def __init__(self, procname, process, inputFileRead, particleData, **options): # list of particles filled from the input yaml file self._inputParticlesList = [] if particleData is not None: @@ -32,18 +33,19 @@ def __init__(self, procname, args, inputFileRead, particleData, **options): # process identifier self.procname = procname - # inputReader - self.settings = inputFileRead - for arg in self._required_args: - setattr(self, arg, self.settings.get(arg)) + setattr(self, arg, inputFileRead.get(arg)) for option, value in options.items(): setattr(self, option, value) - for key, value in args.items(): + for key, value in process.items(): setattr(self, key, value) + # inputReader as deep copy without the process stuff + self.settings = copy.deepcopy(inputFileRead) + delattr(self.settings, "processes") + def prepareProcess(self): # beam particles self._beam1 = Particle.get_info(self.initial[0]) From df5cde8c08501ed80ba1056a511e3c2a300ac6df Mon Sep 17 00:00:00 2001 From: apricePhy <146199496+apricePhy@users.noreply.github.com> Date: Fri, 10 Apr 2026 14:25:07 +0100 Subject: [PATCH 88/88] Update k4GeneratorsConfig.py --- python/k4GeneratorsConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py index 3df32c74..a22cea88 100644 --- a/python/k4GeneratorsConfig.py +++ b/python/k4GeneratorsConfig.py @@ -113,7 +113,7 @@ def __init__(self,arguments=None): help="compare the results of the event generation process by process and produce summary output in outputDir" ) - self.outputDirDefault = "work" + self.outputDirDefault = "Run-Cards" parser.add_argument( "--outputDir", type=str,