diff --git a/.github/workflows/key4hep_build.yml b/.github/workflows/key4hep_build.yml index af2a83e6..1cc492a4 100644 --- a/.github/workflows/key4hep_build.yml +++ b/.github/workflows/key4hep_build.yml @@ -22,6 +22,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Enable verbose CTest output + run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV - uses: key4hep/key4hep-actions/key4hep-build@main with: build_type: ${{ matrix.build_type }} @@ -31,7 +33,8 @@ jobs: with: name: k4GeneratorsConfig_report_${{ matrix.build_type }}_${{ matrix.image }} path: | - test/output/* + test/Run-Cards/*.dat + test/Run-Cards/*.root - name: Show full CTest log on failure if: failure() run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index a8e46cbb..818bc537 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,14 +57,14 @@ install(DIRECTORY ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/ DESTINATION ${C install(DIRECTORY ${PROJECT_SOURCE_DIR}/python DESTINATION ${CMAKE_INSTALL_PREFIX} - REGEX test_.*\\.py|main.py$ EXCLUDE # Don't install python unittests + REGEX test_.*\\.py|k4GeneratorsConfig.py$ EXCLUDE # Don't install python unittests PATTERN __pycache__ EXCLUDE # Or pythons caches ) # Install main as an executable -install(PROGRAMS ${PROJECT_SOURCE_DIR}/python/main.py DESTINATION ${CMAKE_INSTALL_PREFIX}/python) +install(PROGRAMS ${PROJECT_SOURCE_DIR}/python/k4GeneratorsConfig.py DESTINATION ${CMAKE_INSTALL_PREFIX}/python) install(CODE "execute_process(COMMAND bash -c \"set -e - link_target=${CMAKE_INSTALL_PREFIX}/python/main.py + link_target=${CMAKE_INSTALL_PREFIX}/python/k4GeneratorsConfig.py link_name=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/k4GeneratorsConfig rm -f $link_name ln -s $link_target $link_name diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.cxx b/k4GeneratorsConfig/src/eventGenerationCollections.cxx index b6509919..a39dceb5 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.cxx +++ b/k4GeneratorsConfig/src/eventGenerationCollections.cxx @@ -5,7 +5,7 @@ #include #include -k4GeneratorsConfig::eventGenerationCollections::eventGenerationCollections() : m_validCounter(0), m_invalidCounter(0) {} +k4GeneratorsConfig::eventGenerationCollections::eventGenerationCollections() {} k4GeneratorsConfig::eventGenerationCollections::eventGenerationCollections( const eventGenerationCollections& theOriginal) { if (this != &theOriginal) { @@ -29,29 +29,29 @@ k4GeneratorsConfig::eventGenerationCollections::operator=(const eventGenerationC return *this; } k4GeneratorsConfig::eventGenerationCollections::~eventGenerationCollections() {} -void k4GeneratorsConfig::eventGenerationCollections::Execute() { +void k4GeneratorsConfig::eventGenerationCollections::Execute(std::string topDir) { // first make the collection - makeCollections(); + makeCollections(topDir); // second order the collection according to the process orderCollections(); } -void k4GeneratorsConfig::eventGenerationCollections::makeCollections() { +void k4GeneratorsConfig::eventGenerationCollections::makeCollections(std::string topDir) { - for (const auto& generators : std::filesystem::directory_iterator("Run-Cards")) { - std::filesystem::path generatorsPath = generators.path(); - if (!std::filesystem::is_directory(generatorsPath)) + for (const auto& generator : std::filesystem::directory_iterator(topDir)) { + std::filesystem::path generatorPath = generator.path(); + if (!std::filesystem::is_directory(generatorPath)) continue; - for (const auto& procs : std::filesystem::directory_iterator(generatorsPath.string())) { - std::filesystem::path processPath = procs.path(); + for (const auto& process : std::filesystem::directory_iterator(generatorPath.string())) { + std::filesystem::path processPath = process.path(); if (!std::filesystem::is_directory(processPath)) continue; k4GeneratorsConfig::xsection* xsec = new k4GeneratorsConfig::xsection(); - for (const auto& files : std::filesystem::directory_iterator(processPath.string())) { - std::filesystem::path filenamePath = files.path(); + for (const auto& filename : std::filesystem::directory_iterator(processPath.string())) { + std::filesystem::path filenamePath = filename.path(); if (!std::filesystem::is_regular_file(filenamePath)) continue; // take care of the total cross section extracted from the EDM4HEP file @@ -61,13 +61,13 @@ void k4GeneratorsConfig::eventGenerationCollections::makeCollections() { xsec->setProcess(processPath.filename().string()); xsec->setFile(filenamePath.string()); // in some cases the generator name is not available, therefore derive from the filename - xsec->setGenerator(generatorsPath.filename().string()); + xsec->setGenerator(generatorPath.filename().string()); std::cout << "Generator " << xsec->Generator() << " has been processed" << std::endl; m_xsectionCollection.push_back(*xsec); if (xsec->isValid()) - m_validCounter++; + addSuccess(xsec->Generator()); if (!xsec->isValid()) - m_invalidCounter++; + addFailure(xsec->Generator()); } } // we need to keep xsec alive for the analysisHistos distributions @@ -84,7 +84,7 @@ void k4GeneratorsConfig::eventGenerationCollections::makeCollections() { diffDist->setFile(filenamePath.string()); diffDist->setSQRTS(xsec->SQRTS()); // in some cases the generator name is not available, therefore derive from the filename - diffDist->setGenerator(generatorsPath.filename().string()); + diffDist->setGenerator(generatorPath.filename().string()); std::cout << "Generator " << diffDist->Generator() << " has been processed for analysisHistos distributions" << std::endl; m_analysisHistosCollection.push_back(*diffDist); @@ -147,8 +147,36 @@ bool k4GeneratorsConfig::eventGenerationCollections::compareLexical(analysisHist return false; } -unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfSuccesses() { return m_validCounter; } -unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfFailures() { return m_invalidCounter; } +void k4GeneratorsConfig::eventGenerationCollections::addSuccess(std::string generator) { + if (m_validCounter.find(generator) != m_validCounter.end()) { + m_validCounter[generator]++; + } else { + m_validCounter[generator] = 1; + } +} +void k4GeneratorsConfig::eventGenerationCollections::addFailure(std::string generator) { + if (m_invalidCounter.find(generator) != m_invalidCounter.end()) { + m_invalidCounter[generator]++; + } else { + m_invalidCounter[generator] = 1; + } +} +unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfSuccesses() const { + unsigned int validTotal = 0; + std::map::const_iterator imap; + for (imap = m_validCounter.begin(); imap != m_validCounter.end(); imap++) { + validTotal += imap->second; + } + return validTotal; +} +unsigned int k4GeneratorsConfig::eventGenerationCollections::NbOfFailures() const { + unsigned int invalidTotal = 0; + std::map::const_iterator imap; + for (imap = m_invalidCounter.begin(); imap != m_invalidCounter.end(); imap++) { + invalidTotal += imap->second; + } + return invalidTotal; +} void k4GeneratorsConfig::eventGenerationCollections::Write2Root(std::string dirname, std::string filename) { eventGenerationCollections2Root out(dirname, filename); @@ -225,7 +253,20 @@ void k4GeneratorsConfig::eventGenerationCollections::PrintSummary(std::ostream& } output << std::endl; // last thing the invalids - output << "Number of runs : " << m_invalidCounter + m_validCounter << std::endl; - output << "Number of failed runs : " << m_invalidCounter << std::endl; - output << "Number of successful runs: " << m_validCounter << std::endl; + output << "Number of runs : " << NbOfFailures() + NbOfSuccesses() << std::endl; + output << "Number of failed runs : " << NbOfFailures() << std::endl; + output << "Number of successful runs: " << NbOfSuccesses() << std::endl; + // details only for failures: + if (NbOfFailures() > 0) { + output << std::endl << "Detail of Failures:" << std::endl; + std::map::const_iterator failure, success; + for (failure = m_invalidCounter.begin(); failure != m_invalidCounter.end(); failure++) { + output << failure->first << " : " << failure->second << " Failures "; + unsigned int successCount = 0; + if ((success = m_validCounter.find(failure->first)) != m_validCounter.end()) { + successCount = success->second; + } + output << " / " << successCount + failure->second << " Runs" << std::endl; + } + } } diff --git a/k4GeneratorsConfig/src/eventGenerationCollections.h b/k4GeneratorsConfig/src/eventGenerationCollections.h index 897d48c1..4b448242 100644 --- a/k4GeneratorsConfig/src/eventGenerationCollections.h +++ b/k4GeneratorsConfig/src/eventGenerationCollections.h @@ -1,6 +1,7 @@ #ifndef K4GENERATORSCONFIG_EVENTGENERATIONCOLLECTIONS_H #define K4GENERATORSCONFIG_EVENTGENERATIONCOLLECTIONS_H +#include #include #include "analysisHistos.h" @@ -14,15 +15,18 @@ class eventGenerationCollections { eventGenerationCollections& operator=(const eventGenerationCollections&); ~eventGenerationCollections(); - void Execute(); - void makeCollections(); + void Execute(std::string); + void makeCollections(std::string); void orderCollections(); bool compareLength(xsection, xsection); bool compareLexical(xsection, xsection); bool compareLexical(analysisHistos, analysisHistos); - unsigned int NbOfSuccesses(); - unsigned int NbOfFailures(); + void addSuccess(std::string); + void addFailure(std::string); + + unsigned int NbOfSuccesses() const; + unsigned int NbOfFailures() const; void Write2Root(std::string, std::string); @@ -35,8 +39,8 @@ class eventGenerationCollections { private: std::vector m_xsectionCollection; std::vector m_analysisHistosCollection; - unsigned int m_validCounter; - unsigned int m_invalidCounter; + std::map m_validCounter; + std::map m_invalidCounter; }; } // namespace k4GeneratorsConfig diff --git a/k4GeneratorsConfig/src/eventGenerationSummary.cxx b/k4GeneratorsConfig/src/eventGenerationSummary.cxx index 99fce0a4..eb68c032 100644 --- a/k4GeneratorsConfig/src/eventGenerationSummary.cxx +++ b/k4GeneratorsConfig/src/eventGenerationSummary.cxx @@ -7,12 +7,16 @@ // int main(int argc, char** argv) { + std::string topDir = "Run-Cards"; std::string filename = "GenerationSummary.dat"; std::string dirRoot = "."; std::string fileRoot = "eventGenerationSummary.root"; int c; - while ((c = getopt(argc, argv, "hf:d:r:")) != -1) + while ((c = getopt(argc, argv, "hw:f:d:r:")) != -1) switch (c) { + case 'w': + topDir = optarg; + break; case 'f': filename = optarg; break; @@ -23,11 +27,14 @@ int main(int argc, char** argv) { fileRoot = optarg; break; case 'h': - std::cout << "Usage: xsectionSummary -h -f filename" << std::endl; + std::cout << "Usage: xsectionSummary -h -w workdir -f filename -d rootdir -r rootfile" << std::endl; std::cout << "-h: print this help" << std::endl; - std::cout << "-f filename: print the summary information to this file" << std::endl; - std::cout << "-d dirname: write the RootTree and figures to this directory" << std::endl; - std::cout << "-r filename: write the RootTree to this file" << std::endl; + std::cout << "-w dirname: top directory to start the search for generators and processes (default: Run-Cards)" + << std::endl; + std::cout << "-f filename: print the summary information to this file (default: GenerationSummary.dat)" + << std::endl; + std::cout << "-d rootdir: write the RootTree and figures to this directory (default: .)" << std::endl; + std::cout << "-r footfile: write the RootTree to this file (default: eventGenerationSummary.root)" << std::endl; exit(0); default: exit(0); @@ -36,7 +43,7 @@ int main(int argc, char** argv) { // instantiate the collection as pointer k4GeneratorsConfig::eventGenerationCollections* evgenColls = new k4GeneratorsConfig::eventGenerationCollections(); // execute the gathering of information including detailed output - evgenColls->Execute(); + evgenColls->Execute(topDir); // do the root analysis evgenColls->Write2Root(dirRoot, fileRoot); // print the summary on screen diff --git a/python/Generators/Babayaga.py b/python/Generators/Babayaga.py index 7c75fcb6..7e2d84e4 100644 --- a/python/Generators/Babayaga.py +++ b/python/Generators/Babayaga.py @@ -3,8 +3,8 @@ class Babayaga(GeneratorBase): """Babayaga class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "Babayaga", "dat") + def __init__(self, procinfo): + super().__init__(procinfo, "Babayaga", "dat") self.version = "x.y.z" @@ -49,7 +49,7 @@ def write_process(self): self.addOption2GeneratorDatacard("ord", "alpha") self.addOption2GeneratorDatacard("EWKc", "on") - self.addOption2GeneratorDatacard("nev", self.procinfo.get("events")) + self.addOption2GeneratorDatacard("nev", self.procinfo.settings.get_nevents()) self.addOption2GeneratorDatacard("ecms", self.procinfo.get("sqrts")) # output format only hepm2 or hepmc3, the actual version is detected by the linked library, so strip the number @@ -60,12 +60,12 @@ def write_process(self): for key in self.procDB.getDict(): self.addOption2GeneratorDatacard(key,self.procDB.getDict()[key]) - if self.procinfo.eventmode == "unweighted": + if self.procinfo.get("eventmode") == "unweighted": self.addOption2GeneratorDatacard("mode", "unweighted") else: self.addOption2GeneratorDatacard("mode", "weighted") - if self.settings.get_block("selectors"): + if self.procinfo.settings.get_block("selectors"): self.writeAllSelectors() def add_decay(self): diff --git a/python/Generators/GeneratorBase.py b/python/Generators/GeneratorBase.py index 06c70fe4..0513068f 100644 --- a/python/Generators/GeneratorBase.py +++ b/python/Generators/GeneratorBase.py @@ -11,11 +11,10 @@ class GeneratorBase(ABC): """GeneratorBase class""" - def __init__(self, procinfo, settings, name, inputFileExtension): + def __init__(self, procinfo, name, inputFileExtension): # general settings of the class self.procinfo = procinfo - self.settings = settings self.name = name self.procDBName = f"{name}ProcDB" self.inputFileExtension = inputFileExtension @@ -32,9 +31,9 @@ def __init__(self, procinfo, settings, name, inputFileExtension): # check consistency: note that the ParameterSets have been defined, check with the particle masses self.checkModelParameters() - # define the output directory as function of the OutDir spec + generator name + process name + # define OutDir as generator name + process name, we assume that we are in the working directory self.outdir = ( - f"{procinfo.get('OutDir')}/{self.name}/{self.procinfo.get('procname')}" + f"{self.name}/{self.procinfo.get('procname')}" ) # configure the filenames @@ -80,7 +79,7 @@ def __init__(self, procinfo, settings, name, inputFileExtension): self.prepareAnalysisContent() # the generator settings are stored in a public member: - self.gen_settings = settings.get_block(self.name.lower()) + self.gen_settings = self.procinfo.settings.get_block(self.name.lower()) if self.gen_settings is not None: self.gen_settings = {k.lower(): v for k, v in self.gen_settings.items()} @@ -110,7 +109,7 @@ def __init__(self, procinfo, settings, name, inputFileExtension): self.procDBparameters = dict() self.procDBparticles = dict() - if self.settings.get("usedefaults", True): + if self.procinfo.settings.get("usedefaults", True): self.procDB.execute() self.procDBparameters = self.procDB.getDictParameters() self.procDBparticles = self.procDB.getDictParticles() @@ -136,9 +135,9 @@ def validateSelectorsDict(self): raise ValueError(f"{self.name} {key} not found in SelectorKeys list") def writeAllSelectors(self): - selectors = getattr(self.settings, "selectors") + selectors = getattr(self.procinfo.settings, "selectors") try: - procselectors = getattr(self.settings, "procselectors") + procselectors = getattr(self.procinfo.settings, "procselectors") for proc, sel in procselectors.items(): if proc != self.procinfo.get("procname"): continue @@ -231,8 +230,8 @@ def checkModelParameters(self): def isCompatible(self, target, prediction): # maximum relative deviation RelDiffThreshold = 0.001 - if self.settings.get("EWParamDevThreshold".lower()) is not None: - RelDiffThreshold = self.settings.get("EWParamDevThreshold".lower()) + if self.procinfo.settings.get("EWParamDevThreshold".lower()) is not None: + RelDiffThreshold = self.procinfo.settings.get("EWParamDevThreshold".lower()) # do the safe math relDelta = 1. # if the target is zero, then take the absolute deviation, if not the relative deviation @@ -521,10 +520,10 @@ def prepareKey4hepScript(self): def prepareAnalysisContent(self): # analysis is conditioned on the output format - outformat = self.settings.get_output_format() + outformat = self.procinfo.settings.get_output_format() # write the EDM4HEP analysis part based on the final state analysis = "\n" - if outformat == "edm4hep" and self.settings.key4HEPAnalysisON(): + if outformat == "edm4hep" and self.procinfo.settings.key4HEPAnalysisON(): analysis += f"key4HEPAnalysis -i {self.GeneratorDatacardBase}.edm4hep -o {self.GeneratorDatacardBase}.root -p " for pdg in self.procinfo.get_finalstate_pdgList(): @@ -533,10 +532,10 @@ def prepareAnalysisContent(self): analysis +="\n" # write the RIVET analysis - if (outformat == "edm4hep" or outformat == "hepmc3") and self.settings.rivetON(): - yodaout = self.settings.yodaoutput + f"/{self.procinfo.get('procname')}.yoda" + if (outformat == "edm4hep" or outformat == "hepmc3") and self.procinfo.settings.rivetON(): + yodaout = self.procinfo.settings.yodaoutput + f"/{self.procinfo.get('procname')}.yoda" analysis += f"rivet" - for ana in self.settings.analysisname: + for ana in self.procinfo.settings.analysisname: analysis += f" -a {ana}" analysis+=f" -o {yodaout} {self.procinfo.get('procname')}.{self.procinfo.get_output_format()}\n" diff --git a/python/Generators/Generators.py b/python/Generators/Generators.py index 310634fd..1c3e834d 100644 --- a/python/Generators/Generators.py +++ b/python/Generators/Generators.py @@ -4,16 +4,12 @@ class Generators: """Generator class""" def __init__(self, settings): - self.settings = settings - self.generator_list = settings.gens() - - def set_process_info(self, proc_info): - self.proc_info = proc_info + self.generator_list = settings.get_generators() def runGeneratorConfiguration(self, proc_info): # first set process info - self.set_process_info(proc_info) + self.proc_info = proc_info # second import and run the configuration of the generators for generatorName in self.generator_list: @@ -23,7 +19,7 @@ def runGeneratorConfiguration(self, proc_info): # get the ClassObject generatorClass = getattr(generator,generatorName) # execute the object - generatorObj = generatorClass(self.proc_info, self.settings) + generatorObj = generatorClass(self.proc_info) # execute the generator generatorObj.execute() # finalize the generator diff --git a/python/Generators/KKMC.py b/python/Generators/KKMC.py index 7db491f1..0133580f 100644 --- a/python/Generators/KKMC.py +++ b/python/Generators/KKMC.py @@ -6,8 +6,8 @@ class KKMC(GeneratorBase): """KKMC class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "KKMC", "dat") + def __init__(self, procinfo): + super().__init__(procinfo, "KKMC", "dat") self.version = "x.y.z" self.executable = "KKMC-fcc.exe" @@ -57,13 +57,13 @@ def fill_datacard(self): self.replaceOptionInGeneratorDatacard("_besdelta", 0) # TODO add bes self.replaceOptionInGeneratorDatacard("_besmode", 0) - self.replaceOptionInGeneratorDatacard("_ewmode", self.procinfo.get("ewmode")) + self.replaceOptionInGeneratorDatacard("_ewmode", self.procinfo.settings.get_ew_mode()) self.replaceOptionInGeneratorDatacard("_isrmode", self.procinfo.get("isrmode")) - self.replaceOptionInGeneratorDatacard("_fsrmode", self.settings.get("fsrmode", 0)) + self.replaceOptionInGeneratorDatacard("_fsrmode", self.procinfo.settings.get("fsrmode", 0)) self.replaceOptionInGeneratorDatacard("_FINALSTATES", f" {self.finalstate} 1") # output format only hepm2 or hepmc3, the actual version is detected by the linked library, so strip the number - if self.procinfo.eventmode == "unweighted": + if self.procinfo.get("eventmode") == "unweighted": self.replaceOptionInGeneratorDatacard("_wgtmode", "0") else: self.replaceOptionInGeneratorDatacard("_wgtmode", "1") @@ -88,7 +88,7 @@ def fill_key4hepScript(self): key4hepRun += "KKMCee -c {0} --nevts {2} -o {1}.hepmc3\n".format( self.GeneratorDatacardName, self.GeneratorDatacardBase, - self.procinfo.get("events"), + self.procinfo.settings.get_nevents(), ) outformat = self.procinfo.get_output_format() if outformat == "edm4hep": diff --git a/python/Generators/Madgraph.py b/python/Generators/Madgraph.py index f44cd5c7..d4db8136 100644 --- a/python/Generators/Madgraph.py +++ b/python/Generators/Madgraph.py @@ -5,8 +5,8 @@ class Madgraph(GeneratorBase): """Madgraph class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "Madgraph", "dat") + def __init__(self, procinfo): + super().__init__(procinfo, "Madgraph", "dat") self.version = "x.y.z" @@ -73,7 +73,7 @@ def fill_datacard(self): # now add the particles checking for overlap with ProcDB self.prepareParticles() # temporary fix: increase LHE event size - self.addOption2GeneratorDatacard("set nevents", int(self.procinfo.get("events")*1.002)) + self.addOption2GeneratorDatacard("set nevents", int(self.procinfo.settings.get_nevents()*1.002)) if self.procinfo.get("isrmode"): if self.procinfo.get("beamstrahlung") is not None: # if self.gen_settings is None: @@ -93,7 +93,7 @@ def fill_datacard(self): for key in self.procDB.getDict(): self.addOption2GeneratorDatacard(key, self.procDB.getDict()[key]) - # if self.settings.get_block("selectors"): + # if self.procinfo.settings.get_block("selectors"): self.writeAllSelectors() # else: # self.add_default_Selectors() @@ -236,12 +236,12 @@ def fill_key4hepScript(self): def fill_PythiaCMND(self): # append the analysis to the content # for the errors allow 1 permil failures - allowedErrors = int(self.procinfo.get("events")*0.001) + allowedErrors = int(self.procinfo.settings.get_nevents()*0.001) content = f"Main:timesAllowErrors = {allowedErrors}\n" content += "Check:epTolErr = 0.01\n" content += "Main:WriteHepMC = on\n" content += "Beams:frameType = 4\n" - content += "Main:numberOfEvents = {0}\n".format(self.procinfo.get("events")) + content += "Main:numberOfEvents = {0}\n".format(self.procinfo.settings.get_nevents()) self.add2OptionalFile(content) def getModelName(self, model): diff --git a/python/Generators/Pythia.py b/python/Generators/Pythia.py index 7be78dea..e8353c65 100644 --- a/python/Generators/Pythia.py +++ b/python/Generators/Pythia.py @@ -3,15 +3,15 @@ class Pythia(GeneratorBase): """Pythia class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "Pythia", "dat") + def __init__(self, procinfo): + super().__init__(procinfo, "Pythia", "dat") self.version = "x.y.z" self.executable = "pythiaRunner -f" self.setOptionalFileNameAndExtension(self.GeneratorDatacardBase,"selectors") - if settings.get_block("selectors"): + if self.procinfo.settings.get_block("selectors"): self.writeAllSelectors() def setSelectorsDict(self): @@ -72,7 +72,7 @@ def fill_run(self): else: self.addOption2GeneratorDatacard("PartonLevel:FSR", "off") - self.addOption2GeneratorDatacard("Main:numberOfEvents", self.procinfo.get("events")) + self.addOption2GeneratorDatacard("Main:numberOfEvents", self.procinfo.settings.get_nevents()) self.add2GeneratorDatacard("\n") # now add the model parameters diff --git a/python/Generators/Sherpa.py b/python/Generators/Sherpa.py index 6be3b42c..1bb9e58e 100644 --- a/python/Generators/Sherpa.py +++ b/python/Generators/Sherpa.py @@ -3,8 +3,8 @@ class Sherpa(GeneratorBase): """Sherpa class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "Sherpa", "dat") + def __init__(self, procinfo): + super().__init__(procinfo, "Sherpa", "dat") self.version = "3" self.executable = "Sherpa -f" @@ -53,7 +53,7 @@ def write_run(self): self.addOption2GeneratorDatacard("YFS_MODE", "FULL") else: self.addOption2GeneratorDatacard("YFS_MODE", "None") - self.addOption2GeneratorDatacard("EVENTS", self.procinfo.get("events")) + self.addOption2GeneratorDatacard("EVENTS", self.procinfo.settings.get_nevents()) self.add2GeneratorDatacard("\n") # now add the model checking for overlap @@ -67,7 +67,7 @@ def write_run(self): for key in self.procDB.getDictRun(): self.addOption2GeneratorDatacard(key,self.procDB.getDictRun()[key]) - self.addOption2GeneratorDatacard("EVENT_GENERATION_MODE", self.procinfo.eventmode) + self.addOption2GeneratorDatacard("EVENT_GENERATION_MODE", self.procinfo.get("eventmode")) if self.gen_settings is not None: if "run" in self.gen_settings.keys(): for key, value in self.gen_settings["run"].items(): @@ -159,7 +159,7 @@ def fill_datacard(self): if self.procinfo.get("decay"): self.write_decay() # writing selectors depends on the presence of the block - if self.settings.get_block("selectors"): + if self.procinfo.settings.get_block("selectors"): self.add2GeneratorDatacard("\nSELECTORS:\n") self.writeAllSelectors() diff --git a/python/Generators/Whizard.py b/python/Generators/Whizard.py index 7c835012..d241af66 100644 --- a/python/Generators/Whizard.py +++ b/python/Generators/Whizard.py @@ -3,8 +3,8 @@ class Whizard(GeneratorBase): """Whizard class""" - def __init__(self, procinfo, settings): - super().__init__(procinfo, settings, "Whizard", "sin") + def __init__(self, procinfo): + super().__init__(procinfo, "Whizard", "sin") self.version = "x.y.z" @@ -74,7 +74,7 @@ def write_process(self): self.add2GeneratorDatacard(f"process proc = {self.whiz_beam1}, {self.whiz_beam2} => {self.finalstate}\n") - self.addOption2GeneratorDatacard("n_events", self.procinfo.get("events")) + self.addOption2GeneratorDatacard("n_events", self.procinfo.settings.get_nevents()) self.addOption2GeneratorDatacard("sqrts", self.procinfo.get("sqrts")) if self.procinfo.get("decay"): self.add_decay() @@ -95,12 +95,12 @@ def write_process(self): for key in self.procDB.getDict(): self.addOption2GeneratorDatacard(key,self.procDB.getDict()[key]) - if self.procinfo.eventmode == "unweighted": + if self.procinfo.get("eventmode") == "unweighted": self.addOption2GeneratorDatacard("?unweighted", "true") else: self.addOption2GeneratorDatacard("?unweighted", "false") - if self.settings.get_block("selectors"): + if self.procinfo.settings.get_block("selectors"): self.CutKeyWdPresent = False self.aCutIsPresent = False self.writeAllSelectors() diff --git a/python/Parameters.py b/python/Parameters.py index b49da4fa..648ab112 100644 --- a/python/Parameters.py +++ b/python/Parameters.py @@ -2,10 +2,10 @@ class Parameter: - require_args = ["name", "value", "isParticleProperty", "texname"] + require_args = ["name", "value", "isParticleProperty"] - def __init__(self, name, value, isParticleProperty, texname): - args = (name, value, isParticleProperty, texname) + def __init__(self, name, value, isParticleProperty): + args = (name, value, isParticleProperty) for i, prop in enumerate(self.require_args): setattr(self, prop, args[i]) # keep a global list of parameters @@ -38,169 +38,145 @@ def updateList(name): alphaEMMZM1 = Parameter( name="alphaEMMZM1", value=127.9, - isParticleProperty=False, - texname="\\text{aEWM1}" + isParticleProperty=False ) alphaEMMZ = Parameter( name="alphaEMMZ", value=1/alphaEMMZM1.value, - isParticleProperty=False, - texname="\\alpha _{\\text{EW}}" + isParticleProperty=False ) alphaEMLOM1 = Parameter( name="alphaEMLOM1", value=1.32184e+02, - isParticleProperty=False, - texname="\\text{aEWLOM1}" + isParticleProperty=False ) alphaEMLO = Parameter( name="alphaEMLO", value=1/alphaEMLOM1.value, - isParticleProperty=False, - texname="\\alpha _{\\text{aEW LO}}" + isParticleProperty=False ) alphaEMM1 = Parameter( name="alphaEMM1", value=137.035999139, - isParticleProperty=False, - texname="\\text{aEW,Q=0,M1}" + isParticleProperty=False ) alphaEM = Parameter( name="alphaEM", value=1/alphaEMM1.value, - isParticleProperty=False, - texname="\\alpha _{\\text{EW,Q=0}}" + isParticleProperty=False ) GFermi = Parameter( name="GFermi", value=0.0000116637, - isParticleProperty=False, - texname="G_f" + isParticleProperty=False ) sin2thetaLO = Parameter( name="sin2thetaLO", value=0.223013, - isParticleProperty=False, - texname="sin ^{2}\\theta" + isParticleProperty=False ) sin2theta = Parameter( name="sin2theta", value=0.23155, - isParticleProperty=False, - texname="sin ^{2}\\theta" + isParticleProperty=False ) sin2thetaEff = Parameter( name="sin2thetaEff", value=0.23155, - isParticleProperty=False, - texname="sin ^{2}\\theta _{Eff}" + isParticleProperty=False ) alphaSMZ = Parameter( name="alphaSMZ", value=0.1184, - isParticleProperty=False, - texname="\\alpha _s" + isParticleProperty=False ) VEV = Parameter( name="VEV", value=246, - isParticleProperty=False, - texname="\\text{vev}" + isParticleProperty=False ) MZ = Parameter( name="MZ", value=91.1876, - isParticleProperty=True, - texname="\\text{MZ}" + isParticleProperty=True ) WZ = Parameter( name="WZ", value=2.4952, - isParticleProperty=True, - texname="\\text{WZ}" + isParticleProperty=True ) MW = Parameter( name="MW", value=80.379, - isParticleProperty=True, - texname="M_W" + isParticleProperty=True ) WW = Parameter( name="WW", value=2.085, - isParticleProperty=True, - texname="\\text{WW}" + isParticleProperty=True ) MB = Parameter( name="MB", value=4.7, - isParticleProperty=True, - texname="\\text{MB}" + isParticleProperty=True ) ymb = Parameter( name="ymb", value=MB.value/VEV.value, - isParticleProperty=True, - texname="\\text{ymb}" + isParticleProperty=True ) MT = Parameter( name="MT", value=172, - isParticleProperty=True, - texname="\\text{MT}" + isParticleProperty=True ) WT = Parameter( name="WT", value=1.50833649, - isParticleProperty=True, - texname="\\text{WT}" + isParticleProperty=True ) ymt = Parameter( name="ymt", value=MT.value/VEV.value, - isParticleProperty=True, - texname="\\text{ymt}" + isParticleProperty=True ) MH = Parameter( name="MH", value=125, - isParticleProperty=True, - texname="\\text{MH}" + isParticleProperty=True ) WH = Parameter( name="WH", value=0.00407, - isParticleProperty=True, - texname="\\text{WH}" + isParticleProperty=True ) MU_R = Parameter( name="MU_R", value=91.188, - isParticleProperty=False, - texname="\\text{\\mu_r}" + isParticleProperty=False ) diff --git a/python/Particles.py b/python/Particles.py index b0728bbb..e3625315 100644 --- a/python/Particles.py +++ b/python/Particles.py @@ -9,15 +9,13 @@ class Particle: "name", "antiname", "mass", - "width", - "texname", - "antitexname", + "width" ] def __init__( - self, pdg_code, name, antiname, mass, width, texname, antitexname, **options + self, pdg_code, name, antiname, mass, width, **options ): - args = (pdg_code, name, antiname, mass, width, texname, antitexname) + args = (pdg_code, name, antiname, mass, width) assert len(self._required_args) == len(args) for i, name in enumerate(self._required_args): @@ -44,8 +42,6 @@ def anti(self): self.name, self.mass, self.width, - self.antitexname, - self.texname, ) @staticmethod @@ -78,200 +74,176 @@ def name_from_pdg(pdg): print(f"{pdg} code not found") -Photon = Particle( - pdg_code=22, name="a", antiname="a", mass=0, width=0, texname="a", antitexname="a" -) - -Z = Particle( - pdg_code=23, - name="Z", - antiname="Z", - mass=Param.MZ.value, - width=Param.WZ.value, - texname="Z", - antitexname="Z", -) - -W__plus__ = Particle( - pdg_code=24, - name="W+", - antiname="W-", - spin=3, - color=1, - mass=Param.MW.value, - width=Param.WW.value, - texname="W+", - antitexname="W-", -) -W__minus__ = W__plus__.anti() - - -g = Particle( - pdg_code=21, - name="g", - antiname="g", - mass=0, - width=0, - texname="g", - antitexname="g", -) - -ve = Particle( - pdg_code=12, - name="ve", - antiname="ve~", - spin=2, - color=1, - mass=0, - width=0, - texname="ve", - antitexname="ve~", -) - -ve__tilde__ = ve.anti() - -vm = Particle( - pdg_code=14, - name="vm", - antiname="vm~", - mass=0, - width=0, - texname="vm", - antitexname="vm~", -) - -vm__tilde__ = vm.anti() - -vt = Particle( - pdg_code=16, - name="vt", - antiname="vt~", - mass=0, - width=0, - texname="vt", - antitexname="vt~", -) - -vt__tilde__ = vt.anti() - -e__minus__ = Particle( - pdg_code=11, - name="e-", - antiname="e+", - mass=0.0, - width=0, - texname="e-", - antitexname="e+", -) - -e__plus__ = e__minus__.anti() - -mu__minus__ = Particle( - pdg_code=13, - name="mu-", - antiname="mu+", - mass=0, - width=0, - texname="mu-", - antitexname="mu+", -) - -mu__plus__ = mu__minus__.anti() - -ta__minus__ = Particle( - pdg_code=15, - name="ta-", - antiname="ta+", - mass=0, - width=0, - texname="ta-", - antitexname="ta+", -) - -ta__plus__ = ta__minus__.anti() - -u = Particle( - pdg_code=2, - name="u", - antiname="u~", - mass=0, - width=0, - texname="u", - antitexname="u~", -) - -u__tilde__ = u.anti() - -c = Particle( - pdg_code=4, - name="c", - antiname="c~", - mass=0, - width=0, - texname="c", - antitexname="c~", -) - -c__tilde__ = c.anti() - -t = Particle( - pdg_code=6, - name="t", - antiname="t~", - mass=Param.MT.value, - width=Param.WT.value, - texname="t", - antitexname="t~", -) - -t__tilde__ = t.anti() - -d = Particle( - pdg_code=1, - name="d", - antiname="d~", - mass=0, - width=0, - texname="d", - antitexname="d~", -) - -d__tilde__ = d.anti() - -s = Particle( - pdg_code=3, - name="s", - antiname="s~", - spin=2, - color=3, - mass=0, - width=0, - texname="s", - antitexname="s~", -) - -s__tilde__ = s.anti() - -b = Particle( - pdg_code=5, - name="b", - antiname="b~", - spin=2, - color=3, - mass=Param.MB.value, - width=0, - texname="b", - antitexname="b~", -) - -b__tilde__ = b.anti() - -H = Particle( - pdg_code=25, - name="H", - antiname="H", - mass=Param.MH.value, - width=Param.WH.value, - texname="H", - antitexname="H", -) +class ParticleCollection: + """The default Particles are instantiated""" + + def __init__(self): + + globals()['Photon'] = Particle( + pdg_code=22, + name="a", + antiname="a", + mass=0, + width=0 + ) + + globals()['Z'] = Particle( + pdg_code=23, + name="Z", + antiname="Z", + mass=Param.MZ.value, + width=Param.WZ.value, + ) + + globals()['W__plus__'] = Particle( + pdg_code=24, + name="W+", + antiname="W-", + spin=3, + color=1, + mass=Param.MW.value, + width=Param.WW.value, + ) + globals()['W__minus__'] = W__plus__.anti() + + globals()['g'] = Particle( + pdg_code=21, + name="g", + antiname="g", + mass=0, + width=0, + ) + + globals()['ve'] = Particle( + pdg_code=12, + name="ve", + antiname="ve~", + spin=2, + color=1, + mass=0, + width=0, + ) + + globals()['ve__tilde__'] = ve.anti() + + globals()['vm'] = Particle( + pdg_code=14, + name="vm", + antiname="vm~", + mass=0, + width=0, + ) + + globals()['vm__tilde__'] = vm.anti() + + globals()['vt'] = Particle( + pdg_code=16, + name="vt", + antiname="vt~", + mass=0, + width=0, + ) + + globals()['vt__tilde__'] = vt.anti() + + globals()['e__minus__'] = Particle( + pdg_code=11, + name="e-", + antiname="e+", + mass=0.0, + width=0, + ) + + globals()['e__plus__'] = e__minus__.anti() + + globals()['mu__minus__'] = Particle( + pdg_code=13, + name="mu-", + antiname="mu+", + mass=0, + width=0, + ) + + globals()['mu__plus__'] = mu__minus__.anti() + + globals()['ta__minus__'] = Particle( + pdg_code=15, + name="ta-", + antiname="ta+", + mass=0, + width=0, + ) + + globals()['ta__plus__'] = ta__minus__.anti() + + globals()['u'] = Particle( + pdg_code=2, + name="u", + antiname="u~", + mass=0, + width=0, + ) + + globals()['u__tilde__'] = u.anti() + + globals()['c'] = Particle( + pdg_code=4, + name="c", + antiname="c~", + mass=0, + width=0, + ) + + globals()['c__tilde__'] = c.anti() + + globals()['t'] = Particle( + pdg_code=6, + name="t", + antiname="t~", + mass=Param.MT.value, + width=Param.WT.value, + ) + + globals()['t__tilde__'] = t.anti() + + globals()['d'] = Particle( + pdg_code=1, + name="d", + antiname="d~", + mass=0, + width=0, + ) + + globals()['d__tilde__'] = d.anti() + + globals()['s'] = Particle( + pdg_code=3, + name="s", + antiname="s~", + spin=2, + color=3, + mass=0, + width=0, + ) + + globals()['s__tilde__'] = s.anti() + + globals()['b'] = Particle( + pdg_code=5, + name="b", + antiname="b~", + spin=2, + color=3, + mass=Param.MB.value, + width=0, + ) + + globals()['b__tilde__'] = b.anti() + + globals()['H'] = Particle( + pdg_code=25, + name="H", + antiname="H", + mass=Param.MH.value, + width=Param.WH.value, + ) diff --git a/python/Process.py b/python/Process.py index cbd2b691..e1d42b91 100644 --- a/python/Process.py +++ b/python/Process.py @@ -1,3 +1,4 @@ +import copy from Particles import Particle from Generators.CirceHelper import CirceHelper @@ -17,7 +18,7 @@ class Process: "beamstrahlung", ] - def __init__(self, args, procname, params, particleData, **options): + def __init__(self, procname, process, inputFileRead, particleData, **options): # list of particles filled from the input yaml file self._inputParticlesList = [] if particleData is not None: @@ -27,22 +28,24 @@ def __init__(self, args, procname, params, particleData, **options): # all particles in process list self._particlesOfProcessList = [] # label to be used in the generatorDB - self.generatorDBLabel = "" self.generatorDBTag = [] + + # process identifier self.procname = procname for arg in self._required_args: - setattr(self, arg, params.settings.get(arg)) - for setting in dir(params): - if not setting.startswith("__"): - setattr(self, setting, getattr(params, setting)) + setattr(self, arg, inputFileRead.get(arg)) for option, value in options.items(): setattr(self, option, value) - for key, value in args.items(): + for key, value in process.items(): setattr(self, key, value) + # inputReader as deep copy without the process stuff + self.settings = copy.deepcopy(inputFileRead) + delattr(self.settings, "processes") + def prepareProcess(self): # beam particles self._beam1 = Particle.get_info(self.initial[0]) @@ -63,7 +66,7 @@ def prepareProcess(self): # now the new DBTag: initialstate = [self.initial[0], self.initial[1]] initialstate.sort() - finalstate = self.final + finalstate = [x for x in self.final] finalstate.sort() self._DBTag = [initialstate, finalstate] @@ -88,7 +91,10 @@ def get(self, name): try: return getattr(self, name) except: - return None + try: + return self.settings.get(name) + except: + return None def get_args(self): return self._required_args @@ -109,16 +115,16 @@ def get_nlo(self): return self.get("nlo") def get_output_format(self): - return self.output_format + return self.settings.get_output_format() def get_PythiaTune(self): - return self.PythiaTune + return self.settings.get_PythiaTune() def get_PolarisationDensity(self): - return self.PolarisationDensity + return self.settings.get_PolarisationDensity() def get_PolarisationFraction(self): - return self.PolarisationFraction + return self.settings.get_PolarisationFraction() def get_rndmSeed(self): return self.get("randomseed") @@ -128,8 +134,8 @@ def get_BeamstrahlungFile(self): circe = CirceHelper(self.beamstrahlung, self.sqrts) return circe.getFile() - def get_generatorDBLabel(self): - return self.generatorDBLabel + def get_DBTag(self): + return self._DBTag def get_DBTag(self): return self._DBTag @@ -139,16 +145,3 @@ def print_info(self): print("Particles are defined with the following parameters") for part in self._particlesOfProcessList: part.print_info() - - -class ProcessParameters: - def __init__(self, settings): - self.settings = settings - self.model = settings.get_model() - self.events = settings.get_event_number() - self.output_format = settings.get_output_format() - self.PythiaTune = settings.get_PythiaTune() - self.PolarisationDensity = settings.get_PolarisationDensity() - self.PolarisationFraction = settings.get_PolarisationFraction() - self.eventmode = settings.get_weighted_mode() - self.ewmode = settings.get_ew_mode() diff --git a/python/Production.py b/python/Production.py new file mode 100644 index 00000000..cd9ea520 --- /dev/null +++ b/python/Production.py @@ -0,0 +1,301 @@ +from abc import ABC,abstractmethod +from datetime import datetime +import os +import sys +import subprocess +import shutil +import copy +from pathlib import Path +import filecmp +import difflib + +from Yaml2Datacard import Yaml2Datacard + +class ProductionBase(ABC): + """Base class for all k4GeneratorConfig operations""" + def __init__(self, args): + + # consistent processing of names + self._outputDir = os.getcwd()+"/"+args.outputDir + if args.outputDir.startswith('/'): + self._outputDir = args.outputDir + + self._referenceDir = os.getcwd()+"/"+args.refDir + if args.refDir.startswith('/'): + self._referenceDir = args.refDir + + def getGenerators(self, generator): + if generator == "All": + return os.listdir(self._outputDir) + else: + return [generator] + + def getProcesses(self, generator): + return os.listdir(f"{self._outputDir}/{generator}") + + def getFileNames(self, directory): + filenames = os.listdir(directory) + filenames =[name for name in filenames if os.path.isfile(os.path.join(directory,name))] + return filenames + + def makeDirectory(self, dirname): + # Overwrite directory if it exists + try: + if not os.path.exists(dirname): + os.makedirs(dirname) + else: + # copy to a directory with the same name+date added to dir name + shutil.copytree(dirname, dirname+str(datetime.today()).replace(' ','_').replace(':','_').replace('.','_')) + shutil.rmtree(dirname) + os.makedirs(dirname) + except PermissionError: + message = f"k4GeneratorsConfig::ERROR:\n{dirname} cannot be created (full path: {os.path.abspath(dirname)})" + sys.exit(message) + + def process(self, generators): + # where do we start from + cwd = os.getcwd() + + failure = False + for generator in generators: + processes = self.getProcesses(generator) + + for process in processes: + if not self.execute(generator, process): + failure = True + + if failure: + sys.exit("Failed") + # return to the starting point + if (cwd != os.getcwd()): + os.chdir(cwd) + + @abstractmethod + def execute(self, generator, process): + pass + +class makeGeneratorDatacards(ProductionBase): + """Generator Generator Datacards""" + + def __init__(self, args): + super().__init__(args) + + # make the directory for the work, protect against "./" + self.makeDirectory(args.outputDir) + + # the sqrts argument can be a list of sqrts or a list of strings + sqrtsGlobalFileName = str("") + if len(args.sqrts) > 0: + try: + all(float(val) for val in args.sqrts) + # write the values to a file + try: + sqrtsGlobalFileName = f"{self._outputDir}/sqrts.yaml" + sqrtsFile = open(sqrtsGlobalFileName,"x") + sqrtsList = "ecms: [ " + for sqrts in args.sqrts: + sqrtsList += f" {sqrts}," + sqrtsFile.write(sqrtsList.rstrip(",") + "]") + sqrtsFile.close() + except FileExistsError as e: + sys.exit(f"Command line specification --sqrts with a {args.sqrts} triggers writing of {sqrtsGlobalFileName}, but file exists") + except ValueError as e: + if len(args.sqrts) == 1: + if Path(args.sqrts[0]).stem+Path(args.sqrts[0]).suffix == "sqrts.yaml" and os.path.isfile(args.sqrts[0]): + sqrtsGlobalFileName = os.path.abspath(args.sqrts[0]) + + # specific members for DC creation + self._yamlFiles = [] + self._sqrtsFiles = [] + + # prepare yamls: + self.prepareYamls(args.yaml); + + # prepare the Sqrts files if global is not set + if not sqrtsGlobalFileName: + self.prepareSQRTS(args.sqrts); + + # args to transfer: avoid overwrite + self.Yaml2DatacardArgs = copy.deepcopy(args) + + # run all yamls: + self.run(sqrtsGlobalFileName); + + def execute(self, generator, process): + pass + + def prepareYamls(self, yamlList): + # check whether this is a list of directories or mixed or files + yamlDirs = [] + for yaml in yamlList: + if os.path.isfile(yaml) and yaml.endswith('.yaml') and not Path(yaml).stem.startswith('sqrts'): + self._yamlFiles.append(os.path.abspath(yaml)) + elif os.path.isdir(yaml): + yamlDirs.append(os.path.abspath(yaml)) + + # now we have a list of directories and a list of yaml files, extend the list of yaml files in the directories + for yamlDir in yamlDirs: + self._yamlFiles.extend([f"{yamlDir}/{filename}" + for filename in os.listdir(yamlDir) + if filename.endswith('.yaml') and not filename.startswith('sqrts')]) + + def prepareSQRTS(self, sqrtsList): + # check whether this is a list of directories or mixed or files + sqrtsDirs = [] + for sqrts in sqrtsList: + if os.path.isfile(sqrts) and Path(sqrts).stem.startswith('sqrts') and sqrts.endswith('.yaml'): + self._sqrtsFiles.append(os.path.abspath(sqrts)) + elif os.path.isdir(sqrts): + sqrtsDirs.append(os.path.abspath(sqrts)) + + # now we have a list of directories and a list of sqrts files, extend the list of sqrts files in the directories + for sqrtsDir in sqrtsDirs: + self._sqrtsFiles.extend([f"{sqrtsDir}/{filename}" + for filename in os.listdir(sqrtsDir) + if filename.startswith('sqrts') and filename.endswith('.yaml')]) + + def run(self, sqrtsGlobal): + # remember where we start from + cwd = os.getcwd() + # go to the working directory + os.chdir(self._outputDir) + # loop over all files + for filename in self._yamlFiles: + + # check SQRTS: priority: global then comparison with filenames for specific processes + processName = Path(filename).stem + sqrtsName = f"{Path(filename).parent}/sqrts{processName}.yaml" + if sqrtsGlobal: + sqrtsName = sqrtsGlobal + else: + if not any( name == sqrtsName for name in self._sqrtsFiles): + sqrtsName = "" + # everything is prepared, we can run now + self.Yaml2DatacardArgs.yaml = filename + message = f"Processing : {processName} from file {self.Yaml2DatacardArgs.yaml}" + if sqrtsName and os.path.isfile(sqrtsName): + self.Yaml2DatacardArgs.sqrts = sqrtsName + message += f" with {self.Yaml2DatacardArgs.sqrts}" + else: + self.Yaml2DatacardArgs.sqrts = "" + print(message) + Yaml2Datacard(self.Yaml2DatacardArgs).processFile() + # return to the starting point + os.chdir(cwd) + +class checkGeneratorDatacards(ProductionBase): + """Check Generator Datacards""" + + def __init__(self, args): + super().__init__(args) + + # retrieve all generators in the workdirector: + generators = self.getGenerators(args.generator) + + # now compare to reference + self.process(generators) + + def execute(self, generator, process): + genProc = f"{generator}/{process}" + newDir = f"{self._outputDir}/{genProc}" + refDir = f"{self._referenceDir}/{genProc}" + fileNames = self.getFileNames(newDir) + + success = True + # check that all files were created (new) and are available for comparison (ref) + if len(fileNames) != len(self.getFileNames(refDir)): + print(f"Number of files for Generator {generator} process {process} differ between reference {len(fileNames)} and creation {len(self.getFileNames(refDir))}") + success = False + + for name in fileNames: + filenameGenerated = f"{newDir}/{name}" + # new file must exist + if not os.path.isfile(filenameGenerated): + print(f"File {filenameGenerated} not found") + success = False + continue + + filenameRef = f"{refDir}/{name}" + # reference file must exist + if not os.path.isfile(filenameRef): + print(f"File {filenameRef} not found") + continue + success = False + + # both files exist, so we can compare + message = f"Generator {generator} Process {process} File {name}" + if filecmp.cmp(filenameRef, filenameGenerated, shallow=False): + print(f"OK: {message}") + else: + print(f"NOTOK: {message}") + # if that's the case make a detailed comparison + fileRef = open(filenameGenerated, "r").readlines() + fileGenerated = open(filenameRef, "r").readlines() + print("".join(line for line in difflib.Differ().compare(fileRef,fileGenerated) + if line[0] == "-" or line[0] == "+")) + success = False + + return success + +class generate(ProductionBase): + """Run Event Generation""" + + def __init__(self, args): + super().__init__(args) + + # retrieve all generators in the workdirector: + generators = self.getGenerators(args.generator) + + # now compare to reference + self.process(generators) + + def execute(self, generator, process): + # go to the directory + genProcDir = f"{self._outputDir}/{generator}/{process}" + os.chdir(genProcDir) + # retrieve the script + scripts = [script for script in os.listdir(genProcDir) if script.startswith('Run_') and script.endswith('.sh')] + if len(scripts) != 1: + print(f"Found more than one script for generator {generator} with process {process}") + return False + # execute + success = True + for script in scripts: + try: + result = subprocess.run(f"./{script}", capture_output=True, check=True) + except subprocess.CalledProcessError as e: + print(f"Execution error for {generator} in process {process}") + print(e.returncode) + print(e.output) + success = False + + return success + +class summary(ProductionBase): + """Make a summary of all processes""" + + def __init__(self, args): + super().__init__(args) + + print("Extracting the cross sections by reading EDM4HEP files and superposing the differential distributions") + # remember where we start from + cwd = os.getcwd() + os.chdir(self._outputDir) + try: + result = subprocess.run(["eventGenerationSummary", + "-w", f"{self._outputDir}", + "-f", f"{self._outputDir}/GenerationSummary.dat", + "-d", f"{self._outputDir}"], + capture_output=True, check=True) + except subprocess.CalledProcessError as e: + print(f"Execution error eventGenerationSummary") + print(e.returncode) + print(e.output.decode("utf-8")) + sys.exit("Exception thrown by eventGenerationSummary") + + # return tu the starting point + if (cwd != os.getcwd()): + os.chdir(cwd) + + def execute(self, generator, process): + pass diff --git a/python/Yaml2Datacard.py b/python/Yaml2Datacard.py new file mode 100644 index 00000000..134c5ef4 --- /dev/null +++ b/python/Yaml2Datacard.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse +import textwrap +import copy +from datetime import datetime + +import ReleaseSpecs +from ReleaseSpecs import ReleaseSpec +import YamlInputReader as Reader +from Process import Process +from Generators import Generators +from Particles import ParticleCollection + +class Yaml2Datacard: + """Convert Input files into generator datacards""" + + def __init__(self,args): + # copy the arguments + self.args = copy.deepcopy(args) + + ReleaseSpec.set_info("key4hepUseNightlies",self.args.key4hepUseNightlies) + if ReleaseSpecs.key4hepUseNightlies.value: + print(f"key4HEP configuration: NIGHTLIES") + else: + print(f"key4HEP configuration: RELEASE") + + # make sure it's a valid date + if self.args.key4hepVersion is not None: + try: + relDate = datetime.strptime(self.args.key4hepVersion,'%Y-%m-%d') + if (datetime.today() - relDate).days < 0: + raise ValueError() + print(f"key4HEP configuration date: {self.args.key4hepVersion}") + except ValueError: + print(f"Invalid KEY4HEP release argument, YYYY-MM-DD expected, latest possible date {datetime.today().strftime('%Y-%m-%d')}") + print(f"Requested: {self.args.key4hepVersion}") + print("Cannot configure scripts correctly, exiting") + exit() + else: + print(f"key4HEP configuration date: latest") + # store for future use: + ReleaseSpec.set_info("key4hepReleaseDate",self.args.key4hepVersion) + + # sqrts choices from file + self.energies = [0.] + if self.args.sqrts: + sqrtsReader = Reader.SQRTSReader(self.args.sqrts) + self.energies = sqrtsReader.energies() + + # now we read the global settings + try: + # make sure that we follow a symlink to the real location of the parametersets should replace that by share? + parameterSet = Reader.ParameterSetReader(self.args.parameterTagFile, self.args.parameterTag) + except FileNotFoundError as e: + exit(f"ERROR: File {e} with parameters for tag {self.args.parameterTag} not found"f"ERROR: File {e} with parameters for tag {self.args.parameterTag} not found") + + def processFile(self): + # execute file processes + rndmSeed = self.args.seed + for sqrts in self.energies: + # remember where we started from: + cwd = os.getcwd() + # first step reset all particles: + ParticleCollection() + # read the input file + self.processReader = Reader.ProcessReader(self.args.yaml, sqrts) + # set the number of events if present + if self.args.nevts != -1: + self.processReader.set("events", events) + # the datacard outputDir may differ + self.processOutputDir() + # extract information + processes = self.processReader.get_processes(sqrts) + yamlParticleData = self.processReader.get_particle_data() + generators = Generators(self.processReader) + # + for key, value in processes.items(): + + # make the combination of generator directories and in each generator director the process directory + self.makeDirectories4GeneratorsProcess(self.processReader.get_generators(), key) + + # the command line seed supersedes the seed read from file + if self.args.seedOverride: + value["randomseed"] = rndmSeed + else: + # check for seed in the file/process + try: + randomseed = value["randomseed"] + except: + # random seed not present, fall to external default setting and increment for next round + value["randomseed"] = rndmSeed + rndmSeed += 1 + # instantiate the class for each process + process = Process( + key, value, self.processReader, yamlParticleData, OutDir=self.outputDir + ) + process.prepareProcess() + generators.runGeneratorConfiguration(process) + + # increment for next sqrts + rndmSeed += 1 + + # at the end back to the starting point dir for the next file + if (cwd != os.getcwd()): + os.chdir(cwd) + + def processOutputDir(self): + # the output directory + self.outputDir = getattr(self.processReader, "outdir", self.args.outputDir) + if self.args.outputDirOverride: + self.outputDir = self.args.outputDir + # the attribute always has to be reset to be sure.... + setattr(self.processReader, "outdir", self.args.outputDir) + # all the preparatory work has been done in self.args.outputDir + # create the new directory if it does not exist + if not self.args.outputDirOverride: + try: + if not os.path.exists(self.outputDir): + os.makedirs(self.outputDir) + except PermissionError: + message = f"Yaml2Datacard::processOutputDir::ERROR:\n{self.outputDir} cannot be created (full path: {os.path.abspath(self.outputDir)})" + sys.exit(message) + os.chdir(self.outputDir) + + def makeDirectories4GeneratorsProcess(self, generators, procname): + # do not overwrite directory if it exists + for generator in generators: + process_directory = os.path.join(generator, procname) + if not os.path.exists(process_directory): + try: + os.makedirs(process_directory) + except PermissionError: + message = f"Yaml2Datacard::makeDirectories4GeneratorsProcess::ERROR:\n{process_directory} cannot be created (full path: {os.path.abspath(process_directory)})" + sys.exit(message) + +if __name__ == "__main__": + Yaml2Datacard() diff --git a/python/Input.py b/python/YamlInputReader.py similarity index 98% rename from python/Input.py rename to python/YamlInputReader.py index 657a3bd4..555f1198 100644 --- a/python/Input.py +++ b/python/YamlInputReader.py @@ -4,7 +4,7 @@ import Parameters as ParameterModule from Parameters import Parameter as ParameterClass -class Input: +class ProcessReader: """Class for loading YAML files""" def __init__(self, file, sqrts): @@ -59,7 +59,7 @@ def get_subblock(self, k1, k2): except: return None - def gens(self): + def get_generators(self): if not self.is_set("generators"): raise ValueError("No Generators set!") return getattr(self, "generators") @@ -128,7 +128,7 @@ def get_particle_data(self): def get_model(self): return self.get("model", "SM") - def get_event_number(self): + def get_nevents(self): return self.get("events", 0) def get_isr_mode(self): @@ -217,7 +217,7 @@ def key4HEPAnalysisON(self): if self.anatools is not None: return "key4hep" in self.anatools -class ECMSInput: +class SQRTSReader: """Class for loading YAML files with center of mass energies""" def __init__(self, file): @@ -238,7 +238,7 @@ def energies(self): ecmsList.extend(value) return ecmsList -class ParameterSets: +class ParameterSetReader: """Class for loading YAML files with the parameter settings""" def __init__(self, file, tag): diff --git a/python/k4GeneratorsConfig.py b/python/k4GeneratorsConfig.py new file mode 100644 index 00000000..a22cea88 --- /dev/null +++ b/python/k4GeneratorsConfig.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 + +import os +import shutil +import sys +import argparse +import textwrap +import copy + +from Production import makeGeneratorDatacards +from Production import checkGeneratorDatacards +from Production import generate +from Production import summary + +class k4GeneratorsConfig(): + def __init__(self,arguments=None): + # define all command line arguments + parser = argparse.ArgumentParser(prog="k4GeneratorsConfig") + + parser.add_argument( + "--make", + action='store_true', + help="make the generator datacards from the yaml files" + ) + + self.yamlDefault = [os.path.dirname(os.path.realpath(__file__))+'/../examples'] + parser.add_argument( + "--yaml", + nargs="*", + type=str, + default=argparse.SUPPRESS, + help="yamlFiles and director(y/ies) with yaml files (default: k4GeneratorsConfig/examples)" + ) + + self.sqrtsDefault = [] + parser.add_argument( + "--sqrts", + nargs="*", + type=str, + default=argparse.SUPPRESS, + help="either a space separated list of center of mass energies OR file(s) and director(y/ies) with sqrts lists in yaml format (name : sqrtsPROCESS.dat containing eg, sqrts:[91.,240.]), sqrts.yaml as single argument will be applied to all processes", + ) + + self.seedDefault = 4711 + parser.add_argument( + "--seed", + type=int, + default=argparse.SUPPRESS, + help="If specified overrides the random number seed in the yamlFile, incremented for each process and each sqrts of a yaml file, default: yaml file", + ) + + self.nevtsDefault = -1 + parser.add_argument( + "--nevts", + type=int, + default=argparse.SUPPRESS, + help="If specified overrides the number of events to be generated in the yamlFile, default: use yaml", + ) + + self.parameterTagDefault = "latest" + parser.add_argument( + "--parameterTag", + type=str, + default=argparse.SUPPRESS, + help="parameter tag in Parameters.yaml (default: latest)", + ) + + self.parameterTagFileDefault = os.path.dirname(os.path.realpath(__file__))+'/ParameterSets.yaml' + parser.add_argument( + "--parameterTagFile", + type=str, + default=argparse.SUPPRESS, + help="name of file containing the parameter sets of the requested parameterTag, default: ParameterSets.yaml in directory: k4GeneratorsConfig/python", + ) + + parser.add_argument( + "--key4hepUseNightlies", + action='store_true', + help="configures the key4hepscripts to use nightlies instead of releases", + ) + + self.key4hepVersionDefault = None + parser.add_argument( + "--key4hepVersion", + default=argparse.SUPPRESS, + help="force the use of the version : YYYY-MM-DD (default: latest)", + ) + parser.add_argument( + "--check", + action='store_true', + help="check the generator datacards with respect to the reference" + ) + parser.add_argument( + "--refDir", + type=str, + default=os.path.dirname(os.path.realpath(__file__))+'/../test/ref-results', + help="path to the reference files (default: k4GeneratorsConfig/test/ref-results)" + ) + parser.add_argument( + "--generator", + type=str, + default="All", + help="generator to be run (default: All processed)" + ) + parser.add_argument( + "--generate", + action='store_true', + help="run the event generation" + ) + parser.add_argument( + "--summary", + action='store_true', + help="compare the results of the event generation process by process and produce summary output in outputDir" + ) + + self.outputDirDefault = "Run-Cards" + parser.add_argument( + "--outputDir", + type=str, + default=argparse.SUPPRESS, + help=f"path to output directory (default: ./{self.outputDirDefault}, if specified, overrides the outdir key in yaml)" + ) + parser.add_argument( + "--all", + action='store_true', + help="activates --make --generate --summary" + ) + + # decode the arguments + args = parser.parse_args(arguments) + # check the arguments + self.processArguments(args) + # make the GeneratorDatacards + if self.args.make: + makeGeneratorDatacards(self.args) + # compare to the reference + if self.args.check: + checkGeneratorDatacards(self.args) + # run the event generation + if self.args.generate: + generate(self.args) + # produce the summary + if self.args.summary: + summary(self.args) + + def processArguments(self, args): + # make a deep copy of the arguments: + self.args = copy.deepcopy(args) + # OPTION: outputDir + # differentiate between option being given or not + try: + check = args.outputDir + self.args.outputDirOverride = True + except AttributeError: + # argument was not given, set the default and make it known: + self.args.outputDir = self.outputDirDefault + self.args.outputDirOverride = False + + # --all overrides --make --generate --summary + if args.all: + self.args.make = True + self.args.generate = True + self.args.summary = True + print("k4GeneratorsConfig will make generator datacards, generate events, make a summary") + + # outputDir should never be cwd + if ( os.path.abspath(self.args.outputDir) == os.path.abspath(os.getcwd()) ): + message = f"k4GeneratorsConfig::ERROR --outputDir {self.args.outputDir} not allowed \nPlease specify a directory other than the working directory" + sys.exit(message) + + # OPTION all or make&&generate&&summary + if ( args.all or + (args.make and args.check) or + (args.make and args.generate) or + (args.make and args.summary) ): + if not self.args.outputDirOverride: + message = f"k4GeneratorsConfig::ERROR\n" + message += f"--make and (--check and/or --generate and/or --summary) requested\n" + message += f"yamlFiles may define multiple outputDirectory, functionality not foreseen\n" + message += f"BUT: --outputDir {self.args.outputDir} not defined \nPlease define a common output directory" + sys.exit(message) + + # make sure that Warnings are issued when options specific to --make are invoked, make is false + message = "k4GeneratorsConfig::WARNING invoking option specific to --make or --all:" + try: + check = self.args.yaml + if not args.make and not args.all: + print(f"{message} --yaml {self.args.yaml} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.yaml = self.yamlDefault + try: + check = self.args.sqrts + if not args.make and not args.all: + print(f"{message} --sqrts {self.args.sqrts} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.sqrts = self.sqrtsDefault + try: + check = self.args.seed + self.args.seedOverride = True + if check <= 0: + sys.exit(f"k4GeneratorsConfig::ERROR --seed {self.args.seed} specified, must be >0") + if not args.make and not args.all: + print(f"{message} --seed {self.args.seed} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.seed = self.seedDefault + self.args.seedOverride = False + try: + check = self.args.nevts + if not args.make and not args.all: + print(f"{message} --nevts {self.args.nevts} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.nevts = self.nevtsDefault + try: + check = self.args.parameterTag + if not args.make and not args.all: + print(f"{message} --parameterTag {self.args.parameterTag} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.parameterTag = self.parameterTagDefault + try: + check = self.args.parameterTagFile + if not args.make and not args.all: + print(f"{message} --parameterTagFile {self.args.parameterTagFile} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.parameterTagFile = self.parameterTagFileDefault + try: + check = self.args.key4hepVersion + if not args.make and not args.all: + print(f"{message} --key4hepVersion {self.args.key4hepVersion} has no effect") + except AttributeError as e: + if args.make or args.all: + self.args.key4hepVersion = self.key4hepVersionDefault + + # special treatment for nightlies storetrue + if not args.make and not args.all and args.key4hepUseNightlies: + print(f"k4GeneratorsConfig::WARNING invoking option specific to --make or --all : --key4hepUseNightlies has no effect") + +if __name__ == "__main__": + k4GeneratorsConfig() + diff --git a/python/main.py b/python/main.py deleted file mode 100755 index 87971530..00000000 --- a/python/main.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import argparse -import textwrap -from datetime import datetime - -import ReleaseSpecs -from ReleaseSpecs import ReleaseSpec -import Input as Settings -import Process as process_module -import Generators as generators_module - -def make_output_directory(generators, output_directory, procname): - # Overwrite directory if it exists - if not os.path.exists(output_directory): - os.makedirs(output_directory) - for generator in generators: - generator_directory = os.path.join(output_directory, generator, procname) - if not os.path.exists(generator_directory): - os.makedirs(generator_directory) - - -def main(): - # parser = argparse.ArgumentParser(prog='k4gen',description='Process input YAML files.') - parser = argparse.ArgumentParser( - prog="k4GeneratorsConfig", - formatter_class=argparse.RawDescriptionHelpFormatter, - description=textwrap.dedent( - """\ -Process input YAML files. -The following options are available: ------------------------------------- -SqrtS : float (center of mass energy) -ISRmode : int (0: off, 1: on) - OutputFormat : string (format output, available are hepmc2 and hepmc3) -OutDir : string (output directory, default=$PWD/Run-Cards) -Events : unsigned int (Number of Monte-Carlo events to be generated) -RandomSeed : unsigned int (specify a random seed, important when generating multiple files for the same process) -Processes : see README A list of processes which runcards should be generated. Each process should have its own unique name - Processes: - Muon: - Initial: [11, -11] - Final: [13, -13] -ParticleData : overwrite basic particle properties - ParticleData: - 25: - mass: 125 - width: 0 - -For MADGRAPH and Whizard only: -PolarisationDensity : int ([-1 or 1, 1 or -1]) default: [-1, 1] -PolarisationFraction : float ([0...1.,0....1.]), default [0,0] -Beamstrahlung : string (name of accelerator: ILC, FCC, CLIC, C3, HALFHF) - """ - ), - ) - parser.add_argument("inputfiles", nargs="+", type=str, help="Input YAML file") - parser.add_argument( - "--ecms", - nargs="*", - type=float, - default=[], - help="energies to be processed, overrides nominal yaml input file settings", - ) - parser.add_argument( - "--ecmsFiles", - nargs="*", - type=str, - default=[], - help="yaml files with energies format ecms: [energy1,....] ", - ) - parser.add_argument( - "--seed", - nargs=1, - type=int, - default=4711, - help="initial random number seed, increment for each file", - ) - parser.add_argument( - "--nevts", - type=int, - default=-1, - help="Number of events to be generated", - ) - parser.add_argument( - "--parameterTag", - type=str, - default="latest", - help="parameter tag in Parameters.yaml default is: latest", - ) - parser.add_argument( - "--parameterTagFile", - type=str, - default="ParameterSets.yaml", - help="name of file containing the parameter sets of the requested parameterTag, default: ParameterSets.yaml in directory: python", - ) - parser.add_argument( - "--key4hepUseNightlies", - action='store_true', - help="configures the key4hepscripts to use nightlies instead of releases", - ) - parser.add_argument( - "--key4hepVersion", - default=None, - help="force the use of the version in default is latest, format: YYYY-MM-DD", - ) - args = parser.parse_args() - files = args.inputfiles - energies = args.ecms - ecmsfiles = args.ecmsFiles - rndmSeed = args.seed - events = args.nevts - paramTag = args.parameterTag - paramFileName = os.path.dirname(os.path.realpath(__file__))+"/"+args.parameterTagFile - releaseDate = args.key4hepVersion - - ReleaseSpec.set_info("key4hepUseNightlies",args.key4hepUseNightlies) - if ReleaseSpecs.key4hepUseNightlies.value: - print(f"key4HEP configuration: NIGHTLIES") - else: - print(f"key4HEP configuration: RELEASE") - - # make sure it's a valid date - if releaseDate is not None: - try: - relDate = datetime.strptime(releaseDate,'%Y-%m-%d') - if (datetime.today() - relDate).days < 0: - raise ValueError() - print(f"key4HEP configuration date: {releaseDate}") - except ValueError: - print(f"Invalid KEY4HEP release argument, YYYY-MM-DD expected, latest possible date {datetime.today().strftime('%Y-%m-%d')}") - print(f"Requested: {releaseDate}") - print("Cannot configure scripts correctly, exiting") - exit() - else: - print(f"key4HEP configuration date: latest") - # store for future use: - ReleaseSpec.set_info("key4hepReleaseDate",releaseDate) - - # so additionally we read the argument ecmsFile - for ecmsfile in ecmsfiles: - # open and read ecms file and append the energies to the command line arguments - ecmSettings = Settings.ECMSInput(ecmsfile) - energies.extend(ecmSettings.energies()) - - # now we read the global settings - try: - # make sure that we follow a symlink to the real location of the parametersets should replace that by share? - parameterSet = Settings.ParameterSets(paramFileName, paramTag) - except FileNotFoundError as e: - print(f"ERROR: File {e} with parameters for tag {paramTag} not found") - exit() - - # now execute file processes - if len(energies) == 0: - executeFiles(files, 0, rndmSeed, events) - else: - for sqrts in energies: - rndmIncrement = executeFiles(files, sqrts, rndmSeed, events) - # offset for next round by number of yaml files - rndmSeed = rndmSeed + rndmIncrement - - -def executeFiles(files, sqrts, rndmSeedFallback=4711, events=-1): - if sqrts == 0: - print("Generating and writing configuration files") - else: - print("Generating and writing configuration files for ECM= ", sqrts) - - for yaml_file in files: - # read the input file - settings = Settings.Input(yaml_file, sqrts) - # set the number of events if present - if events != -1: - settings.set("events", events) - settings.gens() - processes = settings.get_processes(sqrts) - yamlParticleData = settings.get_particle_data() - generators = generators_module.Generators(settings) - try: - output_dir = getattr(settings, "outdir", "Run-Cards") - except KeyError: - # If no directory set in input, use default - output_dir = "Run-Cards" - - process_instances = {} - rndmIncrement = 0 - for key, value in processes.items(): - make_output_directory(settings.gens(), output_dir, key) - try: - randomseed = value["randomseed"] - except: - randomseed = rndmSeedFallback + rndmIncrement - value["randomseed"] = randomseed - rndmIncrement += 1 - param = process_module.ProcessParameters(settings) - # instantiate the class for each process - process_instances[key] = process_module.Process( - value, key, param, yamlParticleData, OutDir=output_dir - ) - # increment counter for randomseed - for process_instance in process_instances.values(): - process_instance.prepareProcess() - generators.runGeneratorConfiguration(process_instance) - - return rndmIncrement - - -if __name__ == "__main__": - main() diff --git a/setup.csh b/setup.csh index 0fa8d0d9..03769624 100755 --- a/setup.csh +++ b/setup.csh @@ -42,4 +42,4 @@ if (! -d "${K4GeneratorsConfigDir}/install/") then endif # Set executable -alias k4GeneratorsConfig "python3 ${K4GeneratorsConfigDir}/python/main.py" +alias k4GeneratorsConfig "python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" diff --git a/setup.sh b/setup.sh index 66ad3368..eff874de 100755 --- a/setup.sh +++ b/setup.sh @@ -29,4 +29,4 @@ if [[ ! -d "${K4GeneratorsConfigDir}/install/" ]]; then fi # Set executable -alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/main.py" +alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" diff --git a/setup.zsh b/setup.zsh index d0805ef5..f76fa8d5 100755 --- a/setup.zsh +++ b/setup.zsh @@ -29,4 +29,4 @@ if [[ ! -d "${K4GeneratorsConfigDir}/install/" ]]; then fi # Set executable -alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/main.py" +alias k4GeneratorsConfig="python3 ${K4GeneratorsConfigDir}/python/k4GeneratorsConfig.py" diff --git a/share/ECFAHiggsFactories/GammaGamma/ecmsGammaGamma.yaml b/share/ECFAHiggsFactories/GammaGamma/sqrtsGammaGamma.yaml similarity index 100% rename from share/ECFAHiggsFactories/GammaGamma/ecmsGammaGamma.yaml rename to share/ECFAHiggsFactories/GammaGamma/sqrtsGammaGamma.yaml diff --git a/share/ECFAHiggsFactories/Hnunu/ecmsHnunu.yaml b/share/ECFAHiggsFactories/Hnunu/sqrtsHnunu.yaml similarity index 100% rename from share/ECFAHiggsFactories/Hnunu/ecmsHnunu.yaml rename to share/ECFAHiggsFactories/Hnunu/sqrtsHnunu.yaml diff --git a/share/ECFAHiggsFactories/WW/ecmsWW.yaml b/share/ECFAHiggsFactories/WW/sqrtsWW.yaml similarity index 100% rename from share/ECFAHiggsFactories/WW/ecmsWW.yaml rename to share/ECFAHiggsFactories/WW/sqrtsWW.yaml diff --git a/share/ECFAHiggsFactories/ZH/ecmsZH.yaml b/share/ECFAHiggsFactories/ZH/sqrtsZH.yaml similarity index 100% rename from share/ECFAHiggsFactories/ZH/ecmsZH.yaml rename to share/ECFAHiggsFactories/ZH/sqrtsZH.yaml diff --git a/share/ECFAHiggsFactories/ZHH/ecmsZHH.yaml b/share/ECFAHiggsFactories/ZHH/sqrtsZHH.yaml similarity index 100% rename from share/ECFAHiggsFactories/ZHH/ecmsZHH.yaml rename to share/ECFAHiggsFactories/ZHH/sqrtsZHH.yaml diff --git a/share/ECFAHiggsFactories/ZHPolarized/ZHPolarized.yaml b/share/ECFAHiggsFactories/ZHPolarized/sqrtsZHPolarized.yaml similarity index 100% rename from share/ECFAHiggsFactories/ZHPolarized/ZHPolarized.yaml rename to share/ECFAHiggsFactories/ZHPolarized/sqrtsZHPolarized.yaml diff --git a/share/ECFAHiggsFactories/ZZ/ecmsZZ.yaml b/share/ECFAHiggsFactories/ZZ/sqrtsZZ.yaml similarity index 100% rename from share/ECFAHiggsFactories/ZZ/ecmsZZ.yaml rename to share/ECFAHiggsFactories/ZZ/sqrtsZZ.yaml diff --git a/share/ECFAHiggsFactories/ffbar/ecmsffbar.yaml b/share/ECFAHiggsFactories/ffbar/sqrtsffbar.yaml similarity index 100% rename from share/ECFAHiggsFactories/ffbar/ecmsffbar.yaml rename to share/ECFAHiggsFactories/ffbar/sqrtsffbar.yaml diff --git a/share/ECFAHiggsFactories/ttbar/ecmsttbar.yaml b/share/ECFAHiggsFactories/ttbar/sqrtsttbar.yaml similarity index 100% rename from share/ECFAHiggsFactories/ttbar/ecmsttbar.yaml rename to share/ECFAHiggsFactories/ttbar/sqrtsttbar.yaml diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 60402c6b..fc69d5cf 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,10 +3,7 @@ set(TEST_ENVIRONMENT "PATH=${PROJECT_BINARY_DIR}/bin:$ENV{PATH}") set(TEST_TIMEOUT 3500) # we split the execution into four steps: -set(RUN_YAMLS "${CMAKE_CURRENT_SOURCE_DIR}/createGeneratorDatacardsFromYaml.sh") -set(CHECK_DATACARDS "${CMAKE_CURRENT_SOURCE_DIR}/checkGeneratorDatacards.sh") -set(RUN_GENERATORS "${CMAKE_CURRENT_SOURCE_DIR}/runEventGeneration.sh") -set(RUN_SUMMARY "${CMAKE_CURRENT_SOURCE_DIR}/runSummary.sh") +set(EXECUTABLE "${PROJECT_SOURCE_DIR}/python/k4GeneratorsConfig.py") # the generators to be run set(generatorList Babayaga KKMC Madgraph Pythia Sherpa Whizard ) @@ -18,7 +15,7 @@ foreach(generator ${generatorList}) endforeach() # add the first test: setting up the yaml files -add_test(NAME createGeneratorDatacards COMMAND ${RUN_YAMLS} ) +add_test(NAME createGeneratorDatacards COMMAND python3 ${EXECUTABLE} --make --outputDir Run-Cards ) set_tests_properties("createGeneratorDatacards" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -28,7 +25,7 @@ set_tests_properties("createGeneratorDatacards" # add the second test: comparing the yaml files to the reference files function(check_generator_datacards generator) - add_test(NAME checkGeneratorDatacards_${generator} COMMAND ${CHECK_DATACARDS} -g ${generator}) + add_test(NAME checkGeneratorDatacards_${generator} COMMAND python3 ${EXECUTABLE} --check --generator ${generator} --outputDir Run-Cards ) set_tests_properties("checkGeneratorDatacards_${generator}" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -45,7 +42,7 @@ endforeach() # third test running the generators (define the function first then call it function(add_generator_run name generator) - add_test(NAME ${name}_${generator} COMMAND ${RUN_GENERATORS} -g ${generator} ) + add_test(NAME ${name}_${generator} COMMAND python3 ${EXECUTABLE} --generate --generator ${generator} --outputDir Run-Cards ) set_tests_properties(${name}_${generator} PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -60,12 +57,10 @@ foreach(generator ${generatorList}) endforeach() # third test (after all others have been run: gather the xsections -add_test(NAME xsectionRuns COMMAND ${RUN_SUMMARY} ) +add_test(NAME xsectionRuns COMMAND python3 ${EXECUTABLE} --summary --outputDir Run-Cards ) set_tests_properties("xsectionRuns" PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ENVIRONMENT ${TEST_ENVIRONMENT} DEPENDS "${RunList}" ) - -#add_test(NAME Finalize COMMAND ${CMAKE_BINARY_DIR}/bin/eventGenerationSummary -f ${CMAKE_SOURCE_DIR}/test/xsectionSummary.dat) diff --git a/test/check.sh b/test/check.sh deleted file mode 100755 index 211eb580..00000000 --- a/test/check.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env bash - -set -e - -shopt -s expand_aliases -source ../setup.sh - -# decode command line options - -OPTSTRING=":bhr" - -runEvgen="true" -runReducedEvgen="false" -while getopts ${OPTSTRING} opt; do - case ${opt} in -# x) -# echo "Option -x was triggered, Argument: ${OPTARG}" -# ;; - b) - echo "Option -b was triggered, event generation step will not be run" - runEvgen="false" - ;; - r) - echo "Option -r was triggered, event generation step will be run only for one process per yaml file" - runReducedEvgen="true" - ;; - h) - echo "Arguments are:" - echo "-h for help" - echo "-b to block the event generation step fully" - echo "-r to reduce the number of event generation steps to 1 per yaml file" - exit 0 - ;; - ?) - echo "Invalid option: -${OPTARG}." - exit 1 - ;; - esac -done - -# STEP 0 prepare the input - -mkdir -p ci-setups - -CWD=${PWD} -REFDIR="${PWD}/ref-results" -EXAMPLEDIR="${PWD}/../examples" - -cp "$EXAMPLEDIR"/*yaml ci-setups -cd ci-setups - -function checkFile() { - local generator="$1" - local refgenerator="$(basename "$generator")" - local procname="$2" - local outFile="$3" - if [[ -e "$REFDIR/$refgenerator/$procname/$outFile" ]]; then - if diff "$REFDIR/$refgenerator/$procname/$outFile" "$PWD/$generator/$procname/$outFile" &> /dev/null; then - echo "Process " $procname : "Files are identical for file" $outFile - else - echo "Process " $procname "Files are different for file" $outFile - diff "$REFDIR/$refgenerator/$procname/$outFile" "$PWD/$generator/$procname/$outFile" - exit 1 - fi - else - echo "Did not find $outFile. Not checking!" - fi -} - -function checkOutputs() { - for generator in */*; do - [[ -d "$generator" ]] || continue - echo "Checking $generator" - for outFile in "$PWD/$generator"/*/*; do - [[ -f "$outFile" ]] || continue - local fullpath="$(dirname "$outFile")" - local procname="$(basename "$fullpath")" - checkFile "$generator" "$procname" "$(basename "$outFile")" - done - done -} - -function processYAML() { - local yamlFile="$1" - local filename="${yamlFile%.yaml}" - - mkdir -p "test-$filename" - cd "test-$filename" - echo "Processing file: $yamlFile" - k4GeneratorsConfig "../$yamlFile" - checkOutputs - cd .. -} - -# STEP 1: check the input - -for yamlFile in *.yaml; do - processYAML "$yamlFile" -done - -# STEP 2: run the generators - -function processRun() { - isOK=0 - topDir=${PWD} - thepath="$(dirname "$1")" - runfile="$(basename "$1")" - echo Running $runfile in $thepath - # move to the directory where the script is located - cd $thepath - # run the script - k4ConfigRanGen=0 - ./$runfile - if [[ $? -eq 0 ]]; then - echo k4GeneratorsConfig::Event generation succssful for $runfile in directory $thepath - k4ConfigRanGen=1 - else - k4ConfigRanGen=0 - fi - cd $topDir -} - - -# STEP 3 now we can go through the .sh and run them -if [[ $runEvgen = "true" ]]; then - counter=0 - counterRan=0 - for yamlDir in test-*; do - cd $yamlDir - echo $PWD is the current directory - firstProcessRead="false" - lastGenerator="murks" - for aRunScript in */*/*/*.sh; do - proc="$(dirname "$aRunScript")" - generatorWithPath="$(dirname "$proc")" - if [[ $runReducedEvgen = "true" && $firstProcessRead = "true" && $generatorWithPath = "$lastGenerator" ]]; then - continue - fi - firstProcessRead="true" - lastGenerator=$generatorWithPath - if [[ $k4ConfigRanGen -eq 1 ]]; then - counterRan=$((counterRan+1)) - fi - counter=$((counter+1)) - processRun "$aRunScript" - done - cd .. - done - echo k4GeneratorsConfig::EvGen Summary - echo tried $counter generator runs - echo with $counterRan successful executions - - # STEP 4 - # since we have run the generators we can also do the summary now: - echo Extracting the cross sections by reading EDM4HEP files - ${K4GENERATORSCONFIG}/xsectionSummary -f ${CWD}/xsectionSummary.dat -fi - -# STEP 4: clean up -rm -r "${CWD}/ci-setups" - -exit 0 diff --git a/test/checkGeneratorDatacards.sh b/test/checkGeneratorDatacards.sh deleted file mode 100755 index 126dd6cb..00000000 --- a/test/checkGeneratorDatacards.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -set -e -shopt -s expand_aliases -source ../setup.sh - -EXAMPLEDIR="${PWD}/../examples" -REFDIR="${PWD}/ref-results" - -OPTSTRING="g:h" -while getopts ${OPTSTRING} opt; do - case ${opt} in - g) - echo "Option -g was triggered, Argument: ${OPTARG}" - GENERATOR="${OPTARG}" - echo $0 will only check $GENERATOR - ;; - h) - echo "Arguments are:" - echo "-h for help" - echo "-g GENERATOR " - exit 0 - ;; - ?) - echo "Invalid option: -${OPTARG}." - exit 1 - ;; - esac -done - -# if the generator is not requested explicitely, we define a wildcard (=all) -if [ -z $GENERATOR ]; then - GENERATOR=* -fi - -CWD=${PWD} -cd ci - -function checkFile() { - local generator="$1" - local refgenerator="$(basename "$generator")" - local procname="$2" - local outFile="$3" - if [[ -e "$REFDIR/$refgenerator/$procname/$outFile" ]]; then - if diff "$REFDIR/$refgenerator/$procname/$outFile" "$PWD/$generator/$procname/$outFile" &> /dev/null; then - echo "Process " $procname : "Files are identical for file" $outFile - else - echo "Process " $procname "Files are different for file" $outFile - diff "$REFDIR/$refgenerator/$procname/$outFile" "$PWD/$generator/$procname/$outFile" - exit 1 - fi - else - echo "Did not find $outFile. Not checking!" - fi -} - -# loop over the generators requested: -for generator in */$GENERATOR; do - [[ -d "$generator" ]] || continue - for outFile in "$PWD/$generator"/*/*; do - [[ -f "$outFile" ]] || continue - fullpath="$(dirname "$outFile")" - procname="$(basename "$fullpath")" - checkFile "$generator" "$procname" "$(basename "$outFile")" - # copy the files to the output - mkdir -p "${CWD}"/output/"$generator"/"$procname"/ - cp "$outFile" "${CWD}"/output/"$generator"/"$procname"/"$(basename "$outFile")" - done -done - -exit 0 diff --git a/test/createGeneratorDatacardsFromYaml.sh b/test/createGeneratorDatacardsFromYaml.sh deleted file mode 100755 index a71e89ac..00000000 --- a/test/createGeneratorDatacardsFromYaml.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -set -e -shopt -s expand_aliases -source ../setup.sh - -YAMLDIR="${PWD}/../examples" - -OPTSTRING="d:h" -while getopts ${OPTSTRING} opt; do - case ${opt} in - d) - echo "Option -d was triggered, Argument: ${OPTARG}" - YAMLDIR="${OPTARG}" - echo Searching for yaml files in directory $YAMLDIR - ;; - f) - echo "Option -f was triggered, Argument: ${OPTARG}" - YAMLFILE="${OPTARG}" - echo only $YAMLFILE will be processed - ;; - h) - echo "Arguments are:" - echo "-h for help" - echo "-d YAMLDIRECTORY " - exit 0 - ;; - ?) - echo "Invalid option: -${OPTARG}." - exit 1 - ;; - esac -done - -CWD=${PWD} -# only create the directory if it does not exist yet -if [[ ! -d ${CWD}/ci ]]; then - mkdir -p ${CWD}/ci -else - # clean up the content if necessary - rm -Rf ci/* -fi -# only create the directory if it does not exist yet -if [[ ! -d ${CWD}/output ]]; then - mkdir -p ${CWD}/output -else - # clean up the content if necessary - rm -Rf output/* -fi - - -# only copy if the file does not exist yet: -for yamlFileWithPath in "$YAMLDIR"/*.yaml; do - yamlFile="$(basename "$yamlFileWithPath")" - if [[ ! -f ci/"$yamlFile" ]]; then - echo copying $yamlFileWithPath to ci - cp -f "$yamlFileWithPath" ci - fi -done - -# check whether ecms.dat files are available: -for ecmsFileWithPath in "$YAMLDIR"/"ecms"*.dat; do - ecmsFile="$(basename "$ecmsFileWithPath")" - if [[ -f "$ecmsFileWithPath" && ! -f ci/"$ecmsFile" && "$ecmsFile" != "ecms.dat" ]]; then - echo copying $ecmsFileWithPath to ci - cp -f "$ecmsFileWithPath" ci - fi -done - -cd ci - -# STEP 1: check the input - -function processYAML() { - local yamlFile="$1" - local filename="${yamlFile%.yaml}" - - echo "Processing file: $yamlFile" - if [[ ! -f ecms"$filename".dat ]]; then - k4GeneratorsConfig "$yamlFile" - else - k4GeneratorsConfig "$yamlFile" --ecmsFile ecms"$filename".dat - fi -} - - -for yamlFile in *.yaml; do - processYAML "$yamlFile" -done - - -exit 0 diff --git a/test/runEventGeneration.sh b/test/runEventGeneration.sh deleted file mode 100755 index 21013140..00000000 --- a/test/runEventGeneration.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -set -e - -shopt -s expand_aliases -echo the KEY4HEPSTACK is $KEY4HEP_STACK - -# decode command line options - -OPTSTRING="g:h" -GENERATOR="" -while getopts ${OPTSTRING} opt; do - case ${opt} in - g) - echo "Option -g was triggered, event generation step will be run only for ${OPTARG}" - GENERATOR="$OPTARG" - ;; - h) - echo "Arguments are:" - echo "-h for help" - echo "-g GENERATORNAME only run this generator" - exit 0 - ;; - ?) - echo "Invalid option: -${OPTARG}." - exit 1 - ;; - esac -done - -# if the generator is not requested explicitely, we define a wildcard (=all) -if [ -z $GENERATOR ]; then - GENERATOR=* -fi - -CWD=${PWD} -cd ci - -# run a generator: argument is a pathname -function processRun() { - topDir=${PWD} - thepath="$(dirname "$1")" - runfile="$(basename "$1")" - echo Running $runfile in $thepath - # move to the directory where the script is located - cd $thepath - # run the script - ./$runfile - if [[ $? -eq 0 ]]; then - echo k4GeneratorsConfig::Event generation successful for $runfile in directory $thepath - fi - cd $topDir -} - -file_pattern="*/${GENERATOR}/*/*.sh" -if ls $file_pattern 1> /dev/null 2>&1; then - for aRunScript in ${file_pattern}; do - processRun "$aRunScript" - done -fi - -exit 0 diff --git a/test/runSummary.sh b/test/runSummary.sh deleted file mode 100755 index be75a914..00000000 --- a/test/runSummary.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -e - -shopt -s expand_aliases -source ../setup.sh - -CWD=${PWD} -cd ci - -#STEP 4 -#since we have run the generators we can also do the summary now: -echo "Extracting the cross sections by reading EDM4HEP files and superposing the differential distributions" -eventGenerationSummary -f ${CWD}/output/GenerationSummary.dat -d ../output - -exit 0