First implementation of exporting input variables and infer the ParT network with onnxruntime

CMakeLists.txt

@@ -1,7 +1,7 @@
 ########################################################
 # cmake file for building LCFIPlus package
 # @author Tomohiko Tanabe (ICEPP, The University of Tokyo)
-CMAKE_MINIMUM_REQUIRED(VERSION 2.6 FATAL_ERROR)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.5 FATAL_ERROR)
 ########################################################
 
 
@@ -14,28 +14,55 @@ SET( ${PROJECT_NAME}_VERSION_MAJOR 0 )
 SET( ${PROJECT_NAME}_VERSION_MINOR 11 )
 SET( ${PROJECT_NAME}_VERSION_PATCH 0 )
 
-
-
 ### DEPENDENCIES ############################################################
 
 FIND_PACKAGE( ILCUTIL COMPONENTS ILCSOFT_CMAKE_MODULES REQUIRED )
 
 # load default settings from ILCSOFT_CMAKE_MODULES
 INCLUDE( ilcsoft_default_settings )
 
+# Set up C++ Standard
+# ``-DCMAKE_CXX_STANDARD=<standard>`` when invoking CMake
+set(CMAKE_CXX_STANDARD 17 CACHE STRING "")
+
+if(NOT CMAKE_CXX_STANDARD MATCHES "17|20")
+  message(FATAL_ERROR "Unsupported C++ standard: ${CMAKE_CXX_STANDARD}")
+endif()
+
+# Prevent CMake falls back to the latest standard the compiler does support
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+
+# ONNX support option
+OPTION( ENABLE_ONNX "Enable ONNX Runtime support for ML inference" OFF )
 
 FIND_PACKAGE( Marlin 1.0 REQUIRED )
 FIND_PACKAGE( MarlinUtil REQUIRED )
 #FIND_PACKAGE( ROOT REQUIRED COMPONENTS Minuit2 TMVA TreePlayer )
 FIND_PACKAGE( ROOT REQUIRED COMPONENTS Minuit2 TMVA TreePlayer Gui Geom Eve Minuit XMLIO RGL Ged EG MLP )
+#FIND_PACKAGE( Torch REQUIRED )
 FIND_PACKAGE( LCFIVertex REQUIRED )
 
+# Optional ONNX dependencies
+IF( ENABLE_ONNX )
+  MESSAGE( STATUS "ONNX support enabled" )
+  FIND_PACKAGE( onnxruntime 1.17.1 REQUIRED)
+  FIND_PACKAGE( nlohmann_json REQUIRED)
+  ADD_DEFINITIONS( -DENABLE_ONNX )
+ELSE()
+  MESSAGE( STATUS "ONNX support disabled" )
+ENDIF()
+
+#set(JSON_BuildTests OFF CACHE INTERNAL "")
+#add_subdirectory(nlohmann_json)
+
 INCLUDE_DIRECTORIES( SYSTEM ${Marlin_INCLUDE_DIRS} )
 ADD_DEFINITIONS ( ${Marlin_DEFINITIONS} )
 INCLUDE_DIRECTORIES( SYSTEM ${MarlinUtil_INCLUDE_DIRS} )
 ADD_DEFINITIONS ( ${MarlinUtil_DEFINITIONS} )
 INCLUDE_DIRECTORIES( SYSTEM ${ROOT_INCLUDE_DIRS} )
 INCLUDE_DIRECTORIES( SYSTEM ${LCFIVertex_INCLUDE_DIRS} )
+#INCLUDE_DIRECTORIES( SYSTEM ${TORCH_INCLUDE_DIRS} )
 # left here for backwards compatibility
 INCLUDE_DIRECTORIES( SYSTEM ${LCFIVertex_ROOT}/vertex_lcfi ${LCFIVertex_ROOT}/boost )
 
@@ -58,8 +85,20 @@ SET( ROOT_DICT_INPUT_HEADERS
 		${PROJECT_SOURCE_DIR}/include/TrackNtuple.h
 		${PROJECT_SOURCE_DIR}/include/VertexMassRecovery.h
 		${PROJECT_SOURCE_DIR}/include/VertexNtuple.h
-		${PROJECT_SOURCE_DIR}/include/LinkDef.h
+		${PROJECT_SOURCE_DIR}/include/vertexfinderdnn.h
+		${PROJECT_SOURCE_DIR}/include/DNNProvider2.h
 )
+
+# Add ONNX-related headers if enabled
+IF( ENABLE_ONNX )
+  LIST( APPEND ROOT_DICT_INPUT_HEADERS
+		${PROJECT_SOURCE_DIR}/include/MLInputGenerator.h
+		${PROJECT_SOURCE_DIR}/include/MLMakeNtuple.h
+		${PROJECT_SOURCE_DIR}/include/MLInferenceWeaver.h
+  )
+ENDIF()
+
+LIST( APPEND ROOT_DICT_INPUT_HEADERS ${PROJECT_SOURCE_DIR}/include/LinkDef.h )
 SET( ROOT_DICT_INCLUDE_DIRS ${Marlin_INCLUDE_DIRS} ${LCIO_INCLUDE_DIRS})
 GEN_ROOT_DICT_SOURCES( dict.cc )
 INSTALL(FILES ${ROOT_DICT_OUTPUT_DIR}/dict_rdict.pcm DESTINATION lib)
@@ -79,21 +118,53 @@ ENDIF()
 
 # include directories
 INCLUDE_DIRECTORIES( "./include" )
+#LINK_DIRECTORIES( "./lib" )
 
 # definitions to pass to the compiler
 ADD_DEFINITIONS( "-Wno-effc++ -Wno-shadow" )
 ADD_DEFINITIONS( "-Wno-long-long" )
 ADD_DEFINITIONS( "-Wno-strict-aliasing" ) # avoid warnings in dict.cc
+
 # shut up warnings in boost
 #ADD_DEFINITIONS( "-Wno-unused-local-typedefs" )
 
 # add library
 SET( libname ${PROJECT_NAME} )
 AUX_SOURCE_DIRECTORY( ./src library_sources )
 LIST( REMOVE_ITEM library_sources ${binary_sources} )
+
+# Remove ONNX-related source files if ONNX is disabled
+IF( NOT ENABLE_ONNX )
+  LIST( REMOVE_ITEM library_sources
+    ./src/ONNXRuntime.cc
+    ./src/MLInferenceWeaver.cc
+    ./src/MLInputGenerator.cc
+    ./src/MLMakeNtuple.cc
+    ./src/WeaverInterface.cc
+  )
+ENDIF()
+
 ADD_SHARED_LIBRARY( ${libname} ${library_sources} )
 INSTALL_SHARED_LIBRARY( ${libname} DESTINATION lib )
-TARGET_LINK_LIBRARIES( ${libname} ${Marlin_LIBRARIES} ${ROOT_LIBRARIES} ${ROOT_COMPONENT_LIBRARIES} ${MarlinUtil_LIBRARIES} ${LCFIVertex_LIBRARIES} )
+
+# Link libraries
+SET( LINK_LIBS
+	${Marlin_LIBRARIES}
+	${ROOT_LIBRARIES}
+	${ROOT_COMPONENT_LIBRARIES}
+	${MarlinUtil_LIBRARIES}
+	${LCFIVertex_LIBRARIES}
+)
+
+# Add ONNX libraries if enabled
+IF( ENABLE_ONNX )
+  LIST( APPEND LINK_LIBS
+    nlohmann_json::nlohmann_json
+    onnxruntime::onnxruntime
+  )
+ENDIF()
+
+TARGET_LINK_LIBRARIES( ${libname} ${LINK_LIBS} )
 
 ### EXECUTABLE MODE ########################################################
 
@@ -108,7 +179,7 @@ LIST(APPEND binary_sources ${ROOT_DICT_OUTPUT_SOURCES} )
 ADD_EXECUTABLE( lcfiplus_bin EXCLUDE_FROM_ALL ${binary_sources} )
 
 SET_TARGET_PROPERTIES( lcfiplus_bin PROPERTIES COMPILE_FLAGS "-DBUILD_EVE" OUTPUT_NAME lcfiplus )
-TARGET_LINK_LIBRARIES( lcfiplus_bin ${libname} ${ROOT_LIBRARIES} ${ROOT_COMPONENT_LIBRARIES} )
+TARGET_LINK_LIBRARIES( lcfiplus_bin ${libname} torch ${ROOT_LIBRARIES} ${ROOT_COMPONENT_LIBRARIES} )
 
 # display some variables and write them to cache
 DISPLAY_STD_VARIABLES()

doc/ReleaseNotes.md

@@ -1,3 +1,40 @@
+# Development branch additions (merged 2025-12-06)
+
+* 2025-12-05 SUEHARA Taikan
+  - Implement backward compatibility for MC-PFO assignment
+    - Add default parameter to InitMCPPFOCollections for backward compatibility
+    - Use simple method (first element) when Track-MC relation is not available
+    - Use improved method (weight-based, multi-track support) when available
+    - Maintain full backward compatibility with upstream v00-11
+
+* 2025-12-03 SUEHARA Taikan
+  - Add optional ONNX support with backward compatibility
+    - Enable ONNX Runtime support as optional feature (ENABLE_ONNX CMake option, default: OFF)
+    - Make onnxruntime and nlohmann_json optional dependencies
+    - Conditionally compile ONNX-related source files
+    - Full backward compatibility when ENABLE_ONNX=OFF
+
+* SUEHARA Taikan and collaborators (2024-2025)
+  - Machine Learning and ONNX integration
+    - Add MLInputGenerator, MLMakeNtuple, MLInferenceWeaver for ML-based flavor tagging
+    - Add WeaverInterface and ONNXRuntime for ONNX model inference
+    - Add DNNProvider2 for DNN-based vertex finding
+    - Implement event-based classification with jets
+    - Add dEdx support for particle identification
+
+  - Flavor tagging improvements
+    - Improve PFA-track assignment and track-MC assignment
+    - Implement true jet flavor assignment from MC
+    - Add MC-to-jet assignment algorithm (AssignJetsToMC)
+    - Bugfixes on MC flavor assignment
+    - Add sorted track and neutral accessors (getAllTracksSorted, getNeutralsSorted)
+
+  - Code quality and compatibility
+    - Update C++ standard to C++17 with CMake 3.5+ requirement
+    - Compatibility fixes for key4hep environment and onnxruntime
+    - Various bugfixes in weaver output and neutral PF candidate masking
+    - Add event-based input support
+
 # v00-11
 
 * 2025-02-24 Thomas Madlener ([PR#73](https://github.com/LCFIPlus/LCFIPlus/pull/73))

examples/steering/ml_inference_test.xml

@@ -0,0 +1,81 @@
+<marlin>
+
+  <constants>
+    <!-- ***** Global constant for reconstruction ***** -->
+    <!-- The k4geo directory where to look for the ILD model -->
+    <constant name="k4geo_DIR" value="/cvmfs/sw.hsf.org/key4hep/releases/2025-05-29/x86_64-almalinux9-gcc14.2.0-opt/k4geo/00-22-ubhvqv/share/k4geo" />
+    <!-- ILD detector model -->
+    <constant name="DetectorModel" value="ILD_o1_v05" />
+    <!-- The full compact file name -->
+    <constant name="CompactFile" value="${k4geo_DIR}/ILD/compact/${DetectorModel}/${DetectorModel}.xml" />
+  </constants>
+
+<execute>
+  <processor name="InitDD4hep"/>
+	<processor name="MLInferenceWeaver"/>
+	<processor name="LCIOOutput"/>
+</execute>
+
+
+
+<global>
+	<parameter name="LCIOInputFiles">
+	  /group/ilc/users/rtagami/correction_dEdx/bb/output/n000.d_dstm_14986_0.slcio
+	</parameter>
+	<parameter name="MaxRecordNumber" value="10"/>
+	<parameter name="SkipNEvents" value="0" />
+	<parameter name="SupressCheck" value="false" />
+	<parameter name="Verbosity" options="DEBUG0-4,MESSAGE0-4,WARNING0-4,ERROR0-4,SILENT">MESSAGE0</parameter>
+</global>
+
+  <processor name="InitDD4hep" type="InitializeDD4hep">
+    <!--InitializeDD4hep reads a compact xml file and initializes the DD4hep::LCDD object-->
+    <!--Name of the DD4hep compact xml file to load-->
+    <parameter name="DD4hepXMLFile" type="string"> ${CompactFile} </parameter>
+    </processor>
+
+<processor name="MLInferenceWeaver" type="LcfiplusProcessor">
+
+	<!-- run ML inference -->
+	<parameter name="Algorithms" type="stringVec"> MLInferenceWeaver </parameter>
+	<parameter name="ReadSubdetectorEnergies" type="int" value="1"/> <!-- true for ILD -->
+	<parameter name="UpdateVertexRPDaughters" type="int" value="0"/> <!-- false for non-updative PandoraPFOs -->
+	<parameter name="TrackHitOrdering" type="int" value="1"/> <!-- Track hit ordering: 0=ILD-LOI,SID-DBD, 1=ILD-DBD -->
+	<parameter name="PrintEventNumber" type="int" value="1"/> <!-- 0 for not printing event number, n for printing every n events -->
+
+	<parameter name="MagneticField" type="double" value="3.5"/>
+
+	<!-- specify input collection names -->
+	<parameter name="PFOCollection" type="string" value="PandoraPFOs" />
+	<parameter name="PrimaryVertexCollectionName" type="string" value="PrimaryVertex" />
+
+	<!-- ML Inference Weaver parameters -->
+	<parameter name="MLInferenceWeaver.JetCollectionName" type="string" value="RefinedJets" />
+	<parameter name="MLInferenceWeaver.UpdateJetCollectionName" type="string" value="RefinedJets2" />
+	<parameter name="MLInferenceWeaver.JsonFileName" type="string" value="../../onnx/ilc_nnqq_strange_cpidprob_mcpdg/preprocess.json" />
+	<parameter name="MLInferenceWeaver.OnnxFileName" type="string" value="../../onnx/ilc_nnqq_strange_cpidprob_mcpdg/ilc_nnqq_strange_cpidprob_mcpdg.onnx" />
+	<parameter name="MLInferenceWeaver.OutputParameterName" type="string" value="weaver" />
+
+</processor>
+
+<processor name="LCIOOutput" type="LCIOOutputProcessor">
+	<!--Writes the current event to the specified LCIO outputfile-->
+	<!--drops the named collections from the event-->
+	<parameter name="DropCollectionNames" type="StringVec"> </parameter>
+	<!--drops all collections of the given type from the event-->
+	<parameter name="DropCollectionTypes" type="StringVec"> </parameter>
+	<!-- write complete objects in subset collections to the file (i.e. ignore subset flag)-->
+	<parameter name="FullSubsetCollections" type="StringVec"> </parameter>
+	<!--force keep of the named collections - overrules DropCollectionTypes (and DropCollectionNames)-->
+	<parameter name="KeepCollectionNames" type="StringVec"> </parameter>
+	<!-- name of output file -->
+	<parameter name="LCIOOutputFile" type="string"> ml_inference_test_output.slcio </parameter>
+	<!--write mode for output file:  WRITE_APPEND or WRITE_NEW-->
+	<parameter name="LCIOWriteMode" type="string"> WRITE_NEW </parameter>
+	<!--Set it to true to remove intermediate collections and keep only final products-->
+	<parameter name="SplitFilenameByEvent" type="bool"> false </parameter>
+	<!--will split output file if size in kB exceeds given value - doesn't work with APPEND and NEW-->
+	<parameter name="SplitFilenameSizekB" type="int"> 1996488704 </parameter>
+</processor>
+
+</marlin>

examples/steering/ml_training_data_collection.xml

@@ -0,0 +1,64 @@
+<marlin>
+
+  <constants>
+    <!-- ***** Global constant for reconstruction ***** -->
+    <!-- The lcgeo directory where to look for the ILD model -->
+    <constant name="lcgeo_DIR" value="/cvmfs/ilc.desy.de/sw/x86_64_gcc103_centos7/v02-03-03/lcgeo/v00-20-00" />
+    <!-- ILD detector model -->
+    <constant name="DetectorModel" value="ILD_o1_v05" />
+    <!-- The full compact file name -->
+    <constant name="CompactFile" value="${lcgeo_DIR}/ILD/compact/${DetectorModel}/${DetectorModel}.xml" />
+  </constants>
+
+<execute>
+  <processor name="InitDD4hep"/>
+	<processor name="MLMakeNtuple"/>
+</execute>
+
+
+
+<global>
+  <parameter name="LCIOInputFiles">
+test_nnbb.slcio
+	</parameter>
+	<parameter name="MaxRecordNumber" value="100"/>  
+	<parameter name="SkipNEvents" value="0" />  
+	<parameter name="SupressCheck" value="false" />  
+	<parameter name="Verbosity" options="DEBUG0-4,MESSAGE0-4,WARNING0-4,ERROR0-4,SILENT">MESSAGE0</parameter> 
+</global>
+
+  <processor name="InitDD4hep" type="InitializeDD4hep">
+    <!--InitializeDD4hep reads a compact xml file and initializes the DD4hep::LCDD object-->
+    <!--Name of the DD4hep compact xml file to load-->
+    <parameter name="DD4hepXMLFile" type="string"> ${CompactFile} </parameter>
+    </processor>
+
+<processor name="MLMakeNtuple" type="LcfiplusProcessor">
+
+	<!-- run primary and secondary vertex finders -->
+	<parameter name="Algorithms" type="stringVec"> MLMakeNtuple </parameter>
+	<parameter name="ReadSubdetectorEnergies" type="int" value="1"/> <!-- true for ILD -->
+	<parameter name="UpdateVertexRPDaughters" type="int" value="0"/> <!-- false for non-updative PandoraPFOs -->
+	<parameter name="TrackHitOrdering" type="int" value="1"/> <!-- Track hit ordering: 0=ILD-LOI,SID-DBD, 1=ILD-DBD -->
+	<parameter name="PrintEventNumber" type="int" value="1000"/> <!-- 0 for not printing event number, n for printing every n events -->
+
+	<parameter name="MagneticField" type="double" value="3.5"/>
+	<parameter name="PIDAlgorithmName" type="string" value="TMVA_BDT_MC_16bins_singleP" />
+
+	<parameter name="UseMCP" type="int" value="1" />
+	<parameter name="MCPCollection" type="string" value="MCParticlesSkimmed" />
+	<parameter name="MCPFORelation" type="string" value="RecoMCTruthLink" />
+
+	<!-- specify input collection names -->
+	<parameter name="PFOCollection" type="string" value="PandoraPFOs" />
+	<parameter name="PrimaryVertexCollectionName" type="string" value="PrimaryVertex" />
+
+	<parameter name="MLMakeNtuple.JetCollectionName" type="string" value="RefinedJets" />
+	<parameter name="MLMakeNtuple.OutputRootFileName" type="string" value="test_nnbb_mlmakentuple.root" />
+	<parameter name="MLMakeNtuple.TTreeName" type="string" value="ntp" />
+	<parameter name="MLMakeNtuple.Label" type="int" value="0" />
+
+
+</processor>
+
+</marlin>