From d51431a1c39d3740b9c91a827187ec09515b836f Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 2 Jul 2025 18:46:03 +0200 Subject: [PATCH 01/11] refactor CMake scripts: centralize target linking functionality --- CMakeLists.txt | 1 + cmake/gtest.cmake | 12 +++++++ cmake/json.cmake | 11 +++++++ cmake/libenvpp.cmake | 18 ++++++++++ cmake/mpi.cmake | 17 +++++++++- cmake/onetbb.cmake | 14 ++++++++ cmake/openmp.cmake | 10 ++++++ cmake/stb.cmake | 5 +++ modules/core/CMakeLists.txt | 65 +++++-------------------------------- 9 files changed, 95 insertions(+), 58 deletions(-) create mode 100644 cmake/stb.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 88f343287..1a0982a27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ include(cmake/modes.cmake) include(cmake/sanitizers.cmake) include(cmake/json.cmake) include(cmake/libenvpp.cmake) +include(cmake/stb.cmake) ################# Parallel programming technologies ################# diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake index a9bdd6f13..eb41a7cd3 100644 --- a/cmake/gtest.cmake +++ b/cmake/gtest.cmake @@ -24,3 +24,15 @@ ExternalProject_Add( "${CMAKE_COMMAND}" --install "${CMAKE_CURRENT_BINARY_DIR}/ppc_googletest/build" --prefix "${CMAKE_CURRENT_BINARY_DIR}/ppc_googletest/install") + +function(ppc_link_gtest exec_func_lib) + # Add external project include directories + target_include_directories( + ${exec_func_lib} + PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/googletest/googletest/include) + + add_dependencies(${exec_func_lib} ppc_googletest) + target_link_directories(${exec_func_lib} PUBLIC + "${CMAKE_BINARY_DIR}/ppc_googletest/install/lib") + target_link_libraries(${exec_func_lib} PUBLIC gtest gtest_main) +endfunction() diff --git a/cmake/json.cmake b/cmake/json.cmake index 89070d4b7..882553058 100644 --- a/cmake/json.cmake +++ b/cmake/json.cmake @@ -19,3 +19,14 @@ ExternalProject_Add( INSTALL_COMMAND "${CMAKE_COMMAND}" --install "${CMAKE_CURRENT_BINARY_DIR}/ppc_json/build" --prefix "${CMAKE_CURRENT_BINARY_DIR}/ppc_json/install") + +function(ppc_link_json exec_func_lib) + # Add external project include directories + target_include_directories( + ${exec_func_lib} + PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/json/include) + + add_dependencies(${exec_func_lib} ppc_json) + target_link_directories(${exec_func_lib} INTERFACE + "${CMAKE_BINARY_DIR}/ppc_json/install/include") +endfunction() \ No newline at end of file diff --git a/cmake/libenvpp.cmake b/cmake/libenvpp.cmake index 564a7d488..e150de19b 100644 --- a/cmake/libenvpp.cmake +++ b/cmake/libenvpp.cmake @@ -38,3 +38,21 @@ if(WIN32) else() set(PPC_ENVPP_LIB_NAME envpp) endif() + +function(ppc_link_envpp exec_func_lib) + # Add external project include directories + target_include_directories( + ${exec_func_lib} + PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/libenvpp/include) + target_include_directories( + ${exec_func_lib} SYSTEM + PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/libenvpp/external/fmt/include) + + add_dependencies(${exec_func_lib} ppc_libenvpp) + target_link_directories(${exec_func_lib} PUBLIC + "${CMAKE_BINARY_DIR}/ppc_libenvpp/install/lib") + target_link_directories(${exec_func_lib} PUBLIC + "${CMAKE_BINARY_DIR}/ppc_libenvpp/build") + target_link_libraries(${exec_func_lib} PUBLIC ${PPC_ENVPP_LIB_NAME}) + target_link_libraries(${exec_func_lib} PUBLIC ${PPC_FMT_LIB_NAME}) +endfunction() diff --git a/cmake/mpi.cmake b/cmake/mpi.cmake index 8b307ccdd..9394ff932 100644 --- a/cmake/mpi.cmake +++ b/cmake/mpi.cmake @@ -1,4 +1,19 @@ find_package(MPI REQUIRED) if(NOT MPI_FOUND) message(FATAL_ERROR "MPI NOT FOUND") -endif(MPI_FOUND) +endif() + +function(ppc_link_mpi exec_func_lib) + find_package(MPI REQUIRED) + if(MPI_COMPILE_FLAGS) + set_target_properties(${exec_func_lib} PROPERTIES COMPILE_FLAGS + "${MPI_COMPILE_FLAGS}") + endif(MPI_COMPILE_FLAGS) + + if(MPI_LINK_FLAGS) + set_target_properties(${exec_func_lib} PROPERTIES LINK_FLAGS + "${MPI_LINK_FLAGS}") + endif(MPI_LINK_FLAGS) + target_include_directories(${exec_func_lib} PUBLIC ${MPI_INCLUDE_PATH}) + target_link_libraries(${exec_func_lib} PUBLIC ${MPI_LIBRARIES}) +endfunction() diff --git a/cmake/onetbb.cmake b/cmake/onetbb.cmake index df89aa354..b14b2ed0e 100644 --- a/cmake/onetbb.cmake +++ b/cmake/onetbb.cmake @@ -42,3 +42,17 @@ if(cmake_build_type_lower STREQUAL "debug") else() set(PPC_TBB_LIB_NAME tbb) endif() + +function(ppc_link_tbb exec_func_lib) + # Add external project include directories + target_include_directories( + ${exec_func_lib} + PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/onetbb/include) + + add_dependencies(${exec_func_lib} ppc_onetbb) + target_link_directories(${exec_func_lib} PUBLIC + ${CMAKE_BINARY_DIR}/ppc_onetbb/install/lib) + if(NOT MSVC) + target_link_libraries(${exec_func_lib} PUBLIC ${PPC_TBB_LIB_NAME}) + endif() +endfunction() diff --git a/cmake/openmp.cmake b/cmake/openmp.cmake index 445815153..33b56e339 100644 --- a/cmake/openmp.cmake +++ b/cmake/openmp.cmake @@ -23,3 +23,13 @@ if(OpenMP_FOUND) else(OpenMP_FOUND) message(FATAL_ERROR "OpenMP NOT FOUND") endif(OpenMP_FOUND) + +function(ppc_link_threads exec_func_lib) + target_link_libraries(${exec_func_lib} PUBLIC Threads::Threads) +endfunction() + +function(ppc_link_openmp exec_func_lib) + find_package(OpenMP REQUIRED) + target_link_libraries(${exec_func_lib} PUBLIC ${OpenMP_libomp_LIBRARY} + OpenMP::OpenMP_CXX) +endfunction() diff --git a/cmake/stb.cmake b/cmake/stb.cmake new file mode 100644 index 000000000..2770d4440 --- /dev/null +++ b/cmake/stb.cmake @@ -0,0 +1,5 @@ +function(ppc_link_stb exec_func_lib) + add_library(stb_image STATIC ${CMAKE_SOURCE_DIR}/3rdparty/stb_image_wrapper.cpp) + target_include_directories(stb_image PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/stb) + target_link_libraries(${exec_func_lib} PUBLIC stb_image) +endfunction() \ No newline at end of file diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index 318572711..487b2c9f5 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -29,63 +29,14 @@ target_include_directories( ${exec_func_lib} PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty ${CMAKE_SOURCE_DIR}/modules ${CMAKE_SOURCE_DIR}/tasks) -# Add external project include directories -target_include_directories( - ${exec_func_lib} - PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/onetbb/include - ${CMAKE_SOURCE_DIR}/3rdparty/json/include - ${CMAKE_SOURCE_DIR}/3rdparty/googletest/googletest/include - ${CMAKE_SOURCE_DIR}/3rdparty/libenvpp/include) -target_include_directories( - ${exec_func_lib} SYSTEM - PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/libenvpp/external/fmt/include) - -add_dependencies(${exec_func_lib} ppc_libenvpp) -target_link_directories(${exec_func_lib} PUBLIC - "${CMAKE_BINARY_DIR}/ppc_libenvpp/install/lib") -target_link_directories(${exec_func_lib} PUBLIC - "${CMAKE_BINARY_DIR}/ppc_libenvpp/build") -target_link_libraries(${exec_func_lib} PUBLIC ${PPC_ENVPP_LIB_NAME}) -target_link_libraries(${exec_func_lib} PUBLIC ${PPC_FMT_LIB_NAME}) - -add_dependencies(${exec_func_lib} ppc_json) -target_link_directories(${exec_func_lib} INTERFACE - "${CMAKE_BINARY_DIR}/ppc_json/install/include") - -add_dependencies(${exec_func_lib} ppc_googletest) -target_link_directories(${exec_func_lib} PUBLIC - "${CMAKE_BINARY_DIR}/ppc_googletest/install/lib") -target_link_libraries(${exec_func_lib} PUBLIC gtest gtest_main) - -target_link_libraries(${exec_func_lib} PUBLIC Threads::Threads) - -find_package(OpenMP REQUIRED) -target_link_libraries(${exec_func_lib} PUBLIC ${OpenMP_libomp_LIBRARY} - OpenMP::OpenMP_CXX) - -add_dependencies(${exec_func_lib} ppc_onetbb) -target_link_directories(${exec_func_lib} PUBLIC - ${CMAKE_BINARY_DIR}/ppc_onetbb/install/lib) -if(NOT MSVC) - target_link_libraries(${exec_func_lib} PUBLIC ${PPC_TBB_LIB_NAME}) -endif() - -find_package(MPI REQUIRED) -if(MPI_COMPILE_FLAGS) - set_target_properties(${exec_func_lib} PROPERTIES COMPILE_FLAGS - "${MPI_COMPILE_FLAGS}") -endif(MPI_COMPILE_FLAGS) - -if(MPI_LINK_FLAGS) - set_target_properties(${exec_func_lib} PROPERTIES LINK_FLAGS - "${MPI_LINK_FLAGS}") -endif(MPI_LINK_FLAGS) -target_include_directories(${exec_func_lib} PUBLIC ${MPI_INCLUDE_PATH}) -target_link_libraries(${exec_func_lib} PUBLIC ${MPI_LIBRARIES}) - -add_library(stb_image STATIC ${CMAKE_SOURCE_DIR}/3rdparty/stb_image_wrapper.cpp) -target_include_directories(stb_image PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/stb) -target_link_libraries(${exec_func_lib} PUBLIC stb_image) +ppc_link_envpp(${exec_func_lib}) +ppc_link_json(${exec_func_lib}) +ppc_link_gtest(${exec_func_lib}) +ppc_link_threads(${exec_func_lib}) +ppc_link_openmp(${exec_func_lib}) +ppc_link_tbb(${exec_func_lib}) +ppc_link_mpi(${exec_func_lib}) +ppc_link_stb(${exec_func_lib}) add_executable(${exec_func_tests} ${FUNC_TESTS_SOURCE_FILES}) From b1ddc169855ffedbf3c231b949da8069839f4206 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 30 Jul 2025 14:04:40 +0200 Subject: [PATCH 02/11] add LLVM-based coverage generation support and integrate with CI pipeline --- .github/workflows/ubuntu.yml | 42 ++++---- cmake/configure.cmake | 5 +- docker/ubuntu.Dockerfile | 2 +- scripts/generate_llvm_coverage.py | 162 ++++++++++++++++++++++++++++++ 4 files changed, 191 insertions(+), 20 deletions(-) create mode 100755 scripts/generate_llvm_coverage.py diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index b2e9d1219..64cd165d7 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -322,7 +322,7 @@ jobs: env: PPC_NUM_PROC: 1 PPC_ASAN_RUN: 1 - gcc-build-codecov: + clang-build-codecov: needs: - gcc-test-extended - clang-test-extended @@ -339,12 +339,13 @@ jobs: - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ runner.os }}-gcc + key: ${{ runner.os }}-clang-coverage create-symlink: true max-size: 1G - name: CMake configure run: > cmake -S . -B build -G Ninja + -D CMAKE_C_COMPILER=clang-20 -D CMAKE_CXX_COMPILER=clang++-20 -D CMAKE_C_COMPILER_LAUNCHER=ccache -D CMAKE_CXX_COMPILER_LAUNCHER=ccache -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_VERBOSE_MAKEFILE=ON -D USE_COVERAGE=ON @@ -358,31 +359,38 @@ jobs: PPC_NUM_THREADS: 2 OMPI_ALLOW_RUN_AS_ROOT: 1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + LLVM_PROFILE_FILE: "coverage-%p-%m.profraw" - name: Run tests (threads) run: scripts/run_tests.py --running-type="threads" --counts 1 2 3 4 env: PPC_NUM_PROC: 1 - - name: Generate gcovr Coverage Data + LLVM_PROFILE_FILE: "coverage-%p-%m.profraw" + - name: Generate LLVM Coverage Data run: | mkdir cov-report cd build - gcovr --gcov-executable `which gcov-14` \ - -r ../ \ - --exclude '.*3rdparty/.*' \ - --exclude '/usr/.*' \ - --exclude '.*tasks/.*/tests/.*' \ - --exclude '.*modules/.*/tests/.*' \ - --exclude '.*tasks/common/runners/.*' \ - --exclude '.*modules/runners/.*' \ - --exclude '.*modules/util/include/perf_test_util.hpp' \ - --exclude '.*modules/util/include/func_test_util.hpp' \ - --exclude '.*modules/util/src/func_test_util.cpp' \ - --xml --output ../coverage.xml \ - --html=../cov-report/index.html --html-details + # Merge all raw profiles into a single indexed profile + llvm-profdata-20 merge -sparse $(find . -name "*.profraw") -o coverage.profdata + # Find all test executables + BINARIES=$(find bin -type f -executable | tr '\n' ' ') + # Generate coverage report in LCOV format for Codecov + llvm-cov-20 export \ + $BINARIES \ + --format=lcov \ + --ignore-filename-regex='.*3rdparty/.*|/usr/.*|.*tests/.*|.*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|.*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp' \ + --instr-profile=coverage.profdata \ + > ../coverage.lcov + # Generate HTML report + llvm-cov-20 show \ + $BINARIES \ + --format=html \ + --output-dir=../cov-report \ + --ignore-filename-regex='.*3rdparty/.*|/usr/.*|.*tests/.*|.*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|.*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp' \ + --instr-profile=coverage.profdata - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5.4.3 with: - files: coverage.xml + files: coverage.lcov - name: Upload coverage report artifact id: upload-cov uses: actions/upload-artifact@v4 diff --git a/cmake/configure.cmake b/cmake/configure.cmake index 59bdfe674..630372603 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -63,8 +63,9 @@ if(UNIX) endif() if(USE_COVERAGE) - add_compile_options(--coverage) - add_link_options(--coverage) + # Use LLVM source-based code coverage + add_compile_options(-fprofile-instr-generate -fcoverage-mapping) + add_link_options(-fprofile-instr-generate -fcoverage-mapping) endif(USE_COVERAGE) endif() diff --git a/docker/ubuntu.Dockerfile b/docker/ubuntu.Dockerfile index f3ba14f5d..594705e81 100644 --- a/docker/ubuntu.Dockerfile +++ b/docker/ubuntu.Dockerfile @@ -16,7 +16,7 @@ RUN set -e \ openmpi-bin openmpi-common libopenmpi-dev \ libomp-dev \ gcc-14 g++-14 \ - gcovr zip \ + zip \ && wget -q https://apt.llvm.org/llvm.sh \ && chmod +x llvm.sh \ && ./llvm.sh 20 all \ diff --git a/scripts/generate_llvm_coverage.py b/scripts/generate_llvm_coverage.py new file mode 100755 index 000000000..058804375 --- /dev/null +++ b/scripts/generate_llvm_coverage.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Generate LLVM coverage report for the project.""" + +import os +import subprocess +import sys +import glob +import argparse + + +def run_command(cmd, cwd=None): + """Run a command and return its output.""" + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + if result.returncode != 0: + print(f"Error: {result.stderr}") + sys.exit(1) + return result.stdout + + +def main(): + parser = argparse.ArgumentParser(description="Generate LLVM coverage report") + parser.add_argument("--build-dir", default="build", help="Build directory") + parser.add_argument("--output-dir", default="coverage", help="Output directory for coverage report") + parser.add_argument("--llvm-version", default="20", help="LLVM version (default: 20)") + args = parser.parse_args() + + build_dir = os.path.abspath(args.build_dir) + output_dir = os.path.abspath(args.output_dir) + + # Try to find LLVM tools in various locations + llvm_profdata = None + llvm_cov = None + + # List of possible LLVM tool names + if args.llvm_version: + profdata_names = [f"llvm-profdata-{args.llvm_version}", "llvm-profdata"] + cov_names = [f"llvm-cov-{args.llvm_version}", "llvm-cov"] + else: + profdata_names = ["llvm-profdata"] + cov_names = ["llvm-cov"] + + # Try to find the tools + for name in profdata_names: + result = subprocess.run(["which", name], capture_output=True, text=True) + if result.returncode == 0: + llvm_profdata = name + break + + for name in cov_names: + result = subprocess.run(["which", name], capture_output=True, text=True) + if result.returncode == 0: + llvm_cov = name + break + + if not llvm_profdata or not llvm_cov: + print("Error: Could not find llvm-profdata or llvm-cov in PATH") + print("Make sure LLVM tools are installed and in your PATH") + sys.exit(1) + + if not os.path.exists(build_dir): + print(f"Error: Build directory {build_dir} does not exist") + sys.exit(1) + + # Create output directory + os.makedirs(output_dir, exist_ok=True) + + # Find all .profraw files + profraw_files = glob.glob(os.path.join(build_dir, "**", "*.profraw"), recursive=True) + if not profraw_files: + print("No .profraw files found. Make sure to run tests with LLVM_PROFILE_FILE set.") + print("Example: LLVM_PROFILE_FILE='coverage-%p-%m.profraw' ./your_test") + sys.exit(1) + + print(f"Found {len(profraw_files)} .profraw files") + + # Merge profiles + profdata_file = os.path.join(output_dir, "coverage.profdata") + run_command([llvm_profdata, "merge", "-sparse"] + profraw_files + ["-o", profdata_file]) + print(f"Created merged profile: {profdata_file}") + + # Find all executables in bin directory + bin_dir = os.path.join(build_dir, "bin") + if not os.path.exists(bin_dir): + print(f"Error: Bin directory {bin_dir} does not exist") + sys.exit(1) + + executables = [] + for root, dirs, files in os.walk(bin_dir): + for file in files: + filepath = os.path.join(root, file) + if os.access(filepath, os.X_OK) and not file.endswith('.txt'): + executables.append(filepath) + + if not executables: + print("No executables found in bin directory") + sys.exit(1) + + print(f"Found {len(executables)} executables") + + # Get the project root directory (parent of build dir) + project_root = os.path.dirname(build_dir) + + # Generate LCOV report + lcov_file = os.path.join(output_dir, "coverage.lcov") + cmd = [llvm_cov, "export"] + executables + [ + "--format=lcov", + "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tests/.*|" + ".*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" + ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", + f"--instr-profile={profdata_file}" + ] + + with open(lcov_file, "w") as f: + result = subprocess.run(cmd, stdout=f, stderr=subprocess.PIPE, text=True) + if result.returncode != 0: + print(f"Error generating LCOV report: {result.stderr}") + sys.exit(1) + + print(f"Generated LCOV report: {lcov_file}") + + # Post-process LCOV file to use relative paths + with open(lcov_file, 'r') as f: + lcov_content = f.read() + + # Replace absolute paths with relative paths + lcov_content = lcov_content.replace(project_root + '/', '') + + with open(lcov_file, 'w') as f: + f.write(lcov_content) + + print("Post-processed LCOV file to use relative paths") + + # Generate HTML report + html_dir = os.path.join(output_dir, "html") + cmd = [llvm_cov, "show"] + executables + [ + "--format=html", + f"--output-dir={html_dir}", + "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tests/.*|" + ".*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" + ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", + f"--instr-profile={profdata_file}" + ] + + run_command(cmd) + print(f"Generated HTML report: {html_dir}/index.html") + + # Generate summary + cmd = [llvm_cov, "report"] + executables + [ + f"--instr-profile={profdata_file}", + "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tasks/.*/tests/.*|.*modules/.*/tests/.*|" + ".*tasks/common/runners/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" + ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp" + ] + + summary = run_command(cmd) + print("\nCoverage Summary:") + print(summary) + + +if __name__ == "__main__": + main() \ No newline at end of file From 6c33cd87dea65c27eb2cda5e8a892597d8ee275d Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 30 Jul 2025 14:31:03 +0200 Subject: [PATCH 03/11] add LLVM-based coverage generation support and integrate with CI pipeline --- .clang-tidy | 108 +++++++++--------- .github/workflows/ubuntu.yml | 43 +------ scripts/generate_llvm_coverage.py | 162 -------------------------- scripts/run_tests.py | 184 +++++++++++++++++++++++++++++- 4 files changed, 240 insertions(+), 257 deletions(-) delete mode 100755 scripts/generate_llvm_coverage.py diff --git a/.clang-tidy b/.clang-tidy index 55bedf11f..331b540b9 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -32,60 +32,60 @@ WarningsAsErrors: "*" HeaderFilterRegex: '.*/(modules|tasks)/.*' CheckOptions: - - key: readability-identifier-naming.ClassCase - value: CamelCase - - key: readability-identifier-naming.ClassMemberCase - value: lower_case - - key: readability-identifier-naming.ConstexprVariableCase - value: CamelCase - - key: readability-identifier-naming.ConstexprVariablePrefix - value: k - - key: readability-identifier-naming.EnumCase - value: CamelCase - - key: readability-identifier-naming.EnumConstantCase - value: CamelCase - - key: readability-identifier-naming.EnumConstantPrefix - value: k - - key: readability-identifier-naming.FunctionCase - value: CamelCase - - key: readability-identifier-naming.GlobalConstantCase - value: CamelCase - - key: readability-identifier-naming.GlobalConstantPrefix - value: k - - key: readability-identifier-naming.StaticConstantCase - value: CamelCase - - key: readability-identifier-naming.StaticConstantPrefix - value: k - - key: readability-identifier-naming.StaticVariableCase - value: lower_case - - key: readability-identifier-naming.MacroDefinitionCase - value: UPPER_CASE - - key: readability-identifier-naming.MacroDefinitionIgnoredRegexp - value: '^[A-Z]+(_[A-Z]+)*_$' - - key: readability-identifier-naming.MemberCase - value: lower_case - - key: readability-identifier-naming.PrivateMemberSuffix - value: _ - - key: readability-identifier-naming.PublicMemberSuffix - value: '' - - key: readability-identifier-naming.NamespaceCase - value: lower_case - - key: readability-identifier-naming.ParameterCase - value: lower_case - - key: readability-identifier-naming.TypeAliasCase - value: CamelCase - - key: readability-identifier-naming.TypedefCase - value: CamelCase - - key: readability-identifier-naming.VariableCase - value: lower_case - - key: readability-identifier-naming.IgnoreMainLikeFunctions - value: 1 + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.ClassMemberCase + value: lower_case + - key: readability-identifier-naming.ConstexprVariableCase + value: CamelCase + - key: readability-identifier-naming.ConstexprVariablePrefix + value: k + - key: readability-identifier-naming.EnumCase + value: CamelCase + - key: readability-identifier-naming.EnumConstantCase + value: CamelCase + - key: readability-identifier-naming.EnumConstantPrefix + value: k + - key: readability-identifier-naming.FunctionCase + value: CamelCase + - key: readability-identifier-naming.GlobalConstantCase + value: CamelCase + - key: readability-identifier-naming.GlobalConstantPrefix + value: k + - key: readability-identifier-naming.StaticConstantCase + value: CamelCase + - key: readability-identifier-naming.StaticConstantPrefix + value: k + - key: readability-identifier-naming.StaticVariableCase + value: lower_case + - key: readability-identifier-naming.MacroDefinitionCase + value: UPPER_CASE + - key: readability-identifier-naming.MacroDefinitionIgnoredRegexp + value: '^[A-Z]+(_[A-Z]+)*_$' + - key: readability-identifier-naming.MemberCase + value: lower_case + - key: readability-identifier-naming.PrivateMemberSuffix + value: _ + - key: readability-identifier-naming.PublicMemberSuffix + value: '' + - key: readability-identifier-naming.NamespaceCase + value: lower_case + - key: readability-identifier-naming.ParameterCase + value: lower_case + - key: readability-identifier-naming.TypeAliasCase + value: CamelCase + - key: readability-identifier-naming.TypedefCase + value: CamelCase + - key: readability-identifier-naming.VariableCase + value: lower_case + - key: readability-identifier-naming.IgnoreMainLikeFunctions + value: 1 # Functions with scores beyond 15 are typically flagged as potentially problematic (empirically) - - key: readability-function-cognitive-complexity.Threshold - value: 15 # default: 25 - - key: readability-identifier-length.MinimumVariableNameLength - value: 1 - - key: readability-identifier-length.MinimumParameterNameLength - value: 1 + - key: readability-function-cognitive-complexity.Threshold + value: 15 # default: 25 + - key: readability-identifier-length.MinimumVariableNameLength + value: 1 + - key: readability-identifier-length.MinimumParameterNameLength + value: 1 - key: misc-include-cleaner.IgnoreHeaders value: '(__chrono/.*|stdlib\.h|3rdparty/.*)' diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 64cd165d7..782fd3c01 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -352,51 +352,20 @@ jobs: - name: Build project run: | cmake --build build --parallel - - name: Run tests (MPI) - run: scripts/run_tests.py --running-type="processes" --additional-mpi-args="--oversubscribe" - env: - PPC_NUM_PROC: 2 - PPC_NUM_THREADS: 2 - OMPI_ALLOW_RUN_AS_ROOT: 1 - OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 - LLVM_PROFILE_FILE: "coverage-%p-%m.profraw" - - name: Run tests (threads) - run: scripts/run_tests.py --running-type="threads" --counts 1 2 3 4 - env: - PPC_NUM_PROC: 1 - LLVM_PROFILE_FILE: "coverage-%p-%m.profraw" - - name: Generate LLVM Coverage Data - run: | - mkdir cov-report - cd build - # Merge all raw profiles into a single indexed profile - llvm-profdata-20 merge -sparse $(find . -name "*.profraw") -o coverage.profdata - # Find all test executables - BINARIES=$(find bin -type f -executable | tr '\n' ' ') - # Generate coverage report in LCOV format for Codecov - llvm-cov-20 export \ - $BINARIES \ - --format=lcov \ - --ignore-filename-regex='.*3rdparty/.*|/usr/.*|.*tests/.*|.*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|.*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp' \ - --instr-profile=coverage.profdata \ - > ../coverage.lcov - # Generate HTML report - llvm-cov-20 show \ - $BINARIES \ - --format=html \ - --output-dir=../cov-report \ - --ignore-filename-regex='.*3rdparty/.*|/usr/.*|.*tests/.*|.*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|.*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp' \ - --instr-profile=coverage.profdata + - name: Run tests and generate coverage + run: > + python3 scripts/run_tests.py --running-type processes_coverage + --additional-mpi-args "--oversubscribe" --llvm-version 20 - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5.4.3 with: - files: coverage.lcov + files: build/coverage/coverage.lcov - name: Upload coverage report artifact id: upload-cov uses: actions/upload-artifact@v4 with: name: cov-report - path: 'cov-report' + path: 'build/coverage/html' - name: Comment coverage report link # TODO: Support PRs from forks and handle cases with insufficient write permissions continue-on-error: true diff --git a/scripts/generate_llvm_coverage.py b/scripts/generate_llvm_coverage.py deleted file mode 100755 index 058804375..000000000 --- a/scripts/generate_llvm_coverage.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python3 -"""Generate LLVM coverage report for the project.""" - -import os -import subprocess -import sys -import glob -import argparse - - -def run_command(cmd, cwd=None): - """Run a command and return its output.""" - print(f"Running: {' '.join(cmd)}") - result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) - if result.returncode != 0: - print(f"Error: {result.stderr}") - sys.exit(1) - return result.stdout - - -def main(): - parser = argparse.ArgumentParser(description="Generate LLVM coverage report") - parser.add_argument("--build-dir", default="build", help="Build directory") - parser.add_argument("--output-dir", default="coverage", help="Output directory for coverage report") - parser.add_argument("--llvm-version", default="20", help="LLVM version (default: 20)") - args = parser.parse_args() - - build_dir = os.path.abspath(args.build_dir) - output_dir = os.path.abspath(args.output_dir) - - # Try to find LLVM tools in various locations - llvm_profdata = None - llvm_cov = None - - # List of possible LLVM tool names - if args.llvm_version: - profdata_names = [f"llvm-profdata-{args.llvm_version}", "llvm-profdata"] - cov_names = [f"llvm-cov-{args.llvm_version}", "llvm-cov"] - else: - profdata_names = ["llvm-profdata"] - cov_names = ["llvm-cov"] - - # Try to find the tools - for name in profdata_names: - result = subprocess.run(["which", name], capture_output=True, text=True) - if result.returncode == 0: - llvm_profdata = name - break - - for name in cov_names: - result = subprocess.run(["which", name], capture_output=True, text=True) - if result.returncode == 0: - llvm_cov = name - break - - if not llvm_profdata or not llvm_cov: - print("Error: Could not find llvm-profdata or llvm-cov in PATH") - print("Make sure LLVM tools are installed and in your PATH") - sys.exit(1) - - if not os.path.exists(build_dir): - print(f"Error: Build directory {build_dir} does not exist") - sys.exit(1) - - # Create output directory - os.makedirs(output_dir, exist_ok=True) - - # Find all .profraw files - profraw_files = glob.glob(os.path.join(build_dir, "**", "*.profraw"), recursive=True) - if not profraw_files: - print("No .profraw files found. Make sure to run tests with LLVM_PROFILE_FILE set.") - print("Example: LLVM_PROFILE_FILE='coverage-%p-%m.profraw' ./your_test") - sys.exit(1) - - print(f"Found {len(profraw_files)} .profraw files") - - # Merge profiles - profdata_file = os.path.join(output_dir, "coverage.profdata") - run_command([llvm_profdata, "merge", "-sparse"] + profraw_files + ["-o", profdata_file]) - print(f"Created merged profile: {profdata_file}") - - # Find all executables in bin directory - bin_dir = os.path.join(build_dir, "bin") - if not os.path.exists(bin_dir): - print(f"Error: Bin directory {bin_dir} does not exist") - sys.exit(1) - - executables = [] - for root, dirs, files in os.walk(bin_dir): - for file in files: - filepath = os.path.join(root, file) - if os.access(filepath, os.X_OK) and not file.endswith('.txt'): - executables.append(filepath) - - if not executables: - print("No executables found in bin directory") - sys.exit(1) - - print(f"Found {len(executables)} executables") - - # Get the project root directory (parent of build dir) - project_root = os.path.dirname(build_dir) - - # Generate LCOV report - lcov_file = os.path.join(output_dir, "coverage.lcov") - cmd = [llvm_cov, "export"] + executables + [ - "--format=lcov", - "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tests/.*|" - ".*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" - ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", - f"--instr-profile={profdata_file}" - ] - - with open(lcov_file, "w") as f: - result = subprocess.run(cmd, stdout=f, stderr=subprocess.PIPE, text=True) - if result.returncode != 0: - print(f"Error generating LCOV report: {result.stderr}") - sys.exit(1) - - print(f"Generated LCOV report: {lcov_file}") - - # Post-process LCOV file to use relative paths - with open(lcov_file, 'r') as f: - lcov_content = f.read() - - # Replace absolute paths with relative paths - lcov_content = lcov_content.replace(project_root + '/', '') - - with open(lcov_file, 'w') as f: - f.write(lcov_content) - - print("Post-processed LCOV file to use relative paths") - - # Generate HTML report - html_dir = os.path.join(output_dir, "html") - cmd = [llvm_cov, "show"] + executables + [ - "--format=html", - f"--output-dir={html_dir}", - "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tests/.*|" - ".*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" - ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", - f"--instr-profile={profdata_file}" - ] - - run_command(cmd) - print(f"Generated HTML report: {html_dir}/index.html") - - # Generate summary - cmd = [llvm_cov, "report"] + executables + [ - f"--instr-profile={profdata_file}", - "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tasks/.*/tests/.*|.*modules/.*/tests/.*|" - ".*tasks/common/runners/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" - ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp" - ] - - summary = run_command(cmd) - print("\nCoverage Summary:") - print(summary) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 473060490..af08d009c 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -14,8 +14,9 @@ def init_cmd_args(): parser.add_argument( "--running-type", required=True, - choices=["threads", "processes", "performance"], - help="Specify the execution mode. Choose 'threads' for multithreading or 'processes' for multiprocessing.", + choices=["threads", "processes", "performance", "processes_coverage"], + help="Specify the execution mode. Choose 'threads' for multithreading, " + "'processes' for multiprocessing, or 'processes_coverage' for coverage generation.", ) parser.add_argument( "--additional-mpi-args", @@ -32,6 +33,11 @@ def init_cmd_args(): parser.add_argument( "--verbose", action="store_true", help="Print commands executed by the script" ) + parser.add_argument( + "--llvm-version", + default="20", + help="LLVM version for coverage tools (default: 20)", + ) args = parser.parse_args() _args_dict = vars(args) return _args_dict @@ -161,18 +167,162 @@ def run_performance(self): + self.__get_gtest_settings(1, "_" + task_type) ) + def generate_coverage(self, llvm_version="20"): + """Generate LLVM coverage report after running tests.""" + + # Find llvm-profdata and llvm-cov + if llvm_version: + profdata_names = [f"llvm-profdata-{llvm_version}", "llvm-profdata"] + cov_names = [f"llvm-cov-{llvm_version}", "llvm-cov"] + else: + profdata_names = ["llvm-profdata"] + cov_names = ["llvm-cov"] + + llvm_profdata = None + llvm_cov = None + + for name in profdata_names: + result = subprocess.run(["which", name], capture_output=True, text=True) + if result.returncode == 0: + llvm_profdata = name + break + + for name in cov_names: + result = subprocess.run(["which", name], capture_output=True, text=True) + if result.returncode == 0: + llvm_cov = name + break + + if not llvm_profdata or not llvm_cov: + raise Exception("Could not find llvm-profdata or llvm-cov in PATH") + + build_dir = self.work_dir.parent + output_dir = build_dir / "coverage" + output_dir.mkdir(exist_ok=True) + + # Find all .profraw files + profraw_files = list(build_dir.glob("**/*.profraw")) + if not profraw_files: + raise Exception( + "No .profraw files found. Make sure to run tests with LLVM_PROFILE_FILE set." + ) + + print(f"Found {len(profraw_files)} .profraw files") + + # Merge profiles + profdata_file = output_dir / "coverage.profdata" + cmd = ( + [llvm_profdata, "merge", "-sparse"] + + [str(f) for f in profraw_files] + + ["-o", str(profdata_file)] + ) + if self.verbose: + print("Executing:", " ".join(shlex.quote(part) for part in cmd)) + result = subprocess.run(cmd) + if result.returncode != 0: + raise Exception("Failed to merge coverage profiles") + + # Find executables + executables = [] + for f in self.work_dir.iterdir(): + if f.is_file() and os.access(f, os.X_OK) and not f.suffix == ".txt": + executables.append(str(f)) + + if not executables: + raise Exception("No executables found in bin directory") + + print(f"Found {len(executables)} executables") + + # Get project root + project_root = build_dir.parent + + # Generate LCOV report + lcov_file = output_dir / "coverage.lcov" + cmd = ( + [llvm_cov, "export"] + + executables + + [ + "--format=lcov", + "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tests/.*|" + ".*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" + ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", + f"--instr-profile={profdata_file}", + ] + ) + + with open(lcov_file, "w") as f: + result = subprocess.run(cmd, stdout=f, stderr=subprocess.PIPE, text=True) + if result.returncode != 0: + raise Exception(f"Error generating LCOV report: {result.stderr}") + + # Post-process LCOV file to use relative paths + with open(lcov_file, "r") as f: + lcov_content = f.read() + + lcov_content = lcov_content.replace(str(project_root) + "/", "") + + with open(lcov_file, "w") as f: + f.write(lcov_content) + + print(f"Generated LCOV report: {lcov_file}") + + # Generate HTML report + html_dir = output_dir / "html" + cmd = ( + [llvm_cov, "show"] + + executables + + [ + "--format=html", + f"--output-dir={html_dir}", + "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tests/.*|" + ".*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" + ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", + f"--instr-profile={profdata_file}", + ] + ) + + if self.verbose: + print("Executing:", " ".join(shlex.quote(part) for part in cmd)) + result = subprocess.run(cmd) + if result.returncode != 0: + raise Exception("Failed to generate HTML coverage report") + + print(f"Generated HTML report: {html_dir}/index.html") + + # Generate summary + cmd = ( + [llvm_cov, "report"] + + executables + + [ + f"--instr-profile={profdata_file}", + "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tasks/.*/tests/.*|.*modules/.*/tests/.*|" + ".*tasks/common/runners/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" + ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", + ] + ) + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode == 0: + print("\nCoverage Summary:") + print(result.stdout) + def _execute(args_dict, env): runner = PPCRunner(verbose=args_dict.get("verbose", False)) runner.setup_env(env) - if args_dict["running_type"] in ["threads", "processes"]: + if args_dict["running_type"] in ["threads", "processes", "processes_coverage"]: runner.run_core() if args_dict["running_type"] == "threads": runner.run_threads() elif args_dict["running_type"] == "processes": runner.run_processes(args_dict["additional_mpi_args"]) + elif args_dict["running_type"] == "processes_coverage": + # Run processes tests + runner.run_processes(args_dict["additional_mpi_args"]) + # Generate coverage report + runner.generate_coverage(args_dict.get("llvm_version", "20")) elif args_dict["running_type"] == "performance": runner.run_performance() else: @@ -183,7 +333,33 @@ def _execute(args_dict, env): args_dict = init_cmd_args() counts = args_dict.get("counts") - if counts: + if args_dict["running_type"] == "processes_coverage": + # For coverage, set environment variables for profiling + env_copy = os.environ.copy() + env_copy["LLVM_PROFILE_FILE"] = "coverage-%p-%m.profraw" + env_copy["PPC_NUM_PROC"] = "2" + env_copy["PPC_NUM_THREADS"] = "2" + env_copy["OMPI_ALLOW_RUN_AS_ROOT"] = "1" + env_copy["OMPI_ALLOW_RUN_AS_ROOT_CONFIRM"] = "1" + + # Run threads tests with different counts + print("Running thread tests with coverage...") + threads_args = args_dict.copy() + threads_args["running_type"] = "threads" + for count in [1, 2, 3, 4]: + env_threads = env_copy.copy() + env_threads["PPC_NUM_THREADS"] = str(count) + env_threads["PPC_NUM_PROC"] = "1" + print(f"Executing with threads count: {count}") + try: + _execute(threads_args, env_threads) + except Exception as e: + print(f"Warning: Thread tests with count {count} failed: {e}") + + # Now run the process coverage tests + print("\nRunning process tests with coverage...") + _execute(args_dict, env_copy) + elif counts: for count in counts: env_copy = os.environ.copy() From c13b506d065d69b7a0a78b317af0dbf5e606a3c2 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 30 Jul 2025 14:38:59 +0200 Subject: [PATCH 04/11] refactor test script to streamline processes_coverage mode and update CI configuration for coverage run --- .github/workflows/ubuntu.yml | 12 +++++++++--- scripts/run_tests.py | 31 +++---------------------------- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 782fd3c01..67757201f 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -323,9 +323,9 @@ jobs: PPC_NUM_PROC: 1 PPC_ASAN_RUN: 1 clang-build-codecov: - needs: - - gcc-test-extended - - clang-test-extended +# needs: +# - gcc-test-extended +# - clang-test-extended runs-on: ubuntu-24.04 container: image: ghcr.io/learning-process/ppc-ubuntu:latest @@ -356,6 +356,12 @@ jobs: run: > python3 scripts/run_tests.py --running-type processes_coverage --additional-mpi-args "--oversubscribe" --llvm-version 20 + env: + LLVM_PROFILE_FILE: coverage-%p-%m.profraw + PPC_NUM_THREADS: 2 + PPC_NUM_PROC: 2 + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5.4.3 with: diff --git a/scripts/run_tests.py b/scripts/run_tests.py index af08d009c..5976e334c 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -319,7 +319,8 @@ def _execute(args_dict, env): elif args_dict["running_type"] == "processes": runner.run_processes(args_dict["additional_mpi_args"]) elif args_dict["running_type"] == "processes_coverage": - # Run processes tests + # Run both threads and processes tests, then generate coverage + runner.run_threads() runner.run_processes(args_dict["additional_mpi_args"]) # Generate coverage report runner.generate_coverage(args_dict.get("llvm_version", "20")) @@ -333,33 +334,7 @@ def _execute(args_dict, env): args_dict = init_cmd_args() counts = args_dict.get("counts") - if args_dict["running_type"] == "processes_coverage": - # For coverage, set environment variables for profiling - env_copy = os.environ.copy() - env_copy["LLVM_PROFILE_FILE"] = "coverage-%p-%m.profraw" - env_copy["PPC_NUM_PROC"] = "2" - env_copy["PPC_NUM_THREADS"] = "2" - env_copy["OMPI_ALLOW_RUN_AS_ROOT"] = "1" - env_copy["OMPI_ALLOW_RUN_AS_ROOT_CONFIRM"] = "1" - - # Run threads tests with different counts - print("Running thread tests with coverage...") - threads_args = args_dict.copy() - threads_args["running_type"] = "threads" - for count in [1, 2, 3, 4]: - env_threads = env_copy.copy() - env_threads["PPC_NUM_THREADS"] = str(count) - env_threads["PPC_NUM_PROC"] = "1" - print(f"Executing with threads count: {count}") - try: - _execute(threads_args, env_threads) - except Exception as e: - print(f"Warning: Thread tests with count {count} failed: {e}") - - # Now run the process coverage tests - print("\nRunning process tests with coverage...") - _execute(args_dict, env_copy) - elif counts: + if counts: for count in counts: env_copy = os.environ.copy() From 4f483dbaf60595c0ce2b6d719323be59c4760521 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 30 Jul 2025 15:05:52 +0200 Subject: [PATCH 05/11] update CI workflow: re-enable commented dependencies and fix script argument formatting --- .github/workflows/ubuntu.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 67757201f..2e96a93ad 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -323,9 +323,9 @@ jobs: PPC_NUM_PROC: 1 PPC_ASAN_RUN: 1 clang-build-codecov: -# needs: -# - gcc-test-extended -# - clang-test-extended + # needs: + # - gcc-test-extended + # - clang-test-extended runs-on: ubuntu-24.04 container: image: ghcr.io/learning-process/ppc-ubuntu:latest @@ -354,8 +354,8 @@ jobs: cmake --build build --parallel - name: Run tests and generate coverage run: > - python3 scripts/run_tests.py --running-type processes_coverage - --additional-mpi-args "--oversubscribe" --llvm-version 20 + python3 scripts/run_tests.py --running-type=processes_coverage + --additional-mpi-args="--oversubscribe" --llvm-version=20 env: LLVM_PROFILE_FILE: coverage-%p-%m.profraw PPC_NUM_THREADS: 2 From bb9e3d99c69a7670db2435dd77c945422ba43924 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 30 Jul 2025 15:16:38 +0200 Subject: [PATCH 06/11] extend test script to support PPC_DISABLE_VALGRIND flag and update CI workflow accordingly --- .github/workflows/ubuntu.yml | 1 + scripts/run_tests.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 2e96a93ad..794f10d52 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -362,6 +362,7 @@ jobs: PPC_NUM_PROC: 2 OMPI_ALLOW_RUN_AS_ROOT: 1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + PPC_DISABLE_VALGRIND: 1 - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5.4.3 with: diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 5976e334c..6a47158af 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -106,7 +106,11 @@ def __get_gtest_settings(repeats_count, type_task): return command def run_threads(self): - if platform.system() == "Linux" and not self.__ppc_env.get("PPC_ASAN_RUN"): + if ( + platform.system() == "Linux" + and not self.__ppc_env.get("PPC_ASAN_RUN") + and not self.__ppc_env.get("PPC_DISABLE_VALGRIND") + ): for task_type in ["seq", "stl"]: self.__run_exec( shlex.split(self.valgrind_cmd) @@ -121,7 +125,11 @@ def run_threads(self): ) def run_core(self): - if platform.system() == "Linux" and not self.__ppc_env.get("PPC_ASAN_RUN"): + if ( + platform.system() == "Linux" + and not self.__ppc_env.get("PPC_ASAN_RUN") + and not self.__ppc_env.get("PPC_DISABLE_VALGRIND") + ): self.__run_exec( shlex.split(self.valgrind_cmd) + [str(self.work_dir / "core_func_tests")] From 36bdef988e9b31f26c3f1fa155ced81dc68c7a8c Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 30 Jul 2025 16:24:34 +0200 Subject: [PATCH 07/11] update test script to improve .profraw file discovery and error resilience; adjust CI workflow for installation and updated coverage paths --- .github/workflows/ubuntu.yml | 8 ++++++-- scripts/run_tests.py | 26 +++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 794f10d52..6ff524692 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -349,9 +349,13 @@ jobs: -D CMAKE_C_COMPILER_LAUNCHER=ccache -D CMAKE_CXX_COMPILER_LAUNCHER=ccache -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_VERBOSE_MAKEFILE=ON -D USE_COVERAGE=ON + -D CMAKE_INSTALL_PREFIX=install - name: Build project run: | cmake --build build --parallel + - name: Install project + run: | + cmake --build build --target install - name: Run tests and generate coverage run: > python3 scripts/run_tests.py --running-type=processes_coverage @@ -366,13 +370,13 @@ jobs: - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5.4.3 with: - files: build/coverage/coverage.lcov + files: install/coverage/coverage.lcov - name: Upload coverage report artifact id: upload-cov uses: actions/upload-artifact@v4 with: name: cov-report - path: 'build/coverage/html' + path: 'install/coverage/html' - name: Comment coverage report link # TODO: Support PRs from forks and handle cases with insufficient write permissions continue-on-error: true diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 6a47158af..44315c0d2 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -208,14 +208,27 @@ def generate_coverage(self, llvm_version="20"): output_dir = build_dir / "coverage" output_dir.mkdir(exist_ok=True) + print(f"Looking for .profraw files in: {build_dir}") + print(f"Current working directory: {os.getcwd()}") + # Find all .profraw files - profraw_files = list(build_dir.glob("**/*.profraw")) + # First look in current directory (where tests are run) + cwd = Path.cwd() + profraw_files = list(cwd.glob("*.profraw")) + # Also look in build directory if different + if cwd != build_dir: + profraw_files.extend(list(build_dir.glob("*.profraw"))) + # Look recursively if still nothing found + if not profraw_files: + profraw_files = list(build_dir.glob("**/*.profraw")) if not profraw_files: raise Exception( "No .profraw files found. Make sure to run tests with LLVM_PROFILE_FILE set." ) print(f"Found {len(profraw_files)} .profraw files") + for f in profraw_files[:5]: # Show first 5 files + print(f" - {f}") # Merge profiles profdata_file = output_dir / "coverage.profdata" @@ -328,8 +341,15 @@ def _execute(args_dict, env): runner.run_processes(args_dict["additional_mpi_args"]) elif args_dict["running_type"] == "processes_coverage": # Run both threads and processes tests, then generate coverage - runner.run_threads() - runner.run_processes(args_dict["additional_mpi_args"]) + # Continue even if tests fail to generate coverage report + try: + runner.run_threads() + except Exception as e: + print(f"Warning: Thread tests failed: {e}") + try: + runner.run_processes(args_dict["additional_mpi_args"]) + except Exception as e: + print(f"Warning: Process tests failed: {e}") # Generate coverage report runner.generate_coverage(args_dict.get("llvm_version", "20")) elif args_dict["running_type"] == "performance": From e810acfe58cd419dbb9a69726b975a8ed45fdddc Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 30 Jul 2025 18:09:23 +0200 Subject: [PATCH 08/11] update test script: improve regex patterns for coverage exclusion paths --- scripts/run_tests.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 44315c0d2..a1799b08a 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -264,8 +264,9 @@ def generate_coverage(self, llvm_version="20"): + executables + [ "--format=lcov", - "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tests/.*|" - ".*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" + "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tasks/.*/tests/.*|" + ".*modules/.*/tests/.*|.*tasks/common/runners/.*|" + ".*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", f"--instr-profile={profdata_file}", ] @@ -295,8 +296,9 @@ def generate_coverage(self, llvm_version="20"): + [ "--format=html", f"--output-dir={html_dir}", - "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tests/.*|" - ".*tasks/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" + "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tasks/.*/tests/.*|" + ".*modules/.*/tests/.*|.*tasks/common/runners/.*|" + ".*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", f"--instr-profile={profdata_file}", ] @@ -316,8 +318,9 @@ def generate_coverage(self, llvm_version="20"): + executables + [ f"--instr-profile={profdata_file}", - "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tasks/.*/tests/.*|.*modules/.*/tests/.*|" - ".*tasks/common/runners/.*|.*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" + "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tasks/.*/tests/.*|" + ".*modules/.*/tests/.*|.*tasks/common/runners/.*|" + ".*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", ] ) From f52533df370cc1e070c550e14d3d2f8effd80757 Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 30 Jul 2025 18:16:44 +0200 Subject: [PATCH 09/11] update test script: extend coverage generation to include libraries alongside executables --- scripts/run_tests.py | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/scripts/run_tests.py b/scripts/run_tests.py index a1799b08a..9bc613f8c 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -243,16 +243,32 @@ def generate_coverage(self, llvm_version="20"): if result.returncode != 0: raise Exception("Failed to merge coverage profiles") - # Find executables - executables = [] + # Find executables and libraries with coverage + objects = [] + + # Add all executables from bin directory for f in self.work_dir.iterdir(): if f.is_file() and os.access(f, os.X_OK) and not f.suffix == ".txt": - executables.append(str(f)) - - if not executables: - raise Exception("No executables found in bin directory") - - print(f"Found {len(executables)} executables") + objects.append(str(f)) + + # Add all static libraries from arch directory + arch_dir = build_dir / "arch" + if arch_dir.exists(): + for f in arch_dir.glob("*.a"): + objects.append(str(f)) + + # Add all shared libraries from lib directory + lib_dir = build_dir / "lib" + if lib_dir.exists(): + for f in lib_dir.glob("*.so"): + objects.append(str(f)) + for f in lib_dir.glob("*.dylib"): + objects.append(str(f)) + + if not objects: + raise Exception("No executables or libraries found") + + print(f"Found {len(objects)} executables and libraries") # Get project root project_root = build_dir.parent @@ -261,7 +277,7 @@ def generate_coverage(self, llvm_version="20"): lcov_file = output_dir / "coverage.lcov" cmd = ( [llvm_cov, "export"] - + executables + + objects + [ "--format=lcov", "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tasks/.*/tests/.*|" @@ -292,7 +308,7 @@ def generate_coverage(self, llvm_version="20"): html_dir = output_dir / "html" cmd = ( [llvm_cov, "show"] - + executables + + objects + [ "--format=html", f"--output-dir={html_dir}", @@ -315,7 +331,7 @@ def generate_coverage(self, llvm_version="20"): # Generate summary cmd = ( [llvm_cov, "report"] - + executables + + objects + [ f"--instr-profile={profdata_file}", "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tasks/.*/tests/.*|" From 88fc055bd60f8d9eeedd50a4f9cf89b6d433e6fb Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 30 Jul 2025 18:19:50 +0200 Subject: [PATCH 10/11] remove unnecessary whitespace in test script cleanup during coverage object discovery --- scripts/run_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 9bc613f8c..282ce703d 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -245,18 +245,18 @@ def generate_coverage(self, llvm_version="20"): # Find executables and libraries with coverage objects = [] - + # Add all executables from bin directory for f in self.work_dir.iterdir(): if f.is_file() and os.access(f, os.X_OK) and not f.suffix == ".txt": objects.append(str(f)) - + # Add all static libraries from arch directory arch_dir = build_dir / "arch" if arch_dir.exists(): for f in arch_dir.glob("*.a"): objects.append(str(f)) - + # Add all shared libraries from lib directory lib_dir = build_dir / "lib" if lib_dir.exists(): From b55dd19b14a5854accbcf49ac1853b83cc525d8c Mon Sep 17 00:00:00 2001 From: Alexander Nesterov Date: Wed, 30 Jul 2025 22:22:03 +0200 Subject: [PATCH 11/11] remove unnecessary whitespace in test script cleanup during coverage object discovery --- .github/workflows/ubuntu.yml | 6 +-- scripts/run_tests.py | 86 +++++++++++++++++++++++------------- 2 files changed, 59 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 6ff524692..8206a9fe4 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -361,7 +361,7 @@ jobs: python3 scripts/run_tests.py --running-type=processes_coverage --additional-mpi-args="--oversubscribe" --llvm-version=20 env: - LLVM_PROFILE_FILE: coverage-%p-%m.profraw + LLVM_PROFILE_FILE: build/coverage-%p-%m.profraw PPC_NUM_THREADS: 2 PPC_NUM_PROC: 2 OMPI_ALLOW_RUN_AS_ROOT: 1 @@ -370,13 +370,13 @@ jobs: - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5.4.3 with: - files: install/coverage/coverage.lcov + files: build/coverage/coverage.lcov - name: Upload coverage report artifact id: upload-cov uses: actions/upload-artifact@v4 with: name: cov-report - path: 'install/coverage/html' + path: 'build/coverage/html' - name: Comment coverage report link # TODO: Support PRs from forks and handle cases with insufficient write permissions continue-on-error: true diff --git a/scripts/run_tests.py b/scripts/run_tests.py index 282ce703d..bdc271834 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -204,7 +204,8 @@ def generate_coverage(self, llvm_version="20"): if not llvm_profdata or not llvm_cov: raise Exception("Could not find llvm-profdata or llvm-cov in PATH") - build_dir = self.work_dir.parent + # Always use build directory for coverage + build_dir = Path(self.__get_project_path()) / "build" output_dir = build_dir / "coverage" output_dir.mkdir(exist_ok=True) @@ -212,12 +213,12 @@ def generate_coverage(self, llvm_version="20"): print(f"Current working directory: {os.getcwd()}") # Find all .profraw files - # First look in current directory (where tests are run) - cwd = Path.cwd() - profraw_files = list(cwd.glob("*.profraw")) - # Also look in build directory if different - if cwd != build_dir: - profraw_files.extend(list(build_dir.glob("*.profraw"))) + # First look in build directory + profraw_files = list(build_dir.glob("*.profraw")) + # Also look in current directory (for backward compatibility) + if not profraw_files: + cwd = Path.cwd() + profraw_files = list(cwd.glob("*.profraw")) # Look recursively if still nothing found if not profraw_files: profraw_files = list(build_dir.glob("**/*.profraw")) @@ -247,23 +248,31 @@ def generate_coverage(self, llvm_version="20"): objects = [] # Add all executables from bin directory - for f in self.work_dir.iterdir(): - if f.is_file() and os.access(f, os.X_OK) and not f.suffix == ".txt": - objects.append(str(f)) + bin_dir = build_dir / "bin" + if bin_dir.exists(): + for f in bin_dir.iterdir(): + if f.is_file() and os.access(f, os.X_OK) and not f.suffix == ".txt": + objects.append(str(f)) - # Add all static libraries from arch directory + # Add all static libraries from arch directory (excluding third-party) arch_dir = build_dir / "arch" if arch_dir.exists(): for f in arch_dir.glob("*.a"): - objects.append(str(f)) + # Skip third-party libraries + if "gtest" not in f.name and "gmock" not in f.name and "tbb" not in f.name: + objects.append(str(f)) - # Add all shared libraries from lib directory - lib_dir = build_dir / "lib" + # Add all shared libraries from lib directory (excluding third-party) + lib_dir = build_dir / "lib" if lib_dir.exists(): for f in lib_dir.glob("*.so"): - objects.append(str(f)) + # Skip third-party libraries + if "tbb" not in f.name: + objects.append(str(f)) for f in lib_dir.glob("*.dylib"): - objects.append(str(f)) + # Skip third-party libraries + if "tbb" not in f.name: + objects.append(str(f)) if not objects: raise Exception("No executables or libraries found") @@ -271,7 +280,7 @@ def generate_coverage(self, llvm_version="20"): print(f"Found {len(objects)} executables and libraries") # Get project root - project_root = build_dir.parent + project_root = Path(self.__get_project_path()) # Generate LCOV report lcov_file = output_dir / "coverage.lcov" @@ -306,27 +315,44 @@ def generate_coverage(self, llvm_version="20"): # Generate HTML report html_dir = output_dir / "html" - cmd = ( - [llvm_cov, "show"] - + objects - + [ + html_dir.mkdir(exist_ok=True) + + print("Generating HTML coverage report...") + + # Generate HTML report with all objects at once + # Use the first executable as the main binary and others as additional objects + if objects: + cmd = [ + llvm_cov, "show", + objects[0], # Main binary + ] + + # Add other objects with -object flag + for obj in objects[1:]: + cmd.extend(["-object", obj]) + + cmd.extend([ "--format=html", f"--output-dir={html_dir}", + "--show-line-counts-or-regions", + "--show-instantiations", "--ignore-filename-regex=.*3rdparty/.*|/usr/.*|.*tasks/.*/tests/.*|" ".*modules/.*/tests/.*|.*tasks/common/runners/.*|" ".*modules/runners/.*|.*modules/util/include/perf_test_util.hpp|" ".*modules/util/include/func_test_util.hpp|.*modules/util/src/func_test_util.cpp", f"--instr-profile={profdata_file}", - ] - ) + ]) - if self.verbose: - print("Executing:", " ".join(shlex.quote(part) for part in cmd)) - result = subprocess.run(cmd) - if result.returncode != 0: - raise Exception("Failed to generate HTML coverage report") - - print(f"Generated HTML report: {html_dir}/index.html") + if self.verbose: + print("Executing:", " ".join(shlex.quote(part) for part in cmd)) + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"Warning: HTML generation returned non-zero: {result.stderr}") + else: + print(f"Generated HTML report: {html_dir}/index.html") + else: + print("Error: No objects found for HTML generation") # Generate summary cmd = (