From e15597d3c0080f7f5e25f13dd5b5c70cb8a874c9 Mon Sep 17 00:00:00 2001 From: Luca Terracciano Date: Mon, 15 Sep 2025 14:37:08 +0200 Subject: [PATCH] ci: add github ci --- .build-tools/containers/README.md | 1 - .build-tools/containers/build.sh | 22 ---- .build-tools/containers/buildenv/Dockerfile | 22 ---- .build-tools/containers/buildenv/run.sh | 11 -- .build-tools/containers/deploy.sh | 24 ---- .build-tools/containers/remove.sh | 12 -- .build-tools/containers/run.sh | 24 ---- .github/workflows/deployr.yml | 60 --------- .github/workflows/master-test-workflow.yml | 61 +++++++++ .github/workflows/master.yml | 25 ++++ .github/workflows/pr-development-workflow.yml | 118 ++++++++++++++++++ .github/workflows/pr-development.yml | 25 ++++ .github/workflows/style.yml | 2 +- .gitlab-ci.yml | 54 -------- 14 files changed, 230 insertions(+), 231 deletions(-) delete mode 100644 .build-tools/containers/README.md delete mode 100755 .build-tools/containers/build.sh delete mode 100644 .build-tools/containers/buildenv/Dockerfile delete mode 100755 .build-tools/containers/buildenv/run.sh delete mode 100755 .build-tools/containers/deploy.sh delete mode 100755 .build-tools/containers/remove.sh delete mode 100755 .build-tools/containers/run.sh delete mode 100644 .github/workflows/deployr.yml create mode 100644 .github/workflows/master-test-workflow.yml create mode 100644 .github/workflows/master.yml create mode 100644 .github/workflows/pr-development-workflow.yml create mode 100644 .github/workflows/pr-development.yml delete mode 100644 .gitlab-ci.yml diff --git a/.build-tools/containers/README.md b/.build-tools/containers/README.md deleted file mode 100644 index c29d836..0000000 --- a/.build-tools/containers/README.md +++ /dev/null @@ -1 +0,0 @@ -Containers to run the CI/CD pipelines for the Taskr project diff --git a/.build-tools/containers/build.sh b/.build-tools/containers/build.sh deleted file mode 100755 index bc01183..0000000 --- a/.build-tools/containers/build.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -if command -v arch &>/dev/null; then - target_arch=$(arch) - - if [ $target_arch == "aarch64" ]; then - target_arch="arm64v8" - else - target_arch="x86_64" - fi -else - if [[ $# -ne 2 ]]; then - echo "arch not installed. Please provice manually the target architecture. Usage: $0 " - exit 1 - else - target_arch=${2} - fi -fi - -folder=${1} -echo "Building $folder for arch $target_arch" -docker build -t "registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/taskr/${folder}-${target_arch}:latest" --build-arg ARCH=${target_arch} ${folder} diff --git a/.build-tools/containers/buildenv/Dockerfile b/.build-tools/containers/buildenv/Dockerfile deleted file mode 100644 index d710822..0000000 --- a/.build-tools/containers/buildenv/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -ARG ARCH= -FROM registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/hicr/buildenv-${ARCH}:latest - -ENV USER=hicr -ENV HOME=/home/$USER - -USER $USER - -RUN sudo apt purge -y libopenblas-dev -RUN sudo apt purge -y liblapack-dev -RUN sudo apt purge -y liblapacke-dev -RUN sudo apt autoremove -y -RUN rm -rf /usr/share/doc/libopenblas /usr/lib/libopenblas* /usr/lib/pkconfig/openblas.pc -RUN git clone -b v0.3.28 --single-branch --depth 1 https://github.com/OpenMathLib/OpenBLAS.git $HOME/OpenBLAS && \ - cd $HOME/OpenBLAS && \ - sed -i " s/# USE_THREAD = 0/USE_THREAD=0/g " Makefile.rule && \ - sed -i " s/# USE_LOCKING = 1/USE_LOCKING=1/g " Makefile.rule && \ - make && \ - sudo make PREFIX=/usr/local install && \ - sudo rm -rf ${HOME}/OpenBLAS - -WORKDIR $HOME \ No newline at end of file diff --git a/.build-tools/containers/buildenv/run.sh b/.build-tools/containers/buildenv/run.sh deleted file mode 100755 index 51ae1b5..0000000 --- a/.build-tools/containers/buildenv/run.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -if [[ $# -ne 2 ]]; then - echo "Usage: $0 " - exit 1 -fi - -folder=${1} -arch=${2} - -docker run --name taskr --shm-size=1024M --privileged -td "registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/taskr/${folder}-${arch}:latest" bash diff --git a/.build-tools/containers/deploy.sh b/.build-tools/containers/deploy.sh deleted file mode 100755 index ca8f379..0000000 --- a/.build-tools/containers/deploy.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -if command -v arch &>/dev/null; then - target_arch=$(arch) - - if [ $target_arch == "aarch64" ]; then - target_arch="arm64v8" - else - target_arch="x86_64" - fi -else - if [[ $# -ne 2 ]]; then - echo "arch not installed. Please provice manually the target architecture. Usage: $0 " - exit 1 - else - arch=${2} - fi -fi - -folder=${1} -echo "Deploying $folder for arch $target_arch" - -docker login registry.gitlab.huaweirc.ch -docker push "registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/taskr/${folder}-${target_arch}:latest" diff --git a/.build-tools/containers/remove.sh b/.build-tools/containers/remove.sh deleted file mode 100755 index 18f5471..0000000 --- a/.build-tools/containers/remove.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -if [[ $# -ne 2 ]]; then - echo "Usage: $0 " - exit 1 -fi - -folder=${1} -arch=${2} - -docker rmi -f "registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/taskr/${folder}-${arch}:latest" - diff --git a/.build-tools/containers/run.sh b/.build-tools/containers/run.sh deleted file mode 100755 index 8ed89c1..0000000 --- a/.build-tools/containers/run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -if command -v arch &>/dev/null; then - target_arch=$(arch) - - if [ $target_arch == "aarch64" ]; then - target_arch="arm64v8" - else - target_arch="x86_64" - fi -else - if [[ $# -ne 2 ]]; then - echo "arch not installed. Please provice manually the target architecture. Usage: $0 " - exit 1 - else - arch=${2} - fi -fi - -folder=${1} -echo "Running $folder for arch $target_arch" - -build_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -$build_dir/$folder/run.sh ${folder} ${target_arch} diff --git a/.github/workflows/deployr.yml b/.github/workflows/deployr.yml deleted file mode 100644 index a3d357e..0000000 --- a/.github/workflows/deployr.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Build and Run Tests - -on: - pull_request: - branches: [ "master" ] - push: - branches: [ "master" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - - # Build HiCR and run tests - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Updating Apt - run: sudo apt update - - name: Installing apt packages - run: | - sudo apt install -y build-essential - sudo apt install -y git - sudo apt install -y libgtest-dev - sudo apt install -y libhwloc-dev - sudo apt install -y libopenblas-dev - sudo apt install -y liblapack-dev - sudo apt install -y liblapacke-dev - sudo apt install -y libboost-context-dev - sudo apt install -y curl - sudo apt install -y jq - sudo apt install -y pkgconf - sudo apt install -y wget - sudo apt install -y sudo - sudo apt install -y libopenmpi-dev - sudo apt install -y cmake - sudo apt install -y libstb-dev - sudo apt install -y libsfml-dev - sudo apt install -y libtclap-dev - - name: Installing meson, ninja and gcovr - run: python3 -m pip install meson ninja gcovr - - name: Updating submodules - run: git submodule update --init --recursive - - name: Building all modules - run: | - echo "Building..." - mkdir build - meson setup build -Dengines=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true - meson compile -C build - - name: Running tests - run: | - echo "Running Tests..." - meson test -C build - - uses: actions/upload-artifact@v4 - if: always() - with: - name: meson-logs - path: build/meson-logs/ \ No newline at end of file diff --git a/.github/workflows/master-test-workflow.yml b/.github/workflows/master-test-workflow.yml new file mode 100644 index 0000000..a114990 --- /dev/null +++ b/.github/workflows/master-test-workflow.yml @@ -0,0 +1,61 @@ +name: Merge into master - Run Tests Workflow + +# if statements modified to avoid: https://stackoverflow.com/questions/69354003/github-action-job-fire-when-previous-job-skipped + +on: + workflow_call: + inputs: + os: + required: true + type: string + arch: + required: true + type: string + +permissions: + contents: read + +env: + REGISTRY: ghcr.io + DOCKERIMAGE: ghcr.io/algebraic-programming/hicr/buildenv + +defaults: + run: + shell: bash + +jobs: + # Build HiCR and run tests and the remote image + compile-and-test: + runs-on: ${{ inputs.os }} + container: + image: ghcr.io/algebraic-programming/hicr/buildenv:latest + options: --user hicr + credentials: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup + run: source /home/hicr/.bashrc && meson setup build -Dbackends=hwloc,pthreads,mpi,lpf,nosv,boost,opencl,acl -Dfrontends=channel,RPCEngine,tasking,objectStore -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true + + - name: Compile + run: source /home/hicr/.bashrc && meson compile -C build + + - name: Running tests and creating coverage report + shell: bash + run: | + echo "Running Tests..." + source /home/hicr/.bashrc + meson setup build --wipe -Db_coverage=true -Dbackends=hwloc,pthreads,mpi,nosv,boost -Dfrontends=channel,RPCEngine,tasking,objectStore -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true + meson compile -C build + meson test -C build + echo "Creating coverage report..." + ninja -C build coverage + - uses: actions/upload-artifact@v4 + if: always() + with: + name: meson-logs-${{ inputs.arch }} + path: build/meson-logs/ diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml new file mode 100644 index 0000000..6226b92 --- /dev/null +++ b/.github/workflows/master.yml @@ -0,0 +1,25 @@ +name: Merge into master - Build and Run Tests + +# if statements modified to avoid: https://stackoverflow.com/questions/69354003/github-action-job-fire-when-previous-job-skipped + +on: + push: + branches: ["master"] + +env: + REGISTRY: ghcr.io + DOCKERIMAGE: ghcr.io/algebraic-programming/hicr/buildenv + +jobs: + compile-and-test-arm64: + uses: Algebraic-Programming/DeployR/.github/workflows/master-test-workflow.yml@master + with: + os: ubuntu-24.04-arm + arch: arm64 + + compile-and-test-amd64: + uses: Algebraic-Programming/DeployR/.github/workflows/master-test-workflow.yml@master + with: + os: ubuntu-24.04 + arch: amd64 + \ No newline at end of file diff --git a/.github/workflows/pr-development-workflow.yml b/.github/workflows/pr-development-workflow.yml new file mode 100644 index 0000000..7d671fd --- /dev/null +++ b/.github/workflows/pr-development-workflow.yml @@ -0,0 +1,118 @@ +name: PR development - Build and Run Tests Workflow + +# if statements modified to avoid: https://stackoverflow.com/questions/69354003/github-action-job-fire-when-previous-job-skipped + +on: + workflow_call: + inputs: + os: + required: true + type: string + arch: + required: true + type: string + +permissions: + contents: read + +env: + REGISTRY: ghcr.io + DOCKERIMAGE: ghcr.io/algebraic-programming/hicr/buildenv + +defaults: + run: + shell: bash + +jobs: + # Build HiCR and run tests and the remote image + compile-and-test-standard: + runs-on: ${{ inputs.os }} + container: + image: ghcr.io/algebraic-programming/hicr/buildenv:latest + options: --user hicr + credentials: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup + run: source /home/hicr/.bashrc && meson setup build -Dbackends=hwloc,pthreads,mpi,lpf,nosv,boost,opencl,acl -Dfrontends=channel,RPCEngine,tasking,objectStore -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true + + - name: Compile + run: source /home/hicr/.bashrc && meson compile -C build + + - name: Running tests and creating coverage report + shell: bash + run: | + echo "Running Tests..." + source /home/hicr/.bashrc + meson setup build --wipe -Db_coverage=true -Dbackends=hwloc,pthreads,mpi,nosv,boost -Dfrontends=channel,RPCEngine,tasking,objectStore -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true + meson compile -C build + meson test -C build + echo "Creating coverage report..." + ninja -C build coverage + - uses: actions/upload-artifact@v4 + with: + name: build-standard-${{ inputs.arch }} + path: build/ + + analyze-coverage-report: + runs-on: ubuntu-latest + needs: [compile-and-test-standard] + permissions: + pull-requests: write + if: | + always() && + (contains(needs.compile-and-test-standard.result, 'success')) + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Create temp build folder + run: | + mkdir ${{ runner.temp }}/build + - name: Download standard coverage file + if: ${{ needs.compile-and-test-standard.result == 'success' }} + uses: actions/download-artifact@v4 + with: + name: build-standard-${{ inputs.arch }} + path: ${{ runner.temp }}/build + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Copy build folder in repo + run: | + mv ${{ runner.temp }}/build . + + - name: Code Coverage Report + uses: irongut/CodeCoverageSummary@v1.3.0 + with: + filename: build/meson-logs/coverage.xml + badge: true + fail_below_min: true + format: markdown + hide_branch_rate: false + hide_complexity: true + indicators: true + output: both + thresholds: "60 80" + + - name: Add arch to coverage file + run: | + sed -i '1i # Coverage ${{ inputs.arch }}' code-coverage-results.md + + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + if: github.event_name == 'pull_request' || github.event_name == 'pull_request' + with: + header: ${{ inputs.arch }} + message: | + Coverage for ${{ inputs.arch }} + recreate: true + path: code-coverage-results.md + - uses: actions/upload-artifact@v4 + with: + name: meson-logs-${{ inputs.arch }} + path: build/meson-logs/ diff --git a/.github/workflows/pr-development.yml b/.github/workflows/pr-development.yml new file mode 100644 index 0000000..d9ecf69 --- /dev/null +++ b/.github/workflows/pr-development.yml @@ -0,0 +1,25 @@ +name: PR development - Build and Run Tests + +# if statements modified to avoid: https://stackoverflow.com/questions/69354003/github-action-job-fire-when-previous-job-skipped + +on: + pull_request: + branches: ["master"] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + +jobs: + ci-arm64: + uses: Algebraic-Programming/DeployR/.github/workflows/pr-development-workflow.yml@master + with: + os: ubuntu-24.04-arm + arch: arm64 + + ci-amd64: + uses: Algebraic-Programming/DeployR/.github/workflows/pr-development-workflow.yml@master + with: + os: ubuntu-24.04 + arch: amd64 \ No newline at end of file diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 34dc504..4c3c88a 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -26,4 +26,4 @@ jobs: echo "Checking HiCR source and test formatting..." .build-tools/style/check-style.sh check include .build-tools/style/check-style.sh check tests - .build-tools/style/check-style.sh check examples \ No newline at end of file + .build-tools/style/check-style.sh check examples diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index cbd236e..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,54 +0,0 @@ -build: - image: registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/taskr/buildenv-x86_64:latest - variables: - GIT_SUBMODULE_STRATEGY: recursive - tags: - - docker - - x86 - - infiniband - script: - - export HOME=/home/hicr - - source /home/hicr/.hicr-env.sh - - echo "Compiling OVNI..." - - mkdir extern/tracr/extern/ovni/build; pushd extern/tracr/extern/ovni/build; cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTING=FALSE; make -j24; sudo make install; popd - - echo "Building TaskR..." - - mkdir build - - meson setup build -Dbuildtype=debug -Db_coverage=true -DbuildTests=true -DbuildExamples=true -DdistributedEngine=mpi -DbuildInstrumentation=true -DcompileWarningsAsErrors=true - - meson compile -C build - - echo "Running tests..." - - meson test -C build - - echo "Creating coverage report..." - - ninja -C build coverage - coverage: /^\s*lines:\s*\d+.\d+\%/ - artifacts: - name: ${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHA} - expire_in: 2 days - when: always - paths: - - build/meson-logs/* - reports: - coverage_report: - coverage_format: cobertura - path: build/meson-logs/coverage.xml - only: - - main - - merge_requests - - tags - -docs: - image: registry.gitlab.huaweirc.ch/zrc-von-neumann-lab/runtime-system-innovations/hicr/docs:latest - tags: - - docker - - tiny - script: - - echo "Checking TaskR source and test formatting..." - - .build-tools/style/check-style.sh check include - - .build-tools/style/check-style.sh check tests - - .build-tools/style/check-style.sh check examples - - echo "Building code documentation..." - - make -j1 -C docs - only: - - main - - merge_requests - - tags -