diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 820d51404e..3510615faf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,6 @@ jobs: macos_unit_tests: needs: unit_tests uses: ./.github/workflows/macos_unit_tests.yml - valgrind_tests: - needs: unit_tests - uses: ./.github/workflows/valgrind.yml static_check: needs: unit_tests uses: ./.github/workflows/job-static-check.yml diff --git a/.github/workflows/valgrind.sh b/.github/workflows/valgrind.sh deleted file mode 100644 index 73e7a48dd0..0000000000 --- a/.github/workflows/valgrind.sh +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env bash -set -e - -function print_ps { - set +e - echo "CFEngine processes:" - ps aux | grep [c]f- - - echo "Valgrind processes:" - ps aux | grep [v]algrind - set -e -} - -function no_errors { - set +e - grep -i "error" $1 - grep -i "error" $1 && exit 1 - set -e -} - -function check_daemon_output { - echo "Examining $1:" - no_errors $1 -} - -function check_output { - set -e - if [ ! -f "$1" ]; then - echo "$1 does not exists!" - exit 1 - fi - echo "Looking for problems in $1:" - grep -i "ERROR SUMMARY: 0 errors" "$1" - cat $1 | sed -e "/ 0 errors/d" -e "/and suppressed error/d" -e "/a memory error detector/d" -e "/This database contains unknown binary data/d" > filtered.txt - no_errors filtered.txt - set +e - grep -i "at 0x" filtered.txt - grep -i "at 0x" filtered.txt && exit 1 - grep -i "by 0x" filtered.txt - grep -i "by 0x" filtered.txt && exit 1 - grep -i "Failed to connect" filtered.txt - grep -i "Failed to connect" filtered.txt && exit 1 - set -e -} - -function check_serverd_valgrind_output { - if [ ! -f "$1" ]; then - echo "$1 does not exists!" - exit 1 - fi - set -e - echo "Serverd has 1 expected valgrind error because of old glibc" - echo "Because of this we use special assertions on output" - echo "Looking for problems in $1:" - grep -i "definitely lost" $1 - grep -i "indirectly lost" $1 - grep -i "ERROR SUMMARY" $1 - grep -i "definitely lost: 0 bytes in 0 blocks" $1 - grep -i "indirectly lost: 0 bytes in 0 blocks" $1 - grep -i "ERROR SUMMARY: 0 errors" "$1" - - cat $1 | sed -e "/ERROR SUMMARY/d" -e "/and suppressed error/d" -e "/a memory error detector/d" > filtered.txt - - no_errors filtered.txt - set +e - grep -i "at 0x" filtered.txt - grep -i "at 0x" filtered.txt && exit 1 - grep -i "by 0x" filtered.txt - grep -i "by 0x" filtered.txt && exit 1 - set -e -} - -function check_masterfiles_and_inputs { - set -e - echo "Comparing promises.cf from inputs and masterfiles:" - diff /var/cfengine/inputs/promises.cf /var/cfengine/masterfiles/promises.cf -} - -VG_OPTS="--leak-check=full --track-origins=yes --error-exitcode=1" -BOOTSTRAP_IP="$(ifconfig | grep -C1 Ethernet | sed 's/.*inet \([0-9.]*\).*/\1/;t;d')" - -valgrind $VG_OPTS /var/cfengine/bin/cf-key 2>&1 | tee cf-key.txt -check_output cf-key.txt -valgrind $VG_OPTS /var/cfengine/bin/cf-agent -B $BOOTSTRAP_IP 2>&1 | tee bootstrap.txt -check_output bootstrap.txt - -# Validate all databases here, because later, we cannot validate -# cf_lastseen.lmdb: -echo "Running cf-check diagnose --validate on all databases:" -valgrind $VG_OPTS /var/cfengine/bin/cf-check diagnose --validate 2>&1 | tee cf_check_validate_all.txt -check_output cf_check_validate_all.txt - -check_masterfiles_and_inputs - -print_ps - -echo "Stopping service to relaunch under valgrind" -systemctl stop cfengine3 -sleep 10 -print_ps - -# The IP we bootstrapped to cannot actually be used for communication. -# This ensures that cf-serverd binds to loopback interface, and cf-net -# connects to it: -echo "127.0.0.1" > /var/cfengine/policy_server.dat - -echo "Starting cf-serverd with valgrind in background:" -valgrind $VG_OPTS --log-file=serverd.txt /var/cfengine/bin/cf-serverd --no-fork 2>&1 > serverd_output.txt & -server_pid="$!" -sleep 20 - -echo "Starting cf-execd with valgrind in background:" -valgrind $VG_OPTS --log-file=execd.txt /var/cfengine/bin/cf-execd --no-fork 2>&1 > execd_output.txt & -exec_pid="$!" -sleep 10 - -print_ps - -echo "Running cf-net:" -valgrind $VG_OPTS /var/cfengine/bin/cf-net GET /var/cfengine/masterfiles/promises.cf 2>&1 | tee get.txt -check_output get.txt - -echo "Checking promises.cf diff (from cf-net GET):" -diff ./promises.cf /var/cfengine/masterfiles/promises.cf - -echo "Running update.cf:" -valgrind $VG_OPTS /var/cfengine/bin/cf-agent -K -f update.cf 2>&1 | tee update.txt -check_output update.txt -check_masterfiles_and_inputs -echo "Running update.cf without local copy:" -valgrind $VG_OPTS /var/cfengine/bin/cf-agent -K -f update.cf -D mpf_skip_local_copy_optimization 2>&1 | tee update2.txt -check_output update2.txt -check_masterfiles_and_inputs -echo "Running promises.cf:" -valgrind $VG_OPTS /var/cfengine/bin/cf-agent -K -f promises.cf 2>&1 | tee promises.txt -check_output promises.txt - -# Dump all databases, use grep to filter the JSON lines -# (optional whitespace then double quote or curly brackets). -# Some of the databases have strings containing "error" -# which check_output greps for. -echo "Running cf-check dump:" -valgrind $VG_OPTS /var/cfengine/bin/cf-check dump 2>&1 | grep -E '\s*[{}"]' --invert-match | tee cf_check_dump.txt -check_output cf_check_dump.txt - -echo "Running cf-check diagnose on all databases" -valgrind $VG_OPTS /var/cfengine/bin/cf-check diagnose 2>&1 | tee cf_check_diagnose.txt -check_output cf_check_diagnose.txt - -# Because of the hack with bootstrap IP / policy_server.dat above -# lastseen would not pass validation: -echo "Running cf-check diagnose --validate on all databases except cf_lastseen.lmdb:" -find /var/cfengine/state -name '*.lmdb' ! -name 'cf_lastseen.lmdb' -exec \ - valgrind $VG_OPTS /var/cfengine/bin/cf-check diagnose --validate {} + 2>&1 \ - | tee cf_check_validate_no_lastseen.txt -check_output cf_check_validate_no_lastseen.txt - -echo "Checking that bootstrap ID doesn't change" -/var/cfengine/bin/cf-agent --show-evaluated-vars | grep bootstrap_id > id_a -/var/cfengine/bin/cf-agent -K --show-evaluated-vars | grep bootstrap_id > id_b -cat id_a -diff id_a id_b - -echo "Checking that bootstrap ID has expected length" -[ `cat id_a | awk '{print $2}' | wc -c` -eq 41 ] - -print_ps - -echo "Checking that serverd and execd PIDs are still correct/alive:" -ps -p $exec_pid -ps -p $server_pid - -echo "Killing valgrind cf-execd" -kill $exec_pid -echo "Killing valgrind cf-serverd" -kill $server_pid -sleep 30 - -echo "Output from cf-execd in valgrind:" -cat execd.txt -check_output execd.txt -check_daemon_output execd_output.txt - -echo "Output from cf-serverd in valgrind:" -cat serverd.txt -check_serverd_valgrind_output serverd.txt -check_daemon_output serverd_output.txt - -echo "Stopping cfengine3 service" -systemctl stop cfengine3 - -echo "Done killing" -sleep 10 -print_ps - -echo "Check that bootstrap was successful" -grep "This host assumes the role of policy server" bootstrap.txt -grep "completed successfully!" bootstrap.txt - -echo "valgrind_health_check successful! (valgrind.sh)" diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml deleted file mode 100644 index 38b0e71b34..0000000000 --- a/.github/workflows/valgrind.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Valgrind Tests - -on: - workflow_call - -jobs: - valgrind_tests: - runs-on: ubuntu-24.04 - defaults: - run: - working-directory: core - steps: - - uses: actions/checkout@v3 - with: - path: core - submodules: recursive - - name: Get Togethers - uses: cfengine/together-javascript-action@main - id: together - with: - myToken: ${{ secrets.GITHUB_TOKEN }} - - name: Clone masterfiles (master) - uses: actions/checkout@v3 - with: - repository: cfengine/masterfiles - ref: ${{steps.together.outputs.masterfiles || github.base_ref || github.ref}} - path: masterfiles - submodules: recursive - - name: Install dependencies - run: | - ./ci/dependencies.sh - sudo apt install -y valgrind - - name: Run autotools / configure - run: ./autogen.sh --enable-debug --with-systemd-service - - name: Compile and link (make) - run: make -j8 CFLAGS="-Werror -Wall" - - name: Install CFEngine - run: sudo make install - - name: Generate masterfiles - run: ./autogen.sh - working-directory: masterfiles - - name: Install masterfiles - run: sudo make install - working-directory: masterfiles - - name: Reload systemd - run: sudo systemctl daemon-reload - - - name: See if cf-agent runs at all - run: /var/cfengine/bin/cf-agent --version - - - name: Run valgrind.sh - run: sudo bash .github/workflows/valgrind.sh - diff --git a/tests/valgrind-check/valgrind.sh b/tests/valgrind-check/valgrind.sh index 62d02a7fb1..d39b2a4d74 100644 --- a/tests/valgrind-check/valgrind.sh +++ b/tests/valgrind-check/valgrind.sh @@ -1,5 +1,24 @@ #!/bin/bash +# This test script builds and installs CFEngine, and runs our binaries +# with valgrind to look for memory leaks, and similar. +# +# It is normally run inside a container (see Containerfile next to it). +# If you want to run it without a container, you can. +# It assumes a few things: +# 1. Must be run from inside core (CWD) +# 2. Masterfiles must be at ../masterfiles +# 3. Normal build tools must be installed, and valgrind and diff +# 4. Must be run using sudo / as root, or have necessary permissions +# 5. CFEngine should not already be installed +# +# In order to run this in an Ubuntu VM, you can do something like this: +# sudo apt-get install -y build-essential git libtool autoconf automake bison flex libssl-dev libpcre2-dev libbison-dev libacl1 libacl1-dev lmdb-utils liblmdb-dev libpam0g-dev libtool libyaml-dev libxml2-dev librsync-dev valgrind +# git clone https://github.com/cfengine/masterfiles +# git clone --recursive https://github.com/cfengine/core +# cd core +# sudo bash tests/valgrind-check/valgrind.sh + function print_ps { set +e echo "CFEngine processes:"