diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index 02a19fc2..4cf5104c 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -94,6 +94,8 @@ pushd "${HOME}/bin" popd +export DREDD_CLANG_LLVM_DIR=$(pwd)/third_party/clang+llvm + case "$(uname)" in "Linux") @@ -120,17 +122,17 @@ esac mkdir -p build pushd build - cmake -G Ninja .. -DCMAKE_BUILD_TYPE="${CONFIG}" "${CMAKE_OPTIONS[@]}" + cmake -G Ninja .. -DCMAKE_BUILD_TYPE="${CONFIG}" -DDREDD_CLANG_LLVM_DIR="${DREDD_CLANG_LLVM_DIR}" "${CMAKE_OPTIONS[@]}" cmake --build . --config "${CONFIG}" cmake -DCMAKE_INSTALL_PREFIX=./install -DBUILD_TYPE="${CONFIG}" -P cmake_install.cmake # Run the unit tests ./src/libdreddtest/libdreddtest popd -cp build/src/dredd/dredd third_party/clang+llvm/bin/ -DREDD_REPO_ROOT=$(pwd) -export DREDD_REPO_ROOT +export DREDD_REPO_ROOT=$(pwd) +export DREDD_EXECUTABLE=${DREDD_REPO_ROOT}/third_party/clang+llvm/bin/dredd export PATH=${PATH}:${DREDD_REPO_ROOT}/scripts +cp build/src/dredd/dredd ${DREDD_EXECUTABLE} case "$(uname)" in "Linux") @@ -148,6 +150,10 @@ case "$(uname)" in # The C++ code generated by Dredd may require C++20. export DREDD_EXTRA_CXX_ARGS="-std=c++20" export DREDD_EXTRA_C_ARGS="-std=c17" + + ./scripts/check_single_file_tests.sh + ./scripts/check_execute_tests.sh + ./scripts/check_bespoke_tests.sh ;; "Darwin"*) @@ -182,74 +188,15 @@ case "$(uname)" in # The C++ code generated by Dredd may require recent C/C++ support. export DREDD_EXTRA_CXX_ARGS="-std=c++20" export DREDD_EXTRA_C_ARGS="-std=c17" + + ./scripts/check_single_file_tests.sh + ./scripts/check_execute_tests.sh + ./scripts/check_bespoke_tests.sh ;; "MINGW"*|"MSYS_NT"*) - # On Windows, run the single-file tests - export CC=cl.exe - export CXX=cl.exe - - # The following single-file tests give different expected results on Windows - # due to differences in how certain builtin types, such as size_t and - # uint64_t, expand. For simplicity, remove them before running single-file - # tests. - rm test/single_file/do_not_mutate_under_alignof.cc - rm test/single_file/do_not_mutate_under_alignof.cc.expected - rm test/single_file/do_not_mutate_under_alignof.cc.noopt.expected - rm test/single_file/do_not_mutate_under_sizeof.c - rm test/single_file/do_not_mutate_under_sizeof.c.expected - rm test/single_file/do_not_mutate_under_sizeof.c.noopt.expected - rm test/single_file/initializer_list.cc - rm test/single_file/initializer_list.cc.expected - rm test/single_file/initializer_list.cc.noopt.expected - rm test/single_file/add_type_aliases.c - rm test/single_file/add_type_aliases.c.expected - rm test/single_file/add_type_aliases.c.noopt.expected - rm test/single_file/add_type_aliases.cc - rm test/single_file/add_type_aliases.cc.expected - rm test/single_file/add_type_aliases.cc.noopt.expected - rm test/single_file/positive_int_as_minus_one.c - rm test/single_file/positive_int_as_minus_one.c.expected - rm test/single_file/positive_int_as_minus_one.c.noopt.expected - rm test/single_file/positive_int_as_minus_one.cc - rm test/single_file/positive_int_as_minus_one.cc.expected - rm test/single_file/positive_int_as_minus_one.cc.noopt.expected - rm test/single_file/vector_returns_temporary.cc - rm test/single_file/vector_returns_temporary.cc.expected - rm test/single_file/vector_returns_temporary.cc.noopt.expected - - # These tests rely on non-constant sized arrays, which the Microsoft compiler - # does not support. - rm test/single_file/non_const_sized_array.cc - rm test/single_file/non_const_sized_array.cc.expected - rm test/single_file/non_const_sized_array.cc.noopt.expected - rm test/single_file/non_const_sized_array.c - rm test/single_file/non_const_sized_array.c.expected - rm test/single_file/non_const_sized_array.c.noopt.expected - - # These tests expose a difference in how value-dependent types are handled in - # Windows vs. Linux builds of Clang. Not sure what's going on, so removing - # the tests under Windows for simplicity. - rm test/single_file/sizeof_template.cc - rm test/single_file/sizeof_template.cc.expected - rm test/single_file/sizeof_template.cc.noopt.expected - rm test/single_file/sizeof_template2.cc - rm test/single_file/sizeof_template2.cc.expected - rm test/single_file/sizeof_template2.cc.noopt.expected - - # These test relies on `__builtin_function()` not avilable on Windows. - rm test/single_file/builtin_frame_address_with_argument_rewrite.cc - rm test/single_file/builtin_frame_address_with_argument_rewrite.cc.expected - rm test/single_file/builtin_frame_address_with_argument_rewrite.cc.noopt.expected - rm test/single_file/builtin_frame_address.cc - rm test/single_file/builtin_frame_address.cc.expected - rm test/single_file/builtin_frame_address.cc.noopt.expected - rm -r test/execute/constant_function_argument - - # The C code generated by Dredd may require recent C/C++ support. - # Dredd's prelude includes the experimental `stdatomic.h` header. - export DREDD_EXTRA_C_ARGS="/std:c17 /experimental:c11atomics" - export DREDD_EXTRA_CXX_ARGS="/std:c++20" + # Do not run tests on Windows. Dredd doesn't really work on Windows without a proper tool chain; + # performing builds on Windows merely helps by providing one extra compilation target. ;; *) @@ -257,7 +204,3 @@ case "$(uname)" in exit 1 ;; esac - -DREDD_SKIP_COPY_EXECUTABLE=1 ./scripts/check_single_file_tests.sh -DREDD_SKIP_COPY_EXECUTABLE=1 ./scripts/check_execute_tests.sh -DREDD_SKIP_COPY_EXECUTABLE=1 ./scripts/check_bespoke_tests.sh diff --git a/.github/workflows/c_apps.sh b/.github/workflows/c_apps.sh index e9dc7969..648aa26a 100755 --- a/.github/workflows/c_apps.sh +++ b/.github/workflows/c_apps.sh @@ -26,12 +26,15 @@ uname case "$(uname)" in "Linux") NINJA_OS="linux" + + sudo apt update + sudo apt install -y llvm-17 clang-17 clang-tidy-17 clang-format-17 libclang-17-dev + + # Free up some space df -h sudo swapoff -a sudo rm -f /swapfile sudo apt clean - # shellcheck disable=SC2046 - docker rmi $(docker image ls -aq) df -h ;; @@ -50,18 +53,10 @@ pushd "${HOME}/bin" ls popd -# Install clang. -DREDD_LLVM_TAG=$(./scripts/llvm_tag.sh) -pushd ./third_party/clang+llvm - curl -fsSL -o clang+llvm.tar.xz "https://github.com/llvm/llvm-project/releases/download/llvmorg-${DREDD_LLVM_TAG}/clang+llvm-${DREDD_LLVM_TAG}-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - tar xf clang+llvm.tar.xz - mv clang+llvm-${DREDD_LLVM_TAG}-x86_64-linux-gnu-ubuntu-22.04/* . - rm clang+llvm.tar.xz -popd - DREDD_ROOT=$(pwd) -export PATH="${DREDD_ROOT}/third_party/clang+llvm/bin:$PATH" +DREDD_CLANG_LLVM_DIR="/usr/lib/llvm-17" +export PATH="${DREDD_CLANG_LLVM_DIR}/bin:$PATH" export CC=clang export CXX=clang++ @@ -69,16 +64,11 @@ export CXX=clang++ which ${CC} which ${CXX} -mkdir -p build -pushd build - cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Debug - cmake --build . --config Debug - cmake -DCMAKE_INSTALL_PREFIX=./install -DBUILD_TYPE=Debug -P cmake_install.cmake -popd +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DDREDD_CLANG_LLVM_DIR="${DREDD_CLANG_LLVM_DIR}" +cmake --build build --config Debug # Check that dredd works on some projects -DREDD_EXECUTABLE="${DREDD_ROOT}/third_party/clang+llvm/bin/dredd" -cp "${DREDD_ROOT}/build/src/dredd/dredd" "${DREDD_EXECUTABLE}" +DREDD_EXECUTABLE="${DREDD_ROOT}/build/src/dredd/dredd" echo "Curl" date diff --git a/.github/workflows/c_apps.yml b/.github/workflows/c_apps.yml index b70f57a6..f1114a86 100644 --- a/.github/workflows/c_apps.yml +++ b/.github/workflows/c_apps.yml @@ -24,7 +24,7 @@ jobs: fail-fast: false matrix: os: - - ubuntu-22.04 + - ubuntu-24.04 config: - Release diff --git a/.github/workflows/cxx_apps.sh b/.github/workflows/cxx_apps.sh index 043ac5ec..4198f89a 100755 --- a/.github/workflows/cxx_apps.sh +++ b/.github/workflows/cxx_apps.sh @@ -26,12 +26,15 @@ uname case "$(uname)" in "Linux") NINJA_OS="linux" + + sudo apt update + sudo apt install -y llvm-17 clang-17 clang-tidy-17 clang-format-17 libclang-17-dev + + # Free up some space df -h sudo swapoff -a sudo rm -f /swapfile sudo apt clean - # shellcheck disable=SC2046 - docker rmi $(docker image ls -aq) df -h ;; @@ -50,18 +53,10 @@ pushd "${HOME}/bin" ls popd -# Install clang. -DREDD_LLVM_TAG=$(./scripts/llvm_tag.sh) -pushd ./third_party/clang+llvm - curl -fsSL -o clang+llvm.tar.xz "https://github.com/llvm/llvm-project/releases/download/llvmorg-${DREDD_LLVM_TAG}/clang+llvm-${DREDD_LLVM_TAG}-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - tar xf clang+llvm.tar.xz - mv clang+llvm-${DREDD_LLVM_TAG}-x86_64-linux-gnu-ubuntu-22.04/* . - rm clang+llvm.tar.xz -popd - DREDD_ROOT=$(pwd) -export PATH="${DREDD_ROOT}/third_party/clang+llvm/bin:$PATH" +DREDD_CLANG_LLVM_DIR="/usr/lib/llvm-17" +export PATH="${DREDD_CLANG_LLVM_DIR}/bin:$PATH" export CC=clang export CXX=clang++ @@ -69,12 +64,11 @@ export CXX=clang++ which ${CC} which ${CXX} -cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DDREDD_CLANG_LLVM_DIR="${DREDD_CLANG_LLVM_DIR}" cmake --build build --config Debug # Check that dredd works on some projects -DREDD_EXECUTABLE="${DREDD_ROOT}/third_party/clang+llvm/bin/dredd" -cp "${DREDD_ROOT}/build/src/dredd/dredd" "${DREDD_EXECUTABLE}" +DREDD_EXECUTABLE="${DREDD_ROOT}/build/src/dredd/dredd" echo "examples/simple/pi.cc: check that we can build the simple example" date diff --git a/.github/workflows/cxx_apps.yml b/.github/workflows/cxx_apps.yml index 85676d85..f25496a1 100644 --- a/.github/workflows/cxx_apps.yml +++ b/.github/workflows/cxx_apps.yml @@ -24,7 +24,7 @@ jobs: fail-fast: false matrix: os: - - ubuntu-22.04 + - ubuntu-24.04 config: - Release diff --git a/.github/workflows/dev_build.sh b/.github/workflows/dev_build.sh index f04281cc..e9e6b341 100755 --- a/.github/workflows/dev_build.sh +++ b/.github/workflows/dev_build.sh @@ -26,12 +26,15 @@ uname case "$(uname)" in "Linux") NINJA_OS="linux" + + sudo apt update + sudo apt install -y llvm-17 clang-17 clang-tidy-17 clang-format-17 libclang-17-dev + + # Free up some space df -h sudo swapoff -a sudo rm -f /swapfile sudo apt clean - # shellcheck disable=SC2046 - docker rmi $(docker image ls -aq) df -h ;; @@ -53,14 +56,7 @@ pushd "${HOME}/bin" ls popd -# Install clang. -DREDD_LLVM_TAG=$(./scripts/llvm_tag.sh) -pushd ./third_party/clang+llvm - curl -fsSL -o clang+llvm.tar.xz "https://github.com/llvm/llvm-project/releases/download/llvmorg-${DREDD_LLVM_TAG}/clang+llvm-${DREDD_LLVM_TAG}-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - tar xf clang+llvm.tar.xz - mv clang+llvm-${DREDD_LLVM_TAG}-x86_64-linux-gnu-ubuntu-22.04/* . - rm clang+llvm.tar.xz -popd +export DREDD_CLANG_LLVM_DIR="/usr/lib/llvm-17" # Source the dev shell to download clang-tidy and other tools. # Developers should *run* the dev shell, but we want to continue executing this script. diff --git a/.github/workflows/dev_build.yml b/.github/workflows/dev_build.yml index 4b5d8e6f..18b6efe8 100644 --- a/.github/workflows/dev_build.yml +++ b/.github/workflows/dev_build.yml @@ -24,7 +24,7 @@ jobs: fail-fast: false matrix: os: - - ubuntu-22.04 + - ubuntu-24.04 config: - Release diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e4e490e..b760a75b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,7 +67,9 @@ endif() # Third party targets. if(NOT DEFINED DREDD_CLANG_LLVM_DIR) - set(DREDD_CLANG_LLVM_DIR "${CMAKE_SOURCE_DIR}/third_party/clang+llvm") + message( + FATAL_ERROR "DEFINED DREDD_CLANG_LLVM_DIR must refer to an installation of \ +Clang/LLVM.") endif() set(LLVM_DIR "${DREDD_CLANG_LLVM_DIR}/lib/cmake/llvm") diff --git a/README.md b/README.md index baebf1f5..6d4ba098 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ database*](https://clang.llvm.org/docs/JSONCompilationDatabase.html) for the project: a `compile_commands.json` file. Compilation databases can be automatically generated by most modern build systems that support C++ - [here is a great resource on compilation -databases](https://sarcasm.github.io/notes/dev/compilation-database.html). +databases](https://sarcasm.github.io/notes/dev/compilation-database.html). Alternatively the [Bear](https://github.com/rizsotto/Bear) tool can be used to generate a compilation database. You then point Dredd at (a) the C++ source files that you would like to be mutated, and (b) the compilation database, which must provide @@ -44,24 +44,16 @@ To get started, either * [download the latest Dredd release](https://github.com/mc-imperial/dredd/releases/latest) and unzip at a location of your choice, or -* [build Dredd from source](#building-dredd-from-source) and copy the `dredd` executable from `build/src/dredd/` into `third_party/clang+llvm/bin`. +* [build Dredd from source](#building-dredd-from-source). -Set the `DREDD_CHECKOUT` environment variable to refer to the root of the directory into which Dredd is checked out, e.g.: - -``` -export DREDD_CHECKOUT=/path/to/dredd -``` +Either way, ensure that the `dredd` executable is on your path. -Set the `DREDD_CLANG_BIN_DIR` environment variable to refer to the `bin` directory of the build of Clang/LLVM used by Dredd, e.g.: +The instructions that follow also assume that you have the Clang compiler installed, so that `clang` and `clang++` are on your path. -``` -export DREDD_CLANG_BIN_DIR=${DREDD_CHECKOUT}/third_party/clang+llvm/bin -``` - -Set the `DREDD_EXECUTABLE` environment variable to refer to the `dredd` executable at this location, e.g.: +Set the `DREDD_CHECKOUT` environment variable to refer to the root of the directory into which Dredd is checked out, e.g.: ``` -export DREDD_EXECUTABLE=${DREDD_CLANG_BIN_DIR}/dredd +export DREDD_CHECKOUT=/path/to/dredd ``` We first show how to apply Dredd to a simple stand-alone program. We will then show how to apply it to a larger C++ CMake project. @@ -72,9 +64,9 @@ We first show how to apply Dredd to a simple stand-alone program. We will then s ``` # This will modify pi.cc in-place. -${DREDD_EXECUTABLE} examples/simple/pi.cc +dredd examples/simple/pi.cc # Now compile the mutated version of the example -${DREDD_CLANG_BIN_DIR}/clang++ examples/simple/pi.cc -o examples/simple/pi +clang++ examples/simple/pi.cc -o examples/simple/pi ``` Due to the changes made by Dredd, the compile command may lead to warnings being issued but this is expected. The changes to `pi.cc` can be viewed by running `git diff` although, these changes are not meant to be human-readable. @@ -112,7 +104,7 @@ Compile the math library. It is important to use the Clang that ships with Dredd ``` cd examples/math -cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER=${DREDD_CLANG_BIN_DIR}/clang -DCMAKE_CXX_COMPILER=${DREDD_CLANG_BIN_DIR}/clang++ +cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ cmake --build build ``` @@ -130,7 +122,7 @@ All tests should pass. To mutate all of the `.cc` files in the library use the following command: ``` -${DREDD_EXECUTABLE} -p build math/src/*.cc --mutation-info-file mutant-info.json +dredd -p build math/src/*.cc --mutation-info-file mutant-info.json ``` The `-p` option allows the compilation database generated by CMake above to be passed to Dredd: the `compile_commands.json` file in `build` will be used as a compilation database. @@ -258,7 +250,8 @@ The following instructions have been tested on Ubuntu 22.04. ### Prerequisites -- curl (used to fetch a Clang/LLVM release) +- Various Clang/LLVM (version 17) packages, which you can install via: + - `sudo apt install -y llvm-17 clang-17 clang-tidy-17 clang-format-17 libclang-17-dev` - CMake version 3.13 or higher - ninja version 1.10.0 or higher @@ -278,32 +271,24 @@ git clone --recursive git@github.com:mc-imperial/dredd.git The `--recursive` flag ensures that submodules are fetched. -### Downloading Clang/LLVM - -Dredd builds against various Clang and LLVM libraries. Rather than including Clang/LLVM as a submodule of Dredd and building it from source, Dredd assumes that a built release of Clang/LLVM has been downloaded into `third_party`. +### Build steps -From the root of the repository, execute the following commands: +From the root of the repository, execute the following commands, +which assume that LLVM 17 resides at `/usr/lib/llvm-17`. +On Linux, you may change `Release` to `Debug` for a debug build. ``` -cd third_party -# The release file is pretty large, so this download may take a while -curl -Lo clang+llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04.tar.xz -tar xf clang+llvm.tar.xz -mv clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04 clang+llvm -rm clang+llvm.tar.xz -cd .. +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DDREDD_CLANG_LLVM_DIR=/usr/lib/llvm-17 +cmake --build build --config Release +ls -la build/src/dredd/dredd ``` -### Build steps - -From the root of the repository, execute the following commands. -On Linux, you may change `Release` to `Debug` for a debug build; only a release build of Clang/LLVM is available, but linking against this with a Debug build of Dredd still works. +The listed file is the Dredd executable, which you should then add to your path; e.g. ``` -mkdir build && cd build -cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release -cmake --build . --config Release -cp src/dredd/dredd ../third_party/clang+llvm/bin +PATH=$(pwd)/build/src/dredd:${PATH} +# This should find the dredd binary that you just created +which dredd ``` ## Guide for developers @@ -320,34 +305,41 @@ The following guide assumes that you are using Linux for development. ### Scripts Directory The `scripts` directory contains a number of commands that are useful for developers. To make use of these commands, -you must first run `./dev_shell.sh.template` from the root of the Dredd repo. This will ensure that the necessary -environment variables are set as well as building tools that are used in other check commands. +you must first run: -The last four scripts in this section assume that a build Dredd is under `temp/build-Debug`, as they copy the Dredd -binary to `third_party/clang+llvm/bin` to execute the test files. Therefore, to use these scripts, -one should ensure that `check_build.sh` has been executed at least up to the point where a debug build has finished. +``` +# This assumes your LLVM 17 installation is at /usr/lib/llvm-17 +DREDD_CLANG_LLVM_DIR=/usr/lib/llvm-17 ./dev_shell.sh.template +``` + +from the root of the Dredd repo. This will ensure that the necessary +environment variables are set as well as building tools that are used in other check commands. The commands that can be run in isolation in the `scripts` directory are: -- `check_all.sh` : This runs a combination of the commands below to ensure that Dredd is formatted and functioning as expected. -- `check_build.sh` : This checks that Dredd can be build and that the unit tests included in the `test` directory still pass -- `check_clang_tidy.sh` : This runs `clang-tidy` on Dredd's source files. -- `check_clean_build.sh` : This checks that Dredd can be build from scratch by deleting any previous builds first and running +- `check_all.sh`: This runs a combination of the commands below to ensure that Dredd is formatted and functioning as expected. +- `check_build.sh`: This checks that Dredd can be build and that the unit tests included in the `test` directory still pass +- `check_clang_tidy.sh`: This runs `clang-tidy` on Dredd's source files. +- `check_clean_build.sh`: This checks that Dredd can be build from scratch by deleting any previous builds first and running the provided unit tests. -- `check_cmakelint.sh` : This runs cmake-lint on Dredd's cmake files. -- `check_cpplint.sh` : Runs a static check to ensure the code style is correct. -- `check_format.sh` : This uses `clang-format` and `cmake-format` to ensure that all source files and cmake files `src` +- `check_cmakelint.sh`: This runs cmake-lint on Dredd's cmake files. +- `check_cpplint.sh`: Runs a static check to ensure the code style is correct. +- `check_format.sh`: This uses `clang-format` and `cmake-format` to ensure that all source files and cmake files `src` and `examples` are formatted correctly. -- `check_headers.py` : This checks that the necessary files contain a copyright header. -- `fix_format.sh` : This command formats all the Dredd source files and example files using `clang-format` and formats +- `check_headers.py`: This checks that the necessary files contain a copyright header. +- `fix_format.sh`: This command formats all the Dredd source files and example files using `clang-format` and formats the Dredd cmake files and example cmake files using `cmake-format`. -- `check_one_single_file_test.sh` :This will run Dredd on a specific `.c` or `.cc` file in the `tests/single_file` directory +- `check_one_single_file_test.sh`: This will run Dredd on a specific `.c` or `.cc` file in the `tests/single_file` directory and ensure the output is the same as the respective `.expected` file. -- `check_single_file_tests.sh` : This acts the same as the above commands, but will compare the output from Dredd for all +- `check_single_file_tests.sh`: This acts the same as the above commands, but will compare the output from Dredd for all `.c` and `.cc` files in the `test/single_file` directory. -- `regenerate_one_single_file_expectation.sh` : This takes in a `.c` or `.cc` file from the `test/single_file` directory and +- `regenerate_one_single_file_expectation.sh`: This takes in a `.c` or `.cc` file from the `test/single_file` directory and regenerates its respective `.expected` file. You should only do this if you are certain the changes you have made to Dredd are correct. -- `regenerate_single_file_expectations` : This acts the same as the above command but will regenerate all the `.expected` +- `regenerate_single_file_expectations`: This acts the same as the above command but will regenerate all the `.expected` files for the `.c` and `.cc` files in the `test/single_file` file directory. +- `check_one_execute_test.py`: Runs Dredd on a specific end-to-end test. +- `check_execute_tests.sh`: Runs Dredd on all end-to-end tests. +- `check_one_bespoke_test.py`: Runs Dredd on a specific bespoke test. +- `check_bespoke_tests.sh`: Runs Dredd on all bespoke tests. The auxiliary commands in the `scripts` directory are: - `check_cppcheck.sh` : This runs a static check to detect bugs and undefined behaviour. diff --git a/dev_shell.sh.template b/dev_shell.sh.template index 9e01227a..4e7f909b 100755 --- a/dev_shell.sh.template +++ b/dev_shell.sh.template @@ -18,6 +18,8 @@ set -e set -u set -x +: "${DREDD_CLANG_LLVM_DIR?DREDD_CLANG_LLVM_DIR must be set to an installation of Clang/LLVM}" + # Check that it looks like we are in the repo root. test -f ./src/dredd/src/main.cc test -f ./src/.clang-format @@ -26,9 +28,9 @@ test -f ./src/iwyu.imp DREDD_REPO_ROOT="$(pwd)" export DREDD_REPO_ROOT -# Check that the Clang/LLVM release has been downloaded. -CLANG_RELEASE="${DREDD_REPO_ROOT}/third_party/clang+llvm" -test -f ${CLANG_RELEASE}/bin/clang++ +# Check that the Clang/LLVM release looks right. +test -f ${DREDD_CLANG_LLVM_DIR}/bin/clang++ +test -f ${DREDD_CLANG_LLVM_DIR}/lib/cmake/clang/ClangConfig.cmake # If DREDD_SKIP_COMPILER_SET is unset: if test -z ${DREDD_SKIP_COMPILER_SET+x}; then @@ -37,7 +39,7 @@ if test -z ${DREDD_SKIP_COMPILER_SET+x}; then fi # Ensure that the downloaded Clang/LLVM release is on the path -PATH="${CLANG_RELEASE}/bin:${PATH}" +PATH="${DREDD_CLANG_LLVM_DIR}/bin:${PATH}" export PATH cd temp/ @@ -85,20 +87,21 @@ export CPPLINT_PY IWYU_VERSION="0.21" IWYU_OUT="iwyu-${IWYU_VERSION}" +IWYU_INSTALL="$(pwd)/${IWYU_OUT}/build/install" if test ! -f "${IWYU_OUT}.touch"; then git clone --branch ${IWYU_VERSION} --depth 1 https://github.com/include-what-you-use/include-what-you-use.git "${IWYU_OUT}" pushd "${IWYU_OUT}" mkdir build pushd build - cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release "-DCMAKE_INSTALL_PREFIX=${CLANG_RELEASE}" + cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release "-DCMAKE_INSTALL_PREFIX=${IWYU_INSTALL}" cmake --build . --config Release cmake -P cmake_install.cmake --config Release popd popd touch "${IWYU_OUT}.touch" fi -PATH="$(pwd)/${IWYU_OUT}/build/install/bin:${PATH}" +PATH="${IWYU_INSTALL}/bin:${PATH}" export PATH cd "${DREDD_REPO_ROOT}" diff --git a/scripts/check_bespoke_tests.sh b/scripts/check_bespoke_tests.sh index 331271d9..562276a6 100755 --- a/scripts/check_bespoke_tests.sh +++ b/scripts/check_bespoke_tests.sh @@ -20,21 +20,13 @@ set -x if [ -z "${DREDD_SKIP_CHECK_COMPILE_COMMANDS+x}" ] then - DREDD_INSTALLED_EXECUTABLE="${DREDD_REPO_ROOT}/third_party/clang+llvm/bin/dredd" - - cd "${DREDD_REPO_ROOT}" - - if [ -z "${DREDD_SKIP_COPY_EXECUTABLE+x}" ] + if [ -z "${DREDD_EXECUTABLE+x}" ] then - # Ensure that Dredd is in its installed location. This depends on a - # debug build being available - cp temp/build-Debug/src/dredd/dredd "${DREDD_INSTALLED_EXECUTABLE}" + DREDD_EXECUTABLE=${DREDD_REPO_ROOT}/temp/build-Debug/src/dredd/dredd fi - # Avoid copying Dredd to its installed location when invoking the script that - # checks a single test. - export DREDD_SKIP_COPY_EXECUTABLE=1 - + cd "${DREDD_REPO_ROOT}" + # Consider each single-file test case for f in test/bespoke/* do diff --git a/scripts/check_build.sh b/scripts/check_build.sh index 93a7be0d..b869a607 100755 --- a/scripts/check_build.sh +++ b/scripts/check_build.sh @@ -28,7 +28,8 @@ for CONFIG in Debug Release; do cmake \ -G Ninja \ ../.. \ - "-DCMAKE_BUILD_TYPE=${CONFIG}" + "-DCMAKE_BUILD_TYPE=${CONFIG}" \ + "-DDREDD_CLANG_LLVM_DIR=${DREDD_CLANG_LLVM_DIR}" cmake --build . --config "${CONFIG}" diff --git a/scripts/check_execute_tests.sh b/scripts/check_execute_tests.sh index f4ed2c86..0ad464e2 100755 --- a/scripts/check_execute_tests.sh +++ b/scripts/check_execute_tests.sh @@ -20,21 +20,13 @@ set -x if [ -z "${DREDD_SKIP_CHECK_COMPILE_COMMANDS+x}" ] then - DREDD_INSTALLED_EXECUTABLE="${DREDD_REPO_ROOT}/third_party/clang+llvm/bin/dredd" - - cd "${DREDD_REPO_ROOT}" - - if [ -z "${DREDD_SKIP_COPY_EXECUTABLE+x}" ] + if [ -z "${DREDD_EXECUTABLE+x}" ] then - # Ensure that Dredd is in its installed location. This depends on a - # debug build being available - cp temp/build-Debug/src/dredd/dredd "${DREDD_INSTALLED_EXECUTABLE}" + DREDD_EXECUTABLE=${DREDD_REPO_ROOT}/temp/build-Debug/src/dredd/dredd fi - - # Avoid copying Dredd to its installed location when invoking the script that - # checks a single test. - export DREDD_SKIP_COPY_EXECUTABLE=1 + cd "${DREDD_REPO_ROOT}" + # Consider each single-file test case for f in test/execute/* do diff --git a/scripts/check_one_bespoke_test.py b/scripts/check_one_bespoke_test.py index ab5b7d62..7290b6d1 100755 --- a/scripts/check_one_bespoke_test.py +++ b/scripts/check_one_bespoke_test.py @@ -23,7 +23,7 @@ from pathlib import Path DREDD_REPO_ROOT = os.environ['DREDD_REPO_ROOT'] -DREDD_INSTALLED_EXECUTABLE = Path(DREDD_REPO_ROOT, 'third_party', 'clang+llvm', 'bin', 'dredd') +DREDD_EXECUTABLE = Path(DREDD_REPO_ROOT, 'temp', 'build-Debug', 'src', 'dredd', 'dredd') if 'DREDD_EXECUTABLE' not in os.environ else os.environ['DREDD_EXECUTABLE'] test_directory = Path(DREDD_REPO_ROOT, sys.argv[1]) # Back up current directory and move to a temporary directory @@ -37,12 +37,6 @@ Path(bespoke_test_work_dir).mkdir(exist_ok=False) os.chdir(bespoke_test_work_dir) -if 'DREDD_SKIP_COPY_EXECUTABLE' not in os.environ or os.environ['DREDD_SKIP_COPY_EXECUTABLE'] != '1': - # Ensure that Dredd is in its installed location. This depends on a - # debug build being available - shutil.copy(src=Path('../build-Debug', 'src', 'dredd', 'dredd'), - dst=DREDD_INSTALLED_EXECUTABLE) - if not os.path.exists(test_directory / 'test.py'): print(f"No 'test.py' file found for bespoke test {test_directory}") sys.exit(2) diff --git a/scripts/check_one_execute_test.py b/scripts/check_one_execute_test.py index 46981461..a87445a7 100755 --- a/scripts/check_one_execute_test.py +++ b/scripts/check_one_execute_test.py @@ -22,7 +22,7 @@ from pathlib import Path DREDD_REPO_ROOT = os.environ['DREDD_REPO_ROOT'] -DREDD_INSTALLED_EXECUTABLE = Path(DREDD_REPO_ROOT, 'third_party', 'clang+llvm', 'bin', 'dredd') +DREDD_EXECUTABLE = Path(DREDD_REPO_ROOT, 'temp', 'build-Debug', 'src', 'dredd', 'dredd') if 'DREDD_EXECUTABLE' not in os.environ else os.environ['DREDD_EXECUTABLE'] test_directory = Path(DREDD_REPO_ROOT, sys.argv[1]) test_is_cxx = os.path.exists(test_directory / 'harness.cc') extension = 'cc' if test_is_cxx else 'c' @@ -31,12 +31,6 @@ original_dir = os.getcwd() os.chdir(Path(DREDD_REPO_ROOT, 'temp')) -if 'DREDD_SKIP_COPY_EXECUTABLE' not in os.environ or os.environ['DREDD_SKIP_COPY_EXECUTABLE'] != '1': - # Ensure that Dredd is in its installed location. This depends on a - # debug build being available - shutil.copy(src=Path('build-Debug', 'src', 'dredd', 'dredd'), - dst=DREDD_INSTALLED_EXECUTABLE) - # Copy the test files into the temporary directory for mutation shutil.copy(src=test_directory / f'harness.{extension}', dst=f'harness.{extension}') @@ -53,7 +47,7 @@ expected_mutant_outputs.add(component) # Mutate the program using Dredd. -cmd = [DREDD_INSTALLED_EXECUTABLE, '--mutation-info-file', 'temp.json', f'tomutate.{extension}', '--'] +cmd = [DREDD_EXECUTABLE, '--mutation-info-file', 'temp.json', f'tomutate.{extension}', '--'] if test_is_cxx: # This supports execute tests that use C++ 20 features. cmd.append('--std=c++20') diff --git a/scripts/check_one_single_file_test.sh b/scripts/check_one_single_file_test.sh index f013bbcc..6d3296fb 100755 --- a/scripts/check_one_single_file_test.sh +++ b/scripts/check_one_single_file_test.sh @@ -18,18 +18,14 @@ set -e set -u set -x -DREDD_INSTALLED_EXECUTABLE="${DREDD_REPO_ROOT}/third_party/clang+llvm/bin/dredd" +if [ -z "${DREDD_EXECUTABLE+x}" ] +then + DREDD_EXECUTABLE=${DREDD_REPO_ROOT}/temp/build-Debug/src/dredd/dredd +fi # Move to the temporary directory pushd "${DREDD_REPO_ROOT}/temp" - if [ -z "${DREDD_SKIP_COPY_EXECUTABLE+x}" ] - then - # Ensure that Dredd is in its installed location. This depends on a - # debug build being available - cp build-Debug/src/dredd/dredd "${DREDD_INSTALLED_EXECUTABLE}" - fi - f="${DREDD_REPO_ROOT}/${1}" for opt in "opt" "noopt" @@ -52,7 +48,7 @@ pushd "${DREDD_REPO_ROOT}/temp" copy_of_f=$(basename "$f") # Mutate the test case using Dredd - ${DREDD_INSTALLED_EXECUTABLE} ${DREDD_EXTRA_ARGS} --mutation-info-file temp.json "${copy_of_f}" -- + ${DREDD_EXECUTABLE} ${DREDD_EXTRA_ARGS} --mutation-info-file temp.json "${copy_of_f}" -- # Check that the JSON generated by Dredd is valid check_json.py temp.json diff --git a/scripts/check_single_file_tests.sh b/scripts/check_single_file_tests.sh index 5a87fec6..61dc6257 100755 --- a/scripts/check_single_file_tests.sh +++ b/scripts/check_single_file_tests.sh @@ -20,21 +20,13 @@ set -x if [ -z "${DREDD_SKIP_CHECK_COMPILE_COMMANDS+x}" ] then - DREDD_INSTALLED_EXECUTABLE="${DREDD_REPO_ROOT}/third_party/clang+llvm/bin/dredd" - - cd "${DREDD_REPO_ROOT}" - - if [ -z "${DREDD_SKIP_COPY_EXECUTABLE+x}" ] + if [ -z "${DREDD_EXECUTABLE+x}" ] then - # Ensure that Dredd is in its installed location. This depends on a - # debug build being available - cp temp/build-Debug/src/dredd/dredd "${DREDD_INSTALLED_EXECUTABLE}" + DREDD_EXECUTABLE=${DREDD_REPO_ROOT}/temp/build-Debug/src/dredd/dredd fi - # Avoid copying Dredd to its installed location when invoking the script that - # checks a single test. - export DREDD_SKIP_COPY_EXECUTABLE=1 - + cd "${DREDD_REPO_ROOT}" + # Consider each single-file test case for f in test/single_file/*.cc test/single_file/*.c do diff --git a/test/bespoke/mutant_tracking/test.py b/test/bespoke/mutant_tracking/test.py index 2f92bac7..c97bfabc 100644 --- a/test/bespoke/mutant_tracking/test.py +++ b/test/bespoke/mutant_tracking/test.py @@ -6,9 +6,10 @@ from pathlib import Path DREDD_REPO_ROOT = os.environ['DREDD_REPO_ROOT'] -DREDD_INSTALLED_EXECUTABLE = Path(DREDD_REPO_ROOT, 'third_party', 'clang+llvm', 'bin', 'dredd') +DREDD_CLANG_LLVM_DIR = os.environ['DREDD_CLANG_LLVM_DIR'] +DREDD_EXECUTABLE = Path(DREDD_REPO_ROOT, 'temp', 'build-Debug', 'src', 'dredd', 'dredd') if 'DREDD_EXECUTABLE' not in os.environ else os.environ['DREDD_EXECUTABLE'] QUERY_MUTANT_INFO_SCRIPT = Path(DREDD_REPO_ROOT, 'scripts', 'query_mutant_info.py') -CLANG_INSTALLED_EXECUTABLE = Path(DREDD_REPO_ROOT, 'third_party', 'clang+llvm', 'bin', 'clang') +CLANG_EXECUTABLE = Path(DREDD_CLANG_LLVM_DIR, 'bin', 'clang') COMPILED_EXECUTABLE_FILENAME = 'a.exe' if os.name == 'nt' else './a.out' @@ -25,11 +26,11 @@ def run_successfully(cmd): def main(): # Run Dredd without tracking shutil.copyfile(src='example.c', dst='tomutate.c') - run_successfully([DREDD_INSTALLED_EXECUTABLE, '--mutation-info-file', 'info-mutate.json', 'tomutate.c', '--']) + run_successfully([DREDD_EXECUTABLE, '--mutation-info-file', 'info-mutate.json', 'tomutate.c', '--']) # Run Dredd with tracking shutil.copyfile(src='example.c', dst='tomutate.c') - run_successfully([DREDD_INSTALLED_EXECUTABLE, + run_successfully([DREDD_EXECUTABLE, '--mutation-info-file', 'info-track.json', '--only-track-mutant-coverage', @@ -41,7 +42,7 @@ def main(): assert filecmp.cmp('info-mutate.json', 'info-track.json') # The mutant tracking version should compile successfully. - run_successfully([CLANG_INSTALLED_EXECUTABLE, 'tomutate.c']) + run_successfully([CLANG_EXECUTABLE, 'tomutate.c']) # When executed with no command line arguments the program should return 0. dredd_env = os.environ.copy() diff --git a/test/bespoke/nonexistent-file/test.py b/test/bespoke/nonexistent-file/test.py index 76020536..7d4961ab 100644 --- a/test/bespoke/nonexistent-file/test.py +++ b/test/bespoke/nonexistent-file/test.py @@ -4,11 +4,11 @@ from pathlib import Path DREDD_REPO_ROOT = os.environ['DREDD_REPO_ROOT'] -DREDD_INSTALLED_EXECUTABLE = Path(DREDD_REPO_ROOT, 'third_party', 'clang+llvm', 'bin', 'dredd') +DREDD_EXECUTABLE = Path(DREDD_REPO_ROOT, 'temp', 'build-Debug', 'src', 'dredd', 'dredd') if 'DREDD_EXECUTABLE' not in os.environ else os.environ['DREDD_EXECUTABLE'] def main(): - cmd = [DREDD_INSTALLED_EXECUTABLE, 'tomutate.c', '--'] + cmd = [DREDD_EXECUTABLE, 'tomutate.c', '--'] result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # We expect an error due to a non-existent file. assert result.returncode != 0 diff --git a/test/bespoke/query_mutant_info/test.py b/test/bespoke/query_mutant_info/test.py index 077d8995..faadafd2 100644 --- a/test/bespoke/query_mutant_info/test.py +++ b/test/bespoke/query_mutant_info/test.py @@ -6,7 +6,7 @@ from pathlib import Path DREDD_REPO_ROOT = os.environ['DREDD_REPO_ROOT'] -DREDD_INSTALLED_EXECUTABLE = Path(DREDD_REPO_ROOT, 'third_party', 'clang+llvm', 'bin', 'dredd') +DREDD_EXECUTABLE = Path(DREDD_REPO_ROOT, 'temp', 'build-Debug', 'src', 'dredd', 'dredd') if 'DREDD_EXECUTABLE' not in os.environ else os.environ['DREDD_EXECUTABLE'] QUERY_MUTANT_INFO_SCRIPT = Path(DREDD_REPO_ROOT, 'scripts', 'query_mutant_info.py') FILENAME = f'math{os.sep}src{os.sep}exp.cc' @@ -114,7 +114,7 @@ def main(): ]) # Mutate the source files of the math example - run_successfully([DREDD_INSTALLED_EXECUTABLE, + run_successfully([DREDD_EXECUTABLE, '-p', 'build', '--mutation-info-file', diff --git a/test/bespoke/skip_bad_files/test.py b/test/bespoke/skip_bad_files/test.py index f4667675..094e9837 100644 --- a/test/bespoke/skip_bad_files/test.py +++ b/test/bespoke/skip_bad_files/test.py @@ -6,7 +6,7 @@ from pathlib import Path DREDD_REPO_ROOT = os.environ['DREDD_REPO_ROOT'] -DREDD_INSTALLED_EXECUTABLE = Path(DREDD_REPO_ROOT, 'third_party', 'clang+llvm', 'bin', 'dredd') +DREDD_EXECUTABLE = Path(DREDD_REPO_ROOT, 'temp', 'build-Debug', 'src', 'dredd', 'dredd') if 'DREDD_EXECUTABLE' not in os.environ else os.environ['DREDD_EXECUTABLE'] def run_successfully(cmd): @@ -14,7 +14,7 @@ def run_successfully(cmd): def main(): - cmd = [DREDD_INSTALLED_EXECUTABLE, + cmd = [DREDD_EXECUTABLE, '--mutation-info-file', 'info.json', 'good1.c', @@ -26,7 +26,7 @@ def main(): # Dredd should return non-zero, due to 'bad1.c' and 'bad2.c' assert result.returncode != 0 error_output = result.stderr.decode('utf-8') - pattern = "The following files were not mutated due to compile-time errors; see above for details:\s+[^\s]+bad1\.c\s+[^\s]+bad2\.c" + pattern = r"The following files were not mutated due to compile-time errors; see above for details:\s+[^\s]+bad1\.c\s+[^\s]+bad2\.c" match = re.search(pattern, error_output) assert match is not None dictionary = json.load(open('info.json'))