From d74f9964c0af5f4fdb7ae7f98b5c137bf0a654ae Mon Sep 17 00:00:00 2001 From: Filippo Cremonese Date: Tue, 17 Aug 2021 13:03:24 +0200 Subject: [PATCH 1/5] Use orc inspect to list files to copy --- .../config/components/revng_distributable.yml | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/.orchestra/config/components/revng_distributable.yml b/.orchestra/config/components/revng_distributable.yml index 768cdbd2..b2d2826e 100644 --- a/.orchestra/config/components/revng_distributable.yml +++ b/.orchestra/config/components/revng_distributable.yml @@ -13,17 +13,35 @@ install: | rm -rf "${TMP_ROOT}${ORCHESTRA_ROOT}/revng/root" mkdir -p "${TMP_ROOT}${ORCHESTRA_ROOT}/revng/root" cd "$ORCHESTRA_ROOT" - index_files="" - echo share/orchestra/glibc.idx share/orchestra/cmake.idx share/orchestra/boost.idx share/orchestra/clang-release.idx share/orchestra/toolchain_*.idx | sed 's/ /\n/g' | grep -v host | while read index_file; do - if [ -e "$index_file" ]; then - cat "$index_file" | \ - grep -v libgcc_s.so | \ - grep -v 'libc++.so' | \ - grep -v 'libc++abi.so' | \ - grep -v 'llvm-link' | \ - grep -v 'llc' - fi - done | sed 's!^!/!' | rsync \ + + # TODO: This should be derived from orchestra by whitelisting the transitive dependencies of revng-distributable + COMPONENTS_TO_EXCLUDE=( + glibc + boost + clang-release + cmake + llvm-documentation + ) + # Select all installed toolchain components except toolchain/host/* + COMPONENTS_TO_EXCLUDE+=( + $(orc components --installed --json \ + | jq -r ' + .[].name + | select(. | test("^toolchain/.*")) + | select(. | test("^toolchain/host.*") | not)' + ) + ) + + for COMPONENT in "${COMPONENTS_TO_EXCLUDE[@]}"; do + orc inspect component installed-files "$COMPONENT" \ + | grep -v 'libgcc_s.so' \ + | grep -v 'libc++.so' \ + | grep -v 'libc++abi.so' \ + | grep -v 'llvm-link' \ + | grep -v 'llc' + done \ + | sed 's!^!/!' \ + | rsync \ --archive \ --verbose \ --progress \ From d8623419cc95314ade587040f875829df4b50e17 Mon Sep 17 00:00:00 2001 From: Filippo Cremonese Date: Tue, 17 Aug 2021 10:26:10 +0200 Subject: [PATCH 2/5] Fix xcbproto build error with python3.9 A version bump is required to fix a build error in xcb libraries due to them importing gcd from the fractions module, which was deprecated in python 3.5 and removed in python 3.9. --- .orchestra/config/components/ui/libX.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.orchestra/config/components/ui/libX.yml b/.orchestra/config/components/ui/libX.yml index 2deb5291..649d8218 100644 --- a/.orchestra/config/components/ui/libX.yml +++ b/.orchestra/config/components/ui/libX.yml @@ -5,7 +5,7 @@ #@ load("/lib/create_component.lib.yml", "single_build_component") #@ load("/lib/make.lib.yml", "make") -#@ xcb_proto_source_url = "https://xorg.freedesktop.org/archive/individual/proto/xcb-proto-1.14.tar.gz" +#@ xcb_proto_source_url = "https://xorg.freedesktop.org/archive/individual/proto/xcb-proto-1.14.1.tar.gz" #@ libxcb_source_url = "https://xorg.freedesktop.org/archive/individual/lib/libxcb-1.14.tar.gz" #@ libx11_source_url = "https://www.x.org/releases/individual/lib/libX11-1.6.12.tar.gz" #@ xtrans_source_url = "https://xorg.freedesktop.org/releases/individual/lib/xtrans-1.4.0.tar.bz2" From 8cdf8b13faaefd2818d96621faa1c96724e95f77 Mon Sep 17 00:00:00 2001 From: Filippo Cremonese Date: Tue, 17 Aug 2021 13:05:28 +0200 Subject: [PATCH 3/5] Fix libglvnd bug breaking tests --- .orchestra/config/components/ui/libglvnd.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.orchestra/config/components/ui/libglvnd.yml b/.orchestra/config/components/ui/libglvnd.yml index bb4901ca..23ec3a75 100644 --- a/.orchestra/config/components/ui/libglvnd.yml +++ b/.orchestra/config/components/ui/libglvnd.yml @@ -24,6 +24,9 @@ configure: | "$BUILD_DIR" \ "$BUILD_DIR/source" build_system: ninja +post_install: | + # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=881789 + ln -s libGLX_mesa.so.0 "${TMP_ROOT}${ORCHESTRA_ROOT}/lib64/libGLX_indirect.so.0" build_dependencies: - host-c-toolchain - glibc From f5ce701db1961fc8451f5e279c2fcbaf0807bb43 Mon Sep 17 00:00:00 2001 From: Filippo Cremonese Date: Tue, 17 Aug 2021 18:36:18 +0200 Subject: [PATCH 4/5] QT: Use vendored md4c --- .orchestra/config/components/ui/qt.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.orchestra/config/components/ui/qt.yml b/.orchestra/config/components/ui/qt.yml index d9330246..81b24291 100644 --- a/.orchestra/config/components/ui/qt.yml +++ b/.orchestra/config/components/ui/qt.yml @@ -61,6 +61,7 @@ -no-glib \ -no-cups \ -no-gtk \ + -qt-libmd4c \ -qt-harfbuzz \ -fontconfig \ -system-freetype \ From 0352ec9eed7e944a170f30237b809134f0077a1d Mon Sep 17 00:00:00 2001 From: Filippo Cremonese Date: Tue, 17 Aug 2021 13:09:56 +0200 Subject: [PATCH 5/5] Integrate revng-distributable tests with orchestra --- .orchestra/config/components.yml | 1 + .../config/components/revng_distributable.yml | 58 ++- .orchestra/config/global_options.yml | 3 + .../revng-distributable/dockers/README.md | 33 ++ .../dockers/archlinux.docker | 2 +- .../dockers/broken/gentoo.docker | 12 + .../dockers/{ => broken}/opensuse-15.0.docker | 2 +- .../dockers/{ => broken}/opensuse-15.1.docker | 2 +- .../revng-distributable/dockers/build-all | 6 +- .../dockers/centos-7.docker | 2 +- .../dockers/centos-8.docker | 2 +- .../dockers/debian-10.docker | 2 +- .../dockers/debian-7.docker | 4 +- .../dockers/debian-8.docker | 8 +- .../dockers/debian-9.docker | 2 +- .../dockers/fedora-29.docker | 2 +- .../dockers/fedora-30.docker | 2 +- .../dockers/fedora-31.docker | 2 +- .../dockers/fedora-32.docker | 2 +- .../revng-distributable/dockers/gentoo.docker | 9 - .../dockers/install-revng-dependencies | 473 +++++++++++------- .../dockers/opensuse-15.2.docker | 2 +- .../support/revng-distributable/dockers/run | 2 +- .../revng-distributable/dockers/run-all | 5 +- .../dockers/ubuntu-16.04.docker | 2 +- .../dockers/ubuntu-18.04.docker | 2 +- .../dockers/ubuntu-20.04.docker | 2 +- .../dockers/voidlinux.docker | 2 +- .../support/revng-distributable/test.sh | 61 ++- 29 files changed, 495 insertions(+), 212 deletions(-) create mode 100644 .orchestra/support/revng-distributable/dockers/README.md create mode 100644 .orchestra/support/revng-distributable/dockers/broken/gentoo.docker rename .orchestra/support/revng-distributable/dockers/{ => broken}/opensuse-15.0.docker (62%) rename .orchestra/support/revng-distributable/dockers/{ => broken}/opensuse-15.1.docker (62%) delete mode 100644 .orchestra/support/revng-distributable/dockers/gentoo.docker diff --git a/.orchestra/config/components.yml b/.orchestra/config/components.yml index 204d7904..8cc7f8a3 100644 --- a/.orchestra/config/components.yml +++ b/.orchestra/config/components.yml @@ -22,6 +22,7 @@ environment: - HARD_FLAGS_LINK: #@ str(datavalue("hard_flags_link")) - HARD_FLAGS_LINK_LATE: #@ str(datavalue("hard_flags_link_late")) - HARD_FLAGS_LINK_GOLD: #@ str(datavalue("hard_flags_link_gold")) + - COLD_REVNG_TEST_METHOD: #@ str(datavalue("cold_revng_test_method")) - "-LD_LIBRARY_PATH": "" - "-COMPILER_PATH": "" - "-CPATH": "" diff --git a/.orchestra/config/components/revng_distributable.yml b/.orchestra/config/components/revng_distributable.yml index b2d2826e..022a1dfe 100644 --- a/.orchestra/config/components/revng_distributable.yml +++ b/.orchestra/config/components/revng_distributable.yml @@ -1,6 +1,12 @@ #@ load("@ytt:overlay", "overlay") #@ load("/lib/create_component.lib.yml", "single_build_component") +#! Possible values for the cold_revng_test_method user option: +#! - host-xvfb: run the tests on the host machine, using xvfb and fluxbox +#! - host-x11: run the tests on the host machine, using the host running X11 and window manager +#! - podman-xvbf: run the tests on all the distros using podman and xvfb +#! - podman-forward-x11: run the tests on all the distros, forwarding the host X11 server + #@yaml/text-templated-strings --- #@ def revng_distributable(): @@ -43,8 +49,6 @@ install: | | sed 's!^!/!' \ | rsync \ --archive \ - --verbose \ - --progress \ --exclude "/revng/" \ --exclude "/include/" \ --exclude "/share/man/" \ @@ -132,6 +136,56 @@ install: | cd "${TMP_ROOT}${ORCHESTRA_ROOT}" find . -not -type d -not -path './revng/*' -delete find . -type d -empty -delete + + if test "$RUN_TESTS" -eq 1; then + # CI can't run Docker-in-Docker (or Podman-in-Docker) + # We can only test by spawning a local Xvfb server + if [[ ! -z "${GITLAB_CI:-}" ]]; then + export USE_XVFB=1 + cd "${TMP_ROOT}${ORCHESTRA_ROOT}/revng" + "$ORCHESTRA_DOTDIR/support/revng-distributable/test.sh" + + elif [[ "$COLD_REVNG_TEST_METHOD" == host-* ]]; then + if [[ "$COLD_REVNG_TEST_METHOD" == "host-xvfb" ]]; then + export USE_XVFB=1 + fi + cd "${TMP_ROOT}${ORCHESTRA_ROOT}/revng" + + export COLD_REVNG_KILL_METHOD=wmctrl + "$ORCHESTRA_DOTDIR/support/revng-distributable/test.sh" + + else + if [[ "$COLD_REVNG_TEST_METHOD" == "podman-forward-x11" ]]; then + # Remember: the host must allow access to the X11 socket! (xhost +) + PODMAN_OPTIONS="-v /tmp/.X11-unix:/tmp/.X11-unix" + else + PODMAN_OPTIONS="-e USE_XVFB=1" + fi + + for DOCKERFILE in $ORCHESTRA_DOTDIR/support/revng-distributable/dockers/*.docker; do + DISTRO="$(basename "${DOCKERFILE%%.docker}")" + IMAGE_NAME="revng-test-image:$DISTRO" + + echo "Testing revng-distributable in $DISTRO" + podman build --pull -t "$IMAGE_NAME" -f "$DOCKERFILE" + # TODO: fix the test environment so that we can properly close revng-ui without using kill + if ! podman run -t --rm $PODMAN_OPTIONS \ + --security-opt=seccomp=unconfined \ + -e "COLD_REVNG_KILL_METHOD=kill" \ + -w "/orchestra/.orchestra/$(realpath --relative-to "$ORCHESTRA_DOTDIR" "$TMP_ROOT")/$ORCHESTRA_ROOT/revng" \ + -v "$(readlink -f "$ORCHESTRA_DOTDIR/..")":/orchestra \ + "$IMAGE_NAME" \ + bash -c /orchestra/.orchestra/support/revng-distributable/test.sh; + then + echo "[!] $IMAGE_NAME test failed" + exit 1 + else + echo "[+] $IMAGE_NAME test passed" + fi + done + echo "[+] revng-distributable tests passed" + fi + fi build_dependencies: - ui/cold-revng skip_post_install: true diff --git a/.orchestra/config/global_options.yml b/.orchestra/config/global_options.yml index d3daec06..3f7728f5 100644 --- a/.orchestra/config/global_options.yml +++ b/.orchestra/config/global_options.yml @@ -36,3 +36,6 @@ branches: - develop - master nonredistributable_base_url: "https://dummydomain.com/nonredistributable/" + +#! see cold-revng component for the list of admissible test methods +cold_revng_test_method: podman-xvfb diff --git a/.orchestra/support/revng-distributable/dockers/README.md b/.orchestra/support/revng-distributable/dockers/README.md new file mode 100644 index 00000000..56fb7ead --- /dev/null +++ b/.orchestra/support/revng-distributable/dockers/README.md @@ -0,0 +1,33 @@ +# Status of revng-distributable tests + +## How to run the tests + +To run revng-distributable tests, use `orc install --test revng-distributable`. +By default, tests are run on all supported distros using podman and xvfb. +The test mode can be changed by adding the following to `user_options.yml`: + +```yml +#@overlay/replace +cold_revng_test_method: podman-xvfb +``` + +See the supported test modes in `revng_distributable.yml`. + +## Known-working test modes + +The tests work on the following distros (podman+xvfb): + +- Archlinux +- Centos {7,8} +- Debian {7,8,9,10} +- Fedora {29,30,31,32} +- Opensuse 15.2 +- Ubuntu {16,18,20}.04 +- Voidlinux + +The tests also work running revng-ui directly on the host, using the running X11 server (`host-x11` test mode) + +## Broken distros + +- gentoo: need to revise the scripts which install dependencies +- opensuse 15.{0,1}: zypper segfaults diff --git a/.orchestra/support/revng-distributable/dockers/archlinux.docker b/.orchestra/support/revng-distributable/dockers/archlinux.docker index e793b435..a3f23f45 100644 --- a/.orchestra/support/revng-distributable/dockers/archlinux.docker +++ b/.orchestra/support/revng-distributable/dockers/archlinux.docker @@ -1,7 +1,7 @@ FROM archlinux COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/broken/gentoo.docker b/.orchestra/support/revng-distributable/dockers/broken/gentoo.docker new file mode 100644 index 00000000..f694d69c --- /dev/null +++ b/.orchestra/support/revng-distributable/dockers/broken/gentoo.docker @@ -0,0 +1,12 @@ +FROM gentoo/stage3-amd64 AS base-gentoo + +COPY install-revng-dependencies / +RUN echo 'FEATURES="$FEATURES -sandbox -usersandbox -ipc-sandbox -network-sandbox -pid-sandbox"' >> /etc/portage/make.conf +RUN emerge-webrsync + +FROM base-gentoo +RUN echo "sys-auth/elogind" >> "/etc/portage/package.mask" +RUN /install-revng-dependencies --include-test-dependencies + +ENV DISPLAY=:0 +ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/opensuse-15.0.docker b/.orchestra/support/revng-distributable/dockers/broken/opensuse-15.0.docker similarity index 62% rename from .orchestra/support/revng-distributable/dockers/opensuse-15.0.docker rename to .orchestra/support/revng-distributable/dockers/broken/opensuse-15.0.docker index cf1b7711..b5c2615f 100644 --- a/.orchestra/support/revng-distributable/dockers/opensuse-15.0.docker +++ b/.orchestra/support/revng-distributable/dockers/broken/opensuse-15.0.docker @@ -1,7 +1,7 @@ FROM opensuse/leap:15.0 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/opensuse-15.1.docker b/.orchestra/support/revng-distributable/dockers/broken/opensuse-15.1.docker similarity index 62% rename from .orchestra/support/revng-distributable/dockers/opensuse-15.1.docker rename to .orchestra/support/revng-distributable/dockers/broken/opensuse-15.1.docker index 895864ac..dda9ce5d 100644 --- a/.orchestra/support/revng-distributable/dockers/opensuse-15.1.docker +++ b/.orchestra/support/revng-distributable/dockers/broken/opensuse-15.1.docker @@ -1,7 +1,7 @@ FROM opensuse/leap:15.1 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/build-all b/.orchestra/support/revng-distributable/dockers/build-all index 32c820aa..707daa20 100755 --- a/.orchestra/support/revng-distributable/dockers/build-all +++ b/.orchestra/support/revng-distributable/dockers/build-all @@ -3,8 +3,10 @@ set -x set -e -FILES=$(find -name "*.docker") +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +FILES=$(find "$DIR" -name "*.docker") for FILE in $FILES; do - ./build $(basename "${FILE%.*}") + FILE_PATH=$(realpath "$FILE") + "$DIR/build" "$(basename "${FILE%.*}")" done diff --git a/.orchestra/support/revng-distributable/dockers/centos-7.docker b/.orchestra/support/revng-distributable/dockers/centos-7.docker index 5ecdeadf..08b81afd 100644 --- a/.orchestra/support/revng-distributable/dockers/centos-7.docker +++ b/.orchestra/support/revng-distributable/dockers/centos-7.docker @@ -2,7 +2,7 @@ FROM centos:7 COPY install-revng-dependencies / RUN echo '00000000000000000000000000000000' > /etc/machine-id && \ - /install-revng-dependencies + /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/centos-8.docker b/.orchestra/support/revng-distributable/dockers/centos-8.docker index f75dbaf4..5925d734 100644 --- a/.orchestra/support/revng-distributable/dockers/centos-8.docker +++ b/.orchestra/support/revng-distributable/dockers/centos-8.docker @@ -2,7 +2,7 @@ FROM centos:8 COPY install-revng-dependencies / RUN echo '00000000000000000000000000000000' > /etc/machine-id && \ - /install-revng-dependencies + /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/debian-10.docker b/.orchestra/support/revng-distributable/dockers/debian-10.docker index d5a83b58..6abbd5f9 100644 --- a/.orchestra/support/revng-distributable/dockers/debian-10.docker +++ b/.orchestra/support/revng-distributable/dockers/debian-10.docker @@ -1,7 +1,7 @@ FROM debian:10 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/debian-7.docker b/.orchestra/support/revng-distributable/dockers/debian-7.docker index ffa42a89..bcd70000 100644 --- a/.orchestra/support/revng-distributable/dockers/debian-7.docker +++ b/.orchestra/support/revng-distributable/dockers/debian-7.docker @@ -3,7 +3,9 @@ FROM debian:7 COPY install-revng-dependencies / RUN echo "deb http://archive.debian.org/debian/ wheezy contrib main non-free" > /etc/apt/sources.list && \ echo "deb-src http://archive.debian.org/debian/ wheezy contrib main non-free" >> /etc/apt/sources.list && \ - /install-revng-dependencies + echo "deb http://archive.debian.org/debian-security/ wheezy/updates contrib main non-free" >> /etc/apt/sources.list && \ + echo "Acquire::Check-Valid-Until false;" >> /etc/apt/apt.conf && \ + /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/debian-8.docker b/.orchestra/support/revng-distributable/dockers/debian-8.docker index 639d93d4..b0a282f0 100644 --- a/.orchestra/support/revng-distributable/dockers/debian-8.docker +++ b/.orchestra/support/revng-distributable/dockers/debian-8.docker @@ -1,9 +1,11 @@ FROM debian:8 COPY install-revng-dependencies / -RUN echo "deb http://archive.debian.org/debian/ jessie contrib main non-free" > /etc/apt/sources.list && \ - echo "deb-src http://archive.debian.org/debian/ jessie contrib main non-free" >> /etc/apt/sources.list && \ - /install-revng-dependencies +RUN echo "deb http://deb.debian.org/debian/ jessie contrib main non-free" > /etc/apt/sources.list && \ + echo "deb-src http://deb.debian.org/debian/ jessie contrib main non-free" >> /etc/apt/sources.list && \ + echo "deb http://deb.debian.org/debian-security/ jessie/updates contrib main non-free" >> /etc/apt/sources.list && \ + echo "Acquire::Check-Valid-Until false;" >> /etc/apt/apt.conf && \ + /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/debian-9.docker b/.orchestra/support/revng-distributable/dockers/debian-9.docker index c42755a9..08531a87 100644 --- a/.orchestra/support/revng-distributable/dockers/debian-9.docker +++ b/.orchestra/support/revng-distributable/dockers/debian-9.docker @@ -1,7 +1,7 @@ FROM debian:9 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/fedora-29.docker b/.orchestra/support/revng-distributable/dockers/fedora-29.docker index f9cace40..adade953 100644 --- a/.orchestra/support/revng-distributable/dockers/fedora-29.docker +++ b/.orchestra/support/revng-distributable/dockers/fedora-29.docker @@ -1,7 +1,7 @@ FROM fedora:29 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/fedora-30.docker b/.orchestra/support/revng-distributable/dockers/fedora-30.docker index dd1eca0f..9482f927 100644 --- a/.orchestra/support/revng-distributable/dockers/fedora-30.docker +++ b/.orchestra/support/revng-distributable/dockers/fedora-30.docker @@ -1,7 +1,7 @@ FROM fedora:30 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/fedora-31.docker b/.orchestra/support/revng-distributable/dockers/fedora-31.docker index db7e2d99..76a1dc22 100644 --- a/.orchestra/support/revng-distributable/dockers/fedora-31.docker +++ b/.orchestra/support/revng-distributable/dockers/fedora-31.docker @@ -1,7 +1,7 @@ FROM fedora:31 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/fedora-32.docker b/.orchestra/support/revng-distributable/dockers/fedora-32.docker index 8c2966ae..e7143d7e 100644 --- a/.orchestra/support/revng-distributable/dockers/fedora-32.docker +++ b/.orchestra/support/revng-distributable/dockers/fedora-32.docker @@ -1,7 +1,7 @@ FROM fedora:32 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/gentoo.docker b/.orchestra/support/revng-distributable/dockers/gentoo.docker deleted file mode 100644 index 939630fd..00000000 --- a/.orchestra/support/revng-distributable/dockers/gentoo.docker +++ /dev/null @@ -1,9 +0,0 @@ -FROM gentoo/stage3-amd64 - -COPY install-revng-dependencies / -RUN echo 'FEATURES="$FEATURES -sandbox -usersandbox -ipc-sandbox -network-sandbox -pid-sandbox"' >> /etc/portage/make.conf && \ - emerge --sync && \ - /install-revng-dependencies - -ENV DISPLAY=:0 -ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/install-revng-dependencies b/.orchestra/support/revng-distributable/dockers/install-revng-dependencies index 3b7c78d3..f8580dd5 100755 --- a/.orchestra/support/revng-distributable/dockers/install-revng-dependencies +++ b/.orchestra/support/revng-distributable/dockers/install-revng-dependencies @@ -13,11 +13,40 @@ function fail() { exit 1 } -if test "$1" == "--pretend"; then +function print_usage() { + fail "Usage: $0 [--pretend] [--include-test-dependencies] [ID [VERSION_ID]]" +} + +PRETEND=0 +INCLUDE_TEST_DEPS=0 +while [[ "$1" == --* ]]; do + if [[ "$1" == "--pretend" ]]; then + PRETEND=1 + elif [[ "$1" == "--include-test-dependencies" ]]; then + INCLUDE_TEST_DEPS=1 + elif [[ "$1" == "--help" ]]; then + print_usage + else + fail "Unrecognized option: $1" + fi shift - PRETEND=1 +done + +if test "$#" -eq 0; then + # Detect distro automatically + if ! test -e "$RELEASE_INFO"; then + fail "$RELEASE_INFO does not exists, your distribution is not supported." + fi + # The following line is needed to shut up shellcheck + # shellcheck source=/etc/os-release + source "$RELEASE_INFO" + +elif test "$#" -eq 1 || test "$#" -eq 2; then + # Manually specified distro + ID="$1" + VERSION_ID="$2" else - PRETEND=0 + print_usage fi if test "$EUID" -ne 0; then @@ -43,175 +72,251 @@ function asuser() { run "$@" } -if test "$#" -eq 0; then - # Detect distro automatically - if ! test -e "$RELEASE_INFO"; then - fail "$RELEASE_INFO does not exists, your distribution is not supported." - fi - - source /etc/os-release - -elif test "$#" -eq 1 || test "$#" -eq 2; then - - # Manually specified distro - ID="$1" - VERSION_ID="$2" - -else - fail "Usage: $0 [--pretend] [ID VERSION_ID]" -fi +function debian_install_python36() { + # Adapted from https://unix.stackexchange.com/a/332658 + asroot env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + make \ + build-essential \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + wget \ + curl \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libffi-dev \ + liblzma-dev + mkdir /tmp/python3-build + cd /tmp/python3-build + PYTHONVERSION="3.6.14" + wget -O "Python-${PYTHONVERSION}.tgz" "https://www.python.org/ftp/python/${PYTHONVERSION}/Python-${PYTHONVERSION}.tgz" + tar xvf "Python-${PYTHONVERSION}.tgz" + cd "Python-${PYTHONVERSION}" + ./configure --enable-shared --with-ensurepip=install + make "-j$(nproc)" + asroot make altinstall + asroot rm -rf /tmp/python3-build + cd # Needed otherwise pip will crash because the cwd does not exist + asroot ldconfig # Needed otherwise python shared lib is not found + asroot update-alternatives --force --install /usr/bin/python python /usr/local/bin/python3.6 50 + asroot update-alternatives --force --install /usr/bin/python3 python3 /usr/local/bin/python3.6 50 + asroot update-alternatives --force --install /usr/bin/pip3 pip3 /usr/local/bin/pip3.6 50 +} -PIP_PACKAGES="backports.shutil_which pyelftools" +PIP_PACKAGES=( + backports.shutil_which + pyelftools +) -if test "$ID" == "arch"; then +if [[ "$ID" == "arch" ]]; then # Handle Arch Linux - asroot pacman \ - -Syu \ - --noconfirm \ - core/expat \ - core/krb5 \ - core/xz \ - extra/fontconfig \ - extra/ttf-dejavu \ - extra/libsm \ - extra/libxdamage \ - extra/libxfixes \ - extra/libxxf86vm \ - python-pip \ + PACKAGES=( + core/expat + core/krb5 + core/xz + extra/fontconfig + extra/ttf-dejavu + extra/libsm + extra/libxdamage + extra/libxfixes + extra/libxxf86vm + python-pip xdotool - asuser pip3 install --user $PIP_PACKAGES + ) + if [[ "$INCLUDE_TEST_DEPS" == 1 ]]; then + PACKAGES+=( + fluxbox + xorg-server-xvfb + wmctrl + ) + fi + asroot pacman -Syu --noconfirm "${PACKAGES[@]}" + asuser pip3 install --user "${PIP_PACKAGES[@]}" + elif test "$ID" == "centos"; then # Handle CentOS function common() { + PACKAGES=( + libSM + fontconfig + libXext + libXfixes + libXdamage + libXxf86vm + dbus-libs + libXdmcp + xdotool + python3-pip + glibc-devel + ) + if [[ "$INCLUDE_TEST_DEPS" == 1 ]]; then + PACKAGES+=( + fluxbox + xorg-x11-server-Xvfb + wmctrl + ) + fi asroot yum check-update || true asroot yum install -y epel-release asroot yum update -y - asroot yum install -y \ - libSM \ - fontconfig \ - libXext \ - libXfixes \ - libXdamage \ - libXxf86vm \ - dbus-libs \ - libXdmcp \ - xdotool \ - python3-pip \ - glibc-devel - asuser pip3 install --user $PIP_PACKAGES + asroot yum install -y "${PACKAGES[@]}" + asuser pip3 install --user "${PIP_PACKAGES[@]}" } - if test "$VERSION_ID" == "7"; then - common - elif test "$VERSION_ID" == "8"; then + + if [[ "$VERSION_ID" =~ ^(7|8)$ ]]; then common else fail "Your CentOS version is not supported" fi + elif test "$ID" == "debian"; then # Handle Debian function common() { + PACKAGES=( + libgssapi-krb5-2 + libdbus-1-3 + libsm6 + libfontconfig1 + libxext6 + libxfixes3 + libxdamage1 + libxxf86vm1 + python3-pip + xdotool + ) + if [[ "$VERSION_ID" =~ ^7|8|9$ ]]; then + PACKAGES+=( + realpath + ) + fi + if [[ "$INCLUDE_TEST_DEPS" == 1 ]]; then + PACKAGES+=( + fluxbox + xvfb + wmctrl + ) + fi asroot apt-get update - asroot env DEBIAN_FRONTEND=noninteractive apt-get install \ - libgssapi-krb5-2 \ - libdbus-1-3 \ - libsm6 \ - libfontconfig1 \ - libxext6 \ - libxfixes3 \ - libxdamage1 \ - libxxf86vm1 \ - python3-pip \ - xdotool \ - "$@" + asroot env DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes "${PACKAGES[@]}" } - if test "$VERSION_ID" == "7"; then - common -y realpath wget - asroot wget 'http://archive.debian.org/debian/pool/main/e/eglibc/libc6-dev_2.13-38+deb7u10_amd64.deb' - asroot dpkg -i --force-all libc6-dev_2.13-38+deb7u10_amd64.deb - asroot rm libc6-dev_2.13-38+deb7u10_amd64.deb - asuser pip-3.2 install --user --index-url=https://pypi.python.org/simple/ $PIP_PACKAGES - elif test "$VERSION_ID" == "8"; then - common -y --force-yes realpath - asuser pip3 install --user $PIP_PACKAGES - elif test "$VERSION_ID" == "9"; then - common -y --force-yes realpath - asuser pip3 install --user $PIP_PACKAGES - elif test "$VERSION_ID" == "10"; then - common -y - asuser pip3 install --user $PIP_PACKAGES - else + + if ! [[ "$VERSION_ID" =~ ^(7|8|9|10)$ ]]; then fail "Your debian version is not supported" fi + + # Install system-provided packages + common + + # Install python 3.6 where required + if [[ "$VERSION_ID" =~ ^(7|8|9)$ ]]; then + debian_install_python36 + fi + + # Install python libraries + PIP_FLAGS=() + if [[ "$VERSION_ID" == 7 ]]; then + PIP_FLAGS=( + --index-url=https://pypi.python.org/simple/ + ) + fi + asuser pip3 install --user "${PIP_FLAGS[@]}" "${PIP_PACKAGES[@]}" + elif test "$ID" == "fedora"; then # Handle Fedora function common() { - asroot yum check-update || true - asroot yum install -y libSM \ - fontconfig \ - libXext \ - libXfixes \ - libXdamage \ - libXxf86vm \ - dbus-libs \ - libXdmcp \ - xdotool \ - python3-pip \ + PACKAGES=( + libSM + fontconfig + libXext + libXfixes + libXdamage + libXxf86vm + dbus-libs + libXdmcp + xdotool + python3-pip glibc-devel - asuser pip3 install --user $PIP_PACKAGES + ) + if [[ "$INCLUDE_TEST_DEPS" == 1 ]]; then + PACKAGES+=( + fluxbox + xorg-x11-server-Xvfb + wmctrl + ) + fi + asroot yum check-update || true + asroot yum install -y "${PACKAGES[@]}" + asuser pip3 install --user "${PIP_PACKAGES[@]}" } - if test "$VERSION_ID" == "29"; then - common - elif test "$VERSION_ID" == "30"; then - common - elif test "$VERSION_ID" == "31"; then - common - elif test "$VERSION_ID" == "32"; then + if [[ "$VERSION_ID" =~ ^(29|30|31|32)$ ]]; then common else fail "Your fedora version is not supported" fi + elif test "$ID" == "gentoo"; then # Handle Gentoo + PACKAGES=( + x11-libs/libSM + media-libs/fontconfig + app-crypt/mit-krb5 + x11-libs/libXfixes + x11-libs/libXdamage + x11-libs/libXxf86vm + sys-apps/dbus + dev-python/pip + dev-lang/python:3.9 + xdotool + ) + if [[ "$INCLUDE_TEST_DEPS" == 1 ]]; then + PACKAGES+=( + x11-base/xorg-server + x11-wm/fluxbox + x11-misc/wmctrl + ) + echo ">=x11-wm/fluxbox-1.3.7-r4 -imlib -nls -slit -systray -toolbar -truetype" >> /etc/portage/package.use/fluxbox.use + echo ">=media-libs/libglvnd-1.3.3 X" >> /etc/portage/package.use/libglvnd.use + echo ">=media-libs/freetype-2.10.4 harfbuzz" >> /etc/portage/package.use/freetype.use + echo ">=x11-libs/cairo-1.16.0-r4 X" >> /etc/portage/package.use/cairo.use + export USE="xvfb -xorg -elogind" + fi asroot emerge --sync - asroot emerge -v \ - --update \ - --deep \ - --exclude sandbox \ - x11-libs/libSM \ - media-libs/fontconfig \ - app-crypt/mit-krb5 \ - x11-libs/libXfixes \ - x11-libs/libXdamage \ - x11-libs/libXxf86vm \ - sys-apps/dbus \ - dev-python/pip \ - dev-lang/python:3.8 \ - xdotool - asuser pip3 install --user $PIP_PACKAGES + asroot emerge -v --update --deep --exclude sandbox "${PACKAGES[@]}" + asroot eselect python set --python3 python3.9 + asuser pip3 install --user "${PIP_PACKAGES[@]}" elif test "$ID" == "opensuse-leap"; then # Handle openSUSE function common() { + PACKAGES=( + libSM6 + fontconfig + libXext6 + libXfixes3 + libXdamage1 + libXxf86vm1 + libdbus-1-3 + dejavu-fonts + libXdmcp6 + xdotool + python3-pip + glibc-devel + ) + if [[ "$INCLUDE_TEST_DEPS" == 1 ]]; then + PACKAGES+=( + fluxbox + xvfb-run + wmctrl + ) + fi asroot zypper -n refresh - asroot zypper -n install \ - libSM6 \ - fontconfig \ - libXext6 \ - libXfixes3 \ - libXdamage1 \ - libXxf86vm1 \ - libdbus-1-3 \ - dejavu-fonts \ - libXdmcp6 \ - xdotool \ - python3-pip \ - glibc-devel - asuser pip3 install --user $PIP_PACKAGES + asroot zypper -n install "${PACKAGES[@]}" + asuser pip3 install --user "${PIP_PACKAGES[@]}" } - if test "$VERSION_ID" == "15.0"; then - common - elif test "$VERSION_ID" == "15.1"; then - common - elif test "$VERSION_ID" == "15.2"; then + if [[ "$VERSION_ID" =~ ^15\.(0|1|2)$ ]]; then common else fail "Your opensuse version is not supported" @@ -219,49 +324,77 @@ elif test "$ID" == "opensuse-leap"; then elif test "$ID" == "ubuntu"; then # Handle Ubuntu function common() { + PACKAGES=( + libgssapi-krb5-2 + libdbus-1-3 + libsm6 + libfontconfig1 + libxext6 + libxfixes3 + libxdamage1 + libxxf86vm1 + python3-pip + xdotool + ) + if [[ "$VERSION_ID" == "16.04" ]]; then + PACKAGES+=( + realpath + ) + fi + if [[ "$INCLUDE_TEST_DEPS" == 1 ]]; then + PACKAGES+=( + fluxbox + xvfb + wmctrl + ) + fi + asroot apt-get update - asroot env DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libgssapi-krb5-2 \ - libdbus-1-3 \ - libsm6 \ - libfontconfig1 \ - libxext6 \ - libxfixes3 \ - libxdamage1 \ - libxxf86vm1 \ - python3-pip \ - xdotool \ - "$@" - asuser pip3 install --user $PIP_PACKAGES + asroot env DEBIAN_FRONTEND=noninteractive apt-get install -y "${PACKAGES[@]}" } - if test "$VERSION_ID" == "16.04"; then - common realpath - elif test "$VERSION_ID" == "18.04"; then - common - elif test "$VERSION_ID" == "20.04"; then - common - else + if ! [[ "$VERSION_ID" =~ ^(16|18|20)\.04$ ]]; then fail "Your ubuntu version is not supported" fi + + # Install system-provided dependencies + common + + # Install python 3.6 where required + if test "$VERSION_ID" == "16.04"; then + debian_install_python36 + fi + + # Install python libraries + asuser pip3 install --user "${PIP_PACKAGES[@]}" + elif test "$ID" == "void"; then # Handle Void Linux - asroot xbps-install -Suy \ - bash \ - expat \ - mit-krb5 \ - xz \ - fontconfig \ - libSM \ - libXdamage \ - libXfixes \ - libXxf86vm \ - dbus \ - dejavu-fonts-ttf \ - xdotool \ - python3-pip \ - glibc-devel \ - libstdc++ - asuser pip3 install --user $PIP_PACKAGES + PACKAGES=( + bash + expat + mit-krb5 + xz + fontconfig + libSM + libXdamage + libXfixes + libXxf86vm + dbus + dejavu-fonts-ttf + xdotool + python3-pip + glibc-devel + libstdc++ + ) + if [[ "$INCLUDE_TEST_DEPS" == 1 ]]; then + PACKAGES+=( + fluxbox + xorg-server-xvfb + wmctrl + ) + fi + asroot xbps-install -Suy "${PACKAGES[@]}" + asuser pip install --user "${PIP_PACKAGES[@]}" else fail "Your distribution is not supported" fi diff --git a/.orchestra/support/revng-distributable/dockers/opensuse-15.2.docker b/.orchestra/support/revng-distributable/dockers/opensuse-15.2.docker index 10820d39..46de2b09 100644 --- a/.orchestra/support/revng-distributable/dockers/opensuse-15.2.docker +++ b/.orchestra/support/revng-distributable/dockers/opensuse-15.2.docker @@ -1,7 +1,7 @@ FROM opensuse/leap:15.2 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/run b/.orchestra/support/revng-distributable/dockers/run index 06d22931..c400ac22 100755 --- a/.orchestra/support/revng-distributable/dockers/run +++ b/.orchestra/support/revng-distributable/dockers/run @@ -11,6 +11,6 @@ podman \ -ti \ --rm \ -v /tmp/.X11-unix:/tmp/.X11-unix \ - -v $(readlink -f $PWD/../../../..):/orchestra \ + -v "$(readlink -f "$DIR/../../../..")":/orchestra \ revng-user:"$IMAGE" \ bash -c "cd /orchestra; source root/revng/environment; $*" diff --git a/.orchestra/support/revng-distributable/dockers/run-all b/.orchestra/support/revng-distributable/dockers/run-all index 6e31941a..312fa9ec 100755 --- a/.orchestra/support/revng-distributable/dockers/run-all +++ b/.orchestra/support/revng-distributable/dockers/run-all @@ -2,8 +2,9 @@ set -e -FILES=$(find -name "*.docker") +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +FILES=$(find "$DIR" -name "*.docker") for FILE in $FILES; do - ./run $(basename "${FILE%.*}") "$@" + ./run "$(basename "${FILE%.*}")" "$@" done diff --git a/.orchestra/support/revng-distributable/dockers/ubuntu-16.04.docker b/.orchestra/support/revng-distributable/dockers/ubuntu-16.04.docker index 73c45ff5..728aa8af 100644 --- a/.orchestra/support/revng-distributable/dockers/ubuntu-16.04.docker +++ b/.orchestra/support/revng-distributable/dockers/ubuntu-16.04.docker @@ -1,7 +1,7 @@ FROM ubuntu:16.04 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/ubuntu-18.04.docker b/.orchestra/support/revng-distributable/dockers/ubuntu-18.04.docker index 164635b2..80c45a53 100644 --- a/.orchestra/support/revng-distributable/dockers/ubuntu-18.04.docker +++ b/.orchestra/support/revng-distributable/dockers/ubuntu-18.04.docker @@ -1,7 +1,7 @@ FROM ubuntu:18.04 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/ubuntu-20.04.docker b/.orchestra/support/revng-distributable/dockers/ubuntu-20.04.docker index d6180913..2854adcb 100644 --- a/.orchestra/support/revng-distributable/dockers/ubuntu-20.04.docker +++ b/.orchestra/support/revng-distributable/dockers/ubuntu-20.04.docker @@ -1,7 +1,7 @@ FROM ubuntu:20.04 COPY install-revng-dependencies / -RUN /install-revng-dependencies +RUN /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/dockers/voidlinux.docker b/.orchestra/support/revng-distributable/dockers/voidlinux.docker index 6f88dde8..effe6301 100644 --- a/.orchestra/support/revng-distributable/dockers/voidlinux.docker +++ b/.orchestra/support/revng-distributable/dockers/voidlinux.docker @@ -2,7 +2,7 @@ FROM voidlinux/voidlinux COPY install-revng-dependencies / RUN xbps-install -Suy bash && \ - /install-revng-dependencies + /install-revng-dependencies --include-test-dependencies ENV DISPLAY=:0 ENV QT_DEBUG_PLUGINS=1 diff --git a/.orchestra/support/revng-distributable/test.sh b/.orchestra/support/revng-distributable/test.sh index 1db7a9dc..ea00f5ad 100755 --- a/.orchestra/support/revng-distributable/test.sh +++ b/.orchestra/support/revng-distributable/test.sh @@ -1,17 +1,58 @@ #!/bin/bash set -e -set -x +export QT_DEBUG_PLUGINS=1 + +function start_xvfb() { + echo "Starting xvfb" + export XAUTHORITY=/tmp/.Xauthority + export DISPLAY=:99 + + touch "$XAUTHORITY" + Xvfb "$DISPLAY" -screen 0 1920x1080x24 -auth $XAUTHORITY -ac +extension GLX +render -noreset & + XVFB_PID="$!" + # TODO: find a way to reliably tell when Xvfb started up without using sleep (xprop?) + # Wait for Xvfb to start up + sleep 5 + + fluxbox -sync & + WM_PID="$!" + # TODO: find a way to reliably tell when fluxbox started up without using sleep (wmctrl?) + # Wait for the WM to start up + sleep 5 +} + +function stop_xvfb() { + echo "Stopping xvfb" + kill -TERM "$WM_PID" + wait "$WM_PID" + kill -TERM "$XVFB_PID" + wait "$XVFB_PID" +} + +function kill_ui () { + # Kills revng-ui and waits for its termination + # $1: revng-ui PID + local PID="$1" + + if [[ "$COLD_REVNG_KILL_METHOD" == "kill" ]]; then + kill "$PID" + elif [[ "$COLD_REVNG_KILL_METHOD" == "kill9" ]]; then + kill -9 "$PID" + elif [[ "$COLD_REVNG_KILL_METHOD" == "wmctrl" ]]; then + wmctrl -v -c cold-revng || true + else + xdotool search cold-revng windowactivate --sync key --window 0 --clearmodifiers alt+F4 + fi + wait "$PID" || true +} function test_ui() { ./revng ui "$@" & PID="$!" - sleep 3; + sleep 10; if test -d "/proc/$PID"; then - xdotool \ - windowactivate --sync $(xdotool search --classname '.*revng.*') \ - key --clearmodifiers --delay 100 alt+F4 & - wait "$PID" + kill_ui "$PID" else return 1 fi @@ -22,6 +63,10 @@ if test -e "$TARGET"; then cd "$TARGET" fi +if [[ "$USE_XVFB" == 1 ]]; then + start_xvfb +fi + test_ui ls root/share/revng/qa/tests/runtime/*/abi-enforced-for-decompilation/*.bc test_ui root/share/revng/qa/tests/runtime/x86_64/abi-enforced-for-decompilation/calc.bc @@ -51,3 +96,7 @@ test_ui /tmp/calc.for-decompilation.ll -o /tmp/calc.translated /tmp/calc.translated '(+ 3 5)' + +if [[ "$USE_XVFB" == 1 ]]; then + stop_xvfb +fi