From 54f1c1e9c22f74b2a101461861f6fed59f9f57df Mon Sep 17 00:00:00 2001 From: Filippo Cremonese Date: Mon, 12 Jul 2021 12:28:41 +0200 Subject: [PATCH 1/3] Update ci entrypoint --- .orchestra/ci/ci.sh | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.orchestra/ci/ci.sh b/.orchestra/ci/ci.sh index d0155da5..18ab022d 100755 --- a/.orchestra/ci/ci.sh +++ b/.orchestra/ci/ci.sh @@ -2,15 +2,9 @@ # rev.ng CI entrypoint script # This script checks out the correct configuration branch and initializes -# variables for the actual CI script (ci-run.sh) -# -# Parameters are supplied as environment variables. -# -# Optional parameters: -# -# PUSHED_REF: -# orchestra config commit/branch to use. -# Normally set by Gitlab or whoever triggers the CI. +# variables, then it runs the appropriate script for the given CI stage +# Parameters: +# $1 - CI stage: [build|post-build] DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" ORCHESTRA_DIR="$DIR/../.." @@ -38,6 +32,11 @@ function log_err() { set -e +if ! [[ "$1" =~ ^(post-)?build$ ]]; then + log_err "The first parameter of ci.sh must be 'build' or 'post-build'!" + exit 1 +fi + # Determine target branch # # PUSHED_REF contains the git ref that was pushed and triggered the CI. @@ -87,4 +86,8 @@ export COMPONENT_TARGET_BRANCH # Run "true" CI script log "Starting ci-run with COMPONENT_TARGET_BRANCH=$COMPONENT_TARGET_BRANCH" -"$DIR/ci-run.sh" +if [[ "$1" == "build" ]]; then + "$DIR/ci-run.sh" +else + "$DIR/post-build.sh" +fi From b1b332e30ee7e83cd8fe5d9bafdb1d6c58ac6cb6 Mon Sep 17 00:00:00 2001 From: Filippo Cremonese Date: Tue, 19 Jan 2021 10:16:33 +0100 Subject: [PATCH 2/3] Support non optimized builds --- .orchestra/ci/ci-hook/app.py | 21 ++++++++++++------- .orchestra/ci/ci-run.sh | 10 +++++++-- .orchestra/config/components/caliban.yml | 3 ++- .orchestra/config/components/llvm.yml | 2 +- .orchestra/config/components/qemu.yml | 2 +- .orchestra/config/components/revng.yml | 3 ++- .orchestra/config/components/revng_c.yml | 3 ++- .../config/components/ui/adaptagrams.yml | 3 ++- .../config/components/ui/cold_revng.yml | 3 ++- .orchestra/config/components/ui/qt.yml | 2 +- .../config/components/ui/qt_creator.yml | 3 ++- .../config/components/ui/quickqanava.yml | 3 ++- .orchestra/config/global_options.yml | 2 ++ 13 files changed, 40 insertions(+), 20 deletions(-) diff --git a/.orchestra/ci/ci-hook/app.py b/.orchestra/ci/ci-hook/app.py index 5e39eaf3..2dbd498a 100755 --- a/.orchestra/ci/ci-hook/app.py +++ b/.orchestra/ci/ci-hook/app.py @@ -44,6 +44,7 @@ def get_env_or_fail(var_name): mapping = config["github_to_gitlab_mapping"] default_user = config["default_user"] ci_user = config["ci_user"] +revng_components_build_names = ["optimized", "debug", "release"] ORCHESTRA_CONFIG_REPO_HTTP_URL = config["orchestra_config_repo_http_url"] ORCHESTRA_CONFIG_REPO_SSH_URL = config["orchestra_config_repo_ssh_url"] @@ -122,14 +123,18 @@ def trigger_ci(username, repo_url, ref, before, after): variables["SSH_PRIVATE_KEY"] = revng_push_ci_private_key variables["PUSH_CHANGES"] = "1" - parameters = { - "ref": BRANCH, - "variables": [{"key": key, "value": value} - for key, value - in variables.items()] - } - print(json.dumps(parameters, indent=2)) - pipeline = project.pipelines.create(parameters) + # Create a job for each build name + for default_build_name in revng_components_build_names: + variables["REVNG_COMPONENTS_DEFAULT_BUILD"] = default_build_name + + parameters = { + "ref": BRANCH, + "variables": [{"key": key, "value": value} + for key, value + in variables.items()] + } + print(json.dumps(parameters, indent=2)) + pipeline = project.pipelines.create(parameters) @app.route('/ci-hook/gitlab', methods=["POST"]) diff --git a/.orchestra/ci/ci-run.sh b/.orchestra/ci/ci-run.sh index cfbdb5a6..afc57acb 100755 --- a/.orchestra/ci/ci-run.sh +++ b/.orchestra/ci/ci-run.sh @@ -25,6 +25,7 @@ # PROMOTE_BRANCHES: if == 1, promote next-* branches # PUSH_CHANGES: # if == 1, push binary archives and promote next-* branches +# REVNG_COMPONENTS_DEFAULT_BUILD: the default build for revng core components # PUSH_BINARY_ARCHIVE_EMAIL: used as author's email in binary archive commit # PUSH_BINARY_ARCHIVE_NAME: used as author's name in binary archive commit # SSH_PRIVATE_KEY: private key used to push binary archives @@ -139,7 +140,7 @@ if test -n "$TARGET_COMPONENTS_URL"; then | grep '^Component' \ | cut -d' ' -f2)" if test -z "$NEW_COMPONENT"; then - log "Warning: ignoring URL $TARGET_COMPONENT_URL since it doesn't match any component" + log_err "Warning: ignoring URL $TARGET_COMPONENT_URL since it doesn't match any component" else TARGET_COMPONENTS="$NEW_COMPONENT $TARGET_COMPONENTS" fi @@ -175,7 +176,12 @@ cat >> "$USER_OPTIONS" <> "$USER_OPTIONS" + echo "revng_components_default_build: $REVNG_COMPONENTS_DEFAULT_BUILD" >> "$USER_OPTIONS" +fi + log "User options:" cat "$USER_OPTIONS" diff --git a/.orchestra/config/components/caliban.yml b/.orchestra/config/components/caliban.yml index 244083cc..dba392ad 100644 --- a/.orchestra/config/components/caliban.yml +++ b/.orchestra/config/components/caliban.yml @@ -1,3 +1,4 @@ +#@ load("@ytt:data", "data") #@ load("@ytt:overlay", "overlay") #@ load("/lib/cmake.lib.yml", "cmake_boost_configuration", "typical_cmake_builds") @@ -29,5 +30,5 @@ components: repository: caliban license: LICENSE.md binary_archives: private - default_build: optimized + default_build: #@ data.values.revng_components_default_build builds: #@ typical_cmake_builds(**build_args()) diff --git a/.orchestra/config/components/llvm.yml b/.orchestra/config/components/llvm.yml index ca65cfb3..36d8bf04 100644 --- a/.orchestra/config/components/llvm.yml +++ b/.orchestra/config/components/llvm.yml @@ -19,7 +19,7 @@ #@ def llvm_component(use_asan=False): repository: llvm-project license: llvm/LICENSE.TXT -default_build: optimized +default_build: #@ data.values.revng_components_default_build builds: #@ for flavor, f_options in typical_project_flavors(use_asan=use_asan).items(): #@ build_type = f_options["cmake_build_type"] diff --git a/.orchestra/config/components/qemu.yml b/.orchestra/config/components/qemu.yml index a7aca6b4..597e14b8 100644 --- a/.orchestra/config/components/qemu.yml +++ b/.orchestra/config/components/qemu.yml @@ -8,7 +8,7 @@ #@ def qemu_component(use_asan=False): repository: qemu license: LICENSE -default_build: optimized +default_build: #@ data.values.revng_components_default_build builds: #@ for flavor, f_options in typical_project_flavors(use_asan=use_asan).items(): #@ build_type = f_options["cmake_build_type"] diff --git a/.orchestra/config/components/revng.yml b/.orchestra/config/components/revng.yml index 9f2261a4..289bd29f 100644 --- a/.orchestra/config/components/revng.yml +++ b/.orchestra/config/components/revng.yml @@ -1,3 +1,4 @@ +#@ load("@ytt:data", "data") #@ load("@ytt:overlay", "overlay") #@ load("/lib/cmake.lib.yml", "cmake_boost_configuration", "typical_cmake_builds") @@ -26,5 +27,5 @@ components: revng: repository: revng license: LICENSE.md - default_build: optimized + default_build: #@ data.values.revng_components_default_build builds: #@ typical_cmake_builds(**build_args()) diff --git a/.orchestra/config/components/revng_c.yml b/.orchestra/config/components/revng_c.yml index 06e5595b..23abe969 100644 --- a/.orchestra/config/components/revng_c.yml +++ b/.orchestra/config/components/revng_c.yml @@ -1,3 +1,4 @@ +#@ load("@ytt:data", "data") #@ load("@ytt:overlay", "overlay") #@ load("/lib/cmake.lib.yml", "cmake_boost_configuration", "typical_cmake_builds") @@ -24,6 +25,6 @@ components: revng-c: repository: revng-c license: LICENSE.md - default_build: optimized + default_build: #@ data.values.revng_components_default_build binary_archives: private builds: #@ typical_cmake_builds(**build_args()) diff --git a/.orchestra/config/components/ui/adaptagrams.yml b/.orchestra/config/components/ui/adaptagrams.yml index f06a6a60..9988ef50 100644 --- a/.orchestra/config/components/ui/adaptagrams.yml +++ b/.orchestra/config/components/ui/adaptagrams.yml @@ -1,3 +1,4 @@ +#@ load("@ytt:data", "data") #@ load("@ytt:overlay", "overlay") #@ load("/lib/cmake.lib.yml", "typical_cmake_builds") @@ -19,5 +20,5 @@ components: ui/adaptagrams: repository: ui-adaptagrams license: README.md - default_build: optimized + default_build: #@ data.values.revng_components_default_build builds: #@ typical_cmake_builds(**build_args()) diff --git a/.orchestra/config/components/ui/cold_revng.yml b/.orchestra/config/components/ui/cold_revng.yml index 137e5c8e..c8e69f27 100644 --- a/.orchestra/config/components/ui/cold_revng.yml +++ b/.orchestra/config/components/ui/cold_revng.yml @@ -1,3 +1,4 @@ +#@ load("@ytt:data", "data") #@ load("@ytt:overlay", "overlay") #@ load("@ytt:template", "template") @@ -39,5 +40,5 @@ components: repository: ui-cold-revng license: LICENSE.md binary_archives: private - default_build: optimized + default_build: #@ data.values.revng_components_default_build builds: #@ typical_cmake_builds(**build_args()) diff --git a/.orchestra/config/components/ui/qt.yml b/.orchestra/config/components/ui/qt.yml index d9330246..021b9e57 100644 --- a/.orchestra/config/components/ui/qt.yml +++ b/.orchestra/config/components/ui/qt.yml @@ -98,7 +98,7 @@ --- #@ def qt_component(): license: LICENSE.GPLv3 -default_build: optimized +default_build: #@ data.values.revng_components_default_build builds: #@ for flavor, opts in qt_optimization_flavors.items(): #@ extra_qmake_options = opts["extra_qmake_options"] diff --git a/.orchestra/config/components/ui/qt_creator.yml b/.orchestra/config/components/ui/qt_creator.yml index 5caeb309..1d466fef 100644 --- a/.orchestra/config/components/ui/qt_creator.yml +++ b/.orchestra/config/components/ui/qt_creator.yml @@ -1,3 +1,4 @@ +#@ load("@ytt:data", "data") #@ load("@ytt:overlay", "overlay") #@ load("/lib/make.lib.yml", "make") #@ load("/lib/shell.lib.yml", "expand_args") @@ -41,7 +42,7 @@ #@ def qt_creator_component(): repository: ui-qt-creator license: LICENSE.GPL3-EXCEPT -default_build: optimized +default_build: #@ data.values.revng_components_default_build builds: #@ for flavor, opts in qmake_optimization_flavors.items(): #@ extra_qmake_args = opts["extra_qmake_args"] diff --git a/.orchestra/config/components/ui/quickqanava.yml b/.orchestra/config/components/ui/quickqanava.yml index f18051f9..dccdf78b 100644 --- a/.orchestra/config/components/ui/quickqanava.yml +++ b/.orchestra/config/components/ui/quickqanava.yml @@ -1,3 +1,4 @@ +#@ load("@ytt:data", "data") #@ load("@ytt:overlay", "overlay") #@ load("/lib/cmake.lib.yml", "cmake_boost_configuration", "typical_cmake_builds") @@ -22,5 +23,5 @@ components: ui/quickqanava: repository: ui-quickqanava license: licence.txt - default_build: optimized + default_build: #@ data.values.revng_components_default_build builds: #@ typical_cmake_builds(**build_args()) diff --git a/.orchestra/config/global_options.yml b/.orchestra/config/global_options.yml index d3daec06..50fdd963 100644 --- a/.orchestra/config/global_options.yml +++ b/.orchestra/config/global_options.yml @@ -36,3 +36,5 @@ branches: - develop - master nonredistributable_base_url: "https://dummydomain.com/nonredistributable/" + +revng_components_default_build: optimized From 057aa411238f063233e45365af8736841eec43a8 Mon Sep 17 00:00:00 2001 From: Filippo Cremonese Date: Fri, 22 Jan 2021 10:36:56 +0100 Subject: [PATCH 3/3] ui/qt: use -j1 with make install --- .orchestra/config/components/ui/qt.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.orchestra/config/components/ui/qt.yml b/.orchestra/config/components/ui/qt.yml index 021b9e57..f37baa57 100644 --- a/.orchestra/config/components/ui/qt.yml +++ b/.orchestra/config/components/ui/qt.yml @@ -1,6 +1,6 @@ #@ load("@ytt:data", "data") #@ load("@ytt:overlay", "overlay") -#@ load("/lib/make.lib.yml", "make") +#@ load("/lib/make.lib.yml", "make", "serial_make") #@ load("/lib/shell.lib.yml", "expand_args") #@ load("/lib/optimization_flavors.lib.yml", "qt_optimization_flavors") @@ -115,7 +115,7 @@ builds: cd "$BUILD_DIR" (@= make @) - (@= make @) install INSTALL_ROOT=$TMP_ROOT + (@= serial_make @) install INSTALL_ROOT=$TMP_ROOT rm "${TMP_ROOT}${ORCHESTRA_ROOT}/lib64/cmake/Qt5Gui/Qt5GuiConfigExtras.cmake" touch "${TMP_ROOT}${ORCHESTRA_ROOT}/lib64/cmake/Qt5Gui/Qt5GuiConfigExtras.cmake"