diff --git a/dev/ci/list_python_tasks.sh b/dev/ci/list_python_tasks.sh new file mode 100755 index 00000000..3f17968a --- /dev/null +++ b/dev/ci/list_python_tasks.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Outputs the list of all CI python tasks that are available under the current +# working directory. CI python tasks are taskipy task declared in the project +# configuration file (pyproject.toml), whose name ends in "_ci". Tasks whose +# name end in "_ocaml_ci" are additionally identified as requiring the full +# set of FM dependencies to be built. +# +# In the output, each CI python task gets a line +# ,,,, +# where: +# - is the directory of the project defining that task, +# - is the full task name (ending in "_ci"), +# - is either "quick" or "ocaml", +# - is the python project name, as defined in the toml file, +# - is without its "_ci" / "_ocaml_ci" suffix. +# +# The output can be computed with multiple processes, and the number of jobs +# is controlled via the NBJOBS environment variable. + +project_tasks(){ + PROJECT_CONFIG="$1" + PROJECT_DIR="$(dirname "${PROJECT_CONFIG}")" + PROJECT_NAME=$(uvx --quiet --from toml-cli toml get --toml-path "${PROJECT_CONFIG}" project.name) + + for TASK in $( (uvx --quiet --directory ${PROJECT_DIR} --from taskipy task --list || echo -n) | grep -Eo '^[^ ]+_ci\b' ); do + KIND="$( (echo "${TASK}" | grep -Eq '_ocaml_ci$') && echo -n "ocaml" || echo -n "quick" )" + SHORT=$(echo "${TASK}" | sed -re 's/(_ocaml)?_ci$//') + echo "${PROJECT_DIR},${TASK},${KIND},${PROJECT_NAME},${SHORT}" + done +} +export -f project_tasks + +find -type f -name 'pyproject.toml' \ + | xargs -P "${NBJOBS:-1}" -I {} bash -c 'project_tasks "$@"' _ {} \ + | LC_ALL=C sort diff --git a/dev/ci/run_python_tasks.sh b/dev/ci/run_python_tasks.sh new file mode 100755 index 00000000..b34643e8 --- /dev/null +++ b/dev/ci/run_python_tasks.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Script used to run python tasks collected by script "list_python_tasks.sh". +# It expects two arguments: a task list produced by that other script, and the +# kind of tests to run. + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 TASK_FILE TEST_KIND" + echo "Possible values for TEST_KIND: quick, ocaml" + exit 1 +fi + +TASK_FILE="$1" +TEST_KIND="$2" + +if [[ ! -f "${TASK_FILE}" ]]; then + echo "File \"${TASK_FILE}\" does not exist." + exit 1 +fi + +if [[ "${TEST_KIND}" != "quick" && "${TEST_KIND}" != "ocaml" ]]; then + echo "Unknown task kind \"${TEST_KIND}\" (valid options are: quick, ocaml)" + exit 1 +fi + +TASK_COUNT=0 +SELECTED_TASK_COUNT=0 +FAILURE_COUNT=0 + +while IFS="," read -r TASK_DIR TASK_NAME TASK_KIND PROJECT_NAME SHORT_TASK_NAME; do + ((TASK_COUNT++)) + + if [[ "${TASK_KIND}" != "${TEST_KIND}" ]]; then + continue + fi + + ((SELECTED_TASK_COUNT++)) + + echo + echo "============== ${PROJECT_NAME}: ${SHORT_TASK_NAME} ==============" + echo + + PREVIOUS_FAILURE_COUNT=${FAILURE_COUNT} + echo "Running: uv --quiet --directory \"${TASK_DIR}\" run task ${TASK_NAME}" + uv --quiet --directory "${TASK_DIR}" run task ${TASK_NAME} || ((FAILURE_COUNT++)) + + if [[ "${PREVIOUS_FAILURE_COUNT}" -ne "${FAILURE_COUNT}" ]]; then + echo "::error ::The task run with command \`uv --quiet --directory \"${TASK_DIR}\" run task ${TASK_NAME}\` failed." + fi + + echo +done < "${TASK_FILE}" + +((SUCCESS_COUNT=${SELECTED_TASK_COUNT}-${FAILURE_COUNT})) + +echo +echo "============== SUMMARY ==============" +echo +echo "Attempted ${SELECTED_TASK_COUNT} selected tasks over ${TASK_COUNT}." +echo "${SUCCESS_COUNT} / ${SELECTED_TASK_COUNT} succeeded." + +if (( ${FAILURE_COUNT} )); then + echo + echo "Number of failed tasks: ${FAILURE_COUNT} / ${SELECTED_TASK_COUNT}" + exit 1 +fi