Scripts for collecting and running python tasks (locally or in CI).

dev/ci/list_python_tasks.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Outputs the list of all CI python tasks that are available under the current
+# working directory. CI python tasks are taskipy task declared in the project
+# configuration file (pyproject.toml), whose name ends in "_ci". Tasks whose
+# name end in "_ocaml_ci" are additionally identified as requiring the full
+# set of FM dependencies to be built.
+#
+# In the output, each CI python task gets a line
+#   <PROJECT_DIR>,<TASK_NAME>,<TASK_KIND>,<PROJECT_NAME>,<SHORT_TASK_NAME>
+# where:
+# - <PROJECT_DIR> is the directory of the project defining that task,
+# - <TASK_NAME> is the full task name (ending in "_ci"),
+# - <TASK_KIND> is either "quick" or "ocaml",
+# - <PROJECT_NAME> is the python project name, as defined in the toml file,
+# - <SHORT_TASK_NAME> is <TASK_NAME> without its "_ci" / "_ocaml_ci" suffix.
+#
+# The output can be computed with multiple processes, and the number of jobs
+# is controlled via the NBJOBS environment variable.
+
+project_tasks(){
+  PROJECT_CONFIG="$1"
+  PROJECT_DIR="$(dirname "${PROJECT_CONFIG}")"
+  PROJECT_NAME=$(uvx --quiet --from toml-cli toml get --toml-path "${PROJECT_CONFIG}" project.name)
+
+  for TASK in $( (uvx --quiet --directory ${PROJECT_DIR} --from taskipy task --list || echo -n) | grep -Eo '^[^ ]+_ci\b' ); do
+    KIND="$( (echo "${TASK}" | grep -Eq '_ocaml_ci$') && echo -n "ocaml" || echo -n "quick" )"
+    SHORT=$(echo "${TASK}" | sed -re 's/(_ocaml)?_ci$//')
+    echo "${PROJECT_DIR},${TASK},${KIND},${PROJECT_NAME},${SHORT}"
+  done
+}
+export -f project_tasks
+
+find -type f -name 'pyproject.toml' \
+  | xargs -P "${NBJOBS:-1}" -I {} bash -c 'project_tasks "$@"' _ {} \
+  | LC_ALL=C sort

dev/ci/run_python_tasks.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Script used to run python tasks collected by script "list_python_tasks.sh".
+# It expects two arguments: a task list produced by that other script, and the
+# kind of tests to run.
+
+if [[ $# -ne 2 ]]; then
+  echo "Usage: $0 TASK_FILE TEST_KIND"
+  echo "Possible values for TEST_KIND: quick, ocaml"
+  exit 1
+fi
+
+TASK_FILE="$1"
+TEST_KIND="$2"
+
+if [[ ! -f "${TASK_FILE}" ]]; then
+  echo "File \"${TASK_FILE}\" does not exist."
+  exit 1
+fi
+
+if [[ "${TEST_KIND}" != "quick" && "${TEST_KIND}" != "ocaml" ]]; then
+  echo "Unknown task kind \"${TEST_KIND}\" (valid options are: quick, ocaml)"
+  exit 1
+fi
+
+TASK_COUNT=0
+SELECTED_TASK_COUNT=0
+FAILURE_COUNT=0
+
+while IFS="," read -r TASK_DIR TASK_NAME TASK_KIND PROJECT_NAME SHORT_TASK_NAME; do
+  ((TASK_COUNT++))
+
+  if [[ "${TASK_KIND}" != "${TEST_KIND}" ]]; then
+    continue
+  fi
+
+  ((SELECTED_TASK_COUNT++))
+
+  echo
+  echo "============== ${PROJECT_NAME}: ${SHORT_TASK_NAME} =============="
+  echo
+
+  PREVIOUS_FAILURE_COUNT=${FAILURE_COUNT}
+  echo "Running: uv --quiet --directory \"${TASK_DIR}\" run task ${TASK_NAME}"
+  uv --quiet --directory "${TASK_DIR}" run task ${TASK_NAME} || ((FAILURE_COUNT++))
+
+  if [[ "${PREVIOUS_FAILURE_COUNT}" -ne "${FAILURE_COUNT}" ]]; then
+    echo "::error ::The task run with command \`uv --quiet --directory \"${TASK_DIR}\" run task ${TASK_NAME}\` failed."
+  fi
+
+  echo
+done < "${TASK_FILE}"
+
+((SUCCESS_COUNT=${SELECTED_TASK_COUNT}-${FAILURE_COUNT}))
+
+echo
+echo "============== SUMMARY =============="
+echo
+echo "Attempted ${SELECTED_TASK_COUNT} selected tasks over ${TASK_COUNT}."
+echo "${SUCCESS_COUNT} / ${SELECTED_TASK_COUNT} succeeded."
+
+if (( ${FAILURE_COUNT} )); then
+  echo
+  echo "Number of failed tasks: ${FAILURE_COUNT} / ${SELECTED_TASK_COUNT}"
+  exit 1
+fi