Make multiprocessing calls aware of parallelism used during testing

.pylintrc

@@ -90,7 +90,7 @@ ignored-modules=
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use, and will cap the count on Windows to
 # avoid hangs.
-jobs=0
+# jobs=0
 
 # Control the amount of potential inferred values when inferring a single
 # object. This can help the performance when dealing with large functions or

CONTRIBUTING.md

@@ -241,10 +241,6 @@ to run it:
 check/mypy
 ```
 
-If your computer has multiple processor cores, you can add the option `-j 0` to the command above to
-make Mypy run in parallel for a substantial speed increase.
-
-
 ### Linting and formatting
 
 Code should meet common style standards for Python and be free of error-prone constructs. We use
@@ -267,9 +263,12 @@ for formatting code, and provide scripts to run them.
 *   To run the linter:
 
     ```shell
-    check/pylint-changed-files
+    check/pylint-changed-files -j 0
     ```
 
+    (The option `-j 0` tells pylint to use parallel processes; omit it if it causes problems on your
+    system.)
+
 ### Testing and test coverage
 
 When new functions, classes, and files are introduced, they should also have corresponding tests.
@@ -290,14 +289,15 @@ tests, follow these general principles:
     messages.
 
 We use [pytest](https://docs.pytest.org) to run our tests and
-[pytest-cov](https://pytest-cov.readthedocs.io) to compute coverage.
+[pytest-cov](https://pytest-cov.readthedocs.io) to compute coverage. There are wrapper scripts
+in the `check/` subdirectory to run these programs.
 
-*   While developing, periodically check that changes do not break anything. For fast checks, use
-    `pytest -m "not slow" PATH`, where `PATH` is a directory or pytest file to test.
+*   During development, periodically check that code changes do not break anything. For fast checks,
+    run `check/pytest -m "not slow" -n auto PATH ...`, where `PATH ...` is one or more directories
+    or `_test.py` files to test. (The `-n auto` option makes pytest use parallel processes; omit it
+    if it causes problems on your system.)
 
-*   After finishing a task, run `check/pytest` to test all of the OpenFermion code. If your system
-    has multiple processor cores, you can add the option `-n auto` to make pytest use multiple
-    parallel processes for a speed increase. (Beware, though, that this is resource-intensive.)
+*   After finishing a task, run all tests with `check/pytest -n auto`.
 
 We don't require 100% coverage, but coverage should be very high, and any uncovered code must be
 annotated with `# pragma: no cover`. To ignore coverage of a single line, place `# pragma: no cover`
@@ -306,13 +306,12 @@ cover` comment on its own line. Note, however, that these annotations should be
 
 ### Final checks
 
-After a task is finished, run each of the following to make sure everything passes all the tests:
+After the work planned for a pull request is finished, run each of the following to make sure
+everything passes all tests:
 
-*   `check/format-incremental` (and `check/format-incremental --apply` to auto-fix format problems)
-*   `check/pylint -j 0`
-*   `check/mypy`
-*   `check/pytest -n auto`
-*   `check/pytest-and-incremental-coverage`
+```shell
+check/all
+```
 
 ### Pull requests and code reviews
 

check/all

@@ -17,22 +17,34 @@
 # This script is modeled in part on Cirq's check/all script.
 
 declare -r usage="\
-Usage: ${0##*/} [--help] [--only-changed-files] [--apply-format-changes] [BASE_REV]
+Usage: ${0##*/} [OPTIONS] [BASE_REV]
 
-If the first argument given on the command line is the option --help or -h,
-this program prints usage information and then exits.
-
-With no arguments, this runs multiple checks on the code base. These tests
-are based on the programs in the checks/ subdirectory.
+Runs multiple checks on the code base.
 
-If --apply-format-changes is specified, the flag --apply will be passed to
-check/format-incremental to apply the format changes suggested by the
+If --apply-changes (or -a) is given, the flag --apply will be passed to
+check/format-incremental to make it apply format changes suggested by the
 formatter.
 
-You can specify a base git revision to compare against (i.e., to use when
-determining whether or not a file is considered to have changed). If given,
-the argument BASE_REV is passed on to tests that can use it, such as
-check/pytest-and-incremental-coverage."
+If --only-changed-files (or -c) is given, most checks will be limited to the
+files that have changed since the BASE_REV version. If BASE_REV is not given,
+the version at the end of the base branch (origin/main) will be used.
+
+By default, this program runs checks in parallel mode. To prevent this, use the
+option --no-parallel (or -p).
+
+You can specify a base git revision to compare against (that is, the revision
+to be used when determining whether a file is considered to have changed). If
+given, the argument BASE_REV is passed on to tests that can use it, such as
+check/pytest-and-incremental-coverage.
+
+If the first argument given on the command line is the option --help or -h,
+this program prints usage information and then exits.
+
+Summary of available options:
+  --apply-changes       Pass the flag --apply to check/format-incremental
+  --no-parallel         Do not pass the parallelism flags to the check programs
+  --only-changed-files  Limit checks to files that have changed since BASE_REV
+  -h or --help          Show this help message and exit"
 
 set -eo pipefail -o errtrace
 shopt -s inherit_errexit
@@ -51,53 +63,71 @@ function error() {
 declare -a rev=()
 declare -a apply_arg=()
 declare only_changed=""
-for arg in "$@"; do
-    case "${arg}" in
+declare no_parallel=""
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -a | --apply-changes | --apply-format-changes)
+            apply_arg=( "--apply" )
+            ;;
+        -c | --only-changed-files)
+            only_changed="true"
+            ;;
         -h | --help)
             echo "${usage}"
             exit 0
             ;;
-        --apply-format-changes)
-            apply_arg=( "--apply" )
-            shift
-            ;;
-        --only-changed-files)
-            only_changed="true"
-            shift
+        -p | --no-parallel)
+            no_parallel="true"
             ;;
         -*)
-            error "Invalid option '${arg}'"
+            error "Invalid option '$1'"
             error "See '$0 --help' for the list of supported options."
             exit 1
             ;;
         *)
-            if ! rev=( "$(git rev-parse --verify --end-of-options "${arg}^{commit}")" ); then
-                error "No revision '${arg}'"
+            if ! rev=( "$(git rev-parse --verify --end-of-options "$1^{commit}")" ); then
+                error "No revision '$1'"
                 exit 1
             fi
             ;;
     esac
+    shift
 done
 
 # ~~~~ Run the tests ~~~~
 
 declare -a errors=()
+declare -a pylint_j=()
+declare -a pytest_n=()
+declare -a mypy_n=()
 
 function run() {
-    echo "Running $* ..."
+    echo "~~~~ Running $* ~~~~"
     "$@" || errors+=( "$* failed" )
-    echo
 }
 
+# Count only the physical cores (usually fewer than logical cores). This leaves
+# some CPU resources for OpenFermion modules that use multiple processes.
+cpus=$(python3 -c 'import psutil; print(psutil.cpu_count(logical=False))')
+
+if [[ -z "${no_parallel}" ]]; then
+    # Pylint & pytest can auto-detect the number of CPUs, but not mypy. To be
+    # consistent, this uses the same explicit number for all of them.
+    pylint_j=("-j" "${cpus}")
+    pytest_n=("-n" "${cpus}")
+    mypy_n=("-n" "${cpus}")
+fi
+
 if [[ -n "${only_changed}" ]]; then
     run check/format-incremental "${rev[@]}" "${apply_arg[@]}"
-    run check/pylint-changed-files "${rev[@]}"
+    run check/pylint-changed-files "${pylint_j[@]}" "${rev[@]}"
 else
     run check/format-incremental "${rev[@]}" "${apply_arg[@]}" --all
-    run check/pylint "${rev[@]}"
+    run check/pylint "${pylint_j[@]}" "${rev[@]}"
 fi
-run check/mypy
-run check/pytest-and-incremental-coverage "${rev[@]}"
+run check/mypy "${mypy_n[@]}"
+run check/pytest-and-incremental-coverage "${pytest_n[@]}" "${rev[@]}"
 run check/shellcheck
 run check/nbformat
 

check/pylint

@@ -24,4 +24,4 @@
 cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" || exit 1
 cd "$(git rev-parse --show-toplevel)" || exit 1
 
-pylint "$@" src dev_tools
+pylint "$@" src dev_tools && echo "Pylint completed successfully."

check/pylint-changed-files

@@ -43,25 +43,37 @@ thisdir=$(dirname "${BASH_SOURCE[0]:?}") || exit $?
 repo_dir=$(git -C "${thisdir}" rev-parse --show-toplevel) || exit $?
 cd "${repo_dir}" || exit $?
 
-# Figure out which revision to compare against.
-if [ -n "$1" ] && [[ $1 != -* ]]; then
-    if ! git rev-parse --verify --quiet --no-revs "$1^{commit}"; then
-        echo -e "\033[31mNo revision '$1'.\033[0m" >&2
+# Figure out which revision to compare against and gather other arguments.
+declare -a pylint_args=()
+rev=""
+
+for arg in "$@"; do
+    if [[ "${arg}" == -* ]]; then
+        pylint_args+=( "${arg}" )
+    elif [[ -z "${rev}" ]] &&
+        git rev-parse --verify --quiet --no-revs "${arg}^{commit}" &> /dev/null; then
+        rev="${arg}"
+    else
+        pylint_args+=( "${arg}" )
+    fi
+done
+
+if [[ -z "${rev}" ]]; then
+    for candidate in upstream/main origin/main main; do
+        if [[ "$(git cat-file -t "${candidate}" 2>/dev/null)" == "commit" ]]; then
+            rev="${candidate}"
+            break
+        fi
+    done
+    if [[ -z "${rev}" ]]; then
+        echo -e "\033[31mNo default revision found to compare against. \
+Please specify a base revision (e.g. 'origin/main' or 'HEAD~1') as an argument.\033[0m" >&2
         exit 1
     fi
-    rev=$1
-elif [ "$(git cat-file -t "upstream/main" 2> /dev/null)" == "commit" ]; then
-    rev=upstream/main
-elif [ "$(git cat-file -t "origin/main" 2> /dev/null)" == "commit" ]; then
-    rev=origin/main
-elif [ "$(git cat-file -t "main" 2> /dev/null)" == "commit" ]; then
-    rev=main
-else
-    echo -e "\033[31mNo default revision found to compare against. Argument #1 must be what to diff against (e.g. 'origin/main' or 'HEAD~1').\033[0m" >&2
-    exit 1
 fi
+
 base=$(git merge-base "${rev}" HEAD)
-if [ "$(git rev-parse "${rev}")" == "${base}" ]; then
+if [[ "$(git rev-parse "${rev}")" == "${base}" ]]; then
     echo -e "Comparing against revision '${rev}'." >&2
 else
     echo -e "Comparing against revision '${rev}' (merge base ${base})." >&2
@@ -78,7 +90,7 @@ num_changed=${#changed[@]}
 
 # Run it.
 echo "Found ${num_changed} lintable files associated with changes." >&2
-if [ "${num_changed}" -eq 0 ]; then
+if [[ "${num_changed}" -eq 0 ]]; then
     exit 0
 fi
-env PYTHONPATH=dev_tools pylint --jobs=0 "${changed[@]}"
+env PYTHONPATH=dev_tools pylint "${pylint_args[@]}" "${changed[@]}"

check/pytest-and-incremental-coverage

@@ -36,30 +36,40 @@
 ################################################################################
 
 # Get the working directory to the repo root.
-cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
+cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
 cd "$(git rev-parse --show-toplevel)" || exit 1
 
-# Figure out which revision to compare against.
-if [[ -n "$1" && $1 != -* ]]; then
-    if [[ "$(git cat-file -t "$1" 2> /dev/null)" != "commit" ]]; then
-        echo -e "\033[31mNo revision '$1'.\033[0m" >&2
+# Figure out which revision to compare against and gather other arguments.
+declare -a pytest_args=()
+rev=""
+
+for arg in "$@"; do
+    if [[ "${arg}" == -* ]]; then
+        pytest_args+=( "${arg}" )
+    elif [[ -z "${rev}" ]] &&
+        git rev-parse --verify --quiet --no-revs "${arg}^{commit}" &> /dev/null; then
+        rev="${arg}"
+    else
+        pytest_args+=( "${arg}" )
+    fi
+done
+
+if [[ -z "${rev}" ]]; then
+    for candidate in upstream/main origin/main main; do
+        if [[ "$(git cat-file -t "${candidate}" 2>/dev/null)" == "commit" ]]; then
+            rev="${candidate}"
+            break
+        fi
+    done
+    if [[ -z "${rev}" ]]; then
+        echo -e "\033[31mNo default revision found to compare against. \
+Please specify a base revision (e.g. 'origin/main' or 'HEAD~1') as an argument.\033[0m" >&2
         exit 1
     fi
-    rev=$1
-elif [ "$(git cat-file -t upstream/main 2> /dev/null)" == "commit" ]; then
-    rev=upstream/main
-elif [ "$(git cat-file -t origin/main 2> /dev/null)" == "commit" ]; then
-    rev=origin/main
-elif [ "$(git cat-file -t main 2> /dev/null)" == "commit" ]; then
-    rev=main
-else
-    echo -e "\033[31mNo default revision found to compare against. Argument #1 must be what to diff against (e.g. 'origin/main' or 'HEAD~1').\033[0m" >&2
-    exit 1
 fi
-# shellcheck disable=SC2086
-base="$(git merge-base ${rev} HEAD)"
-# shellcheck disable=SC2086
-if [ "$(git rev-parse ${rev})" == "${base}" ]; then
+
+base="$(git merge-base "${rev}" HEAD)"
+if [[ "$(git rev-parse "${rev}")" == "${base}" ]]; then
     echo -e "Comparing against revision '${rev}'." >&2
 else
     echo -e "Comparing against revision '${rev}' (merge base ${base})." >&2
@@ -68,7 +78,7 @@ fi
 
 # Run tests while producing coverage files.
 check/pytest . \
-    -n auto \
+    "${pytest_args[@]}" \
     --actually-quiet \
     --cov \
     --cov-report=annotate
@@ -82,7 +92,7 @@ cover_result=$?
 find . | grep "\.py,cover$" | xargs rm -f
 
 # Report result.
-if [ "${pytest_result}" -ne "0" ] || [ "${cover_result}" -ne "0" ]; then
+if [[ "${pytest_result}" -ne "0" ]] || [[ "${cover_result}" -ne "0" ]]; then
   exit 1
 fi
 exit 0

check/shellcheck

@@ -81,5 +81,6 @@ if (( opt_dry_run )); then
     printf '\\\n  %s ' "${our_shell_scripts[@]}"
     printf '\\\n;\n'
 else
-    shellcheck "${shellcheck_options[@]}" "${our_shell_scripts[@]}"
+    shellcheck "${shellcheck_options[@]}" "${our_shell_scripts[@]}" && \
+        echo "shellcheck completed successfully."
 fi