diff --git a/brev/entrypoint-jupyter-user.bash b/brev/entrypoint-jupyter-user.bash index 5c78eac6..3c25981a 100755 --- a/brev/entrypoint-jupyter-user.bash +++ b/brev/entrypoint-jupyter-user.bash @@ -6,8 +6,8 @@ set -euo pipefail export HOME="${ACH_TARGET_HOME}" -# Generate Jupyter plugin settings -/accelerated-computing-hub/brev/jupyter-generate-plugin-settings.bash +# Generate JupyterLab user settings +/accelerated-computing-hub/brev/jupyter-generate-settings.bash mkdir -p /accelerated-computing-hub/logs diff --git a/brev/entrypoint-jupyter.bash b/brev/entrypoint-jupyter.bash index 2ba3ec67..95674d6c 100755 --- a/brev/entrypoint-jupyter.bash +++ b/brev/entrypoint-jupyter.bash @@ -4,5 +4,46 @@ set -euo pipefail -# Switch to user and run user-level entrypoint -exec gosu "${ACH_TARGET_USER}" /accelerated-computing-hub/brev/entrypoint-jupyter-user.bash "$@" +# Keep this wrapper alive so an unrequested Jupyter exit can restart the +# sibling services before the container restart policy starts Jupyter again. +# An operator stopping the container sends TERM/INT to this wrapper; that path +# deliberately does not restart siblings so `compose down` can finish. +TERMINATING=0 +JUPYTER_PID="" + +# shellcheck disable=SC2317 # Invoked indirectly by the signal traps below. +terminate_jupyter() { + TERMINATING=1 + if [ -n "${JUPYTER_PID}" ] && kill -0 "${JUPYTER_PID}" 2>/dev/null; then + kill -TERM "${JUPYTER_PID}" + fi +} + +trap terminate_jupyter TERM INT + +if [ "$(id -u)" = "0" ] && [ "${ACH_TARGET_USER}" != "$(id -un)" ]; then + gosu "${ACH_TARGET_USER}" /accelerated-computing-hub/brev/entrypoint-jupyter-user.bash "$@" & +else + /accelerated-computing-hub/brev/entrypoint-jupyter-user.bash "$@" & +fi +JUPYTER_PID=$! + +set +e +wait "${JUPYTER_PID}" +JUPYTER_STATUS=$? +if [ "${TERMINATING}" -eq 1 ] && kill -0 "${JUPYTER_PID}" 2>/dev/null; then + wait "${JUPYTER_PID}" + JUPYTER_STATUS=$? +fi +set -e + +if [ "${TERMINATING}" -eq 0 ] && \ + [ "${ACH_RESTART_COMPOSE_SERVICES:-}" = "1" ] && \ + [ -S /var/run/docker.sock ]; then + echo "Jupyter exited with status ${JUPYTER_STATUS}; restarting sibling services." + if ! python3 /accelerated-computing-hub/brev/restart-compose-services.py; then + echo "Error: Failed to restart one or more sibling services." >&2 + fi +fi + +exit "${JUPYTER_STATUS}" diff --git a/brev/jupyter-generate-plugin-settings.bash b/brev/jupyter-generate-plugin-settings.bash deleted file mode 100755 index f5999499..00000000 --- a/brev/jupyter-generate-plugin-settings.bash +++ /dev/null @@ -1,59 +0,0 @@ -#! /bin/bash - -set -eu - -JUPYTER_HOST="jupyter0-${BREV_ENV_ID:-local}.brevlab.com" -NSYS_HTTP_URL="https://nsys0-${BREV_ENV_ID:-local}.brevlab.com" - -# Theme -mkdir -p "${HOME}/.jupyter/lab/user-settings/@jupyterlab/apputils-extension" -cat << EOF > "${HOME}/.jupyter/lab/user-settings/@jupyterlab/apputils-extension/themes.jupyterlab-settings" -{ - "theme": "JupyterLab Dark", - "adaptive-theme": true, - "preferred-light-theme": "JupyterLab Light", - "preferred-dark-theme": "JupyterLab Dark", - "theme-scrollbars": false -} -EOF - -# Load TURN credentials generated by the base service -TURN_CREDENTIALS_FILE="/accelerated-computing-hub/.turn-credentials" -if [ -f "${TURN_CREDENTIALS_FILE}" ]; then - source "${TURN_CREDENTIALS_FILE}" -else - echo "Warning: TURN credentials file not found at ${TURN_CREDENTIALS_FILE}" >&2 -fi - -# Nsight JupyterLab Plugin -mkdir -p "${HOME}/.jupyter/lab/user-settings/jupyterlab-nvidia-nsight" - -TURN_SETTINGS="" -if [ -n "${TURN_USERNAME+x}" ] && [ -n "${TURN_PASSWORD+x}" ]; then - TURN_SETTINGS=$(cat < "${HOME}/.jupyter/lab/user-settings/jupyterlab-nvidia-nsight/plugin.jupyterlab-settings" -{ - "ui": { - "enabled": true, - "suppressServerAddressWarning": true, - "host": "${JUPYTER_HOST}", - "dockerHost": "${JUPYTER_HOST}", - "defaultStreamerAddress": "${NSYS_HTTP_URL}"${TURN_SETTINGS} - } -} -EOF - -# Execution timing -mkdir -p "${HOME}/.jupyter/lab/user-settings/@jupyterlab/notebook-extension" -cat << EOF > "${HOME}/.jupyter/lab/user-settings/@jupyterlab/notebook-extension/tracker.jupyterlab-settings" -{ - "recordTiming": true -} -EOF diff --git a/brev/jupyter-generate-settings.bash b/brev/jupyter-generate-settings.bash new file mode 100755 index 00000000..02912f98 --- /dev/null +++ b/brev/jupyter-generate-settings.bash @@ -0,0 +1,23 @@ +#! /bin/bash + +set -eu + +# Theme +mkdir -p "${HOME}/.jupyter/lab/user-settings/@jupyterlab/apputils-extension" +cat << EOF > "${HOME}/.jupyter/lab/user-settings/@jupyterlab/apputils-extension/themes.jupyterlab-settings" +{ + "theme": "JupyterLab Dark", + "adaptive-theme": true, + "preferred-light-theme": "JupyterLab Light", + "preferred-dark-theme": "JupyterLab Dark", + "theme-scrollbars": false +} +EOF + +# Execution timing +mkdir -p "${HOME}/.jupyter/lab/user-settings/@jupyterlab/notebook-extension" +cat << EOF > "${HOME}/.jupyter/lab/user-settings/@jupyterlab/notebook-extension/tracker.jupyterlab-settings" +{ + "recordTiming": true +} +EOF diff --git a/brev/restart-compose-services.py b/brev/restart-compose-services.py new file mode 100755 index 00000000..0456c65f --- /dev/null +++ b/brev/restart-compose-services.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Restart the current container's persistent Docker Compose siblings.""" + +import http.client +import json +import os +import socket +import sys +import urllib.parse + + +DOCKER_SOCKET = "/var/run/docker.sock" +NON_PERSISTENT_SERVICES = {"base"} + + +class UnixHTTPConnection(http.client.HTTPConnection): + """An HTTP connection transported over a Unix-domain socket.""" + + def __init__(self, socket_path): + super().__init__("localhost") + self.socket_path = socket_path + + def connect(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.connect(self.socket_path) + + +def docker_request(method, path): + """Send one request to the Docker Engine API and return its body.""" + connection = UnixHTTPConnection(DOCKER_SOCKET) + try: + connection.request(method, path) + response = connection.getresponse() + body = response.read() + finally: + connection.close() + + if response.status >= 300: + detail = body.decode("utf-8", errors="replace") + raise RuntimeError( + f"Docker API {method} {path} returned {response.status}: {detail}" + ) + return body + + +def container_details(container_name): + """Return Docker metadata for a container name or ID.""" + container_path = urllib.parse.quote(container_name, safe="") + return json.loads(docker_request("GET", f"/containers/{container_path}/json")) + + +def main(): + if not os.path.exists(DOCKER_SOCKET): + print(f"Docker socket not found at {DOCKER_SOCKET}") + return 0 + + current = container_details(socket.gethostname()) + labels = current.get("Config", {}).get("Labels", {}) or {} + project = labels.get("com.docker.compose.project") + current_service = labels.get("com.docker.compose.service") + if not project or not current_service: + raise RuntimeError( + "Current container has no Docker Compose project/service labels" + ) + + filters = json.dumps({"label": [f"com.docker.compose.project={project}"]}) + query = urllib.parse.urlencode({"all": "1", "filters": filters}) + containers = json.loads(docker_request("GET", f"/containers/json?{query}")) + + siblings = [] + for container in containers: + sibling_labels = container.get("Labels", {}) or {} + service = sibling_labels.get("com.docker.compose.service") + one_off = sibling_labels.get("com.docker.compose.oneoff", "False") + if ( + not service + or service == current_service + or service in NON_PERSISTENT_SERVICES + ): + continue + if one_off.lower() == "true": + continue + siblings.append((service, container["Id"])) + + failed = False + for service, container_id in sorted(siblings): + try: + docker_request("POST", f"/containers/{container_id}/restart?t=10") + except (OSError, RuntimeError) as error: + print( + f"Error: Could not restart Compose service {project}/{service}: " + f"{error}", + file=sys.stderr, + ) + failed = True + else: + print(f"Restarted Compose service {project}/{service}") + + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/brev/test-docker-compose.bash b/brev/test-docker-compose.bash index 8fc94c62..5ce0f304 100755 --- a/brev/test-docker-compose.bash +++ b/brev/test-docker-compose.bash @@ -109,6 +109,133 @@ if [ ! -f "${COMPOSE_FILE}" ]; then exit 1 fi +test_jupyter_shutdown_restart() { + local service + local container_id + local started_at + local state + local deadline + local all_restarted + local restart_group + local service_list + local -a services=() + local -A started_before=() + + # shellcheck disable=SC2016 # Expanded inside the container. + if ! restart_group=$(docker compose -f "${COMPOSE_FILE}" exec -T jupyter \ + sh -c 'printf %s "${ACH_RESTART_COMPOSE_SERVICES:-}"'); then + echo "Error: could not read Jupyter restart configuration" >&2 + return 1 + fi + if [ "${restart_group}" != "1" ]; then + echo "Error: Jupyter service-group restart is not enabled" >&2 + return 1 + fi + + if ! service_list=$(docker compose -f "${COMPOSE_FILE}" config --services); then + echo "Error: could not read Compose services" >&2 + return 1 + fi + mapfile -t services < <( + grep -E '^(jupyter|nsight|nsys|ncu)$' <<<"${service_list}" + ) + if [ "${#services[@]}" -eq 0 ]; then + echo "Error: no persistent web services found" >&2 + return 1 + fi + + echo "🔄 Testing service-group restart after Jupyter shutdown..." + for service in "${services[@]}"; do + container_id=$(docker compose -f "${COMPOSE_FILE}" ps -q "${service}") + if [ -z "${container_id}" ]; then + echo "Error: no running container found for ${service}" >&2 + return 1 + fi + if ! started_at=$( + docker inspect --format '{{.State.StartedAt}}' "${container_id}" + ); then + echo "Error: could not inspect ${service} container" >&2 + return 1 + fi + started_before["${service}"]=${started_at} + done + + if ! docker compose -f "${COMPOSE_FILE}" exec -T jupyter python3 - <<'PY' +import http.cookiejar +import time +import urllib.error +import urllib.request + +cookies = http.cookiejar.CookieJar() +opener = urllib.request.build_opener( + urllib.request.HTTPCookieProcessor(cookies) +) +deadline = time.monotonic() + 60 +while True: + try: + with opener.open("http://127.0.0.1:8888/lab", timeout=10): + break + except (OSError, urllib.error.URLError): + if time.monotonic() >= deadline: + raise + time.sleep(1) + +xsrf = next(cookie.value for cookie in cookies if cookie.name == "_xsrf") +request = urllib.request.Request( + "http://127.0.0.1:8888/api/shutdown", + data=b"", + headers={"X-XSRFToken": xsrf}, + method="POST", +) +with opener.open(request, timeout=10) as response: + if response.status >= 300: + raise RuntimeError(f"Jupyter shutdown returned HTTP {response.status}") +PY + then + echo "Error: failed to request Jupyter shutdown" >&2 + return 1 + fi + + deadline=$((SECONDS + 90)) + while [ "${SECONDS}" -lt "${deadline}" ]; do + all_restarted=1 + for service in "${services[@]}"; do + container_id=$(docker compose -f "${COMPOSE_FILE}" ps -q "${service}") + if [ -z "${container_id}" ]; then + all_restarted=0 + continue + fi + state=$(docker inspect --format '{{.State.Status}}' \ + "${container_id}" 2>/dev/null || true) + started_at=$(docker inspect --format '{{.State.StartedAt}}' \ + "${container_id}" 2>/dev/null || true) + if [ "${state}" != "running" ] || \ + [ "${started_at}" = "${started_before[${service}]}" ]; then + all_restarted=0 + fi + done + if [ "${all_restarted}" -eq 1 ]; then + echo -e "${GREEN}✅ Jupyter shutdown restarted the full service group${NC}" + echo "" + return 0 + fi + sleep 1 + done + + echo "Error: service group did not restart within 90 seconds" >&2 + for service in "${services[@]}"; do + container_id=$(docker compose -f "${COMPOSE_FILE}" ps -aq "${service}" | head -n 1) + if [ -n "${container_id}" ]; then + state=$(docker inspect --format '{{.State.Status}}' "${container_id}") + started_at=$(docker inspect --format '{{.State.StartedAt}}' "${container_id}") + printf ' %s: state=%s started=%s previous=%s\n' \ + "${service}" "${state}" "${started_at}" "${started_before[${service}]}" >&2 + fi + done + docker compose -f "${COMPOSE_FILE}" logs --tail=100 >&2 || true + return 1 +} + echo "================================================================================" echo "Testing Docker Compose: ${COMPOSE_FILE}" echo "================================================================================" @@ -187,7 +314,8 @@ if docker compose -f "${COMPOSE_FILE}" ps | grep -q "Up\|running"; then # Test restart functionality echo "🔄 Testing service restart..." echo "" - if docker compose -f "${COMPOSE_FILE}" restart; then + if test_jupyter_shutdown_restart && \ + docker compose -f "${COMPOSE_FILE}" restart; then echo "" echo -e "${GREEN}✅ Services restarted successfully${NC}" echo "" diff --git a/tutorials/accelerated-python/brev/docker-compose.yml b/tutorials/accelerated-python/brev/docker-compose.yml index aa68e673..fdb08869 100644 --- a/tutorials/accelerated-python/brev/docker-compose.yml +++ b/tutorials/accelerated-python/brev/docker-compose.yml @@ -19,10 +19,10 @@ x-config: count: all capabilities: [gpu] common-env: &common-env - BREV_ENV_ID: ${BREV_ENV_ID:-} ACH_TUTORIAL: *tutorial-name ACH_RUN_TESTS: ${ACH_RUN_TESTS:-} ACH_TEST_ARGS: ${ACH_TEST_ARGS:-} + ACH_RESTART_COMPOSE_SERVICES: "1" ACH_USER: ${ACH_USER:-ach} ACH_UID: ${ACH_UID:-1000} ACH_GID: ${ACH_GID:-1000} diff --git a/tutorials/accelerated-python/brev/requirements.txt b/tutorials/accelerated-python/brev/requirements.txt index fd39887c..63cf79db 100644 --- a/tutorials/accelerated-python/brev/requirements.txt +++ b/tutorials/accelerated-python/brev/requirements.txt @@ -13,7 +13,6 @@ jupyter-dark-detect == 0.1.0 jupyter == 1.1.1 jupyter-server-proxy == 4.4.0 jupyterlab == 4.5.10 -jupyterlab-nvidia-nsight == 1.0.0 jupyterlab-execute-time == 3.3.0 # CUDA diff --git a/tutorials/cuda-cpp/brev/docker-compose.yml b/tutorials/cuda-cpp/brev/docker-compose.yml index a488a05e..d855fabf 100644 --- a/tutorials/cuda-cpp/brev/docker-compose.yml +++ b/tutorials/cuda-cpp/brev/docker-compose.yml @@ -18,10 +18,10 @@ x-config: count: all capabilities: [gpu] common-env: &common-env - BREV_ENV_ID: ${BREV_ENV_ID:-} ACH_TUTORIAL: *tutorial-name ACH_RUN_TESTS: ${ACH_RUN_TESTS:-} ACH_TEST_ARGS: ${ACH_TEST_ARGS:-} + ACH_RESTART_COMPOSE_SERVICES: "1" ACH_USER: ${ACH_USER:-ach} ACH_UID: ${ACH_UID:-1000} ACH_GID: ${ACH_GID:-1000} diff --git a/tutorials/cuda-cpp/brev/requirements.txt b/tutorials/cuda-cpp/brev/requirements.txt index c2ffd9b2..0b08b0c2 100644 --- a/tutorials/cuda-cpp/brev/requirements.txt +++ b/tutorials/cuda-cpp/brev/requirements.txt @@ -7,7 +7,6 @@ jupyter-dark-detect == 0.1.0 # Jupyter jupyter jupyter-server-proxy -jupyterlab-nvidia-nsight jupyterlab-execute-time # NVIDIA devtools diff --git a/tutorials/cuda-tile/brev/docker-compose.yml b/tutorials/cuda-tile/brev/docker-compose.yml index a8c537f3..0c904b2f 100644 --- a/tutorials/cuda-tile/brev/docker-compose.yml +++ b/tutorials/cuda-tile/brev/docker-compose.yml @@ -19,10 +19,10 @@ x-config: count: all capabilities: [gpu] common-env: &common-env - BREV_ENV_ID: ${BREV_ENV_ID:-} ACH_TUTORIAL: *tutorial-name ACH_RUN_TESTS: ${ACH_RUN_TESTS:-} ACH_TEST_ARGS: ${ACH_TEST_ARGS:-} + ACH_RESTART_COMPOSE_SERVICES: "1" ACH_USER: ${ACH_USER:-ach} ACH_UID: ${ACH_UID:-1000} ACH_GID: ${ACH_GID:-1000} diff --git a/tutorials/cuda-tile/brev/requirements.txt b/tutorials/cuda-tile/brev/requirements.txt index 775475ea..6db125f7 100644 --- a/tutorials/cuda-tile/brev/requirements.txt +++ b/tutorials/cuda-tile/brev/requirements.txt @@ -13,7 +13,6 @@ jupyter-dark-detect == 0.1.0 jupyter jupyter-server-proxy jupyterlab -jupyterlab-nvidia-nsight jupyterlab-execute-time # CUDA diff --git a/tutorials/floating-point-emulation/brev/docker-compose.yml b/tutorials/floating-point-emulation/brev/docker-compose.yml index d77a6c43..f40ed606 100644 --- a/tutorials/floating-point-emulation/brev/docker-compose.yml +++ b/tutorials/floating-point-emulation/brev/docker-compose.yml @@ -25,9 +25,9 @@ x-config: - accelerated-computing-hub:/accelerated-computing-hub - /var/run/docker.sock:/var/run/docker.sock environment: - BREV_ENV_ID: ${BREV_ENV_ID:-} ACH_TUTORIAL: *tutorial-name ACH_RUN_TESTS: ${ACH_RUN_TESTS:-} + ACH_RESTART_COMPOSE_SERVICES: "1" ACH_USER: ${ACH_USER:-ach} ACH_UID: ${ACH_UID:-1000} ACH_GID: ${ACH_GID:-1000} diff --git a/tutorials/floating-point-emulation/brev/requirements.txt b/tutorials/floating-point-emulation/brev/requirements.txt index cc6009f5..fd85fe66 100644 --- a/tutorials/floating-point-emulation/brev/requirements.txt +++ b/tutorials/floating-point-emulation/brev/requirements.txt @@ -15,7 +15,6 @@ matplotlib # Jupyter jupyterlab -jupyterlab-nvidia-nsight jupyterlab-execute-time ipywidgets ipykernel diff --git a/tutorials/gpu-deployment/gpu-deployment-from-scratch.md b/tutorials/gpu-deployment/gpu-deployment-from-scratch.md index f22c8003..83732f91 100644 --- a/tutorials/gpu-deployment/gpu-deployment-from-scratch.md +++ b/tutorials/gpu-deployment/gpu-deployment-from-scratch.md @@ -501,7 +501,6 @@ To be able to visualize the file, we can download it an use [nsight-systems](htt Further reading: - [Nsight Documentation](https://developer.nvidia.com/nsight-systems/get-started) -- [Jupyter Lab Nsight extension](https://pypi.org/project/jupyterlab-nvidia-nsight/) - [Towards Data Science community guide](https://medium.com/data-science/profiling-cuda-using-nsight-systems-a-numba-example-fc65003f8c52) diff --git a/tutorials/nvmath-python/brev/docker-compose.yml b/tutorials/nvmath-python/brev/docker-compose.yml index f996db88..7ce4474f 100644 --- a/tutorials/nvmath-python/brev/docker-compose.yml +++ b/tutorials/nvmath-python/brev/docker-compose.yml @@ -19,10 +19,10 @@ x-config: count: all capabilities: [gpu] common-env: &common-env - BREV_ENV_ID: ${BREV_ENV_ID:-} ACH_TUTORIAL: *tutorial-name ACH_RUN_TESTS: ${ACH_RUN_TESTS:-} ACH_TEST_ARGS: ${ACH_TEST_ARGS:-} + ACH_RESTART_COMPOSE_SERVICES: "1" ACH_USER: ${ACH_USER:-ach} ACH_UID: ${ACH_UID:-1000} ACH_GID: ${ACH_GID:-1000} diff --git a/tutorials/nvmath-python/brev/requirements.txt b/tutorials/nvmath-python/brev/requirements.txt index a59d68fd..82327032 100644 --- a/tutorials/nvmath-python/brev/requirements.txt +++ b/tutorials/nvmath-python/brev/requirements.txt @@ -22,7 +22,6 @@ jupyter-dark-detect == 0.1.0 # Jupyter jupyter-server-proxy jupyterlab -jupyterlab-nvidia-nsight jupyterlab-execute-time ipywidgets ipykernel diff --git a/tutorials/stdpar/brev/docker-compose.yml b/tutorials/stdpar/brev/docker-compose.yml index d6277901..834633d1 100644 --- a/tutorials/stdpar/brev/docker-compose.yml +++ b/tutorials/stdpar/brev/docker-compose.yml @@ -19,10 +19,10 @@ x-config: count: all capabilities: [gpu] common-env: &common-env - BREV_ENV_ID: ${BREV_ENV_ID:-} ACH_TUTORIAL: *tutorial-name ACH_RUN_TESTS: ${ACH_RUN_TESTS:-} ACH_TEST_ARGS: ${ACH_TEST_ARGS:-} + ACH_RESTART_COMPOSE_SERVICES: "1" ACH_USER: ${ACH_USER:-ach} ACH_UID: ${ACH_UID:-1000} ACH_GID: ${ACH_GID:-1000} diff --git a/tutorials/stdpar/brev/requirements.txt b/tutorials/stdpar/brev/requirements.txt index 65e05bf5..b6d93ece 100644 --- a/tutorials/stdpar/brev/requirements.txt +++ b/tutorials/stdpar/brev/requirements.txt @@ -18,7 +18,6 @@ conan # Jupyter jupyter jupyter-server-proxy -jupyterlab-nvidia-nsight jupyterlab-execute-time # NVIDIA devtools diff --git a/tutorials/warp/brev/docker-compose.yml b/tutorials/warp/brev/docker-compose.yml index 4bf5d613..ffcd2f47 100644 --- a/tutorials/warp/brev/docker-compose.yml +++ b/tutorials/warp/brev/docker-compose.yml @@ -19,10 +19,10 @@ x-config: count: all capabilities: [gpu] common-env: &common-env - BREV_ENV_ID: ${BREV_ENV_ID:-} ACH_TUTORIAL: *tutorial-name ACH_RUN_TESTS: ${ACH_RUN_TESTS:-} ACH_TEST_ARGS: ${ACH_TEST_ARGS:-} + ACH_RESTART_COMPOSE_SERVICES: "1" ACH_USER: ${ACH_USER:-ach} ACH_UID: ${ACH_UID:-1000} ACH_GID: ${ACH_GID:-1000}