From 73564273a844d7b4a010f158941f29579f83a4fe Mon Sep 17 00:00:00 2001 From: acul71 Date: Sun, 1 Feb 2026 14:51:06 +0100 Subject: [PATCH 1/7] wip: python perf (cherry picked from commit 285b6d3b7d3e4004e19975ae4426a07b27dd7d5f) --- perf/images.yaml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/perf/images.yaml b/perf/images.yaml index a97abbb..2d5a8c2 100644 --- a/perf/images.yaml +++ b/perf/images.yaml @@ -9,7 +9,9 @@ test-aliases: - alias: "none" value: "!~all" - alias: "images" - value: "dotnet-v1.0|go-v0.45|js-v3.x|rust-v0.56" + value: "dotnet-v1.0|go-v0.45|js-v3.x|python-v0.x|rust-v0.56" + - alias: "python" + value: "python-v0.x" - alias: "baselines" value: "https|quic-go|iperf" - alias: "go" @@ -20,7 +22,7 @@ test-aliases: value: "js-v3.x" - alias: "dotnet" value: "dotnet-v1.0" - - alias: "failing" # as of 20 Jan 2026 + - alias: "failing" # as of 29 Jan 2026 value: "go-v0.45|js-v3.x|dotnet-v1.0|rust-v0.56 x rust-v0.56 (webrtc-direct)" # Baseline implementations - Separate from main tests @@ -96,3 +98,22 @@ implementations: secureChannels: [noise, tls] muxers: [yamux] + # Python implementation (github snapshot; Dockerfile uses repo-root context like interop/transport) + - id: python-v0.x + source: + type: github + repo: libp2p/py-libp2p + commit: 5eded13c516380ba42aa5432a437ff7b228454cc + dockerfile: interop/perf/Dockerfile + transports: [quic-v1, tcp, ws] + secureChannels: [noise, tls] + muxers: [yamux, mplex] + # Local python config (same Dockerfile; context = repo root = images/python/v0.x/py-libp2p) + # - id: python-v0.x + # source: + # type: local + # path: images/python/v0.x/py-libp2p + # dockerfile: interop/perf/Dockerfile + # transports: [quic-v1, tcp, ws] + # secureChannels: [noise, tls] + # muxers: [yamux, mplex] From 34f3be71c87c4907b9a067a48c970417a749bf59 Mon Sep 17 00:00:00 2001 From: acul71 Date: Sat, 7 Mar 2026 04:44:16 +0100 Subject: [PATCH 2/7] fix perf python image pin and compose project isolation Pin perf Python tests to the intended py-libp2p snapshot and avoid docker compose project/container name collisions between reruns by using a per-test-pass project suffix. Made-with: Cursor --- perf/images.yaml | 24 ++++++++++++------------ perf/lib/run-single-test.sh | 21 ++++++++++++++------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/perf/images.yaml b/perf/images.yaml index 446b1cc..95f4704 100644 --- a/perf/images.yaml +++ b/perf/images.yaml @@ -100,22 +100,22 @@ implementations: secureChannels: [noise, tls] muxers: [yamux] - # Python implementation (github snapshot; Dockerfile uses repo-root context like interop/transport) - id: python-v0.x source: type: github repo: libp2p/py-libp2p - commit: 5eded13c516380ba42aa5432a437ff7b228454cc + commit: 34c4bd95a0bb170bd37d191c665e10d0f38aad27 dockerfile: interop/perf/Dockerfile - transports: [quic-v1, tcp, ws] + transports: [tcp, ws] secureChannels: [noise, tls] muxers: [yamux, mplex] - # Local python config (same Dockerfile; context = repo root = images/python/v0.x/py-libp2p) - # - id: python-v0.x - # source: - # type: local - # path: images/python/v0.x/py-libp2p - # dockerfile: interop/perf/Dockerfile - # transports: [quic-v1, tcp, ws] - # secureChannels: [noise, tls] - # muxers: [yamux, mplex] + # Local python config (same Dockerfile; context = repo root = images/python/0.x/py-libp2p) + #- id: python-v0.x + # source: + # type: local + # path: images/python/0.x/py-libp2p + # dockerfile: interop/perf/Dockerfile + # # transports: [quic-v1, tcp, ws] + # transports: [tcp, ws] + # secureChannels: [noise, tls] + # muxers: [yamux, mplex] diff --git a/perf/lib/run-single-test.sh b/perf/lib/run-single-test.sh index 5c98af9..6b61415 100755 --- a/perf/lib/run-single-test.sh +++ b/perf/lib/run-single-test.sh @@ -61,9 +61,16 @@ TEST_SLUG=$(echo "${TEST_NAME}" | sed 's/[^a-zA-Z0-9-]/_/g') LOG_FILE="${TEST_PASS_DIR}/logs/${TEST_SLUG}.log" > "${LOG_FILE}" +# Use unique compose project/container names per test pass to avoid stale +# docker compose state collisions when rerunning the same test selection. +RUN_KEY=$(compute_test_key "${TEST_PASS_NAME}-${TEST_NAME}") +COMPOSE_PROJECT_NAME="${TEST_SLUG}_${RUN_KEY}" +CONTAINER_PREFIX="${COMPOSE_PROJECT_NAME}" + print_debug "test key: ${TEST_KEY}" print_debug "test slug: ${TEST_SLUG}" print_debug "log file: ${LOG_FILE}" +print_debug "compose project: ${COMPOSE_PROJECT_NAME}" log_message "[$((${TEST_INDEX} + 1))] ${TEST_NAME} (key: ${TEST_KEY})" @@ -114,7 +121,7 @@ fi if [ "${IS_LEGACY_TEST}" == "true" ]; then # Legacy test: external shared network + Redis proxy service cat > "${COMPOSE_FILE}" < "${COMPOSE_FILE}" < Date: Sat, 7 Mar 2026 20:40:19 +0100 Subject: [PATCH 3/7] feat: TLS optimized --- perf/images.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf/images.yaml b/perf/images.yaml index 95f4704..20ca379 100644 --- a/perf/images.yaml +++ b/perf/images.yaml @@ -104,7 +104,7 @@ implementations: source: type: github repo: libp2p/py-libp2p - commit: 34c4bd95a0bb170bd37d191c665e10d0f38aad27 + commit: ca470efdaf4df982d4e3844d86a515eb18efca7a dockerfile: interop/perf/Dockerfile transports: [tcp, ws] secureChannels: [noise, tls] @@ -115,7 +115,7 @@ implementations: # type: local # path: images/python/0.x/py-libp2p # dockerfile: interop/perf/Dockerfile - # # transports: [quic-v1, tcp, ws] + # transports: [quic-v1, tcp, ws] # transports: [tcp, ws] # secureChannels: [noise, tls] # muxers: [yamux, mplex] From 3505c4fcbaa093241e718927a7af80f90d4bd326 Mon Sep 17 00:00:00 2001 From: acul71 Date: Tue, 26 May 2026 16:54:55 +0200 Subject: [PATCH 4/7] new yamux window fix --- perf/images.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perf/images.yaml b/perf/images.yaml index 20ca379..d1be370 100644 --- a/perf/images.yaml +++ b/perf/images.yaml @@ -104,7 +104,8 @@ implementations: source: type: github repo: libp2p/py-libp2p - commit: ca470efdaf4df982d4e3844d86a515eb18efca7a + #commit: ca470efdaf4df982d4e3844d86a515eb18efca7a + commit: 824654654e1ba0bd1f414c397b724c97906326d2 dockerfile: interop/perf/Dockerfile transports: [tcp, ws] secureChannels: [noise, tls] From 1d6f75d6af5f8031eba127f8c4077c589886a590 Mon Sep 17 00:00:00 2001 From: acul71 Date: Tue, 26 May 2026 17:29:58 +0200 Subject: [PATCH 5/7] fix: normalize perf implementation change detection Compare perf implementation definitions as normalized JSON so comment-only edits do not mark unrelated implementations as changed, while keeping the local Python example in place in perf/images.yaml. Co-authored-by: Cursor --- .../actions/detect-changed-impls/action.yml | 24 ++++++++++++------- perf/images.yaml | 4 ++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/actions/detect-changed-impls/action.yml b/.github/actions/detect-changed-impls/action.yml index bfcb26f..7bd91d5 100644 --- a/.github/actions/detect-changed-impls/action.yml +++ b/.github/actions/detect-changed-impls/action.yml @@ -84,6 +84,14 @@ runs: git show "origin/$BASE_REF:$IMPLS_FILE" > /tmp/impls-old.yaml 2>/dev/null || echo "" > /tmp/impls-old.yaml cp "$IMPLS_FILE" /tmp/impls-new.yaml + # Normalize objects before comparing so YAML comments/formatting + # don't make unchanged entries look modified. + normalize_object() { + local file="$1" + local query="$2" + yq eval -o=json -I=0 "$query" "$file" 2>/dev/null || echo "" + } + # Find changed/new implementations OLD_IMPLS=$(yq eval '.implementations[].id' /tmp/impls-old.yaml 2>/dev/null | sort || echo "") NEW_IMPLS=$(yq eval '.implementations[].id' /tmp/impls-new.yaml 2>/dev/null | sort || echo "") @@ -91,8 +99,8 @@ runs: for impl_id in $NEW_IMPLS; do if echo "$OLD_IMPLS" | grep -q "^${impl_id}$"; then # Exists in old, check if content changed - OLD_IMPL=$(yq eval ".implementations[] | select(.id == \"$impl_id\")" /tmp/impls-old.yaml 2>/dev/null || echo "") - NEW_IMPL=$(yq eval ".implementations[] | select(.id == \"$impl_id\")" /tmp/impls-new.yaml 2>/dev/null || echo "") + OLD_IMPL=$(normalize_object /tmp/impls-old.yaml ".implementations[] | select(.id == \"$impl_id\")") + NEW_IMPL=$(normalize_object /tmp/impls-new.yaml ".implementations[] | select(.id == \"$impl_id\")") if [ "$OLD_IMPL" != "$NEW_IMPL" ]; then CHANGED_IMPLS="${CHANGED_IMPLS:+$CHANGED_IMPLS|}$impl_id" @@ -113,8 +121,8 @@ runs: for relay_id in $NEW_RELAYS; do if echo "$OLD_RELAYS" | grep -q "^${relay_id}$"; then - OLD_RELAY=$(yq eval ".relays[] | select(.id == \"$relay_id\")" /tmp/impls-old.yaml 2>/dev/null || echo "") - NEW_RELAY=$(yq eval ".relays[] | select(.id == \"$relay_id\")" /tmp/impls-new.yaml 2>/dev/null || echo "") + OLD_RELAY=$(normalize_object /tmp/impls-old.yaml ".relays[] | select(.id == \"$relay_id\")") + NEW_RELAY=$(normalize_object /tmp/impls-new.yaml ".relays[] | select(.id == \"$relay_id\")") if [ "$OLD_RELAY" != "$NEW_RELAY" ]; then CHANGED_RELAYS="${CHANGED_RELAYS:+$CHANGED_RELAYS|}$relay_id" echo " → Changed relay: $relay_id" @@ -131,8 +139,8 @@ runs: for router_id in $NEW_ROUTERS; do if echo "$OLD_ROUTERS" | grep -q "^${router_id}$"; then - OLD_ROUTER=$(yq eval ".routers[] | select(.id == \"$router_id\")" /tmp/impls-old.yaml 2>/dev/null || echo "") - NEW_ROUTER=$(yq eval ".routers[] | select(.id == \"$router_id\")" /tmp/impls-new.yaml 2>/dev/null || echo "") + OLD_ROUTER=$(normalize_object /tmp/impls-old.yaml ".routers[] | select(.id == \"$router_id\")") + NEW_ROUTER=$(normalize_object /tmp/impls-new.yaml ".routers[] | select(.id == \"$router_id\")") if [ "$OLD_ROUTER" != "$NEW_ROUTER" ]; then CHANGED_ROUTERS="${CHANGED_ROUTERS:+$CHANGED_ROUTERS|}$router_id" echo " → Changed router: $router_id" @@ -153,8 +161,8 @@ runs: for baseline_id in $NEW_BASELINES; do if echo "$OLD_BASELINES" | grep -q "^${baseline_id}$"; then # Exists in old, check if content changed - OLD_BASELINE=$(yq eval ".baselines[] | select(.id == \"$baseline_id\")" /tmp/impls-old.yaml 2>/dev/null || echo "") - NEW_BASELINE=$(yq eval ".baselines[] | select(.id == \"$baseline_id\")" /tmp/impls-new.yaml 2>/dev/null || echo "") + OLD_BASELINE=$(normalize_object /tmp/impls-old.yaml ".baselines[] | select(.id == \"$baseline_id\")") + NEW_BASELINE=$(normalize_object /tmp/impls-new.yaml ".baselines[] | select(.id == \"$baseline_id\")") if [ "$OLD_BASELINE" != "$NEW_BASELINE" ]; then CHANGED_BASELINES="${CHANGED_BASELINES:+$CHANGED_BASELINES|}$baseline_id" echo " → Changed baseline: $baseline_id" diff --git a/perf/images.yaml b/perf/images.yaml index 36fa277..2775ac2 100644 --- a/perf/images.yaml +++ b/perf/images.yaml @@ -164,13 +164,13 @@ implementations: transports: [tcp, ws] secureChannels: [noise, tls] muxers: [yamux, mplex] - # Local python config (same Dockerfile; context = repo root = images/python/0.x/py-libp2p) + + # Local python config example (same Dockerfile; context = repo root = images/python/0.x/py-libp2p) #- id: python-v0.x # source: # type: local # path: images/python/0.x/py-libp2p # dockerfile: interop/perf/Dockerfile - # # transports: [quic-v1, tcp, ws] # transports: [tcp, ws] # secureChannels: [noise, tls] # muxers: [yamux, mplex] From 64646bd30294333dcffe454a1322cf75ae3fb27c Mon Sep 17 00:00:00 2001 From: acul71 Date: Tue, 26 May 2026 17:52:45 +0200 Subject: [PATCH 6/7] fix: bump python perf image commit Update the pinned py-libp2p perf source to the lighter-image commit so CI builds the smaller Python perf Docker image. Co-authored-by: Cursor --- perf/images.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf/images.yaml b/perf/images.yaml index 2775ac2..29e7e73 100644 --- a/perf/images.yaml +++ b/perf/images.yaml @@ -159,7 +159,7 @@ implementations: type: github repo: libp2p/py-libp2p #commit: ca470efdaf4df982d4e3844d86a515eb18efca7a - commit: 824654654e1ba0bd1f414c397b724c97906326d2 + commit: 64e59731883b11287906ac500d4961978b0743c5 dockerfile: interop/perf/Dockerfile transports: [tcp, ws] secureChannels: [noise, tls] From 775357273990ef400cf67c936953fdded0387321 Mon Sep 17 00:00:00 2001 From: acul71 Date: Fri, 5 Jun 2026 21:07:10 +0200 Subject: [PATCH 7/7] feat(perf): add harness --timeout and python PY_YAMUX_* injection Expose PERF_TEST_TIMEOUT_SECS via --timeout so slow stacks can run longer than the default 300s cap. Forward PY_YAMUX_* tuning env vars into python-v0.x containers only, bump the py-libp2p perf image commit, and clarify timeout vs legacy --duration in the README. Co-authored-by: Cursor --- lib/lib-inputs-yaml.sh | 1 + perf/README.md | 16 +++++++--- perf/images.yaml | 3 +- perf/lib/run-single-test.sh | 63 +++++++++++++++++++++++++++++++------ perf/run.sh | 11 +++++++ 5 files changed, 77 insertions(+), 17 deletions(-) diff --git a/lib/lib-inputs-yaml.sh b/lib/lib-inputs-yaml.sh index 78daf3f..4d7706f 100755 --- a/lib/lib-inputs-yaml.sh +++ b/lib/lib-inputs-yaml.sh @@ -88,6 +88,7 @@ EOF DOWNLOAD_BYTES: "${DOWNLOAD_BYTES:-}" DURATION: "${DURATION:-}" LATENCY_ITERATIONS: "${LATENCY_ITERATIONS:-}" + PERF_TEST_TIMEOUT_SECS: "${PERF_TEST_TIMEOUT_SECS:-}" EOF ;; hole-punch) diff --git a/perf/README.md b/perf/README.md index 79c4274..153e083 100644 --- a/perf/README.md +++ b/perf/README.md @@ -73,8 +73,8 @@ Control test behavior with these options: # Number of iterations for upload/download tests (default: 10) ./run.sh --iterations 10 -# Duration per iteration for throughput tests in seconds (default: 20) -./run.sh --duration 20 +# Max seconds per test before the harness aborts (default: 300) +./run.sh --timeout 900 # Number of iterations for latency tests (default: 100) ./run.sh --latency-iterations 100 @@ -83,7 +83,11 @@ Control test behavior with these options: ./run.sh --cache-dir /srv/cache ``` -**Note**: The `--iterations` flag controls both upload and download iterations. The framework measures throughput by transferring data over a time period and measuring bytes/second. +**Note**: The `--iterations` flag controls both upload and download iterations. Modern dialers measure throughput by transferring a fixed amount of data (`UPLOAD_BYTES` / `DOWNLOAD_BYTES`) per iteration and reporting Gbps. + +**Note**: `--timeout` sets `PERF_TEST_TIMEOUT_SECS` (harness limit for the whole docker-compose test). This is separate from `TEST_TIMEOUT_SECS` inside dialer/listener containers (peer/redis wait limits). + +**Legacy**: `--duration` (default 20s) is passed to dialers as `DURATION` but is unused by current implementations; prefer `--upload-bytes`, `--iterations`, and `--timeout`. ## Test Filtering @@ -586,8 +590,10 @@ For now, all tests run locally using Docker networking. Multi-machine testing su **Tests failing with timeout** - Check container logs: `$TEST_PASS_DIR/logs/.log` -- Increase test duration: `--duration 30` -- Enable debug mode: `--debug` +- Each perf test is capped at **300 seconds** by default (`--timeout`). For slow stacks: `./run.sh --timeout 900 --test-select "..." --yes` +- Do not use `--debug` for throughput runs (adds log I/O and often lowers Gbps) +- `--duration` does not extend the harness limit and is unused by modern dialers (they transfer fixed `UPLOAD_BYTES`/`DOWNLOAD_BYTES` per iteration, not “run for N seconds”) +- Python yamux tuning uses `PY_YAMUX_*` env vars (e.g. `PY_YAMUX_DISABLE_HYSTERESIS=1`); they are injected into **python-v0.x** containers only, not Go/Rust **Cache not working / Matrix regenerates every time** - Check TEST_RUN_KEY in output (should be consistent for same configuration) diff --git a/perf/images.yaml b/perf/images.yaml index 29e7e73..8fe80a1 100644 --- a/perf/images.yaml +++ b/perf/images.yaml @@ -158,8 +158,7 @@ implementations: source: type: github repo: libp2p/py-libp2p - #commit: ca470efdaf4df982d4e3844d86a515eb18efca7a - commit: 64e59731883b11287906ac500d4961978b0743c5 + commit: 21be0a249851276039a0ae2f896c9691e3760ad3 dockerfile: interop/perf/Dockerfile transports: [tcp, ws] secureChannels: [noise, tls] diff --git a/perf/lib/run-single-test.sh b/perf/lib/run-single-test.sh index 6b61415..191179c 100755 --- a/perf/lib/run-single-test.sh +++ b/perf/lib/run-single-test.sh @@ -96,13 +96,55 @@ cleanup() { } trap cleanup EXIT -# Build environment variables per container based on legacy status -# Legacy containers get lowercase env vars pointing to the proxy -# Modern containers get uppercase env vars pointing to global Redis +# Build environment variables per container based on legacy status. +# Legacy containers get lowercase env vars pointing to the proxy. +# Modern containers get uppercase env vars pointing to global Redis. + +# Python-only yamux tuning: forward host PY_YAMUX_* into python-v0.x containers only. +# Read by py-libp2p yamux; ignored by Go/Rust/JS. DEBUG=true sets PY_YAMUX_DEBUG=1. +LISTENER_PY_ENV=() +if [[ "${LISTENER_ID}" == "python-v0.x" ]]; then + if [ "${DEBUG:-false}" = "true" ]; then + LISTENER_PY_ENV+=("PY_YAMUX_DEBUG=1") + fi + if [ -n "${PY_YAMUX_DISABLE_HYSTERESIS:-}" ]; then + LISTENER_PY_ENV+=("PY_YAMUX_DISABLE_HYSTERESIS=${PY_YAMUX_DISABLE_HYSTERESIS}") + fi + if [ -n "${PY_YAMUX_RELEASE_ON_READ:-}" ]; then + LISTENER_PY_ENV+=("PY_YAMUX_RELEASE_ON_READ=${PY_YAMUX_RELEASE_ON_READ}") + fi + if [ -n "${PY_YAMUX_ASSUME_RTT_MS:-}" ]; then + LISTENER_PY_ENV+=("PY_YAMUX_ASSUME_RTT_MS=${PY_YAMUX_ASSUME_RTT_MS}") + fi + if [ -n "${PY_YAMUX_BATCH_THRESHOLD_DIV:-}" ]; then + LISTENER_PY_ENV+=("PY_YAMUX_BATCH_THRESHOLD_DIV=${PY_YAMUX_BATCH_THRESHOLD_DIV}") + fi +fi + +DIALER_PY_ENV=() +if [[ "${DIALER_ID}" == "python-v0.x" ]]; then + if [ "${DEBUG:-false}" = "true" ]; then + DIALER_PY_ENV+=("PY_YAMUX_DEBUG=1") + fi + if [ -n "${PY_YAMUX_DISABLE_HYSTERESIS:-}" ]; then + DIALER_PY_ENV+=("PY_YAMUX_DISABLE_HYSTERESIS=${PY_YAMUX_DISABLE_HYSTERESIS}") + fi + if [ -n "${PY_YAMUX_RELEASE_ON_READ:-}" ]; then + DIALER_PY_ENV+=("PY_YAMUX_RELEASE_ON_READ=${PY_YAMUX_RELEASE_ON_READ}") + fi + if [ -n "${PY_YAMUX_ASSUME_RTT_MS:-}" ]; then + DIALER_PY_ENV+=("PY_YAMUX_ASSUME_RTT_MS=${PY_YAMUX_ASSUME_RTT_MS}") + fi + if [ -n "${PY_YAMUX_BATCH_THRESHOLD_DIV:-}" ]; then + DIALER_PY_ENV+=("PY_YAMUX_BATCH_THRESHOLD_DIV=${PY_YAMUX_BATCH_THRESHOLD_DIV}") + fi +fi + if [ "${LISTENER_LEGACY}" == "true" ]; then LISTENER_ENV=$(generate_legacy_env_vars "false" "proxy-${TEST_KEY}:6379" "${TRANSPORT_NAME}" "${SECURE}" "${MUXER_NAME}") else - LISTENER_ENV=$(generate_modern_env_vars "false" "perf-redis:6379" "${TEST_KEY}" "${TRANSPORT_NAME}" "${SECURE}" "${MUXER_NAME}" "${DEBUG:-false}") + LISTENER_ENV=$(generate_modern_env_vars "false" "perf-redis:6379" "${TEST_KEY}" "${TRANSPORT_NAME}" "${SECURE}" "${MUXER_NAME}" "${DEBUG:-false}" \ + "${LISTENER_PY_ENV[@]}") fi if [ "${DIALER_LEGACY}" == "true" ]; then @@ -114,7 +156,8 @@ else "UPLOAD_ITERATIONS=${upload_iterations}" \ "DOWNLOAD_ITERATIONS=${download_iterations}" \ "LATENCY_ITERATIONS=${latency_iterations}" \ - "DURATION=${duration}") + "DURATION=${duration}" \ + "${DIALER_PY_ENV[@]}") fi # Generate docker-compose file @@ -202,15 +245,15 @@ fi log_debug " Starting containers..." log_message "Running: ${TEST_NAME}" -# Set timeout (300 seconds / 5 minutes) -TEST_TIMEOUT=300 +# Per-test harness timeout (default 300s). Set via run.sh --timeout: +# ./run.sh --timeout 900 --test-select "python-v0" --yes # Track test duration TEST_START=$(date +%s) # Start containers and wait for dialer to exit (with timeout) # WARNING: Do NOT put quotes around this because the command has two parts -if timeout "${TEST_TIMEOUT}" ${DOCKER_COMPOSE_CMD} -f "${COMPOSE_FILE}" up --exit-code-from dialer --abort-on-container-exit >> "${LOG_FILE}" 2>&1; then +if timeout "${PERF_TEST_TIMEOUT_SECS:-300}" ${DOCKER_COMPOSE_CMD} -f "${COMPOSE_FILE}" up --exit-code-from dialer --abort-on-container-exit >> "${LOG_FILE}" 2>&1; then EXIT_CODE=0 log_message " ✓ Test complete" else @@ -218,9 +261,9 @@ else # Check if it was a timeout (exit code 124) if [ "${TEST_EXIT}" -eq 124 ]; then EXIT_CODE=1 - log_error " ✗ Test timed out after ${TEST_TIMEOUT}s" + log_error " ✗ Test timed out after ${PERF_TEST_TIMEOUT_SECS:-300}s" echo "" >> "${LOG_FILE}" - log_error "Test timed out after ${TEST_TIMEOUT} seconds" + log_error "Test timed out after ${PERF_TEST_TIMEOUT_SECS:-300} seconds" else EXIT_CODE="${TEST_EXIT}" log_error " ✗ Test failed (exit code ${TEST_EXIT})" diff --git a/perf/run.sh b/perf/run.sh index 9f59105..40f5993 100755 --- a/perf/run.sh +++ b/perf/run.sh @@ -115,6 +115,9 @@ export DOWNLOAD_BYTES=1073741824 # 1GB default export ITERATIONS=10 export DURATION_PER_ITERATION=20 # seconds per iteration for throughput tests export LATENCY_ITERATIONS=100 # iterations for latency test +# Harness kills compose after this many seconds per test (see run-single-test.sh). +# Set via --timeout or PERF_TEST_TIMEOUT_SECS (default 300). +export PERF_TEST_TIMEOUT_SECS="${PERF_TEST_TIMEOUT_SECS:-300}" # Source common libraries source "${SCRIPT_LIB_DIR}/lib-github-snapshots.sh" @@ -169,6 +172,7 @@ Configuration Options: --iterations VALUE Number of iterations per test (default: 10) --duration VALUE Duration per iteration for throughput (default: 20s) --latency-iterations VALUE Iterations for latency test (default: 100) + --timeout VALUE Max seconds per test before harness abort (default: 300) --cache-dir VALUE Cache directory (default: /srv/cache) Execution Options: @@ -214,6 +218,9 @@ Examples: # Exclude specific baseline ${0} --baseline-ignore "https" + # Slow stacks (e.g. python yamux): 15 minutes per test + ${0} --timeout 900 --impl-select python --yes + # Traditional usage (still supported) ${0} --upload-bytes 5368709120 --download-bytes 5368709120 @@ -258,6 +265,7 @@ while [ $# -gt 0 ]; do --iterations) ITERATIONS="${2}"; shift 2 ;; --duration) DURATION_PER_ITERATION="${2}"; shift 2 ;; --latency-iterations) LATENCY_ITERATIONS="${2}"; shift 2 ;; + --timeout) export PERF_TEST_TIMEOUT_SECS="${2}"; shift 2 ;; --cache-dir) CACHE_DIR="${2}"; shift 2 ;; # Execution options @@ -285,6 +293,8 @@ while [ $# -gt 0 ]; do esac done +export PERF_TEST_TIMEOUT_SECS="${PERF_TEST_TIMEOUT_SECS:-300}" + # Re-derive paths from (possibly updated) CACHE_DIR # --cache-dir may have changed CACHE_DIR after init_common_variables set TEST_RUN_DIR export TEST_RUN_DIR="${CACHE_DIR}/test-run" @@ -482,6 +492,7 @@ print_message "Download Bytes: $(numfmt --to=iec --suffix=B "${DOWNLOAD_BYTES}" print_message "Iterations: ${ITERATIONS}" print_message "Duration per Iteration: ${DURATION_PER_ITERATION}s" print_message "Latency Iterations: ${LATENCY_ITERATIONS}" +print_message "Test Timeout: ${PERF_TEST_TIMEOUT_SECS}s" print_message "Full Matrix Test: ${FULL_MATRIX_TEST}" print_message "Create Snapshot: ${CREATE_SNAPSHOT}" print_message "Export Docker Images: ${EXPORT_DOCKER_IMAGES}"