From cfc995e4800580343e2a0ba8a77ba398d6cdab7a Mon Sep 17 00:00:00 2001 From: tjb Date: Wed, 1 Apr 2026 13:51:51 +0200 Subject: [PATCH 01/19] Add manual FPS benchmark Influx pipeline --- .github/workflows/fps_benchmark_influx.yaml | 91 ++++++++++++ tests/test_benchmark/conftest.py | 96 +++++++++++++ tests/test_benchmark/run_hil_tests.sh | 86 +++++++++++- .../test_benchmark_regression.py | 132 +++++++++++++++++- 4 files changed, 397 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/fps_benchmark_influx.yaml diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml new file mode 100644 index 0000000..a35085f --- /dev/null +++ b/.github/workflows/fps_benchmark_influx.yaml @@ -0,0 +1,91 @@ +name: HIL FPS Benchmark Influx + +on: + workflow_dispatch: + inputs: + testbed: + description: "HIL testbed to run on" + required: true + type: string + depthai_version: + description: "DepthAI version to test against (e.g. 3.3.0)" + required: true + type: string + os_version: + description: "Optional camera OS version to install before the benchmark" + required: false + type: string + hold_reservation: + description: "Hold testbed reservation after the run" + required: false + type: boolean + +env: + TESTBED: ${{ github.event.inputs.testbed }} + DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} + OS_VERSION: ${{ github.event.inputs.os_version }} + HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} + HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} + HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} + INFLUX_BUCKET: fps_metric + +jobs: + fps-benchmark: + runs-on: ["self-hosted", "testbed-runner"] + + steps: + - uses: actions/checkout@v4 + + - name: Run benchmark and push to Influx + run: | + set -euo pipefail + + pip install hil-framework --upgrade \ + --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" + + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark-influx" + RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" + + HOLD_RESERVATION_OPTION="" + if [[ "$HOLD_RESERVATION" == "true" ]]; then + HOLD_RESERVATION_OPTION="--hold-reservation" + fi + + OS_VERSION_OPTION="" + if [[ -n "$OS_VERSION" ]]; then + OS_VERSION_OPTION="--os-version $OS_VERSION" + fi + + if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then + DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION" + else + echo "Invalid depthai version: $DEPTHAI_VERSION" >&2 + exit 1 + fi + + : "${INFLUX_HOST:?INFLUX_HOST is required on the runner}" + : "${INFLUX_ORG:?INFLUX_ORG is required on the runner}" + : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}" + + : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}" + + REMOTE_CMD="export HIL_RUN_ID=\"$GITHUB_RUN_ID\" \ + INFLUX_HOST=\"$INFLUX_HOST\" \ + INFLUX_ORG=\"$INFLUX_ORG\" \ + INFLUX_BUCKET=\"$INFLUX_BUCKET\" \ + INFLUX_TOKEN=\"$INFLUX_TOKEN\" && \ + cd /tmp/modelconverter && \ + ./tests/test_benchmark/run_hil_tests.sh \ + \"$HUBAI_API_KEY\" \ + \"$HIL_FRAMEWORK_TOKEN\" \ + \"$DEPTHAI_VERSION_CHECKED\" \ + \"$TESTBED\"" + + exec hil_runner \ + --testbed "$TESTBED" \ + $HOLD_RESERVATION_OPTION \ + --wait \ + $OS_VERSION_OPTION \ + $RESERVATION_OPTION \ + --sync-workspace --rsync-args="--exclude=venv" \ + --commands "$REMOTE_CMD" diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 9d91a9c..4f4ca3f 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -1,4 +1,6 @@ import os +from datetime import datetime, timezone +from uuid import uuid4 import pytest @@ -16,6 +18,54 @@ def pytest_addoption(parser: pytest.Parser) -> None: default="rvc4", help="Target platform to benchmark (default: rvc4).", ) + parser.addoption( + "--testbed-name", + action="store", + default=None, + help="Logical HIL testbed name for Influx metadata.", + ) + parser.addoption( + "--camera-mxid", + action="store", + default=None, + help="Camera MXID for Influx metadata.", + ) + parser.addoption( + "--camera-os-version", + action="store", + default=None, + help="Camera OS version for Influx metadata.", + ) + parser.addoption( + "--camera-model", + action="store", + default=None, + help="Camera model for Influx metadata.", + ) + parser.addoption( + "--camera-agent-version", + action="store", + default=None, + help="Camera agent version for Influx metadata.", + ) + parser.addoption( + "--runner", + action="store", + default=None, + help="Runner name for Influx metadata.", + ) + parser.addoption( + "--server-os", + action="store", + default=None, + help="Server OS for Influx metadata.", + ) + parser.addoption( + "--run-id", + action="store", + default=None, + help="Optional run identifier for grouping benchmark results in Influx.", + ) def pytest_configure(config: pytest.Config) -> None: @@ -34,3 +84,49 @@ def device_ip(request: pytest.FixtureRequest) -> str | None: @pytest.fixture def benchmark_target(request: pytest.FixtureRequest) -> str: return request.config.getoption("--benchmark-target") + + +def _option_or_env( + request: pytest.FixtureRequest, + option_name: str, + env_name: str, +) -> str | None: + return request.config.getoption(option_name) or os.environ.get(env_name) + + +@pytest.fixture(scope="session") +def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: + return { + "testbed_name": _option_or_env( + request, "--testbed-name", "HIL_TESTBED_NAME" + ), + "camera_mxid": _option_or_env( + request, "--camera-mxid", "HIL_CAMERA_MXID" + ), + "camera_os_version": _option_or_env( + request, "--camera-os-version", "HIL_CAMERA_OS_VERSION" + ), + "camera_model": _option_or_env( + request, "--camera-model", "HIL_CAMERA_MODEL" + ), + "camera_agent_version": _option_or_env( + request, "--camera-agent-version", "HIL_CAMERA_AGENT_VERSION" + ), + "runner": _option_or_env(request, "--runner", "HIL_RUNNER") + or os.environ.get("GITHUB_RUNNER_NAME") + or os.environ.get("HOSTNAME") + or os.environ.get("USER"), + "server_os": _option_or_env(request, "--server-os", "HIL_SERVER_OS"), + } + + +@pytest.fixture(scope="session") +def benchmark_run_id(request: pytest.FixtureRequest) -> str: + configured_run_id = ( + request.config.getoption("--run-id") or os.environ.get("HIL_RUN_ID") + ) + if configured_run_id: + return configured_run_id + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return f"benchmark-{timestamp}-{uuid4().hex[:8]}" diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index e0ae2f0..a5010b2 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -3,8 +3,8 @@ set -e # Exit immediately if a command fails # Check if required arguments were provided -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "Usage: $0 " +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ -z "$4" ]; then + echo "Usage: $0 " exit 1 fi @@ -12,6 +12,7 @@ fi export HUBAI_API_KEY="$1" export PAT_TOKEN="$2" export DEPTHAI_VERSION="$3" +export HIL_TESTBED_NAME="$4" # Navigate to project directory cd /tmp/modelconverter @@ -35,10 +36,81 @@ pip install --upgrade \ --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local \ "depthai==${DEPTHAI_VERSION}" -# Extract hostname of first rvc4 device -hostname=$(hil_camera -t "$HIL_TESTBED" -n test all info -j \ - | jq -r '.[] | select(.platform=="rvc4") | .hostname' \ - | head -n1) +# Cache device metadata once for the whole run using oakctl. If oakctl is not +# available, keep the benchmark runnable but disable Influx push for this run. +oakctl_output=$(oakctl list --format json 2>/dev/null || printf '') + +if [ -z "$oakctl_output" ]; then + echo "Warning: best-effort metadata lookup via oakctl failed; disabling Influx push for this run." >&2 + unset INFLUX_HOST INFLUX_ORG INFLUX_BUCKET INFLUX_TOKEN +fi + +device_hostname=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].ip_addresses[0] // empty' 2>/dev/null \ + | head -n1 +) +camera_mxid=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].serial_number // empty' 2>/dev/null \ + | head -n1 +) +camera_model=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].model // empty' 2>/dev/null \ + | head -n1 +) +camera_agent_version=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].agent_version // empty' 2>/dev/null \ + | head -n1 +) +camera_os=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].os // empty' 2>/dev/null \ + | head -n1 +) +runner_hostname=$(hostname 2>/dev/null || printf 'unknown') +server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') + +if [ -z "$device_hostname" ]; then + device_hostname="unknown" +fi +if [ -z "$camera_mxid" ]; then + camera_mxid="unknown" +fi +if [ -z "$camera_model" ]; then + camera_model="unknown" +fi +if [ -z "$camera_agent_version" ]; then + camera_agent_version="unknown" +fi +if [ -z "$camera_os" ]; then + camera_os="unknown" +fi +if [ -z "$runner_hostname" ]; then + runner_hostname="unknown" +fi +if [ -z "$server_os" ]; then + server_os="unknown" +fi # Run tests -pytest -s -v tests/test_benchmark/ --device-ip "$hostname" \ No newline at end of file +pytest_args=( + -s + -v + tests/test_benchmark/ + --testbed-name "$HIL_TESTBED_NAME" + --camera-mxid "$camera_mxid" + --camera-os-version "$camera_os" + --camera-model "$camera_model" + --camera-agent-version "$camera_agent_version" + --runner "$runner_hostname" + --server-os "$server_os" +) + +if [ "$device_hostname" != "unknown" ]; then + pytest_args+=(--device-ip "$device_hostname") +fi + +pytest "${pytest_args[@]}" diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index bcd91b6..de83120 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -1,5 +1,7 @@ import json +import os from pathlib import Path +from urllib import error, parse, request import pytest @@ -21,6 +23,115 @@ def _model_id(slug: str) -> str: return slug.rsplit("/", 1)[-1] +def _escape_tag(value: str) -> str: + return ( + value.replace("\\", "\\\\") + .replace(",", "\\,") + .replace(" ", "\\ ") + .replace("=", "\\=") + ) + + +def _format_field(value: str | float | bool) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int) and not isinstance(value, bool): + return f"{value}i" + if isinstance(value, float): + return repr(value) + + escaped = str(value).replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def _write_fps_result_to_influx( + *, + model_slug: str, + benchmark_target: str, + benchmark_run_id: str, + device_ip: str | None, + actual_fps: float, + expected_fps: float, + tolerance_low: float, + tolerance_high: float, + fps_min: float, + fps_max: float, + deviation_pct: float, + success: bool, + influx_metadata: dict[str, str | None], +) -> None: + influx_url = os.environ.get("INFLUX_HOST") + influx_org = os.environ.get("INFLUX_ORG") + influx_bucket = os.environ.get("INFLUX_BUCKET") + influx_token = os.environ.get("INFLUX_TOKEN") + + if not all([influx_url, influx_org, influx_bucket, influx_token]): + return + + tags = { + "model_slug": model_slug, + "benchmark_target": benchmark_target, + "run_id": benchmark_run_id, + "status": "passed" if success else "failed", + } + optional_tags = { + "testbed_name": influx_metadata.get("testbed_name"), + "camera_mxid": influx_metadata.get("camera_mxid"), + "camera_os_version": influx_metadata.get("camera_os_version"), + "camera_model": influx_metadata.get("camera_model"), + "camera_agent_version": influx_metadata.get("camera_agent_version"), + "runner": influx_metadata.get("runner"), + "server_os": influx_metadata.get("server_os"), + "device_ip": device_ip, + } + tags.update( + { + key: value + for key, value in optional_tags.items() + if value not in (None, "") + } + ) + + fields = { + "actual_fps": actual_fps, + "expected_fps": expected_fps, + "tolerance_low": tolerance_low, + "tolerance_high": tolerance_high, + "fps_min": fps_min, + "fps_max": fps_max, + "deviation_pct": deviation_pct, + "success": success, + } + + tag_set = ",".join(f"{key}={_escape_tag(value)}" for key, value in tags.items()) + field_set = ",".join( + f"{key}={_format_field(value)}" for key, value in fields.items() + ) + line = f"fps_benchmark,{tag_set} {field_set}" + + write_url = ( + f"{influx_url.rstrip('/')}/api/v2/write?" + f"org={parse.quote(influx_org, safe='')}&" + f"bucket={parse.quote(influx_bucket, safe='')}&precision=ns" + ) + influx_request = request.Request( + write_url, + data=line.encode(), + headers={ + "Authorization": f"Token {influx_token}", + "Content-Type": "text/plain; charset=utf-8", + "Accept": "application/json", + }, + method="POST", + ) + + try: + with request.urlopen(influx_request, timeout=5): + return + except (error.URLError, TimeoutError, OSError) as exc: + print(f"Failed to write benchmark result to InfluxDB: {exc}") + + @pytest.mark.parametrize( "model_slug", _model_slugs("rvc4"), @@ -30,6 +141,8 @@ def test_benchmark_fps( model_slug: str, device_ip: str | None, benchmark_target: str, + benchmark_run_id: str, + influx_metadata: dict[str, str | None], ) -> None: model_config = _targets_data[benchmark_target][model_slug] expected_fps = model_config["expected_fps"] @@ -60,13 +173,30 @@ def test_benchmark_fps( fps_max = expected_fps * (1 + tolerance_high) deviation_pct = ((actual_fps - expected_fps) / expected_fps) * 100 + success = fps_min <= actual_fps <= fps_max print( f"Benchmark result for {model_slug}: " f"actual={actual_fps:.2f} FPS, expected={expected_fps:.2f} FPS. " ) - assert fps_min <= actual_fps <= fps_max, ( + _write_fps_result_to_influx( + model_slug=model_slug, + benchmark_target=benchmark_target, + benchmark_run_id=benchmark_run_id, + device_ip=device_ip, + actual_fps=actual_fps, + expected_fps=expected_fps, + tolerance_low=tolerance_low, + tolerance_high=tolerance_high, + fps_min=fps_min, + fps_max=fps_max, + deviation_pct=deviation_pct, + success=success, + influx_metadata=influx_metadata, + ) + + assert success, ( f"FPS regression for {model_slug}: " f"actual={actual_fps:.2f}, expected={expected_fps:.2f} " f"(deviation: {deviation_pct:+.1f}%, " From ce82fcd4568f4ad8aa5734df47984f1cd453c425 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 08:32:15 +0200 Subject: [PATCH 02/19] Use model-based HIL selection for FPS Influx workflow --- .github/workflows/fps_benchmark_influx.yaml | 10 ++-------- tests/test_benchmark/run_hil_tests.sh | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml index a35085f..2d322a0 100644 --- a/.github/workflows/fps_benchmark_influx.yaml +++ b/.github/workflows/fps_benchmark_influx.yaml @@ -3,10 +3,6 @@ name: HIL FPS Benchmark Influx on: workflow_dispatch: inputs: - testbed: - description: "HIL testbed to run on" - required: true - type: string depthai_version: description: "DepthAI version to test against (e.g. 3.3.0)" required: true @@ -21,7 +17,6 @@ on: type: boolean env: - TESTBED: ${{ github.event.inputs.testbed }} DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} OS_VERSION: ${{ github.event.inputs.os_version }} HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} @@ -78,11 +73,10 @@ jobs: ./tests/test_benchmark/run_hil_tests.sh \ \"$HUBAI_API_KEY\" \ \"$HIL_FRAMEWORK_TOKEN\" \ - \"$DEPTHAI_VERSION_CHECKED\" \ - \"$TESTBED\"" + \"$DEPTHAI_VERSION_CHECKED\"" exec hil_runner \ - --testbed "$TESTBED" \ + --models "oak4_pro or oak4_d or oak4_s" \ $HOLD_RESERVATION_OPTION \ --wait \ $OS_VERSION_OPTION \ diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index a5010b2..9690ee2 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -3,8 +3,8 @@ set -e # Exit immediately if a command fails # Check if required arguments were provided -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ -z "$4" ]; then - echo "Usage: $0 " +if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then + echo "Usage: $0 [TESTBED_NAME]" exit 1 fi @@ -12,7 +12,7 @@ fi export HUBAI_API_KEY="$1" export PAT_TOKEN="$2" export DEPTHAI_VERSION="$3" -export HIL_TESTBED_NAME="$4" +export HIL_TESTBED_NAME="${4:-}" # Navigate to project directory cd /tmp/modelconverter @@ -70,6 +70,11 @@ camera_os=$( | jq -r '.items[0].os // empty' 2>/dev/null \ | head -n1 ) +detected_testbed_name=$( + printf '%s' "$oakctl_output" \ + | jq -r '.items[0].name // .items[0].hostname // .items[0].id // empty' 2>/dev/null \ + | head -n1 +) runner_hostname=$(hostname 2>/dev/null || printf 'unknown') server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') @@ -94,6 +99,12 @@ fi if [ -z "$server_os" ]; then server_os="unknown" fi +if [ -z "$HIL_TESTBED_NAME" ]; then + HIL_TESTBED_NAME="${detected_testbed_name:-}" +fi +if [ -z "$HIL_TESTBED_NAME" ]; then + HIL_TESTBED_NAME="unknown" +fi # Run tests pytest_args=( From f14ff4241b8705053071e375c96b506fdb2ae203 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 08:57:04 +0200 Subject: [PATCH 03/19] Temporarily repurpose rvc4 workflow for FPS Influx dispatch --- .github/workflows/rvc4_test.yaml | 115 +++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 35 deletions(-) diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index 0067be2..2d322a0 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -1,40 +1,85 @@ -name: Test - RVC4 +name: HIL FPS Benchmark Influx on: workflow_dispatch: - pull_request: - branches: [main] - paths: - - requirements.txt - - "modelconverter/__init__.py" - - "modelconverter/packages/rvc4/**" - - "modelconverter/packages/base_exporter.py" - - "modelconverter/packages/base_inferer.py" - - "tests/test_packages/test_rvc4.py" - - "docker/rvc4/Dockerfile" - - "docker/rvc4/entrypoint.sh" - - ".github/workflows/modelconverter_test.yaml" - - ".github/workflows/rvc4_test.yaml" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + inputs: + depthai_version: + description: "DepthAI version to test against (e.g. 3.3.0)" + required: true + type: string + os_version: + description: "Optional camera OS version to install before the benchmark" + required: false + type: string + hold_reservation: + description: "Hold testbed reservation after the run" + required: false + type: boolean + +env: + DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} + OS_VERSION: ${{ github.event.inputs.os_version }} + HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} + HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} + HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} + INFLUX_BUCKET: fps_metric jobs: - rvc4-test: - strategy: - fail-fast: false - matrix: - version: - - "2.23.0" - - "2.24.0" - - "2.25.0" - - "2.26.2" - - "2.27.0" - - "2.32.6" - - uses: ./.github/workflows/modelconverter_test.yaml - secrets: inherit - with: - package: rvc4 - version: ${{ matrix.version }} + fps-benchmark: + runs-on: ["self-hosted", "testbed-runner"] + + steps: + - uses: actions/checkout@v4 + + - name: Run benchmark and push to Influx + run: | + set -euo pipefail + + pip install hil-framework --upgrade \ + --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" + + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark-influx" + RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" + + HOLD_RESERVATION_OPTION="" + if [[ "$HOLD_RESERVATION" == "true" ]]; then + HOLD_RESERVATION_OPTION="--hold-reservation" + fi + + OS_VERSION_OPTION="" + if [[ -n "$OS_VERSION" ]]; then + OS_VERSION_OPTION="--os-version $OS_VERSION" + fi + + if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then + DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION" + else + echo "Invalid depthai version: $DEPTHAI_VERSION" >&2 + exit 1 + fi + + : "${INFLUX_HOST:?INFLUX_HOST is required on the runner}" + : "${INFLUX_ORG:?INFLUX_ORG is required on the runner}" + : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}" + + : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}" + + REMOTE_CMD="export HIL_RUN_ID=\"$GITHUB_RUN_ID\" \ + INFLUX_HOST=\"$INFLUX_HOST\" \ + INFLUX_ORG=\"$INFLUX_ORG\" \ + INFLUX_BUCKET=\"$INFLUX_BUCKET\" \ + INFLUX_TOKEN=\"$INFLUX_TOKEN\" && \ + cd /tmp/modelconverter && \ + ./tests/test_benchmark/run_hil_tests.sh \ + \"$HUBAI_API_KEY\" \ + \"$HIL_FRAMEWORK_TOKEN\" \ + \"$DEPTHAI_VERSION_CHECKED\"" + + exec hil_runner \ + --models "oak4_pro or oak4_d or oak4_s" \ + $HOLD_RESERVATION_OPTION \ + --wait \ + $OS_VERSION_OPTION \ + $RESERVATION_OPTION \ + --sync-workspace --rsync-args="--exclude=venv" \ + --commands "$REMOTE_CMD" From d0bf4d70d05ae64274619c1022a8e7c9d5c19312 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 09:32:22 +0200 Subject: [PATCH 04/19] Map Influx GitHub variables into FPS workflow env --- .github/workflows/rvc4_test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index 2d322a0..16542a9 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -22,6 +22,9 @@ env: HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} + INFLUX_HOST: ${{ vars.INFLUX_HOST }} + INFLUX_ORG: ${{ vars.INFLUX_ORG }} + INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }} INFLUX_BUCKET: fps_metric jobs: From f65c94d44de69fa2317048e7aa07ed925ccc15fc Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 09:42:28 +0200 Subject: [PATCH 05/19] Read Influx workflow settings from GitHub secrets --- .github/workflows/rvc4_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index 16542a9..c360cff 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -22,8 +22,8 @@ env: HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} - INFLUX_HOST: ${{ vars.INFLUX_HOST }} - INFLUX_ORG: ${{ vars.INFLUX_ORG }} + INFLUX_HOST: ${{ secrets.INFLUX_HOST }} + INFLUX_ORG: ${{ secrets.INFLUX_ORG }} INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }} INFLUX_BUCKET: fps_metric From 5fad3b69876d9c1e40f7aa581a5210bb6ed7a5ec Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 10:04:44 +0200 Subject: [PATCH 06/19] Add fake Influx CSV fixture for FPS metrics --- fake_fps_metric.csv | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 fake_fps_metric.csv diff --git a/fake_fps_metric.csv b/fake_fps_metric.csv new file mode 100644 index 0000000..be44f24 --- /dev/null +++ b/fake_fps_metric.csv @@ -0,0 +1,10 @@ +#datatype measurement,tag,tag,tag,tag,tag,field,double,double,double,double,double,tag,dateTime:RFC3339 +#group false,false,false,false,false,false,false,false,false,false,false,false,false +#default fps_benchmark,,,,,,,,,,,, +_measurement,model_slug,benchmark_target,run_id,status,testbed_name,success,actual_fps,expected_fps,tolerance_low,tolerance_high,deviation_pct,device_ip,_time +fps_benchmark,luxonis/yolov6-nano:r2-coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,830.23,750.00,0.10,0.30,10.70,10.12.142.57,2026-04-02T07:53:02Z +fps_benchmark,luxonis/yolov8-instance-segmentation-nano:coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,568.62,554.68,0.10,0.30,2.51,10.12.142.57,2026-04-02T07:53:30Z +fps_benchmark,luxonis/yolov8-nano-pose-estimation:coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,600.26,609.11,0.10,0.30,-1.45,10.12.142.57,2026-04-02T07:53:57Z +fps_benchmark,luxonis/fastsam-s:512x288,rvc4,manual-test-001,passed,testbed-a,true,497.35,475.68,0.10,0.30,4.55,10.12.142.57,2026-04-02T07:54:26Z +fps_benchmark,luxonis/deeplab-v3-plus:512x288,rvc4,manual-test-001,passed,testbed-a,true,317.15,339.42,0.10,0.30,-6.56,10.12.142.57,2026-04-02T07:54:54Z +fps_benchmark,luxonis/deeplab-v3-plus:512x288,rvc4,manual-test-002,failed,testbed-a,false,250.00,339.42,0.10,0.30,-26.35,10.12.142.57,2026-04-02T08:10:00Z From c95f97828a99c9bdefd04b473279e3cb82119c46 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 10:06:28 +0200 Subject: [PATCH 07/19] Fix annotated CSV format for fake Influx fixture --- fake_fps_metric.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fake_fps_metric.csv b/fake_fps_metric.csv index be44f24..fd60727 100644 --- a/fake_fps_metric.csv +++ b/fake_fps_metric.csv @@ -1,6 +1,6 @@ -#datatype measurement,tag,tag,tag,tag,tag,field,double,double,double,double,double,tag,dateTime:RFC3339 -#group false,false,false,false,false,false,false,false,false,false,false,false,false -#default fps_benchmark,,,,,,,,,,,, +#datatype measurement,tag,tag,tag,tag,tag,boolean,double,double,double,double,double,tag,dateTime:RFC3339 +#group false,false,false,false,false,false,false,false,false,false,false,false,false,false +#default ,,,,,,,,,,,,, _measurement,model_slug,benchmark_target,run_id,status,testbed_name,success,actual_fps,expected_fps,tolerance_low,tolerance_high,deviation_pct,device_ip,_time fps_benchmark,luxonis/yolov6-nano:r2-coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,830.23,750.00,0.10,0.30,10.70,10.12.142.57,2026-04-02T07:53:02Z fps_benchmark,luxonis/yolov8-instance-segmentation-nano:coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,568.62,554.68,0.10,0.30,2.51,10.12.142.57,2026-04-02T07:53:30Z From d9308bdfbcab5a3a4bdec6a6e5354b617c9b1483 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 10:09:02 +0200 Subject: [PATCH 08/19] Add fake Influx line protocol fixture --- fake_fps_metric.lp | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 fake_fps_metric.lp diff --git a/fake_fps_metric.lp b/fake_fps_metric.lp new file mode 100644 index 0000000..f8d35d5 --- /dev/null +++ b/fake_fps_metric.lp @@ -0,0 +1,6 @@ +fps_benchmark,model_slug=luxonis/yolov6-nano:r2-coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=830.23,expected_fps=750.00,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=10.70 1775116382000000000 +fps_benchmark,model_slug=luxonis/yolov8-instance-segmentation-nano:coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=568.62,expected_fps=554.68,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=2.51 1775116410000000000 +fps_benchmark,model_slug=luxonis/yolov8-nano-pose-estimation:coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=600.26,expected_fps=609.11,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-1.45 1775116437000000000 +fps_benchmark,model_slug=luxonis/fastsam-s:512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=497.35,expected_fps=475.68,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=4.55 1775116466000000000 +fps_benchmark,model_slug=luxonis/deeplab-v3-plus:512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=317.15,expected_fps=339.42,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-6.56 1775116494000000000 +fps_benchmark,model_slug=luxonis/deeplab-v3-plus:512x288,benchmark_target=rvc4,run_id=manual-test-002,status=failed,testbed_name=testbed-a,device_ip=10.12.142.57 success=false,actual_fps=250.00,expected_fps=339.42,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-26.35 1775117400000000000 From 5e9630903fb2dbf2086511e159592be6ba47d553 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 10:11:37 +0200 Subject: [PATCH 09/19] Use correct Influx bucket name in FPS workflows --- .github/workflows/fps_benchmark_influx.yaml | 2 +- .github/workflows/rvc4_test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml index 2d322a0..46b9bc8 100644 --- a/.github/workflows/fps_benchmark_influx.yaml +++ b/.github/workflows/fps_benchmark_influx.yaml @@ -22,7 +22,7 @@ env: HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} - INFLUX_BUCKET: fps_metric + INFLUX_BUCKET: fps_metrics jobs: fps-benchmark: diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index c360cff..15ffdb5 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -25,7 +25,7 @@ env: INFLUX_HOST: ${{ secrets.INFLUX_HOST }} INFLUX_ORG: ${{ secrets.INFLUX_ORG }} INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }} - INFLUX_BUCKET: fps_metric + INFLUX_BUCKET: fps_metrics jobs: fps-benchmark: From e7e23cd1147c899578c61107a92a5e51c6e474c8 Mon Sep 17 00:00:00 2001 From: tjb Date: Thu, 2 Apr 2026 12:38:25 +0200 Subject: [PATCH 10/19] Normalize FPS Influx tags and testbed fallback --- tests/test_benchmark/run_hil_tests.sh | 3 ++ .../test_benchmark_regression.py | 35 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 9690ee2..e413551 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -102,6 +102,9 @@ fi if [ -z "$HIL_TESTBED_NAME" ]; then HIL_TESTBED_NAME="${detected_testbed_name:-}" fi +if [ -z "$HIL_TESTBED_NAME" ]; then + HIL_TESTBED_NAME="$(hostname 2>/dev/null || printf '')" +fi if [ -z "$HIL_TESTBED_NAME" ]; then HIL_TESTBED_NAME="unknown" fi diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index de83120..5ca88ac 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -32,6 +32,12 @@ def _escape_tag(value: str) -> str: ) +def _normalize_tag(value: str | None) -> str: + if value in (None, ""): + return "unknown" + return str(value) + + def _format_field(value: str | float | bool) -> str: if isinstance(value, bool): return "true" if value else "false" @@ -73,24 +79,19 @@ def _write_fps_result_to_influx( "benchmark_target": benchmark_target, "run_id": benchmark_run_id, "status": "passed" if success else "failed", + "testbed_name": _normalize_tag(influx_metadata.get("testbed_name")), + "camera_mxid": _normalize_tag(influx_metadata.get("camera_mxid")), + "camera_os_version": _normalize_tag( + influx_metadata.get("camera_os_version") + ), + "camera_model": _normalize_tag(influx_metadata.get("camera_model")), + "camera_agent_version": _normalize_tag( + influx_metadata.get("camera_agent_version") + ), + "runner": _normalize_tag(influx_metadata.get("runner")), + "server_os": _normalize_tag(influx_metadata.get("server_os")), + "device_ip": _normalize_tag(device_ip), } - optional_tags = { - "testbed_name": influx_metadata.get("testbed_name"), - "camera_mxid": influx_metadata.get("camera_mxid"), - "camera_os_version": influx_metadata.get("camera_os_version"), - "camera_model": influx_metadata.get("camera_model"), - "camera_agent_version": influx_metadata.get("camera_agent_version"), - "runner": influx_metadata.get("runner"), - "server_os": influx_metadata.get("server_os"), - "device_ip": device_ip, - } - tags.update( - { - key: value - for key, value in optional_tags.items() - if value not in (None, "") - } - ) fields = { "actual_fps": actual_fps, From 45768b9c2ef98c7d2a444c619da0fbe9b4335797 Mon Sep 17 00:00:00 2001 From: tjb Date: Tue, 7 Apr 2026 14:17:49 +0200 Subject: [PATCH 11/19] Temporary debugging logging --- tests/test_benchmark/conftest.py | 9 ++++ tests/test_benchmark/run_hil_tests.sh | 41 ++++++++++++++----- .../test_benchmark_regression.py | 8 ++++ 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 4f4ca3f..8d4484b 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -60,6 +60,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help="Server OS for Influx metadata.", ) + parser.addoption( + "--depthai-version", + action="store", + default=None, + help="DepthAI version used for the benchmark run.", + ) parser.addoption( "--run-id", action="store", @@ -117,6 +123,9 @@ def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: or os.environ.get("HOSTNAME") or os.environ.get("USER"), "server_os": _option_or_env(request, "--server-os", "HIL_SERVER_OS"), + "depthai_version": _option_or_env( + request, "--depthai-version", "DEPTHAI_VERSION" + ), } diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index e413551..28f97b5 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -2,6 +2,8 @@ set -e # Exit immediately if a command fails +COULD_NOT_OBTAIN="could_not_obtain" + # Check if required arguments were provided if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then echo "Usage: $0 [TESTBED_NAME]" @@ -37,12 +39,12 @@ pip install --upgrade \ "depthai==${DEPTHAI_VERSION}" # Cache device metadata once for the whole run using oakctl. If oakctl is not -# available, keep the benchmark runnable but disable Influx push for this run. +# available, keep the benchmark runnable and record explicit placeholder values +# for the missing oakctl-derived metadata. oakctl_output=$(oakctl list --format json 2>/dev/null || printf '') if [ -z "$oakctl_output" ]; then - echo "Warning: best-effort metadata lookup via oakctl failed; disabling Influx push for this run." >&2 - unset INFLUX_HOST INFLUX_ORG INFLUX_BUCKET INFLUX_TOKEN + echo "Warning: best-effort metadata lookup via oakctl failed; using placeholder metadata for this run." >&2 fi device_hostname=$( @@ -79,19 +81,19 @@ runner_hostname=$(hostname 2>/dev/null || printf 'unknown') server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') if [ -z "$device_hostname" ]; then - device_hostname="unknown" + device_hostname="$COULD_NOT_OBTAIN" fi if [ -z "$camera_mxid" ]; then - camera_mxid="unknown" + camera_mxid="$COULD_NOT_OBTAIN" fi if [ -z "$camera_model" ]; then - camera_model="unknown" + camera_model="$COULD_NOT_OBTAIN" fi if [ -z "$camera_agent_version" ]; then - camera_agent_version="unknown" + camera_agent_version="$COULD_NOT_OBTAIN" fi if [ -z "$camera_os" ]; then - camera_os="unknown" + camera_os="$COULD_NOT_OBTAIN" fi if [ -z "$runner_hostname" ]; then runner_hostname="unknown" @@ -106,7 +108,7 @@ if [ -z "$HIL_TESTBED_NAME" ]; then HIL_TESTBED_NAME="$(hostname 2>/dev/null || printf '')" fi if [ -z "$HIL_TESTBED_NAME" ]; then - HIL_TESTBED_NAME="unknown" + HIL_TESTBED_NAME="$COULD_NOT_OBTAIN" fi # Run tests @@ -121,10 +123,29 @@ pytest_args=( --camera-agent-version "$camera_agent_version" --runner "$runner_hostname" --server-os "$server_os" + --depthai-version "$DEPTHAI_VERSION" ) -if [ "$device_hostname" != "unknown" ]; then +if [ "$device_hostname" != "$COULD_NOT_OBTAIN" ]; then pytest_args+=(--device-ip "$device_hostname") fi +echo "Influx metadata debug:" +echo " INFLUX_HOST=${INFLUX_HOST:-}" +echo " INFLUX_ORG=${INFLUX_ORG:-}" +echo " INFLUX_BUCKET=${INFLUX_BUCKET:-}" +echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" +echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" +echo " HIL_TESTBED_NAME=${HIL_TESTBED_NAME:-}" +echo " device_ip=${device_hostname:-}" +echo " camera_mxid=${camera_mxid:-}" +echo " camera_os_version=${camera_os:-}" +echo " camera_model=${camera_model:-}" +echo " camera_agent_version=${camera_agent_version:-}" +echo " runner=${runner_hostname:-}" +echo " server_os=${server_os:-}" +printf ' pytest_args:' +printf ' %q' "${pytest_args[@]}" +printf '\n' + pytest "${pytest_args[@]}" diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 5ca88ac..8544c7b 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -90,6 +90,9 @@ def _write_fps_result_to_influx( ), "runner": _normalize_tag(influx_metadata.get("runner")), "server_os": _normalize_tag(influx_metadata.get("server_os")), + "depthai_version": _normalize_tag( + influx_metadata.get("depthai_version") + ), "device_ip": _normalize_tag(device_ip), } @@ -109,12 +112,17 @@ def _write_fps_result_to_influx( f"{key}={_format_field(value)}" for key, value in fields.items() ) line = f"fps_benchmark,{tag_set} {field_set}" + print(f"Influx write debug: {line}") write_url = ( f"{influx_url.rstrip('/')}/api/v2/write?" f"org={parse.quote(influx_org, safe='')}&" f"bucket={parse.quote(influx_bucket, safe='')}&precision=ns" ) + print( + "Influx request debug: " + f"url={write_url}, token={'' if influx_token else ''}" + ) influx_request = request.Request( write_url, data=line.encode(), From 22fd18c2818050924169724e6b1d8d794a0fed8d Mon Sep 17 00:00:00 2001 From: tjb Date: Wed, 8 Apr 2026 13:06:43 +0200 Subject: [PATCH 12/19] Cleanup Commit --- .github/workflows/rvc4_test.yaml | 6 +++--- fake_fps_metric.csv | 10 ---------- fake_fps_metric.lp | 6 ------ 3 files changed, 3 insertions(+), 19 deletions(-) delete mode 100644 fake_fps_metric.csv delete mode 100644 fake_fps_metric.lp diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index 15ffdb5..570881c 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -1,4 +1,4 @@ -name: HIL FPS Benchmark Influx +name: HIL FPS Benchmark on: workflow_dispatch: @@ -34,14 +34,14 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Run benchmark and push to Influx + - name: Run benchmark run: | set -euo pipefail pip install hil-framework --upgrade \ --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" - export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark-influx" + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark" RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" HOLD_RESERVATION_OPTION="" diff --git a/fake_fps_metric.csv b/fake_fps_metric.csv deleted file mode 100644 index fd60727..0000000 --- a/fake_fps_metric.csv +++ /dev/null @@ -1,10 +0,0 @@ -#datatype measurement,tag,tag,tag,tag,tag,boolean,double,double,double,double,double,tag,dateTime:RFC3339 -#group false,false,false,false,false,false,false,false,false,false,false,false,false,false -#default ,,,,,,,,,,,,, -_measurement,model_slug,benchmark_target,run_id,status,testbed_name,success,actual_fps,expected_fps,tolerance_low,tolerance_high,deviation_pct,device_ip,_time -fps_benchmark,luxonis/yolov6-nano:r2-coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,830.23,750.00,0.10,0.30,10.70,10.12.142.57,2026-04-02T07:53:02Z -fps_benchmark,luxonis/yolov8-instance-segmentation-nano:coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,568.62,554.68,0.10,0.30,2.51,10.12.142.57,2026-04-02T07:53:30Z -fps_benchmark,luxonis/yolov8-nano-pose-estimation:coco-512x288,rvc4,manual-test-001,passed,testbed-a,true,600.26,609.11,0.10,0.30,-1.45,10.12.142.57,2026-04-02T07:53:57Z -fps_benchmark,luxonis/fastsam-s:512x288,rvc4,manual-test-001,passed,testbed-a,true,497.35,475.68,0.10,0.30,4.55,10.12.142.57,2026-04-02T07:54:26Z -fps_benchmark,luxonis/deeplab-v3-plus:512x288,rvc4,manual-test-001,passed,testbed-a,true,317.15,339.42,0.10,0.30,-6.56,10.12.142.57,2026-04-02T07:54:54Z -fps_benchmark,luxonis/deeplab-v3-plus:512x288,rvc4,manual-test-002,failed,testbed-a,false,250.00,339.42,0.10,0.30,-26.35,10.12.142.57,2026-04-02T08:10:00Z diff --git a/fake_fps_metric.lp b/fake_fps_metric.lp deleted file mode 100644 index f8d35d5..0000000 --- a/fake_fps_metric.lp +++ /dev/null @@ -1,6 +0,0 @@ -fps_benchmark,model_slug=luxonis/yolov6-nano:r2-coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=830.23,expected_fps=750.00,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=10.70 1775116382000000000 -fps_benchmark,model_slug=luxonis/yolov8-instance-segmentation-nano:coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=568.62,expected_fps=554.68,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=2.51 1775116410000000000 -fps_benchmark,model_slug=luxonis/yolov8-nano-pose-estimation:coco-512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=600.26,expected_fps=609.11,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-1.45 1775116437000000000 -fps_benchmark,model_slug=luxonis/fastsam-s:512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=497.35,expected_fps=475.68,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=4.55 1775116466000000000 -fps_benchmark,model_slug=luxonis/deeplab-v3-plus:512x288,benchmark_target=rvc4,run_id=manual-test-001,status=passed,testbed_name=testbed-a,device_ip=10.12.142.57 success=true,actual_fps=317.15,expected_fps=339.42,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-6.56 1775116494000000000 -fps_benchmark,model_slug=luxonis/deeplab-v3-plus:512x288,benchmark_target=rvc4,run_id=manual-test-002,status=failed,testbed_name=testbed-a,device_ip=10.12.142.57 success=false,actual_fps=250.00,expected_fps=339.42,tolerance_low=0.10,tolerance_high=0.30,deviation_pct=-26.35 1775117400000000000 From 157906249c1db708e2ff28f28d78fb1619d7f49b Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 07:24:26 +0200 Subject: [PATCH 13/19] Use HIL camera metadata in benchmark tests --- tests/test_benchmark/conftest.py | 9 ++++ tests/test_benchmark/run_hil_tests.sh | 45 +++++++++++-------- .../test_benchmark_regression.py | 3 ++ 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 8d4484b..3d76b1b 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -42,6 +42,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help="Camera model for Influx metadata.", ) + parser.addoption( + "--camera-revision", + action="store", + default=None, + help="Camera revision for Influx metadata.", + ) parser.addoption( "--camera-agent-version", action="store", @@ -115,6 +121,9 @@ def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: "camera_model": _option_or_env( request, "--camera-model", "HIL_CAMERA_MODEL" ), + "camera_revision": _option_or_env( + request, "--camera-revision", "HIL_CAMERA_REVISION" + ), "camera_agent_version": _option_or_env( request, "--camera-agent-version", "HIL_CAMERA_AGENT_VERSION" ), diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 28f97b5..df4cfd7 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -38,43 +38,45 @@ pip install --upgrade \ --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local \ "depthai==${DEPTHAI_VERSION}" -# Cache device metadata once for the whole run using oakctl. If oakctl is not -# available, keep the benchmark runnable and record explicit placeholder values -# for the missing oakctl-derived metadata. -oakctl_output=$(oakctl list --format json 2>/dev/null || printf '') +# Cache device metadata once for the whole run using the HIL camera CLI. If the +# lookup fails, keep the benchmark runnable and record explicit placeholder +# values for the missing camera-derived metadata. +camera_output=$( + camera -t "${HIL_TESTBED_NAME}" -n test all info -j 2>/dev/null || printf '' +) -if [ -z "$oakctl_output" ]; then - echo "Warning: best-effort metadata lookup via oakctl failed; using placeholder metadata for this run." >&2 +if [ -z "$camera_output" ]; then + echo "Warning: best-effort metadata lookup via camera info failed; using placeholder metadata for this run." >&2 fi device_hostname=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].ip_addresses[0] // empty' 2>/dev/null \ + printf '%s' "$camera_output" \ + | jq -r '.[0].hostname // empty' 2>/dev/null \ | head -n1 ) camera_mxid=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].serial_number // empty' 2>/dev/null \ + printf '%s' "$camera_output" \ + | jq -r '.[0].mxid // empty' 2>/dev/null \ | head -n1 ) camera_model=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].model // empty' 2>/dev/null \ + printf '%s' "$camera_output" \ + | jq -r '.[0].model // empty' 2>/dev/null \ | head -n1 ) -camera_agent_version=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].agent_version // empty' 2>/dev/null \ +camera_revision=$( + printf '%s' "$camera_output" \ + | jq -r '.[0].revision // empty' 2>/dev/null \ | head -n1 ) camera_os=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].os // empty' 2>/dev/null \ + printf '%s' "$camera_output" \ + | jq -r '.[0].os_version // empty' 2>/dev/null \ | head -n1 ) detected_testbed_name=$( - printf '%s' "$oakctl_output" \ - | jq -r '.items[0].name // .items[0].hostname // .items[0].id // empty' 2>/dev/null \ + printf '%s' "$camera_output" \ + | jq -r '.[0].name // empty' 2>/dev/null \ | head -n1 ) runner_hostname=$(hostname 2>/dev/null || printf 'unknown') @@ -92,6 +94,9 @@ fi if [ -z "$camera_agent_version" ]; then camera_agent_version="$COULD_NOT_OBTAIN" fi +if [ -z "$camera_revision" ]; then + camera_revision="$COULD_NOT_OBTAIN" +fi if [ -z "$camera_os" ]; then camera_os="$COULD_NOT_OBTAIN" fi @@ -120,6 +125,7 @@ pytest_args=( --camera-mxid "$camera_mxid" --camera-os-version "$camera_os" --camera-model "$camera_model" + --camera-revision "$camera_revision" --camera-agent-version "$camera_agent_version" --runner "$runner_hostname" --server-os "$server_os" @@ -141,6 +147,7 @@ echo " device_ip=${device_hostname:-}" echo " camera_mxid=${camera_mxid:-}" echo " camera_os_version=${camera_os:-}" echo " camera_model=${camera_model:-}" +echo " camera_revision=${camera_revision:-}" echo " camera_agent_version=${camera_agent_version:-}" echo " runner=${runner_hostname:-}" echo " server_os=${server_os:-}" diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 8544c7b..20ec8b2 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -85,6 +85,9 @@ def _write_fps_result_to_influx( influx_metadata.get("camera_os_version") ), "camera_model": _normalize_tag(influx_metadata.get("camera_model")), + "camera_revision": _normalize_tag( + influx_metadata.get("camera_revision") + ), "camera_agent_version": _normalize_tag( influx_metadata.get("camera_agent_version") ), From be2ccf9d47bdf2717e4510d1a4eda768042c08ce Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 07:25:37 +0200 Subject: [PATCH 14/19] Drop obsolete camera agent metadata --- tests/test_benchmark/conftest.py | 9 --------- tests/test_benchmark/run_hil_tests.sh | 5 ----- tests/test_benchmark/test_benchmark_regression.py | 3 --- 3 files changed, 17 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 3d76b1b..5dd8b46 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -48,12 +48,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help="Camera revision for Influx metadata.", ) - parser.addoption( - "--camera-agent-version", - action="store", - default=None, - help="Camera agent version for Influx metadata.", - ) parser.addoption( "--runner", action="store", @@ -124,9 +118,6 @@ def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: "camera_revision": _option_or_env( request, "--camera-revision", "HIL_CAMERA_REVISION" ), - "camera_agent_version": _option_or_env( - request, "--camera-agent-version", "HIL_CAMERA_AGENT_VERSION" - ), "runner": _option_or_env(request, "--runner", "HIL_RUNNER") or os.environ.get("GITHUB_RUNNER_NAME") or os.environ.get("HOSTNAME") diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index df4cfd7..1b10a63 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -91,9 +91,6 @@ fi if [ -z "$camera_model" ]; then camera_model="$COULD_NOT_OBTAIN" fi -if [ -z "$camera_agent_version" ]; then - camera_agent_version="$COULD_NOT_OBTAIN" -fi if [ -z "$camera_revision" ]; then camera_revision="$COULD_NOT_OBTAIN" fi @@ -126,7 +123,6 @@ pytest_args=( --camera-os-version "$camera_os" --camera-model "$camera_model" --camera-revision "$camera_revision" - --camera-agent-version "$camera_agent_version" --runner "$runner_hostname" --server-os "$server_os" --depthai-version "$DEPTHAI_VERSION" @@ -148,7 +144,6 @@ echo " camera_mxid=${camera_mxid:-}" echo " camera_os_version=${camera_os:-}" echo " camera_model=${camera_model:-}" echo " camera_revision=${camera_revision:-}" -echo " camera_agent_version=${camera_agent_version:-}" echo " runner=${runner_hostname:-}" echo " server_os=${server_os:-}" printf ' pytest_args:' diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index 20ec8b2..914e9f0 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -88,9 +88,6 @@ def _write_fps_result_to_influx( "camera_revision": _normalize_tag( influx_metadata.get("camera_revision") ), - "camera_agent_version": _normalize_tag( - influx_metadata.get("camera_agent_version") - ), "runner": _normalize_tag(influx_metadata.get("runner")), "server_os": _normalize_tag(influx_metadata.get("server_os")), "depthai_version": _normalize_tag( From 63fdb794f48dddf83722ed5511a8ee6a8fea6769 Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 07:31:59 +0200 Subject: [PATCH 15/19] Use HIL_TESTBED directly in benchmark script --- tests/test_benchmark/conftest.py | 2 +- tests/test_benchmark/run_hil_tests.sh | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 5dd8b46..a1be4cf 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -104,7 +104,7 @@ def _option_or_env( def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: return { "testbed_name": _option_or_env( - request, "--testbed-name", "HIL_TESTBED_NAME" + request, "--testbed-name", "HIL_TESTBED" ), "camera_mxid": _option_or_env( request, "--camera-mxid", "HIL_CAMERA_MXID" diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 1b10a63..2d1c378 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -6,7 +6,7 @@ COULD_NOT_OBTAIN="could_not_obtain" # Check if required arguments were provided if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then - echo "Usage: $0 [TESTBED_NAME]" + echo "Usage: $0 " exit 1 fi @@ -14,7 +14,6 @@ fi export HUBAI_API_KEY="$1" export PAT_TOKEN="$2" export DEPTHAI_VERSION="$3" -export HIL_TESTBED_NAME="${4:-}" # Navigate to project directory cd /tmp/modelconverter @@ -42,7 +41,7 @@ pip install --upgrade \ # lookup fails, keep the benchmark runnable and record explicit placeholder # values for the missing camera-derived metadata. camera_output=$( - camera -t "${HIL_TESTBED_NAME}" -n test all info -j 2>/dev/null || printf '' + camera -t "${HIL_TESTBED}" -n test all info -j 2>/dev/null || printf '' ) if [ -z "$camera_output" ]; then @@ -103,14 +102,14 @@ fi if [ -z "$server_os" ]; then server_os="unknown" fi -if [ -z "$HIL_TESTBED_NAME" ]; then - HIL_TESTBED_NAME="${detected_testbed_name:-}" +if [ -z "$HIL_TESTBED" ]; then + HIL_TESTBED="${detected_testbed_name:-}" fi -if [ -z "$HIL_TESTBED_NAME" ]; then - HIL_TESTBED_NAME="$(hostname 2>/dev/null || printf '')" +if [ -z "$HIL_TESTBED" ]; then + HIL_TESTBED="$(hostname 2>/dev/null || printf '')" fi -if [ -z "$HIL_TESTBED_NAME" ]; then - HIL_TESTBED_NAME="$COULD_NOT_OBTAIN" +if [ -z "$HIL_TESTBED" ]; then + HIL_TESTBED="$COULD_NOT_OBTAIN" fi # Run tests @@ -118,7 +117,7 @@ pytest_args=( -s -v tests/test_benchmark/ - --testbed-name "$HIL_TESTBED_NAME" + --testbed-name "$HIL_TESTBED" --camera-mxid "$camera_mxid" --camera-os-version "$camera_os" --camera-model "$camera_model" @@ -138,7 +137,7 @@ echo " INFLUX_ORG=${INFLUX_ORG:-}" echo " INFLUX_BUCKET=${INFLUX_BUCKET:-}" echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" -echo " HIL_TESTBED_NAME=${HIL_TESTBED_NAME:-}" +echo " HIL_TESTBED=${HIL_TESTBED:-}" echo " device_ip=${device_hostname:-}" echo " camera_mxid=${camera_mxid:-}" echo " camera_os_version=${camera_os:-}" From a962d3b3b16ec25348bc2da4fb6ea65231264df7 Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 07:35:19 +0200 Subject: [PATCH 16/19] Fail benchmark run on missing camera metadata --- tests/test_benchmark/run_hil_tests.sh | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 2d1c378..43e7aa1 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -2,8 +2,6 @@ set -e # Exit immediately if a command fails -COULD_NOT_OBTAIN="could_not_obtain" - # Check if required arguments were provided if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then echo "Usage: $0 " @@ -45,7 +43,8 @@ camera_output=$( ) if [ -z "$camera_output" ]; then - echo "Warning: best-effort metadata lookup via camera info failed; using placeholder metadata for this run." >&2 + echo "Error: failed to obtain camera metadata via camera info." >&2 + exit 1 fi device_hostname=$( @@ -78,24 +77,31 @@ detected_testbed_name=$( | jq -r '.[0].name // empty' 2>/dev/null \ | head -n1 ) -runner_hostname=$(hostname 2>/dev/null || printf 'unknown') -server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') +missing_metadata=() if [ -z "$device_hostname" ]; then - device_hostname="$COULD_NOT_OBTAIN" + missing_metadata+=("hostname") fi if [ -z "$camera_mxid" ]; then - camera_mxid="$COULD_NOT_OBTAIN" + missing_metadata+=("mxid") fi if [ -z "$camera_model" ]; then - camera_model="$COULD_NOT_OBTAIN" + missing_metadata+=("model") fi if [ -z "$camera_revision" ]; then - camera_revision="$COULD_NOT_OBTAIN" + missing_metadata+=("revision") fi if [ -z "$camera_os" ]; then - camera_os="$COULD_NOT_OBTAIN" + missing_metadata+=("os_version") +fi + +if [ "${#missing_metadata[@]}" -ne 0 ]; then + echo "Error: camera metadata is incomplete; missing fields: ${missing_metadata[*]}" >&2 + exit 1 fi + +runner_hostname=$(hostname 2>/dev/null || printf 'unknown') +server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') if [ -z "$runner_hostname" ]; then runner_hostname="unknown" fi @@ -108,10 +114,6 @@ fi if [ -z "$HIL_TESTBED" ]; then HIL_TESTBED="$(hostname 2>/dev/null || printf '')" fi -if [ -z "$HIL_TESTBED" ]; then - HIL_TESTBED="$COULD_NOT_OBTAIN" -fi - # Run tests pytest_args=( -s @@ -127,9 +129,7 @@ pytest_args=( --depthai-version "$DEPTHAI_VERSION" ) -if [ "$device_hostname" != "$COULD_NOT_OBTAIN" ]; then - pytest_args+=(--device-ip "$device_hostname") -fi +pytest_args+=(--device-ip "$device_hostname") echo "Influx metadata debug:" echo " INFLUX_HOST=${INFLUX_HOST:-}" From 60a4f54be0039543d7748a01cae5abfb6df616cd Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 09:23:13 +0200 Subject: [PATCH 17/19] Use env metadata in benchmark tests --- tests/test_benchmark/conftest.py | 99 ++++----------------------- tests/test_benchmark/run_hil_tests.sh | 21 +++--- 2 files changed, 25 insertions(+), 95 deletions(-) diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index a1be4cf..c7a509a 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -1,6 +1,4 @@ import os -from datetime import datetime, timezone -from uuid import uuid4 import pytest @@ -18,60 +16,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: default="rvc4", help="Target platform to benchmark (default: rvc4).", ) - parser.addoption( - "--testbed-name", - action="store", - default=None, - help="Logical HIL testbed name for Influx metadata.", - ) - parser.addoption( - "--camera-mxid", - action="store", - default=None, - help="Camera MXID for Influx metadata.", - ) - parser.addoption( - "--camera-os-version", - action="store", - default=None, - help="Camera OS version for Influx metadata.", - ) - parser.addoption( - "--camera-model", - action="store", - default=None, - help="Camera model for Influx metadata.", - ) - parser.addoption( - "--camera-revision", - action="store", - default=None, - help="Camera revision for Influx metadata.", - ) - parser.addoption( - "--runner", - action="store", - default=None, - help="Runner name for Influx metadata.", - ) - parser.addoption( - "--server-os", - action="store", - default=None, - help="Server OS for Influx metadata.", - ) - parser.addoption( - "--depthai-version", - action="store", - default=None, - help="DepthAI version used for the benchmark run.", - ) - parser.addoption( - "--run-id", - action="store", - default=None, - help="Optional run identifier for grouping benchmark results in Influx.", - ) def pytest_configure(config: pytest.Config) -> None: @@ -92,50 +36,31 @@ def benchmark_target(request: pytest.FixtureRequest) -> str: return request.config.getoption("--benchmark-target") -def _option_or_env( - request: pytest.FixtureRequest, - option_name: str, - env_name: str, -) -> str | None: - return request.config.getoption(option_name) or os.environ.get(env_name) - - @pytest.fixture(scope="session") def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: return { - "testbed_name": _option_or_env( - request, "--testbed-name", "HIL_TESTBED" - ), - "camera_mxid": _option_or_env( - request, "--camera-mxid", "HIL_CAMERA_MXID" - ), - "camera_os_version": _option_or_env( - request, "--camera-os-version", "HIL_CAMERA_OS_VERSION" - ), - "camera_model": _option_or_env( - request, "--camera-model", "HIL_CAMERA_MODEL" - ), - "camera_revision": _option_or_env( - request, "--camera-revision", "HIL_CAMERA_REVISION" - ), - "runner": _option_or_env(request, "--runner", "HIL_RUNNER") + "testbed_name": os.environ.get("HIL_TESTBED"), + "camera_mxid": os.environ.get("HIL_CAMERA_MXID"), + "camera_os_version": os.environ.get("HIL_CAMERA_OS_VERSION"), + "camera_model": os.environ.get("HIL_CAMERA_MODEL"), + "camera_revision": os.environ.get("HIL_CAMERA_REVISION"), + "runner": os.environ.get("HIL_RUNNER") or os.environ.get("GITHUB_RUNNER_NAME") or os.environ.get("HOSTNAME") or os.environ.get("USER"), - "server_os": _option_or_env(request, "--server-os", "HIL_SERVER_OS"), - "depthai_version": _option_or_env( - request, "--depthai-version", "DEPTHAI_VERSION" - ), + "server_os": os.environ.get("HIL_SERVER_OS"), + "depthai_version": os.environ.get("DEPTHAI_VERSION"), } @pytest.fixture(scope="session") def benchmark_run_id(request: pytest.FixtureRequest) -> str: - configured_run_id = ( - request.config.getoption("--run-id") or os.environ.get("HIL_RUN_ID") - ) + configured_run_id = os.environ.get("HIL_RUN_ID") if configured_run_id: return configured_run_id + from datetime import datetime, timezone + from uuid import uuid4 + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") return f"benchmark-{timestamp}-{uuid4().hex[:8]}" diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index 43e7aa1..ec2f065 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -114,19 +114,19 @@ fi if [ -z "$HIL_TESTBED" ]; then HIL_TESTBED="$(hostname 2>/dev/null || printf '')" fi + +export HIL_TESTBED +export HIL_CAMERA_MXID="$camera_mxid" +export HIL_CAMERA_OS_VERSION="$camera_os" +export HIL_CAMERA_MODEL="$camera_model" +export HIL_CAMERA_REVISION="$camera_revision" +export HIL_SERVER_OS="$server_os" + # Run tests pytest_args=( -s -v tests/test_benchmark/ - --testbed-name "$HIL_TESTBED" - --camera-mxid "$camera_mxid" - --camera-os-version "$camera_os" - --camera-model "$camera_model" - --camera-revision "$camera_revision" - --runner "$runner_hostname" - --server-os "$server_os" - --depthai-version "$DEPTHAI_VERSION" ) pytest_args+=(--device-ip "$device_hostname") @@ -138,6 +138,11 @@ echo " INFLUX_BUCKET=${INFLUX_BUCKET:-}" echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" echo " HIL_TESTBED=${HIL_TESTBED:-}" +echo " HIL_CAMERA_MXID=${HIL_CAMERA_MXID:-}" +echo " HIL_CAMERA_OS_VERSION=${HIL_CAMERA_OS_VERSION:-}" +echo " HIL_CAMERA_MODEL=${HIL_CAMERA_MODEL:-}" +echo " HIL_CAMERA_REVISION=${HIL_CAMERA_REVISION:-}" +echo " HIL_SERVER_OS=${HIL_SERVER_OS:-}" echo " device_ip=${device_hostname:-}" echo " camera_mxid=${camera_mxid:-}" echo " camera_os_version=${camera_os:-}" From ba2e9469644ebc584f92092d2db8d4a9a2f7e9c8 Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 11:33:02 +0200 Subject: [PATCH 18/19] Select rvc4 camera metadata --- tests/test_benchmark/run_hil_tests.sh | 35 ++++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index ec2f065..b9cf0bf 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -47,34 +47,45 @@ if [ -z "$camera_output" ]; then exit 1 fi -device_hostname=$( +rvc4_camera=$( printf '%s' "$camera_output" \ - | jq -r '.[0].hostname // empty' 2>/dev/null \ + | jq -r '.[] | select(.platform == "rvc4") | @json' 2>/dev/null \ + | head -n1 +) + +if [ -z "$rvc4_camera" ]; then + echo "Error: no rvc4 camera found in camera metadata." >&2 + exit 1 +fi + +device_hostname=$( + printf '%s' "$rvc4_camera" \ + | jq -r '.hostname // empty' 2>/dev/null \ | head -n1 ) camera_mxid=$( - printf '%s' "$camera_output" \ - | jq -r '.[0].mxid // empty' 2>/dev/null \ + printf '%s' "$rvc4_camera" \ + | jq -r '.mxid // empty' 2>/dev/null \ | head -n1 ) camera_model=$( - printf '%s' "$camera_output" \ - | jq -r '.[0].model // empty' 2>/dev/null \ + printf '%s' "$rvc4_camera" \ + | jq -r '.model // empty' 2>/dev/null \ | head -n1 ) camera_revision=$( - printf '%s' "$camera_output" \ - | jq -r '.[0].revision // empty' 2>/dev/null \ + printf '%s' "$rvc4_camera" \ + | jq -r '.revision // empty' 2>/dev/null \ | head -n1 ) camera_os=$( - printf '%s' "$camera_output" \ - | jq -r '.[0].os_version // empty' 2>/dev/null \ + printf '%s' "$rvc4_camera" \ + | jq -r '.os_version // empty' 2>/dev/null \ | head -n1 ) detected_testbed_name=$( - printf '%s' "$camera_output" \ - | jq -r '.[0].name // empty' 2>/dev/null \ + printf '%s' "$rvc4_camera" \ + | jq -r '.name // empty' 2>/dev/null \ | head -n1 ) From c24c3ec67684faac7e5cd7b23e4ee6d2332ce9c2 Mon Sep 17 00:00:00 2001 From: tjb Date: Mon, 13 Apr 2026 12:11:17 +0200 Subject: [PATCH 19/19] Restore RVC4 workflow from main --- .github/workflows/rvc4_test.yaml | 118 +++++++++---------------------- 1 file changed, 35 insertions(+), 83 deletions(-) diff --git a/.github/workflows/rvc4_test.yaml b/.github/workflows/rvc4_test.yaml index 570881c..0067be2 100644 --- a/.github/workflows/rvc4_test.yaml +++ b/.github/workflows/rvc4_test.yaml @@ -1,88 +1,40 @@ -name: HIL FPS Benchmark +name: Test - RVC4 on: workflow_dispatch: - inputs: - depthai_version: - description: "DepthAI version to test against (e.g. 3.3.0)" - required: true - type: string - os_version: - description: "Optional camera OS version to install before the benchmark" - required: false - type: string - hold_reservation: - description: "Hold testbed reservation after the run" - required: false - type: boolean - -env: - DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} - OS_VERSION: ${{ github.event.inputs.os_version }} - HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} - HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} - HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} - INFLUX_HOST: ${{ secrets.INFLUX_HOST }} - INFLUX_ORG: ${{ secrets.INFLUX_ORG }} - INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }} - INFLUX_BUCKET: fps_metrics + pull_request: + branches: [main] + paths: + - requirements.txt + - "modelconverter/__init__.py" + - "modelconverter/packages/rvc4/**" + - "modelconverter/packages/base_exporter.py" + - "modelconverter/packages/base_inferer.py" + - "tests/test_packages/test_rvc4.py" + - "docker/rvc4/Dockerfile" + - "docker/rvc4/entrypoint.sh" + - ".github/workflows/modelconverter_test.yaml" + - ".github/workflows/rvc4_test.yaml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - fps-benchmark: - runs-on: ["self-hosted", "testbed-runner"] - - steps: - - uses: actions/checkout@v4 - - - name: Run benchmark - run: | - set -euo pipefail - - pip install hil-framework --upgrade \ - --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" - - export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark" - RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" - - HOLD_RESERVATION_OPTION="" - if [[ "$HOLD_RESERVATION" == "true" ]]; then - HOLD_RESERVATION_OPTION="--hold-reservation" - fi - - OS_VERSION_OPTION="" - if [[ -n "$OS_VERSION" ]]; then - OS_VERSION_OPTION="--os-version $OS_VERSION" - fi - - if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then - DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION" - else - echo "Invalid depthai version: $DEPTHAI_VERSION" >&2 - exit 1 - fi - - : "${INFLUX_HOST:?INFLUX_HOST is required on the runner}" - : "${INFLUX_ORG:?INFLUX_ORG is required on the runner}" - : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}" - - : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}" - - REMOTE_CMD="export HIL_RUN_ID=\"$GITHUB_RUN_ID\" \ - INFLUX_HOST=\"$INFLUX_HOST\" \ - INFLUX_ORG=\"$INFLUX_ORG\" \ - INFLUX_BUCKET=\"$INFLUX_BUCKET\" \ - INFLUX_TOKEN=\"$INFLUX_TOKEN\" && \ - cd /tmp/modelconverter && \ - ./tests/test_benchmark/run_hil_tests.sh \ - \"$HUBAI_API_KEY\" \ - \"$HIL_FRAMEWORK_TOKEN\" \ - \"$DEPTHAI_VERSION_CHECKED\"" - - exec hil_runner \ - --models "oak4_pro or oak4_d or oak4_s" \ - $HOLD_RESERVATION_OPTION \ - --wait \ - $OS_VERSION_OPTION \ - $RESERVATION_OPTION \ - --sync-workspace --rsync-args="--exclude=venv" \ - --commands "$REMOTE_CMD" + rvc4-test: + strategy: + fail-fast: false + matrix: + version: + - "2.23.0" + - "2.24.0" + - "2.25.0" + - "2.26.2" + - "2.27.0" + - "2.32.6" + + uses: ./.github/workflows/modelconverter_test.yaml + secrets: inherit + with: + package: rvc4 + version: ${{ matrix.version }}