diff --git a/.github/workflows/fps_benchmark_influx.yaml b/.github/workflows/fps_benchmark_influx.yaml new file mode 100644 index 0000000..46b9bc8 --- /dev/null +++ b/.github/workflows/fps_benchmark_influx.yaml @@ -0,0 +1,85 @@ +name: HIL FPS Benchmark Influx + +on: + workflow_dispatch: + inputs: + depthai_version: + description: "DepthAI version to test against (e.g. 3.3.0)" + required: true + type: string + os_version: + description: "Optional camera OS version to install before the benchmark" + required: false + type: string + hold_reservation: + description: "Hold testbed reservation after the run" + required: false + type: boolean + +env: + DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }} + OS_VERSION: ${{ github.event.inputs.os_version }} + HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }} + HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }} + HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }} + INFLUX_BUCKET: fps_metrics + +jobs: + fps-benchmark: + runs-on: ["self-hosted", "testbed-runner"] + + steps: + - uses: actions/checkout@v4 + + - name: Run benchmark and push to Influx + run: | + set -euo pipefail + + pip install hil-framework --upgrade \ + --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple" + + export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark-influx" + RESERVATION_OPTION="--reservation-name $RESERVATION_NAME" + + HOLD_RESERVATION_OPTION="" + if [[ "$HOLD_RESERVATION" == "true" ]]; then + HOLD_RESERVATION_OPTION="--hold-reservation" + fi + + OS_VERSION_OPTION="" + if [[ -n "$OS_VERSION" ]]; then + OS_VERSION_OPTION="--os-version $OS_VERSION" + fi + + if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then + DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION" + else + echo "Invalid depthai version: $DEPTHAI_VERSION" >&2 + exit 1 + fi + + : "${INFLUX_HOST:?INFLUX_HOST is required on the runner}" + : "${INFLUX_ORG:?INFLUX_ORG is required on the runner}" + : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}" + + : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}" + + REMOTE_CMD="export HIL_RUN_ID=\"$GITHUB_RUN_ID\" \ + INFLUX_HOST=\"$INFLUX_HOST\" \ + INFLUX_ORG=\"$INFLUX_ORG\" \ + INFLUX_BUCKET=\"$INFLUX_BUCKET\" \ + INFLUX_TOKEN=\"$INFLUX_TOKEN\" && \ + cd /tmp/modelconverter && \ + ./tests/test_benchmark/run_hil_tests.sh \ + \"$HUBAI_API_KEY\" \ + \"$HIL_FRAMEWORK_TOKEN\" \ + \"$DEPTHAI_VERSION_CHECKED\"" + + exec hil_runner \ + --models "oak4_pro or oak4_d or oak4_s" \ + $HOLD_RESERVATION_OPTION \ + --wait \ + $OS_VERSION_OPTION \ + $RESERVATION_OPTION \ + --sync-workspace --rsync-args="--exclude=venv" \ + --commands "$REMOTE_CMD" diff --git a/tests/test_benchmark/conftest.py b/tests/test_benchmark/conftest.py index 9d91a9c..c7a509a 100644 --- a/tests/test_benchmark/conftest.py +++ b/tests/test_benchmark/conftest.py @@ -34,3 +34,33 @@ def device_ip(request: pytest.FixtureRequest) -> str | None: @pytest.fixture def benchmark_target(request: pytest.FixtureRequest) -> str: return request.config.getoption("--benchmark-target") + + +@pytest.fixture(scope="session") +def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]: + return { + "testbed_name": os.environ.get("HIL_TESTBED"), + "camera_mxid": os.environ.get("HIL_CAMERA_MXID"), + "camera_os_version": os.environ.get("HIL_CAMERA_OS_VERSION"), + "camera_model": os.environ.get("HIL_CAMERA_MODEL"), + "camera_revision": os.environ.get("HIL_CAMERA_REVISION"), + "runner": os.environ.get("HIL_RUNNER") + or os.environ.get("GITHUB_RUNNER_NAME") + or os.environ.get("HOSTNAME") + or os.environ.get("USER"), + "server_os": os.environ.get("HIL_SERVER_OS"), + "depthai_version": os.environ.get("DEPTHAI_VERSION"), + } + + +@pytest.fixture(scope="session") +def benchmark_run_id(request: pytest.FixtureRequest) -> str: + configured_run_id = os.environ.get("HIL_RUN_ID") + if configured_run_id: + return configured_run_id + + from datetime import datetime, timezone + from uuid import uuid4 + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return f"benchmark-{timestamp}-{uuid4().hex[:8]}" diff --git a/tests/test_benchmark/run_hil_tests.sh b/tests/test_benchmark/run_hil_tests.sh index e0ae2f0..b9cf0bf 100755 --- a/tests/test_benchmark/run_hil_tests.sh +++ b/tests/test_benchmark/run_hil_tests.sh @@ -3,7 +3,7 @@ set -e # Exit immediately if a command fails # Check if required arguments were provided -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then +if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then echo "Usage: $0 " exit 1 fi @@ -35,10 +35,134 @@ pip install --upgrade \ --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local \ "depthai==${DEPTHAI_VERSION}" -# Extract hostname of first rvc4 device -hostname=$(hil_camera -t "$HIL_TESTBED" -n test all info -j \ - | jq -r '.[] | select(.platform=="rvc4") | .hostname' \ - | head -n1) +# Cache device metadata once for the whole run using the HIL camera CLI. If the +# lookup fails, keep the benchmark runnable and record explicit placeholder +# values for the missing camera-derived metadata. +camera_output=$( + camera -t "${HIL_TESTBED}" -n test all info -j 2>/dev/null || printf '' +) + +if [ -z "$camera_output" ]; then + echo "Error: failed to obtain camera metadata via camera info." >&2 + exit 1 +fi + +rvc4_camera=$( + printf '%s' "$camera_output" \ + | jq -r '.[] | select(.platform == "rvc4") | @json' 2>/dev/null \ + | head -n1 +) + +if [ -z "$rvc4_camera" ]; then + echo "Error: no rvc4 camera found in camera metadata." >&2 + exit 1 +fi + +device_hostname=$( + printf '%s' "$rvc4_camera" \ + | jq -r '.hostname // empty' 2>/dev/null \ + | head -n1 +) +camera_mxid=$( + printf '%s' "$rvc4_camera" \ + | jq -r '.mxid // empty' 2>/dev/null \ + | head -n1 +) +camera_model=$( + printf '%s' "$rvc4_camera" \ + | jq -r '.model // empty' 2>/dev/null \ + | head -n1 +) +camera_revision=$( + printf '%s' "$rvc4_camera" \ + | jq -r '.revision // empty' 2>/dev/null \ + | head -n1 +) +camera_os=$( + printf '%s' "$rvc4_camera" \ + | jq -r '.os_version // empty' 2>/dev/null \ + | head -n1 +) +detected_testbed_name=$( + printf '%s' "$rvc4_camera" \ + | jq -r '.name // empty' 2>/dev/null \ + | head -n1 +) + +missing_metadata=() +if [ -z "$device_hostname" ]; then + missing_metadata+=("hostname") +fi +if [ -z "$camera_mxid" ]; then + missing_metadata+=("mxid") +fi +if [ -z "$camera_model" ]; then + missing_metadata+=("model") +fi +if [ -z "$camera_revision" ]; then + missing_metadata+=("revision") +fi +if [ -z "$camera_os" ]; then + missing_metadata+=("os_version") +fi + +if [ "${#missing_metadata[@]}" -ne 0 ]; then + echo "Error: camera metadata is incomplete; missing fields: ${missing_metadata[*]}" >&2 + exit 1 +fi + +runner_hostname=$(hostname 2>/dev/null || printf 'unknown') +server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown') +if [ -z "$runner_hostname" ]; then + runner_hostname="unknown" +fi +if [ -z "$server_os" ]; then + server_os="unknown" +fi +if [ -z "$HIL_TESTBED" ]; then + HIL_TESTBED="${detected_testbed_name:-}" +fi +if [ -z "$HIL_TESTBED" ]; then + HIL_TESTBED="$(hostname 2>/dev/null || printf '')" +fi + +export HIL_TESTBED +export HIL_CAMERA_MXID="$camera_mxid" +export HIL_CAMERA_OS_VERSION="$camera_os" +export HIL_CAMERA_MODEL="$camera_model" +export HIL_CAMERA_REVISION="$camera_revision" +export HIL_SERVER_OS="$server_os" # Run tests -pytest -s -v tests/test_benchmark/ --device-ip "$hostname" \ No newline at end of file +pytest_args=( + -s + -v + tests/test_benchmark/ +) + +pytest_args+=(--device-ip "$device_hostname") + +echo "Influx metadata debug:" +echo " INFLUX_HOST=${INFLUX_HOST:-}" +echo " INFLUX_ORG=${INFLUX_ORG:-}" +echo " INFLUX_BUCKET=${INFLUX_BUCKET:-}" +echo " INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf ''; else printf ''; fi)" +echo " DEPTHAI_VERSION=${DEPTHAI_VERSION:-}" +echo " HIL_TESTBED=${HIL_TESTBED:-}" +echo " HIL_CAMERA_MXID=${HIL_CAMERA_MXID:-}" +echo " HIL_CAMERA_OS_VERSION=${HIL_CAMERA_OS_VERSION:-}" +echo " HIL_CAMERA_MODEL=${HIL_CAMERA_MODEL:-}" +echo " HIL_CAMERA_REVISION=${HIL_CAMERA_REVISION:-}" +echo " HIL_SERVER_OS=${HIL_SERVER_OS:-}" +echo " device_ip=${device_hostname:-}" +echo " camera_mxid=${camera_mxid:-}" +echo " camera_os_version=${camera_os:-}" +echo " camera_model=${camera_model:-}" +echo " camera_revision=${camera_revision:-}" +echo " runner=${runner_hostname:-}" +echo " server_os=${server_os:-}" +printf ' pytest_args:' +printf ' %q' "${pytest_args[@]}" +printf '\n' + +pytest "${pytest_args[@]}" diff --git a/tests/test_benchmark/test_benchmark_regression.py b/tests/test_benchmark/test_benchmark_regression.py index bcd91b6..914e9f0 100644 --- a/tests/test_benchmark/test_benchmark_regression.py +++ b/tests/test_benchmark/test_benchmark_regression.py @@ -1,5 +1,7 @@ import json +import os from pathlib import Path +from urllib import error, parse, request import pytest @@ -21,6 +23,124 @@ def _model_id(slug: str) -> str: return slug.rsplit("/", 1)[-1] +def _escape_tag(value: str) -> str: + return ( + value.replace("\\", "\\\\") + .replace(",", "\\,") + .replace(" ", "\\ ") + .replace("=", "\\=") + ) + + +def _normalize_tag(value: str | None) -> str: + if value in (None, ""): + return "unknown" + return str(value) + + +def _format_field(value: str | float | bool) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int) and not isinstance(value, bool): + return f"{value}i" + if isinstance(value, float): + return repr(value) + + escaped = str(value).replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def _write_fps_result_to_influx( + *, + model_slug: str, + benchmark_target: str, + benchmark_run_id: str, + device_ip: str | None, + actual_fps: float, + expected_fps: float, + tolerance_low: float, + tolerance_high: float, + fps_min: float, + fps_max: float, + deviation_pct: float, + success: bool, + influx_metadata: dict[str, str | None], +) -> None: + influx_url = os.environ.get("INFLUX_HOST") + influx_org = os.environ.get("INFLUX_ORG") + influx_bucket = os.environ.get("INFLUX_BUCKET") + influx_token = os.environ.get("INFLUX_TOKEN") + + if not all([influx_url, influx_org, influx_bucket, influx_token]): + return + + tags = { + "model_slug": model_slug, + "benchmark_target": benchmark_target, + "run_id": benchmark_run_id, + "status": "passed" if success else "failed", + "testbed_name": _normalize_tag(influx_metadata.get("testbed_name")), + "camera_mxid": _normalize_tag(influx_metadata.get("camera_mxid")), + "camera_os_version": _normalize_tag( + influx_metadata.get("camera_os_version") + ), + "camera_model": _normalize_tag(influx_metadata.get("camera_model")), + "camera_revision": _normalize_tag( + influx_metadata.get("camera_revision") + ), + "runner": _normalize_tag(influx_metadata.get("runner")), + "server_os": _normalize_tag(influx_metadata.get("server_os")), + "depthai_version": _normalize_tag( + influx_metadata.get("depthai_version") + ), + "device_ip": _normalize_tag(device_ip), + } + + fields = { + "actual_fps": actual_fps, + "expected_fps": expected_fps, + "tolerance_low": tolerance_low, + "tolerance_high": tolerance_high, + "fps_min": fps_min, + "fps_max": fps_max, + "deviation_pct": deviation_pct, + "success": success, + } + + tag_set = ",".join(f"{key}={_escape_tag(value)}" for key, value in tags.items()) + field_set = ",".join( + f"{key}={_format_field(value)}" for key, value in fields.items() + ) + line = f"fps_benchmark,{tag_set} {field_set}" + print(f"Influx write debug: {line}") + + write_url = ( + f"{influx_url.rstrip('/')}/api/v2/write?" + f"org={parse.quote(influx_org, safe='')}&" + f"bucket={parse.quote(influx_bucket, safe='')}&precision=ns" + ) + print( + "Influx request debug: " + f"url={write_url}, token={'' if influx_token else ''}" + ) + influx_request = request.Request( + write_url, + data=line.encode(), + headers={ + "Authorization": f"Token {influx_token}", + "Content-Type": "text/plain; charset=utf-8", + "Accept": "application/json", + }, + method="POST", + ) + + try: + with request.urlopen(influx_request, timeout=5): + return + except (error.URLError, TimeoutError, OSError) as exc: + print(f"Failed to write benchmark result to InfluxDB: {exc}") + + @pytest.mark.parametrize( "model_slug", _model_slugs("rvc4"), @@ -30,6 +150,8 @@ def test_benchmark_fps( model_slug: str, device_ip: str | None, benchmark_target: str, + benchmark_run_id: str, + influx_metadata: dict[str, str | None], ) -> None: model_config = _targets_data[benchmark_target][model_slug] expected_fps = model_config["expected_fps"] @@ -60,13 +182,30 @@ def test_benchmark_fps( fps_max = expected_fps * (1 + tolerance_high) deviation_pct = ((actual_fps - expected_fps) / expected_fps) * 100 + success = fps_min <= actual_fps <= fps_max print( f"Benchmark result for {model_slug}: " f"actual={actual_fps:.2f} FPS, expected={expected_fps:.2f} FPS. " ) - assert fps_min <= actual_fps <= fps_max, ( + _write_fps_result_to_influx( + model_slug=model_slug, + benchmark_target=benchmark_target, + benchmark_run_id=benchmark_run_id, + device_ip=device_ip, + actual_fps=actual_fps, + expected_fps=expected_fps, + tolerance_low=tolerance_low, + tolerance_high=tolerance_high, + fps_min=fps_min, + fps_max=fps_max, + deviation_pct=deviation_pct, + success=success, + influx_metadata=influx_metadata, + ) + + assert success, ( f"FPS regression for {model_slug}: " f"actual={actual_fps:.2f}, expected={expected_fps:.2f} " f"(deviation: {deviation_pct:+.1f}%, "