Add FPS benchmark metric to influx

.github/workflows/fps_benchmark_influx.yaml

@@ -0,0 +1,85 @@
+name: HIL FPS Benchmark Influx
+
+on:
+  workflow_dispatch:
+    inputs:
+      depthai_version:
+        description: "DepthAI version to test against (e.g. 3.3.0)"
+        required: true
+        type: string
+      os_version:
+        description: "Optional camera OS version to install before the benchmark"
+        required: false
+        type: string
+      hold_reservation:
+        description: "Hold testbed reservation after the run"
+        required: false
+        type: boolean
+
+env:
+  DEPTHAI_VERSION: ${{ github.event.inputs.depthai_version }}
+  OS_VERSION: ${{ github.event.inputs.os_version }}
+  HIL_FRAMEWORK_TOKEN: ${{ secrets.HIL_FRAMEWORK_TOKEN }}
+  HUBAI_API_KEY: ${{ secrets.HUBAI_API_KEY }}
+  HOLD_RESERVATION: ${{ github.event.inputs.hold_reservation }}
+  INFLUX_BUCKET: fps_metrics
+
+jobs:
+  fps-benchmark:
+    runs-on: ["self-hosted", "testbed-runner"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run benchmark and push to Influx
+        run: |
+          set -euo pipefail
+
+          pip install hil-framework --upgrade \
+            --index-url "https://__token__:$HIL_FRAMEWORK_TOKEN@gitlab.luxonis.com/api/v4/projects/213/packages/pypi/simple"
+
+          export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#fps-benchmark-influx"
+          RESERVATION_OPTION="--reservation-name $RESERVATION_NAME"
+
+          HOLD_RESERVATION_OPTION=""
+          if [[ "$HOLD_RESERVATION" == "true" ]]; then
+            HOLD_RESERVATION_OPTION="--hold-reservation"
+          fi
+
+          OS_VERSION_OPTION=""
+          if [[ -n "$OS_VERSION" ]]; then
+            OS_VERSION_OPTION="--os-version $OS_VERSION"
+          fi
+
+          if [[ "$DEPTHAI_VERSION" =~ ^3\.[0-9]{1,2}\.[0-9]{1,2}$ ]]; then
+            DEPTHAI_VERSION_CHECKED="$DEPTHAI_VERSION"
+          else
+            echo "Invalid depthai version: $DEPTHAI_VERSION" >&2
+            exit 1
+          fi
+
+          : "${INFLUX_HOST:?INFLUX_HOST is required on the runner}"
+          : "${INFLUX_ORG:?INFLUX_ORG is required on the runner}"
+          : "${INFLUX_TOKEN:?INFLUX_TOKEN is required on the runner}"
+
+          : "${INFLUX_BUCKET:?INFLUX_BUCKET must be set by the workflow}"
+
+          REMOTE_CMD="export HIL_RUN_ID=\"$GITHUB_RUN_ID\" \
+            INFLUX_HOST=\"$INFLUX_HOST\" \
+            INFLUX_ORG=\"$INFLUX_ORG\" \
+            INFLUX_BUCKET=\"$INFLUX_BUCKET\" \
+            INFLUX_TOKEN=\"$INFLUX_TOKEN\" && \
+            cd /tmp/modelconverter && \
+            ./tests/test_benchmark/run_hil_tests.sh \
+              \"$HUBAI_API_KEY\" \
+              \"$HIL_FRAMEWORK_TOKEN\" \
+              \"$DEPTHAI_VERSION_CHECKED\""
+
+          exec hil_runner \
+            --models "oak4_pro or oak4_d or oak4_s" \
+            $HOLD_RESERVATION_OPTION \
+            --wait \
+            $OS_VERSION_OPTION \
+            $RESERVATION_OPTION \
+            --sync-workspace --rsync-args="--exclude=venv" \
+            --commands "$REMOTE_CMD"

tests/test_benchmark/conftest.py

@@ -34,3 +34,33 @@ def device_ip(request: pytest.FixtureRequest) -> str | None:
 @pytest.fixture
 def benchmark_target(request: pytest.FixtureRequest) -> str:
     return request.config.getoption("--benchmark-target")
+
+
+@pytest.fixture(scope="session")
+def influx_metadata(request: pytest.FixtureRequest) -> dict[str, str | None]:
+    return {
+        "testbed_name": os.environ.get("HIL_TESTBED"),
+        "camera_mxid": os.environ.get("HIL_CAMERA_MXID"),
+        "camera_os_version": os.environ.get("HIL_CAMERA_OS_VERSION"),
+        "camera_model": os.environ.get("HIL_CAMERA_MODEL"),
+        "camera_revision": os.environ.get("HIL_CAMERA_REVISION"),
+        "runner": os.environ.get("HIL_RUNNER")
+        or os.environ.get("GITHUB_RUNNER_NAME")
+        or os.environ.get("HOSTNAME")
+        or os.environ.get("USER"),
+        "server_os": os.environ.get("HIL_SERVER_OS"),
+        "depthai_version": os.environ.get("DEPTHAI_VERSION"),
+    }
+
+
+@pytest.fixture(scope="session")
+def benchmark_run_id(request: pytest.FixtureRequest) -> str:
+    configured_run_id = os.environ.get("HIL_RUN_ID")
+    if configured_run_id:
+        return configured_run_id
+
+    from datetime import datetime, timezone
+    from uuid import uuid4
+
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    return f"benchmark-{timestamp}-{uuid4().hex[:8]}"

tests/test_benchmark/run_hil_tests.sh

@@ -3,7 +3,7 @@
 set -e  # Exit immediately if a command fails
 
 # Check if required arguments were provided
-if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+if [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ]; then
   echo "Usage: $0 <HUBAI_API_KEY> <PAT_TOKEN> <DAI_VERSION>"
   exit 1
 fi
@@ -35,10 +35,134 @@ pip install --upgrade \
   --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-release-local \
   "depthai==${DEPTHAI_VERSION}"
 
-# Extract hostname of first rvc4 device
-hostname=$(hil_camera -t "$HIL_TESTBED" -n test all info -j \
-  | jq -r '.[] | select(.platform=="rvc4") | .hostname' \
-  | head -n1)
+# Cache device metadata once for the whole run using the HIL camera CLI. If the
+# lookup fails, keep the benchmark runnable and record explicit placeholder
+# values for the missing camera-derived metadata.
+camera_output=$(
+  camera -t "${HIL_TESTBED}" -n test all info -j 2>/dev/null || printf ''
+)
+
+if [ -z "$camera_output" ]; then
+  echo "Error: failed to obtain camera metadata via camera info." >&2
+  exit 1
+fi
+
+rvc4_camera=$(
+  printf '%s' "$camera_output" \
+    | jq -r '.[] | select(.platform == "rvc4") | @json' 2>/dev/null \
+    | head -n1
+)
+
+if [ -z "$rvc4_camera" ]; then
+  echo "Error: no rvc4 camera found in camera metadata." >&2
+  exit 1
+fi
+
+device_hostname=$(
+  printf '%s' "$rvc4_camera" \
+    | jq -r '.hostname // empty' 2>/dev/null \
+    | head -n1
+)
+camera_mxid=$(
+  printf '%s' "$rvc4_camera" \
+    | jq -r '.mxid // empty' 2>/dev/null \
+    | head -n1
+)
+camera_model=$(
+  printf '%s' "$rvc4_camera" \
+    | jq -r '.model // empty' 2>/dev/null \
+    | head -n1
+)
+camera_revision=$(
+  printf '%s' "$rvc4_camera" \
+    | jq -r '.revision // empty' 2>/dev/null \
+    | head -n1
+)
+camera_os=$(
+  printf '%s' "$rvc4_camera" \
+    | jq -r '.os_version // empty' 2>/dev/null \
+    | head -n1
+)
+detected_testbed_name=$(
+  printf '%s' "$rvc4_camera" \
+    | jq -r '.name // empty' 2>/dev/null \
+    | head -n1
+)
+
+missing_metadata=()
+if [ -z "$device_hostname" ]; then
+  missing_metadata+=("hostname")
+fi
+if [ -z "$camera_mxid" ]; then
+  missing_metadata+=("mxid")
+fi
+if [ -z "$camera_model" ]; then
+  missing_metadata+=("model")
+fi
+if [ -z "$camera_revision" ]; then
+  missing_metadata+=("revision")
+fi
+if [ -z "$camera_os" ]; then
+  missing_metadata+=("os_version")
+fi
+
+if [ "${#missing_metadata[@]}" -ne 0 ]; then
+  echo "Error: camera metadata is incomplete; missing fields: ${missing_metadata[*]}" >&2
+  exit 1
+fi
+
+runner_hostname=$(hostname 2>/dev/null || printf 'unknown')
+server_os=$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]' || printf 'unknown')
+if [ -z "$runner_hostname" ]; then
+  runner_hostname="unknown"
+fi
+if [ -z "$server_os" ]; then
+  server_os="unknown"
+fi
+if [ -z "$HIL_TESTBED" ]; then
+  HIL_TESTBED="${detected_testbed_name:-}"
+fi
+if [ -z "$HIL_TESTBED" ]; then
+  HIL_TESTBED="$(hostname 2>/dev/null || printf '')"
+fi
+
+export HIL_TESTBED
+export HIL_CAMERA_MXID="$camera_mxid"
+export HIL_CAMERA_OS_VERSION="$camera_os"
+export HIL_CAMERA_MODEL="$camera_model"
+export HIL_CAMERA_REVISION="$camera_revision"
+export HIL_SERVER_OS="$server_os"
 
 # Run tests
-pytest -s -v tests/test_benchmark/ --device-ip "$hostname"
+pytest_args=(
+  -s
+  -v
+  tests/test_benchmark/
+)
+
+pytest_args+=(--device-ip "$device_hostname")
+
+echo "Influx metadata debug:"
+echo "  INFLUX_HOST=${INFLUX_HOST:-<unset>}"
+echo "  INFLUX_ORG=${INFLUX_ORG:-<unset>}"
+echo "  INFLUX_BUCKET=${INFLUX_BUCKET:-<unset>}"
+echo "  INFLUX_TOKEN=$(if [ -n "${INFLUX_TOKEN:-}" ]; then printf '<set>'; else printf '<unset>'; fi)"
+echo "  DEPTHAI_VERSION=${DEPTHAI_VERSION:-<empty>}"
+echo "  HIL_TESTBED=${HIL_TESTBED:-<empty>}"
+echo "  HIL_CAMERA_MXID=${HIL_CAMERA_MXID:-<empty>}"
+echo "  HIL_CAMERA_OS_VERSION=${HIL_CAMERA_OS_VERSION:-<empty>}"
+echo "  HIL_CAMERA_MODEL=${HIL_CAMERA_MODEL:-<empty>}"
+echo "  HIL_CAMERA_REVISION=${HIL_CAMERA_REVISION:-<empty>}"
+echo "  HIL_SERVER_OS=${HIL_SERVER_OS:-<empty>}"
+echo "  device_ip=${device_hostname:-<empty>}"
+echo "  camera_mxid=${camera_mxid:-<empty>}"
+echo "  camera_os_version=${camera_os:-<empty>}"
+echo "  camera_model=${camera_model:-<empty>}"
+echo "  camera_revision=${camera_revision:-<empty>}"
+echo "  runner=${runner_hostname:-<empty>}"
+echo "  server_os=${server_os:-<empty>}"
+printf '  pytest_args:'
+printf ' %q' "${pytest_args[@]}"
+printf '\n'
+
+pytest "${pytest_args[@]}"