feat(producer): add Rio-style residual-RMS check to regression harness

packages/producer/src/regression-harness.ts

@@ -18,7 +18,11 @@ import { createRenderJob, executeRenderJob } from "./services/renderOrchestrator
 import { compileForRender } from "./services/htmlCompiler.js";
 import { validateCompilation } from "./services/compilationTester.js";
 import { extractMediaMetadata } from "./utils/ffprobe.js";
-import { buildRmsEnvelope, compareAudioEnvelopes } from "./utils/audioRegression.js";
+import {
+  buildRmsEnvelope,
+  compareAudioEnvelopes,
+  computeAudioResidualRmsDb,
+} from "./utils/audioRegression.js";
 import { parseFps, fpsToNumber } from "@hyperframes/core";
 import {
   checkDistributedSupport,
@@ -38,6 +42,15 @@ type TestMetadata = {
   maxFrameFailures: number;
   minAudioCorrelation: number;
   maxAudioLagWindows: number;
+  /**
+   * Optional Rio-style residual-RMS check. Subtracts the rendered audio
+   * from the baseline and reads the residual Overall RMS via `astats`.
+   * A value of `-50` (Rio's convention) treats residuals at-or-below
+   * -50 dBFS as effectively-silent — i.e. the streams are sample-level
+   * equivalent. Omit (undefined) to skip the check; in-process renders
+   * authored before this field was introduced have implicit `undefined`.
+   */
+  maxAudioResidualRmsDb?: number;
   renderConfig: {
     /**
      * Frame rate. Stored on disk as a JSON number (integer fps, e.g. `30`)
@@ -229,6 +242,12 @@ function validateMetadata(meta: unknown): TestMetadata {
   if (typeof m.maxAudioLagWindows !== "number" || m.maxAudioLagWindows < 1) {
     throw new Error("meta.json: 'maxAudioLagWindows' must be >= 1");
   }
+  if (
+    m.maxAudioResidualRmsDb !== undefined &&
+    (typeof m.maxAudioResidualRmsDb !== "number" || !Number.isFinite(m.maxAudioResidualRmsDb))
+  ) {
+    throw new Error("meta.json: 'maxAudioResidualRmsDb' must be a finite number when present");
+  }
   if (!m.renderConfig || typeof m.renderConfig !== "object") {
     throw new Error("meta.json: 'renderConfig' must be an object");
   }
@@ -1051,6 +1070,7 @@ async function runTestSuite(
     let audioPassed = true;
     let audioCorrelation = 1;
     let audioLagWindows = 0;
+    let audioResidualRmsDb: number | null = null;
 
     if (!isPngSequence) {
       logPretty("Comparing audio quality...", "🔊");
@@ -1068,6 +1088,24 @@ async function runTestSuite(
         audioCorrelation = audio.correlation;
         audioLagWindows = audio.lagWindows;
         audioPassed = audio.correlation >= suite.meta.minAudioCorrelation;
+
+        // Rio-style residual RMS check, sample-level rather than
+        // envelope-level. Only runs when the fixture opts in by
+        // setting `maxAudioResidualRmsDb`; the envelope-correlation
+        // gate above stays in place either way for legacy fixtures
+        // (correlation is shape similarity; residual RMS is exact
+        // cancellation — both surface different drift classes).
+        if (suite.meta.maxAudioResidualRmsDb !== undefined) {
+          const residual = computeAudioResidualRmsDb(
+            renderedOutputPath,
+            snapshotVideoPath,
+            suite.meta.maxAudioResidualRmsDb,
+          );
+          audioResidualRmsDb = residual.overallDb;
+          if (!residual.ok) {
+            audioPassed = false;
+          }
+        }
       }
     }
 
@@ -1084,17 +1122,24 @@ async function runTestSuite(
         passed: audioPassed,
         correlation: audioCorrelation,
         lagWindows: audioLagWindows,
+        residualRmsDb: audioResidualRmsDb,
       }),
     );
 
+    const residualSuffix =
+      audioResidualRmsDb === null
+        ? ""
+        : `, residualRMS: ${
+            Number.isFinite(audioResidualRmsDb) ? audioResidualRmsDb.toFixed(2) : "-inf"
+          } dBFS`;
     if (audioPassed) {
       logPretty(
-        `Audio quality: PASSED (correlation: ${audioCorrelation.toFixed(3)}, lag: ${audioLagWindows})`,
+        `Audio quality: PASSED (correlation: ${audioCorrelation.toFixed(3)}, lag: ${audioLagWindows}${residualSuffix})`,
         "✓",
       );
     } else {
       logPretty(
-        `Audio quality: FAILED (correlation: ${audioCorrelation.toFixed(3)}, threshold: ${suite.meta.minAudioCorrelation})`,
+        `Audio quality: FAILED (correlation: ${audioCorrelation.toFixed(3)}, threshold: ${suite.meta.minAudioCorrelation}${residualSuffix})`,
         "✗",
       );
     }

packages/producer/src/utils/audioRegression.test.ts

@@ -1,5 +1,13 @@
-import { describe, expect, it } from "vitest";
-import { buildRmsEnvelope, compareAudioEnvelopes } from "./audioRegression.js";
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import {
+  buildRmsEnvelope,
+  compareAudioEnvelopes,
+  computeAudioResidualRmsDb,
+} from "./audioRegression.js";
 
 describe("compareAudioEnvelopes", () => {
   it("treats silent-vs-silent audio as a perfect match", () => {
@@ -14,3 +22,83 @@ describe("compareAudioEnvelopes", () => {
     });
   });
 });
+
+// Skip the spawn-based tests entirely on hosts without ffmpeg. The
+// regression harness only runs in environments where ffmpeg is present
+// (`Dockerfile.test`, dev boxes with apt's ffmpeg), so an absent ffmpeg
+// is a developer-laptop fact, not a producer regression.
+const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"], { encoding: "utf-8" }).status === 0;
+
+describe.skipIf(!HAS_FFMPEG)("computeAudioResidualRmsDb", () => {
+  let tmp: string;
+
+  beforeAll(() => {
+    tmp = mkdtempSync(join(tmpdir(), "hf-audio-residual-test-"));
+    // Two test wavs: identical 1-second 440 Hz sine, and a 880 Hz sine
+    // that's audibly different from the 440 reference.
+    for (const [name, freq] of [
+      ["sine-440-a.wav", 440],
+      ["sine-440-b.wav", 440],
+      ["sine-880.wav", 880],
+    ] as const) {
+      const result = spawnSync(
+        "ffmpeg",
+        [
+          "-nostdin",
+          "-v",
+          "error",
+          "-f",
+          "lavfi",
+          "-i",
+          `sine=frequency=${freq}:duration=1:sample_rate=48000`,
+          "-ac",
+          "2",
+          "-c:a",
+          "pcm_s16le",
+          join(tmp, name),
+        ],
+        { encoding: "utf-8" },
+      );
+      if (result.status !== 0) {
+        throw new Error(`ffmpeg setup failed for ${name}: ${result.stderr}`);
+      }
+    }
+  });
+
+  afterAll(() => {
+    rmSync(tmp, { recursive: true, force: true });
+  });
+
+  it("returns -inf (or very low dBFS) for two identical streams", () => {
+    const result = computeAudioResidualRmsDb(
+      join(tmp, "sine-440-a.wav"),
+      join(tmp, "sine-440-b.wav"),
+    );
+    expect(result.ok).toBe(true);
+    // 440-vs-440 PCM cancels to silence; ffmpeg reports -inf which we
+    // normalize to NEGATIVE_INFINITY, OR a value well below -90 if the
+    // resampler introduces sub-bit-quantization noise.
+    expect(result.overallDb).toBeLessThan(-80);
+  });
+
+  it("fails when streams are audibly different (440 Hz vs 880 Hz)", () => {
+    const result = computeAudioResidualRmsDb(
+      join(tmp, "sine-440-a.wav"),
+      join(tmp, "sine-880.wav"),
+    );
+    expect(result.ok).toBe(false);
+    // The residual of two uncorrelated unit-amplitude sines is roughly
+    // the sum of both signals at near-full level — typically around
+    // -3 dBFS in this resampled-stereo configuration.
+    expect(result.overallDb).toBeGreaterThan(-30);
+  });
+
+  it("reports ok=false when an input has no audio stream", () => {
+    // A bare empty file: ffmpeg can't probe it, so the function reports
+    // a parse failure (ok=false, NaN). Callers decide whether to treat
+    // that as a pass (no-audio fixture) or a fail (audio expected).
+    const result = computeAudioResidualRmsDb("/dev/null", join(tmp, "sine-440-a.wav"));
+    expect(result.ok).toBe(false);
+    expect(Number.isNaN(result.overallDb)).toBe(true);
+  });
+});

packages/producer/src/utils/audioRegression.ts

@@ -77,3 +77,134 @@ export function compareAudioEnvelopes(
 
   return bestEnvelopeCorrelation(rendered, snapshot, maxLagWindows);
 }
+
+// ── Sample-level residual RMS ───────────────────────────────────────────────
+//
+// Rio-style precise equivalence check: subtract one audio stream from
+// the other, run `astats`, read the residual Overall RMS in dBFS.
+// Perfectly-equivalent streams produce silence (≤ -90 dBFS in practice
+// for AAC-vs-AAC); the Rio convention is `≤ -50 dBFS = effectively
+// identical`.
+//
+// This catches level/phase drift the envelope-correlation check cannot.
+// Correlation measures shape similarity at envelope granularity (2048-
+// sample windows by default); residual RMS measures sample-level
+// cancellation, so it falls out as soon as the two streams disagree by
+// a fraction of a sample in alignment or by a fraction of a dB in
+// level.
+//
+// `astats` is invoked via `ffmpeg` spawned in-process. We require ffmpeg
+// on PATH — the regression harness already requires it for encode +
+// envelope extraction.
+
+import { spawnSync } from "node:child_process";
+
+/**
+ * Result of {@link computeAudioResidualRmsDb}.
+ *
+ * `overallDb` is the residual Overall RMS reading from astats. For
+ * exact-cancellation (truly identical streams), ffmpeg returns `-inf`;
+ * this helper normalizes that to `Number.NEGATIVE_INFINITY` so callers
+ * don't have to special-case the literal string.
+ */
+export interface AudioResidualRms {
+  overallDb: number;
+  ok: boolean;
+  /** Raw stderr lines that mention `RMS level` (one per channel + overall). Useful for debugging unexpected drift. */
+  rmsLines: string[];
+}
+
+/**
+ * Compute the residual Overall RMS (dBFS) of `rendered - snapshot`.
+ *
+ * Both inputs are paths to media files containing an audio stream.
+ * They're resampled to 48 kHz stereo, the snapshot is phase-inverted,
+ * the two are summed via `amix`, and `astats` reports the residual
+ * level.
+ *
+ * Returns `{ ok: false, overallDb: NaN }` if either input lacks an
+ * audio stream, or if ffmpeg's output didn't contain a parseable RMS
+ * line — the caller decides whether that's a pass (no-audio fixture)
+ * or a fail (audio expected but missing).
+ *
+ * `maxResidualRmsDb` defaults to `-50` (Rio convention). Pass `-Infinity`
+ * to compute the value without gating it.
+ */
+export function computeAudioResidualRmsDb(
+  rendered: string,
+  snapshot: string,
+  maxResidualRmsDb = -50,
+): AudioResidualRms {
+  const proc = spawnSync(
+    "ffmpeg",
+    [
+      "-nostdin",
+      "-v",
+      "info",
+      "-i",
+      rendered,
+      "-i",
+      snapshot,
+      "-filter_complex",
+      // Align both streams (resample + stereo + zero-based PTS), invert the
+      // snapshot, sum via amix, run astats. Avoids amix's `normalize`
+      // option (not available on ffmpeg 4.x) — we use volume=-1 + amix to
+      // subtract.
+      [
+        "[0:a]aresample=48000,pan=stereo|c0=c0|c1=c1,asetpts=N/SR/TB[a0]",
+        "[1:a]aresample=48000,pan=stereo|c0=c0|c1=c1,asetpts=N/SR/TB,volume=-1[a1]",
+        "[a0][a1]amix=inputs=2:duration=shortest:dropout_transition=0,astats=metadata=1:reset=1[out]",
+      ].join(";"),
+      "-map",
+      "[out]",
+      "-f",
+      "null",
+      "-",
+    ],
+    { encoding: "utf-8" },
+  );
+
+  const stderr = proc.stderr || "";
+  // Per-channel + overall RMS lines look like:
+  //   [Parsed_astats_8 @ 0x...] Overall RMS level dB: -90.32
+  //   [Parsed_astats_8 @ 0x...] RMS level dB: -90.36         (per-channel; no "Overall" prefix)
+  // Older ffmpeg builds use `Overall RMS level: -inf dB` — handle both shapes.
+  const rmsLines = stderr.split(/\r?\n/).filter((line) => /RMS level/.test(line));
+
+  // Prefer the "Overall" line if it appears; otherwise take the max
+  // per-channel RMS (the most pessimistic channel — that's what Rio
+  // does as its fallback path).
+  const overall = pickRms(rmsLines, /Overall RMS level(?:\s*dB)?:\s*(-?inf|[-\d.]+)/i);
+  const channelMax =
+    pickRms(rmsLines, /RMS level\s*dB:\s*(-?inf|[-\d.]+)/i, "max") ??
+    pickRms(rmsLines, /RMS level:\s*(-?inf|[-\d.]+)/i, "max");
+
+  const value = overall ?? channelMax;
+  if (value === null) {
+    return { overallDb: Number.NaN, ok: false, rmsLines };
+  }
+  return {
+    overallDb: value,
+    ok: value <= maxResidualRmsDb,
+    rmsLines,
+  };
+}
+
+function pickRms(lines: string[], re: RegExp, mode: "first" | "max" = "first"): number | null {
+  const values: number[] = [];
+  for (const line of lines) {
+    const m = re.exec(line);
+    if (!m) continue;
+    const raw = m[1];
+    if (raw === "-inf" || raw === "inf") {
+      values.push(Number.NEGATIVE_INFINITY);
+    } else {
+      const n = Number.parseFloat(raw ?? "");
+      if (!Number.isNaN(n)) values.push(n);
+    }
+    if (mode === "first") break;
+  }
+  if (values.length === 0) return null;
+  if (mode === "max") return Math.max(...values);
+  return values[0] ?? null;
+}