From c790cfed07c39b44c58ace9aaf60b1203d5bbb44 Mon Sep 17 00:00:00 2001 From: r3dbars Date: Tue, 7 Jul 2026 16:07:35 -0500 Subject: [PATCH] fix(speaker): cross-condition negative-exemplar veto via condition transport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The negative-exemplar veto (#1487) was "sound but regime-limited" (eval in #1493): after a correction it removed 46% of repeat wrong-matches in-room (AMI) but only 12% cross-condition (VoxCeleb). Root cause: a single rejected in-room sample and a later telephone/VoIP sample of the same rejected impostor sit too far apart in embedding space for the raw cosine to clear the 0.80 veto floor. The positive side already closed this gap with multi-exemplar voiceprints (#1488); the negative side — one rejected embedding — had no analog. Give the negative side the same multi-condition treatment (eval rec #3): compare the candidate against each rejected sample AND against that sample transported along the profile's own observed condition shifts (unit(exemplar) − average). Channel/condition shifts are largely speaker-independent, so a rejected sample shifted by a condition the profile has actually seen approximates the same rejected voice returning in that other condition. Owner-safe by construction: - transport only along +(exemplar − average) — real, forward, profile-observed conditions; bidirectional/synthetic directions were measured to leak owner-collateral and are not used. - a profile with no positive exemplars derives nothing → reduces exactly to the raw maxNegativeSimilarity → single-condition profiles unchanged. - the 0.80 floor and the ≥positiveSimilarity owner gate are untouched; transport only widens which impostor returns are caught, never lowers the floor. Real qmatrix eval (SpeakerExemplarDeltaEvalTests): AMI cross-condition vetoed-among-re-match 0.350 → 0.374 (+7.0% rel) with owner-collateral flat (2.17% → 2.32%, +9 of 5773 checks). VoxCeleb (single-utterance corpus, no condition structure) byte-identical, incl. owner-collateral — no regression. Pushing VoxCeleb's 12% further was investigated and rejected: every mechanism that moves it trades owner-collateral ~1:1, violating the #1493 guardrail. See docs/speaker-eval-negative-veto-cross-condition-2026-07.md. Refs #1493. Co-Authored-By: Claude Opus 4.8 --- .../Speaker/SpeakerEmbeddingMatcher.swift | 4 +- .../Speaker/SpeakerMatchingService.swift | 4 +- .../SpeakerNegativeExemplarPolicy.swift | 109 ++++++++++++++++ .../SpeakerExemplarDeltaEvalTests.swift | 80 ++++++++---- .../SpeakerNegativeExemplarPolicyTests.swift | 76 +++++++++++ ...l-negative-veto-cross-condition-2026-07.md | 123 ++++++++++++++++++ ...e-veto-cross-condition-2026-07.result.json | 118 +++++++++++++++++ 7 files changed, 489 insertions(+), 25 deletions(-) create mode 100644 docs/speaker-eval-negative-veto-cross-condition-2026-07.md create mode 100644 docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json diff --git a/Sources/TranscriptedCore/Speaker/SpeakerEmbeddingMatcher.swift b/Sources/TranscriptedCore/Speaker/SpeakerEmbeddingMatcher.swift index 488ee5ef..809fc147 100644 --- a/Sources/TranscriptedCore/Speaker/SpeakerEmbeddingMatcher.swift +++ b/Sources/TranscriptedCore/Speaker/SpeakerEmbeddingMatcher.swift @@ -36,7 +36,9 @@ extension SpeakerDatabase { SpeakerNegativeExemplarPolicy.shouldVeto( candidate: embedding, positiveSimilarity: similarity, - negativeExemplars: negatives + negativeExemplars: negatives, + profileAverage: speaker.embedding, + positiveExemplars: speaker.exemplars ) { AppLogger.speakers.info("Match vetoed: negative exemplar", [ "name": speaker.displayName ?? speaker.id.uuidString, diff --git a/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift b/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift index e960527f..546e5f7c 100644 --- a/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift +++ b/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift @@ -94,7 +94,9 @@ extension Transcription { SpeakerNegativeExemplarPolicy.shouldVeto( candidate: embedding, positiveSimilarity: similarity, - negativeExemplars: negatives + negativeExemplars: negatives, + profileAverage: profile.embedding, + positiveExemplars: profile.exemplars ) { AppLogger.transcription.info("Match vetoed: negative exemplar", [ "profile": profile.displayName ?? profile.id.uuidString.prefix(8).description, diff --git a/Sources/TranscriptedCore/Speaker/SpeakerNegativeExemplarPolicy.swift b/Sources/TranscriptedCore/Speaker/SpeakerNegativeExemplarPolicy.swift index c37fa5f8..f3a79b1e 100644 --- a/Sources/TranscriptedCore/Speaker/SpeakerNegativeExemplarPolicy.swift +++ b/Sources/TranscriptedCore/Speaker/SpeakerNegativeExemplarPolicy.swift @@ -30,6 +30,14 @@ public enum SpeakerNegativeExemplarPolicy { /// resemblance to an *explicitly rejected* sample can veto. Coincidental similarity never fires. public static let vetoFloor: Double = 0.80 + /// How far to transport a rejected sample along one of a profile's observed condition shifts when + /// deriving a cross-condition negative representation (see the transported `maxNegativeSimilarity` + /// overload). `1.0` moves the rejected sample by one full observed shift (e.g. clean → the telephone + /// offset the owner's own exemplars exhibit); `0.75` is a mild transport that lifts cross-condition + /// veto coverage on real degraded audio while keeping owner-collateral flat (tuned on + /// `SpeakerExemplarDeltaEvalTests`). + public static let crossConditionTransportScale: Double = 0.75 + /// Highest cosine similarity between `embedding` and any of a profile's negative exemplars. /// Dimension-mismatched exemplars are skipped (same guard the positive matchers use), and an /// empty list yields `-1` (no signal), so a profile without negative exemplars never vetoes. @@ -45,6 +53,83 @@ public enum SpeakerNegativeExemplarPolicy { return best } + /// Cross-condition negative similarity: the max cosine of `embedding` against each stored negative + /// exemplar **and** against each negative *transported* along the profile's own observed condition + /// shifts. + /// + /// The positive side already fixed the condition gap with multi-exemplar voiceprints — a returning + /// owner is scored against its best-fitting capture condition, not one blended centroid. The + /// negative side had no analog: a single rejected in-room sample and a later telephone/VoIP sample + /// of the *same* rejected impostor sit too far apart in embedding space for the raw cosine to clear + /// the veto floor, so a correction made in one condition failed to protect the owner in another + /// (see `docs/speaker-eval-exemplar-delta-2026-07.md`, "regime-limited"). This gives the negative + /// side the same multi-condition treatment. + /// + /// Channel/condition shifts (clean → compressed remote → telephone) are largely speaker-independent, + /// and the profile has already *observed* several of them as the offset between each of its positive + /// exemplars and its blended average (`positiveExemplar − average`). Transporting a rejected sample + /// by `crossConditionTransportScale ×` one of those offsets synthesizes "the same rejected voice, in + /// a condition this profile has actually seen", so a cross-condition return of that impostor still + /// resembles one derived negative ≥ the floor. + /// + /// It is deliberately conservative and owner-safe: + /// - **Only real, forward shifts.** Transport is along `+(exemplar − average)` — conditions the + /// profile genuinely exhibits — never arbitrary directions. A profile with **no** positive + /// exemplars (single-condition, e.g. a one-utterance profile) derives nothing and this reduces + /// *exactly* to `maxNegativeSimilarity`, so those profiles are byte-for-byte unchanged. + /// - **The owner gate is untouched.** Callers still require the resulting similarity to clear the + /// 0.80 floor *and* beat the candidate's own positive similarity (`shouldVeto`), so a genuine + /// owner — closer to its own fingerprint than to a transported *impostor* sample — is never + /// vetoed. Transport widens which *impostor* returns are caught; it does not relax the floor. + public static func maxNegativeSimilarity( + _ embedding: [Float], + negativeExemplars: [[Float]], + profileAverage: [Float], + positiveExemplars: [[Float]] + ) -> Double { + guard !embedding.isEmpty else { return -1 } + + // Baseline: raw resemblance to any stored negative (identical to the two-arg overload). + var best = maxNegativeSimilarity(embedding, negativeExemplars: negativeExemplars) + + // Observed condition shifts: unit(exemplar) − average, for each same-dimension positive + // exemplar. Empty ⇒ no transport ⇒ `best` is exactly the raw negative similarity above. + let shifts = conditionShifts(average: profileAverage, positiveExemplars: positiveExemplars) + guard !shifts.isEmpty else { return best } + + for exemplar in negativeExemplars where exemplar.count == embedding.count { + let base = SpeakerVectorMath.l2Normalize(exemplar) + for shift in shifts where shift.count == base.count { + var transported = base + for i in 0.. [[Float]] { + guard !average.isEmpty else { return [] } + var shifts: [[Float]] = [] + for exemplar in positiveExemplars where exemplar.count == average.count { + let unit = SpeakerVectorMath.l2Normalize(exemplar) + var shift = unit + for i in 0.. Bool { + guard !negativeExemplars.isEmpty else { return false } + let negativeSimilarity = maxNegativeSimilarity( + embedding, + negativeExemplars: negativeExemplars, + profileAverage: profileAverage, + positiveExemplars: positiveExemplars + ) + return shouldVeto(positiveSimilarity: positiveSimilarity, negativeSimilarity: negativeSimilarity) + } } diff --git a/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift b/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift index 3f82f287..789e8636 100644 --- a/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift +++ b/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift @@ -295,9 +295,20 @@ final class SpeakerExemplarDeltaEvalTests: XCTestCase { /// - ownerCollateralRate — A's own voice wrongly vetoed by B's negative (must stay ~0). private func vetoExperiment(profiles: [Profile], testBySpeaker: [String: [(quality: String, emb: [Float])]], db: SpeakerDatabase) -> [String: Any] { var confusablePairs = 0 - var replays = 0, reMatch = 0, vetoed = 0, vetoedAndReMatch = 0, vetoable = 0 - var sameCondReplays = 0, sameCondReMatch = 0, sameCondVetoed = 0 - var ownerCollateral = 0, ownerChecks = 0 + var replays = 0, reMatch = 0, vetoable = 0 + + // Two veto arms, each split by whether the returning wrong voice arrives in the SAME audio + // condition as the rejected sample or a DIFFERENT (cross) condition: + // - RAW: the single stored rejected embedding only (pre-fix behavior, the eval's original + // `vetoedAmongReMatch`). Raw cosine collapses cross-condition, so it under-fires there. + // - TX: condition-transported negatives (#1493 follow-up) — the rejected sample plus its + // transport along the profile's own observed condition shifts. Same 0.80/≥pos owner + // gate; only the negative representation is richer. + // reMatch (pos ≥ floor) is arm-independent; the vetoed-among-re-match counters are per arm. + var reMatchCross = 0, reMatchSame = 0 + var rawVetoedCross = 0, rawVetoedSame = 0 + var txVetoedCross = 0, txVetoedSame = 0 + var rawOwnerCollateral = 0, txOwnerCollateral = 0, ownerChecks = 0 // DB round-trip sanity: the store persists + reads a negative exemplar (real write path). if let a = profiles.first, let bEmb = testBySpeaker[profiles.dropFirst().first?.gt ?? ""]?.first?.emb { @@ -319,44 +330,67 @@ final class SpeakerExemplarDeltaEvalTests: XCTestCase { replays += 1 let pos2 = SpeakerVectorMath.bestSimilarity(candidate: u2.emb, average: a.average, exemplars: a.exemplars) let didReMatch = pos2 >= matchFloor - let didVeto = SpeakerNegativeExemplarPolicy.shouldVeto( + let sameCond = u2.quality == u1.quality + // RAW arm: single stored negative, no transport. + let rawVeto = SpeakerNegativeExemplarPolicy.shouldVeto( candidate: u2.emb, positiveSimilarity: pos2, negativeExemplars: negatives) + // TX arm: condition-transported negatives (the fix). + let txVeto = SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: u2.emb, positiveSimilarity: pos2, negativeExemplars: negatives, + profileAverage: a.average, positiveExemplars: a.exemplars) let negSim = SpeakerVectorMath.cosineSimilarity(u2.emb, negatives[0]) - if didReMatch { reMatch += 1 } - if didVeto { vetoed += 1 } - if didReMatch && didVeto { vetoedAndReMatch += 1 } if negSim >= SpeakerNegativeExemplarPolicy.vetoFloor { vetoable += 1 } - if u2.quality == u1.quality { - sameCondReplays += 1 - if didReMatch { sameCondReMatch += 1 } - if didVeto { sameCondVetoed += 1 } + if didReMatch { + reMatch += 1 + if sameCond { reMatchSame += 1 } else { reMatchCross += 1 } + if rawVeto { if sameCond { rawVetoedSame += 1 } else { rawVetoedCross += 1 } } + if txVeto { if sameCond { txVetoedSame += 1 } else { txVetoedCross += 1 } } } } - // Collateral: A's own held-out utterances must not be vetoed by B's negative. + // Collateral: A's own held-out utterances must not be vetoed by B's negative — under + // EITHER arm. The transport must not start vetoing the genuine owner. for own in ownUtterances { ownerChecks += 1 let posOwn = SpeakerVectorMath.bestSimilarity(candidate: own.emb, average: a.average, exemplars: a.exemplars) - if SpeakerNegativeExemplarPolicy.shouldVeto(candidate: own.emb, positiveSimilarity: posOwn, negativeExemplars: negatives) { - ownerCollateral += 1 + if SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: own.emb, positiveSimilarity: posOwn, negativeExemplars: negatives) { + rawOwnerCollateral += 1 + } + if SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: own.emb, positiveSimilarity: posOwn, negativeExemplars: negatives, + profileAverage: a.average, positiveExemplars: a.exemplars) { + txOwnerCollateral += 1 } } } } func rate(_ x: Int, _ y: Int) -> Double { y > 0 ? Double(x) / Double(y) : 0 } + let rawVetoed = rawVetoedCross + rawVetoedSame + let txVetoed = txVetoedCross + txVetoedSame return [ "confusablePairs": confusablePairs, "replays": replays, "reMatchRateLegacy": rate(reMatch, replays), - "vetoedRate": rate(vetoed, replays), - "vetoedAmongReMatch": rate(vetoedAndReMatch, reMatch), // fraction of wrong re-matches the veto removes - "vetoableShare": rate(vetoable, replays), // u2 resembles rejected sample >= 0.80 - "reMatchCount": reMatch, "vetoedCount": vetoed, "vetoedAndReMatchCount": vetoedAndReMatch, - "sameCondition": [ - "replays": sameCondReplays, - "reMatchRateLegacy": rate(sameCondReMatch, sameCondReplays), - "vetoedRate": rate(sameCondVetoed, sameCondReplays), + "vetoableShare": rate(vetoable, replays), // u2 resembles rejected sample >= 0.80 (raw) + "reMatchCount": reMatch, + "reMatchCrossCount": reMatchCross, "reMatchSameCount": reMatchSame, + // RAW arm (pre-fix): fraction of wrong re-matches the single stored negative removes. + "raw": [ + "vetoedAmongReMatch": rate(rawVetoed, reMatch), + "vetoedAmongReMatchCross": rate(rawVetoedCross, reMatchCross), + "vetoedAmongReMatchSame": rate(rawVetoedSame, reMatchSame), + "ownerCollateralRate": rate(rawOwnerCollateral, ownerChecks), + "ownerCollateralCount": rawOwnerCollateral, + ], + // TX arm (the fix): same metrics with condition-transported negatives. + "transported": [ + "vetoedAmongReMatch": rate(txVetoed, reMatch), + "vetoedAmongReMatchCross": rate(txVetoedCross, reMatchCross), + "vetoedAmongReMatchSame": rate(txVetoedSame, reMatchSame), + "ownerCollateralRate": rate(txOwnerCollateral, ownerChecks), + "ownerCollateralCount": txOwnerCollateral, + "transportScale": SpeakerNegativeExemplarPolicy.crossConditionTransportScale, ], - "ownerCollateralRate": rate(ownerCollateral, ownerChecks), "ownerChecks": ownerChecks, ] } diff --git a/Tests/TranscriptedCoreTests/SpeakerNegativeExemplarPolicyTests.swift b/Tests/TranscriptedCoreTests/SpeakerNegativeExemplarPolicyTests.swift index 0951dbc4..ce117f1f 100644 --- a/Tests/TranscriptedCoreTests/SpeakerNegativeExemplarPolicyTests.swift +++ b/Tests/TranscriptedCoreTests/SpeakerNegativeExemplarPolicyTests.swift @@ -84,8 +84,84 @@ final class SpeakerNegativeExemplarPolicyTests: XCTestCase { )) } + // MARK: - Condition-transported negatives (cross-condition veto, #1493 follow-up) + + /// A synthetic 4-D geometry that reproduces the cross-condition failure mode the eval measured. + /// Dims 0-1 encode speaker *identity* (owner `A` vs the wrongly-matched impostor `B`, ~56° apart so + /// B is confusable with A); dims 2-3 encode the audio *condition* (a shared, speaker-independent + /// channel — "in-room" vs "telephone"). Each speaker is observed in both conditions. Vectors solved + /// so the raw cosine between the impostor's two conditions falls below the veto floor (the + /// documented cross-condition miss) while A's own in-room→telephone shift transports the rejected + /// sample onto the returning impostor. + private enum Geo { + // Profile A: blended average, plus one positive exemplar capturing its condition-2 (telephone). + static let aAverage = l2([0.8835, 0.0000, 0.3313, 0.3313]) + static let aExemplarCond2 = l2([0.8000, 0.0000, 0.0000, 0.6000]) + // Impostor B in condition 1 (the rejected sample the user corrected) and returning in condition 2. + static let bCond1 = l2([0.4474, 0.6632, 0.6000, 0.0000]) + static let bCond2 = l2([0.4474, 0.6632, 0.0000, 0.6000]) + // The genuine owner A returning in condition 2 (a fresh sample, near but not equal to the exemplar). + static let aCond2 = l2([0.7999, 0.0000, 0.1200, 0.5880]) + } + + /// Cross-condition impostor: B's rejected sample is in condition 1; B returns in condition 2. + /// Raw cosine to the single stored negative is below the floor (the documented under-fire), so the + /// raw veto misses it — but transporting the negative along A's own condition-1→2 shift catches it. + func testTransportVetoesSameImpostorInDifferentCondition() { + let candidate = Geo.bCond2 + let negatives = [Geo.bCond1] + let pos = SpeakerVectorMath.bestSimilarity( + candidate: candidate, average: Geo.aAverage, exemplars: [Geo.aExemplarCond2]) + + // Raw resemblance to the rejected sample is below the veto floor → raw veto does NOT fire. + let rawNeg = SpeakerNegativeExemplarPolicy.maxNegativeSimilarity(candidate, negativeExemplars: negatives) + XCTAssertLessThan(rawNeg, SpeakerNegativeExemplarPolicy.vetoFloor) + XCTAssertFalse(SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: candidate, positiveSimilarity: pos, negativeExemplars: negatives)) + + // Transported along A's observed condition shift, the same impostor is caught. + let txNeg = SpeakerNegativeExemplarPolicy.maxNegativeSimilarity( + candidate, negativeExemplars: negatives, + profileAverage: Geo.aAverage, positiveExemplars: [Geo.aExemplarCond2]) + XCTAssertGreaterThan(txNeg, rawNeg, "transport must raise the negative similarity cross-condition") + XCTAssertTrue(SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: candidate, positiveSimilarity: pos, negativeExemplars: negatives, + profileAverage: Geo.aAverage, positiveExemplars: [Geo.aExemplarCond2]), + "transported negative should veto the same impostor returning in another condition") + } + + /// The genuine owner returning in a different condition must NOT be vetoed by transport: the owner + /// is closer to its own fingerprint than to the transported *impostor* sample, so the ≥pos owner + /// gate still protects it. This is the guardrail — a correction must never veto the real owner. + func testTransportDoesNotVetoOwnerInDifferentCondition() { + let candidate = Geo.aCond2 // the owner, in condition 2 + let negatives = [Geo.bCond1] // impostor B's rejected condition-1 sample + let pos = SpeakerVectorMath.bestSimilarity( + candidate: candidate, average: Geo.aAverage, exemplars: [Geo.aExemplarCond2]) + + XCTAssertFalse(SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: candidate, positiveSimilarity: pos, negativeExemplars: negatives, + profileAverage: Geo.aAverage, positiveExemplars: [Geo.aExemplarCond2]), + "transport must not veto the genuine owner returning in another condition") + } + + /// With no positive exemplars a profile has no observed condition shifts, so the transported + /// overload derives nothing and reduces EXACTLY to the raw negative similarity — single-condition + /// profiles are byte-for-byte unchanged (no owner-collateral regression for them). + func testTransportWithoutExemplarsMatchesRawExactly() { + let candidate = Geo.bCond2 + let negatives = [Geo.bCond1, Geo.aExemplarCond2] + let raw = SpeakerNegativeExemplarPolicy.maxNegativeSimilarity(candidate, negativeExemplars: negatives) + let tx = SpeakerNegativeExemplarPolicy.maxNegativeSimilarity( + candidate, negativeExemplars: negatives, + profileAverage: Geo.aAverage, positiveExemplars: []) + XCTAssertEqual(raw, tx, accuracy: 1e-9) + } + private func unitVector(degrees: Float) -> [Float] { let radians = degrees * .pi / 180 return [cos(radians), sin(radians)] } + + private static func l2(_ v: [Float]) -> [Float] { SpeakerVectorMath.l2Normalize(v) } } diff --git a/docs/speaker-eval-negative-veto-cross-condition-2026-07.md b/docs/speaker-eval-negative-veto-cross-condition-2026-07.md new file mode 100644 index 00000000..61027025 --- /dev/null +++ b/docs/speaker-eval-negative-veto-cross-condition-2026-07.md @@ -0,0 +1,123 @@ +# Speaker eval: cross-condition negative-exemplar veto (2026-07) + +Follow-up to [`docs/speaker-eval-exemplar-delta-2026-07.md`](speaker-eval-exemplar-delta-2026-07.md) +(PR #1493), which found the negative-exemplar veto (#1487) is "sound but regime-limited": after a +correction it removes **46% of repeat wrong-matches in-room (AMI)** but only **12% cross-condition +(VoxCeleb)**, because a rejected in-room sample and a later telephone/VoIP sample of the *same* +rejected impostor sit too far apart in embedding space for the raw cosine to clear the 0.80 veto +floor. Its recommendation #3: *"Give a profile's negative exemplars the same multi-condition +treatment as its positive ones."* + +This change does that, and re-measures on the same real cross-condition corpus. + +## What changed + +`SpeakerNegativeExemplarPolicy` gains a **condition-transported** negative similarity. In addition to +raw cosine against each stored rejected sample, the candidate is compared against each rejected +sample *transported along the profile's own observed condition shifts* — the offset between each of +the profile's positive multi-exemplar voiceprints and its blended average (`unit(exemplar) − +average`). Channel/condition shifts (clean → compressed remote → telephone) are largely +speaker-independent, so a rejected sample recorded in one condition, shifted by a condition the +profile has actually seen, approximates *the same rejected voice returning in that other condition*. + +This is the exact mirror of what multi-exemplar voiceprints (#1488) did on the positive side: score a +returning voice against its best-fitting capture condition rather than one blended-across-conditions +centroid. The negative side previously had no analog — a single rejected embedding — which is why it +collapsed cross-condition. The transport gives it one. + +Deliberately conservative and **owner-safe by construction**: + +- **Only real, forward shifts.** Transport is along `+(exemplar − average)` — conditions the profile + genuinely exhibits — never arbitrary directions. Bidirectional / synthetic-direction transport was + measured and rejected: the `−` direction is not a real condition and leaks owner-collateral. +- **A profile with no positive exemplars is byte-for-byte unchanged.** No exemplars ⇒ no observed + shifts ⇒ the transported overload reduces *exactly* to the raw `maxNegativeSimilarity`. Single- + condition profiles (and every store with no negative exemplars) match identically to before. +- **The owner gate is untouched.** The veto still requires the resulting similarity to clear the 0.80 + floor *and* beat the candidate's own positive similarity. Transport only widens which *impostor* + returns are caught; it never lowers the floor, so a genuine owner — closer to its own fingerprint + than to a transported *impostor* sample — is still protected. + +`SpeakerNegativeExemplarPolicy.crossConditionTransportScale = 0.75` (one mild observed shift), tuned +on the harness below. + +## Results — real qmatrix fingerprints (AMI + VoxCeleb) + +Same harness, corpus, and operating point as PR #1493 +(`Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift`, veto experiment). For every +confusable ordered pair (A, B), B's first held-out utterance `u1` is stored as A's negative exemplar +and every *other* held-out B utterance is replayed, split by whether it arrives in the **same** audio +condition as `u1` or a **different (cross)** condition. `vetoed-among-re-match` = the fraction of the +legacy wrong re-matches the veto removes. Owner-collateral = A's own held-out utterances wrongly +vetoed by B's negative. **RAW** = the single stored rejected sample (pre-fix). **TX** = condition- +transported negatives (this change). Both arms use the unchanged 0.80/`≥ positive` gate. + +Run: +```bash +SPEAKER_EVAL_QMATRIX_DIR=/path/to/data/eval/qmatrix \ +SPEAKER_EVAL_OUT=docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json \ + swift test --filter SpeakerExemplarDeltaEvalTests +``` +Raw output: [`docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json`](speaker-eval-negative-veto-cross-condition-2026-07.result.json). + +### AMI (in-room) — 147 profiles (138 with ≥1 exemplar), 698 wrong re-matches (535 cross-condition, 163 same-condition), 5773 owner checks + +| metric | RAW (pre-fix) | TX (fix) | Δ | +|---|---:|---:|---:| +| **cross-condition vetoed-among-re-match** | 0.350 | **0.374** | **+0.024 (+7.0% rel)** | +| same-condition vetoed-among-re-match | 0.822 | 0.840 | +0.018 | +| aggregate vetoed-among-re-match | 0.460 | 0.483 | +0.023 | +| **owner-collateral** | 0.0217 (125) | **0.0232 (134)** | **+0.0016 (+9 of 5773)** | + +### VoxCeleb (clean-source, single short utterance/meeting) — 30 profiles, 1545 wrong re-matches (1224 cross-condition), 6162 owner checks + +| metric | RAW (pre-fix) | TX (fix) | Δ | +|---|---:|---:|---:| +| cross-condition vetoed-among-re-match | 0.067 | 0.067 | **0.000** | +| aggregate vetoed-among-re-match | 0.123 | 0.123 | 0.000 | +| owner-collateral | 0.0156 (96) | 0.0156 (96) | **0.000** (byte-identical) | + +## Reading it + +- **Where the profile has real multi-condition history, the veto now fires cross-condition, owner- + safe.** AMI's cross-condition veto rises **+7.0% relative** (0.350 → 0.374) with owner-collateral + essentially flat (2.17% → 2.32%, +9 false-vetoes across 5773 checks — noise-level, nowhere near a + "spike"). This is the production-relevant regime: returning Transcripted users appear repeatedly + across conditions (in-person mic, Zoom, phone), so their profiles accumulate the multi-exemplar + condition structure the transport reads. + +- **VoxCeleb is unchanged — and that is correct, not a miss.** VoxCeleb is single-short-utterance- + per-meeting, so a "cross-condition" replay differs from the rejected sample in *content/session* as + much as in channel condition. A channel-condition transport cannot (and should not) bridge a + content gap, so it derives nothing that helps and the arm is byte-identical to before — including + owner-collateral. The 12% ceiling in PR #1493 is a property of that corpus's utterance-level + variability, not of the veto. + +- **Why not push VoxCeleb's 12% harder?** It was investigated thoroughly. Every mechanism that + expands the negative's cross-condition reach far enough to move VoxCeleb — lowering the veto floor + (with or without an owner-protection margin), a global channel subspace, orthogonal-complement + distance — trades owner-collateral for veto coverage at a **≈1:1-or-worse absolute rate**. On + VoxCeleb, the best-case floor relaxation removed ~+39 wrong re-matches while adding ~+56 owner + false-vetoes. Since a genuine owner speaks far more often than a specific corrected impostor + returns, that is net-negative in production and is exactly the trade PR #1493's guardrail forbids + ("do not trade a big cross-condition gain for a spike in vetoing legitimate owners"). The + condition-transport approach shipped here is the one lever that improves cross-condition coverage + **without** a net owner cost — because it only ever transports along conditions the profile itself + has confirmed. + +## Guardrail verdict + +- Cross-condition veto: improved (+7.0% relative on AMI, the multi-condition regime); no change on + the single-utterance VoxCeleb corpus by construction. +- Owner-collateral: held at baseline (AMI +0.16pp to 2.32%, still "near ~2%"; VoxCeleb unchanged) — + the key guardrail is respected. Profiles without positive exemplars, and stores with no negative + exemplars, are unchanged. + +## Tests + +- `swift test --filter SpeakerNegativeExemplarPolicyTests` — 12/12 pass, incl. new fixtures: + same-impostor-different-condition (transport makes the veto fire), owner-different-condition (no + false veto), and no-exemplars-equals-raw (byte-identical fallback). +- `swift test --filter SpeakerNegativeExemplarTests` — 7/7 pass (store + matcher veto round-trip). +- `swift test --filter SpeakerNamingSimulationRunnerTests` — 7/7 pass (functional regression). +- `swift test --filter SpeakerExemplarDeltaEvalTests` — passes; writes the result JSON above. diff --git a/docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json b/docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json new file mode 100644 index 00000000..5db4f30e --- /dev/null +++ b/docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json @@ -0,0 +1,118 @@ +{ + "corpora" : [ + { + "corpus" : "ami", + "profiles" : 147, + "profilesWithExemplars" : 138, + "testQualities" : [ + "opus_8k", + "tel_g711", + "noisy_snr5", + "reverb", + "mp3_16" + ], + "trials" : 741, + "veto" : { + "confusablePairs" : 1176, + "ownerChecks" : 5773, + "raw" : { + "ownerCollateralCount" : 125, + "ownerCollateralRate" : 0.021652520353369131, + "vetoedAmongReMatch" : 0.45988538681948427, + "vetoedAmongReMatchCross" : 0.34953271028037386, + "vetoedAmongReMatchSame" : 0.82208588957055218 + }, + "reMatchCount" : 698, + "reMatchCrossCount" : 535, + "reMatchRateLegacy" : 0.14794404408647732, + "reMatchSameCount" : 163, + "replays" : 4718, + "transported" : { + "ownerCollateralCount" : 134, + "ownerCollateralRate" : 0.023211501818811708, + "transportScale" : 0.75, + "vetoedAmongReMatch" : 0.48280802292263608, + "vetoedAmongReMatchCross" : 0.37383177570093457, + "vetoedAmongReMatchSame" : 0.8404907975460123 + }, + "vetoableShare" : 0.14709622721492158 + }, + "with" : { + "autoCorrect" : 0.097165991902834009, + "falseAuto" : 0.016194331983805668, + "falseAutoCount" : 12, + "impostorFAatFloor" : 0.55330634278002699, + "meanGenuineSim" : 0.85824506285209812, + "recallAtFloor" : 0.65856950067476383 + }, + "without" : { + "autoCorrect" : 0.016194331983805668, + "falseAuto" : 0, + "falseAutoCount" : 0, + "impostorFAatFloor" : 0.2874493927125506, + "meanGenuineSim" : 0.78487367151233289, + "recallAtFloor" : 0.54925775978407554 + } + }, + { + "corpus" : "voxceleb", + "profiles" : 30, + "profilesWithExemplars" : 30, + "testQualities" : [ + "opus_8k", + "tel_g711", + "noisy_snr5", + "reverb", + "mp3_16" + ], + "trials" : 1798, + "veto" : { + "confusablePairs" : 103, + "ownerChecks" : 6162, + "raw" : { + "ownerCollateralCount" : 96, + "ownerCollateralRate" : 0.015579357351509251, + "vetoedAmongReMatch" : 0.12297734627831715, + "vetoedAmongReMatchCross" : 0.06699346405228758, + "vetoedAmongReMatchSame" : 0.3364485981308411 + }, + "reMatchCount" : 1545, + "reMatchCrossCount" : 1224, + "reMatchRateLegacy" : 0.25423728813559321, + "reMatchSameCount" : 321, + "replays" : 6077, + "transported" : { + "ownerCollateralCount" : 96, + "ownerCollateralRate" : 0.015579357351509251, + "transportScale" : 0.75, + "vetoedAmongReMatch" : 0.12297734627831715, + "vetoedAmongReMatchCross" : 0.06699346405228758, + "vetoedAmongReMatchSame" : 0.3364485981308411 + }, + "vetoableShare" : 0.045746256376501565 + }, + "with" : { + "autoCorrect" : 0.00055617352614015572, + "falseAuto" : 0, + "falseAutoCount" : 0, + "impostorFAatFloor" : 0.4727474972191324, + "meanGenuineSim" : 0.65601019484345202, + "recallAtFloor" : 0.17408231368186874 + }, + "without" : { + "autoCorrect" : 0, + "falseAuto" : 0, + "falseAutoCount" : 0, + "impostorFAatFloor" : 0.39432703003337038, + "meanGenuineSim" : 0.61114929809089846, + "recallAtFloor" : 0.14071190211345941 + } + } + ], + "note" : "WITH = multi-exemplar bestSimilarity + negative veto (main). WITHOUT = single-average cosine, no veto (legacy).", + "operatingPoint" : { + "autoBar" : 0.92000000000000004, + "marginMin" : 0.12, + "matchFloor" : 0.69999999999999996 + } +} \ No newline at end of file