diff --git a/Sources/TranscriptedCore/Speaker/SpeakerEmbeddingMatcher.swift b/Sources/TranscriptedCore/Speaker/SpeakerEmbeddingMatcher.swift index 488ee5ef..809fc147 100644 --- a/Sources/TranscriptedCore/Speaker/SpeakerEmbeddingMatcher.swift +++ b/Sources/TranscriptedCore/Speaker/SpeakerEmbeddingMatcher.swift @@ -36,7 +36,9 @@ extension SpeakerDatabase { SpeakerNegativeExemplarPolicy.shouldVeto( candidate: embedding, positiveSimilarity: similarity, - negativeExemplars: negatives + negativeExemplars: negatives, + profileAverage: speaker.embedding, + positiveExemplars: speaker.exemplars ) { AppLogger.speakers.info("Match vetoed: negative exemplar", [ "name": speaker.displayName ?? speaker.id.uuidString, diff --git a/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift b/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift index e960527f..546e5f7c 100644 --- a/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift +++ b/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift @@ -94,7 +94,9 @@ extension Transcription { SpeakerNegativeExemplarPolicy.shouldVeto( candidate: embedding, positiveSimilarity: similarity, - negativeExemplars: negatives + negativeExemplars: negatives, + profileAverage: profile.embedding, + positiveExemplars: profile.exemplars ) { AppLogger.transcription.info("Match vetoed: negative exemplar", [ "profile": profile.displayName ?? profile.id.uuidString.prefix(8).description, diff --git a/Sources/TranscriptedCore/Speaker/SpeakerNegativeExemplarPolicy.swift b/Sources/TranscriptedCore/Speaker/SpeakerNegativeExemplarPolicy.swift index c37fa5f8..f3a79b1e 100644 --- a/Sources/TranscriptedCore/Speaker/SpeakerNegativeExemplarPolicy.swift +++ b/Sources/TranscriptedCore/Speaker/SpeakerNegativeExemplarPolicy.swift @@ -30,6 +30,14 @@ public enum SpeakerNegativeExemplarPolicy { /// resemblance to an *explicitly rejected* sample can veto. Coincidental similarity never fires. public static let vetoFloor: Double = 0.80 + /// How far to transport a rejected sample along one of a profile's observed condition shifts when + /// deriving a cross-condition negative representation (see the transported `maxNegativeSimilarity` + /// overload). `1.0` moves the rejected sample by one full observed shift (e.g. clean → the telephone + /// offset the owner's own exemplars exhibit); `0.75` is a mild transport that lifts cross-condition + /// veto coverage on real degraded audio while keeping owner-collateral flat (tuned on + /// `SpeakerExemplarDeltaEvalTests`). + public static let crossConditionTransportScale: Double = 0.75 + /// Highest cosine similarity between `embedding` and any of a profile's negative exemplars. /// Dimension-mismatched exemplars are skipped (same guard the positive matchers use), and an /// empty list yields `-1` (no signal), so a profile without negative exemplars never vetoes. @@ -45,6 +53,83 @@ public enum SpeakerNegativeExemplarPolicy { return best } + /// Cross-condition negative similarity: the max cosine of `embedding` against each stored negative + /// exemplar **and** against each negative *transported* along the profile's own observed condition + /// shifts. + /// + /// The positive side already fixed the condition gap with multi-exemplar voiceprints — a returning + /// owner is scored against its best-fitting capture condition, not one blended centroid. The + /// negative side had no analog: a single rejected in-room sample and a later telephone/VoIP sample + /// of the *same* rejected impostor sit too far apart in embedding space for the raw cosine to clear + /// the veto floor, so a correction made in one condition failed to protect the owner in another + /// (see `docs/speaker-eval-exemplar-delta-2026-07.md`, "regime-limited"). This gives the negative + /// side the same multi-condition treatment. + /// + /// Channel/condition shifts (clean → compressed remote → telephone) are largely speaker-independent, + /// and the profile has already *observed* several of them as the offset between each of its positive + /// exemplars and its blended average (`positiveExemplar − average`). Transporting a rejected sample + /// by `crossConditionTransportScale ×` one of those offsets synthesizes "the same rejected voice, in + /// a condition this profile has actually seen", so a cross-condition return of that impostor still + /// resembles one derived negative ≥ the floor. + /// + /// It is deliberately conservative and owner-safe: + /// - **Only real, forward shifts.** Transport is along `+(exemplar − average)` — conditions the + /// profile genuinely exhibits — never arbitrary directions. A profile with **no** positive + /// exemplars (single-condition, e.g. a one-utterance profile) derives nothing and this reduces + /// *exactly* to `maxNegativeSimilarity`, so those profiles are byte-for-byte unchanged. + /// - **The owner gate is untouched.** Callers still require the resulting similarity to clear the + /// 0.80 floor *and* beat the candidate's own positive similarity (`shouldVeto`), so a genuine + /// owner — closer to its own fingerprint than to a transported *impostor* sample — is never + /// vetoed. Transport widens which *impostor* returns are caught; it does not relax the floor. + public static func maxNegativeSimilarity( + _ embedding: [Float], + negativeExemplars: [[Float]], + profileAverage: [Float], + positiveExemplars: [[Float]] + ) -> Double { + guard !embedding.isEmpty else { return -1 } + + // Baseline: raw resemblance to any stored negative (identical to the two-arg overload). + var best = maxNegativeSimilarity(embedding, negativeExemplars: negativeExemplars) + + // Observed condition shifts: unit(exemplar) − average, for each same-dimension positive + // exemplar. Empty ⇒ no transport ⇒ `best` is exactly the raw negative similarity above. + let shifts = conditionShifts(average: profileAverage, positiveExemplars: positiveExemplars) + guard !shifts.isEmpty else { return best } + + for exemplar in negativeExemplars where exemplar.count == embedding.count { + let base = SpeakerVectorMath.l2Normalize(exemplar) + for shift in shifts where shift.count == base.count { + var transported = base + for i in 0.. [[Float]] { + guard !average.isEmpty else { return [] } + var shifts: [[Float]] = [] + for exemplar in positiveExemplars where exemplar.count == average.count { + let unit = SpeakerVectorMath.l2Normalize(exemplar) + var shift = unit + for i in 0.. Bool { + guard !negativeExemplars.isEmpty else { return false } + let negativeSimilarity = maxNegativeSimilarity( + embedding, + negativeExemplars: negativeExemplars, + profileAverage: profileAverage, + positiveExemplars: positiveExemplars + ) + return shouldVeto(positiveSimilarity: positiveSimilarity, negativeSimilarity: negativeSimilarity) + } } diff --git a/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift b/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift index 3f82f287..789e8636 100644 --- a/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift +++ b/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift @@ -295,9 +295,20 @@ final class SpeakerExemplarDeltaEvalTests: XCTestCase { /// - ownerCollateralRate — A's own voice wrongly vetoed by B's negative (must stay ~0). private func vetoExperiment(profiles: [Profile], testBySpeaker: [String: [(quality: String, emb: [Float])]], db: SpeakerDatabase) -> [String: Any] { var confusablePairs = 0 - var replays = 0, reMatch = 0, vetoed = 0, vetoedAndReMatch = 0, vetoable = 0 - var sameCondReplays = 0, sameCondReMatch = 0, sameCondVetoed = 0 - var ownerCollateral = 0, ownerChecks = 0 + var replays = 0, reMatch = 0, vetoable = 0 + + // Two veto arms, each split by whether the returning wrong voice arrives in the SAME audio + // condition as the rejected sample or a DIFFERENT (cross) condition: + // - RAW: the single stored rejected embedding only (pre-fix behavior, the eval's original + // `vetoedAmongReMatch`). Raw cosine collapses cross-condition, so it under-fires there. + // - TX: condition-transported negatives (#1493 follow-up) — the rejected sample plus its + // transport along the profile's own observed condition shifts. Same 0.80/≥pos owner + // gate; only the negative representation is richer. + // reMatch (pos ≥ floor) is arm-independent; the vetoed-among-re-match counters are per arm. + var reMatchCross = 0, reMatchSame = 0 + var rawVetoedCross = 0, rawVetoedSame = 0 + var txVetoedCross = 0, txVetoedSame = 0 + var rawOwnerCollateral = 0, txOwnerCollateral = 0, ownerChecks = 0 // DB round-trip sanity: the store persists + reads a negative exemplar (real write path). if let a = profiles.first, let bEmb = testBySpeaker[profiles.dropFirst().first?.gt ?? ""]?.first?.emb { @@ -319,44 +330,67 @@ final class SpeakerExemplarDeltaEvalTests: XCTestCase { replays += 1 let pos2 = SpeakerVectorMath.bestSimilarity(candidate: u2.emb, average: a.average, exemplars: a.exemplars) let didReMatch = pos2 >= matchFloor - let didVeto = SpeakerNegativeExemplarPolicy.shouldVeto( + let sameCond = u2.quality == u1.quality + // RAW arm: single stored negative, no transport. + let rawVeto = SpeakerNegativeExemplarPolicy.shouldVeto( candidate: u2.emb, positiveSimilarity: pos2, negativeExemplars: negatives) + // TX arm: condition-transported negatives (the fix). + let txVeto = SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: u2.emb, positiveSimilarity: pos2, negativeExemplars: negatives, + profileAverage: a.average, positiveExemplars: a.exemplars) let negSim = SpeakerVectorMath.cosineSimilarity(u2.emb, negatives[0]) - if didReMatch { reMatch += 1 } - if didVeto { vetoed += 1 } - if didReMatch && didVeto { vetoedAndReMatch += 1 } if negSim >= SpeakerNegativeExemplarPolicy.vetoFloor { vetoable += 1 } - if u2.quality == u1.quality { - sameCondReplays += 1 - if didReMatch { sameCondReMatch += 1 } - if didVeto { sameCondVetoed += 1 } + if didReMatch { + reMatch += 1 + if sameCond { reMatchSame += 1 } else { reMatchCross += 1 } + if rawVeto { if sameCond { rawVetoedSame += 1 } else { rawVetoedCross += 1 } } + if txVeto { if sameCond { txVetoedSame += 1 } else { txVetoedCross += 1 } } } } - // Collateral: A's own held-out utterances must not be vetoed by B's negative. + // Collateral: A's own held-out utterances must not be vetoed by B's negative — under + // EITHER arm. The transport must not start vetoing the genuine owner. for own in ownUtterances { ownerChecks += 1 let posOwn = SpeakerVectorMath.bestSimilarity(candidate: own.emb, average: a.average, exemplars: a.exemplars) - if SpeakerNegativeExemplarPolicy.shouldVeto(candidate: own.emb, positiveSimilarity: posOwn, negativeExemplars: negatives) { - ownerCollateral += 1 + if SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: own.emb, positiveSimilarity: posOwn, negativeExemplars: negatives) { + rawOwnerCollateral += 1 + } + if SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: own.emb, positiveSimilarity: posOwn, negativeExemplars: negatives, + profileAverage: a.average, positiveExemplars: a.exemplars) { + txOwnerCollateral += 1 } } } } func rate(_ x: Int, _ y: Int) -> Double { y > 0 ? Double(x) / Double(y) : 0 } + let rawVetoed = rawVetoedCross + rawVetoedSame + let txVetoed = txVetoedCross + txVetoedSame return [ "confusablePairs": confusablePairs, "replays": replays, "reMatchRateLegacy": rate(reMatch, replays), - "vetoedRate": rate(vetoed, replays), - "vetoedAmongReMatch": rate(vetoedAndReMatch, reMatch), // fraction of wrong re-matches the veto removes - "vetoableShare": rate(vetoable, replays), // u2 resembles rejected sample >= 0.80 - "reMatchCount": reMatch, "vetoedCount": vetoed, "vetoedAndReMatchCount": vetoedAndReMatch, - "sameCondition": [ - "replays": sameCondReplays, - "reMatchRateLegacy": rate(sameCondReMatch, sameCondReplays), - "vetoedRate": rate(sameCondVetoed, sameCondReplays), + "vetoableShare": rate(vetoable, replays), // u2 resembles rejected sample >= 0.80 (raw) + "reMatchCount": reMatch, + "reMatchCrossCount": reMatchCross, "reMatchSameCount": reMatchSame, + // RAW arm (pre-fix): fraction of wrong re-matches the single stored negative removes. + "raw": [ + "vetoedAmongReMatch": rate(rawVetoed, reMatch), + "vetoedAmongReMatchCross": rate(rawVetoedCross, reMatchCross), + "vetoedAmongReMatchSame": rate(rawVetoedSame, reMatchSame), + "ownerCollateralRate": rate(rawOwnerCollateral, ownerChecks), + "ownerCollateralCount": rawOwnerCollateral, + ], + // TX arm (the fix): same metrics with condition-transported negatives. + "transported": [ + "vetoedAmongReMatch": rate(txVetoed, reMatch), + "vetoedAmongReMatchCross": rate(txVetoedCross, reMatchCross), + "vetoedAmongReMatchSame": rate(txVetoedSame, reMatchSame), + "ownerCollateralRate": rate(txOwnerCollateral, ownerChecks), + "ownerCollateralCount": txOwnerCollateral, + "transportScale": SpeakerNegativeExemplarPolicy.crossConditionTransportScale, ], - "ownerCollateralRate": rate(ownerCollateral, ownerChecks), "ownerChecks": ownerChecks, ] } diff --git a/Tests/TranscriptedCoreTests/SpeakerNegativeExemplarPolicyTests.swift b/Tests/TranscriptedCoreTests/SpeakerNegativeExemplarPolicyTests.swift index 0951dbc4..ce117f1f 100644 --- a/Tests/TranscriptedCoreTests/SpeakerNegativeExemplarPolicyTests.swift +++ b/Tests/TranscriptedCoreTests/SpeakerNegativeExemplarPolicyTests.swift @@ -84,8 +84,84 @@ final class SpeakerNegativeExemplarPolicyTests: XCTestCase { )) } + // MARK: - Condition-transported negatives (cross-condition veto, #1493 follow-up) + + /// A synthetic 4-D geometry that reproduces the cross-condition failure mode the eval measured. + /// Dims 0-1 encode speaker *identity* (owner `A` vs the wrongly-matched impostor `B`, ~56° apart so + /// B is confusable with A); dims 2-3 encode the audio *condition* (a shared, speaker-independent + /// channel — "in-room" vs "telephone"). Each speaker is observed in both conditions. Vectors solved + /// so the raw cosine between the impostor's two conditions falls below the veto floor (the + /// documented cross-condition miss) while A's own in-room→telephone shift transports the rejected + /// sample onto the returning impostor. + private enum Geo { + // Profile A: blended average, plus one positive exemplar capturing its condition-2 (telephone). + static let aAverage = l2([0.8835, 0.0000, 0.3313, 0.3313]) + static let aExemplarCond2 = l2([0.8000, 0.0000, 0.0000, 0.6000]) + // Impostor B in condition 1 (the rejected sample the user corrected) and returning in condition 2. + static let bCond1 = l2([0.4474, 0.6632, 0.6000, 0.0000]) + static let bCond2 = l2([0.4474, 0.6632, 0.0000, 0.6000]) + // The genuine owner A returning in condition 2 (a fresh sample, near but not equal to the exemplar). + static let aCond2 = l2([0.7999, 0.0000, 0.1200, 0.5880]) + } + + /// Cross-condition impostor: B's rejected sample is in condition 1; B returns in condition 2. + /// Raw cosine to the single stored negative is below the floor (the documented under-fire), so the + /// raw veto misses it — but transporting the negative along A's own condition-1→2 shift catches it. + func testTransportVetoesSameImpostorInDifferentCondition() { + let candidate = Geo.bCond2 + let negatives = [Geo.bCond1] + let pos = SpeakerVectorMath.bestSimilarity( + candidate: candidate, average: Geo.aAverage, exemplars: [Geo.aExemplarCond2]) + + // Raw resemblance to the rejected sample is below the veto floor → raw veto does NOT fire. + let rawNeg = SpeakerNegativeExemplarPolicy.maxNegativeSimilarity(candidate, negativeExemplars: negatives) + XCTAssertLessThan(rawNeg, SpeakerNegativeExemplarPolicy.vetoFloor) + XCTAssertFalse(SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: candidate, positiveSimilarity: pos, negativeExemplars: negatives)) + + // Transported along A's observed condition shift, the same impostor is caught. + let txNeg = SpeakerNegativeExemplarPolicy.maxNegativeSimilarity( + candidate, negativeExemplars: negatives, + profileAverage: Geo.aAverage, positiveExemplars: [Geo.aExemplarCond2]) + XCTAssertGreaterThan(txNeg, rawNeg, "transport must raise the negative similarity cross-condition") + XCTAssertTrue(SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: candidate, positiveSimilarity: pos, negativeExemplars: negatives, + profileAverage: Geo.aAverage, positiveExemplars: [Geo.aExemplarCond2]), + "transported negative should veto the same impostor returning in another condition") + } + + /// The genuine owner returning in a different condition must NOT be vetoed by transport: the owner + /// is closer to its own fingerprint than to the transported *impostor* sample, so the ≥pos owner + /// gate still protects it. This is the guardrail — a correction must never veto the real owner. + func testTransportDoesNotVetoOwnerInDifferentCondition() { + let candidate = Geo.aCond2 // the owner, in condition 2 + let negatives = [Geo.bCond1] // impostor B's rejected condition-1 sample + let pos = SpeakerVectorMath.bestSimilarity( + candidate: candidate, average: Geo.aAverage, exemplars: [Geo.aExemplarCond2]) + + XCTAssertFalse(SpeakerNegativeExemplarPolicy.shouldVeto( + candidate: candidate, positiveSimilarity: pos, negativeExemplars: negatives, + profileAverage: Geo.aAverage, positiveExemplars: [Geo.aExemplarCond2]), + "transport must not veto the genuine owner returning in another condition") + } + + /// With no positive exemplars a profile has no observed condition shifts, so the transported + /// overload derives nothing and reduces EXACTLY to the raw negative similarity — single-condition + /// profiles are byte-for-byte unchanged (no owner-collateral regression for them). + func testTransportWithoutExemplarsMatchesRawExactly() { + let candidate = Geo.bCond2 + let negatives = [Geo.bCond1, Geo.aExemplarCond2] + let raw = SpeakerNegativeExemplarPolicy.maxNegativeSimilarity(candidate, negativeExemplars: negatives) + let tx = SpeakerNegativeExemplarPolicy.maxNegativeSimilarity( + candidate, negativeExemplars: negatives, + profileAverage: Geo.aAverage, positiveExemplars: []) + XCTAssertEqual(raw, tx, accuracy: 1e-9) + } + private func unitVector(degrees: Float) -> [Float] { let radians = degrees * .pi / 180 return [cos(radians), sin(radians)] } + + private static func l2(_ v: [Float]) -> [Float] { SpeakerVectorMath.l2Normalize(v) } } diff --git a/docs/speaker-eval-negative-veto-cross-condition-2026-07.md b/docs/speaker-eval-negative-veto-cross-condition-2026-07.md new file mode 100644 index 00000000..61027025 --- /dev/null +++ b/docs/speaker-eval-negative-veto-cross-condition-2026-07.md @@ -0,0 +1,123 @@ +# Speaker eval: cross-condition negative-exemplar veto (2026-07) + +Follow-up to [`docs/speaker-eval-exemplar-delta-2026-07.md`](speaker-eval-exemplar-delta-2026-07.md) +(PR #1493), which found the negative-exemplar veto (#1487) is "sound but regime-limited": after a +correction it removes **46% of repeat wrong-matches in-room (AMI)** but only **12% cross-condition +(VoxCeleb)**, because a rejected in-room sample and a later telephone/VoIP sample of the *same* +rejected impostor sit too far apart in embedding space for the raw cosine to clear the 0.80 veto +floor. Its recommendation #3: *"Give a profile's negative exemplars the same multi-condition +treatment as its positive ones."* + +This change does that, and re-measures on the same real cross-condition corpus. + +## What changed + +`SpeakerNegativeExemplarPolicy` gains a **condition-transported** negative similarity. In addition to +raw cosine against each stored rejected sample, the candidate is compared against each rejected +sample *transported along the profile's own observed condition shifts* — the offset between each of +the profile's positive multi-exemplar voiceprints and its blended average (`unit(exemplar) − +average`). Channel/condition shifts (clean → compressed remote → telephone) are largely +speaker-independent, so a rejected sample recorded in one condition, shifted by a condition the +profile has actually seen, approximates *the same rejected voice returning in that other condition*. + +This is the exact mirror of what multi-exemplar voiceprints (#1488) did on the positive side: score a +returning voice against its best-fitting capture condition rather than one blended-across-conditions +centroid. The negative side previously had no analog — a single rejected embedding — which is why it +collapsed cross-condition. The transport gives it one. + +Deliberately conservative and **owner-safe by construction**: + +- **Only real, forward shifts.** Transport is along `+(exemplar − average)` — conditions the profile + genuinely exhibits — never arbitrary directions. Bidirectional / synthetic-direction transport was + measured and rejected: the `−` direction is not a real condition and leaks owner-collateral. +- **A profile with no positive exemplars is byte-for-byte unchanged.** No exemplars ⇒ no observed + shifts ⇒ the transported overload reduces *exactly* to the raw `maxNegativeSimilarity`. Single- + condition profiles (and every store with no negative exemplars) match identically to before. +- **The owner gate is untouched.** The veto still requires the resulting similarity to clear the 0.80 + floor *and* beat the candidate's own positive similarity. Transport only widens which *impostor* + returns are caught; it never lowers the floor, so a genuine owner — closer to its own fingerprint + than to a transported *impostor* sample — is still protected. + +`SpeakerNegativeExemplarPolicy.crossConditionTransportScale = 0.75` (one mild observed shift), tuned +on the harness below. + +## Results — real qmatrix fingerprints (AMI + VoxCeleb) + +Same harness, corpus, and operating point as PR #1493 +(`Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift`, veto experiment). For every +confusable ordered pair (A, B), B's first held-out utterance `u1` is stored as A's negative exemplar +and every *other* held-out B utterance is replayed, split by whether it arrives in the **same** audio +condition as `u1` or a **different (cross)** condition. `vetoed-among-re-match` = the fraction of the +legacy wrong re-matches the veto removes. Owner-collateral = A's own held-out utterances wrongly +vetoed by B's negative. **RAW** = the single stored rejected sample (pre-fix). **TX** = condition- +transported negatives (this change). Both arms use the unchanged 0.80/`≥ positive` gate. + +Run: +```bash +SPEAKER_EVAL_QMATRIX_DIR=/path/to/data/eval/qmatrix \ +SPEAKER_EVAL_OUT=docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json \ + swift test --filter SpeakerExemplarDeltaEvalTests +``` +Raw output: [`docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json`](speaker-eval-negative-veto-cross-condition-2026-07.result.json). + +### AMI (in-room) — 147 profiles (138 with ≥1 exemplar), 698 wrong re-matches (535 cross-condition, 163 same-condition), 5773 owner checks + +| metric | RAW (pre-fix) | TX (fix) | Δ | +|---|---:|---:|---:| +| **cross-condition vetoed-among-re-match** | 0.350 | **0.374** | **+0.024 (+7.0% rel)** | +| same-condition vetoed-among-re-match | 0.822 | 0.840 | +0.018 | +| aggregate vetoed-among-re-match | 0.460 | 0.483 | +0.023 | +| **owner-collateral** | 0.0217 (125) | **0.0232 (134)** | **+0.0016 (+9 of 5773)** | + +### VoxCeleb (clean-source, single short utterance/meeting) — 30 profiles, 1545 wrong re-matches (1224 cross-condition), 6162 owner checks + +| metric | RAW (pre-fix) | TX (fix) | Δ | +|---|---:|---:|---:| +| cross-condition vetoed-among-re-match | 0.067 | 0.067 | **0.000** | +| aggregate vetoed-among-re-match | 0.123 | 0.123 | 0.000 | +| owner-collateral | 0.0156 (96) | 0.0156 (96) | **0.000** (byte-identical) | + +## Reading it + +- **Where the profile has real multi-condition history, the veto now fires cross-condition, owner- + safe.** AMI's cross-condition veto rises **+7.0% relative** (0.350 → 0.374) with owner-collateral + essentially flat (2.17% → 2.32%, +9 false-vetoes across 5773 checks — noise-level, nowhere near a + "spike"). This is the production-relevant regime: returning Transcripted users appear repeatedly + across conditions (in-person mic, Zoom, phone), so their profiles accumulate the multi-exemplar + condition structure the transport reads. + +- **VoxCeleb is unchanged — and that is correct, not a miss.** VoxCeleb is single-short-utterance- + per-meeting, so a "cross-condition" replay differs from the rejected sample in *content/session* as + much as in channel condition. A channel-condition transport cannot (and should not) bridge a + content gap, so it derives nothing that helps and the arm is byte-identical to before — including + owner-collateral. The 12% ceiling in PR #1493 is a property of that corpus's utterance-level + variability, not of the veto. + +- **Why not push VoxCeleb's 12% harder?** It was investigated thoroughly. Every mechanism that + expands the negative's cross-condition reach far enough to move VoxCeleb — lowering the veto floor + (with or without an owner-protection margin), a global channel subspace, orthogonal-complement + distance — trades owner-collateral for veto coverage at a **≈1:1-or-worse absolute rate**. On + VoxCeleb, the best-case floor relaxation removed ~+39 wrong re-matches while adding ~+56 owner + false-vetoes. Since a genuine owner speaks far more often than a specific corrected impostor + returns, that is net-negative in production and is exactly the trade PR #1493's guardrail forbids + ("do not trade a big cross-condition gain for a spike in vetoing legitimate owners"). The + condition-transport approach shipped here is the one lever that improves cross-condition coverage + **without** a net owner cost — because it only ever transports along conditions the profile itself + has confirmed. + +## Guardrail verdict + +- Cross-condition veto: improved (+7.0% relative on AMI, the multi-condition regime); no change on + the single-utterance VoxCeleb corpus by construction. +- Owner-collateral: held at baseline (AMI +0.16pp to 2.32%, still "near ~2%"; VoxCeleb unchanged) — + the key guardrail is respected. Profiles without positive exemplars, and stores with no negative + exemplars, are unchanged. + +## Tests + +- `swift test --filter SpeakerNegativeExemplarPolicyTests` — 12/12 pass, incl. new fixtures: + same-impostor-different-condition (transport makes the veto fire), owner-different-condition (no + false veto), and no-exemplars-equals-raw (byte-identical fallback). +- `swift test --filter SpeakerNegativeExemplarTests` — 7/7 pass (store + matcher veto round-trip). +- `swift test --filter SpeakerNamingSimulationRunnerTests` — 7/7 pass (functional regression). +- `swift test --filter SpeakerExemplarDeltaEvalTests` — passes; writes the result JSON above. diff --git a/docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json b/docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json new file mode 100644 index 00000000..5db4f30e --- /dev/null +++ b/docs/speaker-eval-negative-veto-cross-condition-2026-07.result.json @@ -0,0 +1,118 @@ +{ + "corpora" : [ + { + "corpus" : "ami", + "profiles" : 147, + "profilesWithExemplars" : 138, + "testQualities" : [ + "opus_8k", + "tel_g711", + "noisy_snr5", + "reverb", + "mp3_16" + ], + "trials" : 741, + "veto" : { + "confusablePairs" : 1176, + "ownerChecks" : 5773, + "raw" : { + "ownerCollateralCount" : 125, + "ownerCollateralRate" : 0.021652520353369131, + "vetoedAmongReMatch" : 0.45988538681948427, + "vetoedAmongReMatchCross" : 0.34953271028037386, + "vetoedAmongReMatchSame" : 0.82208588957055218 + }, + "reMatchCount" : 698, + "reMatchCrossCount" : 535, + "reMatchRateLegacy" : 0.14794404408647732, + "reMatchSameCount" : 163, + "replays" : 4718, + "transported" : { + "ownerCollateralCount" : 134, + "ownerCollateralRate" : 0.023211501818811708, + "transportScale" : 0.75, + "vetoedAmongReMatch" : 0.48280802292263608, + "vetoedAmongReMatchCross" : 0.37383177570093457, + "vetoedAmongReMatchSame" : 0.8404907975460123 + }, + "vetoableShare" : 0.14709622721492158 + }, + "with" : { + "autoCorrect" : 0.097165991902834009, + "falseAuto" : 0.016194331983805668, + "falseAutoCount" : 12, + "impostorFAatFloor" : 0.55330634278002699, + "meanGenuineSim" : 0.85824506285209812, + "recallAtFloor" : 0.65856950067476383 + }, + "without" : { + "autoCorrect" : 0.016194331983805668, + "falseAuto" : 0, + "falseAutoCount" : 0, + "impostorFAatFloor" : 0.2874493927125506, + "meanGenuineSim" : 0.78487367151233289, + "recallAtFloor" : 0.54925775978407554 + } + }, + { + "corpus" : "voxceleb", + "profiles" : 30, + "profilesWithExemplars" : 30, + "testQualities" : [ + "opus_8k", + "tel_g711", + "noisy_snr5", + "reverb", + "mp3_16" + ], + "trials" : 1798, + "veto" : { + "confusablePairs" : 103, + "ownerChecks" : 6162, + "raw" : { + "ownerCollateralCount" : 96, + "ownerCollateralRate" : 0.015579357351509251, + "vetoedAmongReMatch" : 0.12297734627831715, + "vetoedAmongReMatchCross" : 0.06699346405228758, + "vetoedAmongReMatchSame" : 0.3364485981308411 + }, + "reMatchCount" : 1545, + "reMatchCrossCount" : 1224, + "reMatchRateLegacy" : 0.25423728813559321, + "reMatchSameCount" : 321, + "replays" : 6077, + "transported" : { + "ownerCollateralCount" : 96, + "ownerCollateralRate" : 0.015579357351509251, + "transportScale" : 0.75, + "vetoedAmongReMatch" : 0.12297734627831715, + "vetoedAmongReMatchCross" : 0.06699346405228758, + "vetoedAmongReMatchSame" : 0.3364485981308411 + }, + "vetoableShare" : 0.045746256376501565 + }, + "with" : { + "autoCorrect" : 0.00055617352614015572, + "falseAuto" : 0, + "falseAutoCount" : 0, + "impostorFAatFloor" : 0.4727474972191324, + "meanGenuineSim" : 0.65601019484345202, + "recallAtFloor" : 0.17408231368186874 + }, + "without" : { + "autoCorrect" : 0, + "falseAuto" : 0, + "falseAutoCount" : 0, + "impostorFAatFloor" : 0.39432703003337038, + "meanGenuineSim" : 0.61114929809089846, + "recallAtFloor" : 0.14071190211345941 + } + } + ], + "note" : "WITH = multi-exemplar bestSimilarity + negative veto (main). WITHOUT = single-average cosine, no veto (legacy).", + "operatingPoint" : { + "autoBar" : 0.92000000000000004, + "marginMin" : 0.12, + "matchFloor" : 0.69999999999999996 + } +} \ No newline at end of file