diff --git a/Sources/TranscriptedCore/Models/TranscriptionTypes.swift b/Sources/TranscriptedCore/Models/TranscriptionTypes.swift index 672b6cac..f5e5c197 100644 --- a/Sources/TranscriptedCore/Models/TranscriptionTypes.swift +++ b/Sources/TranscriptedCore/Models/TranscriptionTypes.swift @@ -351,19 +351,30 @@ public struct ChannelSpeakerContext: Sendable { /// Similarity of the next-closest profile that cleared the match floor (the runner-up), /// or nil if unknown / no runner-up. Feeds `SpeakerNamingPolicy`'s auto-accept margin guard. public let matchSecondSimilarity: Double? + /// Winner's cosine to its blended *average* representative (exemplars excluded), and the + /// highest average cosine among the other candidate profiles. Feed + /// `SpeakerNamingPolicy.shouldAutoAccept`'s `marginSimilarities` so the auto-accept margin is + /// computed on the average, not the best exemplar (see + /// docs/speaker-eval-exemplar-delta-2026-07.md). nil on paths that don't carry a match result. + public let matchAverageSimilarity: Double? + public let matchSecondBestAverageSimilarity: Double? public init( persistentSpeakerId: UUID, sessionEmbedding: [Float]?, matchedProfileSnapshot: SpeakerProfile?, matchSimilarity: Double?, - matchSecondSimilarity: Double? = nil + matchSecondSimilarity: Double? = nil, + matchAverageSimilarity: Double? = nil, + matchSecondBestAverageSimilarity: Double? = nil ) { self.persistentSpeakerId = persistentSpeakerId self.sessionEmbedding = sessionEmbedding self.matchedProfileSnapshot = matchedProfileSnapshot self.matchSimilarity = matchSimilarity self.matchSecondSimilarity = matchSecondSimilarity + self.matchAverageSimilarity = matchAverageSimilarity + self.matchSecondBestAverageSimilarity = matchSecondBestAverageSimilarity } } diff --git a/Sources/TranscriptedCore/Pipeline/TranscriptionPipeline.swift b/Sources/TranscriptedCore/Pipeline/TranscriptionPipeline.swift index d99e47e9..ba4d4fa5 100644 --- a/Sources/TranscriptedCore/Pipeline/TranscriptionPipeline.swift +++ b/Sources/TranscriptedCore/Pipeline/TranscriptionPipeline.swift @@ -253,7 +253,7 @@ extension Transcription { // Match each speaker's mean embedding against the DB once // (existingProfiles was already snapshotted above for post-processing) - var speakerMatchResults: [Int: (persistentId: UUID, similarity: Double, secondSimilarity: Double)] = [:] + var speakerMatchResults: [Int: (persistentId: UUID, similarity: Double, secondSimilarity: Double, averageSimilarity: Double, secondAverageSimilarity: Double)] = [:] var speakerNewProfiles: [Int: UUID] = [:] var speakerIdRemap: [Int: Int] = [:] // Session mean embedding actually matched for each non-ghost speaker, kept so the @@ -320,7 +320,7 @@ extension Transcription { // the shared profile. Matching reads only the `existingProfiles` snapshot, so // deferring the blend cannot change any match decision. if let matchResult = Self.matchAgainstProfiles(meanEmbedding, profiles: existingProfiles, threshold: adaptiveThreshold, negativeExemplarsByProfile: negativeExemplarsByProfile) { - speakerMatchResults[speakerId] = (matchResult.profileId, matchResult.similarity, matchResult.secondBestSimilarity) + speakerMatchResults[speakerId] = (matchResult.profileId, matchResult.similarity, matchResult.secondBestSimilarity, matchResult.averageSimilarity, matchResult.secondBestAverageSimilarity) let matchedProfile = existingProfiles.first(where: { $0.id == matchResult.profileId }) AppLogger.transcription.info("Speaker matched DB profile", [ "speakerId": "\(speakerId)", @@ -440,7 +440,9 @@ extension Transcription { sessionEmbedding: sessionEmbedding, matchedProfileSnapshot: matchedProfileSnapshot, matchSimilarity: speakerMatchResults[effectiveSpeakerId]?.similarity, - matchSecondSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondSimilarity + matchSecondSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondSimilarity, + matchAverageSimilarity: speakerMatchResults[effectiveSpeakerId]?.averageSimilarity, + matchSecondBestAverageSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondAverageSimilarity ) } @@ -890,7 +892,7 @@ extension Transcription { } } - var speakerMatchResults: [Int: (persistentId: UUID, similarity: Double, secondSimilarity: Double)] = [:] + var speakerMatchResults: [Int: (persistentId: UUID, similarity: Double, secondSimilarity: Double, averageSimilarity: Double, secondAverageSimilarity: Double)] = [:] var speakerNewProfiles: [Int: UUID] = [:] var speakerIdRemap: [Int: Int] = [:] var newlyCreatedProfileIds: Set = [] @@ -930,7 +932,7 @@ extension Transcription { // deferring the blend changes no match decision, and a spun-off distinct voice never // contaminates the shared profile). if let matchResult = Self.matchAgainstProfiles(meanEmbedding, profiles: existingProfiles, threshold: adaptiveThreshold, negativeExemplarsByProfile: negativeExemplarsByProfile) { - speakerMatchResults[speakerId] = (matchResult.profileId, matchResult.similarity, matchResult.secondBestSimilarity) + speakerMatchResults[speakerId] = (matchResult.profileId, matchResult.similarity, matchResult.secondBestSimilarity, matchResult.averageSimilarity, matchResult.secondBestAverageSimilarity) } else { let newProfile = speakerDB.addOrUpdateSpeaker(embedding: meanEmbedding, existingId: nil) speakerNewProfiles[speakerId] = newProfile.id @@ -997,7 +999,9 @@ extension Transcription { sessionEmbedding: sessionEmbedding, matchedProfileSnapshot: matchedProfileSnapshot, matchSimilarity: speakerMatchResults[effectiveSpeakerId]?.similarity, - matchSecondSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondSimilarity + matchSecondSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondSimilarity, + matchAverageSimilarity: speakerMatchResults[effectiveSpeakerId]?.averageSimilarity, + matchSecondBestAverageSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondAverageSimilarity ) } diff --git a/Sources/TranscriptedCore/Pipeline/TranscriptionPipelineRunner.swift b/Sources/TranscriptedCore/Pipeline/TranscriptionPipelineRunner.swift index 9f3d4567..9e599af1 100644 --- a/Sources/TranscriptedCore/Pipeline/TranscriptionPipelineRunner.swift +++ b/Sources/TranscriptedCore/Pipeline/TranscriptionPipelineRunner.swift @@ -12,6 +12,18 @@ extension TranscriptionTaskManager { /// Runner-up similarity for the auto-accept margin guard; nil when unknown (the /// utterance fallback path), which `shouldAutoAccept` treats as "confirm, don't auto". let secondSimilarity: Double? + /// Winner's average-only cosine and the runner-up's average-only cosine, when carried by + /// the match context. Feed `SpeakerNamingPolicy.shouldAutoAccept`'s `marginSimilarities` + /// so the auto-accept margin is judged on the blended average, not the best exemplar. + let averageSimilarity: Double? + let secondAverageSimilarity: Double? + + /// Average-based margin pair for `shouldAutoAccept`, or nil to fall back to the legacy + /// (exemplar) margin when the average sims weren't carried (e.g. utterance fallback). + var marginSimilarities: (best: Double, secondBest: Double)? { + guard let best = averageSimilarity, let second = secondAverageSimilarity else { return nil } + return (best: best, secondBest: second) + } } nonisolated static func speakerClassificationKnowledge( @@ -31,7 +43,9 @@ extension TranscriptionTaskManager { speakerId: sid, profile: snapshot, similarity: similarity, - secondSimilarity: context.matchSecondSimilarity + secondSimilarity: context.matchSecondSimilarity, + averageSimilarity: context.matchAverageSimilarity, + secondAverageSimilarity: context.matchSecondBestAverageSimilarity )) continue } @@ -44,7 +58,9 @@ extension TranscriptionTaskManager { speakerId: sid, profile: profile, similarity: similarity, - secondSimilarity: nil // runner-up not carried on the utterance fallback + secondSimilarity: nil, // runner-up not carried on the utterance fallback + averageSimilarity: nil, + secondAverageSimilarity: nil )) } } @@ -330,7 +346,8 @@ extension TranscriptionTaskManager { profile: entry.profile, similarity: entry.similarity, secondBestSimilarity: entry.secondSimilarity, - recentOutcomes: cachedRecentOutcomes(entry.profile) + recentOutcomes: cachedRecentOutcomes(entry.profile), + marginSimilarities: entry.marginSimilarities ) if canAutoAccept { autoAcceptedIds.insert(sid) @@ -358,7 +375,8 @@ extension TranscriptionTaskManager { profile: entry.profile, similarity: entry.similarity, secondBestSimilarity: entry.secondSimilarity, - recentOutcomes: recentOutcomesByProfile[entry.profile.id] ?? [] + recentOutcomes: recentOutcomesByProfile[entry.profile.id] ?? [], + marginSimilarities: entry.marginSimilarities ) speakerMappings[key] = mapping speakerSources[key] = autoAcceptedIds.contains(entry.speakerId) ? "db" : "db_pending" @@ -403,7 +421,8 @@ extension TranscriptionTaskManager { profile: entry.profile, similarity: entry.similarity, secondBestSimilarity: entry.secondSimilarity, - recentOutcomes: cachedRecentOutcomes(entry.profile) + recentOutcomes: cachedRecentOutcomes(entry.profile), + marginSimilarities: entry.marginSimilarities ) if canAutoAccept { micAutoAcceptedIds.insert(sid) @@ -435,7 +454,8 @@ extension TranscriptionTaskManager { profile: entry.profile, similarity: entry.similarity, secondBestSimilarity: entry.secondSimilarity, - recentOutcomes: recentOutcomesByProfile[entry.profile.id] ?? [] + recentOutcomes: recentOutcomesByProfile[entry.profile.id] ?? [], + marginSimilarities: entry.marginSimilarities ) speakerMappings[key] = mapping speakerSources["mic_\(entry.speakerId)"] = micAutoAcceptedIds.contains(entry.speakerId) ? "db" : "db_pending" diff --git a/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift b/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift index 70e862a5..e960527f 100644 --- a/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift +++ b/Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift @@ -10,9 +10,18 @@ extension Transcription { let profileId: UUID let similarity: Double /// Similarity of the next-closest non-disputed profile that cleared the floor, or -1 - /// if there was no runner-up. Used by `SpeakerNamingPolicy.shouldAutoAccept` as the - /// "clear winner" margin guard. + /// if there was no runner-up. Best-of-representatives (exemplar) score; still used for + /// write-back blend gating and logging. let secondBestSimilarity: Double + /// Winner's cosine to its *blended average only* (exemplars excluded). Top of the + /// average-based auto-accept margin (`SpeakerNamingPolicy.shouldAutoAccept` + /// `marginSimilarities`). Equal to `similarity` for legacy single-average profiles. + let averageSimilarity: Double + /// Highest average-only cosine among the OTHER candidate profiles (the runner-up on the + /// average representation), or -1 if there was none. Bottom of the average-based margin. + /// Decoupling the margin from the best exemplar keeps a lucky-exemplar impostor from + /// clearing the auto-accept gate — see docs/speaker-eval-exemplar-delta-2026-07.md. + let secondBestAverageSimilarity: Double } // MARK: - Embedding Utilities @@ -61,6 +70,11 @@ extension Transcription { var bestProfile: SpeakerProfile? var bestSimilarity: Double = -1 var secondBestSimilarity: Double = -1 + // Average-only cosine for every candidate profile (survives dispute/dim/veto gating), + // used to build the average-based auto-accept margin after the winner is chosen. Kept + // separate from `bestSimilarity` (best-of-exemplars) so the recall win from exemplar + // scoring is untouched while the margin is computed against the blended average. + var averageSimilarityByProfile: [UUID: Double] = [:] for profile in profiles { // A disputed profile was explicitly rejected by the user; don't @@ -88,6 +102,10 @@ extension Transcription { ]) continue } + // Record the average-only cosine for the margin computation (all non-vetoed candidates, + // not just floor-clearers, so the runner-up on the average representation is captured + // even when it didn't clear the exemplar floor). + averageSimilarityByProfile[profile.id] = SpeakerVectorMath.cosineSimilarity(embedding, profile.embedding) if similarity >= threshold { if similarity > bestSimilarity { secondBestSimilarity = bestSimilarity @@ -129,8 +147,20 @@ extension Transcription { return nil } + // Average-based margin: the winner's cosine to its blended average vs the highest average + // cosine among the other candidates. A genuine owner is close on both average and best + // exemplar (margin holds); a lucky-exemplar impostor is far from the average (margin + // collapses), so `SpeakerNamingPolicy.shouldAutoAccept` withholds the silent auto-name. + let averageSimilarity = averageSimilarityByProfile[matched.id] ?? bestSimilarity + var secondBestAverageSimilarity = -1.0 + for (id, avg) in averageSimilarityByProfile where id != matched.id { + secondBestAverageSimilarity = max(secondBestAverageSimilarity, avg) + } + return SnapshotMatchResult(profileId: matched.id, similarity: bestSimilarity, - secondBestSimilarity: secondBestSimilarity) + secondBestSimilarity: secondBestSimilarity, + averageSimilarity: averageSimilarity, + secondBestAverageSimilarity: secondBestAverageSimilarity) } // MARK: - Cross-Cluster Link/Merge (#8) diff --git a/Sources/TranscriptedCore/Speaker/SpeakerNamingPolicy.swift b/Sources/TranscriptedCore/Speaker/SpeakerNamingPolicy.swift index 5a8410ee..a3820fff 100644 --- a/Sources/TranscriptedCore/Speaker/SpeakerNamingPolicy.swift +++ b/Sources/TranscriptedCore/Speaker/SpeakerNamingPolicy.swift @@ -39,13 +39,33 @@ public enum SpeakerNamingPolicy { ) == .trusted } + /// - Parameters: + /// - similarity: best-of-representatives similarity (blended average OR any stored + /// multi-exemplar voiceprint, `SpeakerVectorMath.bestSimilarity`). Checked against the + /// 0.92 bar so the multi-exemplar recall win is preserved on degraded audio. + /// - secondBestSimilarity: legacy runner-up similarity used for the margin when + /// `marginSimilarities` is not supplied (and the sentinel: `< 0` ⇒ no runner-up, `nil` ⇒ + /// unknown ⇒ conservative). + /// - marginSimilarities: when supplied, the best-vs-second **margin** is computed against + /// each profile's *average* representative (`best` = winner's cosine to its blended + /// average, `secondBest` = highest average cosine among the other candidates, `< 0` ⇒ + /// none). Decoupling the margin from the best-exemplar score restores the "0 false-auto" + /// guarantee: a genuine owner is close on BOTH its average and its best exemplar so the + /// margin still holds, while an impostor that only cleared the 0.92 bar via one lucky + /// exemplar is far from the average, so its average-based margin collapses and auto-accept + /// is withheld (routed to suggest/ask). Legacy callers and single-average profiles (where + /// average == best exemplar) pass `nil` and behave exactly as before. See + /// `docs/speaker-eval-exemplar-delta-2026-07.md`. public static func shouldAutoAccept( profile: SpeakerProfile, similarity: Double, - secondBestSimilarity: Double? + secondBestSimilarity: Double?, + marginSimilarities: (best: Double, secondBest: Double)? = nil ) -> Bool { + let marginTop = marginSimilarities?.best ?? similarity + let marginRunnerUp: Double? = marginSimilarities.map { $0.secondBest } ?? secondBestSimilarity let marginOK: Bool - switch secondBestSimilarity { + switch marginRunnerUp { case .none: // Runner-up unknown (e.g. a fallback path that didn't carry it) — be conservative // and route to confirm rather than silently auto-name. @@ -53,7 +73,7 @@ public enum SpeakerNamingPolicy { case .some(let second) where second < 0: marginOK = true // no confusable runner-up cleared the match floor case .some(let second): - marginOK = (similarity - second) >= autoAcceptMarginMin + marginOK = (marginTop - second) >= autoAcceptMarginMin } return isAutoRecognizable(profile: profile, recentOutcomes: []) && similarity > autoAcceptSimilarityThreshold @@ -69,7 +89,8 @@ public enum SpeakerNamingPolicy { profile: SpeakerProfile, similarity: Double, secondBestSimilarity: Double?, - recentOutcomes: [SpeakerMatchOutcomeKind] + recentOutcomes: [SpeakerMatchOutcomeKind], + marginSimilarities: (best: Double, secondBest: Double)? = nil ) -> Bool { guard isAutoRecognizable(profile: profile, recentOutcomes: recentOutcomes) else { return false @@ -77,7 +98,8 @@ public enum SpeakerNamingPolicy { return shouldAutoAccept( profile: profile, similarity: similarity, - secondBestSimilarity: secondBestSimilarity + secondBestSimilarity: secondBestSimilarity, + marginSimilarities: marginSimilarities ) } @@ -90,13 +112,15 @@ public enum SpeakerNamingPolicy { profile: SpeakerProfile, similarity: Double, secondBestSimilarity: Double?, - recentOutcomes: [SpeakerMatchOutcomeKind] = [] + recentOutcomes: [SpeakerMatchOutcomeKind] = [], + marginSimilarities: (best: Double, secondBest: Double)? = nil ) -> SpeakerMapping { guard shouldAutoAccept( profile: profile, similarity: similarity, secondBestSimilarity: secondBestSimilarity, - recentOutcomes: recentOutcomes + recentOutcomes: recentOutcomes, + marginSimilarities: marginSimilarities ), let name = profile.displayName, !name.isEmpty else { diff --git a/Sources/TranscriptedCore/Speaker/SpeakerNamingSimulationRunner.swift b/Sources/TranscriptedCore/Speaker/SpeakerNamingSimulationRunner.swift index 4adbdbb5..d7085277 100644 --- a/Sources/TranscriptedCore/Speaker/SpeakerNamingSimulationRunner.swift +++ b/Sources/TranscriptedCore/Speaker/SpeakerNamingSimulationRunner.swift @@ -632,7 +632,9 @@ public final class SpeakerNamingSimulationRunner { sessionEmbedding: meanEmbedding, matchedProfileSnapshot: snapshot, matchSimilarity: match.similarity, - matchSecondSimilarity: match.secondBestSimilarity + matchSecondSimilarity: match.secondBestSimilarity, + matchAverageSimilarity: match.averageSimilarity, + matchSecondBestAverageSimilarity: match.secondBestAverageSimilarity ) } else { let profile = speakerDB.addOrUpdateSpeaker(embedding: meanEmbedding, existingId: nil) @@ -771,18 +773,24 @@ public final class SpeakerNamingSimulationRunner { profileId: snapshot.id, limit: SpeakerProfileHealth.recentOutcomeWindow ).map(\.kind) + let marginSimilarities: (best: Double, secondBest: Double)? = + context.matchAverageSimilarity.flatMap { best in + context.matchSecondBestAverageSimilarity.map { (best: best, secondBest: $0) } + } let canAutoAccept = SpeakerNamingPolicy.shouldAutoAccept( profile: snapshot, similarity: similarity, secondBestSimilarity: context.matchSecondSimilarity, - recentOutcomes: recentOutcomes + recentOutcomes: recentOutcomes, + marginSimilarities: marginSimilarities ) speakerMappings[key] = SpeakerNamingPolicy.initialMapping( speakerId: speakerId, profile: snapshot, similarity: similarity, secondBestSimilarity: context.matchSecondSimilarity, - recentOutcomes: recentOutcomes + recentOutcomes: recentOutcomes, + marginSimilarities: marginSimilarities ) speakerSources[key] = canAutoAccept ? "db" : "db_pending" } else { diff --git a/Tests/TranscriptedCoreTests/SpeakerAutoAcceptMarginTests.swift b/Tests/TranscriptedCoreTests/SpeakerAutoAcceptMarginTests.swift new file mode 100644 index 00000000..873f484c --- /dev/null +++ b/Tests/TranscriptedCoreTests/SpeakerAutoAcceptMarginTests.swift @@ -0,0 +1,135 @@ +import XCTest +@testable import TranscriptedCore + +/// Guards the release-critical retune from #1493's findings: the multi-exemplar feature (#1488) +/// let a distinct speaker clear the auto-accept gate via ONE lucky exemplar (similarity 0.93–0.97, +/// best-vs-second margin 0.12–0.27), producing 12 silent mislabels on degraded AMI audio where the +/// average-only matcher had 0. The fix decouples the auto-accept MARGIN from the best-exemplar +/// score: keep best-of-exemplars for match SELECTION (the recall win), but judge the best-vs-second +/// margin against each profile's blended AVERAGE. A genuine owner is close on both average and best +/// exemplar (margin holds); a lucky-exemplar impostor is far from the average (margin collapses, +/// auto-accept withheld). See docs/speaker-eval-exemplar-delta-2026-07.md. +/// +/// These are the in-tree, corpus-free proof of that behavior (the full A/B lives in +/// `SpeakerExemplarDeltaEvalTests`, which is XCTSkip'd without the 11 GB qmatrix corpus). +@available(macOS 14.0, *) +final class SpeakerAutoAcceptMarginTests: XCTestCase { + + private func matureProfile( + _ name: String, + embedding: [Float], + exemplars: [[Float]] = [], + callCount: Int = 10, + disputeCount: Int = 0 + ) -> SpeakerProfile { + SpeakerProfile( + id: UUID(), + displayName: name, + nameSource: NameSource.userManual, + embedding: embedding, + firstSeen: Date(), + lastSeen: Date(), + callCount: callCount, + confidence: 0.9, + disputeCount: disputeCount, + exemplars: exemplars) + } + + // MARK: - Policy gate, explicit scores + + /// The exact failure shape: best exemplar 0.95, exemplar runner-up 0.80 (legacy margin 0.15 ≥ + /// 0.12 → the pre-fix gate silently auto-names), but on the AVERAGE the winner is 0.60 and the + /// runner-up 0.55 (avg margin 0.05 < 0.12). The retuned gate must withhold. + func testLuckyExemplarImpostorRejectedByAverageMargin() { + let profile = matureProfile("Alice", embedding: [1, 0, 0]) + + // Pre-fix behavior (regression witness): best-exemplar margin waves the impostor through. + XCTAssertTrue( + SpeakerNamingPolicy.shouldAutoAccept( + profile: profile, similarity: 0.95, secondBestSimilarity: 0.80), + "pre-fix best-exemplar margin should have auto-accepted (this is the regression)") + + // Fixed behavior: the collapsed average margin withholds the auto-accept. + XCTAssertFalse( + SpeakerNamingPolicy.shouldAutoAccept( + profile: profile, similarity: 0.95, secondBestSimilarity: 0.80, + marginSimilarities: (best: 0.60, secondBest: 0.55)), + "average-based margin (0.05) is below 0.12 → auto-accept must be withheld") + } + + /// The genuine owner: same 0.95 best exemplar and 0.80 runner-up, but close to the profile on + /// its AVERAGE too (0.93 vs 0.55 → avg margin 0.38). The retune must NOT cost this auto-name. + func testGenuineOwnerStillAutoAcceptedUnderAverageMargin() { + let profile = matureProfile("Alice", embedding: [1, 0, 0]) + XCTAssertTrue( + SpeakerNamingPolicy.shouldAutoAccept( + profile: profile, similarity: 0.95, secondBestSimilarity: 0.80, + marginSimilarities: (best: 0.93, secondBest: 0.55)), + "genuine owner is close on the average too → margin holds, auto-accept preserved") + } + + /// Legacy single-average profiles (no exemplars) pass nil and behave exactly as before, so the + /// change is a no-op off the exemplar path. + func testNilMarginPreservesLegacyBehavior() { + let profile = matureProfile("Alice", embedding: [1, 0, 0]) + XCTAssertTrue(SpeakerNamingPolicy.shouldAutoAccept( + profile: profile, similarity: 0.95, secondBestSimilarity: 0.80)) + XCTAssertFalse(SpeakerNamingPolicy.shouldAutoAccept( + profile: profile, similarity: 0.95, secondBestSimilarity: 0.90)) // margin 0.05 + XCTAssertTrue(SpeakerNamingPolicy.shouldAutoAccept( + profile: profile, similarity: 0.95, secondBestSimilarity: -1)) // no runner-up + } + + // MARK: - Full match path: matchAgainstProfiles computes the average margin + + /// Constructs the fixture as real profiles + embedding and drives the production matcher: the + /// impostor wins on a lucky exemplar (best-of-exemplars 1.0), a runner-up clears the floor on + /// exemplar (0.85 → legacy margin 0.15), yet both sit low on the average (0.60 vs 0.55). The + /// SnapshotMatchResult must carry the collapsed average margin, and the gate fed those values + /// must withhold — while the legacy call on the same result would auto-name. + func testMatchAgainstProfilesCarriesCollapsedAverageMargin() throws { + let candidate: [Float] = [1, 0, 0] + // Impostor target: far on the average (0.60), but one exemplar sits right on the candidate. + let alice = matureProfile("Alice", embedding: [0.6, 0.8, 0], exemplars: [[1, 0, 0]]) + // Runner-up: clears the 0.70 floor on its exemplar (0.85) but is low on the average (0.55). + let bob = matureProfile("Bob", embedding: [0.55, 0, 0.835], exemplars: [[0.85, 0.5268, 0]]) + + let match = try XCTUnwrap( + Transcription.matchAgainstProfiles(candidate, profiles: [alice, bob], threshold: 0.70)) + XCTAssertEqual(match.profileId, alice.id, "impostor wins match selection via the lucky exemplar") + XCTAssertEqual(match.similarity, 1.0, accuracy: 1e-4, "best-of-exemplars score clears the 0.92 bar") + XCTAssertEqual(match.secondBestSimilarity, 0.85, accuracy: 1e-4, "exemplar runner-up (legacy margin 0.15)") + XCTAssertEqual(match.averageSimilarity, 0.60, accuracy: 1e-3, "winner is far on the average") + XCTAssertEqual(match.secondBestAverageSimilarity, 0.55, accuracy: 1e-3, "runner-up is far on the average too") + + // Pre-fix gate (exemplar margin 1.0 − 0.85 = 0.15) auto-names the impostor. + XCTAssertTrue(SpeakerNamingPolicy.shouldAutoAccept( + profile: alice, similarity: match.similarity, secondBestSimilarity: match.secondBestSimilarity), + "legacy margin on the same match would silently mislabel") + + // Retuned gate (average margin 0.60 − 0.55 = 0.05) withholds. + XCTAssertFalse(SpeakerNamingPolicy.shouldAutoAccept( + profile: alice, similarity: match.similarity, secondBestSimilarity: match.secondBestSimilarity, + marginSimilarities: (best: match.averageSimilarity, secondBest: match.secondBestAverageSimilarity)), + "average-based margin collapses → the fix withholds the auto-name") + } + + /// The genuine owner through the same production matcher: high on both exemplar and average, so + /// the average margin (0.95 − 0.55) stays wide and the auto-accept survives the retune. + func testMatchAgainstProfilesGenuineOwnerStillAutoAccepts() throws { + let candidate: [Float] = [1, 0, 0] + let alice = matureProfile("Alice", embedding: [0.95, 0.3122, 0], exemplars: [[1, 0, 0]]) + let bob = matureProfile("Bob", embedding: [0.55, 0, 0.835], exemplars: [[0.85, 0.5268, 0]]) + + let match = try XCTUnwrap( + Transcription.matchAgainstProfiles(candidate, profiles: [alice, bob], threshold: 0.70)) + XCTAssertEqual(match.profileId, alice.id) + XCTAssertEqual(match.averageSimilarity, 0.95, accuracy: 1e-3) + XCTAssertEqual(match.secondBestAverageSimilarity, 0.55, accuracy: 1e-3) + + XCTAssertTrue(SpeakerNamingPolicy.shouldAutoAccept( + profile: alice, similarity: match.similarity, secondBestSimilarity: match.secondBestSimilarity, + marginSimilarities: (best: match.averageSimilarity, secondBest: match.secondBestAverageSimilarity)), + "genuine owner clears the average margin → auto-accept preserved") + } +} diff --git a/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift b/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift index 8fafe149..3f82f287 100644 --- a/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift +++ b/Tests/TranscriptedCoreTests/SpeakerExemplarDeltaEvalTests.swift @@ -199,15 +199,17 @@ final class SpeakerExemplarDeltaEvalTests: XCTestCase { /// Identification metrics for one arm at the certified operating point. private func scoreArm(profiles: [Profile], trials: [(gt: String, emb: [Float])], useExemplars: Bool) -> [String: Any] { var recallAtFloor = 0 - var autoCorrect = 0 // genuine, silently + correctly recognized + var autoCorrect = 0 // genuine, silently + correctly recognized (avg-margin gate) var falseAuto = 0 // WRONG identity silently auto-named (the dangerous case) + var autoCorrectLegacyMargin = 0 // same, under the pre-fix best-exemplar margin + var falseAutoLegacyMargin = 0 // WRONG auto-names under the pre-fix best-exemplar margin var impostorFAatFloor = 0 // any non-self profile clears the match floor var genuineSimSum = 0.0 var genuineSimN = 0 for trial in trials { // Score against every gallery profile (best rep for WITH, average for WITHOUT). - var best = (sim: -2.0, gt: "", cc: 0, ex: [[Float]]()) + var best = (sim: -2.0, gt: "", cc: 0, ex: [[Float]](), avg: [Float]()) var second = -2.0 var bestImpostor = -2.0 for p in profiles { @@ -217,25 +219,50 @@ final class SpeakerExemplarDeltaEvalTests: XCTestCase { if p.gt != trial.gt { bestImpostor = max(bestImpostor, sim) } if sim > best.sim { second = best.sim - best = (sim, p.gt, p.callCount, p.exemplars) + best = (sim, p.gt, p.callCount, p.exemplars, p.average) } else if sim > second { second = sim } } if bestImpostor >= matchFloor { impostorFAatFloor += 1 } + // Auto-accept margin decoupled from best-of-exemplars (the fix under test): the 0.92 + // bar is still checked against the best-exemplar `best.sim` (so the recall win holds), + // but the best-vs-second MARGIN is checked against the profile AVERAGE — winner's + // cosine-to-average vs the highest average cosine among the other profiles. A genuine + // owner is close on both, so it still clears; a lucky-exemplar impostor is far from the + // average, so its margin collapses and auto-accept is withheld. WITHOUT arm scores on + // the average already, so passing nil keeps it exactly the legacy single-average gate. + let marginSims: (best: Double, secondBest: Double)? + if useExemplars { + let winnerAvg = SpeakerVectorMath.cosineSimilarity(trial.emb, best.avg) + var secondAvg = -1.0 + for p in profiles where p.gt != best.gt { + secondAvg = max(secondAvg, SpeakerVectorMath.cosineSimilarity(trial.emb, p.average)) + } + marginSims = (best: winnerAvg, secondBest: secondAvg) + } else { + marginSims = nil + } + // Real auto-accept decision on the argmax profile (named + mature so eligibility holds // exactly when callCount > 4; margin/similarity gates come from production). let named = namedMatureProfile(gt: best.gt, callCount: best.cc, average: [], exemplars: best.ex) let autoAccept = SpeakerNamingPolicy.shouldAutoAccept( + profile: named, similarity: best.sim, secondBestSimilarity: second, + marginSimilarities: marginSims) + // Pre-fix behavior for the A/B: the same gate with the best-exemplar margin (nil). + let autoAcceptLegacyMargin = SpeakerNamingPolicy.shouldAutoAccept( profile: named, similarity: best.sim, secondBestSimilarity: second) if best.gt == trial.gt { genuineSimSum += best.sim; genuineSimN += 1 if best.sim >= matchFloor { recallAtFloor += 1 } if autoAccept { autoCorrect += 1 } - } else if autoAccept { - falseAuto += 1 + if autoAcceptLegacyMargin { autoCorrectLegacyMargin += 1 } + } else { + if autoAccept { falseAuto += 1 } + if autoAcceptLegacyMargin { falseAutoLegacyMargin += 1 } } } let n = max(1, trials.count) @@ -244,6 +271,11 @@ final class SpeakerExemplarDeltaEvalTests: XCTestCase { "autoCorrect": Double(autoCorrect) / Double(n), "falseAuto": Double(falseAuto) / Double(n), "falseAutoCount": falseAuto, + // Pre-fix best-exemplar margin, for the before/after within one artifact. Equal to the + // new counters in the WITHOUT arm (which never supplies an average margin). + "autoCorrectLegacyMargin": Double(autoCorrectLegacyMargin) / Double(n), + "falseAutoLegacyMargin": Double(falseAutoLegacyMargin) / Double(n), + "falseAutoLegacyMarginCount": falseAutoLegacyMargin, "impostorFAatFloor": Double(impostorFAatFloor) / Double(n), "meanGenuineSim": genuineSimN > 0 ? genuineSimSum / Double(genuineSimN) : 0, ] diff --git a/docs/speaker-eval-exemplar-delta-2026-07.after-fix.result.json b/docs/speaker-eval-exemplar-delta-2026-07.after-fix.result.json new file mode 100644 index 00000000..92955586 --- /dev/null +++ b/docs/speaker-eval-exemplar-delta-2026-07.after-fix.result.json @@ -0,0 +1,116 @@ +{ + "corpora" : [ + { + "corpus" : "ami", + "profiles" : 147, + "profilesWithExemplars" : 138, + "testQualities" : [ + "opus_8k", + "tel_g711", + "noisy_snr5", + "reverb", + "mp3_16" + ], + "trials" : 741, + "veto" : { + "confusablePairs" : 1176, + "ownerChecks" : 5773, + "ownerCollateralRate" : 0.021652520353369131, + "reMatchCount" : 698, + "reMatchRateLegacy" : 0.14794404408647732, + "replays" : 4718, + "sameCondition" : { + "reMatchRateLegacy" : 0.19757575757575757, + "replays" : 825, + "vetoedRate" : 0.37696969696969695 + }, + "vetoableShare" : 0.14709622721492158, + "vetoedAmongReMatch" : 0.45988538681948427, + "vetoedAndReMatchCount" : 321, + "vetoedCount" : 578, + "vetoedRate" : 0.122509537939805 + }, + "with" : { + "autoCorrect" : 0.070175438596491224, + "autoCorrectLegacyMargin" : 0.097165991902834009, + "falseAuto" : 0.0026990553306342779, + "falseAutoCount" : 2, + "falseAutoLegacyMargin" : 0.016194331983805668, + "falseAutoLegacyMarginCount" : 12, + "impostorFAatFloor" : 0.55330634278002699, + "meanGenuineSim" : 0.85824506285209812, + "recallAtFloor" : 0.65856950067476383 + }, + "without" : { + "autoCorrect" : 0.016194331983805668, + "autoCorrectLegacyMargin" : 0.016194331983805668, + "falseAuto" : 0, + "falseAutoCount" : 0, + "falseAutoLegacyMargin" : 0, + "falseAutoLegacyMarginCount" : 0, + "impostorFAatFloor" : 0.2874493927125506, + "meanGenuineSim" : 0.78487367151233289, + "recallAtFloor" : 0.54925775978407554 + } + }, + { + "corpus" : "voxceleb", + "profiles" : 30, + "profilesWithExemplars" : 30, + "testQualities" : [ + "opus_8k", + "tel_g711", + "noisy_snr5", + "reverb", + "mp3_16" + ], + "trials" : 1798, + "veto" : { + "confusablePairs" : 103, + "ownerChecks" : 6162, + "ownerCollateralRate" : 0.015579357351509251, + "reMatchCount" : 1545, + "reMatchRateLegacy" : 0.25423728813559321, + "replays" : 6077, + "sameCondition" : { + "reMatchRateLegacy" : 0.28331862312444839, + "replays" : 1133, + "vetoedRate" : 0.1209179170344219 + }, + "vetoableShare" : 0.045746256376501565, + "vetoedAmongReMatch" : 0.12297734627831715, + "vetoedAndReMatchCount" : 190, + "vetoedCount" : 249, + "vetoedRate" : 0.040974164883988813 + }, + "with" : { + "autoCorrect" : 0.00055617352614015572, + "autoCorrectLegacyMargin" : 0.00055617352614015572, + "falseAuto" : 0, + "falseAutoCount" : 0, + "falseAutoLegacyMargin" : 0, + "falseAutoLegacyMarginCount" : 0, + "impostorFAatFloor" : 0.4727474972191324, + "meanGenuineSim" : 0.65601019484345202, + "recallAtFloor" : 0.17408231368186874 + }, + "without" : { + "autoCorrect" : 0, + "autoCorrectLegacyMargin" : 0, + "falseAuto" : 0, + "falseAutoCount" : 0, + "falseAutoLegacyMargin" : 0, + "falseAutoLegacyMarginCount" : 0, + "impostorFAatFloor" : 0.39432703003337038, + "meanGenuineSim" : 0.61114929809089846, + "recallAtFloor" : 0.14071190211345941 + } + } + ], + "note" : "WITH = multi-exemplar bestSimilarity + negative veto (main). WITHOUT = single-average cosine, no veto (legacy).", + "operatingPoint" : { + "autoBar" : 0.92000000000000004, + "marginMin" : 0.12, + "matchFloor" : 0.69999999999999996 + } +} \ No newline at end of file diff --git a/docs/speaker-eval-exemplar-delta-2026-07.md b/docs/speaker-eval-exemplar-delta-2026-07.md index 1c51b3f6..496a0d52 100644 --- a/docs/speaker-eval-exemplar-delta-2026-07.md +++ b/docs/speaker-eval-exemplar-delta-2026-07.md @@ -1,5 +1,11 @@ # Speaker eval: multi-exemplar + negative-exemplar accuracy delta (2026-07) +> **Update (2026-07, retune shipped):** Recommendation #2 below is now implemented — the auto-accept +> **margin** is computed against each profile's blended *average* representative instead of its best +> exemplar, while match selection keeps best-of-exemplars. On the same degraded AMI slice this cuts +> the multi-exemplar false-auto count from **12 → 2** with the recall gain **fully retained** +> (0.659, unchanged). See [Retune shipped](#retune-shipped-2026-07--average-based-auto-accept-margin). + ## Verdict The two speaker-identity features that merged into `main` in 2026-07 were measured against the @@ -170,6 +176,57 @@ rep, not the best exemplar (see below). false-auto stays ~0 (consistent with the ladder's clean findings); the 1.6% is the degraded-slice upper bound, not the expected production rate. +## Retune shipped (2026-07) — average-based auto-accept margin + +Recommendation #2 shipped as the "lowest-risk option": the auto-accept **margin** is now judged +against each profile's blended *average* representative, decoupled from the best-of-exemplars score +that still drives match **selection** (and thus the recall win). `SpeakerMatchingService` +(`matchAgainstProfiles`) now carries, alongside the exemplar `similarity`/`secondBestSimilarity`, the +winner's average-only cosine and the highest average cosine among the other candidates. +`SpeakerNamingPolicy.shouldAutoAccept` gained a `marginSimilarities:` parameter: the 0.92 bar is still +checked against the best exemplar (preserving auto-recognition of returning voices), but the +best-vs-second **margin** is checked on the average. A genuine owner is close on *both* its average and +its best exemplar, so its margin still clears; a lucky-exemplar impostor is far from the average, so +its average-based margin collapses and the auto-accept is withheld (routed to suggest/ask). Legacy +single-average profiles pass `nil` and behave exactly as before. + +Re-run of this harness on the same corpus, WITH arm (multi-exemplar), AMI degraded slice — raw output +in [`speaker-eval-exemplar-delta-2026-07.after-fix.result.json`](speaker-eval-exemplar-delta-2026-07.after-fix.result.json): + +| metric | pre-fix (best-exemplar margin) | post-fix (average margin) | Δ | +|---|---:|---:|---:| +| **false-auto (silent mislabels)** @ (0.92, 0.12) | **12** (1.6%) | **2** (0.27%) | **−10** | +| genuine recall @ floor 0.70 | 0.659 | **0.659** | 0.000 | +| correct silent auto-recognition @ (0.92, 0.12) | 0.097 | 0.070 | −0.027 | +| mean genuine similarity | 0.858 | 0.858 | 0.000 | + +Recall is untouched by construction — the fix does not change match selection, the match floor, or +`bestSimilarity`, so `recallAtFloor` and `meanGenuineSim` are identical to the pre-fix WITH arm. The +harness reports both counters per run (`falseAutoLegacyMarginCount` = pre-fix, `falseAutoCount` = +post-fix) so the A/B is reproducible in one artifact. The WITHOUT (legacy single-average) arm and the +VoxCeleb slice are unchanged (0 false-auto in both arms; VoxCeleb never reaches the 0.92 bar on +degraded audio, so it has no auto-recognition to regress). + +**Residual 2 (characterized).** The two remaining false-autos are *not* lucky-exemplar artifacts: +their average-similarity to the wrong profile is genuinely high (`avgBest` 0.82 and 0.70) with a real +average margin (0.172 and 0.137) — distinct speakers who are genuinely confusable on the *blended +average*, the error class the average-only WITHOUT arm would also risk if its bar-clearing weren't +gated by the (lower) average score. They are irreducible at this operating point without harming the +genuine experience: probed tighteners on the same slice removed at most one of the two while costing +2.5×–15× as many *genuine* auto-recognitions as false-autos they eliminated — + +| lever | false-auto | genuine autos kept | +|---|---:|---:| +| average margin (shipped) | 2 | 52 | +| + raise margin 0.12→0.15 | 1 | 47 (−5) | +| + exemplar-vs-average gap ≤ 0.15 | 1 | 25 (−27) | +| + require avgBest ≥ 0.80 | 1 | 23 (−29) | + +So the shipped lever is the knee of the curve: it removes the entire lucky-exemplar class (10/10) at +zero recall and near-zero genuine-auto cost, and stops before trading the genuine experience away to +chase two irreducibly-confusable cases. The residual 0.27% remains a worst-case degraded-slice upper +bound (§ caveats), not an expected production rate. + ## Risks / caveats - **Fingerprint-level, not full-pipeline.** The eval operates on per-(speaker, meeting) mean @@ -192,4 +249,8 @@ rep, not the best exemplar (see below). - `swift test --filter SpeakerNamingSimulationRunnerTests` — 7/7 pass (existing functional regression; confirms the Core test target builds + runs with the exemplar features on `main`). - `swift test --filter SpeakerExemplarDeltaEvalTests` — passes (this eval; ~3.4 s), writes - `docs/speaker-eval-exemplar-delta-2026-07.result.json`. + `docs/speaker-eval-exemplar-delta-2026-07.result.json` (pre-fix) and, re-run after the retune, + `docs/speaker-eval-exemplar-delta-2026-07.after-fix.result.json` (12 → 2 false-auto, recall held). +- `swift test --filter SpeakerAutoAcceptMarginTests` — corpus-free, in-tree proof of the retune: + constructs the lucky-exemplar impostor fixture and asserts the average-based margin withholds it + while a genuine owner still auto-accepts (both at the policy gate and through `matchAgainstProfiles`).