Skip to content

fix(speaker): average-based auto-accept margin — restore 0-false-auto without losing the multi-exemplar recall win #1496

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
r3dbars merged 1 commit into from
Jul 7, 2026
Merged

fix(speaker): average-based auto-accept margin — restore 0-false-auto without losing the multi-exemplar recall win #1496

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion Sources/TranscriptedCore/Models/TranscriptionTypes.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,19 +351,30 @@ public struct ChannelSpeakerContext: Sendable {
/// Similarity of the next-closest profile that cleared the match floor (the runner-up),
/// or nil if unknown / no runner-up. Feeds `SpeakerNamingPolicy`'s auto-accept margin guard.
public let matchSecondSimilarity: Double?
/// Winner's cosine to its blended *average* representative (exemplars excluded), and the
/// highest average cosine among the other candidate profiles. Feed
/// `SpeakerNamingPolicy.shouldAutoAccept`'s `marginSimilarities` so the auto-accept margin is
/// computed on the average, not the best exemplar (see
/// docs/speaker-eval-exemplar-delta-2026-07.md). nil on paths that don't carry a match result.
public let matchAverageSimilarity: Double?
public let matchSecondBestAverageSimilarity: Double?

public init(
persistentSpeakerId: UUID,
sessionEmbedding: [Float]?,
matchedProfileSnapshot: SpeakerProfile?,
matchSimilarity: Double?,
matchSecondSimilarity: Double? = nil
matchSecondSimilarity: Double? = nil,
matchAverageSimilarity: Double? = nil,
matchSecondBestAverageSimilarity: Double? = nil
) {
self.persistentSpeakerId = persistentSpeakerId
self.sessionEmbedding = sessionEmbedding
self.matchedProfileSnapshot = matchedProfileSnapshot
self.matchSimilarity = matchSimilarity
self.matchSecondSimilarity = matchSecondSimilarity
self.matchAverageSimilarity = matchAverageSimilarity
self.matchSecondBestAverageSimilarity = matchSecondBestAverageSimilarity
}
}

Expand Down
16 changes: 10 additions & 6 deletions Sources/TranscriptedCore/Pipeline/TranscriptionPipeline.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ extension Transcription {

// Match each speaker's mean embedding against the DB once
// (existingProfiles was already snapshotted above for post-processing)
var speakerMatchResults: [Int: (persistentId: UUID, similarity: Double, secondSimilarity: Double)] = [:]
var speakerMatchResults: [Int: (persistentId: UUID, similarity: Double, secondSimilarity: Double, averageSimilarity: Double, secondAverageSimilarity: Double)] = [:]
var speakerNewProfiles: [Int: UUID] = [:]
var speakerIdRemap: [Int: Int] = [:]
// Session mean embedding actually matched for each non-ghost speaker, kept so the
Expand Down Expand Up @@ -320,7 +320,7 @@ extension Transcription {
// the shared profile. Matching reads only the `existingProfiles` snapshot, so
// deferring the blend cannot change any match decision.
if let matchResult = Self.matchAgainstProfiles(meanEmbedding, profiles: existingProfiles, threshold: adaptiveThreshold, negativeExemplarsByProfile: negativeExemplarsByProfile) {
speakerMatchResults[speakerId] = (matchResult.profileId, matchResult.similarity, matchResult.secondBestSimilarity)
speakerMatchResults[speakerId] = (matchResult.profileId, matchResult.similarity, matchResult.secondBestSimilarity, matchResult.averageSimilarity, matchResult.secondBestAverageSimilarity)
let matchedProfile = existingProfiles.first(where: { $0.id == matchResult.profileId })
AppLogger.transcription.info("Speaker matched DB profile", [
"speakerId": "\(speakerId)",
Expand Down Expand Up @@ -440,7 +440,9 @@ extension Transcription {
sessionEmbedding: sessionEmbedding,
matchedProfileSnapshot: matchedProfileSnapshot,
matchSimilarity: speakerMatchResults[effectiveSpeakerId]?.similarity,
matchSecondSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondSimilarity
matchSecondSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondSimilarity,
matchAverageSimilarity: speakerMatchResults[effectiveSpeakerId]?.averageSimilarity,
matchSecondBestAverageSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondAverageSimilarity
)
}

Expand Down Expand Up @@ -890,7 +892,7 @@ extension Transcription {
}
}

var speakerMatchResults: [Int: (persistentId: UUID, similarity: Double, secondSimilarity: Double)] = [:]
var speakerMatchResults: [Int: (persistentId: UUID, similarity: Double, secondSimilarity: Double, averageSimilarity: Double, secondAverageSimilarity: Double)] = [:]
var speakerNewProfiles: [Int: UUID] = [:]
var speakerIdRemap: [Int: Int] = [:]
var newlyCreatedProfileIds: Set<UUID> = []
Expand Down Expand Up @@ -930,7 +932,7 @@ extension Transcription {
// deferring the blend changes no match decision, and a spun-off distinct voice never
// contaminates the shared profile).
if let matchResult = Self.matchAgainstProfiles(meanEmbedding, profiles: existingProfiles, threshold: adaptiveThreshold, negativeExemplarsByProfile: negativeExemplarsByProfile) {
speakerMatchResults[speakerId] = (matchResult.profileId, matchResult.similarity, matchResult.secondBestSimilarity)
speakerMatchResults[speakerId] = (matchResult.profileId, matchResult.similarity, matchResult.secondBestSimilarity, matchResult.averageSimilarity, matchResult.secondBestAverageSimilarity)
} else {
let newProfile = speakerDB.addOrUpdateSpeaker(embedding: meanEmbedding, existingId: nil)
speakerNewProfiles[speakerId] = newProfile.id
Expand Down Expand Up @@ -997,7 +999,9 @@ extension Transcription {
sessionEmbedding: sessionEmbedding,
matchedProfileSnapshot: matchedProfileSnapshot,
matchSimilarity: speakerMatchResults[effectiveSpeakerId]?.similarity,
matchSecondSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondSimilarity
matchSecondSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondSimilarity,
matchAverageSimilarity: speakerMatchResults[effectiveSpeakerId]?.averageSimilarity,
matchSecondBestAverageSimilarity: speakerMatchResults[effectiveSpeakerId]?.secondAverageSimilarity
)
}

Expand Down
32 changes: 26 additions & 6 deletions Sources/TranscriptedCore/Pipeline/TranscriptionPipelineRunner.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,18 @@ extension TranscriptionTaskManager {
/// Runner-up similarity for the auto-accept margin guard; nil when unknown (the
/// utterance fallback path), which `shouldAutoAccept` treats as "confirm, don't auto".
let secondSimilarity: Double?
/// Winner's average-only cosine and the runner-up's average-only cosine, when carried by
/// the match context. Feed `SpeakerNamingPolicy.shouldAutoAccept`'s `marginSimilarities`
/// so the auto-accept margin is judged on the blended average, not the best exemplar.
let averageSimilarity: Double?
let secondAverageSimilarity: Double?

/// Average-based margin pair for `shouldAutoAccept`, or nil to fall back to the legacy
/// (exemplar) margin when the average sims weren't carried (e.g. utterance fallback).
var marginSimilarities: (best: Double, secondBest: Double)? {
guard let best = averageSimilarity, let second = secondAverageSimilarity else { return nil }
return (best: best, secondBest: second)
}
}

nonisolated static func speakerClassificationKnowledge(
Expand All @@ -31,7 +43,9 @@ extension TranscriptionTaskManager {
speakerId: sid,
profile: snapshot,
similarity: similarity,
secondSimilarity: context.matchSecondSimilarity
secondSimilarity: context.matchSecondSimilarity,
averageSimilarity: context.matchAverageSimilarity,
secondAverageSimilarity: context.matchSecondBestAverageSimilarity
))
continue
}
Expand All @@ -44,7 +58,9 @@ extension TranscriptionTaskManager {
speakerId: sid,
profile: profile,
similarity: similarity,
secondSimilarity: nil // runner-up not carried on the utterance fallback
secondSimilarity: nil, // runner-up not carried on the utterance fallback
averageSimilarity: nil,
secondAverageSimilarity: nil
))
}
}
Expand Down Expand Up @@ -330,7 +346,8 @@ extension TranscriptionTaskManager {
profile: entry.profile,
similarity: entry.similarity,
secondBestSimilarity: entry.secondSimilarity,
recentOutcomes: cachedRecentOutcomes(entry.profile)
recentOutcomes: cachedRecentOutcomes(entry.profile),
marginSimilarities: entry.marginSimilarities
)
if canAutoAccept {
autoAcceptedIds.insert(sid)
Expand Down Expand Up @@ -358,7 +375,8 @@ extension TranscriptionTaskManager {
profile: entry.profile,
similarity: entry.similarity,
secondBestSimilarity: entry.secondSimilarity,
recentOutcomes: recentOutcomesByProfile[entry.profile.id] ?? []
recentOutcomes: recentOutcomesByProfile[entry.profile.id] ?? [],
marginSimilarities: entry.marginSimilarities
)
speakerMappings[key] = mapping
speakerSources[key] = autoAcceptedIds.contains(entry.speakerId) ? "db" : "db_pending"
Expand Down Expand Up @@ -403,7 +421,8 @@ extension TranscriptionTaskManager {
profile: entry.profile,
similarity: entry.similarity,
secondBestSimilarity: entry.secondSimilarity,
recentOutcomes: cachedRecentOutcomes(entry.profile)
recentOutcomes: cachedRecentOutcomes(entry.profile),
marginSimilarities: entry.marginSimilarities
)
if canAutoAccept {
micAutoAcceptedIds.insert(sid)
Expand Down Expand Up @@ -435,7 +454,8 @@ extension TranscriptionTaskManager {
profile: entry.profile,
similarity: entry.similarity,
secondBestSimilarity: entry.secondSimilarity,
recentOutcomes: recentOutcomesByProfile[entry.profile.id] ?? []
recentOutcomes: recentOutcomesByProfile[entry.profile.id] ?? [],
marginSimilarities: entry.marginSimilarities
)
speakerMappings[key] = mapping
speakerSources["mic_\(entry.speakerId)"] = micAutoAcceptedIds.contains(entry.speakerId) ? "db" : "db_pending"
Expand Down
36 changes: 33 additions & 3 deletions Sources/TranscriptedCore/Speaker/SpeakerMatchingService.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,18 @@ extension Transcription {
let profileId: UUID
let similarity: Double
/// Similarity of the next-closest non-disputed profile that cleared the floor, or -1
/// if there was no runner-up. Used by `SpeakerNamingPolicy.shouldAutoAccept` as the
/// "clear winner" margin guard.
/// if there was no runner-up. Best-of-representatives (exemplar) score; still used for
/// write-back blend gating and logging.
let secondBestSimilarity: Double
/// Winner's cosine to its *blended average only* (exemplars excluded). Top of the
/// average-based auto-accept margin (`SpeakerNamingPolicy.shouldAutoAccept`
/// `marginSimilarities`). Equal to `similarity` for legacy single-average profiles.
let averageSimilarity: Double
/// Highest average-only cosine among the OTHER candidate profiles (the runner-up on the
/// average representation), or -1 if there was none. Bottom of the average-based margin.
/// Decoupling the margin from the best exemplar keeps a lucky-exemplar impostor from
/// clearing the auto-accept gate — see docs/speaker-eval-exemplar-delta-2026-07.md.
let secondBestAverageSimilarity: Double
}

// MARK: - Embedding Utilities
Expand Down Expand Up @@ -61,6 +70,11 @@ extension Transcription {
var bestProfile: SpeakerProfile?
var bestSimilarity: Double = -1
var secondBestSimilarity: Double = -1
// Average-only cosine for every candidate profile (survives dispute/dim/veto gating),
// used to build the average-based auto-accept margin after the winner is chosen. Kept
// separate from `bestSimilarity` (best-of-exemplars) so the recall win from exemplar
// scoring is untouched while the margin is computed against the blended average.
var averageSimilarityByProfile: [UUID: Double] = [:]

for profile in profiles {
// A disputed profile was explicitly rejected by the user; don't
Expand Down Expand Up @@ -88,6 +102,10 @@ extension Transcription {
])
continue
}
// Record the average-only cosine for the margin computation (all non-vetoed candidates,
// not just floor-clearers, so the runner-up on the average representation is captured
// even when it didn't clear the exemplar floor).
averageSimilarityByProfile[profile.id] = SpeakerVectorMath.cosineSimilarity(embedding, profile.embedding)
if similarity >= threshold {
if similarity > bestSimilarity {
secondBestSimilarity = bestSimilarity
Expand Down Expand Up @@ -129,8 +147,20 @@ extension Transcription {
return nil
}

// Average-based margin: the winner's cosine to its blended average vs the highest average
// cosine among the other candidates. A genuine owner is close on both average and best
// exemplar (margin holds); a lucky-exemplar impostor is far from the average (margin
// collapses), so `SpeakerNamingPolicy.shouldAutoAccept` withholds the silent auto-name.
let averageSimilarity = averageSimilarityByProfile[matched.id] ?? bestSimilarity
var secondBestAverageSimilarity = -1.0
for (id, avg) in averageSimilarityByProfile where id != matched.id {
secondBestAverageSimilarity = max(secondBestAverageSimilarity, avg)
}

return SnapshotMatchResult(profileId: matched.id, similarity: bestSimilarity,
secondBestSimilarity: secondBestSimilarity)
secondBestSimilarity: secondBestSimilarity,
averageSimilarity: averageSimilarity,
secondBestAverageSimilarity: secondBestAverageSimilarity)
}

// MARK: - Cross-Cluster Link/Merge (#8)
Expand Down
38 changes: 31 additions & 7 deletions Sources/TranscriptedCore/Speaker/SpeakerNamingPolicy.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,21 +39,41 @@ public enum SpeakerNamingPolicy {
) == .trusted
}

/// - Parameters:
/// - similarity: best-of-representatives similarity (blended average OR any stored
/// multi-exemplar voiceprint, `SpeakerVectorMath.bestSimilarity`). Checked against the
/// 0.92 bar so the multi-exemplar recall win is preserved on degraded audio.
/// - secondBestSimilarity: legacy runner-up similarity used for the margin when
/// `marginSimilarities` is not supplied (and the sentinel: `< 0` ⇒ no runner-up, `nil` ⇒
/// unknown ⇒ conservative).
/// - marginSimilarities: when supplied, the best-vs-second **margin** is computed against
/// each profile's *average* representative (`best` = winner's cosine to its blended
/// average, `secondBest` = highest average cosine among the other candidates, `< 0` ⇒
/// none). Decoupling the margin from the best-exemplar score restores the "0 false-auto"
/// guarantee: a genuine owner is close on BOTH its average and its best exemplar so the
/// margin still holds, while an impostor that only cleared the 0.92 bar via one lucky
/// exemplar is far from the average, so its average-based margin collapses and auto-accept
/// is withheld (routed to suggest/ask). Legacy callers and single-average profiles (where
/// average == best exemplar) pass `nil` and behave exactly as before. See
/// `docs/speaker-eval-exemplar-delta-2026-07.md`.
public static func shouldAutoAccept(
profile: SpeakerProfile,
similarity: Double,
secondBestSimilarity: Double?
secondBestSimilarity: Double?,
marginSimilarities: (best: Double, secondBest: Double)? = nil
) -> Bool {
let marginTop = marginSimilarities?.best ?? similarity
let marginRunnerUp: Double? = marginSimilarities.map { $0.secondBest } ?? secondBestSimilarity
let marginOK: Bool
switch secondBestSimilarity {
switch marginRunnerUp {
case .none:
// Runner-up unknown (e.g. a fallback path that didn't carry it) — be conservative
// and route to confirm rather than silently auto-name.
marginOK = false
case .some(let second) where second < 0:
marginOK = true // no confusable runner-up cleared the match floor
case .some(let second):
marginOK = (similarity - second) >= autoAcceptMarginMin
marginOK = (marginTop - second) >= autoAcceptMarginMin
}
return isAutoRecognizable(profile: profile, recentOutcomes: [])
&& similarity > autoAcceptSimilarityThreshold
Expand All @@ -69,15 +89,17 @@ public enum SpeakerNamingPolicy {
profile: SpeakerProfile,
similarity: Double,
secondBestSimilarity: Double?,
recentOutcomes: [SpeakerMatchOutcomeKind]
recentOutcomes: [SpeakerMatchOutcomeKind],
marginSimilarities: (best: Double, secondBest: Double)? = nil
) -> Bool {
guard isAutoRecognizable(profile: profile, recentOutcomes: recentOutcomes) else {
return false
}
return shouldAutoAccept(
profile: profile,
similarity: similarity,
secondBestSimilarity: secondBestSimilarity
secondBestSimilarity: secondBestSimilarity,
marginSimilarities: marginSimilarities
)
}

Expand All @@ -90,13 +112,15 @@ public enum SpeakerNamingPolicy {
profile: SpeakerProfile,
similarity: Double,
secondBestSimilarity: Double?,
recentOutcomes: [SpeakerMatchOutcomeKind] = []
recentOutcomes: [SpeakerMatchOutcomeKind] = [],
marginSimilarities: (best: Double, secondBest: Double)? = nil
) -> SpeakerMapping {
guard shouldAutoAccept(
profile: profile,
similarity: similarity,
secondBestSimilarity: secondBestSimilarity,
recentOutcomes: recentOutcomes
recentOutcomes: recentOutcomes,
marginSimilarities: marginSimilarities
),
let name = profile.displayName,
!name.isEmpty else {
Expand Down
Loading
Loading