diff --git a/WindowTranslator.Abstractions/Modules/IOcrModule.cs b/WindowTranslator.Abstractions/Modules/IOcrModule.cs index 743ad060..13b68cad 100644 --- a/WindowTranslator.Abstractions/Modules/IOcrModule.cs +++ b/WindowTranslator.Abstractions/Modules/IOcrModule.cs @@ -94,22 +94,4 @@ public class BasicOcrParam : IPluginParam [Category("MergeThrethold")] public bool IsAvoidMergeList { get; set; } = false; - /// - /// バッファサイズ - /// - [Category("Buffer")] - [Spinnable] - public int BufferSize { get; set; } = 3; - - /// - /// フォントサイズの振動を抑制する - /// - [Category("Buffer")] - public bool IsSuppressVibe { get; set; } = true; - - /// - /// 復元を有効にするかどうか - /// - [Category("Buffer")] - public bool IsEnableRecover { get; set; } = true; } diff --git a/WindowTranslator.Abstractions/Properties/Resources.Designer.cs b/WindowTranslator.Abstractions/Properties/Resources.Designer.cs index 70baa10b..b172bf8d 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.Designer.cs +++ b/WindowTranslator.Abstractions/Properties/Resources.Designer.cs @@ -72,16 +72,6 @@ internal Resources() { /// public static string Brightness => ResourceManager.GetString("Brightness", resourceCulture) ?? string.Empty; - /// - /// "obt@@\" ɗގĂ郍[JCYꂽ܂B - /// - public static string Buffer => ResourceManager.GetString("Buffer", resourceCulture) ?? string.Empty; - - /// - /// "obt@TCY" ɗގĂ郍[JCYꂽ܂B - /// - public static string BufferSize => ResourceManager.GetString("BufferSize", resourceCulture) ?? string.Empty; - /// /// "LbVW[" ɗގĂ郍[JCYꂽ܂B /// @@ -112,16 +102,6 @@ internal Resources() { /// public static string IsAvoidMergeList => ResourceManager.GetString("IsAvoidMergeList", resourceCulture) ?? string.Empty; - /// - /// "F~X̕␳" ɗގĂ郍[JCYꂽ܂B - /// - public static string IsEnableRecover => ResourceManager.GetString("IsEnableRecover", resourceCulture) ?? string.Empty; - - /// - /// "U̗}" ɗގĂ郍[JCYꂽ܂B - /// - public static string IsSuppressVibe => ResourceManager.GetString("IsSuppressVibe", resourceCulture) ?? string.Empty; - /// /// "s臒l" ɗގĂ郍[JCYꂽ܂B /// diff --git a/WindowTranslator.Abstractions/Properties/Resources.ar.resx b/WindowTranslator.Abstractions/Properties/Resources.ar.resx index af413ccd..3208883f 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.ar.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.ar.resx @@ -104,22 +104,10 @@ تجنب دمج القائمة - - حجم المخزن المؤقت - إعدادات OCR الأساسية أخرى - - وظيفة المخزن المؤقت - - - تصحيح أخطاء التعرف - - - قمع الاهتزازات - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.cs.resx b/WindowTranslator.Abstractions/Properties/Resources.cs.resx index 6f904362..1f02740c 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.cs.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.cs.resx @@ -162,24 +162,12 @@ Vyhnout se slučování seznamů - - Velikost vyrovnávací paměti - Základní nastavení OCR Ostatní - - Funkce vyrovnávací paměti - - - Oprava chyb rozpoznávání - - - Potlačení chvění - Nastavení jazyka diff --git a/WindowTranslator.Abstractions/Properties/Resources.de.resx b/WindowTranslator.Abstractions/Properties/Resources.de.resx index f596f986..557324f0 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.de.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.de.resx @@ -162,22 +162,10 @@ Vermeiden Sie das Zusammenführen von Listen - - Puffergröße - Grundlegende OCR-Einstellungen Sonstiges - - Pufferfunktion - - - Erkennungsfehlerkorrektur - - - Vibrationsunterdrückung - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.en.resx b/WindowTranslator.Abstractions/Properties/Resources.en.resx index 7696f6a6..52cdbffb 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.en.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.en.resx @@ -162,22 +162,10 @@ Avoid merging lists - - Buffer size - Basic OCR Setup Other - - Buffer function - - - Recognition error correction - - - Vibration suppression - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.es.resx b/WindowTranslator.Abstractions/Properties/Resources.es.resx index be99d709..0447bb74 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.es.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.es.resx @@ -104,22 +104,10 @@ Evitar fusión de lista - - Tamaño del búfer - Configuración básica de OCR Otros - - Función de búfer - - - Corrección de errores de reconocimiento - - - Supresión de vibraciones - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.fa.resx b/WindowTranslator.Abstractions/Properties/Resources.fa.resx index 8537365a..56ebb681 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.fa.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.fa.resx @@ -104,22 +104,10 @@ جلوگیری از ادغام لیست - - اندازه بافر - تنظیمات پایه OCR متفرقه - - قابلیت بافر - - - تصحیح خطای تشخیص - - - جلوگیری از لرزش - diff --git a/WindowTranslator.Abstractions/Properties/Resources.fil.resx b/WindowTranslator.Abstractions/Properties/Resources.fil.resx index 1d2cc9ff..1e955451 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.fil.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.fil.resx @@ -104,24 +104,12 @@ Mga Karakter na Iwasan sa Pagsama - - Laki ng Buffer - Mga Basic na Parameter ng OCR Iba pa - - Pag-andar ng Buffer - - - Pagwawasto ng Mali sa Pagkilala - - - Pagsupil ng Panginginig - Mga Setting ng Wika diff --git a/WindowTranslator.Abstractions/Properties/Resources.fr.resx b/WindowTranslator.Abstractions/Properties/Resources.fr.resx index 40496e5c..7eb50e2d 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.fr.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.fr.resx @@ -104,22 +104,10 @@ Éviter la fusion de liste - - Taille du tampon - Paramètres OCR de base Autres - - Fonction de tampon - - - Correction des erreurs de reconnaissance - - - Suppression des vibrations - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.hi.resx b/WindowTranslator.Abstractions/Properties/Resources.hi.resx index c9535152..46f3a27c 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.hi.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.hi.resx @@ -104,22 +104,10 @@ सूची को मर्ज करने से बचें - - बफर साइज़ - बुनियादी OCR सेटअप अन्य - - बफर फ़ंक्शन - - - पहचान त्रुटि सुधार - - - कंपन दमन - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.hu.resx b/WindowTranslator.Abstractions/Properties/Resources.hu.resx index 84ff6caa..768144d8 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.hu.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.hu.resx @@ -49,24 +49,12 @@ Listák összevonásának elkerülése - - Puffer mérete - Alapvető OCR beállítások Egyéb - - Puffer funkció - - - Felismerési hibák javítása - - - Rezgéselnyomás - Nyelvi beállítások diff --git a/WindowTranslator.Abstractions/Properties/Resources.id.resx b/WindowTranslator.Abstractions/Properties/Resources.id.resx index c735ec97..5d58d378 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.id.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.id.resx @@ -104,22 +104,10 @@ Hindari penggabungan daftar - - Ukuran buffer - Pengaturan OCR dasar Lainnya - - Fungsi buffer - - - Koreksi kesalahan pengenalan - - - Penekanan getaran - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.ko.resx b/WindowTranslator.Abstractions/Properties/Resources.ko.resx index 5044b495..37a8e32a 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.ko.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.ko.resx @@ -162,22 +162,10 @@ 병합 목록을 피하십시오 - - 버퍼 크기 - 기본 OCR 설정 기타 - - 버퍼 기능 - - - 인식 오류 보정 - - - 진동 억제 - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.ms.resx b/WindowTranslator.Abstractions/Properties/Resources.ms.resx index 66891355..1c414402 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.ms.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.ms.resx @@ -104,22 +104,10 @@ Elakkan penggabungan senarai - - Saiz penimbal - Tetapan OCR asas Lain-lain - - Fungsi penimbal - - - Pembetulan ralat pengecaman - - - Penindasan getaran - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.pl.resx b/WindowTranslator.Abstractions/Properties/Resources.pl.resx index 6fc64c50..4f528c4b 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.pl.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.pl.resx @@ -162,24 +162,12 @@ Unikanie scalania list - - Rozmiar bufora - Podstawowe ustawienia OCR Inne - - Funkcja bufora - - - Korekta błędów rozpoznawania - - - Tłumienie drgań - Ustawienia języka diff --git a/WindowTranslator.Abstractions/Properties/Resources.pt-BR.resx b/WindowTranslator.Abstractions/Properties/Resources.pt-BR.resx index 2ecc2d78..4ae33922 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.pt-BR.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.pt-BR.resx @@ -104,22 +104,10 @@ Evitar mesclagem de listas - - Tamanho do buffer - Configuração básica de OCR Outros - - Função do buffer - - - Correção de erros de reconhecimento - - - Supressão de vibração - diff --git a/WindowTranslator.Abstractions/Properties/Resources.resx b/WindowTranslator.Abstractions/Properties/Resources.resx index d7abd701..14ed80dc 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.resx @@ -162,24 +162,12 @@ リストのマージを避ける - - バッファサイズ - 基本的なOCRの設定 その他 - - バッファ機能 - - - 認識ミスの補正 - - - 振動の抑制 - 言語設定 diff --git a/WindowTranslator.Abstractions/Properties/Resources.ru.resx b/WindowTranslator.Abstractions/Properties/Resources.ru.resx index 22302086..10836a2c 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.ru.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.ru.resx @@ -104,24 +104,12 @@ Символы, которых следует избегать при объединении - - Размер буфера - Основные параметры OCR Прочее - - Функция буфера - - - Исправление ошибок распознавания - - - Подавление вибрации - Настройки языка diff --git a/WindowTranslator.Abstractions/Properties/Resources.th.resx b/WindowTranslator.Abstractions/Properties/Resources.th.resx index 275379b4..6fd287c5 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.th.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.th.resx @@ -104,24 +104,12 @@ อักขระที่ต้องหลีกเลี่ยงการรวม - - ขนาดบัฟเฟอร์ - พารามิเตอร์ OCR พื้นฐาน อื่นๆ - - ฟังก์ชันบัฟเฟอร์ - - - การแก้ไขข้อผิดพลาดในการรู้จำ - - - การระงับการสั่นสะเทือน - การตั้งค่าภาษา diff --git a/WindowTranslator.Abstractions/Properties/Resources.tr.resx b/WindowTranslator.Abstractions/Properties/Resources.tr.resx index 3ac7be6b..f18ab918 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.tr.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.tr.resx @@ -104,24 +104,12 @@ Birleştirmeden Kaçınılacak Karakterler - - Tampon Boyutu - Temel OCR Parametreleri Diğer - - Tampon İşlevi - - - Tanıma Hatası Düzeltmesi - - - Titreşim Bastırma - Dil Ayarları diff --git a/WindowTranslator.Abstractions/Properties/Resources.vi.resx b/WindowTranslator.Abstractions/Properties/Resources.vi.resx index 53ef277f..a05765bf 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.vi.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.vi.resx @@ -162,22 +162,10 @@ Tránh danh sách hợp nhất - - Kích thước bộ đệm - 基本的なOCRの設定 người khác - - Chức năng bộ đệm - - - Hiệu chỉnh lỗi nhận dạng - - - Ngăn chặn rung động - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.zh-CN.resx b/WindowTranslator.Abstractions/Properties/Resources.zh-CN.resx index 93ba90b8..93a29a8d 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.zh-CN.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.zh-CN.resx @@ -162,22 +162,10 @@ 避免合并列表 - - 缓冲区大小 - 基本 OCR 设置 其他 - - 缓冲功能 - - - 识别错误纠正 - - - 振动抑制 - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/Properties/Resources.zh-TW.resx b/WindowTranslator.Abstractions/Properties/Resources.zh-TW.resx index e1eb6d88..7f54a9bb 100644 --- a/WindowTranslator.Abstractions/Properties/Resources.zh-TW.resx +++ b/WindowTranslator.Abstractions/Properties/Resources.zh-TW.resx @@ -162,22 +162,10 @@ 避免合併列表 - - 緩衝區大小 - 基本 OCR 設定 其他 - - 緩衝功能 - - - 識別錯誤修正 - - - 振動抑制 - \ No newline at end of file diff --git a/WindowTranslator.Abstractions/UserSettings.cs b/WindowTranslator.Abstractions/UserSettings.cs index 5e1e8166..244405b1 100644 --- a/WindowTranslator.Abstractions/UserSettings.cs +++ b/WindowTranslator.Abstractions/UserSettings.cs @@ -82,11 +82,6 @@ public class TargetSettings /// public bool DisplayBusy { get; set; } = true; - /// - /// ワンショットモードを有効にするか（ユーザー操作の初回のみ処理） - /// - public bool IsOneShotMode { get; set; } - /// /// マウスポインター判定の余白（WPF上のピクセル値） /// diff --git a/WindowTranslator.Tests/OcrTextTrackerAccuracyTests.cs b/WindowTranslator.Tests/OcrTextTrackerAccuracyTests.cs new file mode 100644 index 00000000..3d47d3b8 --- /dev/null +++ b/WindowTranslator.Tests/OcrTextTrackerAccuracyTests.cs @@ -0,0 +1,1366 @@ +using System.Diagnostics; +using System.Drawing; +using Microsoft.Extensions.Logging.Abstractions; +using Quickenshtein; +using WindowTranslator.Modules.Ocr; +using Xunit.Abstractions; + +namespace WindowTranslator.Tests; + +public class OcrTextTrackerAccuracyTests(ITestOutputHelper output) +{ + [Fact] + public void TrackingImprovesAccuracyOverTheExistingBufferFilter() + { + IReadOnlyList> legacyFrames = RunLegacyBuffer(); + IReadOnlyList> trackedFrames = RunTracker(); + + double legacyAccuracy = OcrTrackingAccuracyScenarios.Measure(legacyFrames); + double trackingAccuracy = OcrTrackingAccuracyScenarios.Measure(trackedFrames); + output.WriteLine($"OcrBufferFilter baseline accuracy: {legacyAccuracy:P2}"); + output.WriteLine($"OcrTextTracker accuracy: {trackingAccuracy:P2}"); + + Assert.InRange(legacyAccuracy, 0.8005, 0.8007); + Assert.True(trackingAccuracy >= 0.95, $"Tracking accuracy was {trackingAccuracy:P2}."); + Assert.True(trackingAccuracy - legacyAccuracy >= 0.14, + $"Expected at least a 14 point improvement, but measured {(trackingAccuracy - legacyAccuracy):P2}."); + } + + [Fact] + public void TrackingImprovesAccuracyAcrossSeededRandomizedTraces() + { + List legacyAccuracies = []; + List trackingAccuracies = []; + foreach (int seed in RandomizedOcrTrackingAccuracyScenarios.Seeds) + { + RandomizedScenario scenario = RandomizedOcrTrackingAccuracyScenarios.Create(seed); + LegacyOcrBufferModel legacy = new(); + OcrTextTracker tracker = new(NullLogger.Instance); + List> legacyFrames = []; + List> trackedFrames = []; + for (int frame = 0; frame < scenario.Observations.Count; frame++) + { + IReadOnlyList observations = scenario.Observations[frame]; + legacyFrames.Add(legacy.Update(observations, scenario.ImageSize)); + trackedFrames.Add(tracker.Update( + observations, + scenario.ImageSize, + TimeSpan.FromMilliseconds(frame * 350))); + } + + double legacyAccuracy = RandomizedOcrTrackingAccuracyScenarios.Measure( + scenario.Expected, + legacyFrames); + double trackingAccuracy = RandomizedOcrTrackingAccuracyScenarios.Measure( + scenario.Expected, + trackedFrames); + legacyAccuracies.Add(legacyAccuracy); + trackingAccuracies.Add(trackingAccuracy); + output.WriteLine( + $"Seed {seed}: OcrBufferFilter {legacyAccuracy:P2}, OcrTextTracker {trackingAccuracy:P2}"); + Assert.True(trackingAccuracy - legacyAccuracy >= 0.08, + $"Seed {seed} improved by only {(trackingAccuracy - legacyAccuracy):P2}."); + } + + double averageLegacyAccuracy = legacyAccuracies.Average(); + double averageTrackingAccuracy = trackingAccuracies.Average(); + output.WriteLine( + $"Randomized average: OcrBufferFilter {averageLegacyAccuracy:P2}, " + + $"OcrTextTracker {averageTrackingAccuracy:P2}"); + + Assert.True(averageTrackingAccuracy > averageLegacyAccuracy, + $"Tracking accuracy {averageTrackingAccuracy:P2} did not improve on {averageLegacyAccuracy:P2}."); + Assert.True(averageTrackingAccuracy - averageLegacyAccuracy >= 0.10, + $"Randomized accuracy improved by only {(averageTrackingAccuracy - averageLegacyAccuracy):P2}."); + Assert.InRange(averageTrackingAccuracy, 0.70, 0.995); + } + + [Fact] + public void RandomizedTraceDoesNotLeaveSplitFragmentsAfterTheWholeRegionReturns() + { + RandomizedScenario scenario = RandomizedOcrTrackingAccuracyScenarios.Create(20260719); + OcrTextTracker tracker = new(NullLogger.Instance); + IReadOnlyList result = []; + for (int frame = 0; frame <= 95; frame++) + { + result = tracker.Update( + scenario.Observations[frame], + scenario.ImageSize, + TimeSpan.FromMilliseconds(frame * 350)); + } + + TextRect longMenu = Assert.Single( + result, + rect => rect.Context?.StartsWith("long-menu", StringComparison.Ordinal) == true); + Assert.Equal("Alpha Beta Gamma Delta Epsilon", longMenu.SourceText); + } + + [Fact] + public void TextAndWidthChangeDoesNotCreateOverlappingTracks() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect initial = new("AAAA", 100, 100, 100, 20, 16, false); + TextRect changed = new("ZZZZZZZZ", 100, 100, 125, 20, 16, false); + tracker.Update([initial], imageSize, TimeSpan.Zero); + + TextRect pending = Assert.Single(tracker.Update( + [changed], imageSize, TimeSpan.FromMilliseconds(500))); + TextRect confirmed = Assert.Single(tracker.Update( + [changed], imageSize, TimeSpan.FromMilliseconds(1000))); + + Assert.Equal("AAAA", pending.SourceText); + Assert.Equal("ZZZZZZZZ", confirmed.SourceText); + } + + [Theory] + [InlineData(145)] + [InlineData(200)] + public void LargeTextAndWidthChangeDoesNotCreateOverlappingTracks(double changedWidth) + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect initial = new("AAAA", 100, 100, 100, 20, 16, false); + TextRect changed = new("ZZZZZZZZ", 100, 100, changedWidth, 20, 16, false); + tracker.Update([initial], imageSize, TimeSpan.Zero); + + TextRect pending = Assert.Single(tracker.Update( + [changed], imageSize, TimeSpan.FromMilliseconds(500))); + TextRect confirmed = Assert.Single(tracker.Update( + [changed], imageSize, TimeSpan.FromMilliseconds(1000))); + + Assert.Equal("AAAA", pending.SourceText); + Assert.Equal("ZZZZZZZZ", confirmed.SourceText); + } + + [Theory] + [InlineData(5)] + [InlineData(6)] + public void SplitUpToTheCandidateLimitRemainsOneLogicalTrack(int partCount) + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + string text = new(Enumerable.Range(0, partCount) + .Select(index => (char)('A' + index)) + .ToArray()); + tracker.Update([new(text, 0, 100, partCount * 20, 20, 16, false)], imageSize, TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + Enumerable.Range(0, partCount) + .Select(index => new TextRect( + ((char)('A' + index)).ToString(), + index * 20, + 100, + 20, + 20, + 16, + false)), + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal(text, Assert.Single(result).SourceText); + } + + [Theory] + [InlineData(5)] + [InlineData(6)] + public void MergeUpToTheCandidateLimitConvergesWithoutDuplicate(int trackCount) + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect[] fragments = Enumerable.Range(0, trackCount) + .Select(index => new TextRect( + ((char)('A' + index)).ToString(), + index * 20, + 100, + 20, + 20, + 16, + false)) + .ToArray(); + string text = string.Concat(fragments.Select(fragment => fragment.SourceText)); + TextRect merged = new(text, 0, 100, trackCount * 20, 20, 16, false); + tracker.Update(fragments, imageSize, TimeSpan.Zero); + + Assert.Equal(trackCount, tracker.Update( + [merged], imageSize, TimeSpan.FromMilliseconds(500)).Count); + IReadOnlyList result = tracker.Update( + [merged], imageSize, TimeSpan.FromMilliseconds(1000)); + + Assert.Equal(text, Assert.Single(result).SourceText); + } + + [Fact] + public void PersistentMicroGeometryChangeEventuallyConverges() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect initial = new("Panel", 100, 100, 100, 30, 24, false) { Angle = 359 }; + TextRect changed = initial with { X = 101, Angle = 1 }; + tracker.Update([initial], imageSize, TimeSpan.Zero); + + TextRect result = initial; + for (int frame = 1; frame <= 6; frame++) + { + result = Assert.Single(tracker.Update( + [changed], imageSize, TimeSpan.FromMilliseconds(frame * 500))); + } + + Assert.Equal(101, result.X); + Assert.Equal(1, result.Angle); + } + + [Fact] + public void NonFiniteGeometryIsRejected() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect valid = new("Invalid", 10, 10, 100, 30, 24, false); + TextRect[] invalid = + [ + valid with { X = double.NaN }, + valid with { Y = double.PositiveInfinity }, + valid with { Width = double.PositiveInfinity }, + valid with { Height = double.PositiveInfinity }, + valid with { FontSize = double.NaN }, + valid with { Angle = double.NegativeInfinity }, + ]; + + Assert.Empty(tracker.Update(invalid, imageSize, TimeSpan.Zero)); + } + + [Fact] + public void MissingObservationsExpireAfterTheRetentionWindow() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect observation = new("Temporary", 20, 20, 120, 30, 24, false); + + Assert.Single(tracker.Update([observation], imageSize)); + Assert.Single(tracker.Update([], imageSize)); + Assert.Single(tracker.Update([], imageSize)); + Assert.Single(tracker.Update([], imageSize)); + Assert.Empty(tracker.Update([], imageSize)); + } + + [Fact] + public void PersistentMergeConvergesAfterConfirmation() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect first = new("New", 50, 400, 70, 32, 25, false); + TextRect second = new("Game", 125, 400, 90, 32, 25, false); + TextRect merged = new("New Game", 50, 400, 165, 32, 25, false); + + Assert.Equal(2, tracker.Update([first, second], imageSize).Count); + Assert.Equal(2, tracker.Update([merged], imageSize).Count); + TextRect result = Assert.Single(tracker.Update([merged], imageSize)); + Assert.Equal("New Game", result.SourceText); + } + + [Fact] + public void TrackingWorkDoesNotDominateAnOcrFrame() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1920, 1080); + TextRect[] observations = Enumerable.Range(0, 30) + .Select(index => new TextRect($"Text {index}", (index % 10) * 180, (index / 10) * 100, 120, 32, 25, false)) + .ToArray(); + tracker.Update(observations, imageSize); + + Stopwatch stopwatch = Stopwatch.StartNew(); + for (int frame = 0; frame < 100; frame++) + { + TextRect[] jittered = observations + .Select(rect => rect with { X = rect.X + (frame % 3) - 1, Y = rect.Y + (frame % 2) }) + .ToArray(); + Assert.Equal(30, tracker.Update(jittered, imageSize).Count); + } + stopwatch.Stop(); + + output.WriteLine($"100 frames with 30 tracks: {stopwatch.ElapsedMilliseconds} ms"); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(2), $"Tracking took {stopwatch.Elapsed}."); + } + + [Fact] + public void SpatiallyDistantLongTextsAreRejectedWithoutDominatingTheFrame() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(3840, 2160); + string initialPrefix = new('A', 2000); + string distantPrefix = new('B', 2000); + TextRect[] initial = Enumerable.Range(0, 64) + .Select(index => new TextRect( + $"{initialPrefix}-{index}", + index % 8 * 20, + index / 8 * 20, + 120, + 30, + 24, + false)) + .ToArray(); + TextRect[] distant = Enumerable.Range(0, 64) + .Select(index => new TextRect( + $"{distantPrefix}-{index}", + 3000 + (index % 8 * 20), + 1500 + (index / 8 * 20), + 120, + 30, + 24, + false)) + .ToArray(); + tracker.Update(initial, imageSize); + + Stopwatch stopwatch = Stopwatch.StartNew(); + tracker.Update(distant, imageSize); + stopwatch.Stop(); + + output.WriteLine($"64x64 spatially distant long-text candidates: {stopwatch.ElapsedMilliseconds} ms"); + Assert.True(stopwatch.Elapsed < TimeSpan.FromMilliseconds(500), $"Spatial rejection took {stopwatch.Elapsed}."); + } + + [Fact] + public void NearbyLengthIncompatibleLongTextsAreRejectedBeforeEditDistanceDominates() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(3840, 2160); + string longPrefix = new('A', 20000); + TextRect[] initial = Enumerable.Range(0, 64) + .Select(index => new TextRect( + $"{longPrefix}-{index}", + index % 8 * 5, + index / 8 * 5, + 120, + 30, + 24, + false)) + .ToArray(); + TextRect[] nearby = Enumerable.Range(0, 64) + .Select(index => new TextRect( + $"B-{index}", + 200 + (index % 8 * 5), + index / 8 * 5, + 120, + 30, + 24, + false)) + .ToArray(); + tracker.Update(initial, imageSize); + + Stopwatch stopwatch = Stopwatch.StartNew(); + IReadOnlyList result = tracker.Update(nearby, imageSize); + stopwatch.Stop(); + + output.WriteLine($"64x64 nearby length-incompatible candidates: {stopwatch.ElapsedMilliseconds} ms"); + Assert.Equal(128, result.Count); + Assert.True(stopwatch.Elapsed < TimeSpan.FromMilliseconds(500), $"Length rejection took {stopwatch.Elapsed}."); + } + + [Fact] + public void DenseOneToOneAssignmentPreservesMaximumCardinality() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect[] tracks = Enumerable.Range(0, 9) + .Select(index => new TextRect( + "A", + index < 3 ? 160 + index : 100 + index - 3, + 100, + 100, + 30, + 24, + false)) + .ToArray(); + TextRect[] observations = Enumerable.Range(0, 9) + .Select(index => new TextRect( + "A", + index < 3 ? 100 + index : 160 + index - 3, + 100, + 100, + 30, + 24, + false)) + .ToArray(); + tracker.Update(tracks, imageSize, TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + observations, + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal(9, result.Count); + } + + [Fact] + public void DenseOneToOneAssignmentPreservesMaximumCardinalityWithTenTracks() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect[] tracks = Enumerable.Range(0, 10) + .Select(index => new TextRect( + "A", + index < 3 ? 160 + index : 100 + index - 3, + 100, + 100, + 30, + 24, + false)) + .ToArray(); + TextRect[] observations = Enumerable.Range(0, 10) + .Select(index => new TextRect( + "A", + index < 3 ? 100 + index : 160 + index - 3, + 100, + 100, + 30, + 24, + false)) + .ToArray(); + tracker.Update(tracks, imageSize, TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + observations, + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal(10, result.Count); + } + + [Fact] + public void MaximumCardinalityAssignmentStillMaximizesTheTotalScore() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect left = new TextRect("A", 0, 100, 100, 30, 24, false) { Context = "Left" }; + TextRect right = new TextRect("A", 30, 100, 100, 30, 24, false) { Context = "Right" }; + tracker.Update([left, right], imageSize, TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + [ + left with { X = 10, Context = "NearLeft" }, + right with { X = 50, Context = "NearRight" }, + ], + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal(["NearLeft", "NearRight"], result.Select(rect => rect.Context)); + } + + [Fact] + public void WeakAssignmentDoesNotScaleWithReservedObservations() + { + Size imageSize = new(1920, 1080); + TextRect[] observations = Enumerable.Range(0, 200) + .Select(index => new TextRect( + new string((char)(0x1000 + index), 8), + (index % 20) * 90, + (index / 20) * 90, + 60, + 30, + 24, + false)) + .ToArray(); + TextRect[] oneWeakObservation = observations.ToArray(); + oneWeakObservation[^1] = oneWeakObservation[^1] with + { + SourceText = oneWeakObservation[^1].SourceText + "X", + }; + + MeasureAllocation(observations); + MeasureAllocation(oneWeakObservation); + long strongAllocation = MeasureAllocation(observations); + long weakAllocation = MeasureAllocation(oneWeakObservation); + long weakAssignmentAllocation = weakAllocation - strongAllocation; + + output.WriteLine($"Additional allocation for one weak assignment: {weakAssignmentAllocation:N0} bytes"); + Assert.True(weakAssignmentAllocation < 256 * 1024, + $"One weak assignment allocated an additional {weakAssignmentAllocation:N0} bytes."); + + long MeasureAllocation(TextRect[] current) + { + OcrTextTracker tracker = new(NullLogger.Instance); + tracker.Update(observations, imageSize, TimeSpan.Zero); + long before = GC.GetAllocatedBytesForCurrentThread(); + tracker.Update(current, imageSize, TimeSpan.FromMilliseconds(500)); + return GC.GetAllocatedBytesForCurrentThread() - before; + } + } + + [Fact] + public void IndependentOneToOneAssignmentsDoNotAllocateAsADenseMatrix() + { + MeasureAllocation(80); + MeasureAllocation(320); + long smallAllocation = MeasureAllocation(80); + long largeAllocation = MeasureAllocation(320); + + output.WriteLine($"80 independent assignments: {smallAllocation:N0} bytes"); + output.WriteLine($"320 independent assignments: {largeAllocation:N0} bytes"); + Assert.True(largeAllocation < smallAllocation * 8, + $"Quadrupling independent assignments increased allocation from " + + $"{smallAllocation:N0} to {largeAllocation:N0} bytes."); + + static long MeasureAllocation(int count) + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect[] observations = Enumerable.Range(0, count) + .Select(index => new TextRect( + new string((char)(0x1000 + index), 8), + index * 200, + 100, + 30, + 20, + 16, + false)) + .ToArray(); + tracker.Update(observations, imageSize, TimeSpan.Zero); + long before = GC.GetAllocatedBytesForCurrentThread(); + tracker.Update(observations, imageSize, TimeSpan.FromMilliseconds(500)); + return GC.GetAllocatedBytesForCurrentThread() - before; + } + } + + [Fact] + public void MixedStructureSelectionDoesNotDominateAFrame() + { + Size imageSize = new(1000, 600); + TextRect[] tracks = Enumerable.Range(0, 9) + .Select(index => new TextRect("AA", 100 + index, 100, 100, 30, 24, false)) + .ToArray(); + TextRect[] observations = Enumerable.Range(0, 9) + .Select(index => new TextRect("A", 100 + index, 100, 100, 30, 24, false)) + .ToArray(); + RunFrame(); + Stopwatch stopwatch = Stopwatch.StartNew(); + for (int iteration = 0; iteration < 10; iteration++) + { + Assert.Equal(9, RunFrame().Count); + } + stopwatch.Stop(); + + output.WriteLine($"10 frames of 9x9 mixed structure candidates: {stopwatch.ElapsedMilliseconds} ms"); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(1), + $"10 mixed structure frames took {stopwatch.Elapsed}."); + + IReadOnlyList RunFrame() + { + OcrTextTracker tracker = new(NullLogger.Instance); + tracker.Update(tracks, imageSize, TimeSpan.Zero); + return tracker.Update(observations, imageSize, TimeSpan.FromMilliseconds(500)); + } + } + + [Fact] + public void DenseRejectedLongStructureCandidatesDoNotDominateAFrame() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + string previousText = new('B', 1000); + string currentText = new('A', 1000); + TextRect[] tracks = Enumerable.Range(0, 6) + .Select(index => new TextRect(previousText, 100 + index, 100, 100, 30, 24, false)) + .ToArray(); + TextRect[] observations = Enumerable.Range(0, 6) + .Select(index => new TextRect(currentText, 100 + index, 100, 100, 30, 24, false)) + .ToArray(); + tracker.Update(tracks, imageSize, TimeSpan.Zero); + + Stopwatch stopwatch = Stopwatch.StartNew(); + tracker.Update(observations, imageSize, TimeSpan.FromMilliseconds(500)); + stopwatch.Stop(); + + output.WriteLine($"Dense rejected long structure candidates: {stopwatch.ElapsedMilliseconds} ms"); + Assert.True(stopwatch.Elapsed < TimeSpan.FromSeconds(1), $"Dense long-text rejection took {stopwatch.Elapsed}."); + } + + [Fact] + public void StructureSelectionPrefersGreaterCoverageOverHigherIndividualScores() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect[] tracks = + [ + new("A", 0, 100, 10, 20, 16, false), + new("B", 10, 100, 10, 20, 16, false), + new("C", 20, 100, 10, 20, 16, false), + new("D", 30, 100, 10, 20, 16, false), + new("E", 40, 100, 10, 20, 16, false), + new("F", 50, 100, 10, 20, 16, false), + ]; + tracker.Update(tracks, imageSize, TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + [ + new("ABC", 0, 100, 30, 20, 16, false), + new("DEF", 30, 100, 30, 20, 16, false), + new("AD", 0, 100, 44, 20, 16, false), + new("BE", 10, 100, 44, 20, 16, false), + new("CF", 20, 100, 44, 20, 16, false), + ], + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal( + ["A", "B", "C", "D", "E", "F", "ABC", "DEF"], + result.Select(rect => rect.SourceText)); + } + + [Fact] + public void StructureSelectionEscapesLargeAndCompactGreedyTraps() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + tracker.Update( + [ + new("A", -20, 100, 20, 20, 16, false), + new("BCDE", 0, 100, 100, 20, 16, false), + new("B", 0, 100, 25, 20, 16, false), + new("C", 25, 100, 25, 20, 16, false), + new("D", 50, 100, 25, 20, 16, false), + new("E", 75, 100, 25, 20, 16, false), + ], + imageSize, + TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + [ + new("ABCDE", -20, 100, 124, 20, 16, false), + new("BC", 0, 100, 50, 20, 16, false), + new("DE", 50, 100, 50, 20, 16, false), + new("ABD", -20, 100, 95, 20, 16, false), + new("CBCDEE", 0, 100, 100, 20, 16, false), + ], + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal( + ["A", "BCDE", "B", "C", "D", "E", "ABD", "CBCDEE"], + result.Select(rect => rect.SourceText)); + } + + [Fact] + public void StructureSelectionReplacesOneBlockingCandidateWithTwoMatches() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + tracker.Update( + [ + new("A", 0, 100, 10, 20, 16, false), + new("B", 10, 100, 10, 20, 16, false), + new("C", 20, 100, 10, 20, 16, false), + new("D", 30, 100, 10, 20, 16, false), + ], + imageSize, + TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + [ + new("AB", 0, 100, 24, 20, 16, false), + new("CD", 20, 100, 24, 20, 16, false), + new("ABC", 0, 100, 30, 20, 16, false), + ], + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal( + ["A", "B", "C", "D", "ABC"], + result.Select(rect => rect.SourceText)); + } + + [Fact] + public void StructureSelectionCanReplaceCoupledBlockingCandidates() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + string longTail = new('A', 10); + tracker.Update( + [ + new("XYZ", 0, 100, 30, 20, 16, false), + new("Q", 300, 100, 10, 20, 16, false), + new("B", 30, 100, 10, 20, 16, false), + new("C", 40, 100, 10, 20, 16, false), + new("D", 50, 100, 10, 20, 16, false), + new(longTail, 60, 100, 100, 20, 16, false), + ], + imageSize, + TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + [ + new("Novel", 500, 100, 50, 20, 16, false), + new("X", 0, 100, 10, 20, 16, false), + new("Y", 10, 100, 10, 20, 16, false), + new("XYZBC", 0, 100, 50, 20, 16, false), + new($"BCD{longTail}", 30, 100, 130, 20, 16, false), + new("Z", 20, 100, 10, 20, 16, false), + ], + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal( + ["XYZ", "Q", "B", "C", "D", longTail, "Novel", "XYZBC"], + result.Select(rect => rect.SourceText)); + } + + [Fact] + public void IndependentStructureConflictsDoNotConsumeEachOthersSelectionBudget() + { + const int groupCount = 8; + const int spacing = 4000; + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(groupCount * spacing, 600); + List tracks = []; + List observations = []; + for (int group = 0; group < groupCount; group++) + { + double x = group * spacing; + tracks.AddRange( + [ + new("A", x, 100, 10, 20, 16, false), + new("B", x + 10, 100, 10, 20, 16, false), + new("C", x + 20, 100, 10, 20, 16, false), + new("D", x + 30, 100, 10, 20, 16, false), + ]); + observations.AddRange( + [ + new("AB", x, 100, 24, 20, 16, false), + new("CD", x + 20, 100, 24, 20, 16, false), + new("ABC", x, 100, 30, 20, 16, false), + ]); + } + tracker.Update(tracks, imageSize, TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + observations, + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal(groupCount, result.Count(rect => rect.SourceText == "ABC")); + Assert.DoesNotContain(result, rect => rect.SourceText is "AB" or "CD"); + Assert.Equal(groupCount * 5, result.Count); + } + + [Fact] + public void StaleSplitFragmentDoesNotRemainAfterTheWholeTrackReturns() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect whole = new("ABCDEFGHIJ", 0, 100, 100, 20, 16, false); + tracker.Update([whole], imageSize, TimeSpan.Zero); + tracker.Update( + [ + new("AB", 0, 100, 20, 20, 16, false), + new("CD", 20, 100, 20, 20, 16, false), + new("EF", 40, 100, 20, 20, 48, false), + new("GH", 60, 100, 20, 20, 16, false), + new("IJ", 80, 100, 20, 20, 16, false), + ], + imageSize, + TimeSpan.FromMilliseconds(500)); + + IReadOnlyList result = tracker.Update( + [whole], + imageSize, + TimeSpan.FromMilliseconds(1000)); + + Assert.Equal("ABCDEFGHIJ", Assert.Single(result).SourceText); + } + + [Fact] + public void WholeRegionRemovesFragmentsCreatedWhileTheParentWasUnmatched() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect whole = new("ABCDEFGHIJ", 0, 100, 100, 20, 16, false); + tracker.Update([whole], imageSize, TimeSpan.Zero); + + IReadOnlyList split = tracker.Update( + [ + new("BC", 10, 100, 20, 20, 8, false), + new("DE", 30, 100, 20, 20, 16, false), + new("FG", 50, 100, 20, 20, 32, false), + new("HI", 70, 100, 20, 20, 64, false), + ], + imageSize, + TimeSpan.FromMilliseconds(500)); + Assert.True(split.Count > 1, "The adversarial split must leave its parent unmatched."); + + IReadOnlyList result = tracker.Update( + [whole], + imageSize, + TimeSpan.FromMilliseconds(1000)); + + Assert.Equal("ABCDEFGHIJ", Assert.Single(result).SourceText); + } + + [Fact] + public void FourPartSplitRemainsOneLogicalTrack() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + tracker.Update([new("ABCD", 0, 100, 80, 20, 16, false)], imageSize, TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + [ + new("A", 0, 100, 20, 20, 16, false), + new("B", 20, 100, 20, 20, 16, false), + new("C", 40, 100, 20, 20, 16, false), + new("D", 60, 100, 20, 20, 16, false), + ], + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal("ABCD", Assert.Single(result).SourceText); + } + + [Fact] + public void FourTrackMergeConvergesWithoutDuplicate() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect[] fragments = + [ + new("A", 0, 100, 20, 20, 16, false), + new("B", 20, 100, 20, 20, 16, false), + new("C", 40, 100, 20, 20, 16, false), + new("D", 60, 100, 20, 20, 16, false), + ]; + TextRect merged = new("ABCD", 0, 100, 80, 20, 16, false); + tracker.Update(fragments, imageSize, TimeSpan.Zero); + + Assert.Equal(4, tracker.Update([merged], imageSize, TimeSpan.FromMilliseconds(500)).Count); + IReadOnlyList result = tracker.Update( + [merged], + imageSize, + TimeSpan.FromMilliseconds(1000)); + + Assert.Equal("ABCD", Assert.Single(result).SourceText); + } + + [Fact] + public void EquivalentStructureCandidatesDoNotDependOnObservationOrder() + { + Size imageSize = new(1000, 600); + TextRect[] observations = + [ + new TextRect("A", 40, 100, 18, 20, 16, false) { Context = "Left" }, + new("B", 62, 100, 18, 20, 16, false), + new TextRect("A", 120, 100, 18, 20, 16, false) { Context = "Right" }, + new("B", 142, 100, 18, 20, 16, false), + ]; + + string? forward = SelectContext(observations); + string? reversed = SelectContext(observations.Reverse().ToArray()); + + Assert.Equal("Left", forward); + Assert.Equal(forward, reversed); + + string? SelectContext(TextRect[] current) + { + OcrTextTracker tracker = new(NullLogger.Instance); + tracker.Update([new("AB", 80, 100, 40, 20, 16, false)], imageSize, TimeSpan.Zero); + IReadOnlyList result = tracker.Update( + current, + imageSize, + TimeSpan.FromMilliseconds(500)); + return Assert.Single(result, rect => rect.SourceText == "AB").Context; + } + } + + [Fact] + public void MotionPredictionKeepsIdentityWhenTracksCrossBetweenSlowOcrFrames() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + + tracker.Update( + [ + RectWithContext("Item", 0, "A"), + RectWithContext("Item", 300, "B"), + ], imageSize, TimeSpan.Zero); + tracker.Update( + [ + RectWithContext("Item", 100, "A"), + RectWithContext("Item", 200, "B"), + ], imageSize, TimeSpan.FromMilliseconds(500)); + + IReadOnlyList crossed = tracker.Update( + [ + RectWithContext("Item", 220, "A"), + RectWithContext("Item", 80, "B"), + ], imageSize, TimeSpan.FromMilliseconds(1000)); + + Assert.Equal(["A", "B"], crossed.Select(rect => rect.Context)); + } + + [Fact] + public void PersistentSmallGeometryChangeEventuallyConverges() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect initial = new("Panel", 100, 100, 100, 30, 24, false); + TextRect changed = initial with { X = 110, Width = 110 }; + tracker.Update([initial], imageSize, TimeSpan.Zero); + + TextRect first = Assert.Single(tracker.Update( + [changed], + imageSize, + TimeSpan.FromMilliseconds(500))); + TextRect confirmed = Assert.Single(tracker.Update( + [changed], + imageSize, + TimeSpan.FromMilliseconds(1000))); + + Assert.Equal((100, 100), (first.X, first.Width)); + Assert.Equal((110, 110), (confirmed.X, confirmed.Width)); + } + + [Fact] + public void GlobalSelectionPrefersExactSplitOverPartialOneToOneMatch() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + tracker.Update([new("New Game", 0, 0, 200, 30, 24, false)], imageSize, TimeSpan.Zero); + + TextRect result = Assert.Single(tracker.Update( + [ + new("New", 0, 0, 65, 30, 24, false), + new("Game", 70, 0, 130, 30, 24, false), + ], imageSize, TimeSpan.FromMilliseconds(500))); + + Assert.Equal("New Game", result.SourceText); + } + + [Fact] + public void DominantFragmentDoesNotSuppressAnExactSplitCandidate() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + tracker.Update([new("ABCDEFG", 0, 100, 200, 30, 24, false)], imageSize, TimeSpan.Zero); + + TextRect result = Assert.Single(tracker.Update( + [ + new("ABCDEF", 0, 100, 180, 30, 24, false), + new("G", 185, 100, 15, 30, 24, false), + ], + imageSize, + TimeSpan.FromMilliseconds(500))); + + Assert.Equal("ABCDEFG", result.SourceText); + } + + [Fact] + public void StrongOneToOneMatchIsReservedBeforeMergeCandidates() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect first = new("A", 0, 100, 40, 30, 24, false); + TextRect second = new("B", 50, 100, 40, 30, 24, false); + tracker.Update([first, second], imageSize, TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + [ + first, + new("AB", 0, 100, 90, 30, 24, false), + ], + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal(["A", "B", "AB"], result.Select(rect => rect.SourceText)); + } + + [Fact] + public void CompatibilityEquivalentTextRemainsOneTrackWhileMoving() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + tracker.Update( + [new("ＡＢＣＤ", 0, 100, 100, 30, 24, false)], + imageSize, + TimeSpan.Zero); + + IReadOnlyList result = tracker.Update( + [new("ABCD", 60, 100, 100, 30, 24, false)], + imageSize, + TimeSpan.FromMilliseconds(500)); + + Assert.Equal("ＡＢＣＤ", Assert.Single(result).SourceText); + } + + [Fact] + public void DormantChildrenReturnWhenAConfirmedMergeSplitsAgain() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect first = new("New", 50, 400, 70, 32, 25, false); + TextRect second = new("Game", 125, 400, 90, 32, 25, false); + TextRect merged = new("New Game", 50, 400, 165, 32, 25, false); + + tracker.Update([first, second], imageSize, TimeSpan.Zero); + tracker.Update([merged], imageSize, TimeSpan.FromMilliseconds(500)); + Assert.Single(tracker.Update([merged], imageSize, TimeSpan.FromMilliseconds(1000))); + Assert.Single(tracker.Update([first, second], imageSize, TimeSpan.FromMilliseconds(1500))); + + IReadOnlyList restored = tracker.Update( + [first, second], imageSize, TimeSpan.FromMilliseconds(2000)); + Assert.Equal(["New", "Game"], restored.Select(rect => rect.SourceText)); + } + + [Fact] + public void DormantChildrenReturnAfterTheMergedParentMoves() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect first = new("New", 0, 100, 40, 30, 24, false); + TextRect second = new("Game", 50, 100, 50, 30, 24, false); + TextRect merged = new("New Game", 0, 100, 100, 30, 24, false); + TextRect movedMerged = merged with { X = 200 }; + TextRect movedFirst = first with { X = 200 }; + TextRect movedSecond = second with { X = 250 }; + + tracker.Update([first, second], imageSize, TimeSpan.Zero); + tracker.Update([merged], imageSize, TimeSpan.FromMilliseconds(500)); + Assert.Single(tracker.Update([merged], imageSize, TimeSpan.FromMilliseconds(1000))); + Assert.Single(tracker.Update([movedMerged], imageSize, TimeSpan.FromMilliseconds(1500))); + Assert.Single(tracker.Update([movedFirst, movedSecond], imageSize, TimeSpan.FromMilliseconds(2000))); + + IReadOnlyList restored = tracker.Update( + [movedFirst, movedSecond], imageSize, TimeSpan.FromMilliseconds(2500)); + Assert.Equal(["New", "Game"], restored.Select(rect => rect.SourceText)); + } + + [Fact] + public void DormantChildrenReturnWhenMovementAndSplitOccurTogether() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect first = new("New", 0, 100, 40, 30, 24, false); + TextRect second = new("Game", 50, 100, 50, 30, 24, false); + TextRect merged = new("New Game", 0, 100, 100, 30, 24, false); + TextRect movedFirst = first with { X = 200 }; + TextRect movedSecond = second with { X = 250 }; + + tracker.Update([first, second], imageSize, TimeSpan.Zero); + tracker.Update([merged], imageSize, TimeSpan.FromMilliseconds(500)); + Assert.Single(tracker.Update([merged], imageSize, TimeSpan.FromMilliseconds(1000))); + + Assert.Single(tracker.Update( + [movedFirst, movedSecond], imageSize, TimeSpan.FromMilliseconds(1500))); + IReadOnlyList restored = tracker.Update( + [movedFirst, movedSecond], imageSize, TimeSpan.FromMilliseconds(2000)); + Assert.Equal(["New", "Game"], restored.Select(rect => rect.SourceText)); + } + + [Fact] + public void SimilarButDifferentOcrErrorsDoNotShareAStringVote() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + + tracker.Update([new("Start Game", 100, 100, 200, 40, 32, false)], imageSize, TimeSpan.Zero); + tracker.Update([new("Start Garne", 100, 100, 200, 40, 32, false)], imageSize, TimeSpan.FromMilliseconds(500)); + TextRect afterDifferentErrors = Assert.Single(tracker.Update( + [new("Start Ganne", 100, 100, 200, 40, 32, false)], imageSize, TimeSpan.FromMilliseconds(1000))); + Assert.Equal("Start Game", afterDifferentErrors.SourceText); + + TextRect afterRepeatedError = Assert.Single(tracker.Update( + [new("Start Ganne", 100, 100, 200, 40, 32, false)], imageSize, TimeSpan.FromMilliseconds(1500))); + Assert.Equal("Start Ganne", afterRepeatedError.SourceText); + } + + [Theory] + [InlineData(60, 24, 0, 60, 24, 0)] + [InlineData(30, 48, 0, 30, 36, 0)] + [InlineData(30, 24, 20, 30, 24, 10)] + public void SplitAndMergeCandidatesRequireCompatibleMemberStyles( + double secondHeight, + double secondFontSize, + double secondAngle, + double combinedHeight, + double combinedFontSize, + double combinedAngle) + { + Size imageSize = new(1000, 600); + TextRect first = new("New", 0, 100, 95, 30, 24, false) { Angle = 0 }; + TextRect second = new("Game", 100, 100, 100, secondHeight, secondFontSize, false) { Angle = secondAngle }; + TextRect combined = new("New Game", 0, 100, 200, combinedHeight, combinedFontSize, false) { Angle = combinedAngle }; + + OcrTextTracker splitTracker = new(NullLogger.Instance); + splitTracker.Update([combined], imageSize, TimeSpan.Zero); + Assert.True( + splitTracker.Update([first, second], imageSize, TimeSpan.FromMilliseconds(500)).Count > 1, + "Incompatible OCR fragments must not be collapsed into one logical track."); + + OcrTextTracker mergeTracker = new(NullLogger.Instance); + mergeTracker.Update([first, second], imageSize, TimeSpan.Zero); + mergeTracker.Update([combined], imageSize, TimeSpan.FromMilliseconds(500)); + Assert.True( + mergeTracker.Update([combined], imageSize, TimeSpan.FromMilliseconds(1000)).Count > 1, + "Incompatible tracks must not converge into one logical track."); + } + + [Fact] + public void SplitUsesCircularMeanForAnglesAcrossTheWrapBoundary() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect combined = new("AB", 0, 100, 200, 30, 24, false) { Angle = 0 }; + TextRect first = new("A", 0, 100, 95, 30, 24, false) { Angle = 359 }; + TextRect second = new("B", 100, 100, 100, 30, 24, false) { Angle = 1 }; + tracker.Update([combined], imageSize, TimeSpan.Zero); + + tracker.Update([first, second], imageSize, TimeSpan.FromMilliseconds(500)); + TextRect result = Assert.Single(tracker.Update( + [first, second], imageSize, TimeSpan.FromMilliseconds(1000))); + + Assert.InRange(Math.Abs(result.Angle), 0, 0.001); + } + + [Fact] + public void SmallAngleJitterAcrossTheWrapBoundaryIsSuppressed() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect initial = new("A", 0, 100, 100, 30, 24, false) { Angle = 359 }; + TextRect jittered = initial with { Angle = 1 }; + tracker.Update([initial], imageSize, TimeSpan.Zero); + + tracker.Update([jittered], imageSize, TimeSpan.FromMilliseconds(500)); + TextRect result = Assert.Single(tracker.Update( + [jittered], imageSize, TimeSpan.FromMilliseconds(1000))); + + Assert.Equal(359, result.Angle); + } + + [Fact] + public void SplitOrdersFragmentsAlongTheRotatedReadingDirection() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect combined = new("AB", 105, 100, 205, 30, 24, false) { Angle = 180 }; + TextRect first = new("A", 105, 100, 95, 30, 24, false) { Angle = 180 }; + TextRect second = new("B", 0, 100, 100, 30, 24, false) { Angle = 180 }; + tracker.Update([combined], imageSize, TimeSpan.Zero); + + TextRect result = Assert.Single(tracker.Update( + [first, second], imageSize, TimeSpan.FromMilliseconds(500))); + + Assert.Equal("AB", result.SourceText); + } + + [Fact] + public void PersistentMergeOrdersTracksAlongTheRotatedReadingDirection() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect first = new("A", 105, 100, 95, 30, 24, false) { Angle = 180 }; + TextRect second = new("B", 0, 100, 100, 30, 24, false) { Angle = 180 }; + TextRect combined = new("AB", 105, 100, 205, 30, 24, false) { Angle = 180 }; + tracker.Update([first, second], imageSize, TimeSpan.Zero); + + Assert.Equal(2, tracker.Update( + [combined], imageSize, TimeSpan.FromMilliseconds(500)).Count); + TextRect result = Assert.Single(tracker.Update( + [combined], imageSize, TimeSpan.FromMilliseconds(1000))); + + Assert.Equal("AB", result.SourceText); + } + + [Fact] + public void SplitTreatsObliqueFragmentsOnTheSameReadingAxisAsAdjacent() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + double offset = 105 / Math.Sqrt(2); + TextRect combined = new("AB", 0, 0, 205, 20, 18, false) { Angle = 45 }; + TextRect first = new("A", 0, 0, 100, 20, 18, false) { Angle = 45 }; + TextRect second = new("B", offset, offset, 100, 20, 18, false) { Angle = 45 }; + tracker.Update([combined], imageSize, TimeSpan.Zero); + + TextRect result = Assert.Single(tracker.Update( + [first, second], imageSize, TimeSpan.FromMilliseconds(500))); + + Assert.Equal("AB", result.SourceText); + } + + [Fact] + public void PersistentMergeCombinesOrientedRectsAlongTheReadingAxis() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + double offset = 105 / Math.Sqrt(2); + TextRect first = new("A", 0, 0, 100, 20, 18, false) { Angle = 45 }; + TextRect second = new("B", offset, offset, 100, 20, 18, false) { Angle = 45 }; + TextRect combined = new("AB", 0, 0, 205, 20, 18, false) { Angle = 45 }; + tracker.Update([first, second], imageSize, TimeSpan.Zero); + + Assert.Equal(2, tracker.Update( + [combined], imageSize, TimeSpan.FromMilliseconds(500)).Count); + TextRect result = Assert.Single(tracker.Update( + [combined], imageSize, TimeSpan.FromMilliseconds(1000))); + + Assert.Equal("AB", result.SourceText); + } + + [Fact] + public void DormantChildrenDoNotConsumeAnActiveTracksObservation() + { + OcrTextTracker tracker = new(NullLogger.Instance); + Size imageSize = new(1000, 600); + TextRect childNew = RectWithContext("New", 0, "ChildNew"); + TextRect childGame = RectWithContext("Game", 50, "ChildGame"); + TextRect merged = new("New Game", 0, 100, 100, 30, 24, false) { Context = "Merged" }; + + tracker.Update( + [childNew, childGame, RectWithContext("New", 100, "Active")], + imageSize, + TimeSpan.Zero); + tracker.Update( + [merged, RectWithContext("New", 100, "Active")], + imageSize, + TimeSpan.FromMilliseconds(500)); + tracker.Update( + [merged, RectWithContext("New", 100, "Active")], + imageSize, + TimeSpan.FromMilliseconds(1000)); + + TextRect[] observations = + [ + RectWithContext("New", 100, "Active"), + childGame, + ]; + tracker.Update(observations, imageSize, TimeSpan.FromMilliseconds(1500)); + IReadOnlyList result = tracker.Update( + observations, + imageSize, + TimeSpan.FromMilliseconds(2000)); + + Assert.Single(result, rect => rect.Context == "Active"); + } + + [Fact] + public void RemovedFeaturesAreNotExposed() + { + const System.Reflection.BindingFlags flags = System.Reflection.BindingFlags.Static + | System.Reflection.BindingFlags.Public + | System.Reflection.BindingFlags.NonPublic; + Type appResources = typeof(OcrTextTracker).Assembly.GetType("WindowTranslator.Properties.Resources", throwOnError: true)!; + Type abstractionResources = typeof(TextRect).Assembly.GetType("WindowTranslator.Properties.Resources", throwOnError: true)!; + + Assert.Null(appResources.GetProperty("IsOneShotMode", flags)); + Assert.Null(abstractionResources.GetProperty("Buffer", flags)); + Assert.Null(abstractionResources.GetProperty("BufferSize", flags)); + Assert.Null(abstractionResources.GetProperty("IsSuppressVibe", flags)); + Assert.Null(abstractionResources.GetProperty("IsEnableRecover", flags)); + Assert.DoesNotContain( + typeof(IOcrTextTracker).GetMethods(), + method => method.GetParameters().Any(parameter => parameter.ParameterType == typeof(TimeSpan))); + } + + private static IReadOnlyList> RunLegacyBuffer() + { + List> actual = []; + foreach (Scenario scenario in OcrTrackingAccuracyScenarios.All) + { + LegacyOcrBufferModel filter = new(); + foreach (IReadOnlyList frame in scenario.Observations) + { + actual.Add(filter.Update(frame, new(1000, 600))); + } + } + return actual; + } + + private static IReadOnlyList> RunTracker() + { + List> actual = []; + foreach (Scenario scenario in OcrTrackingAccuracyScenarios.All) + { + OcrTextTracker tracker = new(NullLogger.Instance); + foreach (IReadOnlyList frame in scenario.Observations) + { + actual.Add(tracker.Update(frame, new(1000, 600))); + } + } + return actual; + } + + private static TextRect RectWithContext(string text, double x, string context) + => new TextRect(text, x, 100, 40, 30, 24, false) { Context = context }; +} + +internal sealed class LegacyOcrBufferModel +{ + private const int BufferSize = 3; + private readonly Queue> buffer = []; + + public IReadOnlyList Update(IReadOnlyList observations, Size imageSize) + { + Size threshold = new((int)(imageSize.Width * 0.02), (int)(imageSize.Height * 0.02)); + List bufferedTexts = []; + foreach (TextRect rect in this.buffer.SelectMany(frame => frame)) + { + if (!bufferedTexts.Any(existing => AreSimilar(existing, rect, threshold))) + { + bufferedTexts.Add(rect); + } + } + + if (this.buffer.Count == BufferSize) + { + this.buffer.Dequeue(); + } + + List current = []; + List output = []; + foreach (TextRect observation in observations) + { + TextRect text = observation; + TextRect? past = bufferedTexts.FirstOrDefault(buffered => AreSimilar(buffered, text, threshold)); + if (past is not null) + { + text = text with + { + X = past.X, + Y = past.Y, + Width = Math.Max(text.Width, past.Width), + Height = past.Height, + FontSize = past.FontSize, + }; + } + current.Add(text); + output.Add(text); + } + this.buffer.Enqueue(current); + + foreach (TextRect buffered in bufferedTexts) + { + if (!current.Any(text => AreSimilar(text, buffered, threshold) || Intersects(text, buffered)) + && !output.Any(existing => Intersects(existing, buffered))) + { + output.Add(buffered); + } + } + return output; + } + + private static bool Intersects(TextRect first, TextRect second) + => first.X < second.X + second.Width + && first.X + first.Width > second.X + && first.Y < second.Y + second.Height + && first.Y + first.Height > second.Y; + + private static bool AreSimilar(TextRect first, TextRect second, Size threshold) + { + int maximumLength = Math.Max(first.SourceText.Length, second.SourceText.Length); + double textSimilarity = maximumLength == 0 + ? 1 + : 1 - ((double)Levenshtein.GetDistance(first.SourceText, second.SourceText, CalculationOptions.DefaultWithThreading) / maximumLength); + return textSimilarity >= 0.8 + && Math.Abs(first.X - second.X) <= threshold.Width + && Math.Abs(first.Y - second.Y) <= threshold.Height + && Math.Abs(first.Width - second.Width) <= threshold.Width + && Math.Abs(first.Height - second.Height) <= threshold.Height; + } +} diff --git a/WindowTranslator.Tests/OcrTrackingAccuracyScenarios.cs b/WindowTranslator.Tests/OcrTrackingAccuracyScenarios.cs new file mode 100644 index 00000000..c8c576c5 --- /dev/null +++ b/WindowTranslator.Tests/OcrTrackingAccuracyScenarios.cs @@ -0,0 +1,144 @@ +using System.Drawing; + +namespace WindowTranslator.Tests; + +internal static class OcrTrackingAccuracyScenarios +{ + public static IReadOnlyList All { get; } = + [ + new("static jitter and transient typo", + [ + [Rect("Start Game", 100, 100, 200, 40)], + [Rect("Start Game", 102, 99, 198, 41)], + [Rect("Start Garne", 99, 101, 203, 39)], + [], + [Rect("Start Game", 101, 100, 201, 40)], + ], + [ + [Rect("Start Game", 100, 100, 200, 40)], + [Rect("Start Game", 100, 100, 200, 40)], + [Rect("Start Game", 100, 100, 200, 40)], + [Rect("Start Game", 100, 100, 200, 40)], + [Rect("Start Game", 100, 100, 200, 40)], + ]), + new("one-to-one assignment", + [ + [Rect("Score", 100, 200, 80, 30), Rect("Score", 125, 200, 80, 30)], + [Rect("Score", 113, 200, 80, 30), Rect("Score", 111, 200, 80, 30)], + ], + [ + [Rect("Score", 100, 200, 80, 30), Rect("Score", 125, 200, 80, 30)], + [Rect("Score", 100, 200, 80, 30), Rect("Score", 125, 200, 80, 30)], + ]), + new("temporary split", + [ + [Rect("New Game", 300, 300, 180, 36)], + [Rect("New", 300, 300, 76, 36), Rect("Game", 382, 300, 98, 36)], + [Rect("New Game", 301, 300, 179, 36)], + ], + [ + [Rect("New Game", 300, 300, 180, 36)], + [Rect("New Game", 300, 300, 180, 36)], + [Rect("New Game", 300, 300, 180, 36)], + ]), + new("temporary merge", + [ + [Rect("New", 50, 400, 70, 32), Rect("Game", 125, 400, 90, 32)], + [Rect("New Game", 50, 400, 165, 32)], + [Rect("New", 50, 400, 70, 32), Rect("Game", 125, 400, 90, 32)], + ], + [ + [Rect("New", 50, 400, 70, 32), Rect("Game", 125, 400, 90, 32)], + [Rect("New", 50, 400, 70, 32), Rect("Game", 125, 400, 90, 32)], + [Rect("New", 50, 400, 70, 32), Rect("Game", 125, 400, 90, 32)], + ]), + new("transient and sustained text change", + [ + [Rect("Pause", 600, 100, 120, 34)], + [Rect("Resume", 600, 100, 120, 34)], + [Rect("Pause", 600, 100, 120, 34)], + [Rect("Resume", 600, 100, 120, 34)], + [Rect("Resume", 600, 100, 120, 34)], + ], + [ + [Rect("Pause", 600, 100, 120, 34)], + [Rect("Pause", 600, 100, 120, 34)], + [Rect("Pause", 600, 100, 120, 34)], + [Rect("Pause", 600, 100, 120, 34)], + [Rect("Resume", 600, 100, 120, 34)], + ]), + new("real movement", + [ + [Rect("Moving", 100, 500, 130, 30)], + [Rect("Moving", 102, 501, 130, 30)], + [Rect("Moving", 350, 500, 130, 30)], + [Rect("Moving", 352, 500, 130, 30)], + ], + [ + [Rect("Moving", 100, 500, 130, 30)], + [Rect("Moving", 100, 500, 130, 30)], + [Rect("Moving", 350, 500, 130, 30)], + [Rect("Moving", 350, 500, 130, 30)], + ]), + ]; + + public static double Measure(IReadOnlyList> actualFrames) + { + var expectedFrames = All.SelectMany(s => s.Expected).ToArray(); + var actual = actualFrames.ToArray(); + Assert.Equal(expectedFrames.Length, actual.Length); + + return expectedFrames.Zip(actual, ScoreFrame).Average(); + } + + private static double ScoreFrame(IReadOnlyList expected, IReadOnlyList actual) + { + if (expected.Count == 0) + { + return actual.Count == 0 ? 1 : 0; + } + + var remaining = actual.ToList(); + var score = 0d; + foreach (var expectedRect in expected) + { + var best = remaining + .Select((rect, index) => (rect, index, score: ScoreRect(expectedRect, rect))) + .OrderByDescending(candidate => candidate.score) + .FirstOrDefault(); + if (best.rect is not null) + { + score += best.score; + remaining.RemoveAt(best.index); + } + } + + return score / Math.Max(expected.Count, actual.Count); + } + + private static double ScoreRect(TextRect expected, TextRect actual) + { + var text = expected.SourceText == actual.SourceText ? 0.5 : 0; + var geometry = IntersectionOverUnion(expected, actual) * 0.5; + return text + geometry; + } + + private static double IntersectionOverUnion(TextRect first, TextRect second) + { + var left = Math.Max(first.X, second.X); + var top = Math.Max(first.Y, second.Y); + var right = Math.Min(first.X + first.Width, second.X + second.Width); + var bottom = Math.Min(first.Y + first.Height, second.Y + second.Height); + var intersection = Math.Max(0, right - left) * Math.Max(0, bottom - top); + var union = first.Width * first.Height + second.Width * second.Height - intersection; + return union <= 0 ? 0 : intersection / union; + } + + private static TextRect Rect(string text, double x, double y, double width, double height) + => new(text, x, y, width, height, height * 0.8, false, Color.White, Color.Black); +} + +internal sealed record Scenario( + string Name, + IReadOnlyList> Observations, + IReadOnlyList> Expected); diff --git a/WindowTranslator.Tests/RandomizedOcrTrackingAccuracyScenarios.cs b/WindowTranslator.Tests/RandomizedOcrTrackingAccuracyScenarios.cs new file mode 100644 index 00000000..1e460d07 --- /dev/null +++ b/WindowTranslator.Tests/RandomizedOcrTrackingAccuracyScenarios.cs @@ -0,0 +1,423 @@ +using System.Drawing; + +namespace WindowTranslator.Tests; + +internal static class RandomizedOcrTrackingAccuracyScenarios +{ + public static IReadOnlyList Seeds { get; } = [643, 0x5EED, 20260719]; + + public static RandomizedScenario Create(int seed) + { + Size imageSize = new(1920, 1080); + const int frameCount = 96; + Entity[] entities = + [ + new("long-menu", "Alpha Beta Gamma Delta Epsilon", null, int.MaxValue, 80, 80, 360, 34, 0, 0, 0, false), + new("new", "New", null, int.MaxValue, 80, 190, 70, 32, 0, 0, 0, false), + new("game", "Game", null, int.MaxValue, 155, 190, 90, 32, 0, 0, 0, false), + new("score-left", "Score", null, int.MaxValue, 300, 300, 100, 30, 3, 0, 0, false), + new("score-right", "Score", null, int.MaxValue, 600, 300, 100, 30, -3, 0, 0, false), + new("inventory", "Inventory", null, int.MaxValue, 80, 430, 150, 34, 0, 0, 0, false), + new("quest", "Quest Log", null, int.MaxValue, 300, 430, 150, 34, 0.4, 0.15, 359, false), + new("hp", "HP 100", "HP 075", 38, 540, 430, 115, 34, 0, 0, 0, false, 130), + new("resume", "Resume", "Settings Menu", 57, 80, 560, 120, 36, 0, 0, 0, false, 190), + new("map", "Map", null, int.MaxValue, 340, 560, 90, 36, 0, 0, -8, false), + new("continue", "Continue", null, int.MaxValue, 520, 560, 140, 36, 0, 0, 8, false), + new("options", "Options", null, int.MaxValue, 760, 560, 130, 36, 0, 0, 0, false), + ]; + StableRandom random = new(seed); + int[] missedFrames = new int[entities.Length]; + List> observations = []; + List> expected = []; + int mergeOffset = random.Next(7); + + for (int frame = 0; frame < frameCount; frame++) + { + TextRect[] truth = entities + .Select(entity => CreateTruth(entity, frame, stabilizeChanges: false)) + .ToArray(); + expected.Add(entities + .Select(entity => CreateTruth(entity, frame, stabilizeChanges: true)) + .ToArray()); + + List current = []; + bool mergeNewGame = IsMergeFrame(frame, mergeOffset); + if (mergeNewGame) + { + TextRect first = truth[1]; + TextRect second = truth[2]; + current.Add(AddNoise( + first with + { + SourceText = "New Game", + Width = (second.X + second.Width) - first.X, + Context = "new+game", + }, + random, + allowTextError: true)); + missedFrames[1] = 0; + missedFrames[2] = 0; + } + + for (int index = 0; index < entities.Length; index++) + { + if (mergeNewGame && index is 1 or 2) + { + continue; + } + + bool drop = missedFrames[index] < 2 && random.Chance(0.08); + if (drop) + { + missedFrames[index]++; + continue; + } + missedFrames[index] = 0; + + TextRect noisy = AddNoise(truth[index], random, allowTextError: true); + bool fivePartBoundary = index == 0 && (frame + mergeOffset) % 29 == 11; + bool split = fivePartBoundary || (noisy.SourceText.Length >= 4 && random.Chance(0.07)); + if (split) + { + int partCount = fivePartBoundary ? 5 : 2 + random.Next(3); + current.AddRange(Split(noisy, partCount)); + } + else + { + current.Add(noisy); + } + } + + if ((frame + mergeOffset) % 17 == 6 || random.Chance(0.025)) + { + current.Add(new( + $"Noise-{frame}", + 1100 + random.Next(600), + 100 + random.Next(700), + 70 + random.Next(100), + 24 + random.Next(18), + 20, + false) + { + Angle = random.Next(-20, 21), + Context = "false-positive", + }); + } + + Shuffle(current, random); + observations.Add(current); + } + + return new(seed, imageSize, observations, expected); + } + + public static double Measure( + IReadOnlyList> expectedFrames, + IReadOnlyList> actualFrames) + { + Assert.Equal(expectedFrames.Count, actualFrames.Count); + return expectedFrames.Zip(actualFrames, ScoreFrame).Average(); + } + + private static TextRect CreateTruth(Entity entity, int frame, bool stabilizeChanges) + { + bool changed = frame >= entity.ChangeFrame; + if (stabilizeChanges && frame == entity.ChangeFrame) + { + changed = false; + } + double angle = entity.Id == "quest" && frame >= 48 + ? 1 + : entity.Angle; + if (stabilizeChanges && entity.Id == "quest" && frame < 51) + { + angle = entity.Angle; + } + return new( + changed ? entity.ChangedText! : entity.InitialText, + entity.X + (entity.VelocityX * frame), + entity.Y + (entity.VelocityY * frame), + changed ? entity.EffectiveChangedWidth : entity.Width, + entity.Height, + entity.Height * 0.8, + entity.MultiLine) + { + Angle = angle, + Context = entity.Id, + }; + } + + private static TextRect AddNoise(TextRect source, StableRandom random, bool allowTextError) + { + bool outlier = random.Chance(0.018); + double xNoise = outlier ? random.Next(-24, 25) : random.Next(-3, 4); + double yNoise = outlier ? random.Next(-16, 17) : random.Next(-2, 3); + string text = allowTextError && random.Chance(0.11) + ? Mutate(source.SourceText, random) + : source.SourceText; + return source with + { + SourceText = text, + X = source.X + xNoise, + Y = source.Y + yNoise, + Width = Math.Max(8, source.Width + random.Next(-5, 6)), + Height = Math.Max(8, source.Height + random.Next(-2, 3)), + FontSize = Math.Max(6, source.FontSize + (random.Next(-2, 3) * 0.5)), + Angle = source.Angle + (random.Next(-3, 4) * 0.5), + MultiLine = random.Chance(0.015) ? !source.MultiLine : source.MultiLine, + }; + } + + private static IEnumerable Split(TextRect source, int requestedPartCount) + { + string[] parts = CreateParts(source.SourceText, requestedPartCount); + double partWidth = source.Width / parts.Length; + for (int index = 0; index < parts.Length; index++) + { + yield return source with + { + SourceText = parts[index], + X = source.X + (partWidth * index), + Width = partWidth, + Context = $"{source.Context}:part-{index}", + }; + } + } + + private static string[] CreateParts(string text, int requestedPartCount) + { + string[] words = text.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (words.Length >= 2 && words.Length <= requestedPartCount) + { + return words; + } + + string compact = string.Concat(words); + int partCount = Math.Min(requestedPartCount, compact.Length); + List parts = []; + int offset = 0; + for (int part = 0; part < partCount; part++) + { + int remainingParts = partCount - part; + int length = (compact.Length - offset + remainingParts - 1) / remainingParts; + parts.Add(compact.Substring(offset, length)); + offset += length; + } + return parts.ToArray(); + } + + private static string Mutate(string text, StableRandom random) + { + if (text.Length == 0) + { + return text; + } + int index = random.Next(text.Length); + char[] characters = text.ToCharArray(); + characters[index] = characters[index] == '8' ? 'B' : '8'; + return new(characters); + } + + private static bool IsMergeFrame(int frame, int offset) + => (frame + offset) % 31 is 8 or 9; + + private static void Shuffle(List values, StableRandom random) + { + for (int index = values.Count - 1; index > 0; index--) + { + int other = random.Next(index + 1); + (values[index], values[other]) = (values[other], values[index]); + } + } + + private static double ScoreFrame( + IReadOnlyList expected, + IReadOnlyList actual) + { + int count = Math.Max(expected.Count, actual.Count); + if (count == 0) + { + return 1; + } + + double[,] scores = new double[count + 1, count + 1]; + for (int expectedIndex = 0; expectedIndex < expected.Count; expectedIndex++) + { + for (int actualIndex = 0; actualIndex < actual.Count; actualIndex++) + { + scores[expectedIndex + 1, actualIndex + 1] = ScoreRect( + expected[expectedIndex], + actual[actualIndex]); + } + } + + int[] assignment = SolveMaximumWeightAssignment(scores, count); + double score = 0; + for (int expectedIndex = 0; expectedIndex < expected.Count; expectedIndex++) + { + int actualIndex = assignment[expectedIndex]; + if (actualIndex < actual.Count) + { + score += scores[expectedIndex + 1, actualIndex + 1]; + } + } + return score / count; + } + + private static double ScoreRect(TextRect expected, TextRect actual) + { + double text = expected.SourceText == actual.SourceText ? 1 : 0; + double geometry = IntersectionOverUnion(expected, actual); + double fontSize = RatioSimilarity(expected.FontSize, actual.FontSize); + double angle = Math.Max(0, 1 - (AngleDifference(expected.Angle, actual.Angle) / 10)); + double multiLine = expected.MultiLine == actual.MultiLine ? 1 : 0; + return (text * 0.45) + + (geometry * 0.35) + + (fontSize * 0.08) + + (angle * 0.07) + + (multiLine * 0.05); + } + + private static int[] SolveMaximumWeightAssignment(double[,] weights, int count) + { + const double maximumWeight = 1; + double[] rowPotential = new double[count + 1]; + double[] columnPotential = new double[count + 1]; + int[] matching = new int[count + 1]; + int[] path = new int[count + 1]; + for (int row = 1; row <= count; row++) + { + matching[0] = row; + int column = 0; + double[] minimum = Enumerable.Repeat(double.PositiveInfinity, count + 1).ToArray(); + bool[] used = new bool[count + 1]; + do + { + used[column] = true; + int currentRow = matching[column]; + double delta = double.PositiveInfinity; + int nextColumn = 0; + for (int candidateColumn = 1; candidateColumn <= count; candidateColumn++) + { + if (used[candidateColumn]) + { + continue; + } + double reducedCost = maximumWeight - weights[currentRow, candidateColumn] + - rowPotential[currentRow] + - columnPotential[candidateColumn]; + if (reducedCost < minimum[candidateColumn]) + { + minimum[candidateColumn] = reducedCost; + path[candidateColumn] = column; + } + if (minimum[candidateColumn] < delta) + { + delta = minimum[candidateColumn]; + nextColumn = candidateColumn; + } + } + for (int candidateColumn = 0; candidateColumn <= count; candidateColumn++) + { + if (used[candidateColumn]) + { + rowPotential[matching[candidateColumn]] += delta; + columnPotential[candidateColumn] -= delta; + } + else + { + minimum[candidateColumn] -= delta; + } + } + column = nextColumn; + } + while (matching[column] != 0); + + do + { + int previousColumn = path[column]; + matching[column] = matching[previousColumn]; + column = previousColumn; + } + while (column != 0); + } + + int[] result = new int[count]; + for (int column = 1; column <= count; column++) + { + result[matching[column] - 1] = column - 1; + } + return result; + } + + private static double IntersectionOverUnion(TextRect first, TextRect second) + { + double left = Math.Max(first.X, second.X); + double top = Math.Max(first.Y, second.Y); + double right = Math.Min(first.X + first.Width, second.X + second.Width); + double bottom = Math.Min(first.Y + first.Height, second.Y + second.Height); + double intersection = Math.Max(0, right - left) * Math.Max(0, bottom - top); + double union = (first.Width * first.Height) + (second.Width * second.Height) - intersection; + return union <= 0 ? 0 : intersection / union; + } + + private static double RatioSimilarity(double first, double second) + { + double maximum = Math.Max(Math.Abs(first), Math.Abs(second)); + return maximum <= double.Epsilon ? 1 : Math.Min(Math.Abs(first), Math.Abs(second)) / maximum; + } + + private static double AngleDifference(double first, double second) + { + double difference = Math.Abs(first - second) % 360; + return Math.Min(difference, 360 - difference); + } + + private sealed record Entity( + string Id, + string InitialText, + string? ChangedText, + int ChangeFrame, + double X, + double Y, + double Width, + double Height, + double VelocityX, + double VelocityY, + double Angle, + bool MultiLine, + double ChangedWidth = 0) + { + public double EffectiveChangedWidth { get; } = ChangedWidth > 0 ? ChangedWidth : Width; + } + + private sealed class StableRandom(int seed) + { + private uint state = seed == 0 ? 0x9E3779B9 : unchecked((uint)seed); + + public int Next(int maximum) + => maximum <= 0 ? throw new ArgumentOutOfRangeException(nameof(maximum)) : (int)(this.NextUInt32() % maximum); + + public int Next(int minimum, int maximum) + => minimum + this.Next(maximum - minimum); + + public bool Chance(double probability) + => this.NextUInt32() / ((double)uint.MaxValue + 1) < probability; + + private uint NextUInt32() + { + uint value = this.state; + value ^= value << 13; + value ^= value >> 17; + value ^= value << 5; + this.state = value; + return value; + } + } +} + +internal sealed record RandomizedScenario( + int Seed, + Size ImageSize, + IReadOnlyList> Observations, + IReadOnlyList> Expected); diff --git a/WindowTranslator.Tests/WindowTranslator.Tests.csproj b/WindowTranslator.Tests/WindowTranslator.Tests.csproj new file mode 100644 index 00000000..9843f464 --- /dev/null +++ b/WindowTranslator.Tests/WindowTranslator.Tests.csproj @@ -0,0 +1,23 @@ + + + net10.0-windows10.0.20348.0 + false + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + diff --git a/WindowTranslator.sln b/WindowTranslator.sln index c57a980e..bd6f0f72 100644 --- a/WindowTranslator.sln +++ b/WindowTranslator.sln @@ -71,6 +71,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WindowTranslator.Plugin.PLa EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WindowTranslator.Plugin.GitHubCopilotPlugin", "Plugins\WindowTranslator.Plugin.GitHubCopilotPlugin\WindowTranslator.Plugin.GitHubCopilotPlugin.csproj", "{B3C4D5E6-F7A8-4B9C-8D0E-1F2A3B4C5D6E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WindowTranslator.Tests", "WindowTranslator.Tests\WindowTranslator.Tests.csproj", "{BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -357,6 +359,18 @@ Global {B3C4D5E6-F7A8-4B9C-8D0E-1F2A3B4C5D6E}.Release|x64.Build.0 = Release|Any CPU {B3C4D5E6-F7A8-4B9C-8D0E-1F2A3B4C5D6E}.Release|x86.ActiveCfg = Release|Any CPU {B3C4D5E6-F7A8-4B9C-8D0E-1F2A3B4C5D6E}.Release|x86.Build.0 = Release|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Debug|x64.ActiveCfg = Debug|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Debug|x64.Build.0 = Debug|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Debug|x86.ActiveCfg = Debug|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Debug|x86.Build.0 = Debug|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Release|Any CPU.Build.0 = Release|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Release|x64.ActiveCfg = Release|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Release|x64.Build.0 = Release|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Release|x86.ActiveCfg = Release|Any CPU + {BB9CDCE9-DB41-4C9B-9AAA-729BC5340482}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/WindowTranslator/FilterPriority.cs b/WindowTranslator/FilterPriority.cs index fd5086b5..fe9bb74d 100644 --- a/WindowTranslator/FilterPriority.cs +++ b/WindowTranslator/FilterPriority.cs @@ -2,5 +2,4 @@ public static class FilterPriority { public static double OcrCommonFilter => -110.0; - public static double OcrBufferFilter => -100.0; } diff --git a/WindowTranslator/Modules/ErrorReport/ErrorReportViewModel.cs b/WindowTranslator/Modules/ErrorReport/ErrorReportViewModel.cs index 08ca8d61..9a97c2a7 100644 --- a/WindowTranslator/Modules/ErrorReport/ErrorReportViewModel.cs +++ b/WindowTranslator/Modules/ErrorReport/ErrorReportViewModel.cs @@ -148,7 +148,6 @@ private static string GetInfo(Exception ex, UserSettings? settings, string targe sb.AppendLine(CultureInfo.CurrentCulture, $"OverlayShortcut: {value.OverlayShortcut ?? "N/A"}"); sb.AppendLine(CultureInfo.CurrentCulture, $"OverlayOpacity: {value.OverlayOpacity}"); sb.AppendLine(CultureInfo.CurrentCulture, $"DisplayBusy: {value.DisplayBusy}"); - sb.AppendLine(CultureInfo.CurrentCulture, $"IsOneShotMode: {value.IsOneShotMode}"); if (value.SelectedPlugins?.Count > 0) { diff --git a/WindowTranslator/Modules/Main/MainViewModelBase.cs b/WindowTranslator/Modules/Main/MainViewModelBase.cs index 02c49580..f6cdfb2c 100644 --- a/WindowTranslator/Modules/Main/MainViewModelBase.cs +++ b/WindowTranslator/Modules/Main/MainViewModelBase.cs @@ -12,6 +12,7 @@ using WindowTranslator.ComponentModel; using WindowTranslator.Extensions; using WindowTranslator.Modules.Capture; +using WindowTranslator.Modules.Ocr; using WindowTranslator.Properties; using WindowTranslator.Stores; using static System.Windows.Forms.VisualStyles.VisualStyleElement; @@ -23,6 +24,7 @@ public abstract partial class MainViewModelBase : IDisposable { private readonly Timer timer; private readonly IOcrModule ocr; + private readonly IOcrTextTracker ocrTextTracker; private readonly ITranslateModule translator; private readonly ICacheModule cache; private readonly IColorModule color; @@ -36,10 +38,8 @@ public abstract partial class MainViewModelBase : IDisposable private readonly IProcessInfoStore processInfoStore; private readonly double fontScale; private readonly double overlayOpacity; - private readonly bool isOneShotModeEnabled; private readonly double mousePointerHitTestPadding; private TextRect[]? lastRequested; - private bool isFirstCapture = true; [ObservableProperty] private string title; @@ -71,6 +71,7 @@ public MainViewModelBase( IProcessInfoStore processInfoStore, ICaptureModule capture, IOcrModule ocr, + IOcrTextTracker ocrTextTracker, ITranslateModule translator, ICacheModule cache, IColorModule color, @@ -83,19 +84,17 @@ public MainViewModelBase( this.Font = options.Value.Font; this.fontScale = options.Value.FontScale; this.overlayOpacity = options.Value.OverlayOpacity; - this.isOneShotModeEnabled = options.Value.IsOneShotMode; this.mousePointerHitTestPadding = options.Value.MousePointerHitTestPadding; this.DisplayBusy = options.Value.DisplayBusy; this.capture = capture ?? throw new ArgumentNullException(nameof(capture)); this.capture.Captured += Capture_CapturedAsync; this.ocr = ocr ?? throw new ArgumentNullException(nameof(ocr)); + this.ocrTextTracker = ocrTextTracker ?? throw new ArgumentNullException(nameof(ocrTextTracker)); this.translator = translator ?? throw new ArgumentNullException(nameof(translator)); this.cache = cache ?? throw new ArgumentNullException(nameof(cache)); this.color = color ?? throw new ArgumentNullException(nameof(color)); this.filters = filters.ToArray(); this.logger = logger; - // Capture will be started/stopped based on OverlayVisible property changes - this.isFirstCapture = this.isOneShotModeEnabled; // Only set to true if feature is enabled this.capture.StartCapture(processInfoStore.MainWindowHandle); this.timer = new(_ => Application.Current.Dispatcher.Invoke(() => CreateTextOverlayAsync().Forget()), null, 0, 500); var transAsm = this.translator.GetType().Assembly; @@ -107,15 +106,9 @@ partial void OnOverlayVisibleChanged(bool value) if (value) { this.OcrTexts.Clear(); + this.ocrTextTracker.Reset(); // Start capture when overlay becomes visible this.capture.StartCapture(this.processInfoStore.MainWindowHandle); - - // Reset first capture flag for one-shot mode - if (this.isOneShotModeEnabled) - { - this.isFirstCapture = true; - this.logger.LogDebug("Overlay became visible - first capture flag reset (one shot mode enabled)"); - } } else { @@ -165,53 +158,40 @@ private async Task CreateTextOverlayAsync() return; } - IEnumerable texts = []; - - if (!this.isOneShotModeEnabled || isFirstCapture) + IEnumerable texts; + using (this.Recognizing.EnterBusy()) { - if (this.isOneShotModeEnabled) + try { - this.logger.LogDebug("OCRトリガー実行"); - this.isFirstCapture = false; + texts = await this.ocr.RecognizeAsync(sbmp); + texts = this.ocrTextTracker.Update(texts, new(sbmp.PixelWidth, sbmp.PixelHeight)); } - - using (this.Recognizing.EnterBusy()) + catch (ObjectDisposedException) { - try - { - texts = await this.ocr.RecognizeAsync(sbmp); - } - catch (ObjectDisposedException) - { - // すでに破棄されている場合は何もしない - this.timer.DisposeAsync().Forget(); - this.capture.StopCapture(); - return; - } - catch (OperationCanceledException) - { - // キャンセルされた場合は何もしない - this.timer.DisposeAsync().Forget(); - this.capture.StopCapture(); - return; - } - catch (Exception e) - { - this.timer.DisposeAsync().Forget(); - this.capture.StopCapture(); - var path = Path.Combine(PathUtility.UserDir, $"ocr_error", $"{DateTime.UtcNow:yyyyMMdd'T'HHmmss'Z'}.png"); - await sbmp.TrySaveImage(path); - await this.presentationService.OpenErrorDialogAsync(Resources.FaildOcr, e, this.name, path); - StrongReferenceMessenger.Default.Send(new(this)); - return; - } + // すでに破棄されている場合は何もしない + this.timer.DisposeAsync().Forget(); + this.capture.StopCapture(); + return; + } + catch (OperationCanceledException) + { + // キャンセルされた場合は何もしない + this.timer.DisposeAsync().Forget(); + this.capture.StopCapture(); + return; + } + catch (Exception e) + { + this.timer.DisposeAsync().Forget(); + this.capture.StopCapture(); + var path = Path.Combine(PathUtility.UserDir, $"ocr_error", $"{DateTime.UtcNow:yyyyMMdd'T'HHmmss'Z'}.png"); + await sbmp.TrySaveImage(path); + await this.presentationService.OpenErrorDialogAsync(Resources.FaildOcr, e, this.name, path); + StrongReferenceMessenger.Default.Send(new(this)); + return; } - texts = texts.Select(t => t with { FontSize = t.FontSize * this.fontScale }); - } - else - { - texts = this.OcrTexts.ToArray(); } + texts = texts.Select(t => t with { FontSize = t.FontSize * this.fontScale }); // フィルター&翻訳処理は必ず通す using (this.Filtering.EnterBusy()) @@ -345,12 +325,13 @@ public sealed class CaptureMainViewModel( [Inject] IProcessInfoStore processInfoStore, [Inject] ICaptureModule capture, [Inject] IOcrModule ocr, + [Inject] IOcrTextTracker ocrTextTracker, [Inject] ITranslateModule translator, [Inject] ICacheModule cache, [Inject] IColorModule color, [Inject] IEnumerable filters, [Inject] ILogger logger) - : MainViewModelBase(presentationService, options, processInfoStore, capture, ocr, translator, cache, color, filters, logger) + : MainViewModelBase(presentationService, options, processInfoStore, capture, ocr, ocrTextTracker, translator, cache, color, filters, logger) { public ICaptureModule Capture { get; } = capture ?? throw new ArgumentNullException(nameof(capture)); } @@ -362,11 +343,12 @@ public sealed class OverlayMainViewModel( [Inject] IProcessInfoStore processInfoStore, [Inject] ICaptureModule capture, [Inject] IOcrModule ocr, + [Inject] IOcrTextTracker ocrTextTracker, [Inject] ITranslateModule translator, [Inject] ICacheModule cache, [Inject] IColorModule color, [Inject] IEnumerable filters, [Inject] ILogger logger) - : MainViewModelBase(presentationService, options, processInfoStore, capture, ocr, translator, cache, color, filters, logger) + : MainViewModelBase(presentationService, options, processInfoStore, capture, ocr, ocrTextTracker, translator, cache, color, filters, logger) { } diff --git a/WindowTranslator/Modules/Ocr/IOcrTextTracker.cs b/WindowTranslator/Modules/Ocr/IOcrTextTracker.cs new file mode 100644 index 00000000..5beb2711 --- /dev/null +++ b/WindowTranslator/Modules/Ocr/IOcrTextTracker.cs @@ -0,0 +1,19 @@ +using System.Drawing; + +namespace WindowTranslator.Modules.Ocr; + +/// +/// OCRの観測結果をフレーム間で追跡し、安定化したテキスト領域を返す。 +/// +public interface IOcrTextTracker +{ + /// + /// 現在フレームのOCR観測でトラックを更新する。 + /// + IReadOnlyList Update(IEnumerable observations, Size imageSize); + + /// + /// 保持しているすべてのトラックを破棄する。 + /// + void Reset(); +} diff --git a/WindowTranslator/Modules/Ocr/OcrBufferFilter.cs b/WindowTranslator/Modules/Ocr/OcrBufferFilter.cs deleted file mode 100644 index ea71a6b2..00000000 --- a/WindowTranslator/Modules/Ocr/OcrBufferFilter.cs +++ /dev/null @@ -1,140 +0,0 @@ -using System.Drawing; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.ObjectPool; -using Microsoft.Extensions.Options; -using Quickenshtein; -using WindowTranslator.Extensions; - -namespace WindowTranslator.Modules.Ocr; - -public class OcrBufferFilter(IOptionsSnapshot target, IOptionsSnapshot options, ILogger logger) : IFilterModule -{ - private static readonly ObjectPool> listPool = ObjectPool.Create(new ListPolicy()); - private readonly ILogger logger = logger; - private readonly Queue> buffer = new(); - private readonly bool isOneShotMode = target.Value.IsOneShotMode; - private readonly int bufferSize = options.Value.BufferSize; - private readonly bool isSuppressVibe = options.Value.IsSuppressVibe; - private readonly bool isEnableRecover = options.Value.IsEnableRecover; - - public double Priority => FilterPriority.OcrBufferFilter; - - public async IAsyncEnumerable ExecutePreTranslate(IAsyncEnumerable texts, FilterContext context) - { - if (this.bufferSize <= 0 || this.isOneShotMode) - { - await foreach (var text in texts.ConfigureAwait(false)) - { - yield return text; - } - yield break; - } - using var l = this.logger.LogDebugTime("OcrBufferFilter"); - var threshold = (context.ImageSize * 0.02f).ToSize(); - // バッファ内の全テキストを集め、AreSimilarで重複チェックして重複を排除 - var bufferedTexts = listPool.Get(); - foreach (var rect in this.buffer.SelectMany(b => b)) - { - if (!bufferedTexts.Any(existing => AreSimilar(existing, rect, threshold))) - { - bufferedTexts.Add(rect); - } - } - - List currentTextsList; - if (buffer.Count == this.bufferSize) - { - // 既にプールされたリストを再利用 - currentTextsList = buffer.Dequeue(); - currentTextsList.Clear(); - } - else - { - // 新規作成ではなく、プールから取得 - currentTextsList = listPool.Get(); - } - - // 現在のテキストを列挙しながら処理 - await foreach (var t in texts.ConfigureAwait(false)) - { - var text = t; - // 過去のバッファ内に類似するテキストがあるか確認 - // もしあって、かつフォントサイズが異なる場合は過去のテキストのフォントサイズを使用 - if (this.isSuppressVibe && bufferedTexts.FirstOrDefault(buf => AreSimilar(buf, text, threshold)) is { } pastText) - { - // フォントサイズを平均化 - text = text with - { - X = pastText.X, - Y = pastText.Y, - Width = Math.Max(text.Width, pastText.Width), - Height = pastText.Height, - FontSize = pastText.FontSize - }; - } - currentTextsList.Add(text); - yield return text; - } - - // バッファに現在のテキストを追加 - this.buffer.Enqueue(currentTextsList); - - // finalBufferedCacheを再利用: 事前にクリアしてから、bufferedTextsから候補を追加 - var finalBuffered = listPool.Get(); - if (this.isEnableRecover) - { - foreach (var buf in bufferedTexts) - { - if (!currentTextsList.Any(t => AreSimilar(t, buf, threshold) || Intersects(t, buf)) && - !finalBuffered.Any(existing => Intersects(existing, buf))) - { - finalBuffered.Add(buf); - this.logger.LogDebug($"Buffered: {buf.SourceText}"); - yield return buf; - } - } - } - - // リストを返却 - listPool.Return(bufferedTexts); - listPool.Return(finalBuffered); - } - - public IAsyncEnumerable ExecutePostTranslate(IAsyncEnumerable texts, FilterContext context) => texts; - - // 矩形同士が重なっているかを判定する追加メソッド - private static bool Intersects(TextRect rect1, TextRect rect2) - => rect1.X < rect2.X + rect2.Width && - rect1.X + rect1.Width > rect2.X && - rect1.Y < rect2.Y + rect2.Height && - rect1.Y + rect1.Height > rect2.Y; - - private static bool AreSimilar(TextRect rect1, TextRect rect2, Size threshold) - { - // テキストの一致率が80%未満なら別扱い - var p = 1 - ((float)Levenshtein.GetDistance(rect1.SourceText, rect2.SourceText, CalculationOptions.DefaultWithThreading) - / Math.Max(rect1.SourceText.Length, rect2.SourceText.Length)); - - if (p < 0.8) - { - return false; - } - - // 位置とサイズのずれがしきい値以下だったら同じとみなす - return Math.Abs(rect1.X - rect2.X) <= threshold.Width && - Math.Abs(rect1.Y - rect2.Y) <= threshold.Height && - Math.Abs(rect1.Width - rect2.Width) <= threshold.Width && - Math.Abs(rect1.Height - rect2.Height) <= threshold.Height; - } -} - -file class ListPolicy : IPooledObjectPolicy> -{ - public List Create() => []; - - public bool Return(List obj) - { - obj.Clear(); - return true; - } -} diff --git a/WindowTranslator/Modules/Ocr/OcrTextTracker.cs b/WindowTranslator/Modules/Ocr/OcrTextTracker.cs new file mode 100644 index 00000000..42ddd469 --- /dev/null +++ b/WindowTranslator/Modules/Ocr/OcrTextTracker.cs @@ -0,0 +1,1837 @@ +using System.Diagnostics; +using System.Drawing; +using System.Numerics; +using System.Text; +using Microsoft.Extensions.Logging; +using Quickenshtein; + +namespace WindowTranslator.Modules.Ocr; + +/// +/// OCRテキスト領域を1対1割当てと分割・統合候補によって継続追跡する。 +/// +public sealed class OcrTextTracker(ILogger logger) : IOcrTextTracker +{ + private const int MaxMissedFrames = 3; + private const int StructureConfirmationFrames = 2; + private const int MicroGeometryConfirmationFrames = 4; + private const int MaxStructureCandidates = 6; + private const int MaxStructureSelectionStates = 1024; + private const int MaxOneToOneCandidatesPerResource = 3; + private const double MinimumAssignmentScore = 0.58; + private const double StructureAssignmentBonus = 0.05; + private const double TextVoteDecay = 0.75; + private const double TextVoteThreshold = 1.5; + private const int TextVoteHistorySize = 5; + private const double MinimumStructureSizeRatio = 0.65; + private const double MinimumStructureOverlap = 0.45; + private const double MinimumStructureTextSimilarity = 0.65; + private const double MinimumMovingStructureTextSimilarity = 0.9; + private const double MaximumStructureAngleDifference = 8; + private const double StrongOneToOneScore = 0.9; + private const double AngleVectorEpsilon = 0.000000000001; + private static readonly TimeSpan dormantRetention = TimeSpan.FromSeconds(5); + + private readonly ILogger logger = logger; + private readonly object syncRoot = new(); + private readonly List tracks = []; + private Dictionary mergeCandidates = []; + private Dictionary dormantRestoreCandidates = []; + private long nextTrackId; + + public IReadOnlyList Update(IEnumerable observations, Size imageSize) + => this.Update( + observations, + imageSize, + TimeSpan.FromSeconds((double)Stopwatch.GetTimestamp() / Stopwatch.Frequency)); + + public IReadOnlyList Update(IEnumerable observations, Size imageSize, TimeSpan timestamp) + { + ArgumentNullException.ThrowIfNull(observations); + lock (this.syncRoot) + { + return this.UpdateCore(observations, imageSize, timestamp); + } + } + + public void Reset() + { + lock (this.syncRoot) + { + this.tracks.Clear(); + this.mergeCandidates.Clear(); + this.dormantRestoreCandidates.Clear(); + this.nextTrackId = 0; + } + } + + private TextRect[] UpdateCore(IEnumerable observations, Size imageSize, TimeSpan timestamp) + { + TextRect[] current = observations.Where(IsValid).ToArray(); + if (this.tracks.Count == 0) + { + foreach (TextRect observation in current) + { + this.CreateTrack(observation, timestamp); + } + return this.GetOutput(); + } + + string[] normalizedCurrent = current + .Select(observation => NormalizeText(observation.SourceText)) + .ToArray(); + + HashSet matchedTracks = []; + HashSet matchedObservations = []; + TextTrack[] activeTracks = this.tracks + .Where(track => !track.IsDormant) + .ToArray(); + List oneToOneCandidates = BuildOneToOneCandidates( + activeTracks, + current, + normalizedCurrent, + imageSize, + timestamp); + MatchCandidate[] strongMatches = SelectOneToOne( + oneToOneCandidates + .Where(candidate => IsStrongOneToOneCandidate(candidate, normalizedCurrent)) + .ToArray()); + HashSet reservedTracks = strongMatches + .SelectMany(candidate => candidate.Tracks) + .ToHashSet(); + HashSet reservedObservations = strongMatches + .SelectMany(candidate => candidate.ObservationIndices) + .ToHashSet(); + + TextTrack[] unresolvedTracks = activeTracks + .Where(track => !reservedTracks.Contains(track)) + .ToArray(); + List structureCandidates = BuildStructureCandidates( + unresolvedTracks, + current, + normalizedCurrent, + imageSize, + reservedObservations); + structureCandidates.AddRange(this.BuildRestorationCandidates( + unresolvedTracks, + current, + normalizedCurrent, + imageSize, + timestamp, + reservedObservations)); + List selected = [.. strongMatches, .. SelectStructures(structureCandidates)]; + reservedTracks.UnionWith(selected.SelectMany(candidate => candidate.Tracks)); + reservedObservations.UnionWith(selected.SelectMany(candidate => candidate.ObservationIndices)); + + selected.AddRange(SelectOneToOne( + oneToOneCandidates + .Where(candidate => !reservedTracks.Contains(candidate.Tracks[0])) + .Where(candidate => !reservedObservations.Contains(candidate.ObservationIndices[0])) + .ToArray())); + this.ApplyMatches(selected, timestamp, matchedTracks, matchedObservations); + + for (int i = 0; i < current.Length; i++) + { + if (!matchedObservations.Contains(i)) + { + TextTrack? coveringTrack = activeTracks.FirstOrDefault(track => + IsCoveredTextFragment(current[i], track.Stabilized)); + TextTrack track = this.CreateTrack(current[i], timestamp, coveringTrack?.Id); + matchedTracks.Add(track); + } + } + + foreach (TextTrack track in this.tracks.Where(track => !track.IsDormant).ToArray()) + { + if (!matchedTracks.Contains(track)) + { + TextTrack? coveringTrack = track.CoveringTrackId is long coveringTrackId + ? matchedTracks.FirstOrDefault(candidate => candidate.Id == coveringTrackId && !candidate.IsDormant) + : null; + if (coveringTrack is not null + && IsCoveredTextFragment(track.Stabilized, coveringTrack.Stabilized)) + { + this.logger.LogDebug( + "OCR track {TrackId} removed as a stale fragment of a matched track", + track.Id); + this.RemoveTrackTree(track); + continue; + } + track.MarkMissed(); + } + } + + foreach (TextTrack track in this.tracks + .Where(track => (!track.IsDormant && track.MissedFrames > MaxMissedFrames) + || (track.IsDormant + && !matchedTracks.Contains(track) + && timestamp - track.DormantSince > dormantRetention)) + .ToArray()) + { + this.logger.LogDebug("OCR track {TrackId} expired after {MissedFrames} missed frames", track.Id, track.MissedFrames); + this.RemoveTrackTree(track); + } + + return this.GetOutput(); + } + + private static bool IsValid(TextRect rect) + => !string.IsNullOrWhiteSpace(rect.SourceText) + && double.IsFinite(rect.X) + && double.IsFinite(rect.Y) + && double.IsFinite(rect.Width) + && double.IsFinite(rect.Height) + && double.IsFinite(rect.FontSize) + && double.IsFinite(rect.Angle) + && rect.Width > 0 + && rect.Height > 0; + + private static List BuildStructureCandidates( + TextTrack[] tracks, + TextRect[] observations, + string[] normalizedObservations, + Size imageSize, + HashSet excludedObservations) + { + List result = []; + foreach (TextTrack track in tracks) + { + int[] nearby = Enumerable.Range(0, observations.Length) + .Where(index => !excludedObservations.Contains(index)) + .Where(index => IsPotentialStructureMember(track.Stabilized, observations[index], imageSize)) + .OrderBy(index => CenterDistance(track.Stabilized, observations[index])) + .Take(MaxStructureCandidates) + .ToArray(); + for (int memberCount = 2; memberCount <= nearby.Length; memberCount++) + { + foreach (IReadOnlyList members in Combinations(nearby, memberCount)) + { + TextRect[] memberRects = members.Select(index => observations[index]).ToArray(); + if (!MembersAreAdjacent(memberRects) || !MembersHaveCompatibleStyle(memberRects)) + { + continue; + } + if (TryCombineStructure( + memberRects, + members.Select(index => normalizedObservations[index]).ToArray(), + track.Stabilized, + track.ConfirmedText, + track.NormalizedConfirmedText, + imageSize, + out TextRect combined, + out double score)) + { + result.Add(new(MatchKind.Split, [track], members.ToArray(), combined, score)); + } + } + } + } + + foreach (int observationIndex in Enumerable.Range(0, observations.Length) + .Where(index => !excludedObservations.Contains(index))) + { + TextRect observation = observations[observationIndex]; + TextTrack[] nearby = tracks + .Where(track => IsPotentialStructureMember(observation, track.Stabilized, imageSize)) + .OrderBy(track => CenterDistance(observation, track.Stabilized)) + .Take(MaxStructureCandidates) + .ToArray(); + for (int memberCount = 2; memberCount <= nearby.Length; memberCount++) + { + foreach (IReadOnlyList members in Combinations(nearby, memberCount)) + { + TextRect[] memberRects = members.Select(track => track.Stabilized).ToArray(); + if (!MembersAreAdjacent(memberRects) || !MembersHaveCompatibleStyle(memberRects)) + { + continue; + } + if (TryCombineStructure( + memberRects, + members.Select(track => track.NormalizedConfirmedText).ToArray(), + observation, + observation.SourceText, + normalizedObservations[observationIndex], + imageSize, + out _, + out double score)) + { + result.Add(new(MatchKind.Merge, members.ToArray(), [observationIndex], observation, score)); + } + } + } + } + return result; + } + + private static List BuildOneToOneCandidates( + IReadOnlyList tracks, + TextRect[] observations, + string[] normalizedObservations, + Size imageSize, + TimeSpan timestamp, + Func? geometryOverride = null, + Func? distanceGateDimensionOverride = null, + HashSet? excludedObservations = null) + { + List result = []; + foreach (TextTrack track in tracks) + { + TextRect? geometry = geometryOverride?.Invoke(track); + for (int observationIndex = 0; observationIndex < observations.Length; observationIndex++) + { + if (excludedObservations?.Contains(observationIndex) == true) + { + continue; + } + string normalizedObservation = normalizedObservations[observationIndex]; + double score = ScoreAssignment( + track, + observations[observationIndex], + normalizedObservation, + imageSize, + timestamp, + geometry, + distanceGateDimensionOverride?.Invoke(track)); + if (score >= MinimumAssignmentScore) + { + result.Add(new(MatchKind.OneToOne, [track], [observationIndex], observations[observationIndex], score)); + } + } + } + return result; + } + + private static bool IsStrongOneToOneCandidate( + MatchCandidate candidate, + string[] normalizedObservations) + => candidate.Score >= StrongOneToOneScore + && candidate.Tracks[0].NormalizedConfirmedText + == normalizedObservations[candidate.ObservationIndices[0]]; + + private static MatchCandidate[] SelectOneToOne(IReadOnlyList candidates) + { + if (candidates.Count == 0) + { + return []; + } + + Dictionary candidatesByTrack = candidates + .GroupBy(candidate => candidate.Tracks[0]) + .ToDictionary(group => group.Key, group => group.ToArray()); + Dictionary candidatesByObservation = candidates + .GroupBy(candidate => candidate.ObservationIndices[0]) + .ToDictionary(group => group.Key, group => group.ToArray()); + HashSet visited = new(ReferenceEqualityComparer.Instance); + List selected = []; + foreach (MatchCandidate first in candidates) + { + if (!visited.Add(first)) + { + continue; + } + + List component = []; + Queue pending = new([first]); + while (pending.TryDequeue(out MatchCandidate? candidate)) + { + component.Add(candidate); + foreach (MatchCandidate neighbor in candidatesByTrack[candidate.Tracks[0]]) + { + if (visited.Add(neighbor)) + { + pending.Enqueue(neighbor); + } + } + foreach (MatchCandidate neighbor in candidatesByObservation[candidate.ObservationIndices[0]]) + { + if (visited.Add(neighbor)) + { + pending.Enqueue(neighbor); + } + } + } + selected.AddRange(SelectOneToOneComponent(component)); + } + return selected.ToArray(); + } + + private static MatchCandidate[] SelectOneToOneComponent(IReadOnlyList candidates) + { + if (candidates.Count == 1) + { + return [candidates[0]]; + } + + TextTrack[] tracks = candidates + .Select(candidate => candidate.Tracks[0]) + .Distinct() + .ToArray(); + int[] observations = candidates + .Select(candidate => candidate.ObservationIndices[0]) + .Distinct() + .Order() + .ToArray(); + int count = Math.Max(tracks.Length, observations.Length); + Dictionary<(TextTrack track, int observation), MatchCandidate> byPair = candidates + .ToDictionary(candidate => (candidate.Tracks[0], candidate.ObservationIndices[0])); + double cardinalityBonus = count + 1; + double maximumWeight = cardinalityBonus + 1; + double[,] costs = new double[count + 1, count + 1]; + for (int trackIndex = 0; trackIndex < count; trackIndex++) + { + for (int observationIndex = 0; observationIndex < count; observationIndex++) + { + double weight = trackIndex < tracks.Length + && observationIndex < observations.Length + && byPair.TryGetValue( + (tracks[trackIndex], observations[observationIndex]), + out MatchCandidate? candidate) + ? cardinalityBonus + candidate.Score + : 0; + costs[trackIndex + 1, observationIndex + 1] = maximumWeight - weight; + } + } + + int[] assignment = SolveMinimumCostAssignment(costs, count); + return tracks + .Select((track, index) => assignment[index] < observations.Length + ? byPair.GetValueOrDefault((track, observations[assignment[index]])) + : null) + .OfType() + .ToArray(); + } + + private static int[] SolveMinimumCostAssignment(double[,] costs, int count) + { + double[] rowPotential = new double[count + 1]; + double[] columnPotential = new double[count + 1]; + int[] matching = new int[count + 1]; + int[] path = new int[count + 1]; + + for (int row = 1; row <= count; row++) + { + matching[0] = row; + int column = 0; + double[] minimum = Enumerable.Repeat(double.PositiveInfinity, count + 1).ToArray(); + bool[] used = new bool[count + 1]; + do + { + used[column] = true; + int currentRow = matching[column]; + double delta = double.PositiveInfinity; + int nextColumn = 0; + for (int candidateColumn = 1; candidateColumn <= count; candidateColumn++) + { + if (used[candidateColumn]) + { + continue; + } + double reducedCost = costs[currentRow, candidateColumn] + - rowPotential[currentRow] + - columnPotential[candidateColumn]; + if (reducedCost < minimum[candidateColumn]) + { + minimum[candidateColumn] = reducedCost; + path[candidateColumn] = column; + } + if (minimum[candidateColumn] < delta) + { + delta = minimum[candidateColumn]; + nextColumn = candidateColumn; + } + } + + for (int candidateColumn = 0; candidateColumn <= count; candidateColumn++) + { + if (used[candidateColumn]) + { + rowPotential[matching[candidateColumn]] += delta; + columnPotential[candidateColumn] -= delta; + } + else + { + minimum[candidateColumn] -= delta; + } + } + column = nextColumn; + } + while (matching[column] != 0); + + do + { + int previousColumn = path[column]; + matching[column] = matching[previousColumn]; + column = previousColumn; + } + while (column != 0); + } + + int[] result = new int[count]; + for (int column = 1; column <= count; column++) + { + result[matching[column] - 1] = column - 1; + } + return result; + } + + private static double ScoreAssignment( + TextTrack track, + TextRect observation, + string normalizedObservation, + Size imageSize, + TimeSpan timestamp, + TextRect? geometryOverride = null, + double? distanceGateDimensionOverride = null) + { + TextRect previous = geometryOverride ?? track.LatestObservation; + TextRect predicted = geometryOverride ?? track.Predict(timestamp); + double centerDistance = CenterDistance(predicted, observation); + double maximumDimension = Math.Max(previous.Width, previous.Height); + double distanceGate = TrackingDistanceGate(previous, imageSize, distanceGateDimensionOverride); + double overlap = IntersectionOverUnion(predicted, observation); + if (overlap == 0 && centerDistance > distanceGate) + { + return -1; + } + + string trackText = track.NormalizedConfirmedText; + string observationText = normalizedObservation; + if (overlap == 0 + && !CanReachTextSimilarity(trackText.Length, observationText.Length, 0.9) + && centerDistance > maximumDimension * 1.25) + { + return -1; + } + + double center = Math.Max(0, 1 - (centerDistance / distanceGate)); + double width = RatioSimilarity(previous.Width, observation.Width); + double height = RatioSimilarity(previous.Height, observation.Height); + double size = (width + height) / 2; + double coverage = IntersectionOverSmallerArea(predicted, observation); + bool sameRegionGeometry = HasSameRegionGeometry(predicted, observation, coverage, height); + if (!CanReachTextSimilarity(trackText.Length, observationText.Length, 0.15) + && overlap < 0.3 + && !sameRegionGeometry) + { + return -1; + } + if (!CanReachTextSimilarity(trackText.Length, observationText.Length, 0.65) + && size < 0.8 + && !sameRegionGeometry) + { + return -1; + } + + double text = NormalizedTextSimilarity(trackText, observationText); + if (overlap == 0 && text < 0.9 && centerDistance > maximumDimension * 1.25) + { + return -1; + } + bool sameRegionContentChange = text < 0.65 && sameRegionGeometry; + if (text < 0.15 && overlap < 0.3 && !sameRegionContentChange) + { + return -1; + } + if (text < 0.65 && size < 0.8 && !sameRegionContentChange) + { + return -1; + } + double aspect = RatioSimilarity(previous.Width / previous.Height, observation.Width / observation.Height); + double recency = Math.Max(0, 1 - ((double)track.MissedFrames / (MaxMissedFrames + 1))); + double score = (text * 0.35) + + (overlap * 0.25) + + (center * 0.2) + + (size * 0.1) + + (aspect * 0.05) + + (recency * 0.05); + if (!sameRegionContentChange) + { + return score; + } + + double angle = Math.Max(0, 1 - (AngleDifference(predicted.Angle, observation.Angle) / 10)); + double replacementScore = (overlap * 0.15) + + (center * 0.2) + + (height * 0.2) + + (coverage * 0.25) + + (angle * 0.1) + + (recency * 0.05); + return Math.Max(score, replacementScore); + } + + private static bool HasSameRegionGeometry( + TextRect previous, + TextRect observation, + double coverage, + double heightSimilarity) + { + double positionTolerance = Math.Max(3, Math.Min(previous.Height, observation.Height) * 0.2); + bool readingOriginAligned = Math.Abs(previous.X - observation.X) <= positionTolerance + && Math.Abs(previous.Y - observation.Y) <= positionTolerance; + bool centerAligned = CenterDistance(previous, observation) <= positionTolerance; + return coverage >= 0.8 + && heightSimilarity >= MinimumStructureSizeRatio + && AngleDifference(previous.Angle, observation.Angle) <= MaximumStructureAngleDifference + && (readingOriginAligned || centerAligned); + } + + private static bool TryCombineStructure( + TextRect[] source, + string[] normalizedSource, + TextRect targetRect, + string targetText, + string normalizedTargetText, + Size imageSize, + out TextRect combined, + out double score) + { + (TextRect Rect, string Normalized)[] orderedMembers = OrderForReading( + source.Select((rect, index) => (Rect: rect, Normalized: normalizedSource[index])), + member => member.Rect) + .ToArray(); + TextRect[] members = orderedMembers.Select(member => member.Rect).ToArray(); + TextRect geometry = Union(members, string.Empty); + double overlap = IntersectionOverUnion(geometry, targetRect); + double center = 0; + double size = 0; + double aspect = 0; + double minimumTextSimilarity = MinimumStructureTextSimilarity; + if (overlap < MinimumStructureOverlap) + { + double distanceGate = TrackingDistanceGate(targetRect, imageSize); + double centerDistance = CenterDistance(geometry, targetRect); + size = (RatioSimilarity(geometry.Width, targetRect.Width) + + RatioSimilarity(geometry.Height, targetRect.Height)) / 2; + aspect = RatioSimilarity( + geometry.Width / geometry.Height, + targetRect.Width / targetRect.Height); + if (centerDistance > distanceGate + || size < MinimumStructureSizeRatio + || aspect < MinimumStructureSizeRatio) + { + combined = default!; + score = -1; + return false; + } + center = Math.Max(0, 1 - (centerDistance / distanceGate)); + minimumTextSimilarity = MinimumMovingStructureTextSimilarity; + } + + if (!TrySelectCombinedText( + members, + orderedMembers.Select(member => member.Normalized).ToArray(), + targetText, + normalizedTargetText, + minimumTextSimilarity, + out string text, + out double textSimilarity)) + { + combined = default!; + score = -1; + return false; + } + + combined = geometry with { SourceText = text }; + score = overlap >= MinimumStructureOverlap + ? (overlap * 0.55) + (textSimilarity * 0.45) + StructureAssignmentBonus + : (textSimilarity * 0.45) + + (center * 0.25) + + (size * 0.2) + + (aspect * 0.1) + + StructureAssignmentBonus; + return true; + } + + private static bool TrySelectCombinedText( + TextRect[] members, + string[] normalizedMembers, + string targetText, + string normalizedTarget, + double minimumSimilarity, + out string text, + out double similarity) + { + int withoutSpacesLength = normalizedMembers.Sum(member => member.Length); + int withSpacesLength = withoutSpacesLength + members.Length - 1; + double withSpacesSimilarity = -1; + double withoutSpacesSimilarity = -1; + if (CanReachTextSimilarity(withSpacesLength, normalizedTarget.Length, minimumSimilarity)) + { + string normalizedWithSpaces = string.Join(' ', normalizedMembers); + withSpacesSimilarity = NormalizedTextSimilarity(normalizedWithSpaces, normalizedTarget); + } + if (CanReachTextSimilarity(withoutSpacesLength, normalizedTarget.Length, minimumSimilarity)) + { + string normalizedWithoutSpaces = string.Concat(normalizedMembers); + withoutSpacesSimilarity = NormalizedTextSimilarity(normalizedWithoutSpaces, normalizedTarget); + } + if (withSpacesSimilarity < minimumSimilarity && withoutSpacesSimilarity < minimumSimilarity) + { + text = string.Empty; + similarity = -1; + return false; + } + + bool useSpaces = withSpacesSimilarity >= withoutSpacesSimilarity; + text = useSpaces + ? string.Join(' ', members.Select(member => member.SourceText)) + : string.Concat(members.Select(member => member.SourceText)); + similarity = useSpaces ? withSpacesSimilarity : withoutSpacesSimilarity; + return true; + } + + private static bool CanReachTextSimilarity(int firstLength, int secondLength, double minimumSimilarity) + { + int maximumLength = Math.Max(firstLength, secondLength); + return maximumLength == 0 + || (double)Math.Min(firstLength, secondLength) / maximumLength >= minimumSimilarity; + } + + private static List SelectStructures(IReadOnlyList candidates) + { + if (candidates.Count == 0) + { + return []; + } + + Dictionary> candidatesByTrack = []; + Dictionary> candidatesByObservation = []; + foreach (MatchCandidate candidate in candidates) + { + foreach (TextTrack track in candidate.Tracks) + { + if (!candidatesByTrack.TryGetValue(track, out List? trackCandidates)) + { + trackCandidates = []; + candidatesByTrack[track] = trackCandidates; + } + trackCandidates.Add(candidate); + } + foreach (int observation in candidate.ObservationIndices) + { + if (!candidatesByObservation.TryGetValue(observation, out List? observationCandidates)) + { + observationCandidates = []; + candidatesByObservation[observation] = observationCandidates; + } + observationCandidates.Add(candidate); + } + } + + HashSet visited = new(ReferenceEqualityComparer.Instance); + List selected = []; + MatchCandidate[] orderedCandidates = OrderStructureCandidates(candidates); + Dictionary candidateOrder = new(ReferenceEqualityComparer.Instance); + for (int index = 0; index < orderedCandidates.Length; index++) + { + candidateOrder[orderedCandidates[index]] = index; + } + foreach (MatchCandidate first in orderedCandidates) + { + if (!visited.Add(first)) + { + continue; + } + + List component = []; + Queue pending = new([first]); + while (pending.TryDequeue(out MatchCandidate? candidate)) + { + component.Add(candidate); + foreach (TextTrack track in candidate.Tracks) + { + foreach (MatchCandidate neighbor in candidatesByTrack[track]) + { + if (visited.Add(neighbor)) + { + pending.Enqueue(neighbor); + } + } + } + foreach (int observation in candidate.ObservationIndices) + { + foreach (MatchCandidate neighbor in candidatesByObservation[observation]) + { + if (visited.Add(neighbor)) + { + pending.Enqueue(neighbor); + } + } + } + } + component.Sort((first, second) => candidateOrder[first].CompareTo(candidateOrder[second])); + selected.AddRange(SelectStructureComponent(component)); + } + selected.Sort((first, second) => candidateOrder[first].CompareTo(candidateOrder[second])); + return selected; + } + + private static MatchCandidate[] OrderStructureCandidates(IEnumerable candidates) + => candidates + .Select(candidate => ( + Candidate: candidate, + TrackKey: string.Join(',', candidate.Tracks.Select(track => track.Id).Order()))) + .OrderByDescending(item => item.Candidate.ResourceCount) + .ThenByDescending(item => item.Candidate.Score) + .ThenBy(item => item.Candidate.Kind) + .ThenBy(item => item.Candidate.Combined.Y) + .ThenBy(item => item.Candidate.Combined.X) + .ThenBy(item => item.Candidate.Combined.Height) + .ThenBy(item => item.Candidate.Combined.Width) + .ThenBy(item => item.Candidate.Combined.Angle) + .ThenBy(item => item.Candidate.Combined.SourceText, StringComparer.Ordinal) + .ThenBy(item => item.TrackKey, StringComparer.Ordinal) + .Select(item => item.Candidate) + .ToArray(); + + private static List SelectStructureComponent(IReadOnlyList ordered) + { + Dictionary trackResources = ordered + .SelectMany(candidate => candidate.Tracks) + .Distinct() + .OrderBy(track => track.Id) + .Select((track, index) => (track, index)) + .ToDictionary(item => item.track, item => item.index); + Dictionary observationResources = ordered + .SelectMany(candidate => candidate.ObservationIndices) + .Distinct() + .Order() + .Select((observation, index) => (observation, index: trackResources.Count + index)) + .ToDictionary(item => item.observation, item => item.index); + Dictionary resourceMasks = new(ReferenceEqualityComparer.Instance); + Dictionary selectionOrderMasks = new(ReferenceEqualityComparer.Instance); + for (int candidateIndex = 0; candidateIndex < ordered.Count; candidateIndex++) + { + MatchCandidate candidate = ordered[candidateIndex]; + BigInteger mask = BigInteger.Zero; + foreach (TextTrack track in candidate.Tracks) + { + mask |= BigInteger.One << trackResources[track]; + } + foreach (int observation in candidate.ObservationIndices) + { + mask |= BigInteger.One << observationResources[observation]; + } + resourceMasks[candidate] = mask; + selectionOrderMasks[candidate] = BigInteger.One << (ordered.Count - candidateIndex - 1); + } + + Dictionary states = new() + { + [BigInteger.Zero] = new(BigInteger.Zero, BigInteger.Zero, 0, 0, null, null), + }; + foreach (MatchCandidate candidate in ordered) + { + BigInteger candidateMask = resourceMasks[candidate]; + foreach (StructureSelectionState state in states.Values.ToArray()) + { + if ((state.UsedResources & candidateMask) != BigInteger.Zero) + { + continue; + } + BigInteger combinedMask = state.UsedResources | candidateMask; + StructureSelectionState proposed = new( + combinedMask, + state.SelectionOrder | selectionOrderMasks[candidate], + state.ResourceCount + candidate.ResourceCount, + state.Score + candidate.Score, + state, + candidate); + if (!states.TryGetValue(combinedMask, out StructureSelectionState? existing) + || IsBetterStructureSelection(proposed, existing)) + { + states[combinedMask] = proposed; + } + } + + if (states.Count > MaxStructureSelectionStates * 2) + { + states = states.Values + .OrderByDescending(state => state.ResourceCount) + .ThenByDescending(state => state.Score) + .ThenByDescending(state => state.SelectionOrder) + .ThenBy(state => state.UsedResources) + .Take(MaxStructureSelectionStates) + .ToDictionary(state => state.UsedResources); + } + } + + StructureSelectionState best = states.Values + .OrderByDescending(state => state.ResourceCount) + .ThenByDescending(state => state.Score) + .ThenByDescending(state => state.SelectionOrder) + .ThenBy(state => state.UsedResources) + .First(); + HashSet finalSelection = new(ReferenceEqualityComparer.Instance); + for (StructureSelectionState? state = best; state?.Candidate is not null; state = state.Previous) + { + finalSelection.Add(state.Candidate); + } + return ordered.Where(finalSelection.Contains).ToList(); + } + + private static bool IsBetterStructureSelection( + StructureSelectionState proposed, + StructureSelectionState current) + => proposed.ResourceCount > current.ResourceCount + || (proposed.ResourceCount == current.ResourceCount + && (proposed.Score > current.Score + 0.000001 + || (Math.Abs(proposed.Score - current.Score) <= 0.000001 + && proposed.SelectionOrder > current.SelectionOrder))); + + private List BuildRestorationCandidates( + IReadOnlyList activeTracks, + TextRect[] observations, + string[] normalizedObservations, + Size imageSize, + TimeSpan timestamp, + HashSet excludedObservations) + { + List result = []; + foreach (TextTrack parent in activeTracks) + { + TextTrack[] children = this.tracks + .Where(track => track.IsDormant && track.DormantParentId == parent.Id) + .OrderBy(track => track.Id) + .ToArray(); + if (children.Length < 2) + { + continue; + } + + List childCandidates = BuildOneToOneCandidates( + children, + observations, + normalizedObservations, + imageSize, + timestamp, + child => child.RestoreGeometry(parent.LatestObservation), + _ => Math.Max(parent.LatestObservation.Width, parent.LatestObservation.Height), + excludedObservations); + Dictionary candidatesByChild = children.ToDictionary( + child => child, + child => childCandidates + .Where(candidate => candidate.Tracks[0] == child) + .OrderByDescending(candidate => candidate.Score) + .Take(MaxOneToOneCandidatesPerResource) + .ToArray()); + if (candidatesByChild.Values.Any(candidates => candidates.Length == 0)) + { + continue; + } + + AddAssignments(0, [], [], 0); + + void AddAssignments( + int childIndex, + List assignments, + HashSet usedObservations, + double score) + { + if (childIndex == children.Length) + { + int[] observationIndices = assignments.Select(assignment => assignment.ObservationIndex).ToArray(); + TextRect[] assignedObservations = observationIndices + .Select(index => observations[index]) + .ToArray(); + if (!TryCombineStructure( + assignedObservations, + observationIndices.Select(index => normalizedObservations[index]).ToArray(), + parent.Stabilized, + parent.ConfirmedText, + parent.NormalizedConfirmedText, + imageSize, + out TextRect combined, + out double structureScore)) + { + return; + } + result.Add(new( + MatchKind.Restore, + [parent, .. children], + observationIndices, + combined, + Math.Max((score / children.Length) + StructureAssignmentBonus, structureScore)) + { + RestorationAssignments = assignments.ToArray(), + }); + return; + } + + TextTrack child = children[childIndex]; + foreach (MatchCandidate candidate in candidatesByChild[child]) + { + int observationIndex = candidate.ObservationIndices[0]; + if (!usedObservations.Add(observationIndex)) + { + continue; + } + assignments.Add(new(child, observationIndex, candidate.Combined)); + AddAssignments(childIndex + 1, assignments, usedObservations, score + candidate.Score); + assignments.RemoveAt(assignments.Count - 1); + usedObservations.Remove(observationIndex); + } + } + } + return result; + } + + private void ApplyMatches( + IReadOnlyList selected, + TimeSpan timestamp, + HashSet matchedTracks, + HashSet matchedObservations) + { + Dictionary currentMergeCandidates = []; + Dictionary currentRestoreCandidates = []; + foreach (MatchCandidate candidate in selected) + { + foreach (int observationIndex in candidate.ObservationIndices) + { + matchedObservations.Add(observationIndex); + } + + if (candidate.Kind == MatchKind.Restore) + { + TextTrack parent = candidate.Tracks[0]; + TextTrack[] children = candidate.Tracks.Skip(1).ToArray(); + int restoreCount = this.dormantRestoreCandidates.GetValueOrDefault(parent.Id) + 1; + currentRestoreCandidates[parent.Id] = restoreCount; + matchedTracks.Add(parent); + matchedTracks.UnionWith(children); + if (restoreCount >= StructureConfirmationFrames) + { + foreach (RestorationAssignment assignment in candidate.RestorationAssignments) + { + assignment.Track.Reactivate(timestamp); + assignment.Track.Observe(assignment.Observation, timestamp); + } + this.tracks.Remove(parent); + this.logger.LogDebug("OCR track {TrackId} restored {ChildCount} dormant child tracks", parent.Id, children.Length); + } + else + { + parent.Observe(candidate.Combined, timestamp); + } + continue; + } + + if (candidate.Kind is MatchKind.OneToOne or MatchKind.Split) + { + TextTrack track = candidate.Tracks[0]; + track.Observe(candidate.Combined, timestamp); + matchedTracks.Add(track); + if (candidate.Kind == MatchKind.Split) + { + this.logger.LogDebug("OCR track {TrackId} retained across a {PartCount}-part split", track.Id, candidate.ObservationIndices.Count); + } + continue; + } + + string key = string.Join(',', candidate.Tracks.Select(track => track.Id).Order()); + int count = this.mergeCandidates.GetValueOrDefault(key) + 1; + currentMergeCandidates[key] = count; + foreach (TextTrack track in candidate.Tracks) + { + matchedTracks.Add(track); + } + if (count >= StructureConfirmationFrames) + { + TextTrack parent = this.CreateTrack(candidate.Combined, timestamp); + matchedTracks.Add(parent); + foreach (TextTrack child in candidate.Tracks) + { + child.MarkDormant(parent, timestamp); + } + this.logger.LogDebug("OCR tracks {TrackIds} converged into track {TrackId}", key, parent.Id); + } + else + { + foreach (TextTrack track in candidate.Tracks) + { + track.MarkObservedWithoutChangingStructure(timestamp); + } + this.logger.LogDebug("OCR tracks {TrackIds} retained during a temporary merge", key); + } + } + this.mergeCandidates = currentMergeCandidates; + this.dormantRestoreCandidates = currentRestoreCandidates; + } + + private void RemoveTrackTree(TextTrack track) + { + foreach (TextTrack child in this.tracks.Where(candidate => candidate.DormantParentId == track.Id).ToArray()) + { + this.RemoveTrackTree(child); + } + this.tracks.Remove(track); + } + + private static IEnumerable> Combinations(IReadOnlyList source, int count) + { + int[] indexes = Enumerable.Range(0, count).ToArray(); + while (true) + { + yield return indexes.Select(index => source[index]).ToArray(); + int position = count - 1; + while (position >= 0 && indexes[position] == source.Count - count + position) + { + position--; + } + if (position < 0) + { + yield break; + } + indexes[position]++; + for (int i = position + 1; i < count; i++) + { + indexes[i] = indexes[i - 1] + 1; + } + } + } + + private static IEnumerable OrderForReading(IEnumerable source, Func getRect) + { + T[] items = source.ToArray(); + TextRect[] members = items.Select(getRect).ToArray(); + double angle = AverageAngle(members); + (double cosine, double sine) = ReadingAxis(angle); + var projected = items + .Select((item, index) => + { + TextRect rect = members[index]; + (double along, double perpendicular) = ProjectCenter(rect, cosine, sine); + return (item, along, perpendicular); + }) + .ToArray(); + double perpendicularSpan = projected.Max(item => item.perpendicular) + - projected.Min(item => item.perpendicular); + double averageHeight = members.Average(rect => rect.Height); + return perpendicularSpan <= averageHeight * 0.75 + ? projected.OrderBy(item => item.along).Select(item => item.item) + : projected.OrderBy(item => item.perpendicular).ThenBy(item => item.along).Select(item => item.item); + } + + private static TextRect Union(TextRect[] members, string text) + { + double angle = AverageAngle(members); + (double cosine, double sine) = ReadingAxis(angle); + (double along, double perpendicular)[] projectedCorners = members + .SelectMany(GetCorners) + .Select(point => ProjectPoint(point.x, point.y, cosine, sine)) + .ToArray(); + double minimumAlong = projectedCorners.Min(point => point.along); + double maximumAlong = projectedCorners.Max(point => point.along); + double minimumPerpendicular = projectedCorners.Min(point => point.perpendicular); + double maximumPerpendicular = projectedCorners.Max(point => point.perpendicular); + (double x, double y) = FromReadingCoordinates( + minimumAlong, + minimumPerpendicular, + cosine, + sine); + double[] perpendicularPositions = members + .Select(rect => ProjectCenter(rect, cosine, sine).perpendicular) + .ToArray(); + double perpendicularSpan = perpendicularPositions.Max() - perpendicularPositions.Min(); + TextRect first = members[0]; + return first with + { + SourceText = text, + X = x, + Y = y, + Width = maximumAlong - minimumAlong, + Height = maximumPerpendicular - minimumPerpendicular, + FontSize = members.Average(rect => rect.FontSize), + MultiLine = perpendicularSpan > members.Average(rect => rect.Height) * 0.75 + || members.Any(rect => rect.MultiLine), + Angle = angle, + }; + } + + private static bool MembersAreAdjacent(TextRect[] members) + { + HashSet connected = [0]; + Queue pending = new([0]); + while (pending.TryDequeue(out int current)) + { + for (int candidate = 0; candidate < members.Length; candidate++) + { + if (!connected.Contains(candidate) && AreAdjacent(members[current], members[candidate])) + { + connected.Add(candidate); + pending.Enqueue(candidate); + } + } + } + return connected.Count == members.Length; + } + + private static bool MembersHaveCompatibleStyle(TextRect[] members) + { + for (int first = 0; first < members.Length; first++) + { + for (int second = first + 1; second < members.Length; second++) + { + if (RatioSimilarity(members[first].Height, members[second].Height) < MinimumStructureSizeRatio + || RatioSimilarity(members[first].FontSize, members[second].FontSize) < MinimumStructureSizeRatio + || AngleDifference(members[first].Angle, members[second].Angle) > MaximumStructureAngleDifference) + { + return false; + } + } + } + return true; + } + + private static double AngleDifference(double first, double second) + { + double difference = Math.Abs(first - second) % 360; + return Math.Min(difference, 360 - difference); + } + + private static double NormalizeAngle(double angle) + { + double normalized = angle % 360; + if (normalized > 180) + { + normalized -= 360; + } + else if (normalized <= -180) + { + normalized += 360; + } + return Math.Abs(normalized) <= AngleVectorEpsilon ? 0 : normalized; + } + + private static bool AreAdjacent(TextRect first, TextRect second) + { + double angle = AverageAngle(first.Angle, second.Angle); + (double cosine, double sine) = ReadingAxis(angle); + (double firstCenterX, double firstCenterY) = Center(first); + (double secondCenterX, double secondCenterY) = Center(second); + double deltaX = secondCenterX - firstCenterX; + double deltaY = secondCenterY - firstCenterY; + double alongDistance = Math.Abs((deltaX * cosine) + (deltaY * sine)); + double perpendicularDistance = Math.Abs((-deltaX * sine) + (deltaY * cosine)); + double alongGap = Math.Max(0, alongDistance - ((first.Width + second.Width) / 2)); + double perpendicularGap = Math.Max(0, perpendicularDistance - ((first.Height + second.Height) / 2)); + bool sameLine = perpendicularDistance <= Math.Max(first.Height, second.Height) * 0.75 + && alongGap <= Math.Max(first.Height, second.Height) * 2; + bool neighboringLine = alongDistance <= Math.Max(first.Width, second.Width) * 0.75 + && perpendicularGap <= Math.Max(first.Width, second.Width); + return sameLine || neighboringLine; + } + + private static double AverageAngle(TextRect[] members) + { + double sine = 0; + double cosine = 0; + foreach (TextRect member in members) + { + double radians = member.Angle * Math.PI / 180; + sine += Math.Sin(radians); + cosine += Math.Cos(radians); + } + return ResolveAverageAngle(sine, cosine, members[0].Angle); + } + + private static double AverageAngle(double first, double second) + { + double firstRadians = first * Math.PI / 180; + double secondRadians = second * Math.PI / 180; + return ResolveAverageAngle( + Math.Sin(firstRadians) + Math.Sin(secondRadians), + Math.Cos(firstRadians) + Math.Cos(secondRadians), + first); + } + + private static double ResolveAverageAngle(double sine, double cosine, double fallback) + { + if (Math.Abs(sine) <= AngleVectorEpsilon && Math.Abs(cosine) <= AngleVectorEpsilon) + { + return fallback; + } + double angle = Math.Atan2(sine, cosine) * 180 / Math.PI; + return Math.Abs(angle) <= AngleVectorEpsilon ? 0 : angle; + } + + private static (double cosine, double sine) ReadingAxis(double angle) + { + double radians = angle * Math.PI / 180; + return (Math.Cos(radians), Math.Sin(radians)); + } + + private static (double along, double perpendicular) ProjectCenter(TextRect rect, double cosine, double sine) + { + (double centerX, double centerY) = Center(rect); + return ProjectPoint(centerX, centerY, cosine, sine); + } + + private static (double along, double perpendicular) ProjectPoint( + double x, + double y, + double cosine, + double sine) + => ( + (x * cosine) + (y * sine), + (-x * sine) + (y * cosine)); + + private static (double x, double y) FromReadingCoordinates( + double along, + double perpendicular, + double cosine, + double sine) + => ( + (along * cosine) - (perpendicular * sine), + (along * sine) + (perpendicular * cosine)); + + private static IEnumerable<(double x, double y)> GetCorners(TextRect rect) + { + if (Math.Abs(rect.Angle) <= AngleVectorEpsilon) + { + return + [ + (rect.X, rect.Y), + (rect.X + rect.Width, rect.Y), + (rect.X, rect.Y + rect.Height), + (rect.X + rect.Width, rect.Y + rect.Height), + ]; + } + (double cosine, double sine) = ReadingAxis(rect.Angle); + double alongX = rect.Width * cosine; + double alongY = rect.Width * sine; + double perpendicularX = -rect.Height * sine; + double perpendicularY = rect.Height * cosine; + return + [ + (rect.X, rect.Y), + (rect.X + alongX, rect.Y + alongY), + (rect.X + perpendicularX, rect.Y + perpendicularY), + (rect.X + alongX + perpendicularX, rect.Y + alongY + perpendicularY), + ]; + } + + private static bool IsNear(TextRect first, TextRect second) + { + if (IntersectionOverUnion(first, second) > 0) + { + return true; + } + double gate = Math.Max(Math.Max(first.Width, first.Height), Math.Max(second.Width, second.Height)); + return CenterDistance(first, second) <= gate * 1.5; + } + + private static bool IsPotentialStructureMember(TextRect target, TextRect member, Size imageSize) + => IsNear(target, member) + || CenterDistance(target, member) <= TrackingDistanceGate(target, imageSize); + + private static double TrackingDistanceGate( + TextRect geometry, + Size imageSize, + double? maximumDimensionOverride = null) + { + double imageDiagonal = Math.Sqrt( + ((double)imageSize.Width * imageSize.Width) + + ((double)imageSize.Height * imageSize.Height)); + double maximumDimension = maximumDimensionOverride + ?? Math.Max(geometry.Width, geometry.Height); + return Math.Max(imageDiagonal * 0.08, maximumDimension * 4); + } + + private static double NormalizedTextSimilarity(string first, string second) + { + if (string.Equals(first, second, StringComparison.Ordinal)) + { + return 1; + } + int maximumLength = Math.Max(first.Length, second.Length); + if (maximumLength == 0) + { + return 1; + } + return 1 - ((double)Levenshtein.GetDistance(first, second, CalculationOptions.Default) / maximumLength); + } + + private static string NormalizeText(string text) + { + StringBuilder result = new(text.Length); + bool previousWasWhitespace = false; + foreach (char character in text.Normalize(NormalizationForm.FormKC)) + { + if (char.IsWhiteSpace(character)) + { + if (!previousWasWhitespace && result.Length > 0) + { + result.Append(' '); + } + previousWasWhitespace = true; + } + else + { + result.Append(character); + previousWasWhitespace = false; + } + } + return result.ToString().TrimEnd(); + } + + private static string NormalizeCompactText(string text) + => string.Concat(NormalizeText(text).Where(character => !char.IsWhiteSpace(character))); + + private static bool IsCoveredTextFragment(TextRect fragment, TextRect container) + { + if (AngleDifference(fragment.Angle, container.Angle) > MaximumStructureAngleDifference + || IntersectionOverSmallerArea(fragment, container) < 0.8) + { + return false; + } + + string fragmentText = NormalizeCompactText(fragment.SourceText); + string containerText = NormalizeCompactText(container.SourceText); + return fragmentText.Length > 0 + && fragmentText.Length < containerText.Length + && containerText.Contains(fragmentText, StringComparison.Ordinal); + } + + private static double IntersectionOverUnion(TextRect first, TextRect second) + { + if (Math.Abs(first.Angle) <= AngleVectorEpsilon + && Math.Abs(second.Angle) <= AngleVectorEpsilon) + { + double axisLeft = Math.Max(first.X, second.X); + double axisTop = Math.Max(first.Y, second.Y); + double axisRight = Math.Min(first.X + first.Width, second.X + second.Width); + double axisBottom = Math.Min(first.Y + first.Height, second.Y + second.Height); + double axisIntersection = Math.Max(0, axisRight - axisLeft) * Math.Max(0, axisBottom - axisTop); + double axisUnion = (first.Width * first.Height) + (second.Width * second.Height) - axisIntersection; + return axisUnion <= 0 ? 0 : axisIntersection / axisUnion; + } + RectInfo firstBounds = first.GetRotatedBoundingBox(); + RectInfo secondBounds = second.GetRotatedBoundingBox(); + double left = Math.Max(firstBounds.X, secondBounds.X); + double top = Math.Max(firstBounds.Y, secondBounds.Y); + double right = Math.Min(firstBounds.X + firstBounds.Width, secondBounds.X + secondBounds.Width); + double bottom = Math.Min(firstBounds.Y + firstBounds.Height, secondBounds.Y + secondBounds.Height); + double intersection = Math.Max(0, right - left) * Math.Max(0, bottom - top); + double union = (firstBounds.Width * firstBounds.Height) + + (secondBounds.Width * secondBounds.Height) + - intersection; + return union <= 0 ? 0 : intersection / union; + } + + private static double IntersectionOverSmallerArea(TextRect first, TextRect second) + { + RectInfo firstBounds = first.GetRotatedBoundingBox(); + RectInfo secondBounds = second.GetRotatedBoundingBox(); + double left = Math.Max(firstBounds.X, secondBounds.X); + double top = Math.Max(firstBounds.Y, secondBounds.Y); + double right = Math.Min(firstBounds.X + firstBounds.Width, secondBounds.X + secondBounds.Width); + double bottom = Math.Min(firstBounds.Y + firstBounds.Height, secondBounds.Y + secondBounds.Height); + double intersection = Math.Max(0, right - left) * Math.Max(0, bottom - top); + double smallerArea = Math.Min( + firstBounds.Width * firstBounds.Height, + secondBounds.Width * secondBounds.Height); + return smallerArea <= 0 ? 0 : intersection / smallerArea; + } + + private static double CenterDistance(TextRect first, TextRect second) + { + if (Math.Abs(first.Angle) <= AngleVectorEpsilon + && Math.Abs(second.Angle) <= AngleVectorEpsilon) + { + double axisX = (first.X + (first.Width / 2)) - (second.X + (second.Width / 2)); + double axisY = (first.Y + (first.Height / 2)) - (second.Y + (second.Height / 2)); + return Math.Sqrt((axisX * axisX) + (axisY * axisY)); + } + (double firstCenterX, double firstCenterY) = Center(first); + (double secondCenterX, double secondCenterY) = Center(second); + double x = firstCenterX - secondCenterX; + double y = firstCenterY - secondCenterY; + return Math.Sqrt((x * x) + (y * y)); + } + + private static (double x, double y) Center(TextRect rect) + { + if (Math.Abs(rect.Angle) <= AngleVectorEpsilon) + { + return (rect.X + (rect.Width / 2), rect.Y + (rect.Height / 2)); + } + (double cosine, double sine) = ReadingAxis(rect.Angle); + return ( + rect.X + (rect.Width * cosine / 2) - (rect.Height * sine / 2), + rect.Y + (rect.Width * sine / 2) + (rect.Height * cosine / 2)); + } + + private static double RatioSimilarity(double first, double second) + { + double maximum = Math.Max(Math.Abs(first), Math.Abs(second)); + return maximum <= double.Epsilon ? 1 : Math.Min(Math.Abs(first), Math.Abs(second)) / maximum; + } + + private TextTrack CreateTrack(TextRect observation, TimeSpan timestamp, long? coveringTrackId = null) + { + TextTrack track = new(++this.nextTrackId, observation, timestamp, coveringTrackId); + this.tracks.Add(track); + this.logger.LogDebug("OCR track {TrackId} created for {SourceText}", track.Id, observation.SourceText); + return track; + } + + private TextRect[] GetOutput() + => this.tracks + .Where(track => !track.IsDormant) + .OrderBy(track => track.Id) + .Select(track => track.Stabilized) + .ToArray(); + + private enum MatchKind + { + OneToOne, + Split, + Merge, + Restore, + } + + private sealed record RestorationAssignment(TextTrack Track, int ObservationIndex, TextRect Observation); + + private sealed record StructureSelectionState( + BigInteger UsedResources, + BigInteger SelectionOrder, + int ResourceCount, + double Score, + StructureSelectionState? Previous, + MatchCandidate? Candidate); + + private sealed record MatchCandidate( + MatchKind Kind, + IReadOnlyList Tracks, + IReadOnlyList ObservationIndices, + TextRect Combined, + double Score) + { + public int ResourceCount => this.Tracks.Count + this.ObservationIndices.Count; + + public IReadOnlyList RestorationAssignments { get; init; } = []; + } + + private sealed class TextTrack( + long id, + TextRect observation, + TimeSpan timestamp, + long? coveringTrackId) + { + private TextRect? geometryCandidate; + private int geometryCandidateCount; + private readonly List textVotes = []; + private string normalizedConfirmedText = NormalizeText(observation.SourceText); + private DormantGeometry? dormantGeometry; + private double velocityX; + private double velocityY; + private int motionSamples; + + public long Id { get; } = id; + + public TextRect LatestObservation { get; private set; } = observation; + + public TextRect Stabilized { get; private set; } = observation; + + public string ConfirmedText { get; private set; } = observation.SourceText; + + public string NormalizedConfirmedText => this.normalizedConfirmedText; + + public int MissedFrames { get; private set; } + + public TimeSpan LastObservationTime { get; private set; } = timestamp; + + public bool IsDormant { get; private set; } + + public long? DormantParentId { get; private set; } + + public long? CoveringTrackId { get; } = coveringTrackId; + + public TimeSpan DormantSince { get; private set; } + + public void Observe(TextRect current, TimeSpan timestamp) + { + this.MissedFrames = 0; + this.UpdateMotion(current, timestamp); + this.LatestObservation = current; + this.LastObservationTime = timestamp; + this.UpdateText(current.SourceText); + this.UpdateGeometry(current); + this.Stabilized = current with + { + SourceText = this.ConfirmedText, + X = this.Stabilized.X, + Y = this.Stabilized.Y, + Width = this.Stabilized.Width, + Height = this.Stabilized.Height, + FontSize = this.Stabilized.FontSize, + MultiLine = this.Stabilized.MultiLine, + Angle = this.Stabilized.Angle, + }; + } + + public void MarkObservedWithoutChangingStructure(TimeSpan timestamp) + { + this.MissedFrames = 0; + this.LastObservationTime = timestamp; + } + + public void MarkMissed() + { + this.MissedFrames++; + } + + public TextRect Predict(TimeSpan timestamp) + { + double seconds = (timestamp - this.LastObservationTime).TotalSeconds; + if (this.motionSamples == 0 || seconds <= 0 || seconds > 2) + { + return this.LatestObservation; + } + + double maximumDisplacement = Math.Max(this.LatestObservation.Width, this.LatestObservation.Height) * 3; + double x = Math.Clamp(this.velocityX * seconds, -maximumDisplacement, maximumDisplacement); + double y = Math.Clamp(this.velocityY * seconds, -maximumDisplacement, maximumDisplacement); + return this.LatestObservation with + { + X = this.LatestObservation.X + x, + Y = this.LatestObservation.Y + y, + }; + } + + public TextRect RestoreGeometry(TextRect parent) + => this.dormantGeometry?.Restore(this.Stabilized, parent) ?? this.Stabilized; + + public void MarkDormant(TextTrack parent, TimeSpan timestamp) + { + this.dormantGeometry = DormantGeometry.Create(this.Stabilized, parent.Stabilized); + this.IsDormant = true; + this.DormantParentId = parent.Id; + this.DormantSince = timestamp; + this.MissedFrames = 0; + this.ResetMotion(timestamp); + } + + public void Reactivate(TimeSpan timestamp) + { + this.IsDormant = false; + this.DormantParentId = null; + this.dormantGeometry = null; + this.MissedFrames = 0; + this.ResetMotion(timestamp); + } + + private void UpdateText(string current) + { + string normalizedCurrent = NormalizeText(current); + if (normalizedCurrent == this.normalizedConfirmedText) + { + this.textVotes.Clear(); + return; + } + + this.textVotes.Add(new(normalizedCurrent, current)); + if (this.textVotes.Count > TextVoteHistorySize) + { + this.textVotes.RemoveAt(0); + } + + (string normalized, int count, double weight)? winner = this.textVotes + .Select((vote, index) => (vote, age: this.textVotes.Count - index - 1)) + .GroupBy(item => item.vote.Normalized) + .Select(group => ( + normalized: group.Key, + count: group.Count(), + weight: group.Sum(item => Math.Pow(TextVoteDecay, item.age)))) + .OrderByDescending(candidate => candidate.weight) + .ThenByDescending(candidate => candidate.count) + .Cast<(string normalized, int count, double weight)?>() + .FirstOrDefault(); + if (winner is { count: >= 2, weight: >= TextVoteThreshold }) + { + this.ConfirmedText = this.textVotes.Last(vote => vote.Normalized == winner.Value.normalized).Original; + this.normalizedConfirmedText = winner.Value.normalized; + this.textVotes.Clear(); + } + } + + private void UpdateMotion(TextRect current, TimeSpan timestamp) + { + double seconds = (timestamp - this.LastObservationTime).TotalSeconds; + if (seconds < 0.05 || seconds > 2) + { + if (seconds > 2 || seconds < 0) + { + this.ResetMotion(timestamp); + } + return; + } + + double movementX = current.X - this.LatestObservation.X; + double movementY = current.Y - this.LatestObservation.Y; + double jitterTolerance = Math.Max(1, Math.Min(this.LatestObservation.Width, this.LatestObservation.Height) * 0.05); + if (Math.Abs(movementX) <= jitterTolerance && Math.Abs(movementY) <= jitterTolerance) + { + this.velocityX *= 0.5; + this.velocityY *= 0.5; + return; + } + + double instantX = movementX / seconds; + double instantY = movementY / seconds; + double newWeight = this.motionSamples == 0 ? 1 : 0.8; + this.velocityX = (this.velocityX * (1 - newWeight)) + (instantX * newWeight); + this.velocityY = (this.velocityY * (1 - newWeight)) + (instantY * newWeight); + this.motionSamples++; + } + + private void ResetMotion(TimeSpan timestamp) + { + this.velocityX = 0; + this.velocityY = 0; + this.motionSamples = 0; + this.LastObservationTime = timestamp; + } + + private void UpdateGeometry(TextRect current) + { + bool isNoise = IsGeometryNoise(this.Stabilized, current); + bool matchesStable = isNoise && GeometrySamplesMatch(this.Stabilized, current); + if (matchesStable && GeometryValuesEqual(this.Stabilized, current)) + { + this.geometryCandidate = null; + this.geometryCandidateCount = 0; + return; + } + + double movement = CenterDistance(this.Stabilized, current); + if (movement > Math.Max(this.Stabilized.Width, this.Stabilized.Height) * 0.75) + { + this.ApplyGeometry(current); + return; + } + + if (this.geometryCandidate is not null + && (isNoise + ? GeometrySamplesMatch(this.geometryCandidate, current) + : GeometryCandidateMatches(this.geometryCandidate, current))) + { + this.geometryCandidate = current; + this.geometryCandidateCount++; + } + else + { + this.geometryCandidate = current; + this.geometryCandidateCount = 1; + } + + int confirmationFrames = matchesStable + ? MicroGeometryConfirmationFrames + : StructureConfirmationFrames; + if (this.geometryCandidateCount >= confirmationFrames) + { + this.ApplyGeometry(current); + } + } + + private static bool IsGeometryNoise(TextRect stable, TextRect current) + { + double xTolerance = Math.Max(3, stable.Width * 0.15); + double yTolerance = Math.Max(3, stable.Height * 0.15); + return Math.Abs(stable.X - current.X) <= xTolerance + && Math.Abs(stable.Y - current.Y) <= yTolerance + && Math.Abs(stable.Width - current.Width) <= Math.Max(3, stable.Width * 0.12) + && Math.Abs(stable.Height - current.Height) <= Math.Max(3, stable.Height * 0.12) + && Math.Abs(stable.FontSize - current.FontSize) <= Math.Max(1, stable.FontSize * 0.12) + && AngleDifference(stable.Angle, current.Angle) <= 2 + && stable.MultiLine == current.MultiLine; + } + + private static bool GeometrySamplesMatch(TextRect first, TextRect second) + => Math.Abs(first.X - second.X) <= 1 + && Math.Abs(first.Y - second.Y) <= 1 + && Math.Abs(first.Width - second.Width) <= 1 + && Math.Abs(first.Height - second.Height) <= 1 + && Math.Abs(first.FontSize - second.FontSize) <= 0.5 + && AngleDifference(first.Angle, second.Angle) <= 2 + && first.MultiLine == second.MultiLine; + + private static bool GeometryValuesEqual(TextRect first, TextRect second) + => first.X == second.X + && first.Y == second.Y + && first.Width == second.Width + && first.Height == second.Height + && first.FontSize == second.FontSize + && first.Angle == second.Angle + && first.MultiLine == second.MultiLine; + + private static bool GeometryCandidateMatches(TextRect candidate, TextRect current) + => CenterDistance(candidate, current) <= Math.Max(3, Math.Max(candidate.Width, candidate.Height) * 0.15) + && RatioSimilarity(candidate.Width, current.Width) >= 0.85 + && RatioSimilarity(candidate.Height, current.Height) >= 0.85 + && AngleDifference(candidate.Angle, current.Angle) <= 3 + && candidate.MultiLine == current.MultiLine; + + private void ApplyGeometry(TextRect current) + { + this.Stabilized = this.Stabilized with + { + X = current.X, + Y = current.Y, + Width = current.Width, + Height = current.Height, + FontSize = current.FontSize, + MultiLine = current.MultiLine, + Angle = current.Angle, + }; + this.geometryCandidate = null; + this.geometryCandidateCount = 0; + } + + private sealed record DormantGeometry( + double AlongOffsetRatio, + double PerpendicularOffsetRatio, + double WidthRatio, + double HeightRatio, + double FontSizeRatio, + double AngleOffset) + { + public static DormantGeometry Create(TextRect child, TextRect parent) + { + (double cosine, double sine) = ReadingAxis(parent.Angle); + double deltaX = child.X - parent.X; + double deltaY = child.Y - parent.Y; + double alongOffset = (deltaX * cosine) + (deltaY * sine); + double perpendicularOffset = (-deltaX * sine) + (deltaY * cosine); + return new( + alongOffset / parent.Width, + perpendicularOffset / parent.Height, + child.Width / parent.Width, + child.Height / parent.Height, + parent.FontSize > double.Epsilon ? child.FontSize / parent.FontSize : 0, + NormalizeAngle(child.Angle - parent.Angle)); + } + + public TextRect Restore(TextRect child, TextRect parent) + { + (double cosine, double sine) = ReadingAxis(parent.Angle); + double alongOffset = this.AlongOffsetRatio * parent.Width; + double perpendicularOffset = this.PerpendicularOffsetRatio * parent.Height; + return child with + { + X = parent.X + (alongOffset * cosine) - (perpendicularOffset * sine), + Y = parent.Y + (alongOffset * sine) + (perpendicularOffset * cosine), + Width = this.WidthRatio * parent.Width, + Height = this.HeightRatio * parent.Height, + FontSize = this.FontSizeRatio > 0 + ? this.FontSizeRatio * parent.FontSize + : child.FontSize, + Angle = NormalizeAngle(parent.Angle + this.AngleOffset), + }; + } + } + + private sealed record TextVote(string Normalized, string Original); + } +} diff --git a/WindowTranslator/Modules/Settings/AllSettingsViewModel.cs b/WindowTranslator/Modules/Settings/AllSettingsViewModel.cs index 4c17bfb6..e034de78 100644 --- a/WindowTranslator/Modules/Settings/AllSettingsViewModel.cs +++ b/WindowTranslator/Modules/Settings/AllSettingsViewModel.cs @@ -262,7 +262,6 @@ public async Task SaveAsync(object window) }, PluginParams = t.Params.ToDictionary(p => p.GetType().Name), DisplayBusy = t.DisplayBusy, - IsOneShotMode = t.IsOneShotMode, OverlayOpacity = t.OverlayOpacity, MousePointerHitTestPadding = t.MousePointerHitTestPadding, }), @@ -468,15 +467,10 @@ public partial class TargetSettingsViewModel( [ObservableProperty] private bool displayBusy = settings.DisplayBusy; - [property: Category("SettingsViewModel|Misc")] - [property: SortIndex(9)] - [ObservableProperty] - private bool isOneShotMode = settings.IsOneShotMode; - [property: Category("SettingsViewModel|Misc")] [property: LocalizedDescription(typeof(Resources), $"{nameof(MousePointerHitTestPadding)}_Desc")] [property: Slidable(0, 100, 1, 10, true, 1)] - [property: SortIndex(10)] + [property: SortIndex(9)] [ObservableProperty] private double mousePointerHitTestPadding = settings.MousePointerHitTestPadding; diff --git a/WindowTranslator/Program.cs b/WindowTranslator/Program.cs index 6f3f247d..41217ded 100644 --- a/WindowTranslator/Program.cs +++ b/WindowTranslator/Program.cs @@ -26,6 +26,7 @@ using WindowTranslator.Modules.ErrorReport; using WindowTranslator.Modules.LogView; using WindowTranslator.Modules.Main; +using WindowTranslator.Modules.Ocr; using WindowTranslator.Modules.Settings; using WindowTranslator.Modules.Startup; using WindowTranslator.Modules.Validate; @@ -134,6 +135,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddScoped(); builder.Services.AddHostedService(); if (builder.Configuration.GetValue("IgnoreUpdate")) { diff --git a/WindowTranslator/Properties/Resources.Designer.cs b/WindowTranslator/Properties/Resources.Designer.cs index 37d4cd4d..ff43b584 100644 --- a/WindowTranslator/Properties/Resources.Designer.cs +++ b/WindowTranslator/Properties/Resources.Designer.cs @@ -312,11 +312,6 @@ internal Resources() { /// public static string IsLatest => ResourceManager.GetString("IsLatest", resourceCulture) ?? string.Empty; - /// - /// "I[oCLɂ^C~Ôݖ|󂷂" ɗގĂ郍[JCYꂽ܂B - /// - public static string IsOneShotMode => ResourceManager.GetString("IsOneShotMode", resourceCulture) ?? string.Empty; - /// /// "}EX|C^[ʒũeLXĝ݃I[oC|\" ɗގĂ郍[JCYꂽ܂B /// diff --git a/WindowTranslator/Properties/Resources.ar.resx b/WindowTranslator/Properties/Resources.ar.resx index 580aa430..7a99470f 100644 --- a/WindowTranslator/Properties/Resources.ar.resx +++ b/WindowTranslator/Properties/Resources.ar.resx @@ -306,9 +306,6 @@ عرض أيقونة مشغول - - الترجمة فقط عند تفعيل الطبقة العلوية - عرض الترجمة فقط للنص تحت المؤشر diff --git a/WindowTranslator/Properties/Resources.cs.resx b/WindowTranslator/Properties/Resources.cs.resx index 83b05eb1..67983248 100644 --- a/WindowTranslator/Properties/Resources.cs.resx +++ b/WindowTranslator/Properties/Resources.cs.resx @@ -193,9 +193,6 @@ Zobrazit ikonu zaneprázdnění - - Překládat pouze při zapnutém překryvu - Zobrazovat překlad v překryvu pouze pro text v pozici ukazatele myši diff --git a/WindowTranslator/Properties/Resources.de.resx b/WindowTranslator/Properties/Resources.de.resx index a7d8dee4..6576f0ac 100644 --- a/WindowTranslator/Properties/Resources.de.resx +++ b/WindowTranslator/Properties/Resources.de.resx @@ -306,9 +306,6 @@ Zeige das Beschäftigt-Symbol - - Nur übersetzen, wenn Overlay aktiviert ist - Overlay-Übersetzung nur für Text an der Mauszeiger-Position anzeigen diff --git a/WindowTranslator/Properties/Resources.en.resx b/WindowTranslator/Properties/Resources.en.resx index d34ea235..ce3a7abc 100644 --- a/WindowTranslator/Properties/Resources.en.resx +++ b/WindowTranslator/Properties/Resources.en.resx @@ -306,9 +306,6 @@ Show busy icon - - Translate only when overlay is enabled - Display overlay translation only for text at mouse pointer position diff --git a/WindowTranslator/Properties/Resources.es.resx b/WindowTranslator/Properties/Resources.es.resx index 1d5bc4e8..c30dc38c 100644 --- a/WindowTranslator/Properties/Resources.es.resx +++ b/WindowTranslator/Properties/Resources.es.resx @@ -306,9 +306,6 @@ Mostrar icono de ocupado - - Traducir solo cuando la superposición está habilitada - Mostrar traducción solo para texto bajo el puntero diff --git a/WindowTranslator/Properties/Resources.fa.resx b/WindowTranslator/Properties/Resources.fa.resx index ce535156..64db2f3d 100644 --- a/WindowTranslator/Properties/Resources.fa.resx +++ b/WindowTranslator/Properties/Resources.fa.resx @@ -306,9 +306,6 @@ نمایش آیکون در حال پردازش - - ترجمه فقط هنگام فعال‌سازی پوشش - نمایش ترجمه فقط برای متن زیر مکان‌نما diff --git a/WindowTranslator/Properties/Resources.fil.resx b/WindowTranslator/Properties/Resources.fil.resx index 5a0c8803..fd294aef 100644 --- a/WindowTranslator/Properties/Resources.fil.resx +++ b/WindowTranslator/Properties/Resources.fil.resx @@ -306,9 +306,6 @@ Ipakita ang processing icon - - Magsalin lamang kapag pinagana ang overlay - Ipakita ang overlay translation para sa teksto sa posisyon ng mouse pointer lamang diff --git a/WindowTranslator/Properties/Resources.fr.resx b/WindowTranslator/Properties/Resources.fr.resx index 121478f7..752caae5 100644 --- a/WindowTranslator/Properties/Resources.fr.resx +++ b/WindowTranslator/Properties/Resources.fr.resx @@ -306,9 +306,6 @@ Afficher l'icône occupé - - Traduire uniquement lorsque la superposition est activée - Afficher la traduction uniquement pour le texte sous le pointeur diff --git a/WindowTranslator/Properties/Resources.hi.resx b/WindowTranslator/Properties/Resources.hi.resx index e93cc645..16c99848 100644 --- a/WindowTranslator/Properties/Resources.hi.resx +++ b/WindowTranslator/Properties/Resources.hi.resx @@ -306,9 +306,6 @@ व्यस्त आइकन दिखाएं - - केवल तब अनुवाद करें जब ओवरले सक्षम हो - केवल माउस पॉइंटर स्थिति पर टेक्स्ट के लिए ओवरले अनुवाद प्रदर्शित करें diff --git a/WindowTranslator/Properties/Resources.hu.resx b/WindowTranslator/Properties/Resources.hu.resx index f1e11152..a8348275 100644 --- a/WindowTranslator/Properties/Resources.hu.resx +++ b/WindowTranslator/Properties/Resources.hu.resx @@ -193,9 +193,6 @@ Foglalt ikon megjelenítése - - Fordítás csak akkor, ha az átfedő réteg engedélyezve van - Átfedő fordítás megjelenítése csak az egérmutató pozíciójában lévő szövegnél diff --git a/WindowTranslator/Properties/Resources.id.resx b/WindowTranslator/Properties/Resources.id.resx index 857c2012..2c08790a 100644 --- a/WindowTranslator/Properties/Resources.id.resx +++ b/WindowTranslator/Properties/Resources.id.resx @@ -306,9 +306,6 @@ Tampilkan ikon sibuk - - Terjemahkan hanya saat hamparan diaktifkan - Tampilkan terjemahan hamparan hanya untuk teks di posisi pointer mouse diff --git a/WindowTranslator/Properties/Resources.ko.resx b/WindowTranslator/Properties/Resources.ko.resx index ac4ccca1..3a25aeb6 100644 --- a/WindowTranslator/Properties/Resources.ko.resx +++ b/WindowTranslator/Properties/Resources.ko.resx @@ -306,9 +306,6 @@ 처리 중 아이콘을 표시합니다 - - 오버레이가 활성화된 시점에만 번역 - 마우스 포인터 위치의 텍스트에만 오버레이 번역을 표시 diff --git a/WindowTranslator/Properties/Resources.ms.resx b/WindowTranslator/Properties/Resources.ms.resx index c4e894cc..4db56c9a 100644 --- a/WindowTranslator/Properties/Resources.ms.resx +++ b/WindowTranslator/Properties/Resources.ms.resx @@ -306,9 +306,6 @@ Tunjukkan ikon sibuk - - Terjemah hanya apabila hamparan didayakan - Paparkan terjemahan hamparan hanya untuk teks pada kedudukan penunjuk tetikus diff --git a/WindowTranslator/Properties/Resources.pl.resx b/WindowTranslator/Properties/Resources.pl.resx index 12d726e2..6803fd29 100644 --- a/WindowTranslator/Properties/Resources.pl.resx +++ b/WindowTranslator/Properties/Resources.pl.resx @@ -306,9 +306,6 @@ Wyświetlaj ikonę zajętości - - Tłumacz tylko gdy nakładka jest włączona - Wyświetlaj tłumaczenie nakładki tylko dla tekstu pod pozycją wskaźnika myszy diff --git a/WindowTranslator/Properties/Resources.pt-BR.resx b/WindowTranslator/Properties/Resources.pt-BR.resx index c67b535e..57eac78f 100644 --- a/WindowTranslator/Properties/Resources.pt-BR.resx +++ b/WindowTranslator/Properties/Resources.pt-BR.resx @@ -306,9 +306,6 @@ Mostrar ícone de ocupado - - Traduzir apenas quando a sobreposição estiver ativada - Exibir tradução da sobreposição apenas para texto na posição do ponteiro do mouse diff --git a/WindowTranslator/Properties/Resources.resx b/WindowTranslator/Properties/Resources.resx index abc7f099..9990ab8c 100644 --- a/WindowTranslator/Properties/Resources.resx +++ b/WindowTranslator/Properties/Resources.resx @@ -306,9 +306,6 @@ 処理中アイコンを表示する - - オーバレイを有効にしたタイミングのみ翻訳する - マウスポインター位置のテキストのみオーバレイ翻訳を表示する diff --git a/WindowTranslator/Properties/Resources.ru.resx b/WindowTranslator/Properties/Resources.ru.resx index 30166ed9..91e17c08 100644 --- a/WindowTranslator/Properties/Resources.ru.resx +++ b/WindowTranslator/Properties/Resources.ru.resx @@ -306,9 +306,6 @@ Отображать значок обработки - - Переводить только при включении оверлея - Отображать перевод оверлея только для текста в позиции указателя мыши diff --git a/WindowTranslator/Properties/Resources.th.resx b/WindowTranslator/Properties/Resources.th.resx index a8fa27c7..bfb7de13 100644 --- a/WindowTranslator/Properties/Resources.th.resx +++ b/WindowTranslator/Properties/Resources.th.resx @@ -306,9 +306,6 @@ แสดงไอคอนกำลังประมวลผล - - แปลเฉพาะเมื่อเปิดใช้งานโอเวอร์เลย์เท่านั้น - แสดงการแปลโอเวอร์เลย์เฉพาะข้อความที่ตำแหน่งเมาส์เท่านั้น diff --git a/WindowTranslator/Properties/Resources.tr.resx b/WindowTranslator/Properties/Resources.tr.resx index 5c3f3397..ea62219d 100644 --- a/WindowTranslator/Properties/Resources.tr.resx +++ b/WindowTranslator/Properties/Resources.tr.resx @@ -306,9 +306,6 @@ İşlem simgesini göster - - Yalnızca kaplama etkinleştirildiğinde çevir - Kaplama çevirisini yalnızca fare işaretçisi konumundaki metin için göster diff --git a/WindowTranslator/Properties/Resources.vi.resx b/WindowTranslator/Properties/Resources.vi.resx index a98af764..922c5fde 100644 --- a/WindowTranslator/Properties/Resources.vi.resx +++ b/WindowTranslator/Properties/Resources.vi.resx @@ -306,9 +306,6 @@ Hiển thị biểu tượng bận - - Chỉ dịch khi lớp phủ được kích hoạt - Chỉ hiển thị bản dịch lớp phủ cho văn bản tại vị trí con trỏ chuột diff --git a/WindowTranslator/Properties/Resources.zh-CN.resx b/WindowTranslator/Properties/Resources.zh-CN.resx index 2d4e955f..fbbf5a4d 100644 --- a/WindowTranslator/Properties/Resources.zh-CN.resx +++ b/WindowTranslator/Properties/Resources.zh-CN.resx @@ -306,9 +306,6 @@ 显示忙碌图标 - - 仅在启用覆盖层时翻译 - 仅在鼠标指针位置的文本上显示覆盖翻译 diff --git a/WindowTranslator/Properties/Resources.zh-TW.resx b/WindowTranslator/Properties/Resources.zh-TW.resx index eff0a897..632e3971 100644 --- a/WindowTranslator/Properties/Resources.zh-TW.resx +++ b/WindowTranslator/Properties/Resources.zh-TW.resx @@ -306,9 +306,6 @@ 顯示忙碌圖標 - - 僅在啟用覆蓋層時翻譯 - 僅在滑鼠指標位置的文字上顯示重疊翻譯