From b7da6024a7cc7ad6ab341a0e4f45e298b83710d1 Mon Sep 17 00:00:00 2001 From: joshp123 Date: Sun, 7 Jun 2026 20:46:12 +0200 Subject: [PATCH 1/2] Add imsgcrawl built-in crawler --- README.md | 1 + .../BuiltInCrawlApps+Imsgcrawl.swift | 39 +++++++++++++++++++ .../BuiltInApps/BuiltInCrawlApps.swift | 2 + Sources/CrawlBarSelfTest/SelfTestConfig.swift | 7 ++++ 4 files changed, 49 insertions(+) create mode 100644 Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps+Imsgcrawl.swift diff --git a/README.md b/README.md index 18ab157..531d7b1 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Built-in manifests ship for: - `slacrawl` - `discrawl` - `telecrawl` +- `imsgcrawl` - `notcrawl` - `gogcli` through the `gog` executable - `wacli` diff --git a/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps+Imsgcrawl.swift b/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps+Imsgcrawl.swift new file mode 100644 index 0000000..b4151c6 --- /dev/null +++ b/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps+Imsgcrawl.swift @@ -0,0 +1,39 @@ +import Foundation + +public extension BuiltInCrawlApps { + static let imsgcrawl = CrawlAppManifest( + id: Self.imsgcrawlID, + displayName: "iMessage", + description: "Local-first iMessage archive crawler", + binary: .init(name: "imsgcrawl"), + branding: .init( + symbolName: "message.fill", + accentColor: "#34C759", + bundleIdentifier: "com.apple.MobileSMS"), + paths: .init( + defaultDatabase: "~/.imsgcrawl/archive.db", + defaultCache: "~/.imsgcrawl/cache", + defaultLogs: "~/.imsgcrawl/logs"), + commands: [ + "metadata": ["--json", "metadata"], + "status": ["--json", "status"], + "refresh": ["--json", "sync"], + "chats": ["--json", "chats"], + "messages": ["--json", "messages"], + "search": ["--json", "search"], + "contact-export": ["--json", "contacts", "export"], + ], + capabilities: [.status, .refresh, .search], + statusRequiresSecrets: false, + privacy: .init( + containsPrivateMessages: true, + exportsSecrets: false, + localOnlyScopes: [ + "apple-messages", + "sqlite", + "contact-handles", + "message-archive", + "message-text-search", + ])) + .withSuggestion(Self.appSuggest("Messages", ["com.apple.MobileSMS"])) +} diff --git a/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps.swift b/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps.swift index b98a83b..d92e410 100644 --- a/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps.swift +++ b/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps.swift @@ -5,6 +5,7 @@ public enum BuiltInCrawlApps { public static let slacrawlID = CrawlAppID(rawValue: "slacrawl") public static let discrawlID = CrawlAppID(rawValue: "discrawl") public static let telecrawlID = CrawlAppID(rawValue: "telecrawl") + public static let imsgcrawlID = CrawlAppID(rawValue: "imsgcrawl") public static let notcrawlID = CrawlAppID(rawValue: "notcrawl") public static let gogcliID = CrawlAppID(rawValue: "gogcli") public static let wacliID = CrawlAppID(rawValue: "wacli") @@ -16,6 +17,7 @@ public enum BuiltInCrawlApps { Self.slacrawl, Self.discrawl, Self.telecrawl, + Self.imsgcrawl, Self.notcrawl, Self.gogcli, Self.wacli, diff --git a/Sources/CrawlBarSelfTest/SelfTestConfig.swift b/Sources/CrawlBarSelfTest/SelfTestConfig.swift index 388a88c..5ff1fcc 100644 --- a/Sources/CrawlBarSelfTest/SelfTestConfig.swift +++ b/Sources/CrawlBarSelfTest/SelfTestConfig.swift @@ -160,6 +160,13 @@ extension CrawlBarSelfTest { try Self.expect(BuiltInCrawlApps.telecrawl.install?.package == "steipete/tap/telecrawl", "telecrawl install metadata exists") try Self.expect(BuiltInCrawlApps.telecrawl.paths.defaultConfig == "~/.telecrawl/backup.json", "telecrawl config path maps") try Self.expect(BuiltInCrawlApps.graincrawl.availability == .available, "graincrawl is available") + try Self.expect(BuiltInCrawlApps.imsgcrawl.displayName == "iMessage", "imsgcrawl uses user-facing iMessage name") + try Self.expect(BuiltInCrawlApps.imsgcrawl.commands["status"] == ["--json", "status"], "imsgcrawl uses crawlkit status command") + try Self.expect(BuiltInCrawlApps.imsgcrawl.commands["refresh"] == ["--json", "sync"], "imsgcrawl sync is wired as refresh") + try Self.expect(BuiltInCrawlApps.imsgcrawl.commands["contact-export"] == ["--json", "contacts", "export"], "imsgcrawl contact export is wired") + try Self.expect(BuiltInCrawlApps.imsgcrawl.privacy.containsPrivateMessages, "imsgcrawl privacy metadata flags iMessage data") + try Self.expect(BuiltInCrawlApps.imsgcrawl.branding.bundleIdentifier == "com.apple.MobileSMS", "imsgcrawl uses native Messages app icon") + try Self.expect(BuiltInCrawlApps.imsgcrawl.suggestion?.name == "Messages", "imsgcrawl suggests from the native Messages app") try Self.expect(BuiltInCrawlApps.graincrawl.commands["status"] == ["status", "--json"], "graincrawl uses crawlkit status command") try Self.expect( BuiltInCrawlApps.graincrawl.commands["refresh"] == ["sync", "--json"], From 8c110d4723e659967fa7fc269342fb5a453d5b9f Mon Sep 17 00:00:00 2001 From: Vincent Koc <25068+vincentkoc@users.noreply.github.com> Date: Wed, 17 Jun 2026 18:22:52 +0800 Subject: [PATCH 2/2] fix: protect imessage crawler integration --- .../BuiltInCrawlApps+Imsgcrawl.swift | 3 --- .../CrawlBarCore/StatusMapperCrawlKit.swift | 4 +++- Sources/CrawlBarCore/StatusMapperJSON.swift | 10 +++++++- Sources/CrawlBarSelfTest/SelfTestConfig.swift | 5 +++- .../SelfTestStatusMapping.swift | 24 +++++++++++++++++++ 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps+Imsgcrawl.swift b/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps+Imsgcrawl.swift index b4151c6..0a62ca7 100644 --- a/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps+Imsgcrawl.swift +++ b/Sources/CrawlBarCore/BuiltInApps/BuiltInCrawlApps+Imsgcrawl.swift @@ -18,10 +18,7 @@ public extension BuiltInCrawlApps { "metadata": ["--json", "metadata"], "status": ["--json", "status"], "refresh": ["--json", "sync"], - "chats": ["--json", "chats"], - "messages": ["--json", "messages"], "search": ["--json", "search"], - "contact-export": ["--json", "contacts", "export"], ], capabilities: [.status, .refresh, .search], statusRequiresSecrets: false, diff --git a/Sources/CrawlBarCore/StatusMapperCrawlKit.swift b/Sources/CrawlBarCore/StatusMapperCrawlKit.swift index 19c5680..b688d9f 100644 --- a/Sources/CrawlBarCore/StatusMapperCrawlKit.swift +++ b/Sources/CrawlBarCore/StatusMapperCrawlKit.swift @@ -27,7 +27,9 @@ extension CrawlStatusMapper { share: self.shareStatus(in: object), remote: remote, sqliteObject: self.sqliteObjectStatus(in: object), - sqliteBundle: self.sqliteBundleStatus(in: object)) + sqliteBundle: self.sqliteBundleStatus(in: object), + warnings: self.stringValues(["warnings"], in: object), + errors: self.stringValues(["errors"], in: object)) } func isCrawlKitStatus(_ object: [String: Any]) -> Bool { diff --git a/Sources/CrawlBarCore/StatusMapperJSON.swift b/Sources/CrawlBarCore/StatusMapperJSON.swift index 4d6e555..67ab9fe 100644 --- a/Sources/CrawlBarCore/StatusMapperJSON.swift +++ b/Sources/CrawlBarCore/StatusMapperJSON.swift @@ -53,13 +53,21 @@ extension CrawlStatusMapper { return .current case "warn", "warning", "degraded": return .stale - case "failed", "failure": + case "failed", "failure", "source_error", "archive_error": return .error default: return nil } } + func stringValues(_ keys: [String], in object: [String: Any]) -> [String] { + for key in keys { + guard let values = self.firstValue(key, in: object) as? [Any] else { continue } + return values.compactMap { ($0 as? String)?.nilIfBlank } + } + return [] + } + func dateValue(_ keys: [String], in object: [String: Any]) -> Date? { for key in keys { guard let value = self.firstValue(key, in: object) else { continue } diff --git a/Sources/CrawlBarSelfTest/SelfTestConfig.swift b/Sources/CrawlBarSelfTest/SelfTestConfig.swift index 5ff1fcc..e9197ae 100644 --- a/Sources/CrawlBarSelfTest/SelfTestConfig.swift +++ b/Sources/CrawlBarSelfTest/SelfTestConfig.swift @@ -163,7 +163,10 @@ extension CrawlBarSelfTest { try Self.expect(BuiltInCrawlApps.imsgcrawl.displayName == "iMessage", "imsgcrawl uses user-facing iMessage name") try Self.expect(BuiltInCrawlApps.imsgcrawl.commands["status"] == ["--json", "status"], "imsgcrawl uses crawlkit status command") try Self.expect(BuiltInCrawlApps.imsgcrawl.commands["refresh"] == ["--json", "sync"], "imsgcrawl sync is wired as refresh") - try Self.expect(BuiltInCrawlApps.imsgcrawl.commands["contact-export"] == ["--json", "contacts", "export"], "imsgcrawl contact export is wired") + try Self.expect(BuiltInCrawlApps.imsgcrawl.commands["search"] == ["--json", "search"], "imsgcrawl search is wired") + try Self.expect(BuiltInCrawlApps.imsgcrawl.commands["chats"] == nil, "imsgcrawl chats stay outside persisted actions") + try Self.expect(BuiltInCrawlApps.imsgcrawl.commands["messages"] == nil, "imsgcrawl messages stay outside persisted actions") + try Self.expect(BuiltInCrawlApps.imsgcrawl.commands["contact-export"] == nil, "imsgcrawl contact export stays outside persisted actions") try Self.expect(BuiltInCrawlApps.imsgcrawl.privacy.containsPrivateMessages, "imsgcrawl privacy metadata flags iMessage data") try Self.expect(BuiltInCrawlApps.imsgcrawl.branding.bundleIdentifier == "com.apple.MobileSMS", "imsgcrawl uses native Messages app icon") try Self.expect(BuiltInCrawlApps.imsgcrawl.suggestion?.name == "Messages", "imsgcrawl suggests from the native Messages app") diff --git a/Sources/CrawlBarSelfTest/SelfTestStatusMapping.swift b/Sources/CrawlBarSelfTest/SelfTestStatusMapping.swift index b23060b..39f32c8 100644 --- a/Sources/CrawlBarSelfTest/SelfTestStatusMapping.swift +++ b/Sources/CrawlBarSelfTest/SelfTestStatusMapping.swift @@ -315,6 +315,30 @@ extension CrawlBarSelfTest { let failedStatus = CrawlStatusMapper().status(from: failedResult, manifest: BuiltInCrawlApps.graincrawl) try Self.expect(failedStatus.state == .error, "crawlkit failed state maps to error") + let imsgcrawlSourceErrorResult = CrawlCommandResult( + appID: BuiltInCrawlApps.imsgcrawlID, + action: "status", + exitCode: 0, + stdout: """ + { + "schema_version": "crawlkit.control.v1", + "app_id": "imsgcrawl", + "state": "source_error", + "summary": "Messages source could not be read.", + "warnings": ["archive has not been synced"], + "errors": ["Messages database access was denied"] + } + """, + stderr: "", + startedAt: Date(), + finishedAt: Date()) + let imsgcrawlSourceErrorStatus = CrawlStatusMapper().status( + from: imsgcrawlSourceErrorResult, + manifest: BuiltInCrawlApps.imsgcrawl) + try Self.expect(imsgcrawlSourceErrorStatus.state == .error, "crawlkit source errors map to error") + try Self.expect(imsgcrawlSourceErrorStatus.warnings == ["archive has not been synced"], "crawlkit warnings are preserved") + try Self.expect(imsgcrawlSourceErrorStatus.errors == ["Messages database access was denied"], "crawlkit errors are preserved") + let githubAuthMessage = """ [github] request GET /repos/openclaw/openclaw github GET /repos/openclaw/openclaw failed with status 401: {