From 2a6315281a685d111809641dddaa58ec573570cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Sun, 12 Apr 2026 14:36:57 +0200 Subject: [PATCH 1/9] Retry oversized Matrix messages as chunks Bump package version and add handling for oversized Matrix messages and thread reply metadata. sendRoomMessage now accepts an explicit reply target and applies thread reply fallback metadata when posting into Matrix threads. If a Matrix send fails with M_TOO_LARGE/413 for rich-text content, the adapter will split splittable text content into UTF-8 byte-sized chunks (DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES = 12000) and retry as plain-text message chunks, logging a warning; non-text oversized errors are rethrown and logged. Added helper methods for splitting by UTF-8 bytes, detecting splittable content, and improved error handling and client send logic. Updated and added tests to cover reply fallbacks, explicit reply targets, chunked retries, and M_TOO_LARGE behavior. Version bumped to 0.2.1. --- package.json | 8 +- pnpm-lock.yaml | 34 +++---- src/index.test.ts | 171 ++++++++++++++++++++++++++++++- src/index.ts | 252 ++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 424 insertions(+), 41 deletions(-) diff --git a/package.json b/package.json index ea1b884..2f9f198 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@beeper/chat-adapter-matrix", - "version": "0.2.0", + "version": "0.2.1", "description": "Matrix adapter for chat", "engines": { "node": ">=22" @@ -33,9 +33,9 @@ "clean": "rm -rf dist" }, "dependencies": { - "@chat-adapter/state-memory": "^4.17.0", - "@chat-adapter/state-redis": "^4.17.0", - "chat": "^4.17.0", + "@chat-adapter/state-memory": "^4.25.0", + "@chat-adapter/state-redis": "^4.25.0", + "chat": "^4.25.0", "marked": "^15.0.12", "matrix-js-sdk": "^41.0.0", "node-html-parser": "^7.1.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 30910b8..6e9c5a0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,14 +9,14 @@ importers: .: dependencies: '@chat-adapter/state-memory': - specifier: ^4.17.0 - version: 4.17.0 + specifier: ^4.25.0 + version: 4.25.0 '@chat-adapter/state-redis': - specifier: ^4.17.0 - version: 4.17.0 + specifier: ^4.25.0 + version: 4.25.0 chat: - specifier: ^4.17.0 - version: 4.17.0 + specifier: ^4.25.0 + version: 4.25.0 marked: specifier: ^15.0.12 version: 15.0.12 @@ -82,11 +82,11 @@ packages: '@bcoe/v8-coverage@0.2.3': resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} - '@chat-adapter/state-memory@4.17.0': - resolution: {integrity: sha512-7LewkFY6gQVEXxAycaWUXax9Pt7Prmn2mBdoIBbTEAzV5bSQBaXqKyrd6VWCd9gEyxNlBUainyQul4iGNPw1sQ==} + '@chat-adapter/state-memory@4.25.0': + resolution: {integrity: sha512-bNM4ycFPfGHUI5TN3yo9wGG8ahgUXlKS7WLzhr8WnjmH6g6o8OebknihNh3WDw07xckJUoKDpOtE1uwChvgZag==} - '@chat-adapter/state-redis@4.17.0': - resolution: {integrity: sha512-sJ2V/pZESZKrGkuafMqg4gqyvHwz8d0JDUvqeZ+FVfEgT/h0g1kYfSc++qfEayzmPpSJxC6gzGpBGTiL/5utCw==} + '@chat-adapter/state-redis@4.25.0': + resolution: {integrity: sha512-XHZ6Kv9vbKP19YaRKXaurnsxzizHpQOkfY0P4JPS8oEGVKSCJ5a9ZEnlsT3guLG8zcO6cFqNR2Br+PmX6fh1Dw==} '@esbuild/aix-ppc64@0.21.5': resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} @@ -849,8 +849,8 @@ packages: character-entities@2.0.2: resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} - chat@4.17.0: - resolution: {integrity: sha512-kX9jIXmEU2ksnF1YshM+qfI/6pBy6k8pELkuRaE2AJyF/0nt4GUc0McBhn2DNpXLDI5sVwRGB3PmI+q5wUtYFg==} + chat@4.25.0: + resolution: {integrity: sha512-QM8ex4Gpn8zYIPyQXh41Who6R9Wq3WcQeOjAy4EuR1m1ha0tASuzHkLQfjaTAGLgrgrThV0Zh5KKoH0S92iwNA==} check-error@2.1.3: resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} @@ -1797,15 +1797,15 @@ snapshots: '@bcoe/v8-coverage@0.2.3': {} - '@chat-adapter/state-memory@4.17.0': + '@chat-adapter/state-memory@4.25.0': dependencies: - chat: 4.17.0 + chat: 4.25.0 transitivePeerDependencies: - supports-color - '@chat-adapter/state-redis@4.17.0': + '@chat-adapter/state-redis@4.25.0': dependencies: - chat: 4.17.0 + chat: 4.25.0 redis: 5.11.0 transitivePeerDependencies: - '@node-rs/xxhash' @@ -2373,7 +2373,7 @@ snapshots: character-entities@2.0.2: {} - chat@4.17.0: + chat@4.25.0: dependencies: '@workflow/serde': 4.1.0-beta.2 mdast-util-to-string: 4.0.0 diff --git a/src/index.test.ts b/src/index.test.ts index 4f98493..30acd70 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { Chat, getEmoji, stringifyMarkdown } from "chat"; -import type { ChatInstance, Logger, StateAdapter } from "chat"; +import type { AdapterPostableMessage, ChatInstance, Logger, StateAdapter } from "chat"; import { createMemoryState } from "@chat-adapter/state-memory"; import { EventType, MsgType, RelationType, type MatrixClient } from "matrix-js-sdk"; import { MatrixError } from "matrix-js-sdk/lib/http-api/errors"; @@ -310,6 +310,8 @@ function makeStateAdapter(initial: Record = {}): StateAdapter { return run(); }; const get: StateAdapter["get"] = (key) => afterReady(() => base.get(key)); + const getList: StateAdapter["getList"] = (key) => + afterReady(() => base.getList(key)); const set: StateAdapter["set"] = (key, value, ttlMs) => afterReady(() => base.set(key, value, ttlMs)); const setIfNotExists: StateAdapter["setIfNotExists"] = (key, value, ttlMs) => @@ -319,14 +321,26 @@ function makeStateAdapter(initial: Record = {}): StateAdapter { acquireLock: vi.fn((threadId, ttlMs) => afterReady(() => base.acquireLock(threadId, ttlMs)) ), + appendToList: vi.fn((key, value, options) => + afterReady(() => base.appendToList(key, value, options)) + ), connect: vi.fn(connect), delete: vi.fn((key) => afterReady(() => base.delete(key))), + dequeue: vi.fn((threadId) => afterReady(() => base.dequeue(threadId))), disconnect: vi.fn(() => afterReady(() => base.disconnect())), + enqueue: vi.fn((threadId, entry, maxSize) => + afterReady(() => base.enqueue(threadId, entry, maxSize)) + ), extendLock: vi.fn((lock, ttlMs) => afterReady(() => base.extendLock(lock, ttlMs)) ), + forceReleaseLock: vi.fn((threadId) => + afterReady(() => base.forceReleaseLock(threadId)) + ), get, + getList, isSubscribed: vi.fn((threadId) => afterReady(() => base.isSubscribed(threadId))), + queueDepth: vi.fn((threadId) => afterReady(() => base.queueDepth(threadId))), releaseLock: vi.fn((lock) => afterReady(() => base.releaseLock(lock))), set: vi.fn(set), setIfNotExists: vi.fn(setIfNotExists), @@ -1069,6 +1083,53 @@ describe("MatrixAdapter", () => { expect(channel.metadata).toMatchObject(thread.metadata ?? {}); }); + it("adds reply fallback metadata when posting to a Matrix thread", async () => { + const fakeClient = makeClient(); + const adapter = await makeInitializedAdapter(fakeClient); + + await adapter.postMessage("matrix:!room%3Abeeper.com:%24root", "hello"); + + expect(fakeClient.sendEvent).toHaveBeenCalledWith( + "!room:beeper.com", + "$root", + EventType.RoomMessage, + expect.objectContaining({ + body: "hello", + msgtype: MsgType.Text, + "m.relates_to": { + "m.in_reply_to": { + event_id: "$root", + }, + }, + }) + ); + }); + + it("uses an explicit reply target when posting within a Matrix thread", async () => { + const fakeClient = makeClient(); + const adapter = await makeInitializedAdapter(fakeClient); + + await adapter.postMessage("matrix:!room%3Abeeper.com:%24root", { + raw: "hello", + matrixReplyToEventId: "$reply", + } as AdapterPostableMessage); + + expect(fakeClient.sendEvent).toHaveBeenCalledWith( + "!room:beeper.com", + "$root", + EventType.RoomMessage, + expect.objectContaining({ + body: "hello", + msgtype: MsgType.Text, + "m.relates_to": { + "m.in_reply_to": { + event_id: "$reply", + }, + }, + }) + ); + }); + it("uploads file payloads and posts Matrix media events", async () => { const fakeClient = makeClient(); fakeClient.sendEvent = vi @@ -1199,6 +1260,114 @@ describe("MatrixAdapter", () => { ); }); + it("retries oversized rich-text events as split plain-text messages", async () => { + const fakeClient = makeClient(); + const logger = makeTestLogger(); + const tooLargeError = new MatrixError( + { errcode: "M_TOO_LARGE", error: "event too large" }, + 413 + ); + fakeClient.sendEvent = vi + .fn() + .mockRejectedValueOnce(tooLargeError) + .mockResolvedValueOnce({ event_id: "$chunk-1" }) + .mockResolvedValueOnce({ event_id: "$chunk-2" }); + + const adapter = new MatrixAdapter({ + baseURL: "https://hs.beeper.com", + auth: { + type: "accessToken", + accessToken: "token", + userID: "@bot:beeper.com", + }, + logger: logger.logger, + createClient: () => asMatrixClient(fakeClient), + }); + await adapter.initialize(makeChatInstance()); + + const repeatedMarkdown = Array.from({ length: 2_000 }, () => "**hello** `world`").join("\n"); + const sent = await adapter.postMessage("matrix:!room%3Abeeper.com", { + markdown: repeatedMarkdown, + }); + + expect(sent.id).toBe("$chunk-1"); + expect(fakeClient.sendEvent).toHaveBeenCalledTimes(3); + expect(fakeClient.sendEvent).toHaveBeenNthCalledWith( + 1, + "!room:beeper.com", + EventType.RoomMessage, + expect.objectContaining({ + format: "org.matrix.custom.html", + formatted_body: expect.any(String), + }) + ); + + const fallbackBodies = (fakeClient.sendEvent.mock.calls as unknown as Array) + .slice(1) + .map((call) => call[2] as unknown as Record); + expect(fallbackBodies).toHaveLength(2); + expect(fallbackBodies.map((content) => content.body).join("\n")) + .toBe(Array.from({ length: 2_000 }, () => "hello world").join("\n")); + for (const content of fallbackBodies) { + expect(content).toMatchObject({ msgtype: MsgType.Text }); + expect(content).not.toHaveProperty("format"); + expect(content).not.toHaveProperty("formatted_body"); + expect(content).not.toHaveProperty("m.mentions"); + } + + expect(logger.warn).toHaveBeenCalledWith( + "Matrix message exceeded size limit; retrying as split plain-text chunks", + expect.objectContaining({ + roomId: "!room:beeper.com", + chunkCount: 2, + msgtype: MsgType.Text, + }) + ); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it("rethrows M_TOO_LARGE for non-text Matrix events", async () => { + const fakeClient = makeClient(); + const logger = makeTestLogger(); + const tooLargeError = new MatrixError( + { errcode: "M_TOO_LARGE", error: "event too large" }, + 413 + ); + fakeClient.sendEvent = vi.fn().mockRejectedValue(tooLargeError); + + const adapter = new MatrixAdapter({ + baseURL: "https://hs.beeper.com", + auth: { + type: "accessToken", + accessToken: "token", + userID: "@bot:beeper.com", + }, + logger: logger.logger, + createClient: () => asMatrixClient(fakeClient), + }); + await adapter.initialize(makeChatInstance()); + + await expect( + adapter.postMessage("matrix:!room%3Abeeper.com", { + files: [ + { + data: new Uint8Array([1, 2, 3]).buffer, + filename: "report.png", + mimeType: "image/png", + }, + ], + } as AdapterPostableMessage) + ).rejects.toBe(tooLargeError); + expect(logger.error).toHaveBeenCalledWith( + "Matrix send message failed", + expect.objectContaining({ + roomId: "!room:beeper.com", + msgtype: MsgType.Image, + error: tooLargeError, + }) + ); + }); + it("logs and rethrows Matrix upload failures", async () => { const fakeClient = makeClient(); const logger = makeTestLogger(); diff --git a/src/index.ts b/src/index.ts index f22131d..ce1fb22 100644 --- a/src/index.ts +++ b/src/index.ts @@ -80,6 +80,7 @@ const DEFAULT_COMMAND_PREFIX = "/"; const DEFAULT_PERSISTENCE_KEY_PREFIX = "matrix"; const TYPING_TIMEOUT_MS = 30_000; const DEFAULT_MATRIX_STORE_PERSIST_INTERVAL_MS = 30_000; +const DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES = 12_000; const FAST_SYNC_DEFAULTS: NonNullable = { initialSyncLimit: 1, lazyLoadMembers: true, @@ -420,14 +421,20 @@ export class MatrixAdapter implements Adapter { message: AdapterPostableMessage ): Promise> { const { roomID, rootEventID } = this.decodeThreadId(threadId); + const replyEventID = this.extractReplyEventID(message); const contents = await this.toRoomMessageContents(message); const [firstContent, ...extraContents] = contents; if (!firstContent) { throw new Error("Cannot post an empty Matrix message."); } - const response = await this.sendRoomMessage(roomID, rootEventID, firstContent); + const response = await this.sendRoomMessage( + roomID, + rootEventID, + replyEventID, + firstContent, + ); for (const content of extraContents) { - await this.sendRoomMessage(roomID, rootEventID, content); + await this.sendRoomMessage(roomID, rootEventID, replyEventID, content); } return { @@ -472,7 +479,7 @@ export class MatrixAdapter implements Adapter { body: `* ${baseContent.body}`, }; - const response = await this.sendRoomMessage(roomID, rootEventID, editContent); + const response = await this.sendRoomMessage(roomID, rootEventID, undefined, editContent); return { id: response.event_id, @@ -1504,27 +1511,56 @@ export class MatrixAdapter implements Adapter { private async sendRoomMessage( roomID: string, rootEventID: string | undefined, + replyEventID: string | undefined, content: MatrixOutboundMessageContent ) { - const response = await this.withLoggedMatrixOperation( - "Matrix send message failed", - { + const threadContent = this.applyThreadReplyMetadata(content, rootEventID, replyEventID); + + try { + const response = await this.performSendRoomMessage(roomID, rootEventID, threadContent); + void this.maybePersistSecretsBundle(); + return response; + } catch (error) { + if (this.isTooLargeMatrixError(error)) { + const splitContents = this.splitOversizedTextContent(threadContent); + if (splitContents.length > 1) { + this.logger.warn( + "Matrix message exceeded size limit; retrying as split plain-text chunks", + { + roomId: roomID, + rootEventId: rootEventID, + originalLength: typeof threadContent.body === "string" ? threadContent.body.length : undefined, + chunkCount: splitContents.length, + msgtype: threadContent.msgtype, + } + ); + + let firstResponse: Awaited> | undefined; + for (const splitContent of splitContents) { + const response = await this.sendRoomMessage( + roomID, + rootEventID, + replyEventID, + splitContent + ); + firstResponse ??= response; + } + + if (firstResponse) { + return firstResponse; + } + } + } + + this.logger.error("Matrix send message failed", { roomId: roomID, rootEventId: rootEventID, eventType: EventType.RoomMessage, msgtype: content.msgtype, - }, - async () => { - const client = this.requireClient(); - if (rootEventID) { - return client.sendEvent(roomID, rootEventID, EventType.RoomMessage, content); - } - - return client.sendEvent(roomID, EventType.RoomMessage, content); - } - ); - void this.maybePersistSecretsBundle(); - return response; + error, + }); + throw error; + } } private async toRoomMessageContents( @@ -1860,7 +1896,7 @@ export class MatrixAdapter implements Adapter { raw: event, user: message.author, triggerId: event.getId(), - }); + }, undefined); } } @@ -2594,6 +2630,184 @@ export class MatrixAdapter implements Adapter { }; } + private extractReplyEventID(message: AdapterPostableMessage): string | undefined { + if (typeof message !== "object" || message === null || isCardElement(message)) { + return undefined; + } + + const replyEventID = (message as { matrixReplyToEventId?: unknown }).matrixReplyToEventId; + return typeof replyEventID === "string" && replyEventID.length > 0 + ? replyEventID + : undefined; + } + + private applyThreadReplyMetadata( + content: MatrixOutboundMessageContent, + rootEventID: string | undefined, + replyEventID: string | undefined + ): MatrixOutboundMessageContent { + const threadableContent = content as MatrixOutboundMessageContent & { + "m.relates_to"?: { + rel_type?: string; + "m.in_reply_to"?: { event_id: string }; + [key: string]: unknown; + }; + }; + + if (!rootEventID || threadableContent["m.relates_to"]?.rel_type) { + return threadableContent; + } + + return { + ...threadableContent, + "m.relates_to": { + ...threadableContent["m.relates_to"], + "m.in_reply_to": { + event_id: replyEventID ?? rootEventID, + }, + }, + } as MatrixOutboundMessageContent; + } + + private async performSendRoomMessage( + roomID: string, + rootEventID: string | undefined, + content: MatrixOutboundMessageContent + ) { + const client = this.requireClient(); + if (rootEventID) { + return client.sendEvent( + roomID, + rootEventID, + EventType.RoomMessage, + content as RoomMessageEventContent + ); + } + + return client.sendEvent(roomID, EventType.RoomMessage, content); + } + + private isTooLargeMatrixError(error: unknown): error is MatrixError { + return ( + error instanceof MatrixError && + (error.errcode === "M_TOO_LARGE" || error.httpStatus === 413) + ); + } + + private splitOversizedTextContent( + content: MatrixOutboundMessageContent + ): MatrixTextMessageContent[] { + if (!this.isSplittableTextContent(content)) { + return []; + } + + const body = content.body; + if (this.utf8ByteLength(body) <= DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES) { + return []; + } + + const parts = this.splitTextByUtf8Bytes(body, DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES); + if (parts.length <= 1) { + return []; + } + + return parts.map((part) => ({ + body: part, + msgtype: content.msgtype, + })); + } + + private isSplittableTextContent( + content: MatrixOutboundMessageContent + ): content is MatrixTextMessageContent { + if ("url" in content || "info" in content) { + return false; + } + + if (typeof content.body !== "string" || content.body.length <= 1) { + return false; + } + + if ("m.new_content" in content) { + return false; + } + + return ( + content.msgtype === MsgType.Text || + content.msgtype === MsgType.Notice || + content.msgtype === MsgType.Emote + ); + } + + private splitTextByUtf8Bytes(text: string, maxBytes: number): string[] { + const normalizedMaxBytes = Math.max(1, Math.floor(maxBytes)); + const chunks: string[] = []; + let remaining = text; + + while (remaining.length > 0) { + if (this.utf8ByteLength(remaining) <= normalizedMaxBytes) { + chunks.push(remaining); + remaining = ""; + break; + } + + const boundary = this.findSplitBoundary(remaining, normalizedMaxBytes); + const head = remaining.slice(0, boundary).trimEnd(); + const tail = remaining.slice(boundary).trimStart(); + + if (!head || head === remaining) { + break; + } + + chunks.push(head); + remaining = tail; + } + + if (remaining.length > 0 && chunks.at(-1) !== remaining) { + chunks.push(remaining); + } + + return chunks.filter((chunk) => chunk.length > 0); + } + + private findSplitBoundary(text: string, maxBytes: number): number { + let low = 1; + let high = text.length; + let best = 1; + + while (low <= high) { + const mid = Math.floor((low + high) / 2); + const candidate = text.slice(0, mid); + if (this.utf8ByteLength(candidate) <= maxBytes) { + best = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + + for (const pattern of [/\n{2,}/gu, /\n/gu, /\s+/gu]) { + let match: RegExpExecArray | null = null; + let lastBoundary: number | null = null; + pattern.lastIndex = 0; + while ((match = pattern.exec(text)) !== null) { + if (match.index >= best) { + break; + } + lastBoundary = match.index + match[0].length; + } + if (lastBoundary && lastBoundary > 0) { + return lastBoundary; + } + } + + return best; + } + + private utf8ByteLength(text: string): number { + return Buffer.byteLength(text, "utf8"); + } + private toRoomMessageContent( message: AdapterPostableMessage ): MatrixTextMessageContent { From 2db39ccccf7dce840e91f3554de8ed52b4a5aebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Sun, 12 Apr 2026 14:39:38 +0200 Subject: [PATCH 2/9] Move state adapters to devDependencies Move @chat-adapter/state-memory and @chat-adapter/state-redis from dependencies to devDependencies in package.json because they are only needed for development/test environments. --- package.json | 4 ++-- pnpm-lock.yaml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 2f9f198..17d2385 100644 --- a/package.json +++ b/package.json @@ -33,14 +33,14 @@ "clean": "rm -rf dist" }, "dependencies": { - "@chat-adapter/state-memory": "^4.25.0", - "@chat-adapter/state-redis": "^4.25.0", "chat": "^4.25.0", "marked": "^15.0.12", "matrix-js-sdk": "^41.0.0", "node-html-parser": "^7.1.0" }, "devDependencies": { + "@chat-adapter/state-memory": "^4.25.0", + "@chat-adapter/state-redis": "^4.25.0", "@eslint/js": "^10.0.1", "@types/node": "^22.10.2", "@vitest/coverage-v8": "^2.1.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6e9c5a0..2e3dafc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,12 +8,6 @@ importers: .: dependencies: - '@chat-adapter/state-memory': - specifier: ^4.25.0 - version: 4.25.0 - '@chat-adapter/state-redis': - specifier: ^4.25.0 - version: 4.25.0 chat: specifier: ^4.25.0 version: 4.25.0 @@ -27,6 +21,12 @@ importers: specifier: ^7.1.0 version: 7.1.0 devDependencies: + '@chat-adapter/state-memory': + specifier: ^4.25.0 + version: 4.25.0 + '@chat-adapter/state-redis': + specifier: ^4.25.0 + version: 4.25.0 '@eslint/js': specifier: ^10.0.1 version: 10.0.1(eslint@10.0.3) From 017a878ea418892107a0cc131624ccc7a7e950a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Sun, 12 Apr 2026 14:49:56 +0200 Subject: [PATCH 3/9] MatrixAdapter: improve messaging & evictions Multiple refactors and bug fixes in MatrixAdapter: - Send logic: pass undefined for reply target on extra contents, apply thread/reply metadata before sending, use performSendRoomMessage and persist secrets where appropriate. - Message chunking: replace utf8ByteLength helper with Buffer.byteLength and simplify boundary search for splitting oversized messages. - Session keys: remove sessionBasePrefix indirection and use persistenceSessionPrefix directly for session storage keys and username temporary key. - Eviction and state fixes: only evict processedTimelineEventIDs and reactionByEventID when sizes exceed 10,000, avoiding unnecessary evictions; avoid reassigning userID during login device lookup; simplify joinRoomWithRetry error handling. - HTML parsing: stop stripping reply fallback markup in parseMatrixHTML and remove the unused stripReplyFallbackFromHTML helper. - Logging: ensure matrixSDKLogConfigured is set when configuring SDK logger. These changes improve correctness, reduce unexpected state mutations, and optimize memory/IO behavior during message sending. --- src/index.ts | 75 +++++++++++++++------------------------------------- 1 file changed, 22 insertions(+), 53 deletions(-) diff --git a/src/index.ts b/src/index.ts index ce1fb22..b82ad44 100644 --- a/src/index.ts +++ b/src/index.ts @@ -434,7 +434,7 @@ export class MatrixAdapter implements Adapter { firstContent, ); for (const content of extraContents) { - await this.sendRoomMessage(roomID, rootEventID, replyEventID, content); + await this.sendRoomMessage(roomID, rootEventID, undefined, content); } return { @@ -1297,7 +1297,7 @@ export class MatrixAdapter implements Adapter { this.auth.password ); - let userID = this.auth.userID ?? loginResponse.user_id; + const userID = this.auth.userID ?? loginResponse.user_id; if (!userID) { throw new Error("Password login succeeded but no user ID was returned."); } @@ -1305,7 +1305,6 @@ export class MatrixAdapter implements Adapter { let authDeviceID = normalizeOptionalString(loginResponse.device_id); if (!authDeviceID) { const whoami = await this.lookupWhoAmIFromAccessToken(loginResponse.access_token); - userID = userID ?? whoami.userID; authDeviceID = whoami.deviceID; } @@ -1537,12 +1536,9 @@ export class MatrixAdapter implements Adapter { let firstResponse: Awaited> | undefined; for (const splitContent of splitContents) { - const response = await this.sendRoomMessage( - roomID, - rootEventID, - replyEventID, - splitContent - ); + const chunkWithMeta = this.applyThreadReplyMetadata(splitContent, rootEventID, replyEventID); + const response = await this.performSendRoomMessage(roomID, rootEventID, chunkWithMeta); + void this.maybePersistSecretsBundle(); firstResponse ??= response; } @@ -1814,7 +1810,9 @@ export class MatrixAdapter implements Adapter { return; } this.processedTimelineEventIDs.add(eventID); - evictOldestEntries(this.processedTimelineEventIDs); + if (this.processedTimelineEventIDs.size > 10_000) { + evictOldestEntries(this.processedTimelineEventIDs); + } } const roomID = room?.roomId ?? event.getRoomId(); @@ -1943,14 +1941,11 @@ export class MatrixAdapter implements Adapter { } private async joinRoomWithRetry(roomID: string, maxAttempts = 3): Promise { - let lastError: unknown; - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { try { await this.requireClient().joinRoom(roomID); return; } catch (error) { - lastError = error; if (!this.isRetryableJoinError(error) || attempt === maxAttempts) { throw error; } @@ -1968,8 +1963,6 @@ export class MatrixAdapter implements Adapter { ); } } - - throw lastError; } private shouldAcceptInvite( @@ -2057,7 +2050,9 @@ export class MatrixAdapter implements Adapter { userID: sender, }); - evictOldestEntries(this.reactionByEventID); + if (this.reactionByEventID.size > 10_000) { + evictOldestEntries(this.reactionByEventID); + } } this.requireChat().processReaction({ @@ -2232,7 +2227,7 @@ export class MatrixAdapter implements Adapter { private parseMatrixHTML( html: string ): { markdown: string; mentionedUserIDs: Set } { - const root = parseHTML(this.stripReplyFallbackFromHTML(html)); + const root = parseHTML(html); const mentionedUserIDs = new Set(); const markdown = this.normalizeMarkdownSpacing( this.renderHTMLNodesToMarkdown(root.childNodes, mentionedUserIDs) @@ -2422,16 +2417,6 @@ export class MatrixAdapter implements Adapter { return lines.slice(index + 1).join("\n"); } - private stripReplyFallbackFromHTML(html: string): string { - const root = parseHTML(html); - for (const child of [...root.childNodes]) { - if (child instanceof HTMLElement && child.tagName.toLowerCase() === "mx-reply") { - child.remove(); - } - } - return root.toString(); - } - private extractAttachments(content: MatrixMessageContent) { const url = typeof content.url === "string" ? content.url : undefined; if (!url) { @@ -2702,7 +2687,7 @@ export class MatrixAdapter implements Adapter { } const body = content.body; - if (this.utf8ByteLength(body) <= DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES) { + if (Buffer.byteLength(body, "utf8") <= DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES) { return []; } @@ -2745,7 +2730,7 @@ export class MatrixAdapter implements Adapter { let remaining = text; while (remaining.length > 0) { - if (this.utf8ByteLength(remaining) <= normalizedMaxBytes) { + if (Buffer.byteLength(remaining, "utf8") <= normalizedMaxBytes) { chunks.push(remaining); remaining = ""; break; @@ -2778,7 +2763,7 @@ export class MatrixAdapter implements Adapter { while (low <= high) { const mid = Math.floor((low + high) / 2); const candidate = text.slice(0, mid); - if (this.utf8ByteLength(candidate) <= maxBytes) { + if (Buffer.byteLength(candidate, "utf8") <= maxBytes) { best = mid; low = mid + 1; } else { @@ -2786,28 +2771,16 @@ export class MatrixAdapter implements Adapter { } } - for (const pattern of [/\n{2,}/gu, /\n/gu, /\s+/gu]) { - let match: RegExpExecArray | null = null; - let lastBoundary: number | null = null; - pattern.lastIndex = 0; - while ((match = pattern.exec(text)) !== null) { - if (match.index >= best) { - break; - } - lastBoundary = match.index + match[0].length; - } - if (lastBoundary && lastBoundary > 0) { - return lastBoundary; + for (let i = best - 1; i > 0; i--) { + const ch = text[i]; + if (ch === "\n" || ch === " " || ch === "\t" || ch === "\r") { + return i + 1; } } return best; } - private utf8ByteLength(text: string): number { - return Buffer.byteLength(text, "utf8"); - } - private toRoomMessageContent( message: AdapterPostableMessage ): MatrixTextMessageContent { @@ -3283,19 +3256,15 @@ export class MatrixAdapter implements Adapter { return `${this.persistenceConfig.keyPrefix}:store`; } - private get sessionBasePrefix(): string { - return this.persistenceSessionPrefix; - } - private getSessionStorageKey(userID: string): string { - return `${this.sessionBasePrefix}:user:${encodeURIComponent(userID)}`; + return `${this.persistenceSessionPrefix}:user:${encodeURIComponent(userID)}`; } private getSessionUsernameTemporaryKey(): string | null { if (this.auth.type !== "password") { return null; } - return `${this.sessionBasePrefix}:username:${encodeURIComponent(this.auth.username)}`; + return `${this.persistenceSessionPrefix}:username:${encodeURIComponent(this.auth.username)}`; } private async loadPersistedSession(): Promise { @@ -3571,8 +3540,8 @@ export class MatrixAdapter implements Adapter { const setLevel = Reflect.get(matrixSDKLogger, "setLevel"); if (typeof setLevel === "function") { setLevel.call(matrixSDKLogger, numericLevel, false); + matrixSDKLogConfigured = true; } - matrixSDKLogConfigured = true; } } From 6130839faa96894092fde3c07b0e37226e57fd87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Sun, 12 Apr 2026 14:59:31 +0200 Subject: [PATCH 4/9] Update CHANGELOG.md --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d53aa6..d4f1433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 0.2.1 + +### New + +- Automatic oversized message splitting: text messages rejected with `M_TOO_LARGE` (413) are retried as plain-text chunks (~12 KB each) +- Thread reply metadata: messages posted to threads now include `m.in_reply_to`, with optional `matrixReplyToEventId` override + +### Fixes + +- Attachments sent alongside text no longer incorrectly carry the reply-to relationship +- Incoming formatted messages were parsed twice; removed redundant `` pre-strip pass +- `matrixSDKLogConfigured` flag no longer latches when `setLevel` is missing from the SDK logger + +### Changes + +- Bump `chat` SDK to 4.25.0 +- Move `@chat-adapter/state-memory` and `@chat-adapter/state-redis` to devDependencies + ## 0.2.0 ### New From bd6a906ead9b439bd2dd046d473f44224f68a4f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Sun, 12 Apr 2026 19:22:54 +0200 Subject: [PATCH 5/9] Refactor: split Matrix adapter into modules Extract core functionality out of src/index.ts into focused modules to reduce file size and improve organization: add src/config.ts (config parsing/validation and env helpers), src/history/cursor.ts (thread/cursor encoding & decoding), src/messages/inbound.ts and src/messages/outbound.ts (message parsing/formatting, upload/attachment helpers), and src/shared/utils.ts (general helpers). Update imports throughout index.ts to use the new modules and remove duplicated helper implementations. Intended to be a pure refactor with no behavioral changes: centralizes logic for easier testing and maintenance and simplifies createMatrixAdapter/config handling. --- src/config.ts | 174 ++++++ src/history/cursor.ts | 113 ++++ src/index.ts | 1275 +++----------------------------------- src/messages/inbound.ts | 319 ++++++++++ src/messages/outbound.ts | 566 +++++++++++++++++ src/shared/utils.ts | 81 +++ 6 files changed, 1356 insertions(+), 1172 deletions(-) create mode 100644 src/config.ts create mode 100644 src/history/cursor.ts create mode 100644 src/messages/inbound.ts create mode 100644 src/messages/outbound.ts create mode 100644 src/shared/utils.ts diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..3e27ec1 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,174 @@ +import type { MatrixAdapterConfig, MatrixAuthConfig, MatrixAccessTokenAuthConfig, MatrixPersistenceConfig } from "./types"; +import { normalizeOptionalString } from "./shared/utils"; + +export const DEFAULT_COMMAND_PREFIX = "/"; +export const DEFAULT_PERSISTENCE_KEY_PREFIX = "matrix"; +export const DEFAULT_MATRIX_STORE_PERSIST_INTERVAL_MS = 30_000; +export const FAST_SYNC_DEFAULTS: NonNullable = { + initialSyncLimit: 1, + lazyLoadMembers: true, + disablePresence: true, + pollTimeout: 10_000, +}; + +export type SDKLogLevel = NonNullable; + +export type ResolvedPersistenceConfig = { + keyPrefix: string; + session: Pick, "decrypt" | "encrypt" | "ttlMs">; + sync: Required, "persistIntervalMs">> & + Pick, "snapshotTtlMs">; +}; + +export function validateConfig(config: MatrixAdapterConfig): void { + if (!config.baseURL?.trim()) { + throw new Error("baseURL is required."); + } + if (config.persistence?.session?.ttlMs !== undefined && config.persistence.session.ttlMs <= 0) { + throw new Error("persistence.session.ttlMs must be a positive number."); + } + if ( + config.persistence?.sync?.persistIntervalMs !== undefined && + config.persistence.sync.persistIntervalMs <= 0 + ) { + throw new Error("persistence.sync.persistIntervalMs must be a positive number."); + } + if ( + config.persistence?.sync?.snapshotTtlMs !== undefined && + config.persistence.sync.snapshotTtlMs <= 0 + ) { + throw new Error("persistence.sync.snapshotTtlMs must be a positive number."); + } + if ( + (config.persistence?.session?.encrypt && !config.persistence?.session?.decrypt) || + (!config.persistence?.session?.encrypt && config.persistence?.session?.decrypt) + ) { + throw new Error( + "persistence.session.encrypt and persistence.session.decrypt must be provided together." + ); + } +} + +export function normalizePersistenceConfig( + config?: MatrixPersistenceConfig +): ResolvedPersistenceConfig { + return { + keyPrefix: + normalizeOptionalString(config?.keyPrefix) ?? DEFAULT_PERSISTENCE_KEY_PREFIX, + session: { + decrypt: config?.session?.decrypt, + encrypt: config?.session?.encrypt, + ttlMs: config?.session?.ttlMs, + }, + sync: { + persistIntervalMs: + config?.sync?.persistIntervalMs ?? + DEFAULT_MATRIX_STORE_PERSIST_INTERVAL_MS, + snapshotTtlMs: config?.sync?.snapshotTtlMs, + }, + }; +} + +export function resolveAuthFromEnv(): MatrixAuthConfig { + const username = process.env.MATRIX_USERNAME; + const password = process.env.MATRIX_PASSWORD; + + if (username && password) { + return { + type: "password", + username, + password, + userID: process.env.MATRIX_USER_ID, + }; + } + + const accessToken = process.env.MATRIX_ACCESS_TOKEN; + const userID = process.env.MATRIX_USER_ID; + + if (!accessToken) { + throw new Error( + "Set MATRIX_USERNAME+MATRIX_PASSWORD for password auth, or MATRIX_ACCESS_TOKEN for access token auth." + ); + } + + const auth: MatrixAccessTokenAuthConfig = { + type: "accessToken", + accessToken, + userID, + }; + + return auth; +} + +function envBool(value: string | undefined, fallback = false): boolean { + if (!value) { + return fallback; + } + + const normalized = value.trim().toLowerCase(); + return ( + normalized === "1" || + normalized === "true" || + normalized === "yes" || + normalized === "on" + ); +} + +function parseEnvList(value: string | undefined): string[] { + if (!value) { + return []; + } + + return value + .split(",") + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); +} + +function isSDKLogLevel(value: string): value is SDKLogLevel { + return value === "trace" || + value === "debug" || + value === "info" || + value === "warn" || + value === "error"; +} + +export function parseSDKLogLevel(value: string | undefined): SDKLogLevel | undefined { + if (!value) { + return undefined; + } + const normalized = value.trim().toLowerCase(); + return isSDKLogLevel(normalized) ? normalized : undefined; +} + +export function createMatrixAdapterConfigFromEnv(): MatrixAdapterConfig { + const baseURL = process.env.MATRIX_BASE_URL; + if (!baseURL) { + throw new Error("baseURL is required. Set MATRIX_BASE_URL."); + } + + const recoveryKey = process.env.MATRIX_RECOVERY_KEY; + const inviteAutoJoinInviterAllowlist = parseEnvList( + process.env.MATRIX_INVITE_AUTOJOIN_ALLOWLIST + ); + const inviteAutoJoinEnabled = envBool( + process.env.MATRIX_INVITE_AUTOJOIN, + inviteAutoJoinInviterAllowlist.length > 0 + ); + + return { + baseURL, + auth: resolveAuthFromEnv(), + userName: process.env.MATRIX_BOT_USERNAME ?? "bot", + deviceID: normalizeOptionalString(process.env.MATRIX_DEVICE_ID), + commandPrefix: process.env.MATRIX_COMMAND_PREFIX, + recoveryKey, + inviteAutoJoin: inviteAutoJoinEnabled + ? { + inviterAllowlist: inviteAutoJoinInviterAllowlist, + } + : undefined, + matrixSDKLogLevel: + parseSDKLogLevel(process.env.MATRIX_SDK_LOG_LEVEL) ?? "error", + }; +} diff --git a/src/history/cursor.ts b/src/history/cursor.ts new file mode 100644 index 0000000..01b0514 --- /dev/null +++ b/src/history/cursor.ts @@ -0,0 +1,113 @@ +import { Direction } from "matrix-js-sdk"; +import type { MatrixThreadID } from "../types"; +import { isRecord } from "../shared/utils"; + +export const MATRIX_PREFIX = "matrix"; +export const MATRIX_CURSOR_PREFIX = "mxv1:"; + +export type CursorKind = "room_messages" | "thread_relations" | "thread_list"; + +export type CursorDirection = "forward" | "backward"; + +export type CursorV1Payload = { + dir: CursorDirection; + kind: CursorKind; + roomID: string; + rootEventID?: string; + token: string; +}; + +export function encodeThreadId(platformData: MatrixThreadID): string { + const room = encodeURIComponent(platformData.roomID); + if (platformData.rootEventID) { + return `${MATRIX_PREFIX}:${room}:${encodeURIComponent(platformData.rootEventID)}`; + } + return `${MATRIX_PREFIX}:${room}`; +} + +export function decodeThreadId(threadId: string): MatrixThreadID { + const parts = threadId.split(":"); + if (parts.length < 2 || parts[0] !== MATRIX_PREFIX) { + throw new Error(`Invalid Matrix thread ID: ${threadId}`); + } + + const roomID = decodeURIComponent(parts[1]); + const rootEventID = parts[2] ? decodeURIComponent(parts[2]) : undefined; + + return { roomID, rootEventID }; +} + +export function channelIdFromThreadId(threadId: string): string { + const { roomID } = decodeThreadId(threadId); + return encodeThreadId({ roomID }); +} + +export function encodeCursorV1(payload: CursorV1Payload): string { + return `${MATRIX_CURSOR_PREFIX}${Buffer.from( + JSON.stringify(payload), + "utf8" + ).toString("base64url")}`; +} + +export function decodeCursorV1( + cursor: string, + expectedKind: CursorKind, + expectedRoomID: string, + expectedRootEventID?: string, + expectedDirection?: CursorDirection +): CursorV1Payload { + if (!cursor.startsWith(MATRIX_CURSOR_PREFIX)) { + throw new Error("Invalid cursor format. Expected mxv1 cursor."); + } + + let parsed: unknown; + try { + parsed = JSON.parse( + Buffer.from(cursor.slice(MATRIX_CURSOR_PREFIX.length), "base64url").toString("utf8") + ); + } catch (error) { + throw new Error(`Invalid cursor format. ${String(error)}`); + } + + if (!isRecord(parsed)) { + throw new Error("Invalid cursor format. Cursor payload must be an object."); + } + + if (parsed.kind !== expectedKind) { + throw new Error(`Invalid cursor kind. Expected ${expectedKind}.`); + } + if (parsed.roomID !== expectedRoomID) { + throw new Error("Invalid cursor context. Room mismatch."); + } + if (parsed.dir !== "forward" && parsed.dir !== "backward") { + throw new Error("Invalid cursor format. Invalid direction."); + } + if (expectedDirection && parsed.dir !== expectedDirection) { + throw new Error(`Invalid cursor direction. Expected ${expectedDirection}.`); + } + if (typeof parsed.token !== "string" || parsed.token.length === 0) { + throw new Error("Invalid cursor format. Missing token."); + } + + const rootEventID = + typeof parsed.rootEventID === "string" ? parsed.rootEventID : undefined; + if (expectedRootEventID) { + if (rootEventID !== expectedRootEventID) { + throw new Error("Invalid cursor context. Thread mismatch."); + } + } else if (rootEventID) { + throw new Error("Invalid cursor context. Unexpected thread scope."); + } + + return { + dir: parsed.dir, + kind: expectedKind, + roomID: expectedRoomID, + rootEventID, + token: parsed.token, + }; +} + +export function toSDKDirection(dir: CursorDirection): Direction { + return dir === "forward" ? Direction.Forward : Direction.Backward; +} diff --git a/src/index.ts b/src/index.ts index b82ad44..bb58f4d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,3 @@ -import { randomBytes } from "node:crypto"; import type { Adapter, AdapterPostableMessage, @@ -6,7 +5,6 @@ import type { ChannelInfo, ChatInstance, EmojiValue, - FileUpload, FetchOptions, FetchResult, FormattedContent, @@ -22,9 +20,7 @@ import type { import { ConsoleLogger, getEmoji, - isCardElement, Message, - markdownToPlainText, parseMarkdown, stringifyMarkdown, } from "chat"; @@ -49,45 +45,80 @@ import sdk, { import type { IStore } from "matrix-js-sdk/lib/store"; import type { RoomMessageEventContent, - RoomMessageTextEventContent, } from "matrix-js-sdk/lib/@types/events"; import type { MediaEventContent } from "matrix-js-sdk/lib/@types/media"; import { MatrixError } from "matrix-js-sdk/lib/http-api/errors"; import { decodeRecoveryKey } from "matrix-js-sdk/lib/crypto-api/recovery-key"; import { logger as matrixSDKLogger } from "matrix-js-sdk/lib/logger"; -import { marked } from "marked"; import { - HTMLElement, - NodeType, - parse as parseHTML, - type Node as HTMLNode, -} from "node-html-parser"; + DEFAULT_COMMAND_PREFIX, + FAST_SYNC_DEFAULTS, + type ResolvedPersistenceConfig, + type SDKLogLevel, + createMatrixAdapterConfigFromEnv, + normalizePersistenceConfig, + validateConfig, +} from "./config"; +import { + channelIdFromThreadId, + decodeCursorV1, + decodeThreadId, + encodeCursorV1, + encodeThreadId, + toSDKDirection, + type CursorDirection, + type CursorKind, + type CursorV1Payload, +} from "./history/cursor"; +import { + isMentioned, + parseMatrixContent, + type MatrixMessageContent, + type ParsedMatrixContent, +} from "./messages/inbound"; +import { + applyThreadReplyMetadata, + binarySize, + collectLinkOnlyAttachmentLines, + defaultAttachmentName, + extractAttachmentsFromMessage, + extractFilesFromMessage, + extractReplyEventID, + isTooLargeMatrixError, + mergeTextAndLinks, + messageTypeForAttachment, + messageTypeForMimeType, + normalizeFileUpload, + normalizeUploadData, + readAttachmentData, + splitOversizedTextContent, + toRoomMessageContent, + type MatrixMediaMsgType, + type MatrixOutboundMessageContent, + type MatrixRoomMessageContent, + type MatrixTextMessageContent, + type OutboundUpload, +} from "./messages/outbound"; +import { + evictOldestEntries, + generateDeviceID, + hasIndexedDB, + isRecord, + matrixLocalpart, + normalizeOptionalString, + normalizeStringList, + readStringValue, +} from "./shared/utils"; import { ChatStateMatrixStore } from "./store/chat-state-matrix-store"; import type { MatrixAuthBootstrapClient, - MatrixAccessTokenAuthConfig, MatrixAdapterConfig, MatrixAuthConfig, MatrixCreateStoreOptions, - MatrixPersistenceConfig, - MatrixPersistenceSyncConfig, MatrixThreadID, } from "./types"; -const MATRIX_PREFIX = "matrix"; -const MATRIX_CURSOR_PREFIX = "mxv1:"; -const DEFAULT_COMMAND_PREFIX = "/"; -const DEFAULT_PERSISTENCE_KEY_PREFIX = "matrix"; const TYPING_TIMEOUT_MS = 30_000; -const DEFAULT_MATRIX_STORE_PERSIST_INTERVAL_MS = 30_000; -const DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES = 12_000; -const FAST_SYNC_DEFAULTS: NonNullable = { - initialSyncLimit: 1, - lazyLoadMembers: true, - disablePresence: true, - pollTimeout: 10_000, -}; -type SDKLogLevel = NonNullable; const MATRIX_SDK_LOG_LEVELS: Record = { trace: 0, debug: 1, @@ -97,31 +128,6 @@ const MATRIX_SDK_LOG_LEVELS: Record = { }; let matrixSDKLogConfigured = false; -type MatrixMessageContent = { - body?: string; - format?: string; - formatted_body?: string; - msgtype?: string; - "m.mentions"?: { - room?: boolean; - user_ids?: string[]; - }; - [key: string]: unknown; -}; - -type MatrixTextMessageContent = RoomMessageTextEventContent & { - "com.beeper.dont_render_edited"?: boolean; -}; - -type MatrixRoomMessageContent = RoomMessageEventContent & { - "com.beeper.dont_render_edited"?: boolean; - "m.new_content"?: RoomMessageEventContent & { - "com.beeper.dont_render_edited"?: boolean; - }; -}; - -type MatrixOutboundMessageContent = MatrixRoomMessageContent | MediaEventContent; - type StoredReaction = { emoji: EmojiValue; messageID: string; @@ -137,13 +143,6 @@ type ResolvedAuth = { userID: string; }; -type ResolvedPersistenceConfig = { - keyPrefix: string; - session: Pick, "decrypt" | "encrypt" | "ttlMs">; - sync: Required, "persistIntervalMs">> & - Pick, "snapshotTtlMs">; -}; - type StoredSession = { accessToken?: string; authType: MatrixAuthConfig["type"]; @@ -158,55 +157,8 @@ type StoredSession = { username?: string; }; -type CursorKind = "room_messages" | "thread_relations" | "thread_list"; - -type CursorDirection = "forward" | "backward"; - -type CursorV1Payload = { - dir: CursorDirection; - kind: CursorKind; - roomID: string; - rootEventID?: string; - token: string; -}; - type DirectAccountData = Record; -type OutboundUpload = { - data: Blob; - fileName: string; - info?: { - h?: number; - mimetype?: string; - size?: number; - w?: number; - }; - msgtype: MatrixMediaMsgType; - type?: string; -}; - -type MatrixMediaMsgType = - | MsgType.Audio - | MsgType.File - | MsgType.Image - | MsgType.Video; - -type ParsedMatrixContent = { - markdown: string; - mentionsRoom: boolean; - mentionedUserIDs: Set; - text: string; -}; - -type RenderedMatrixMessage = { - body: string; - formattedBody?: string; - mentions?: { - room?: boolean; - user_ids?: string[]; - }; -}; - type MatrixRoomMetadata = { avatarURL?: string; canonicalAlias?: string; @@ -257,7 +209,7 @@ export class MatrixAdapter implements Adapter { private shuttingDown = false; constructor(config: MatrixAdapterConfig) { - this.validateConfig(config); + validateConfig(config); this.baseURL = config.baseURL; this.auth = config.auth; this.userID = config.auth.userID ?? ""; @@ -384,28 +336,15 @@ export class MatrixAdapter implements Adapter { } encodeThreadId(platformData: MatrixThreadID): string { - const room = encodeURIComponent(platformData.roomID); - if (platformData.rootEventID) { - return `${MATRIX_PREFIX}:${room}:${encodeURIComponent(platformData.rootEventID)}`; - } - return `${MATRIX_PREFIX}:${room}`; + return encodeThreadId(platformData); } decodeThreadId(threadId: string): MatrixThreadID { - const parts = threadId.split(":"); - if (parts.length < 2 || parts[0] !== MATRIX_PREFIX) { - throw new Error(`Invalid Matrix thread ID: ${threadId}`); - } - - const roomID = decodeURIComponent(parts[1]); - const rootEventID = parts[2] ? decodeURIComponent(parts[2]) : undefined; - - return { roomID, rootEventID }; + return decodeThreadId(threadId); } channelIdFromThreadId(threadId: string): string { - const { roomID } = this.decodeThreadId(threadId); - return this.encodeThreadId({ roomID }); + return channelIdFromThreadId(threadId); } renderFormatted(content: FormattedContent): string { @@ -421,7 +360,7 @@ export class MatrixAdapter implements Adapter { message: AdapterPostableMessage ): Promise> { const { roomID, rootEventID } = this.decodeThreadId(threadId); - const replyEventID = this.extractReplyEventID(message); + const replyEventID = extractReplyEventID(message); const contents = await this.toRoomMessageContents(message); const [firstContent, ...extraContents] = contents; if (!firstContent) { @@ -462,7 +401,7 @@ export class MatrixAdapter implements Adapter { message: AdapterPostableMessage ): Promise> { const { roomID, rootEventID } = this.decodeThreadId(threadId); - const baseContent = this.toRoomMessageContent(message); + const baseContent = toRoomMessageContent(message); const newContent: MatrixTextMessageContent = { ...baseContent, "com.beeper.dont_render_edited": true, @@ -801,7 +740,7 @@ export class MatrixAdapter implements Adapter { const content = raw.getContent(); const edited = this.extractEditedContent(raw); const effectiveContent = edited?.content ?? content; - const parsed = this.parseMatrixContent(effectiveContent); + const parsed = parseMatrixContent(effectiveContent); const sender = raw.getSender() ?? "unknown"; return new Message({ @@ -817,15 +756,17 @@ export class MatrixAdapter implements Adapter { }, attachments: this.extractAttachments(effectiveContent), raw, - isMention: this.isMentioned(effectiveContent, parsed), + isMention: isMentioned({ + content: effectiveContent, + parsed, + userID: this.userID, + userName: this.userName, + }), }); } private encodeCursorV1(payload: CursorV1Payload): string { - return `${MATRIX_CURSOR_PREFIX}${Buffer.from( - JSON.stringify(payload), - "utf8" - ).toString("base64url")}`; + return encodeCursorV1(payload); } private decodeCursorV1( @@ -835,60 +776,17 @@ export class MatrixAdapter implements Adapter { expectedRootEventID?: string, expectedDirection?: CursorDirection ): CursorV1Payload { - if (!cursor.startsWith(MATRIX_CURSOR_PREFIX)) { - throw new Error("Invalid cursor format. Expected mxv1 cursor."); - } - - let parsed: unknown; - try { - parsed = JSON.parse( - Buffer.from(cursor.slice(MATRIX_CURSOR_PREFIX.length), "base64url").toString("utf8") - ); - } catch (error) { - throw new Error(`Invalid cursor format. ${String(error)}`); - } - - if (!isRecord(parsed)) { - throw new Error("Invalid cursor format. Cursor payload must be an object."); - } - - if (parsed.kind !== expectedKind) { - throw new Error(`Invalid cursor kind. Expected ${expectedKind}.`); - } - if (parsed.roomID !== expectedRoomID) { - throw new Error("Invalid cursor context. Room mismatch."); - } - if (parsed.dir !== "forward" && parsed.dir !== "backward") { - throw new Error("Invalid cursor format. Invalid direction."); - } - if (expectedDirection && parsed.dir !== expectedDirection) { - throw new Error(`Invalid cursor direction. Expected ${expectedDirection}.`); - } - if (typeof parsed.token !== "string" || parsed.token.length === 0) { - throw new Error("Invalid cursor format. Missing token."); - } - - const rootEventID = - typeof parsed.rootEventID === "string" ? parsed.rootEventID : undefined; - if (expectedRootEventID) { - if (rootEventID !== expectedRootEventID) { - throw new Error("Invalid cursor context. Thread mismatch."); - } - } else if (rootEventID) { - throw new Error("Invalid cursor context. Unexpected thread scope."); - } - - return { - dir: parsed.dir, - kind: expectedKind, - roomID: expectedRoomID, - rootEventID, - token: parsed.token, - }; + return decodeCursorV1( + cursor, + expectedKind, + expectedRoomID, + expectedRootEventID, + expectedDirection + ); } private toSDKDirection(dir: CursorDirection): Direction { - return dir === "forward" ? Direction.Forward : Direction.Backward; + return toSDKDirection(dir); } private async fetchRoomMessagesPage(args: { @@ -1513,15 +1411,15 @@ export class MatrixAdapter implements Adapter { replyEventID: string | undefined, content: MatrixOutboundMessageContent ) { - const threadContent = this.applyThreadReplyMetadata(content, rootEventID, replyEventID); + const threadContent = applyThreadReplyMetadata(content, rootEventID, replyEventID); try { const response = await this.performSendRoomMessage(roomID, rootEventID, threadContent); void this.maybePersistSecretsBundle(); return response; } catch (error) { - if (this.isTooLargeMatrixError(error)) { - const splitContents = this.splitOversizedTextContent(threadContent); + if (isTooLargeMatrixError(error)) { + const splitContents = splitOversizedTextContent(threadContent); if (splitContents.length > 1) { this.logger.warn( "Matrix message exceeded size limit; retrying as split plain-text chunks", @@ -1536,7 +1434,11 @@ export class MatrixAdapter implements Adapter { let firstResponse: Awaited> | undefined; for (const splitContent of splitContents) { - const chunkWithMeta = this.applyThreadReplyMetadata(splitContent, rootEventID, replyEventID); + const chunkWithMeta = applyThreadReplyMetadata( + splitContent, + rootEventID, + replyEventID + ); const response = await this.performSendRoomMessage(roomID, rootEventID, chunkWithMeta); void this.maybePersistSecretsBundle(); firstResponse ??= response; @@ -1562,11 +1464,11 @@ export class MatrixAdapter implements Adapter { private async toRoomMessageContents( message: AdapterPostableMessage ): Promise { - const textContent = this.toRoomMessageContent(message); - const attachments = this.extractAttachmentsFromMessage(message); + const textContent = toRoomMessageContent(message); + const attachments = extractAttachmentsFromMessage(message); const uploads = await this.collectUploads(message, attachments); - const linkLines = this.collectLinkOnlyAttachmentLines(attachments); - const textBody = this.mergeTextAndLinks(textContent, linkLines); + const linkLines = collectLinkOnlyAttachmentLines(attachments); + const textBody = mergeTextAndLinks(textContent, linkLines); const contents: MatrixOutboundMessageContent[] = []; if ((normalizeOptionalString(textBody.body) ?? "").length > 0) { @@ -2193,230 +2095,6 @@ export class MatrixAdapter implements Adapter { return room.getMember(userId) ?? undefined; } - private parseMatrixContent(content: MatrixMessageContent): ParsedMatrixContent { - const mentionedUserIDs = this.extractMentionedUserIDs(content); - const mentionsRoom = this.extractRoomMention(content); - const formattedBody = normalizeOptionalString(content.formatted_body); - if (formattedBody) { - const htmlMarkdown = this.parseMatrixHTML(formattedBody); - for (const mentionedUserID of htmlMarkdown.mentionedUserIDs) { - mentionedUserIDs.add(mentionedUserID); - } - - if (htmlMarkdown.markdown.length > 0) { - return { - text: markdownToPlainText(htmlMarkdown.markdown), - markdown: htmlMarkdown.markdown, - mentionedUserIDs, - mentionsRoom, - }; - } - } - - const body = this.stripReplyFallbackFromBody( - normalizeOptionalString(content.body) ?? "" - ); - return { - text: body, - markdown: this.markdownForPlainText(body, content.msgtype), - mentionedUserIDs, - mentionsRoom, - }; - } - - private parseMatrixHTML( - html: string - ): { markdown: string; mentionedUserIDs: Set } { - const root = parseHTML(html); - const mentionedUserIDs = new Set(); - const markdown = this.normalizeMarkdownSpacing( - this.renderHTMLNodesToMarkdown(root.childNodes, mentionedUserIDs) - ); - return { - markdown, - mentionedUserIDs, - }; - } - - private renderHTMLNodesToMarkdown( - nodes: HTMLNode[], - mentionedUserIDs: Set - ): string { - return nodes - .map((node) => this.renderHTMLNodeToMarkdown(node, mentionedUserIDs)) - .join(""); - } - - private renderHTMLNodeToMarkdown( - node: HTMLNode, - mentionedUserIDs: Set - ): string { - if (node.nodeType === NodeType.TEXT_NODE) { - return node.text; - } - - if (!(node instanceof HTMLElement)) { - return ""; - } - - const tagName = node.tagName.toLowerCase(); - const children = this.renderHTMLNodesToMarkdown(node.childNodes, mentionedUserIDs); - - switch (tagName) { - case "mx-reply": - return ""; - case "html": - case "body": - case "span": - return children; - case "br": - return "\n"; - case "p": - case "div": - return children.trim() ? `${children.trim()}\n\n` : ""; - case "strong": - case "b": - return children ? `**${children}**` : ""; - case "em": - case "i": - return children ? `*${children}*` : ""; - case "del": - case "s": - return children ? `~~${children}~~` : ""; - case "code": - return node.parentNode instanceof HTMLElement && - node.parentNode.tagName.toLowerCase() === "pre" - ? children - : `\`${children}\``; - case "pre": { - const codeContent = children.replace(/\n+$/u, ""); - return codeContent ? `\n\`\`\`\n${codeContent}\n\`\`\`\n\n` : ""; - } - case "blockquote": { - const quoted = children.trim(); - if (!quoted) { - return ""; - } - return `${quoted - .split("\n") - .map((line) => `> ${line}`) - .join("\n")}\n\n`; - } - case "ul": - return `${node.childNodes - .map((child) => this.renderListItemToMarkdown(child, mentionedUserIDs, null)) - .filter(Boolean) - .join("\n")}\n\n`; - case "ol": - return `${node.childNodes - .map((child, index) => - this.renderListItemToMarkdown(child, mentionedUserIDs, index + 1) - ) - .filter(Boolean) - .join("\n")}\n\n`; - case "a": - return this.renderHTMLLinkToMarkdown(node, children, mentionedUserIDs); - case "img": - return normalizeOptionalString(node.getAttribute("alt")) ?? "image"; - default: - return children; - } - } - - private renderListItemToMarkdown( - node: HTMLNode, - mentionedUserIDs: Set, - ordinal: number | null - ): string { - if (!(node instanceof HTMLElement) || node.tagName.toLowerCase() !== "li") { - return ""; - } - const content = this.normalizeMarkdownSpacing( - this.renderHTMLNodesToMarkdown(node.childNodes, mentionedUserIDs) - ); - if (!content) { - return ""; - } - return `${ordinal === null ? "-" : `${ordinal}.`} ${content}`; - } - - private renderHTMLLinkToMarkdown( - node: HTMLElement, - children: string, - mentionedUserIDs: Set - ): string { - const href = normalizeOptionalString(node.getAttribute("href")); - const text = children || node.text; - if (!href) { - return text; - } - - const mentionedUserID = this.parseMatrixToUserID(href); - if (mentionedUserID) { - mentionedUserIDs.add(mentionedUserID); - return text || this.matrixMentionDisplayText(mentionedUserID); - } - - return `[${text || href}](${href})`; - } - - private parseMatrixToUserID(href: string): string | null { - let url: URL; - try { - url = new URL(href); - } catch { - return null; - } - - if (url.hostname !== "matrix.to") { - return null; - } - - const rawPath = url.hash.startsWith("#/") ? url.hash.slice(2) : url.hash; - const firstSegment = rawPath.split("/")[0]; - if (!firstSegment) { - return null; - } - - const identifier = decodeURIComponent(firstSegment); - return identifier.startsWith("@") ? identifier : null; - } - - private extractMentionedUserIDs(content: MatrixMessageContent): Set { - const mentions = new Set(); - const matrixMentions = content["m.mentions"]; - if (!isRecord(matrixMentions) || !Array.isArray(matrixMentions.user_ids)) { - return mentions; - } - - for (const userID of matrixMentions.user_ids) { - if (typeof userID === "string" && userID.length > 0) { - mentions.add(userID); - } - } - - return mentions; - } - - private extractRoomMention(content: MatrixMessageContent): boolean { - const matrixMentions = content["m.mentions"]; - return isRecord(matrixMentions) && matrixMentions.room === true; - } - - private stripReplyFallbackFromBody(body: string): string { - const lines = body.split("\n"); - let index = 0; - while (index < lines.length && lines[index]?.startsWith(">")) { - index += 1; - } - - if (index === 0 || index >= lines.length || lines[index] !== "") { - return body; - } - - return lines.slice(index + 1).join("\n"); - } - private extractAttachments(content: MatrixMessageContent) { const url = typeof content.url === "string" ? content.url : undefined; if (!url) { @@ -2481,7 +2159,7 @@ export class MatrixAdapter implements Adapter { case MsgType.File: return "file"; default: { - const mediaType = this.messageTypeForMimeType(mimeType); + const mediaType = messageTypeForMimeType(mimeType); switch (mediaType) { case MsgType.Image: return "image"; @@ -2566,34 +2244,6 @@ export class MatrixAdapter implements Adapter { return relation?.rel_type === RelationType.Replace; } - private isMentioned(content: MatrixMessageContent, parsed: ParsedMatrixContent): boolean { - if (parsed.mentionsRoom) { - return true; - } - if (this.userID && parsed.mentionedUserIDs.has(this.userID)) { - return true; - } - - const formatted = - typeof content.formatted_body === "string" ? content.formatted_body : ""; - - const hasUserID = this.userID - ? parsed.text.includes(this.userID) || formatted.includes(this.userID) - : false; - const hasMatrixTo = this.userID - ? formatted.includes(`matrix.to/#/${encodeURIComponent(this.userID)}`) - : false; - - const usernameMention = this.userName.startsWith("@") - ? this.userName - : `@${this.userName}`; - - const hasUserName = - parsed.text.includes(usernameMention) || formatted.includes(usernameMention); - - return hasUserID || hasMatrixTo || hasUserName; - } - private parseSlashCommand( text: string ): { command: string; text: string } | null { @@ -2615,45 +2265,6 @@ export class MatrixAdapter implements Adapter { }; } - private extractReplyEventID(message: AdapterPostableMessage): string | undefined { - if (typeof message !== "object" || message === null || isCardElement(message)) { - return undefined; - } - - const replyEventID = (message as { matrixReplyToEventId?: unknown }).matrixReplyToEventId; - return typeof replyEventID === "string" && replyEventID.length > 0 - ? replyEventID - : undefined; - } - - private applyThreadReplyMetadata( - content: MatrixOutboundMessageContent, - rootEventID: string | undefined, - replyEventID: string | undefined - ): MatrixOutboundMessageContent { - const threadableContent = content as MatrixOutboundMessageContent & { - "m.relates_to"?: { - rel_type?: string; - "m.in_reply_to"?: { event_id: string }; - [key: string]: unknown; - }; - }; - - if (!rootEventID || threadableContent["m.relates_to"]?.rel_type) { - return threadableContent; - } - - return { - ...threadableContent, - "m.relates_to": { - ...threadableContent["m.relates_to"], - "m.in_reply_to": { - event_id: replyEventID ?? rootEventID, - }, - }, - } as MatrixOutboundMessageContent; - } - private async performSendRoomMessage( roomID: string, rootEventID: string | undefined, @@ -2672,360 +2283,42 @@ export class MatrixAdapter implements Adapter { return client.sendEvent(roomID, EventType.RoomMessage, content); } - private isTooLargeMatrixError(error: unknown): error is MatrixError { - return ( - error instanceof MatrixError && - (error.errcode === "M_TOO_LARGE" || error.httpStatus === 413) - ); - } - - private splitOversizedTextContent( - content: MatrixOutboundMessageContent - ): MatrixTextMessageContent[] { - if (!this.isSplittableTextContent(content)) { - return []; - } - - const body = content.body; - if (Buffer.byteLength(body, "utf8") <= DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES) { - return []; - } - - const parts = this.splitTextByUtf8Bytes(body, DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES); - if (parts.length <= 1) { - return []; - } - - return parts.map((part) => ({ - body: part, - msgtype: content.msgtype, - })); - } - - private isSplittableTextContent( - content: MatrixOutboundMessageContent - ): content is MatrixTextMessageContent { - if ("url" in content || "info" in content) { - return false; - } - - if (typeof content.body !== "string" || content.body.length <= 1) { - return false; - } - - if ("m.new_content" in content) { - return false; - } - - return ( - content.msgtype === MsgType.Text || - content.msgtype === MsgType.Notice || - content.msgtype === MsgType.Emote - ); - } - - private splitTextByUtf8Bytes(text: string, maxBytes: number): string[] { - const normalizedMaxBytes = Math.max(1, Math.floor(maxBytes)); - const chunks: string[] = []; - let remaining = text; - - while (remaining.length > 0) { - if (Buffer.byteLength(remaining, "utf8") <= normalizedMaxBytes) { - chunks.push(remaining); - remaining = ""; - break; - } - - const boundary = this.findSplitBoundary(remaining, normalizedMaxBytes); - const head = remaining.slice(0, boundary).trimEnd(); - const tail = remaining.slice(boundary).trimStart(); - - if (!head || head === remaining) { - break; - } - - chunks.push(head); - remaining = tail; - } - - if (remaining.length > 0 && chunks.at(-1) !== remaining) { - chunks.push(remaining); - } - - return chunks.filter((chunk) => chunk.length > 0); - } - - private findSplitBoundary(text: string, maxBytes: number): number { - let low = 1; - let high = text.length; - let best = 1; - - while (low <= high) { - const mid = Math.floor((low + high) / 2); - const candidate = text.slice(0, mid); - if (Buffer.byteLength(candidate, "utf8") <= maxBytes) { - best = mid; - low = mid + 1; - } else { - high = mid - 1; - } - } - - for (let i = best - 1; i > 0; i--) { - const ch = text[i]; - if (ch === "\n" || ch === " " || ch === "\t" || ch === "\r") { - return i + 1; - } - } - - return best; - } - - private toRoomMessageContent( - message: AdapterPostableMessage - ): MatrixTextMessageContent { - const rendered = this.renderTextMessage(message); - const content: MatrixTextMessageContent = { - body: rendered.body, - msgtype: MsgType.Text, - }; - if (rendered.formattedBody) { - content.format = "org.matrix.custom.html"; - content.formatted_body = rendered.formattedBody; - } - if (rendered.mentions) { - content["m.mentions"] = rendered.mentions; - } - - return content; - } - - private renderTextMessage(message: AdapterPostableMessage): RenderedMatrixMessage { - if (typeof message === "string") { - return this.renderPlainTextMessage(message); - } - - if (isCardElement(message)) { - return this.renderPlainTextMessage("[Card message]"); - } - - if (typeof message === "object" && message !== null) { - if ("raw" in message && typeof message.raw === "string") { - return this.renderPlainTextMessage(message.raw); - } - if ("markdown" in message && typeof message.markdown === "string") { - return this.renderMarkdownMessage(message.markdown); - } - if ("ast" in message) { - return this.renderMarkdownMessage(stringifyMarkdown(message.ast)); - } - if ("card" in message) { - return this.renderPlainTextMessage(message.fallbackText ?? "[Card message]"); - } - } - - return { body: "" }; - } - - private renderPlainTextMessage(text: string): RenderedMatrixMessage { - const rendered = this.replaceMentionPlaceholdersInPlainText(text); - if (rendered.mentionedUserIDs.size === 0) { - return { - body: rendered.body, - }; - } - - return { - body: rendered.body, - formattedBody: this.renderMarkdownToMatrixHTML(rendered.markdown), - mentions: this.buildMentionsContent(rendered.mentionedUserIDs), - }; - } - - private renderMarkdownMessage(markdown: string): RenderedMatrixMessage { - const rendered = this.replaceMentionPlaceholdersInMarkdown(markdown); - return { - body: markdownToPlainText(rendered.markdown), - formattedBody: this.renderMarkdownToMatrixHTML(rendered.markdown), - mentions: this.buildMentionsContent(rendered.mentionedUserIDs), - }; - } - - private replaceMentionPlaceholdersInPlainText(text: string): { - body: string; - markdown: string; - mentionedUserIDs: Set; - } { - const mentionedUserIDs = new Set(); - const pattern = /<@(@[^>\s]+:[^>\s]+)>/gu; - let body = ""; - let markdown = ""; - let lastIndex = 0; - - for (const match of text.matchAll(pattern)) { - const [token, userID] = match; - const index = match.index ?? 0; - const plainSegment = text.slice(lastIndex, index); - body += plainSegment; - markdown += escapeMarkdownText(plainSegment); - - const mentionText = this.matrixMentionDisplayText(userID); - body += mentionText; - markdown += `[${escapeMarkdownLinkText(mentionText)}](${this.matrixToUserLink(userID)})`; - mentionedUserIDs.add(userID); - lastIndex = index + token.length; - } - - const trailing = text.slice(lastIndex); - body += trailing; - markdown += escapeMarkdownText(trailing); - - return { body, markdown, mentionedUserIDs }; - } - - private replaceMentionPlaceholdersInMarkdown(markdown: string): { - markdown: string; - mentionedUserIDs: Set; - } { - const mentionedUserIDs = new Set(); - const transformed = markdown.replace( - /<@(@[^>\s]+:[^>\s]+)>/gu, - (_match, userID: string) => { - mentionedUserIDs.add(userID); - return `[${escapeMarkdownLinkText(this.matrixMentionDisplayText(userID))}](${this.matrixToUserLink( - userID - )})`; - } - ); - - return { - markdown: transformed, - mentionedUserIDs, - }; - } - - private renderMarkdownToMatrixHTML(markdown: string): string { - return marked.parse(markdown, { - async: false, - breaks: true, - gfm: true, - }); - } - - private buildMentionsContent( - mentionedUserIDs: Set - ): { room?: boolean; user_ids?: string[] } | undefined { - if (mentionedUserIDs.size === 0) { - return undefined; - } - - return { - user_ids: [...mentionedUserIDs], - }; - } - - private matrixToUserLink(userID: string): string { - return `https://matrix.to/#/${encodeURIComponent(userID)}`; - } - - private matrixMentionDisplayText(userID: string): string { - return `@${matrixLocalpart(userID)}`; - } - - private markdownForPlainText(text: string, msgtype?: string): string { - const escaped = escapeMarkdownText(text); - if (msgtype === "m.emote" && escaped.length > 0) { - return `*${escaped}*`; - } - return escaped; - } - - private normalizeMarkdownSpacing(markdown: string): string { - return markdown.replace(/\n{3,}/gu, "\n\n").trim(); - } - - private mergeTextAndLinks( - content: MatrixTextMessageContent, - linkLines: string[] - ): MatrixTextMessageContent { - if (linkLines.length === 0) { - return content; - } - - const suffix = linkLines.join("\n"); - const body = content.body ?? ""; - const mergedBody = body ? `${body}\n\n${suffix}` : suffix; - if (!content.formatted_body) { - return { - ...content, - body: mergedBody, - }; - } - - const formattedSuffix = linkLines - .map((line) => `

${escapeHTML(line)}

`) - .join(""); - - return { - ...content, - body: mergedBody, - formatted_body: `${content.formatted_body}${formattedSuffix}`, - }; - } - - private collectLinkOnlyAttachmentLines(attachments: Attachment[]): string[] { - const lines: string[] = []; - for (const attachment of attachments) { - const hasLocalData = - Boolean(attachment.data) || typeof attachment.fetchData === "function"; - if (hasLocalData) { - continue; - } - if (!attachment.url) { - continue; - } - const label = attachment.name ?? attachment.type; - lines.push(`${label}: ${attachment.url}`); - } - return lines; - } - private async collectUploads( message: AdapterPostableMessage, attachments: Attachment[] ): Promise { const uploads: OutboundUpload[] = []; - const files = this.extractFilesFromMessage(message); + const files = extractFilesFromMessage(message, this.logger); for (const file of files) { uploads.push({ - data: this.normalizeUploadData(file.data), + data: normalizeUploadData(file.data), fileName: file.filename, info: { mimetype: normalizeOptionalString(file.mimeType), - size: this.binarySize(file.data), + size: binarySize(file.data), }, - msgtype: this.messageTypeForMimeType(normalizeOptionalString(file.mimeType)), + msgtype: messageTypeForMimeType(normalizeOptionalString(file.mimeType)), }); } for (const attachment of attachments) { - const data = await this.readAttachmentData(attachment); + const data = await readAttachmentData(attachment); if (!data) { continue; } const fileName = normalizeOptionalString(attachment.name) ?? - this.defaultAttachmentName(attachment); + defaultAttachmentName(attachment); uploads.push({ - data: this.normalizeUploadData(data), + data: normalizeUploadData(data), fileName, info: { h: attachment.height, mimetype: normalizeOptionalString(attachment.mimeType), - size: attachment.size ?? this.binarySize(data), + size: attachment.size ?? binarySize(data), w: attachment.width, }, - msgtype: this.messageTypeForAttachment(attachment), + msgtype: messageTypeForAttachment(attachment), type: attachment.type, }); } @@ -3033,141 +2326,6 @@ export class MatrixAdapter implements Adapter { return uploads; } - private extractFilesFromMessage(message: AdapterPostableMessage): FileUpload[] { - if (typeof message !== "object" || message === null) { - return []; - } - if (!("files" in message) || !Array.isArray(message.files)) { - return []; - } - return message.files.flatMap((file): FileUpload[] => { - const normalized = this.normalizeFileUpload(file); - return normalized ? [normalized] : []; - }); - } - - private normalizeFileUpload(file: unknown): FileUpload | null { - if (!isRecord(file)) { - return null; - } - - const filename = normalizeOptionalString( - typeof file.filename === "string" ? file.filename : undefined - ); - if (!filename) { - return null; - } - - const data = this.normalizeFileUploadData(file.data); - if (!data) { - this.logger.warn("Skipping invalid Matrix file upload", { filename }); - return null; - } - - return { - filename, - data, - mimeType: - typeof file.mimeType === "string" ? normalizeOptionalString(file.mimeType) : undefined, - }; - } - - private normalizeFileUploadData(data: unknown): Buffer | Blob | ArrayBuffer | null { - if (Buffer.isBuffer(data) || data instanceof Blob || data instanceof ArrayBuffer) { - return data; - } - - if (data instanceof Uint8Array) { - return Buffer.from(data); - } - - return null; - } - - private extractAttachmentsFromMessage(message: AdapterPostableMessage): Attachment[] { - if (typeof message !== "object" || message === null) { - return []; - } - if (!("attachments" in message) || !Array.isArray(message.attachments)) { - return []; - } - return message.attachments.filter((a): a is Attachment => isRecord(a)); - } - - private async readAttachmentData( - attachment: Attachment - ): Promise { - if (typeof attachment.fetchData === "function") { - return attachment.fetchData(); - } - return attachment.data ?? null; - } - - private normalizeUploadData(data: Buffer | Blob | ArrayBuffer): Blob { - if (data instanceof Blob) { - return data; - } - if (this.isNodeBuffer(data)) { - return new Blob([new Uint8Array(data)]); - } - return new Blob([data]); - } - - private binarySize(data: Buffer | Blob | ArrayBuffer): number { - if (data instanceof ArrayBuffer) { - return data.byteLength; - } - if (this.isNodeBuffer(data)) { - return data.length; - } - return data.size; - } - - private isNodeBuffer(value: unknown): value is Buffer { - return typeof Buffer !== "undefined" && Buffer.isBuffer(value); - } - - private messageTypeForAttachment(attachment: Attachment): MatrixMediaMsgType { - switch (attachment.type) { - case "image": - return MsgType.Image; - case "video": - return MsgType.Video; - case "audio": - return MsgType.Audio; - default: - return this.messageTypeForMimeType(normalizeOptionalString(attachment.mimeType)); - } - } - - private messageTypeForMimeType(mimeType?: string): MatrixMediaMsgType { - if (!mimeType) { - return MsgType.File; - } - if (mimeType.startsWith("image/")) { - return MsgType.Image; - } - if (mimeType.startsWith("video/")) { - return MsgType.Video; - } - if (mimeType.startsWith("audio/")) { - return MsgType.Audio; - } - return MsgType.File; - } - - private defaultAttachmentName(attachment: Attachment): string { - switch (attachment.type) { - case "image": - return "image"; - case "video": - return "video"; - case "audio": - return "audio"; - default: - return "file"; - } - } private mustGetEventByID(roomID: string, eventID: string): MatrixEvent { const room = this.requireRoom(roomID); @@ -3404,35 +2562,6 @@ export class MatrixAdapter implements Adapter { } } - private validateConfig(config: MatrixAdapterConfig): void { - if (!config.baseURL?.trim()) { - throw new Error("baseURL is required."); - } - if (config.persistence?.session?.ttlMs !== undefined && config.persistence.session.ttlMs <= 0) { - throw new Error("persistence.session.ttlMs must be a positive number."); - } - if ( - config.persistence?.sync?.persistIntervalMs !== undefined && - config.persistence.sync.persistIntervalMs <= 0 - ) { - throw new Error("persistence.sync.persistIntervalMs must be a positive number."); - } - if ( - config.persistence?.sync?.snapshotTtlMs !== undefined && - config.persistence.sync.snapshotTtlMs <= 0 - ) { - throw new Error("persistence.sync.snapshotTtlMs must be a positive number."); - } - if ( - (config.persistence?.session?.encrypt && !config.persistence?.session?.decrypt) || - (!config.persistence?.session?.encrypt && config.persistence?.session?.decrypt) - ) { - throw new Error( - "persistence.session.encrypt and persistence.session.decrypt must be provided together." - ); - } - } - private async resolveDeviceID(...candidates: Array): Promise { const configuredDeviceID = normalizeOptionalString(this.deviceID); if (configuredDeviceID) { @@ -3556,37 +2685,7 @@ export function createMatrixAdapter(config?: MatrixAdapterConfig): MatrixAdapter }); } - const baseURL = process.env.MATRIX_BASE_URL; - if (!baseURL) { - throw new Error("baseURL is required. Set MATRIX_BASE_URL."); - } - - const recoveryKey = process.env.MATRIX_RECOVERY_KEY; - const inviteAutoJoinInviterAllowlist = parseEnvList( - process.env.MATRIX_INVITE_AUTOJOIN_ALLOWLIST - ); - const inviteAutoJoinEnabled = envBool( - process.env.MATRIX_INVITE_AUTOJOIN, - inviteAutoJoinInviterAllowlist.length > 0 - ); - - const auth = resolveAuthFromEnv(); - - return new MatrixAdapter({ - baseURL, - auth, - userName: process.env.MATRIX_BOT_USERNAME ?? "bot", - deviceID: normalizeOptionalString(process.env.MATRIX_DEVICE_ID), - commandPrefix: process.env.MATRIX_COMMAND_PREFIX, - recoveryKey, - inviteAutoJoin: inviteAutoJoinEnabled - ? { - inviterAllowlist: inviteAutoJoinInviterAllowlist, - } - : undefined, - matrixSDKLogLevel: - parseSDKLogLevel(process.env.MATRIX_SDK_LOG_LEVEL) ?? "error", - }); + return new MatrixAdapter(createMatrixAdapterConfigFromEnv()); } export type { @@ -3596,171 +2695,3 @@ export type { MatrixPersistenceSyncConfig, MatrixThreadID, } from "./types"; - -function normalizePersistenceConfig( - config?: MatrixPersistenceConfig -): ResolvedPersistenceConfig { - return { - keyPrefix: - normalizeOptionalString(config?.keyPrefix) ?? DEFAULT_PERSISTENCE_KEY_PREFIX, - session: { - decrypt: config?.session?.decrypt, - encrypt: config?.session?.encrypt, - ttlMs: config?.session?.ttlMs, - }, - sync: { - persistIntervalMs: - config?.sync?.persistIntervalMs ?? - DEFAULT_MATRIX_STORE_PERSIST_INTERVAL_MS, - snapshotTtlMs: config?.sync?.snapshotTtlMs, - }, - }; -} - -function resolveAuthFromEnv(): MatrixAuthConfig { - const username = process.env.MATRIX_USERNAME; - const password = process.env.MATRIX_PASSWORD; - - if (username && password) { - return { - type: "password", - username, - password, - userID: process.env.MATRIX_USER_ID, - }; - } - - const accessToken = process.env.MATRIX_ACCESS_TOKEN; - const userID = process.env.MATRIX_USER_ID; - - if (!accessToken) { - throw new Error( - "Set MATRIX_USERNAME+MATRIX_PASSWORD for password auth, or MATRIX_ACCESS_TOKEN for access token auth." - ); - } - - const auth: MatrixAccessTokenAuthConfig = { - type: "accessToken", - accessToken, - userID, - }; - - return auth; -} - -function envBool(value: string | undefined, fallback = false): boolean { - if (!value) { - return fallback; - } - - const normalized = value.trim().toLowerCase(); - return ( - normalized === "1" || - normalized === "true" || - normalized === "yes" || - normalized === "on" - ); -} - -function parseEnvList(value: string | undefined): string[] { - if (!value) { - return []; - } - - return value - .split(",") - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0); -} - -function generateDeviceID(length = 8): string { - const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - const bytes = randomBytes(length); - let out = "chatsdk_"; - - for (let i = 0; i < length; i += 1) { - out += alphabet[bytes[i] % alphabet.length]; - } - - return out; -} - -function normalizeOptionalString(value: string | null | undefined): string | undefined { - if (!value) { - return undefined; - } - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : undefined; -} - -function readStringValue(value: unknown): string | undefined { - return typeof value === "string" ? normalizeOptionalString(value) : undefined; -} - -function matrixLocalpart(userID: string): string { - return userID.startsWith("@") ? userID.slice(1).split(":")[0] ?? userID : userID; -} - -function escapeMarkdownText(value: string): string { - return value.replace(/([\\`*_{}\[\]()#+\-.!|>])/gu, "\\$1"); -} - -function escapeMarkdownLinkText(value: string): string { - return value.replace(/([\\\]])/gu, "\\$1"); -} - -function escapeHTML(value: string): string { - return value - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">") - .replaceAll('"', """) - .replaceAll("'", "'"); -} - -function normalizeStringList(values: string[] | undefined): string[] { - if (!values || values.length === 0) { - return []; - } - - return values - .map((value) => normalizeOptionalString(value)) - .filter((value): value is string => Boolean(value)); -} - -function hasIndexedDB(): boolean { - return typeof globalThis.indexedDB !== "undefined" && globalThis.indexedDB !== null; -} - -function isSDKLogLevel(value: string): value is SDKLogLevel { - return value in MATRIX_SDK_LOG_LEVELS; -} - -function parseSDKLogLevel(value: string | undefined): SDKLogLevel | undefined { - if (!value) { - return undefined; - } - const normalized = value.trim().toLowerCase(); - return isSDKLogLevel(normalized) ? normalized : undefined; -} - -function evictOldestEntries( - collection: { size: number; keys(): Iterable; delete(key: string): unknown }, - maxSize = 10_000, - targetSize = 5_000 -): void { - if (collection.size <= maxSize) return; - const toDelete = collection.size - targetSize; - let deleted = 0; - // Map and Set iteration is insertion ordered, so keys() yields the oldest - // entries first for the collections used by this adapter. - for (const key of collection.keys()) { - if (deleted >= toDelete) break; - collection.delete(key); - deleted++; - } -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} diff --git a/src/messages/inbound.ts b/src/messages/inbound.ts new file mode 100644 index 0000000..de0a7a9 --- /dev/null +++ b/src/messages/inbound.ts @@ -0,0 +1,319 @@ +import { markdownToPlainText } from "chat"; +import { MsgType } from "matrix-js-sdk"; +import { + HTMLElement, + NodeType, + parse as parseHTML, + type Node as HTMLNode, +} from "node-html-parser"; +import { + escapeMarkdownText, + isRecord, + normalizeOptionalString, +} from "../shared/utils"; + +export type MatrixMessageContent = { + body?: string; + format?: string; + formatted_body?: string; + msgtype?: string; + "m.mentions"?: { + room?: boolean; + user_ids?: string[]; + }; + [key: string]: unknown; +}; + +export type ParsedMatrixContent = { + markdown: string; + mentionsRoom: boolean; + mentionedUserIDs: Set; + text: string; +}; + +export function parseMatrixContent(content: MatrixMessageContent): ParsedMatrixContent { + const mentionedUserIDs = extractMentionedUserIDs(content); + const mentionsRoom = extractRoomMention(content); + const formattedBody = normalizeOptionalString(content.formatted_body); + if (formattedBody) { + const htmlMarkdown = parseMatrixHTML(formattedBody); + for (const mentionedUserID of htmlMarkdown.mentionedUserIDs) { + mentionedUserIDs.add(mentionedUserID); + } + + if (htmlMarkdown.markdown.length > 0) { + return { + text: markdownToPlainText(htmlMarkdown.markdown), + markdown: htmlMarkdown.markdown, + mentionedUserIDs, + mentionsRoom, + }; + } + } + + const body = stripReplyFallbackFromBody( + normalizeOptionalString(content.body) ?? "" + ); + return { + text: body, + markdown: markdownForPlainText(body, content.msgtype), + mentionedUserIDs, + mentionsRoom, + }; +} + +export function isMentioned(args: { + content: MatrixMessageContent; + parsed: ParsedMatrixContent; + userID: string; + userName: string; +}): boolean { + const { content, parsed, userID, userName } = args; + if (parsed.mentionsRoom) { + return true; + } + if (userID && parsed.mentionedUserIDs.has(userID)) { + return true; + } + + const formatted = + typeof content.formatted_body === "string" ? content.formatted_body : ""; + + const hasUserID = userID + ? parsed.text.includes(userID) || formatted.includes(userID) + : false; + const hasMatrixTo = userID + ? formatted.includes(`matrix.to/#/${encodeURIComponent(userID)}`) + : false; + + const usernameMention = userName.startsWith("@") + ? userName + : `@${userName}`; + + const hasUserName = + parsed.text.includes(usernameMention) || formatted.includes(usernameMention); + + return hasUserID || hasMatrixTo || hasUserName; +} + +function parseMatrixHTML( + html: string +): { markdown: string; mentionedUserIDs: Set } { + const root = parseHTML(stripReplyFallbackFromHTML(html)); + const mentionedUserIDs = new Set(); + const markdown = normalizeMarkdownSpacing( + renderHTMLNodesToMarkdown(root.childNodes, mentionedUserIDs) + ); + return { + markdown, + mentionedUserIDs, + }; +} + +function renderHTMLNodesToMarkdown( + nodes: HTMLNode[], + mentionedUserIDs: Set +): string { + return nodes + .map((node) => renderHTMLNodeToMarkdown(node, mentionedUserIDs)) + .join(""); +} + +function renderHTMLNodeToMarkdown( + node: HTMLNode, + mentionedUserIDs: Set +): string { + if (node.nodeType === NodeType.TEXT_NODE) { + return node.text; + } + + if (!(node instanceof HTMLElement)) { + return ""; + } + + const tagName = node.tagName.toLowerCase(); + const children = renderHTMLNodesToMarkdown(node.childNodes, mentionedUserIDs); + + switch (tagName) { + case "mx-reply": + return ""; + case "html": + case "body": + case "span": + return children; + case "br": + return "\n"; + case "p": + case "div": + return children.trim() ? `${children.trim()}\n\n` : ""; + case "strong": + case "b": + return children ? `**${children}**` : ""; + case "em": + case "i": + return children ? `*${children}*` : ""; + case "del": + case "s": + return children ? `~~${children}~~` : ""; + case "code": + return node.parentNode instanceof HTMLElement && + node.parentNode.tagName.toLowerCase() === "pre" + ? children + : `\`${children}\``; + case "pre": { + const codeContent = children.replace(/\n+$/u, ""); + return codeContent ? `\n\`\`\`\n${codeContent}\n\`\`\`\n\n` : ""; + } + case "blockquote": { + const quoted = children.trim(); + if (!quoted) { + return ""; + } + return `${quoted + .split("\n") + .map((line) => `> ${line}`) + .join("\n")}\n\n`; + } + case "ul": + return `${node.childNodes + .map((child) => renderListItemToMarkdown(child, mentionedUserIDs, null)) + .filter(Boolean) + .join("\n")}\n\n`; + case "ol": + return `${node.childNodes + .map((child, index) => + renderListItemToMarkdown(child, mentionedUserIDs, index + 1) + ) + .filter(Boolean) + .join("\n")}\n\n`; + case "a": + return renderHTMLLinkToMarkdown(node, children, mentionedUserIDs); + case "img": + return normalizeOptionalString(node.getAttribute("alt")) ?? "image"; + default: + return children; + } +} + +function renderListItemToMarkdown( + node: HTMLNode, + mentionedUserIDs: Set, + ordinal: number | null +): string { + if (!(node instanceof HTMLElement) || node.tagName.toLowerCase() !== "li") { + return ""; + } + const content = normalizeMarkdownSpacing( + renderHTMLNodesToMarkdown(node.childNodes, mentionedUserIDs) + ); + if (!content) { + return ""; + } + return `${ordinal === null ? "-" : `${ordinal}.`} ${content}`; +} + +function renderHTMLLinkToMarkdown( + node: HTMLElement, + children: string, + mentionedUserIDs: Set +): string { + const href = normalizeOptionalString(node.getAttribute("href")); + const text = children || node.text; + if (!href) { + return text; + } + + const mentionedUserID = parseMatrixToUserID(href); + if (mentionedUserID) { + mentionedUserIDs.add(mentionedUserID); + return text || matrixMentionDisplayText(mentionedUserID); + } + + return `[${text || href}](${href})`; +} + +function parseMatrixToUserID(href: string): string | null { + let url: URL; + try { + url = new URL(href); + } catch { + return null; + } + + if (url.hostname !== "matrix.to") { + return null; + } + + const rawPath = url.hash.startsWith("#/") ? url.hash.slice(2) : url.hash; + const firstSegment = rawPath.split("/")[0]; + if (!firstSegment) { + return null; + } + + const identifier = decodeURIComponent(firstSegment); + return identifier.startsWith("@") ? identifier : null; +} + +function extractMentionedUserIDs(content: MatrixMessageContent): Set { + const mentions = new Set(); + const matrixMentions = content["m.mentions"]; + if (!isRecord(matrixMentions) || !Array.isArray(matrixMentions.user_ids)) { + return mentions; + } + + for (const userID of matrixMentions.user_ids) { + if (typeof userID === "string" && userID.length > 0) { + mentions.add(userID); + } + } + + return mentions; +} + +function extractRoomMention(content: MatrixMessageContent): boolean { + const matrixMentions = content["m.mentions"]; + return isRecord(matrixMentions) && matrixMentions.room === true; +} + +function stripReplyFallbackFromBody(body: string): string { + const lines = body.split("\n"); + let index = 0; + while (index < lines.length && lines[index]?.startsWith(">")) { + index += 1; + } + + if (index === 0 || index >= lines.length || lines[index] !== "") { + return body; + } + + return lines.slice(index + 1).join("\n"); +} + +function stripReplyFallbackFromHTML(html: string): string { + const root = parseHTML(html); + for (const child of [...root.childNodes]) { + if (child instanceof HTMLElement && child.tagName.toLowerCase() === "mx-reply") { + child.remove(); + } + } + return root.toString(); +} + +function matrixMentionDisplayText(userID: string): string { + const localpart = userID.startsWith("@") + ? userID.slice(1).split(":")[0] ?? userID + : userID; + return `@${localpart}`; +} + +function markdownForPlainText(text: string, msgtype?: string): string { + const escaped = escapeMarkdownText(text); + if (msgtype === MsgType.Emote && escaped.length > 0) { + return `*${escaped}*`; + } + return escaped; +} + +function normalizeMarkdownSpacing(markdown: string): string { + return markdown.replace(/\n{3,}/gu, "\n\n").trim(); +} diff --git a/src/messages/outbound.ts b/src/messages/outbound.ts new file mode 100644 index 0000000..37c3d22 --- /dev/null +++ b/src/messages/outbound.ts @@ -0,0 +1,566 @@ +import type { + AdapterPostableMessage, + Attachment, + EmojiValue, + FileUpload, + Logger, +} from "chat"; +import { + isCardElement, + markdownToPlainText, + stringifyMarkdown, +} from "chat"; +import { marked } from "marked"; +import { MatrixError } from "matrix-js-sdk/lib/http-api/errors"; +import { MsgType, RelationType } from "matrix-js-sdk"; +import type { + RoomMessageEventContent, + RoomMessageTextEventContent, +} from "matrix-js-sdk/lib/@types/events"; +import type { MediaEventContent } from "matrix-js-sdk/lib/@types/media"; +import { + escapeHTML, + escapeMarkdownLinkText, + escapeMarkdownText, + isRecord, + matrixLocalpart, + normalizeOptionalString, +} from "../shared/utils"; + +const DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES = 12_000; + +export type MatrixTextMessageContent = RoomMessageTextEventContent & { + "com.beeper.dont_render_edited"?: boolean; +}; + +export type MatrixRoomMessageContent = RoomMessageEventContent & { + "com.beeper.dont_render_edited"?: boolean; + "m.new_content"?: RoomMessageEventContent & { + "com.beeper.dont_render_edited"?: boolean; + }; +}; + +export type MatrixOutboundMessageContent = MatrixRoomMessageContent | MediaEventContent; + +export type MatrixMediaMsgType = + | MsgType.Audio + | MsgType.File + | MsgType.Image + | MsgType.Video; + +type RenderedMatrixMessage = { + body: string; + formattedBody?: string; + mentions?: { + room?: boolean; + user_ids?: string[]; + }; +}; + +export type OutboundUpload = { + data: Blob; + fileName: string; + info?: { + h?: number; + mimetype?: string; + size?: number; + w?: number; + }; + msgtype: MatrixMediaMsgType; + type?: string; +}; + +export function extractReplyEventID( + message: AdapterPostableMessage +): string | undefined { + if (typeof message !== "object" || message === null || isCardElement(message)) { + return undefined; + } + + const replyEventID = (message as { matrixReplyToEventId?: unknown }).matrixReplyToEventId; + return typeof replyEventID === "string" && replyEventID.length > 0 + ? replyEventID + : undefined; +} + +export function applyThreadReplyMetadata( + content: MatrixOutboundMessageContent, + rootEventID: string | undefined, + replyEventID: string | undefined +): MatrixOutboundMessageContent { + const threadableContent = content as MatrixOutboundMessageContent & { + "m.relates_to"?: { + rel_type?: string; + "m.in_reply_to"?: { event_id: string }; + [key: string]: unknown; + }; + }; + + if (!rootEventID || threadableContent["m.relates_to"]?.rel_type) { + return threadableContent; + } + + return { + ...threadableContent, + "m.relates_to": { + ...threadableContent["m.relates_to"], + "m.in_reply_to": { + event_id: replyEventID ?? rootEventID, + }, + }, + } as MatrixOutboundMessageContent; +} + +export function isTooLargeMatrixError(error: unknown): error is MatrixError { + return ( + error instanceof MatrixError && + (error.errcode === "M_TOO_LARGE" || error.httpStatus === 413) + ); +} + +export function splitOversizedTextContent( + content: MatrixOutboundMessageContent +): MatrixTextMessageContent[] { + if (!isSplittableTextContent(content)) { + return []; + } + + const body = content.body; + if (Buffer.byteLength(body, "utf8") <= DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES) { + return []; + } + + const parts = splitTextByUtf8Bytes(body, DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES); + if (parts.length <= 1) { + return []; + } + + return parts.map((part) => ({ + body: part, + msgtype: content.msgtype, + })); +} + +export function toRoomMessageContent( + message: AdapterPostableMessage +): MatrixTextMessageContent { + const rendered = renderTextMessage(message); + const content: MatrixTextMessageContent = { + body: rendered.body, + msgtype: MsgType.Text, + }; + if (rendered.formattedBody) { + content.format = "org.matrix.custom.html"; + content.formatted_body = rendered.formattedBody; + } + if (rendered.mentions) { + content["m.mentions"] = rendered.mentions; + } + + return content; +} + +export function mergeTextAndLinks( + content: MatrixTextMessageContent, + linkLines: string[] +): MatrixTextMessageContent { + if (linkLines.length === 0) { + return content; + } + + const suffix = linkLines.join("\n"); + const body = content.body ?? ""; + const mergedBody = body ? `${body}\n\n${suffix}` : suffix; + if (!content.formatted_body) { + return { + ...content, + body: mergedBody, + }; + } + + const formattedSuffix = linkLines + .map((line) => `

${escapeHTML(line)}

`) + .join(""); + + return { + ...content, + body: mergedBody, + formatted_body: `${content.formatted_body}${formattedSuffix}`, + }; +} + +export function collectLinkOnlyAttachmentLines(attachments: Attachment[]): string[] { + const lines: string[] = []; + for (const attachment of attachments) { + const hasLocalData = + Boolean(attachment.data) || typeof attachment.fetchData === "function"; + if (hasLocalData) { + continue; + } + if (!attachment.url) { + continue; + } + const label = attachment.name ?? attachment.type; + lines.push(`${label}: ${attachment.url}`); + } + return lines; +} + +export function extractFilesFromMessage( + message: AdapterPostableMessage, + logger?: Logger +): FileUpload[] { + if (typeof message !== "object" || message === null) { + return []; + } + if (!("files" in message) || !Array.isArray(message.files)) { + return []; + } + return message.files.flatMap((file): FileUpload[] => { + const normalized = normalizeFileUpload(file, logger); + return normalized ? [normalized] : []; + }); +} + +export function normalizeFileUpload( + file: unknown, + logger?: Logger +): FileUpload | null { + if (!isRecord(file)) { + return null; + } + + const filename = normalizeOptionalString( + typeof file.filename === "string" ? file.filename : undefined + ); + if (!filename) { + return null; + } + + const data = normalizeFileUploadData(file.data); + if (!data) { + logger?.warn("Skipping invalid Matrix file upload", { filename }); + return null; + } + + return { + filename, + data, + mimeType: + typeof file.mimeType === "string" ? normalizeOptionalString(file.mimeType) : undefined, + }; +} + +export function normalizeFileUploadData( + data: unknown +): Buffer | Blob | ArrayBuffer | null { + if (Buffer.isBuffer(data) || data instanceof Blob || data instanceof ArrayBuffer) { + return data; + } + + if (data instanceof Uint8Array) { + return Buffer.from(data); + } + + return null; +} + +export function extractAttachmentsFromMessage( + message: AdapterPostableMessage +): Attachment[] { + if (typeof message !== "object" || message === null) { + return []; + } + if (!("attachments" in message) || !Array.isArray(message.attachments)) { + return []; + } + return message.attachments.filter((a): a is Attachment => isRecord(a)); +} + +export async function readAttachmentData( + attachment: Attachment +): Promise { + if (typeof attachment.fetchData === "function") { + return attachment.fetchData(); + } + return attachment.data ?? null; +} + +export function normalizeUploadData(data: Buffer | Blob | ArrayBuffer): Blob { + if (data instanceof Blob) { + return data; + } + if (isNodeBuffer(data)) { + return new Blob([new Uint8Array(data)]); + } + return new Blob([data]); +} + +export function binarySize(data: Buffer | Blob | ArrayBuffer): number { + if (data instanceof ArrayBuffer) { + return data.byteLength; + } + if (isNodeBuffer(data)) { + return data.length; + } + return data.size; +} + +export function messageTypeForAttachment( + attachment: Attachment +): MatrixMediaMsgType { + switch (attachment.type) { + case "image": + return MsgType.Image; + case "video": + return MsgType.Video; + case "audio": + return MsgType.Audio; + default: + return messageTypeForMimeType(normalizeOptionalString(attachment.mimeType)); + } +} + +export function messageTypeForMimeType(mimeType?: string): MatrixMediaMsgType { + if (!mimeType) { + return MsgType.File; + } + if (mimeType.startsWith("image/")) { + return MsgType.Image; + } + if (mimeType.startsWith("video/")) { + return MsgType.Video; + } + if (mimeType.startsWith("audio/")) { + return MsgType.Audio; + } + return MsgType.File; +} + +export function defaultAttachmentName(attachment: Attachment): string { + switch (attachment.type) { + case "image": + return "image"; + case "video": + return "video"; + case "audio": + return "audio"; + default: + return "file"; + } +} + +function renderTextMessage(message: AdapterPostableMessage): RenderedMatrixMessage { + if (typeof message === "string") { + return renderPlainTextMessage(message); + } + + if (isCardElement(message)) { + return renderPlainTextMessage("[Card message]"); + } + + if (typeof message === "object" && message !== null) { + if ("raw" in message && typeof message.raw === "string") { + return renderPlainTextMessage(message.raw); + } + if ("markdown" in message && typeof message.markdown === "string") { + return renderMarkdownMessage(message.markdown); + } + if ("ast" in message) { + return renderMarkdownMessage(stringifyMarkdown(message.ast)); + } + if ("card" in message) { + return renderPlainTextMessage(message.fallbackText ?? "[Card message]"); + } + } + + return { body: "" }; +} + +function renderPlainTextMessage(text: string): RenderedMatrixMessage { + const rendered = replaceMentionPlaceholdersInPlainText(text); + if (rendered.mentionedUserIDs.size === 0) { + return { + body: rendered.body, + }; + } + + return { + body: rendered.body, + formattedBody: renderMarkdownToMatrixHTML(rendered.markdown), + mentions: buildMentionsContent(rendered.mentionedUserIDs), + }; +} + +function renderMarkdownMessage(markdown: string): RenderedMatrixMessage { + const rendered = replaceMentionPlaceholdersInMarkdown(markdown); + return { + body: markdownToPlainText(rendered.markdown), + formattedBody: renderMarkdownToMatrixHTML(rendered.markdown), + mentions: buildMentionsContent(rendered.mentionedUserIDs), + }; +} + +function replaceMentionPlaceholdersInPlainText(text: string): { + body: string; + markdown: string; + mentionedUserIDs: Set; +} { + const mentionedUserIDs = new Set(); + const pattern = /<@(@[^>\s]+:[^>\s]+)>/gu; + let body = ""; + let markdown = ""; + let lastIndex = 0; + + for (const match of text.matchAll(pattern)) { + const [token, userID] = match; + const index = match.index ?? 0; + const plainSegment = text.slice(lastIndex, index); + body += plainSegment; + markdown += escapeMarkdownText(plainSegment); + + const mentionText = matrixMentionDisplayText(userID); + body += mentionText; + markdown += `[${escapeMarkdownLinkText(mentionText)}](${matrixToUserLink(userID)})`; + mentionedUserIDs.add(userID); + lastIndex = index + token.length; + } + + const trailing = text.slice(lastIndex); + body += trailing; + markdown += escapeMarkdownText(trailing); + + return { body, markdown, mentionedUserIDs }; +} + +function replaceMentionPlaceholdersInMarkdown(markdown: string): { + markdown: string; + mentionedUserIDs: Set; +} { + const mentionedUserIDs = new Set(); + const transformed = markdown.replace( + /<@(@[^>\s]+:[^>\s]+)>/gu, + (_match, userID: string) => { + mentionedUserIDs.add(userID); + return `[${escapeMarkdownLinkText(matrixMentionDisplayText(userID))}](${matrixToUserLink( + userID + )})`; + } + ); + + return { + markdown: transformed, + mentionedUserIDs, + }; +} + +function renderMarkdownToMatrixHTML(markdown: string): string { + return marked.parse(markdown, { + async: false, + breaks: true, + gfm: true, + }); +} + +function buildMentionsContent( + mentionedUserIDs: Set +): { room?: boolean; user_ids?: string[] } | undefined { + if (mentionedUserIDs.size === 0) { + return undefined; + } + + return { + user_ids: [...mentionedUserIDs], + }; +} + +function matrixToUserLink(userID: string): string { + return `https://matrix.to/#/${encodeURIComponent(userID)}`; +} + +function matrixMentionDisplayText(userID: string): string { + return `@${matrixLocalpart(userID)}`; +} + +function isSplittableTextContent( + content: MatrixOutboundMessageContent +): content is MatrixTextMessageContent { + if ("url" in content || "info" in content) { + return false; + } + + if (typeof content.body !== "string" || content.body.length <= 1) { + return false; + } + + if ("m.new_content" in content) { + return false; + } + + return ( + content.msgtype === MsgType.Text || + content.msgtype === MsgType.Notice || + content.msgtype === MsgType.Emote + ); +} + +function splitTextByUtf8Bytes(text: string, maxBytes: number): string[] { + const normalizedMaxBytes = Math.max(1, Math.floor(maxBytes)); + const chunks: string[] = []; + let remaining = text; + + while (remaining.length > 0) { + if (Buffer.byteLength(remaining, "utf8") <= normalizedMaxBytes) { + chunks.push(remaining); + remaining = ""; + break; + } + + const boundary = findSplitBoundary(remaining, normalizedMaxBytes); + const head = remaining.slice(0, boundary).trimEnd(); + const tail = remaining.slice(boundary).trimStart(); + + if (!head || head === remaining) { + break; + } + + chunks.push(head); + remaining = tail; + } + + if (remaining.length > 0 && chunks.at(-1) !== remaining) { + chunks.push(remaining); + } + + return chunks.filter((chunk) => chunk.length > 0); +} + +function findSplitBoundary(text: string, maxBytes: number): number { + let low = 1; + let high = text.length; + let best = 1; + + while (low <= high) { + const mid = Math.floor((low + high) / 2); + const candidate = text.slice(0, mid); + if (Buffer.byteLength(candidate, "utf8") <= maxBytes) { + best = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + + for (let i = best - 1; i > 0; i--) { + const ch = text[i]; + if (ch === "\n" || ch === " " || ch === "\t" || ch === "\r") { + return i + 1; + } + } + + return best; +} + +function isNodeBuffer(value: unknown): value is Buffer { + return typeof Buffer !== "undefined" && Buffer.isBuffer(value); +} diff --git a/src/shared/utils.ts b/src/shared/utils.ts new file mode 100644 index 0000000..c7da5ca --- /dev/null +++ b/src/shared/utils.ts @@ -0,0 +1,81 @@ +import { randomBytes } from "node:crypto"; + +export function generateDeviceID(length = 8): string { + const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + const bytes = randomBytes(length); + let out = "chatsdk_"; + + for (let i = 0; i < length; i += 1) { + out += alphabet[bytes[i] % alphabet.length]; + } + + return out; +} + +export function normalizeOptionalString( + value: string | null | undefined +): string | undefined { + if (!value) { + return undefined; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +export function readStringValue(value: unknown): string | undefined { + return typeof value === "string" ? normalizeOptionalString(value) : undefined; +} + +export function matrixLocalpart(userID: string): string { + return userID.startsWith("@") ? userID.slice(1).split(":")[0] ?? userID : userID; +} + +export function escapeMarkdownText(value: string): string { + return value.replace(/([\\`*_{}\[\]()#+\-.!|>])/gu, "\\$1"); +} + +export function escapeMarkdownLinkText(value: string): string { + return value.replace(/([\\\]])/gu, "\\$1"); +} + +export function escapeHTML(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +export function normalizeStringList(values: string[] | undefined): string[] { + if (!values || values.length === 0) { + return []; + } + + return values + .map((value) => normalizeOptionalString(value)) + .filter((value): value is string => Boolean(value)); +} + +export function hasIndexedDB(): boolean { + return typeof globalThis.indexedDB !== "undefined" && globalThis.indexedDB !== null; +} + +export function evictOldestEntries( + collection: { size: number; keys(): Iterable; delete(key: string): unknown }, + maxSize = 10_000, + targetSize = 5_000 +): void { + if (collection.size <= maxSize) return; + const toDelete = collection.size - targetSize; + let deleted = 0; + for (const key of collection.keys()) { + if (deleted >= toDelete) break; + collection.delete(key); + deleted++; + } +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} From bf66abbe6f4a2c80d70fc2543dc8a044f0f93892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Sun, 12 Apr 2026 19:54:04 +0200 Subject: [PATCH 6/9] Fix thread reply handling and text splitting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Omit reply fallback metadata for follow-up attachments and avoid cutting UTF-16 surrogate pairs when splitting oversized text. sendRoomMessage now returns both the send response and the actual sent content so resolved events use the correct content; applyThreadReplyMetadata treats replyEventID === null as “do not attach reply metadata” (fixes attachments in threads). Text-splitting logic was hardened: surrogate-aware boundary clamping, trim helpers, and adjusted findSplitBoundary ensure emojis and other surrogate pairs are preserved across chunks. Other refactors: moved matrixMentionDisplayText to shared utils, parallelized attachment upload prep with Promise.all, removed thin cursor wrapper methods in favor of direct helpers, and tightened secrets-bundle persist throttling. Added tests for reply-fallback omission and surrogate-safe splitting. --- src/index.test.ts | 62 ++++++++++++++++++ src/index.ts | 135 +++++++++++++++++++-------------------- src/messages/inbound.ts | 24 ++----- src/messages/outbound.ts | 65 +++++++++++++++---- src/shared/utils.ts | 4 ++ 5 files changed, 193 insertions(+), 97 deletions(-) diff --git a/src/index.test.ts b/src/index.test.ts index 30acd70..fc3628e 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -6,6 +6,7 @@ import { EventType, MsgType, RelationType, type MatrixClient } from "matrix-js-s import { MatrixError } from "matrix-js-sdk/lib/http-api/errors"; import { encodeRecoveryKey } from "matrix-js-sdk/lib/crypto-api/recovery-key"; import { createMatrixAdapter, MatrixAdapter } from "./index"; +import { splitOversizedTextContent } from "./messages/outbound"; type RawEventLike = { content?: Record; @@ -1181,6 +1182,43 @@ describe("MatrixAdapter", () => { ); }); + it("omits reply fallback metadata for follow-up attachments in Matrix threads", async () => { + const fakeClient = makeClient(); + fakeClient.sendEvent = vi + .fn() + .mockResolvedValueOnce({ event_id: "$text" }) + .mockResolvedValueOnce({ event_id: "$file" }); + fakeClient.uploadContent = vi + .fn() + .mockResolvedValueOnce({ content_uri: "mxc://beeper.com/file-1" }); + + const adapter = await makeInitializedAdapter(fakeClient); + + await adapter.postMessage("matrix:!room%3Abeeper.com:%24root", { + markdown: "File incoming", + files: [ + { + data: new Uint8Array([1, 2, 3]).buffer, + filename: "report.png", + mimeType: "image/png", + }, + ], + }); + + const sendCalls = fakeClient.sendEvent.mock.calls as unknown as Array; + const firstContent = sendCalls[0]?.[3] as Record; + const secondContent = sendCalls[1]?.[3] as Record; + + expect(firstContent).toMatchObject({ + "m.relates_to": { + "m.in_reply_to": { + event_id: "$root", + }, + }, + }); + expect(secondContent).not.toHaveProperty("m.relates_to"); + }); + it("skips invalid file uploads instead of passing malformed entries downstream", async () => { const fakeClient = makeClient(); const logger = makeTestLogger(); @@ -1267,6 +1305,11 @@ describe("MatrixAdapter", () => { { errcode: "M_TOO_LARGE", error: "event too large" }, 413 ); + fakeClient.getRoom.mockReturnValue( + makeRoom({ + findEventById: () => null, + }) + ); fakeClient.sendEvent = vi .fn() .mockRejectedValueOnce(tooLargeError) @@ -1314,6 +1357,7 @@ describe("MatrixAdapter", () => { expect(content).not.toHaveProperty("formatted_body"); expect(content).not.toHaveProperty("m.mentions"); } + expect(sent.raw.getContent()).toMatchObject(fallbackBodies[0] ?? {}); expect(logger.warn).toHaveBeenCalledWith( "Matrix message exceeded size limit; retrying as split plain-text chunks", @@ -1446,6 +1490,24 @@ describe("MatrixAdapter", () => { ); }); + it("splits oversized text without cutting surrogate pairs", () => { + const content = splitOversizedTextContent({ + body: "😀".repeat(4_000), + msgtype: MsgType.Text, + }); + + expect(content.length).toBeGreaterThan(1); + expect(content.map((part) => part.body).join("")).toBe("😀".repeat(4_000)); + + for (const part of content) { + const body = part.body; + const lastCodeUnit = body.charCodeAt(body.length - 1); + const firstCodeUnit = body.charCodeAt(0); + expect(lastCodeUnit < 0xd800 || lastCodeUnit > 0xdbff).toBe(true); + expect(firstCodeUnit < 0xdc00 || firstCodeUnit > 0xdfff).toBe(true); + } + }); + it("generates and persists a device id when one is not provided", async () => { const adapter = getInternals( new MatrixAdapter({ diff --git a/src/index.ts b/src/index.ts index bb58f4d..4a3f06f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -170,6 +170,11 @@ type MatrixRoomMetadata = { topic?: string; }; +type SentRoomMessage = { + response: Awaited>; + sentContent: MatrixOutboundMessageContent; +}; + // Intentionally unsupported in this adapter: postEphemeral, openModal, and native stream. export class MatrixAdapter implements Adapter { readonly name = "matrix"; @@ -366,21 +371,21 @@ export class MatrixAdapter implements Adapter { if (!firstContent) { throw new Error("Cannot post an empty Matrix message."); } - const response = await this.sendRoomMessage( + const { response, sentContent } = await this.sendRoomMessage( roomID, rootEventID, replyEventID, firstContent, ); for (const content of extraContents) { - await this.sendRoomMessage(roomID, rootEventID, undefined, content); + await this.sendRoomMessage(roomID, rootEventID, null, content); } return { id: response.event_id, threadId, raw: this.resolveSentEvent(roomID, response.event_id, { - content: firstContent, + content: sentContent, roomID, sender: this.userID, }), @@ -418,13 +423,18 @@ export class MatrixAdapter implements Adapter { body: `* ${baseContent.body}`, }; - const response = await this.sendRoomMessage(roomID, rootEventID, undefined, editContent); + const { response, sentContent } = await this.sendRoomMessage( + roomID, + rootEventID, + null, + editContent + ); return { id: response.event_id, threadId, raw: this.resolveSentEvent(roomID, response.event_id, { - content: editContent, + content: sentContent, roomID, sender: this.userID, }), @@ -559,7 +569,7 @@ export class MatrixAdapter implements Adapter { const direction = options.direction ?? "backward"; const limit = options.limit ?? 50; const cursor = options.cursor - ? this.decodeCursorV1( + ? decodeCursorV1( options.cursor, rootEventID ? "thread_relations" : "room_messages", roomID, @@ -580,7 +590,7 @@ export class MatrixAdapter implements Adapter { return { messages: response.events.map((event) => this.parseMessageInternal(event)), nextCursor: response.nextToken - ? this.encodeCursorV1({ + ? encodeCursorV1({ kind: "room_messages", dir: direction, token: response.nextToken, @@ -605,7 +615,7 @@ export class MatrixAdapter implements Adapter { this.parseMessageInternal(event, this.encodeThreadId({ roomID, rootEventID })) ), nextCursor: response.nextToken - ? this.encodeCursorV1({ + ? encodeCursorV1({ kind: "thread_relations", dir: direction, token: response.nextToken, @@ -682,7 +692,7 @@ export class MatrixAdapter implements Adapter { const roomID = this.decodeThreadId(channelId).roomID; const limit = options.limit ?? 50; const cursor = options.cursor - ? this.decodeCursorV1(options.cursor, "thread_list", roomID, undefined, "backward") + ? decodeCursorV1(options.cursor, "thread_list", roomID, undefined, "backward") : null; const listResponse = await this.requireClient().createThreadListMessagesRequest( roomID, @@ -717,7 +727,7 @@ export class MatrixAdapter implements Adapter { return { threads: summaries, nextCursor: listResponse.end - ? this.encodeCursorV1({ + ? encodeCursorV1({ kind: "thread_list", dir: "backward", token: listResponse.end, @@ -765,30 +775,6 @@ export class MatrixAdapter implements Adapter { }); } - private encodeCursorV1(payload: CursorV1Payload): string { - return encodeCursorV1(payload); - } - - private decodeCursorV1( - cursor: string, - expectedKind: CursorKind, - expectedRoomID: string, - expectedRootEventID?: string, - expectedDirection?: CursorDirection - ): CursorV1Payload { - return decodeCursorV1( - cursor, - expectedKind, - expectedRoomID, - expectedRootEventID, - expectedDirection - ); - } - - private toSDKDirection(dir: CursorDirection): Direction { - return toSDKDirection(dir); - } - private async fetchRoomMessagesPage(args: { roomID: string; includeThreadReplies: boolean; @@ -800,7 +786,7 @@ export class MatrixAdapter implements Adapter { args.roomID, args.fromToken, args.limit, - this.toSDKDirection(args.direction) + toSDKDirection(args.direction) ); const messageChunk = (response.chunk ?? []).filter( (raw) => @@ -847,7 +833,7 @@ export class MatrixAdapter implements Adapter { THREAD_RELATION_TYPE.name, null, { - dir: this.toSDKDirection(args.direction), + dir: toSDKDirection(args.direction), from: args.fromToken ?? undefined, limit: relationLimit, } @@ -1408,15 +1394,18 @@ export class MatrixAdapter implements Adapter { private async sendRoomMessage( roomID: string, rootEventID: string | undefined, - replyEventID: string | undefined, + replyEventID: string | null | undefined, content: MatrixOutboundMessageContent - ) { + ): Promise { const threadContent = applyThreadReplyMetadata(content, rootEventID, replyEventID); try { const response = await this.performSendRoomMessage(roomID, rootEventID, threadContent); void this.maybePersistSecretsBundle(); - return response; + return { + response, + sentContent: threadContent, + }; } catch (error) { if (isTooLargeMatrixError(error)) { const splitContents = splitOversizedTextContent(threadContent); @@ -1432,7 +1421,7 @@ export class MatrixAdapter implements Adapter { } ); - let firstResponse: Awaited> | undefined; + let firstSentMessage: SentRoomMessage | undefined; for (const splitContent of splitContents) { const chunkWithMeta = applyThreadReplyMetadata( splitContent, @@ -1441,11 +1430,14 @@ export class MatrixAdapter implements Adapter { ); const response = await this.performSendRoomMessage(roomID, rootEventID, chunkWithMeta); void this.maybePersistSecretsBundle(); - firstResponse ??= response; + firstSentMessage ??= { + response, + sentContent: chunkWithMeta, + }; } - if (firstResponse) { - return firstResponse; + if (firstSentMessage) { + return firstSentMessage; } } } @@ -1609,6 +1601,10 @@ export class MatrixAdapter implements Adapter { } private async maybePersistSecretsBundle(force = false): Promise { + if (!force && Date.now() - this.lastSecretsBundlePersistAt < 60_000) { + return; + } + if (!this.stateAdapter) { return; } @@ -1624,9 +1620,6 @@ export class MatrixAdapter implements Adapter { } const now = Date.now(); - if (!force && now - this.lastSecretsBundlePersistAt < 60_000) { - return; - } try { const bundle = await crypto.exportSecretsBundle(); @@ -2301,26 +2294,33 @@ export class MatrixAdapter implements Adapter { }); } - for (const attachment of attachments) { - const data = await readAttachmentData(attachment); - if (!data) { - continue; + const attachmentUploads = await Promise.all( + attachments.map(async (attachment): Promise => { + const data = await readAttachmentData(attachment); + if (!data) { + return null; + } + const fileName = + normalizeOptionalString(attachment.name) ?? + defaultAttachmentName(attachment); + return { + data: normalizeUploadData(data), + fileName, + info: { + h: attachment.height, + mimetype: normalizeOptionalString(attachment.mimeType), + size: attachment.size ?? binarySize(data), + w: attachment.width, + }, + msgtype: messageTypeForAttachment(attachment), + type: attachment.type, + }; + }) + ); + for (const upload of attachmentUploads) { + if (upload) { + uploads.push(upload); } - const fileName = - normalizeOptionalString(attachment.name) ?? - defaultAttachmentName(attachment); - uploads.push({ - data: normalizeUploadData(data), - fileName, - info: { - h: attachment.height, - mimetype: normalizeOptionalString(attachment.mimeType), - size: attachment.size ?? binarySize(data), - w: attachment.width, - }, - msgtype: messageTypeForAttachment(attachment), - type: attachment.type, - }); } return uploads; @@ -2340,7 +2340,7 @@ export class MatrixAdapter implements Adapter { roomID: string, eventID: string, fallback: { - content: MatrixRoomMessageContent; + content: MatrixOutboundMessageContent; roomID: string; sender?: string; } @@ -2611,9 +2611,6 @@ export class MatrixAdapter implements Adapter { ]); for (const key of candidates) { - if (!key) { - continue; - } const value = await this.stateAdapter.get(key); const normalized = normalizeOptionalString(value); if (normalized) { diff --git a/src/messages/inbound.ts b/src/messages/inbound.ts index de0a7a9..5bc9e4a 100644 --- a/src/messages/inbound.ts +++ b/src/messages/inbound.ts @@ -9,6 +9,7 @@ import { import { escapeMarkdownText, isRecord, + matrixMentionDisplayText, normalizeOptionalString, } from "../shared/utils"; @@ -99,7 +100,12 @@ export function isMentioned(args: { function parseMatrixHTML( html: string ): { markdown: string; mentionedUserIDs: Set } { - const root = parseHTML(stripReplyFallbackFromHTML(html)); + const root = parseHTML(html); + for (const child of [...root.childNodes]) { + if (child instanceof HTMLElement && child.tagName.toLowerCase() === "mx-reply") { + child.remove(); + } + } const mentionedUserIDs = new Set(); const markdown = normalizeMarkdownSpacing( renderHTMLNodesToMarkdown(root.childNodes, mentionedUserIDs) @@ -289,22 +295,6 @@ function stripReplyFallbackFromBody(body: string): string { return lines.slice(index + 1).join("\n"); } -function stripReplyFallbackFromHTML(html: string): string { - const root = parseHTML(html); - for (const child of [...root.childNodes]) { - if (child instanceof HTMLElement && child.tagName.toLowerCase() === "mx-reply") { - child.remove(); - } - } - return root.toString(); -} - -function matrixMentionDisplayText(userID: string): string { - const localpart = userID.startsWith("@") - ? userID.slice(1).split(":")[0] ?? userID - : userID; - return `@${localpart}`; -} function markdownForPlainText(text: string, msgtype?: string): string { const escaped = escapeMarkdownText(text); diff --git a/src/messages/outbound.ts b/src/messages/outbound.ts index 37c3d22..930eb34 100644 --- a/src/messages/outbound.ts +++ b/src/messages/outbound.ts @@ -23,7 +23,7 @@ import { escapeMarkdownLinkText, escapeMarkdownText, isRecord, - matrixLocalpart, + matrixMentionDisplayText, normalizeOptionalString, } from "../shared/utils"; @@ -86,7 +86,7 @@ export function extractReplyEventID( export function applyThreadReplyMetadata( content: MatrixOutboundMessageContent, rootEventID: string | undefined, - replyEventID: string | undefined + replyEventID: string | null | undefined ): MatrixOutboundMessageContent { const threadableContent = content as MatrixOutboundMessageContent & { "m.relates_to"?: { @@ -100,6 +100,10 @@ export function applyThreadReplyMetadata( return threadableContent; } + if (replyEventID === null) { + return threadableContent; + } + return { ...threadableContent, "m.relates_to": { @@ -478,10 +482,6 @@ function matrixToUserLink(userID: string): string { return `https://matrix.to/#/${encodeURIComponent(userID)}`; } -function matrixMentionDisplayText(userID: string): string { - return `@${matrixLocalpart(userID)}`; -} - function isSplittableTextContent( content: MatrixOutboundMessageContent ): content is MatrixTextMessageContent { @@ -516,9 +516,14 @@ function splitTextByUtf8Bytes(text: string, maxBytes: number): string[] { break; } - const boundary = findSplitBoundary(remaining, normalizedMaxBytes); - const head = remaining.slice(0, boundary).trimEnd(); - const tail = remaining.slice(boundary).trimStart(); + const boundary = clampSurrogateBoundary( + remaining, + findSplitBoundary(remaining, normalizedMaxBytes) + ); + const rawHead = remaining.slice(0, boundary); + const rawTail = remaining.slice(boundary); + const head = trimEndPreservingSurrogates(rawHead); + const tail = trimStartPreservingSurrogates(rawTail); if (!head || head === remaining) { break; @@ -542,9 +547,9 @@ function findSplitBoundary(text: string, maxBytes: number): number { while (low <= high) { const mid = Math.floor((low + high) / 2); - const candidate = text.slice(0, mid); + const candidate = text.slice(0, clampSurrogateBoundary(text, mid)); if (Buffer.byteLength(candidate, "utf8") <= maxBytes) { - best = mid; + best = candidate.length; low = mid + 1; } else { high = mid - 1; @@ -561,6 +566,44 @@ function findSplitBoundary(text: string, maxBytes: number): number { return best; } +function trimEndPreservingSurrogates(text: string): string { + let boundary = text.length; + while (boundary > 0 && /\s/.test(text[boundary - 1] ?? "")) { + boundary -= 1; + } + return text.slice(0, clampSurrogateBoundary(text, boundary)); +} + +function trimStartPreservingSurrogates(text: string): string { + let boundary = 0; + while (boundary < text.length && /\s/.test(text[boundary] ?? "")) { + boundary += 1; + } + return text.slice(clampSurrogateBoundary(text, boundary)); +} + +function clampSurrogateBoundary(text: string, boundary: number): number { + if (boundary <= 0 || boundary >= text.length) { + return boundary; + } + + const current = text.charCodeAt(boundary); + const previous = text.charCodeAt(boundary - 1); + if (isLowSurrogateCodeUnit(current) && isHighSurrogateCodeUnit(previous)) { + return boundary - 1; + } + + return boundary; +} + +function isHighSurrogateCodeUnit(code: number): boolean { + return code >= 0xd800 && code <= 0xdbff; +} + +function isLowSurrogateCodeUnit(code: number): boolean { + return code >= 0xdc00 && code <= 0xdfff; +} + function isNodeBuffer(value: unknown): value is Buffer { return typeof Buffer !== "undefined" && Buffer.isBuffer(value); } diff --git a/src/shared/utils.ts b/src/shared/utils.ts index c7da5ca..9e7af7d 100644 --- a/src/shared/utils.ts +++ b/src/shared/utils.ts @@ -76,6 +76,10 @@ export function evictOldestEntries( } } +export function matrixMentionDisplayText(userID: string): string { + return `@${matrixLocalpart(userID)}`; +} + export function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } From f867b64b30f63de695cc8c5ae604a1178499ab65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 13 Apr 2026 12:24:09 +0200 Subject: [PATCH 7/9] Validate Matrix thread IDs; clamp split boundary Add validation to decodeThreadId to throw when the decoded room ID is empty (e.g. "matrix:") to prevent returning invalid MatrixThreadID objects. Add a unit test to assert this behavior. Also update findSplitBoundary to use clampSurrogateBoundary on its return values to avoid splitting inside Unicode surrogate pairs when chopping long outbound messages. --- src/history/cursor.ts | 3 +++ src/index.test.ts | 15 +++++++++++++++ src/messages/outbound.ts | 4 ++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/history/cursor.ts b/src/history/cursor.ts index 01b0514..8457314 100644 --- a/src/history/cursor.ts +++ b/src/history/cursor.ts @@ -32,6 +32,9 @@ export function decodeThreadId(threadId: string): MatrixThreadID { } const roomID = decodeURIComponent(parts[1]); + if (!roomID) { + throw new Error(`Invalid Matrix thread ID: ${threadId}`); + } const rootEventID = parts[2] ? decodeURIComponent(parts[2]) : undefined; return { roomID, rootEventID }; diff --git a/src/index.test.ts b/src/index.test.ts index fc3628e..e00a6fd 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -450,6 +450,21 @@ describe("MatrixAdapter", () => { }); }); + it("rejects thread IDs with an empty room ID", () => { + const adapter = new MatrixAdapter({ + baseURL: "https://hs.beeper.com", + auth: { + type: "accessToken", + accessToken: "token", + userID: "@bot:beeper.com", + }, + }); + + expect(() => adapter.decodeThreadId("matrix:")).toThrow( + "Invalid Matrix thread ID: matrix:" + ); + }); + it("parses slash commands from timeline messages", async () => { const fakeClient = makeClient(); diff --git a/src/messages/outbound.ts b/src/messages/outbound.ts index 930eb34..e59682c 100644 --- a/src/messages/outbound.ts +++ b/src/messages/outbound.ts @@ -559,11 +559,11 @@ function findSplitBoundary(text: string, maxBytes: number): number { for (let i = best - 1; i > 0; i--) { const ch = text[i]; if (ch === "\n" || ch === " " || ch === "\t" || ch === "\r") { - return i + 1; + return clampSurrogateBoundary(text, i + 1); } } - return best; + return clampSurrogateBoundary(text, best); } function trimEndPreservingSurrogates(text: string): string { From a17ee5a3541bf9f35537c5d7be16952264222c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 13 Apr 2026 14:15:05 +0200 Subject: [PATCH 8/9] Fix threads, E2E IndexedDB, and splitting Multiple changes to improve thread metadata, E2E test setup, and message splitting behavior: - listThreads: use local room data (requireRoom/processThreadRoots/getThread) to prefer live thread metadata, fall back to fetching the latest reply when bundled summaries lag, and compute replyCount using both room and bundled data. - Added fetchLatestThreadReplySummary helper to pull a latest reply when needed. - Sending retries: treat any split chunk result as valid fallback (retry even when there's a single plain-text chunk) and keep the existing warning logging. - outbound splitting: splitOversizedTextContent now returns the original part when it already fits and preserves boundary whitespace when splitting (removed trimming logic that dropped boundary whitespace). - inbound mention handling: isMentioned ignores empty usernames and normalizes username whitespace before matching. - E2E/tests & helpers: add fake-indexeddb to tests, enable indexedDB for e2ee in createParticipantFromSession via a per-session cryptoDatabasePrefix (with sanitization), add waitForCondition async support and switch tests to waitForMatchingMessage/waitForCondition where appropriate, plus several new/updated tests for thread behavior and splitting. - package.json: add fake-indexeddb test dependency. These changes improve accuracy of thread summaries, ensure E2E crypto state isolation across sessions, and avoid losing whitespace when splitting large text messages. --- e2e/e2e.test.ts | 23 ++-- e2e/helpers.ts | 17 ++- package.json | 1 + pnpm-lock.yaml | 9 ++ src/index.test.ts | 273 ++++++++++++++++++++++++++++++++++++++- src/index.ts | 54 +++++++- src/messages/inbound.ts | 19 ++- src/messages/outbound.ts | 32 ++--- 8 files changed, 378 insertions(+), 50 deletions(-) diff --git a/e2e/e2e.test.ts b/e2e/e2e.test.ts index da5670a..4180b55 100644 --- a/e2e/e2e.test.ts +++ b/e2e/e2e.test.ts @@ -11,6 +11,7 @@ import { nonce, shutdownParticipant, sleep, + waitForCondition, waitForEvent, waitForEncryptedRoom, waitForFetchedMessage, @@ -470,11 +471,11 @@ describe.skipIf(!hasCoreCredentials)("E2E Matrix Adapter", () => { ]); const latestOffline = offlinePosts[offlinePosts.length - 1]; - const caughtUpMessage = await waitForFetchedMessage( + const caughtUpMessage = await waitForMatchingMessage( bot.adapter, bot.adapter.encodeThreadId({ roomID: restartRoomID }), - latestOffline.id, - (message) => message.text.includes(restartTag), + (message) => + message.id === latestOffline.id && message.text.includes(restartTag), 60_000 ); expect(caughtUpMessage.text).toContain(restartTag); @@ -626,11 +627,10 @@ describe.skipIf(!hasCoreCredentials)("E2E Matrix Adapter", () => { }); await sender.adapter.postMessage(threadId, `Thread reply ${replyTag}`); - await waitForFetchedMessage( + await waitForMatchingMessage( bot.adapter, bot.adapter.encodeThreadId({ roomID: threadListRoomID }), - rootPosted.id, - (message) => message.text.includes(rootTag) + (message) => message.id === rootPosted.id && message.text.includes(rootTag) ); await waitForMatchingMessage( bot.adapter, @@ -653,8 +653,15 @@ describe.skipIf(!hasCoreCredentials)("E2E Matrix Adapter", () => { expect(threadInfo.isDM).toBe(false); expect(threadInfo.metadata?.roomID).toBe(threadListRoomID); - const threads = await bot.adapter.listThreads(channelId, { limit: 20 }); - const summary = threads.threads.find((thread) => thread.id === threadId); + let summary: + | Awaited>["threads"][number] + | undefined; + await waitForCondition(async () => { + const threads = await bot.adapter.listThreads(channelId, { limit: 20 }); + summary = threads.threads.find((thread) => thread.id === threadId); + return Boolean(summary && (summary.replyCount ?? 0) >= 1); + }, 45_000); + expect(summary).toBeTruthy(); expect(summary?.rootMessage.id).toBe(rootPosted.id); expect(summary?.rootMessage.text).toContain(rootTag); diff --git a/e2e/helpers.ts b/e2e/helpers.ts index 08f258d..d181dc6 100644 --- a/e2e/helpers.ts +++ b/e2e/helpers.ts @@ -2,6 +2,7 @@ import { randomBytes } from "node:crypto"; import { Chat, type Message, type ReactionEvent, type StateAdapter } from "chat"; import { createMemoryState } from "@chat-adapter/state-memory"; import { createRedisState } from "@chat-adapter/state-redis"; +import "fake-indexeddb/auto"; import { EventType } from "matrix-js-sdk"; import type { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk"; import { MatrixAdapter } from "../src/index"; @@ -61,6 +62,7 @@ export async function createParticipantFromSession(opts: { state?: StateAdapter; }): Promise { const state = opts.state ?? createE2EState(opts.name); + const cryptoDatabasePrefix = createE2EIndexedDBPrefix(opts.session); const adapter = new MatrixAdapter({ baseURL: env.baseURL, auth: { @@ -71,7 +73,8 @@ export async function createParticipantFromSession(opts: { deviceID: opts.session.deviceID, inviteAutoJoin: {}, e2ee: { - useIndexedDB: false, + cryptoDatabasePrefix, + useIndexedDB: true, }, recoveryKey: opts.recoveryKey, }); @@ -136,6 +139,14 @@ type MatrixLoginResponse = { userID: string; }; +function createE2EIndexedDBPrefix(session: MatrixLoginResponse): string { + return `matrix-chat-adapter-e2e-${sanitizeForIndexedDBName(session.userID)}-${sanitizeForIndexedDBName(session.deviceID)}`; +} + +function sanitizeForIndexedDBName(value: string): string { + return value.replace(/[^a-zA-Z0-9_-]+/gu, "_"); +} + function generateDeviceID(): string { return `E2E_${randomBytes(8).toString("hex").toUpperCase()}`; } @@ -283,14 +294,14 @@ export function waitForEvent( } export async function waitForCondition( - condition: () => boolean, + condition: () => boolean | Promise, timeoutMs = 10_000, intervalMs = 250 ): Promise { const startedAt = Date.now(); while (true) { - if (condition()) { + if (await condition()) { return; } diff --git a/package.json b/package.json index 17d2385..e6f5466 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "@types/node": "^22.10.2", "@vitest/coverage-v8": "^2.1.8", "eslint": "^10.0.2", + "fake-indexeddb": "^6.2.4", "tsup": "^8.3.5", "typescript": "^5.7.2", "typescript-eslint": "^8.56.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2e3dafc..86c71a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -39,6 +39,9 @@ importers: eslint: specifier: ^10.0.2 version: 10.0.3 + fake-indexeddb: + specifier: ^6.2.4 + version: 6.2.5 tsup: specifier: ^8.3.5 version: 8.5.1(postcss@8.5.8)(typescript@5.9.3) @@ -1026,6 +1029,10 @@ packages: extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fake-indexeddb@6.2.5: + resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==} + engines: {node: '>=18'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2602,6 +2609,8 @@ snapshots: extend@3.0.2: {} + fake-indexeddb@6.2.5: {} + fast-deep-equal@3.1.3: {} fast-json-stable-stringify@2.1.0: {} diff --git a/src/index.test.ts b/src/index.test.ts index e00a6fd..eddceb0 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -6,6 +6,7 @@ import { EventType, MsgType, RelationType, type MatrixClient } from "matrix-js-s import { MatrixError } from "matrix-js-sdk/lib/http-api/errors"; import { encodeRecoveryKey } from "matrix-js-sdk/lib/crypto-api/recovery-key"; import { createMatrixAdapter, MatrixAdapter } from "./index"; +import { isMentioned } from "./messages/inbound"; import { splitOversizedTextContent } from "./messages/outbound"; type RawEventLike = { @@ -49,6 +50,9 @@ type RoomStateLike = { type RoomLike = { currentState: RoomStateLike; getMember: (userId: string) => MemberLike | null; + getThread: (eventId: string) => { length: number; replyToEvent: ReturnType | null } | null; + getThreads: () => Array<{ id: string }>; + processThreadRoots: (events: Array>, toStartOfTimeline: boolean) => void; roomId: string; name: string; timeline: unknown[]; @@ -273,6 +277,9 @@ function makeRoom(overrides: Partial = {}): RoomLike { getStateEvents: () => null, }, getMember: () => null, + getThread: () => null, + getThreads: () => [], + processThreadRoots: () => undefined, hasEncryptionStateEvent: () => false, getJoinedMembers: () => [{}, {}], getMyMembership: () => "join", @@ -1364,7 +1371,7 @@ describe("MatrixAdapter", () => { .slice(1) .map((call) => call[2] as unknown as Record); expect(fallbackBodies).toHaveLength(2); - expect(fallbackBodies.map((content) => content.body).join("\n")) + expect(fallbackBodies.map((content) => content.body).join("")) .toBe(Array.from({ length: 2_000 }, () => "hello world").join("\n")); for (const content of fallbackBodies) { expect(content).toMatchObject({ msgtype: MsgType.Text }); @@ -1385,6 +1392,68 @@ describe("MatrixAdapter", () => { expect(logger.error).not.toHaveBeenCalled(); }); + it("retries oversized rich-text events as a single plain-text fallback when body fits", async () => { + const fakeClient = makeClient(); + const logger = makeTestLogger(); + const tooLargeError = new MatrixError( + { errcode: "M_TOO_LARGE", error: "event too large" }, + 413 + ); + fakeClient.getRoom.mockReturnValue( + makeRoom({ + findEventById: () => null, + }) + ); + fakeClient.sendEvent = vi + .fn() + .mockRejectedValueOnce(tooLargeError) + .mockResolvedValueOnce({ event_id: "$plain-fallback" }); + + const adapter = new MatrixAdapter({ + baseURL: "https://hs.beeper.com", + auth: { + type: "accessToken", + accessToken: "token", + userID: "@bot:beeper.com", + }, + logger: logger.logger, + createClient: () => asMatrixClient(fakeClient), + }); + await adapter.initialize(makeChatInstance()); + + const sent = await adapter.postMessage("matrix:!room%3Abeeper.com", { + markdown: "**hello**", + }); + + expect(sent.id).toBe("$plain-fallback"); + expect(fakeClient.sendEvent).toHaveBeenCalledTimes(2); + expect(fakeClient.sendEvent).toHaveBeenNthCalledWith( + 2, + "!room:beeper.com", + EventType.RoomMessage, + expect.objectContaining({ + body: "hello", + msgtype: MsgType.Text, + }) + ); + + const fallbackCalls = fakeClient.sendEvent.mock.calls as unknown as Array; + const fallbackContent = fallbackCalls[1]?.[2] as Record | undefined; + expect(fallbackContent).not.toHaveProperty("format"); + expect(fallbackContent).not.toHaveProperty("formatted_body"); + expect(fallbackContent).not.toHaveProperty("m.mentions"); + expect(sent.raw.getContent()).toMatchObject(fallbackContent ?? {}); + + expect(logger.warn).toHaveBeenCalledWith( + "Matrix message exceeded size limit; retrying as split plain-text chunks", + expect.objectContaining({ + roomId: "!room:beeper.com", + chunkCount: 1, + msgtype: MsgType.Text, + }) + ); + }); + it("rethrows M_TOO_LARGE for non-text Matrix events", async () => { const fakeClient = makeClient(); const logger = makeTestLogger(); @@ -1523,6 +1592,36 @@ describe("MatrixAdapter", () => { } }); + it("preserves boundary whitespace when splitting oversized text", () => { + const text = `${"a".repeat(11_999)} ${"b".repeat(32)}`; + const content = splitOversizedTextContent({ + body: text, + msgtype: MsgType.Text, + }); + + expect(content.length).toBeGreaterThan(1); + expect(content.map((part) => part.body).join("")).toBe(text); + }); + + it("does not treat an empty username as a mention", () => { + expect( + isMentioned({ + content: { + body: "@someone said hi", + msgtype: MsgType.Text, + }, + parsed: { + markdown: "@someone said hi", + mentionedUserIDs: new Set(), + mentionsRoom: false, + text: "@someone said hi", + }, + userID: "", + userName: "", + }) + ).toBe(false); + }); + it("generates and persists a device id when one is not provided", async () => { const adapter = getInternals( new MatrixAdapter({ @@ -2597,4 +2696,176 @@ describe("MatrixAdapter", () => { dir: "backward", }); }); + + it("prefers live room thread metadata when bundled thread counts lag", async () => { + const fakeClient = makeClient(); + fakeClient.createThreadListMessagesRequest.mockResolvedValue({ + chunk: [ + makeRawEvent({ + event_id: "$root1", + origin_server_ts: 1_700_000_000_100, + content: { body: "Root 1" }, + unsigned: { + "m.relations": { + "m.thread": { + count: 0, + latest_event: { origin_server_ts: 1_700_000_000_500 }, + }, + }, + }, + }), + ], + end: undefined, + }); + + const latestReply = makeEvent({ + getId: () => "$reply1", + getTs: () => 1_700_000_000_900, + }); + fakeClient.getRoom.mockReturnValue( + makeRoom({ + getThread: (eventId: string) => + eventId === "$root1" + ? { + length: 2, + replyToEvent: latestReply, + } + : null, + }) + ); + + const adapter = await makeInitializedAdapter(fakeClient); + + const result = await adapter.listThreads("matrix:!room%3Abeeper.com", { limit: 2 }); + + expect(result.threads).toHaveLength(1); + expect(result.threads[0]?.replyCount).toBe(2); + expect(result.threads[0]?.lastReplyAt?.toISOString()).toBe( + "2023-11-14T22:13:20.900Z" + ); + }); + + it("hydrates room thread metadata from thread-list roots before summarizing", async () => { + const fakeClient = makeClient(); + fakeClient.createThreadListMessagesRequest.mockResolvedValue({ + chunk: [ + makeRawEvent({ + event_id: "$root1", + origin_server_ts: 1_700_000_000_100, + content: { body: "Root 1" }, + unsigned: { + "m.relations": { + "m.thread": { + count: 0, + latest_event: { origin_server_ts: 1_700_000_000_500 }, + }, + }, + }, + }), + ], + end: undefined, + }); + + const latestReply = makeEvent({ + getId: () => "$reply1", + getTs: () => 1_700_000_000_900, + }); + let hydrated = false; + const processThreadRoots = vi.fn( + (events: Array>, toStartOfTimeline: boolean) => { + expect(events.map((event) => event.getId())).toEqual(["$root1"]); + expect(toStartOfTimeline).toBe(true); + hydrated = true; + } + ); + fakeClient.getRoom.mockReturnValue( + makeRoom({ + processThreadRoots, + getThread: (eventId: string) => + hydrated && eventId === "$root1" + ? { + length: 1, + replyToEvent: latestReply, + } + : null, + }) + ); + + const adapter = await makeInitializedAdapter(fakeClient); + + const result = await adapter.listThreads("matrix:!room%3Abeeper.com", { limit: 2 }); + + expect(processThreadRoots).toHaveBeenCalledOnce(); + expect(result.threads).toHaveLength(1); + expect(result.threads[0]?.replyCount).toBe(1); + expect(result.threads[0]?.lastReplyAt?.toISOString()).toBe( + "2023-11-14T22:13:20.900Z" + ); + }); + + it("falls back to fetching the latest thread reply when thread summaries are missing", async () => { + const fakeClient = makeClient(); + fakeClient.createThreadListMessagesRequest.mockResolvedValue({ + chunk: [ + makeRawEvent({ + event_id: "$root1", + origin_server_ts: 1_700_000_000_100, + content: { body: "Root 1" }, + unsigned: {}, + }), + ], + end: undefined, + }); + fakeClient.relations.mockResolvedValue({ + originalEvent: null, + events: [ + makeEvent({ + getId: () => "$reply1", + getTs: () => 1_700_000_000_900, + getContent: () => ({ body: "Reply 1" }), + getRelation: () => ({ + rel_type: RelationType.Thread, + event_id: "$root1", + }), + isRelation: (relType?: string) => relType === RelationType.Thread, + threadRootId: "$root1", + }), + ], + nextBatch: null, + prevBatch: null, + }); + fakeClient.getRoom.mockReturnValue( + makeRoom({ + findEventById: (eventId?: string) => + eventId === "$root1" + ? makeEvent({ + getId: () => "$root1", + getTs: () => 1_700_000_000_100, + getContent: () => ({ body: "Root 1" }), + }) + : null, + }) + ); + + const adapter = await makeInitializedAdapter(fakeClient); + + const result = await adapter.listThreads("matrix:!room%3Abeeper.com", { limit: 2 }); + + expect(fakeClient.relations).toHaveBeenCalledWith( + "!room:beeper.com", + "$root1", + "m.thread", + null, + { + dir: "b", + from: undefined, + limit: 1, + } + ); + expect(result.threads).toHaveLength(1); + expect(result.threads[0]?.replyCount).toBe(1); + expect(result.threads[0]?.lastReplyAt?.toISOString()).toBe( + "2023-11-14T22:13:20.900Z" + ); + }); }); diff --git a/src/index.ts b/src/index.ts index 4a3f06f..042be70 100644 --- a/src/index.ts +++ b/src/index.ts @@ -690,6 +690,7 @@ export class MatrixAdapter implements Adapter { options: ListThreadsOptions = {} ): Promise> { const roomID = this.decodeThreadId(channelId).roomID; + const room = this.requireRoom(roomID); const limit = options.limit ?? 50; const cursor = options.cursor ? decodeCursorV1(options.cursor, "thread_list", roomID, undefined, "backward") @@ -702,6 +703,7 @@ export class MatrixAdapter implements Adapter { ThreadFilterType.All ); const events = await this.mapRawEvents(listResponse.chunk ?? [], roomID); + room.processThreadRoots(events, true); const summaries: ThreadSummary[] = []; for (const rootEvent of events) { @@ -710,16 +712,32 @@ export class MatrixAdapter implements Adapter { continue; } - const bundled = rootEvent.getServerAggregatedRelation( - THREAD_RELATION_TYPE.name - ); - const latestTS = bundled?.latest_event?.origin_server_ts; + const localRootEvent = room.findEventById(rootID); + const summaryRootEvent = localRootEvent ?? rootEvent; + const bundled = + summaryRootEvent.getServerAggregatedRelation( + THREAD_RELATION_TYPE.name + ) ?? + rootEvent.getServerAggregatedRelation( + THREAD_RELATION_TYPE.name + ); + const roomThread = room.getThread(rootID); + const latestReply = roomThread?.replyToEvent; + let latestTS = latestReply?.getTs() ?? bundled?.latest_event?.origin_server_ts; + let replyCount = Math.max(roomThread?.length ?? 0, bundled?.count ?? 0); + + if (replyCount === 0 && typeof latestTS !== "number") { + const fallback = await this.fetchLatestThreadReplySummary(roomID, rootID); + replyCount = Math.max(replyCount, fallback.replyCount); + latestTS = latestTS ?? fallback.latestReplyTS; + } + const threadID = this.encodeThreadId({ roomID, rootEventID: rootID }); summaries.push({ id: threadID, - rootMessage: this.parseMessageInternal(rootEvent, threadID), - replyCount: bundled?.count ?? 0, + rootMessage: this.parseMessageInternal(summaryRootEvent, threadID), + replyCount, lastReplyAt: typeof latestTS === "number" ? new Date(latestTS) : undefined, }); } @@ -737,6 +755,28 @@ export class MatrixAdapter implements Adapter { }; } + private async fetchLatestThreadReplySummary( + roomID: string, + rootEventID: string + ): Promise<{ replyCount: number; latestReplyTS?: number }> { + const response = await this.fetchThreadMessagesPage({ + roomID, + rootEventID, + includeRoot: false, + limit: 1, + direction: "backward", + fromToken: null, + }); + const latestReply = response.events.at(-1); + + return latestReply + ? { + replyCount: 1, + latestReplyTS: latestReply.getTs(), + } + : { replyCount: 0 }; + } + private parseMessageInternal( raw: MatrixEvent, overrideThreadID?: string @@ -1409,7 +1449,7 @@ export class MatrixAdapter implements Adapter { } catch (error) { if (isTooLargeMatrixError(error)) { const splitContents = splitOversizedTextContent(threadContent); - if (splitContents.length > 1) { + if (splitContents.length > 0) { this.logger.warn( "Matrix message exceeded size limit; retrying as split plain-text chunks", { diff --git a/src/messages/inbound.ts b/src/messages/inbound.ts index 5bc9e4a..35342ae 100644 --- a/src/messages/inbound.ts +++ b/src/messages/inbound.ts @@ -86,13 +86,18 @@ export function isMentioned(args: { const hasMatrixTo = userID ? formatted.includes(`matrix.to/#/${encodeURIComponent(userID)}`) : false; - - const usernameMention = userName.startsWith("@") - ? userName - : `@${userName}`; - - const hasUserName = - parsed.text.includes(usernameMention) || formatted.includes(usernameMention); + const normalizedUserName = userName.trim(); + const hasUserName = normalizedUserName + ? (() => { + const usernameMention = normalizedUserName.startsWith("@") + ? normalizedUserName + : `@${normalizedUserName}`; + return ( + parsed.text.includes(usernameMention) || + formatted.includes(usernameMention) + ); + })() + : false; return hasUserID || hasMatrixTo || hasUserName; } diff --git a/src/messages/outbound.ts b/src/messages/outbound.ts index e59682c..97241a9 100644 --- a/src/messages/outbound.ts +++ b/src/messages/outbound.ts @@ -131,13 +131,15 @@ export function splitOversizedTextContent( const body = content.body; if (Buffer.byteLength(body, "utf8") <= DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES) { - return []; + return [ + { + body, + msgtype: content.msgtype, + }, + ]; } const parts = splitTextByUtf8Bytes(body, DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES); - if (parts.length <= 1) { - return []; - } return parts.map((part) => ({ body: part, @@ -520,10 +522,8 @@ function splitTextByUtf8Bytes(text: string, maxBytes: number): string[] { remaining, findSplitBoundary(remaining, normalizedMaxBytes) ); - const rawHead = remaining.slice(0, boundary); - const rawTail = remaining.slice(boundary); - const head = trimEndPreservingSurrogates(rawHead); - const tail = trimStartPreservingSurrogates(rawTail); + const head = remaining.slice(0, boundary); + const tail = remaining.slice(boundary); if (!head || head === remaining) { break; @@ -566,22 +566,6 @@ function findSplitBoundary(text: string, maxBytes: number): number { return clampSurrogateBoundary(text, best); } -function trimEndPreservingSurrogates(text: string): string { - let boundary = text.length; - while (boundary > 0 && /\s/.test(text[boundary - 1] ?? "")) { - boundary -= 1; - } - return text.slice(0, clampSurrogateBoundary(text, boundary)); -} - -function trimStartPreservingSurrogates(text: string): string { - let boundary = 0; - while (boundary < text.length && /\s/.test(text[boundary] ?? "")) { - boundary += 1; - } - return text.slice(clampSurrogateBoundary(text, boundary)); -} - function clampSurrogateBoundary(text: string, boundary: number): number { if (boundary <= 0 || boundary >= text.length) { return boundary; From 37bbc929e1271bb3864fde54c28810711beb2c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?batuhan=20i=C3=A7=C3=B6z?= Date: Mon, 13 Apr 2026 14:30:33 +0200 Subject: [PATCH 9/9] Escape link text, robust Matrix parsing & e2e fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • messages/inbound.ts: Escape markdown link text when rendering HTML anchors (escapeMarkdownLinkText) to avoid accidental markdown injection, and make parseMatrixToUserID resilient to malformed percent-encoding by wrapping decodeURIComponent in a try/catch. • e2e/helpers.ts: Replace sanitizeForIndexedDBName with base64url encoding (encodeForIndexedDBName) for safer IndexedDB keys, and rework waitForCondition to respect timeouts using Promise.race, clearing timers and rejecting on timeout. • src/index.test.ts: Add tests ensuring HTML anchor text is escaped when re-emitting markdown links and that malformed matrix.to links are ignored instead of aborting formatted-body parsing. These changes improve robustness around link handling, encoding for storage, and test coverage for edge cases. --- e2e/helpers.ts | 32 +++++++++++++++++++---- src/index.test.ts | 56 +++++++++++++++++++++++++++++++++++++++++ src/messages/inbound.ts | 11 +++++--- 3 files changed, 91 insertions(+), 8 deletions(-) diff --git a/e2e/helpers.ts b/e2e/helpers.ts index d181dc6..7157863 100644 --- a/e2e/helpers.ts +++ b/e2e/helpers.ts @@ -140,11 +140,11 @@ type MatrixLoginResponse = { }; function createE2EIndexedDBPrefix(session: MatrixLoginResponse): string { - return `matrix-chat-adapter-e2e-${sanitizeForIndexedDBName(session.userID)}-${sanitizeForIndexedDBName(session.deviceID)}`; + return `matrix-chat-adapter-e2e-${encodeForIndexedDBName(session.userID)}-${encodeForIndexedDBName(session.deviceID)}`; } -function sanitizeForIndexedDBName(value: string): string { - return value.replace(/[^a-zA-Z0-9_-]+/gu, "_"); +function encodeForIndexedDBName(value: string): string { + return Buffer.from(value, "utf8").toString("base64url"); } function generateDeviceID(): string { @@ -301,8 +301,30 @@ export async function waitForCondition( const startedAt = Date.now(); while (true) { - if (await condition()) { - return; + const remainingMs = timeoutMs - (Date.now() - startedAt); + if (remainingMs <= 0) { + throw new Error(`waitForCondition timed out after ${timeoutMs}ms`); + } + + let timeout: ReturnType | undefined; + try { + const matched = await Promise.race([ + Promise.resolve().then(condition), + new Promise((_, reject) => { + timeout = setTimeout(() => { + reject(new Error(`waitForCondition timed out after ${timeoutMs}ms`)); + }, remainingMs); + timeout.unref?.(); + }), + ]); + + if (matched) { + return; + } + } finally { + if (timeout) { + clearTimeout(timeout); + } } if (Date.now() - startedAt >= timeoutMs) { diff --git a/src/index.test.ts b/src/index.test.ts index eddceb0..cc27915 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -972,6 +972,62 @@ describe("MatrixAdapter", () => { }); }); + it("escapes HTML anchor text before re-emitting markdown links", async () => { + const fakeClient = makeClient(); + fakeClient.fetchRoomEvent = vi.fn(async () => + makeRawEvent({ + event_id: "$escaped-link", + content: { + body: "literal [text](https://example.com)", + msgtype: MsgType.Text, + format: "org.matrix.custom.html", + formatted_body: + '

Docs [v1] (draft) \\\\ notes

', + }, + }) + ); + + const adapter = await makeInitializedAdapter(fakeClient); + const message = await adapter.fetchMessage( + "matrix:!room%3Abeeper.com", + "$escaped-link" + ); + + expect(message).toBeTruthy(); + expect( + stringifyMarkdown(requireValue(message, "escaped link message").formatted).trim() + ).toBe("\\[Docs [v1\\] (draft) \\\\\\ notes](https://example.com)"); + }); + + it("ignores malformed matrix.to links instead of aborting formatted-body parsing", async () => { + const fakeClient = makeClient(); + fakeClient.fetchRoomEvent = vi.fn(async () => + makeRawEvent({ + event_id: "$bad-matrix-to", + content: { + body: "Broken mention link", + msgtype: MsgType.Text, + format: "org.matrix.custom.html", + formatted_body: + '

broken] still parsed

', + }, + }) + ); + + const adapter = await makeInitializedAdapter(fakeClient); + const message = await adapter.fetchMessage( + "matrix:!room%3Abeeper.com", + "$bad-matrix-to" + ); + + expect(message).toBeTruthy(); + expect(message?.text).toBe("broken] still parsed"); + expect( + stringifyMarkdown(requireValue(message, "bad matrix.to message").formatted).trim() + ).toBe("[broken\\]](https://matrix.to/#/%E0%A4%A) **still parsed**"); + expect(message?.isMention).toBe(false); + }); + it("strips Matrix reply fallback from plain body text", async () => { const fakeClient = makeClient(); fakeClient.fetchRoomEvent = vi.fn(async () => diff --git a/src/messages/inbound.ts b/src/messages/inbound.ts index 35342ae..938e123 100644 --- a/src/messages/inbound.ts +++ b/src/messages/inbound.ts @@ -7,6 +7,7 @@ import { type Node as HTMLNode, } from "node-html-parser"; import { + escapeMarkdownLinkText, escapeMarkdownText, isRecord, matrixMentionDisplayText, @@ -240,7 +241,7 @@ function renderHTMLLinkToMarkdown( return text || matrixMentionDisplayText(mentionedUserID); } - return `[${text || href}](${href})`; + return `[${escapeMarkdownLinkText(text || href)}](${href})`; } function parseMatrixToUserID(href: string): string | null { @@ -261,8 +262,12 @@ function parseMatrixToUserID(href: string): string | null { return null; } - const identifier = decodeURIComponent(firstSegment); - return identifier.startsWith("@") ? identifier : null; + try { + const identifier = decodeURIComponent(firstSegment); + return identifier.startsWith("@") ? identifier : null; + } catch { + return null; + } } function extractMentionedUserIDs(content: MatrixMessageContent): Set {