diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d53aa6..d4f1433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 0.2.1 + +### New + +- Automatic oversized message splitting: text messages rejected with `M_TOO_LARGE` (413) are retried as plain-text chunks (~12 KB each) +- Thread reply metadata: messages posted to threads now include `m.in_reply_to`, with optional `matrixReplyToEventId` override + +### Fixes + +- Attachments sent alongside text no longer incorrectly carry the reply-to relationship +- Incoming formatted messages were parsed twice; removed redundant `` pre-strip pass +- `matrixSDKLogConfigured` flag no longer latches when `setLevel` is missing from the SDK logger + +### Changes + +- Bump `chat` SDK to 4.25.0 +- Move `@chat-adapter/state-memory` and `@chat-adapter/state-redis` to devDependencies + ## 0.2.0 ### New diff --git a/e2e/e2e.test.ts b/e2e/e2e.test.ts index da5670a..4180b55 100644 --- a/e2e/e2e.test.ts +++ b/e2e/e2e.test.ts @@ -11,6 +11,7 @@ import { nonce, shutdownParticipant, sleep, + waitForCondition, waitForEvent, waitForEncryptedRoom, waitForFetchedMessage, @@ -470,11 +471,11 @@ describe.skipIf(!hasCoreCredentials)("E2E Matrix Adapter", () => { ]); const latestOffline = offlinePosts[offlinePosts.length - 1]; - const caughtUpMessage = await waitForFetchedMessage( + const caughtUpMessage = await waitForMatchingMessage( bot.adapter, bot.adapter.encodeThreadId({ roomID: restartRoomID }), - latestOffline.id, - (message) => message.text.includes(restartTag), + (message) => + message.id === latestOffline.id && message.text.includes(restartTag), 60_000 ); expect(caughtUpMessage.text).toContain(restartTag); @@ -626,11 +627,10 @@ describe.skipIf(!hasCoreCredentials)("E2E Matrix Adapter", () => { }); await sender.adapter.postMessage(threadId, `Thread reply ${replyTag}`); - await waitForFetchedMessage( + await waitForMatchingMessage( bot.adapter, bot.adapter.encodeThreadId({ roomID: threadListRoomID }), - rootPosted.id, - (message) => message.text.includes(rootTag) + (message) => message.id === rootPosted.id && message.text.includes(rootTag) ); await waitForMatchingMessage( bot.adapter, @@ -653,8 +653,15 @@ describe.skipIf(!hasCoreCredentials)("E2E Matrix Adapter", () => { expect(threadInfo.isDM).toBe(false); expect(threadInfo.metadata?.roomID).toBe(threadListRoomID); - const threads = await bot.adapter.listThreads(channelId, { limit: 20 }); - const summary = threads.threads.find((thread) => thread.id === threadId); + let summary: + | Awaited>["threads"][number] + | undefined; + await waitForCondition(async () => { + const threads = await bot.adapter.listThreads(channelId, { limit: 20 }); + summary = threads.threads.find((thread) => thread.id === threadId); + return Boolean(summary && (summary.replyCount ?? 0) >= 1); + }, 45_000); + expect(summary).toBeTruthy(); expect(summary?.rootMessage.id).toBe(rootPosted.id); expect(summary?.rootMessage.text).toContain(rootTag); diff --git a/e2e/helpers.ts b/e2e/helpers.ts index 08f258d..7157863 100644 --- a/e2e/helpers.ts +++ b/e2e/helpers.ts @@ -2,6 +2,7 @@ import { randomBytes } from "node:crypto"; import { Chat, type Message, type ReactionEvent, type StateAdapter } from "chat"; import { createMemoryState } from "@chat-adapter/state-memory"; import { createRedisState } from "@chat-adapter/state-redis"; +import "fake-indexeddb/auto"; import { EventType } from "matrix-js-sdk"; import type { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk"; import { MatrixAdapter } from "../src/index"; @@ -61,6 +62,7 @@ export async function createParticipantFromSession(opts: { state?: StateAdapter; }): Promise { const state = opts.state ?? createE2EState(opts.name); + const cryptoDatabasePrefix = createE2EIndexedDBPrefix(opts.session); const adapter = new MatrixAdapter({ baseURL: env.baseURL, auth: { @@ -71,7 +73,8 @@ export async function createParticipantFromSession(opts: { deviceID: opts.session.deviceID, inviteAutoJoin: {}, e2ee: { - useIndexedDB: false, + cryptoDatabasePrefix, + useIndexedDB: true, }, recoveryKey: opts.recoveryKey, }); @@ -136,6 +139,14 @@ type MatrixLoginResponse = { userID: string; }; +function createE2EIndexedDBPrefix(session: MatrixLoginResponse): string { + return `matrix-chat-adapter-e2e-${encodeForIndexedDBName(session.userID)}-${encodeForIndexedDBName(session.deviceID)}`; +} + +function encodeForIndexedDBName(value: string): string { + return Buffer.from(value, "utf8").toString("base64url"); +} + function generateDeviceID(): string { return `E2E_${randomBytes(8).toString("hex").toUpperCase()}`; } @@ -283,15 +294,37 @@ export function waitForEvent( } export async function waitForCondition( - condition: () => boolean, + condition: () => boolean | Promise, timeoutMs = 10_000, intervalMs = 250 ): Promise { const startedAt = Date.now(); while (true) { - if (condition()) { - return; + const remainingMs = timeoutMs - (Date.now() - startedAt); + if (remainingMs <= 0) { + throw new Error(`waitForCondition timed out after ${timeoutMs}ms`); + } + + let timeout: ReturnType | undefined; + try { + const matched = await Promise.race([ + Promise.resolve().then(condition), + new Promise((_, reject) => { + timeout = setTimeout(() => { + reject(new Error(`waitForCondition timed out after ${timeoutMs}ms`)); + }, remainingMs); + timeout.unref?.(); + }), + ]); + + if (matched) { + return; + } + } finally { + if (timeout) { + clearTimeout(timeout); + } } if (Date.now() - startedAt >= timeoutMs) { diff --git a/package.json b/package.json index ea1b884..e6f5466 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@beeper/chat-adapter-matrix", - "version": "0.2.0", + "version": "0.2.1", "description": "Matrix adapter for chat", "engines": { "node": ">=22" @@ -33,18 +33,19 @@ "clean": "rm -rf dist" }, "dependencies": { - "@chat-adapter/state-memory": "^4.17.0", - "@chat-adapter/state-redis": "^4.17.0", - "chat": "^4.17.0", + "chat": "^4.25.0", "marked": "^15.0.12", "matrix-js-sdk": "^41.0.0", "node-html-parser": "^7.1.0" }, "devDependencies": { + "@chat-adapter/state-memory": "^4.25.0", + "@chat-adapter/state-redis": "^4.25.0", "@eslint/js": "^10.0.1", "@types/node": "^22.10.2", "@vitest/coverage-v8": "^2.1.8", "eslint": "^10.0.2", + "fake-indexeddb": "^6.2.4", "tsup": "^8.3.5", "typescript": "^5.7.2", "typescript-eslint": "^8.56.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 30910b8..86c71a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,15 +8,9 @@ importers: .: dependencies: - '@chat-adapter/state-memory': - specifier: ^4.17.0 - version: 4.17.0 - '@chat-adapter/state-redis': - specifier: ^4.17.0 - version: 4.17.0 chat: - specifier: ^4.17.0 - version: 4.17.0 + specifier: ^4.25.0 + version: 4.25.0 marked: specifier: ^15.0.12 version: 15.0.12 @@ -27,6 +21,12 @@ importers: specifier: ^7.1.0 version: 7.1.0 devDependencies: + '@chat-adapter/state-memory': + specifier: ^4.25.0 + version: 4.25.0 + '@chat-adapter/state-redis': + specifier: ^4.25.0 + version: 4.25.0 '@eslint/js': specifier: ^10.0.1 version: 10.0.1(eslint@10.0.3) @@ -39,6 +39,9 @@ importers: eslint: specifier: ^10.0.2 version: 10.0.3 + fake-indexeddb: + specifier: ^6.2.4 + version: 6.2.5 tsup: specifier: ^8.3.5 version: 8.5.1(postcss@8.5.8)(typescript@5.9.3) @@ -82,11 +85,11 @@ packages: '@bcoe/v8-coverage@0.2.3': resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} - '@chat-adapter/state-memory@4.17.0': - resolution: {integrity: sha512-7LewkFY6gQVEXxAycaWUXax9Pt7Prmn2mBdoIBbTEAzV5bSQBaXqKyrd6VWCd9gEyxNlBUainyQul4iGNPw1sQ==} + '@chat-adapter/state-memory@4.25.0': + resolution: {integrity: sha512-bNM4ycFPfGHUI5TN3yo9wGG8ahgUXlKS7WLzhr8WnjmH6g6o8OebknihNh3WDw07xckJUoKDpOtE1uwChvgZag==} - '@chat-adapter/state-redis@4.17.0': - resolution: {integrity: sha512-sJ2V/pZESZKrGkuafMqg4gqyvHwz8d0JDUvqeZ+FVfEgT/h0g1kYfSc++qfEayzmPpSJxC6gzGpBGTiL/5utCw==} + '@chat-adapter/state-redis@4.25.0': + resolution: {integrity: sha512-XHZ6Kv9vbKP19YaRKXaurnsxzizHpQOkfY0P4JPS8oEGVKSCJ5a9ZEnlsT3guLG8zcO6cFqNR2Br+PmX6fh1Dw==} '@esbuild/aix-ppc64@0.21.5': resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} @@ -849,8 +852,8 @@ packages: character-entities@2.0.2: resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} - chat@4.17.0: - resolution: {integrity: sha512-kX9jIXmEU2ksnF1YshM+qfI/6pBy6k8pELkuRaE2AJyF/0nt4GUc0McBhn2DNpXLDI5sVwRGB3PmI+q5wUtYFg==} + chat@4.25.0: + resolution: {integrity: sha512-QM8ex4Gpn8zYIPyQXh41Who6R9Wq3WcQeOjAy4EuR1m1ha0tASuzHkLQfjaTAGLgrgrThV0Zh5KKoH0S92iwNA==} check-error@2.1.3: resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} @@ -1026,6 +1029,10 @@ packages: extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fake-indexeddb@6.2.5: + resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==} + engines: {node: '>=18'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -1797,15 +1804,15 @@ snapshots: '@bcoe/v8-coverage@0.2.3': {} - '@chat-adapter/state-memory@4.17.0': + '@chat-adapter/state-memory@4.25.0': dependencies: - chat: 4.17.0 + chat: 4.25.0 transitivePeerDependencies: - supports-color - '@chat-adapter/state-redis@4.17.0': + '@chat-adapter/state-redis@4.25.0': dependencies: - chat: 4.17.0 + chat: 4.25.0 redis: 5.11.0 transitivePeerDependencies: - '@node-rs/xxhash' @@ -2373,7 +2380,7 @@ snapshots: character-entities@2.0.2: {} - chat@4.17.0: + chat@4.25.0: dependencies: '@workflow/serde': 4.1.0-beta.2 mdast-util-to-string: 4.0.0 @@ -2602,6 +2609,8 @@ snapshots: extend@3.0.2: {} + fake-indexeddb@6.2.5: {} + fast-deep-equal@3.1.3: {} fast-json-stable-stringify@2.1.0: {} diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..3e27ec1 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,174 @@ +import type { MatrixAdapterConfig, MatrixAuthConfig, MatrixAccessTokenAuthConfig, MatrixPersistenceConfig } from "./types"; +import { normalizeOptionalString } from "./shared/utils"; + +export const DEFAULT_COMMAND_PREFIX = "/"; +export const DEFAULT_PERSISTENCE_KEY_PREFIX = "matrix"; +export const DEFAULT_MATRIX_STORE_PERSIST_INTERVAL_MS = 30_000; +export const FAST_SYNC_DEFAULTS: NonNullable = { + initialSyncLimit: 1, + lazyLoadMembers: true, + disablePresence: true, + pollTimeout: 10_000, +}; + +export type SDKLogLevel = NonNullable; + +export type ResolvedPersistenceConfig = { + keyPrefix: string; + session: Pick, "decrypt" | "encrypt" | "ttlMs">; + sync: Required, "persistIntervalMs">> & + Pick, "snapshotTtlMs">; +}; + +export function validateConfig(config: MatrixAdapterConfig): void { + if (!config.baseURL?.trim()) { + throw new Error("baseURL is required."); + } + if (config.persistence?.session?.ttlMs !== undefined && config.persistence.session.ttlMs <= 0) { + throw new Error("persistence.session.ttlMs must be a positive number."); + } + if ( + config.persistence?.sync?.persistIntervalMs !== undefined && + config.persistence.sync.persistIntervalMs <= 0 + ) { + throw new Error("persistence.sync.persistIntervalMs must be a positive number."); + } + if ( + config.persistence?.sync?.snapshotTtlMs !== undefined && + config.persistence.sync.snapshotTtlMs <= 0 + ) { + throw new Error("persistence.sync.snapshotTtlMs must be a positive number."); + } + if ( + (config.persistence?.session?.encrypt && !config.persistence?.session?.decrypt) || + (!config.persistence?.session?.encrypt && config.persistence?.session?.decrypt) + ) { + throw new Error( + "persistence.session.encrypt and persistence.session.decrypt must be provided together." + ); + } +} + +export function normalizePersistenceConfig( + config?: MatrixPersistenceConfig +): ResolvedPersistenceConfig { + return { + keyPrefix: + normalizeOptionalString(config?.keyPrefix) ?? DEFAULT_PERSISTENCE_KEY_PREFIX, + session: { + decrypt: config?.session?.decrypt, + encrypt: config?.session?.encrypt, + ttlMs: config?.session?.ttlMs, + }, + sync: { + persistIntervalMs: + config?.sync?.persistIntervalMs ?? + DEFAULT_MATRIX_STORE_PERSIST_INTERVAL_MS, + snapshotTtlMs: config?.sync?.snapshotTtlMs, + }, + }; +} + +export function resolveAuthFromEnv(): MatrixAuthConfig { + const username = process.env.MATRIX_USERNAME; + const password = process.env.MATRIX_PASSWORD; + + if (username && password) { + return { + type: "password", + username, + password, + userID: process.env.MATRIX_USER_ID, + }; + } + + const accessToken = process.env.MATRIX_ACCESS_TOKEN; + const userID = process.env.MATRIX_USER_ID; + + if (!accessToken) { + throw new Error( + "Set MATRIX_USERNAME+MATRIX_PASSWORD for password auth, or MATRIX_ACCESS_TOKEN for access token auth." + ); + } + + const auth: MatrixAccessTokenAuthConfig = { + type: "accessToken", + accessToken, + userID, + }; + + return auth; +} + +function envBool(value: string | undefined, fallback = false): boolean { + if (!value) { + return fallback; + } + + const normalized = value.trim().toLowerCase(); + return ( + normalized === "1" || + normalized === "true" || + normalized === "yes" || + normalized === "on" + ); +} + +function parseEnvList(value: string | undefined): string[] { + if (!value) { + return []; + } + + return value + .split(",") + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); +} + +function isSDKLogLevel(value: string): value is SDKLogLevel { + return value === "trace" || + value === "debug" || + value === "info" || + value === "warn" || + value === "error"; +} + +export function parseSDKLogLevel(value: string | undefined): SDKLogLevel | undefined { + if (!value) { + return undefined; + } + const normalized = value.trim().toLowerCase(); + return isSDKLogLevel(normalized) ? normalized : undefined; +} + +export function createMatrixAdapterConfigFromEnv(): MatrixAdapterConfig { + const baseURL = process.env.MATRIX_BASE_URL; + if (!baseURL) { + throw new Error("baseURL is required. Set MATRIX_BASE_URL."); + } + + const recoveryKey = process.env.MATRIX_RECOVERY_KEY; + const inviteAutoJoinInviterAllowlist = parseEnvList( + process.env.MATRIX_INVITE_AUTOJOIN_ALLOWLIST + ); + const inviteAutoJoinEnabled = envBool( + process.env.MATRIX_INVITE_AUTOJOIN, + inviteAutoJoinInviterAllowlist.length > 0 + ); + + return { + baseURL, + auth: resolveAuthFromEnv(), + userName: process.env.MATRIX_BOT_USERNAME ?? "bot", + deviceID: normalizeOptionalString(process.env.MATRIX_DEVICE_ID), + commandPrefix: process.env.MATRIX_COMMAND_PREFIX, + recoveryKey, + inviteAutoJoin: inviteAutoJoinEnabled + ? { + inviterAllowlist: inviteAutoJoinInviterAllowlist, + } + : undefined, + matrixSDKLogLevel: + parseSDKLogLevel(process.env.MATRIX_SDK_LOG_LEVEL) ?? "error", + }; +} diff --git a/src/history/cursor.ts b/src/history/cursor.ts new file mode 100644 index 0000000..8457314 --- /dev/null +++ b/src/history/cursor.ts @@ -0,0 +1,116 @@ +import { Direction } from "matrix-js-sdk"; +import type { MatrixThreadID } from "../types"; +import { isRecord } from "../shared/utils"; + +export const MATRIX_PREFIX = "matrix"; +export const MATRIX_CURSOR_PREFIX = "mxv1:"; + +export type CursorKind = "room_messages" | "thread_relations" | "thread_list"; + +export type CursorDirection = "forward" | "backward"; + +export type CursorV1Payload = { + dir: CursorDirection; + kind: CursorKind; + roomID: string; + rootEventID?: string; + token: string; +}; + +export function encodeThreadId(platformData: MatrixThreadID): string { + const room = encodeURIComponent(platformData.roomID); + if (platformData.rootEventID) { + return `${MATRIX_PREFIX}:${room}:${encodeURIComponent(platformData.rootEventID)}`; + } + return `${MATRIX_PREFIX}:${room}`; +} + +export function decodeThreadId(threadId: string): MatrixThreadID { + const parts = threadId.split(":"); + if (parts.length < 2 || parts[0] !== MATRIX_PREFIX) { + throw new Error(`Invalid Matrix thread ID: ${threadId}`); + } + + const roomID = decodeURIComponent(parts[1]); + if (!roomID) { + throw new Error(`Invalid Matrix thread ID: ${threadId}`); + } + const rootEventID = parts[2] ? decodeURIComponent(parts[2]) : undefined; + + return { roomID, rootEventID }; +} + +export function channelIdFromThreadId(threadId: string): string { + const { roomID } = decodeThreadId(threadId); + return encodeThreadId({ roomID }); +} + +export function encodeCursorV1(payload: CursorV1Payload): string { + return `${MATRIX_CURSOR_PREFIX}${Buffer.from( + JSON.stringify(payload), + "utf8" + ).toString("base64url")}`; +} + +export function decodeCursorV1( + cursor: string, + expectedKind: CursorKind, + expectedRoomID: string, + expectedRootEventID?: string, + expectedDirection?: CursorDirection +): CursorV1Payload { + if (!cursor.startsWith(MATRIX_CURSOR_PREFIX)) { + throw new Error("Invalid cursor format. Expected mxv1 cursor."); + } + + let parsed: unknown; + try { + parsed = JSON.parse( + Buffer.from(cursor.slice(MATRIX_CURSOR_PREFIX.length), "base64url").toString("utf8") + ); + } catch (error) { + throw new Error(`Invalid cursor format. ${String(error)}`); + } + + if (!isRecord(parsed)) { + throw new Error("Invalid cursor format. Cursor payload must be an object."); + } + + if (parsed.kind !== expectedKind) { + throw new Error(`Invalid cursor kind. Expected ${expectedKind}.`); + } + if (parsed.roomID !== expectedRoomID) { + throw new Error("Invalid cursor context. Room mismatch."); + } + if (parsed.dir !== "forward" && parsed.dir !== "backward") { + throw new Error("Invalid cursor format. Invalid direction."); + } + if (expectedDirection && parsed.dir !== expectedDirection) { + throw new Error(`Invalid cursor direction. Expected ${expectedDirection}.`); + } + if (typeof parsed.token !== "string" || parsed.token.length === 0) { + throw new Error("Invalid cursor format. Missing token."); + } + + const rootEventID = + typeof parsed.rootEventID === "string" ? parsed.rootEventID : undefined; + if (expectedRootEventID) { + if (rootEventID !== expectedRootEventID) { + throw new Error("Invalid cursor context. Thread mismatch."); + } + } else if (rootEventID) { + throw new Error("Invalid cursor context. Unexpected thread scope."); + } + + return { + dir: parsed.dir, + kind: expectedKind, + roomID: expectedRoomID, + rootEventID, + token: parsed.token, + }; +} + +export function toSDKDirection(dir: CursorDirection): Direction { + return dir === "forward" ? Direction.Forward : Direction.Backward; +} diff --git a/src/index.test.ts b/src/index.test.ts index 4f98493..cc27915 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1,11 +1,13 @@ import { describe, expect, it, vi } from "vitest"; import { Chat, getEmoji, stringifyMarkdown } from "chat"; -import type { ChatInstance, Logger, StateAdapter } from "chat"; +import type { AdapterPostableMessage, ChatInstance, Logger, StateAdapter } from "chat"; import { createMemoryState } from "@chat-adapter/state-memory"; import { EventType, MsgType, RelationType, type MatrixClient } from "matrix-js-sdk"; import { MatrixError } from "matrix-js-sdk/lib/http-api/errors"; import { encodeRecoveryKey } from "matrix-js-sdk/lib/crypto-api/recovery-key"; import { createMatrixAdapter, MatrixAdapter } from "./index"; +import { isMentioned } from "./messages/inbound"; +import { splitOversizedTextContent } from "./messages/outbound"; type RawEventLike = { content?: Record; @@ -48,6 +50,9 @@ type RoomStateLike = { type RoomLike = { currentState: RoomStateLike; getMember: (userId: string) => MemberLike | null; + getThread: (eventId: string) => { length: number; replyToEvent: ReturnType | null } | null; + getThreads: () => Array<{ id: string }>; + processThreadRoots: (events: Array>, toStartOfTimeline: boolean) => void; roomId: string; name: string; timeline: unknown[]; @@ -272,6 +277,9 @@ function makeRoom(overrides: Partial = {}): RoomLike { getStateEvents: () => null, }, getMember: () => null, + getThread: () => null, + getThreads: () => [], + processThreadRoots: () => undefined, hasEncryptionStateEvent: () => false, getJoinedMembers: () => [{}, {}], getMyMembership: () => "join", @@ -310,6 +318,8 @@ function makeStateAdapter(initial: Record = {}): StateAdapter { return run(); }; const get: StateAdapter["get"] = (key) => afterReady(() => base.get(key)); + const getList: StateAdapter["getList"] = (key) => + afterReady(() => base.getList(key)); const set: StateAdapter["set"] = (key, value, ttlMs) => afterReady(() => base.set(key, value, ttlMs)); const setIfNotExists: StateAdapter["setIfNotExists"] = (key, value, ttlMs) => @@ -319,14 +329,26 @@ function makeStateAdapter(initial: Record = {}): StateAdapter { acquireLock: vi.fn((threadId, ttlMs) => afterReady(() => base.acquireLock(threadId, ttlMs)) ), + appendToList: vi.fn((key, value, options) => + afterReady(() => base.appendToList(key, value, options)) + ), connect: vi.fn(connect), delete: vi.fn((key) => afterReady(() => base.delete(key))), + dequeue: vi.fn((threadId) => afterReady(() => base.dequeue(threadId))), disconnect: vi.fn(() => afterReady(() => base.disconnect())), + enqueue: vi.fn((threadId, entry, maxSize) => + afterReady(() => base.enqueue(threadId, entry, maxSize)) + ), extendLock: vi.fn((lock, ttlMs) => afterReady(() => base.extendLock(lock, ttlMs)) ), + forceReleaseLock: vi.fn((threadId) => + afterReady(() => base.forceReleaseLock(threadId)) + ), get, + getList, isSubscribed: vi.fn((threadId) => afterReady(() => base.isSubscribed(threadId))), + queueDepth: vi.fn((threadId) => afterReady(() => base.queueDepth(threadId))), releaseLock: vi.fn((lock) => afterReady(() => base.releaseLock(lock))), set: vi.fn(set), setIfNotExists: vi.fn(setIfNotExists), @@ -435,6 +457,21 @@ describe("MatrixAdapter", () => { }); }); + it("rejects thread IDs with an empty room ID", () => { + const adapter = new MatrixAdapter({ + baseURL: "https://hs.beeper.com", + auth: { + type: "accessToken", + accessToken: "token", + userID: "@bot:beeper.com", + }, + }); + + expect(() => adapter.decodeThreadId("matrix:")).toThrow( + "Invalid Matrix thread ID: matrix:" + ); + }); + it("parses slash commands from timeline messages", async () => { const fakeClient = makeClient(); @@ -935,6 +972,62 @@ describe("MatrixAdapter", () => { }); }); + it("escapes HTML anchor text before re-emitting markdown links", async () => { + const fakeClient = makeClient(); + fakeClient.fetchRoomEvent = vi.fn(async () => + makeRawEvent({ + event_id: "$escaped-link", + content: { + body: "literal [text](https://example.com)", + msgtype: MsgType.Text, + format: "org.matrix.custom.html", + formatted_body: + '

Docs [v1] (draft) \\\\ notes

', + }, + }) + ); + + const adapter = await makeInitializedAdapter(fakeClient); + const message = await adapter.fetchMessage( + "matrix:!room%3Abeeper.com", + "$escaped-link" + ); + + expect(message).toBeTruthy(); + expect( + stringifyMarkdown(requireValue(message, "escaped link message").formatted).trim() + ).toBe("\\[Docs [v1\\] (draft) \\\\\\ notes](https://example.com)"); + }); + + it("ignores malformed matrix.to links instead of aborting formatted-body parsing", async () => { + const fakeClient = makeClient(); + fakeClient.fetchRoomEvent = vi.fn(async () => + makeRawEvent({ + event_id: "$bad-matrix-to", + content: { + body: "Broken mention link", + msgtype: MsgType.Text, + format: "org.matrix.custom.html", + formatted_body: + '

broken] still parsed

', + }, + }) + ); + + const adapter = await makeInitializedAdapter(fakeClient); + const message = await adapter.fetchMessage( + "matrix:!room%3Abeeper.com", + "$bad-matrix-to" + ); + + expect(message).toBeTruthy(); + expect(message?.text).toBe("broken] still parsed"); + expect( + stringifyMarkdown(requireValue(message, "bad matrix.to message").formatted).trim() + ).toBe("[broken\\]](https://matrix.to/#/%E0%A4%A) **still parsed**"); + expect(message?.isMention).toBe(false); + }); + it("strips Matrix reply fallback from plain body text", async () => { const fakeClient = makeClient(); fakeClient.fetchRoomEvent = vi.fn(async () => @@ -1069,6 +1162,53 @@ describe("MatrixAdapter", () => { expect(channel.metadata).toMatchObject(thread.metadata ?? {}); }); + it("adds reply fallback metadata when posting to a Matrix thread", async () => { + const fakeClient = makeClient(); + const adapter = await makeInitializedAdapter(fakeClient); + + await adapter.postMessage("matrix:!room%3Abeeper.com:%24root", "hello"); + + expect(fakeClient.sendEvent).toHaveBeenCalledWith( + "!room:beeper.com", + "$root", + EventType.RoomMessage, + expect.objectContaining({ + body: "hello", + msgtype: MsgType.Text, + "m.relates_to": { + "m.in_reply_to": { + event_id: "$root", + }, + }, + }) + ); + }); + + it("uses an explicit reply target when posting within a Matrix thread", async () => { + const fakeClient = makeClient(); + const adapter = await makeInitializedAdapter(fakeClient); + + await adapter.postMessage("matrix:!room%3Abeeper.com:%24root", { + raw: "hello", + matrixReplyToEventId: "$reply", + } as AdapterPostableMessage); + + expect(fakeClient.sendEvent).toHaveBeenCalledWith( + "!room:beeper.com", + "$root", + EventType.RoomMessage, + expect.objectContaining({ + body: "hello", + msgtype: MsgType.Text, + "m.relates_to": { + "m.in_reply_to": { + event_id: "$reply", + }, + }, + }) + ); + }); + it("uploads file payloads and posts Matrix media events", async () => { const fakeClient = makeClient(); fakeClient.sendEvent = vi @@ -1120,6 +1260,43 @@ describe("MatrixAdapter", () => { ); }); + it("omits reply fallback metadata for follow-up attachments in Matrix threads", async () => { + const fakeClient = makeClient(); + fakeClient.sendEvent = vi + .fn() + .mockResolvedValueOnce({ event_id: "$text" }) + .mockResolvedValueOnce({ event_id: "$file" }); + fakeClient.uploadContent = vi + .fn() + .mockResolvedValueOnce({ content_uri: "mxc://beeper.com/file-1" }); + + const adapter = await makeInitializedAdapter(fakeClient); + + await adapter.postMessage("matrix:!room%3Abeeper.com:%24root", { + markdown: "File incoming", + files: [ + { + data: new Uint8Array([1, 2, 3]).buffer, + filename: "report.png", + mimeType: "image/png", + }, + ], + }); + + const sendCalls = fakeClient.sendEvent.mock.calls as unknown as Array; + const firstContent = sendCalls[0]?.[3] as Record; + const secondContent = sendCalls[1]?.[3] as Record; + + expect(firstContent).toMatchObject({ + "m.relates_to": { + "m.in_reply_to": { + event_id: "$root", + }, + }, + }); + expect(secondContent).not.toHaveProperty("m.relates_to"); + }); + it("skips invalid file uploads instead of passing malformed entries downstream", async () => { const fakeClient = makeClient(); const logger = makeTestLogger(); @@ -1199,6 +1376,182 @@ describe("MatrixAdapter", () => { ); }); + it("retries oversized rich-text events as split plain-text messages", async () => { + const fakeClient = makeClient(); + const logger = makeTestLogger(); + const tooLargeError = new MatrixError( + { errcode: "M_TOO_LARGE", error: "event too large" }, + 413 + ); + fakeClient.getRoom.mockReturnValue( + makeRoom({ + findEventById: () => null, + }) + ); + fakeClient.sendEvent = vi + .fn() + .mockRejectedValueOnce(tooLargeError) + .mockResolvedValueOnce({ event_id: "$chunk-1" }) + .mockResolvedValueOnce({ event_id: "$chunk-2" }); + + const adapter = new MatrixAdapter({ + baseURL: "https://hs.beeper.com", + auth: { + type: "accessToken", + accessToken: "token", + userID: "@bot:beeper.com", + }, + logger: logger.logger, + createClient: () => asMatrixClient(fakeClient), + }); + await adapter.initialize(makeChatInstance()); + + const repeatedMarkdown = Array.from({ length: 2_000 }, () => "**hello** `world`").join("\n"); + const sent = await adapter.postMessage("matrix:!room%3Abeeper.com", { + markdown: repeatedMarkdown, + }); + + expect(sent.id).toBe("$chunk-1"); + expect(fakeClient.sendEvent).toHaveBeenCalledTimes(3); + expect(fakeClient.sendEvent).toHaveBeenNthCalledWith( + 1, + "!room:beeper.com", + EventType.RoomMessage, + expect.objectContaining({ + format: "org.matrix.custom.html", + formatted_body: expect.any(String), + }) + ); + + const fallbackBodies = (fakeClient.sendEvent.mock.calls as unknown as Array) + .slice(1) + .map((call) => call[2] as unknown as Record); + expect(fallbackBodies).toHaveLength(2); + expect(fallbackBodies.map((content) => content.body).join("")) + .toBe(Array.from({ length: 2_000 }, () => "hello world").join("\n")); + for (const content of fallbackBodies) { + expect(content).toMatchObject({ msgtype: MsgType.Text }); + expect(content).not.toHaveProperty("format"); + expect(content).not.toHaveProperty("formatted_body"); + expect(content).not.toHaveProperty("m.mentions"); + } + expect(sent.raw.getContent()).toMatchObject(fallbackBodies[0] ?? {}); + + expect(logger.warn).toHaveBeenCalledWith( + "Matrix message exceeded size limit; retrying as split plain-text chunks", + expect.objectContaining({ + roomId: "!room:beeper.com", + chunkCount: 2, + msgtype: MsgType.Text, + }) + ); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it("retries oversized rich-text events as a single plain-text fallback when body fits", async () => { + const fakeClient = makeClient(); + const logger = makeTestLogger(); + const tooLargeError = new MatrixError( + { errcode: "M_TOO_LARGE", error: "event too large" }, + 413 + ); + fakeClient.getRoom.mockReturnValue( + makeRoom({ + findEventById: () => null, + }) + ); + fakeClient.sendEvent = vi + .fn() + .mockRejectedValueOnce(tooLargeError) + .mockResolvedValueOnce({ event_id: "$plain-fallback" }); + + const adapter = new MatrixAdapter({ + baseURL: "https://hs.beeper.com", + auth: { + type: "accessToken", + accessToken: "token", + userID: "@bot:beeper.com", + }, + logger: logger.logger, + createClient: () => asMatrixClient(fakeClient), + }); + await adapter.initialize(makeChatInstance()); + + const sent = await adapter.postMessage("matrix:!room%3Abeeper.com", { + markdown: "**hello**", + }); + + expect(sent.id).toBe("$plain-fallback"); + expect(fakeClient.sendEvent).toHaveBeenCalledTimes(2); + expect(fakeClient.sendEvent).toHaveBeenNthCalledWith( + 2, + "!room:beeper.com", + EventType.RoomMessage, + expect.objectContaining({ + body: "hello", + msgtype: MsgType.Text, + }) + ); + + const fallbackCalls = fakeClient.sendEvent.mock.calls as unknown as Array; + const fallbackContent = fallbackCalls[1]?.[2] as Record | undefined; + expect(fallbackContent).not.toHaveProperty("format"); + expect(fallbackContent).not.toHaveProperty("formatted_body"); + expect(fallbackContent).not.toHaveProperty("m.mentions"); + expect(sent.raw.getContent()).toMatchObject(fallbackContent ?? {}); + + expect(logger.warn).toHaveBeenCalledWith( + "Matrix message exceeded size limit; retrying as split plain-text chunks", + expect.objectContaining({ + roomId: "!room:beeper.com", + chunkCount: 1, + msgtype: MsgType.Text, + }) + ); + }); + + it("rethrows M_TOO_LARGE for non-text Matrix events", async () => { + const fakeClient = makeClient(); + const logger = makeTestLogger(); + const tooLargeError = new MatrixError( + { errcode: "M_TOO_LARGE", error: "event too large" }, + 413 + ); + fakeClient.sendEvent = vi.fn().mockRejectedValue(tooLargeError); + + const adapter = new MatrixAdapter({ + baseURL: "https://hs.beeper.com", + auth: { + type: "accessToken", + accessToken: "token", + userID: "@bot:beeper.com", + }, + logger: logger.logger, + createClient: () => asMatrixClient(fakeClient), + }); + await adapter.initialize(makeChatInstance()); + + await expect( + adapter.postMessage("matrix:!room%3Abeeper.com", { + files: [ + { + data: new Uint8Array([1, 2, 3]).buffer, + filename: "report.png", + mimeType: "image/png", + }, + ], + } as AdapterPostableMessage) + ).rejects.toBe(tooLargeError); + expect(logger.error).toHaveBeenCalledWith( + "Matrix send message failed", + expect.objectContaining({ + roomId: "!room:beeper.com", + msgtype: MsgType.Image, + error: tooLargeError, + }) + ); + }); + it("logs and rethrows Matrix upload failures", async () => { const fakeClient = makeClient(); const logger = makeTestLogger(); @@ -1277,6 +1630,54 @@ describe("MatrixAdapter", () => { ); }); + it("splits oversized text without cutting surrogate pairs", () => { + const content = splitOversizedTextContent({ + body: "😀".repeat(4_000), + msgtype: MsgType.Text, + }); + + expect(content.length).toBeGreaterThan(1); + expect(content.map((part) => part.body).join("")).toBe("😀".repeat(4_000)); + + for (const part of content) { + const body = part.body; + const lastCodeUnit = body.charCodeAt(body.length - 1); + const firstCodeUnit = body.charCodeAt(0); + expect(lastCodeUnit < 0xd800 || lastCodeUnit > 0xdbff).toBe(true); + expect(firstCodeUnit < 0xdc00 || firstCodeUnit > 0xdfff).toBe(true); + } + }); + + it("preserves boundary whitespace when splitting oversized text", () => { + const text = `${"a".repeat(11_999)} ${"b".repeat(32)}`; + const content = splitOversizedTextContent({ + body: text, + msgtype: MsgType.Text, + }); + + expect(content.length).toBeGreaterThan(1); + expect(content.map((part) => part.body).join("")).toBe(text); + }); + + it("does not treat an empty username as a mention", () => { + expect( + isMentioned({ + content: { + body: "@someone said hi", + msgtype: MsgType.Text, + }, + parsed: { + markdown: "@someone said hi", + mentionedUserIDs: new Set(), + mentionsRoom: false, + text: "@someone said hi", + }, + userID: "", + userName: "", + }) + ).toBe(false); + }); + it("generates and persists a device id when one is not provided", async () => { const adapter = getInternals( new MatrixAdapter({ @@ -2351,4 +2752,176 @@ describe("MatrixAdapter", () => { dir: "backward", }); }); + + it("prefers live room thread metadata when bundled thread counts lag", async () => { + const fakeClient = makeClient(); + fakeClient.createThreadListMessagesRequest.mockResolvedValue({ + chunk: [ + makeRawEvent({ + event_id: "$root1", + origin_server_ts: 1_700_000_000_100, + content: { body: "Root 1" }, + unsigned: { + "m.relations": { + "m.thread": { + count: 0, + latest_event: { origin_server_ts: 1_700_000_000_500 }, + }, + }, + }, + }), + ], + end: undefined, + }); + + const latestReply = makeEvent({ + getId: () => "$reply1", + getTs: () => 1_700_000_000_900, + }); + fakeClient.getRoom.mockReturnValue( + makeRoom({ + getThread: (eventId: string) => + eventId === "$root1" + ? { + length: 2, + replyToEvent: latestReply, + } + : null, + }) + ); + + const adapter = await makeInitializedAdapter(fakeClient); + + const result = await adapter.listThreads("matrix:!room%3Abeeper.com", { limit: 2 }); + + expect(result.threads).toHaveLength(1); + expect(result.threads[0]?.replyCount).toBe(2); + expect(result.threads[0]?.lastReplyAt?.toISOString()).toBe( + "2023-11-14T22:13:20.900Z" + ); + }); + + it("hydrates room thread metadata from thread-list roots before summarizing", async () => { + const fakeClient = makeClient(); + fakeClient.createThreadListMessagesRequest.mockResolvedValue({ + chunk: [ + makeRawEvent({ + event_id: "$root1", + origin_server_ts: 1_700_000_000_100, + content: { body: "Root 1" }, + unsigned: { + "m.relations": { + "m.thread": { + count: 0, + latest_event: { origin_server_ts: 1_700_000_000_500 }, + }, + }, + }, + }), + ], + end: undefined, + }); + + const latestReply = makeEvent({ + getId: () => "$reply1", + getTs: () => 1_700_000_000_900, + }); + let hydrated = false; + const processThreadRoots = vi.fn( + (events: Array>, toStartOfTimeline: boolean) => { + expect(events.map((event) => event.getId())).toEqual(["$root1"]); + expect(toStartOfTimeline).toBe(true); + hydrated = true; + } + ); + fakeClient.getRoom.mockReturnValue( + makeRoom({ + processThreadRoots, + getThread: (eventId: string) => + hydrated && eventId === "$root1" + ? { + length: 1, + replyToEvent: latestReply, + } + : null, + }) + ); + + const adapter = await makeInitializedAdapter(fakeClient); + + const result = await adapter.listThreads("matrix:!room%3Abeeper.com", { limit: 2 }); + + expect(processThreadRoots).toHaveBeenCalledOnce(); + expect(result.threads).toHaveLength(1); + expect(result.threads[0]?.replyCount).toBe(1); + expect(result.threads[0]?.lastReplyAt?.toISOString()).toBe( + "2023-11-14T22:13:20.900Z" + ); + }); + + it("falls back to fetching the latest thread reply when thread summaries are missing", async () => { + const fakeClient = makeClient(); + fakeClient.createThreadListMessagesRequest.mockResolvedValue({ + chunk: [ + makeRawEvent({ + event_id: "$root1", + origin_server_ts: 1_700_000_000_100, + content: { body: "Root 1" }, + unsigned: {}, + }), + ], + end: undefined, + }); + fakeClient.relations.mockResolvedValue({ + originalEvent: null, + events: [ + makeEvent({ + getId: () => "$reply1", + getTs: () => 1_700_000_000_900, + getContent: () => ({ body: "Reply 1" }), + getRelation: () => ({ + rel_type: RelationType.Thread, + event_id: "$root1", + }), + isRelation: (relType?: string) => relType === RelationType.Thread, + threadRootId: "$root1", + }), + ], + nextBatch: null, + prevBatch: null, + }); + fakeClient.getRoom.mockReturnValue( + makeRoom({ + findEventById: (eventId?: string) => + eventId === "$root1" + ? makeEvent({ + getId: () => "$root1", + getTs: () => 1_700_000_000_100, + getContent: () => ({ body: "Root 1" }), + }) + : null, + }) + ); + + const adapter = await makeInitializedAdapter(fakeClient); + + const result = await adapter.listThreads("matrix:!room%3Abeeper.com", { limit: 2 }); + + expect(fakeClient.relations).toHaveBeenCalledWith( + "!room:beeper.com", + "$root1", + "m.thread", + null, + { + dir: "b", + from: undefined, + limit: 1, + } + ); + expect(result.threads).toHaveLength(1); + expect(result.threads[0]?.replyCount).toBe(1); + expect(result.threads[0]?.lastReplyAt?.toISOString()).toBe( + "2023-11-14T22:13:20.900Z" + ); + }); }); diff --git a/src/index.ts b/src/index.ts index f22131d..042be70 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,3 @@ -import { randomBytes } from "node:crypto"; import type { Adapter, AdapterPostableMessage, @@ -6,7 +5,6 @@ import type { ChannelInfo, ChatInstance, EmojiValue, - FileUpload, FetchOptions, FetchResult, FormattedContent, @@ -22,9 +20,7 @@ import type { import { ConsoleLogger, getEmoji, - isCardElement, Message, - markdownToPlainText, parseMarkdown, stringifyMarkdown, } from "chat"; @@ -49,44 +45,80 @@ import sdk, { import type { IStore } from "matrix-js-sdk/lib/store"; import type { RoomMessageEventContent, - RoomMessageTextEventContent, } from "matrix-js-sdk/lib/@types/events"; import type { MediaEventContent } from "matrix-js-sdk/lib/@types/media"; import { MatrixError } from "matrix-js-sdk/lib/http-api/errors"; import { decodeRecoveryKey } from "matrix-js-sdk/lib/crypto-api/recovery-key"; import { logger as matrixSDKLogger } from "matrix-js-sdk/lib/logger"; -import { marked } from "marked"; import { - HTMLElement, - NodeType, - parse as parseHTML, - type Node as HTMLNode, -} from "node-html-parser"; + DEFAULT_COMMAND_PREFIX, + FAST_SYNC_DEFAULTS, + type ResolvedPersistenceConfig, + type SDKLogLevel, + createMatrixAdapterConfigFromEnv, + normalizePersistenceConfig, + validateConfig, +} from "./config"; +import { + channelIdFromThreadId, + decodeCursorV1, + decodeThreadId, + encodeCursorV1, + encodeThreadId, + toSDKDirection, + type CursorDirection, + type CursorKind, + type CursorV1Payload, +} from "./history/cursor"; +import { + isMentioned, + parseMatrixContent, + type MatrixMessageContent, + type ParsedMatrixContent, +} from "./messages/inbound"; +import { + applyThreadReplyMetadata, + binarySize, + collectLinkOnlyAttachmentLines, + defaultAttachmentName, + extractAttachmentsFromMessage, + extractFilesFromMessage, + extractReplyEventID, + isTooLargeMatrixError, + mergeTextAndLinks, + messageTypeForAttachment, + messageTypeForMimeType, + normalizeFileUpload, + normalizeUploadData, + readAttachmentData, + splitOversizedTextContent, + toRoomMessageContent, + type MatrixMediaMsgType, + type MatrixOutboundMessageContent, + type MatrixRoomMessageContent, + type MatrixTextMessageContent, + type OutboundUpload, +} from "./messages/outbound"; +import { + evictOldestEntries, + generateDeviceID, + hasIndexedDB, + isRecord, + matrixLocalpart, + normalizeOptionalString, + normalizeStringList, + readStringValue, +} from "./shared/utils"; import { ChatStateMatrixStore } from "./store/chat-state-matrix-store"; import type { MatrixAuthBootstrapClient, - MatrixAccessTokenAuthConfig, MatrixAdapterConfig, MatrixAuthConfig, MatrixCreateStoreOptions, - MatrixPersistenceConfig, - MatrixPersistenceSyncConfig, MatrixThreadID, } from "./types"; -const MATRIX_PREFIX = "matrix"; -const MATRIX_CURSOR_PREFIX = "mxv1:"; -const DEFAULT_COMMAND_PREFIX = "/"; -const DEFAULT_PERSISTENCE_KEY_PREFIX = "matrix"; const TYPING_TIMEOUT_MS = 30_000; -const DEFAULT_MATRIX_STORE_PERSIST_INTERVAL_MS = 30_000; -const FAST_SYNC_DEFAULTS: NonNullable = { - initialSyncLimit: 1, - lazyLoadMembers: true, - disablePresence: true, - pollTimeout: 10_000, -}; -type SDKLogLevel = NonNullable; const MATRIX_SDK_LOG_LEVELS: Record = { trace: 0, debug: 1, @@ -96,31 +128,6 @@ const MATRIX_SDK_LOG_LEVELS: Record = { }; let matrixSDKLogConfigured = false; -type MatrixMessageContent = { - body?: string; - format?: string; - formatted_body?: string; - msgtype?: string; - "m.mentions"?: { - room?: boolean; - user_ids?: string[]; - }; - [key: string]: unknown; -}; - -type MatrixTextMessageContent = RoomMessageTextEventContent & { - "com.beeper.dont_render_edited"?: boolean; -}; - -type MatrixRoomMessageContent = RoomMessageEventContent & { - "com.beeper.dont_render_edited"?: boolean; - "m.new_content"?: RoomMessageEventContent & { - "com.beeper.dont_render_edited"?: boolean; - }; -}; - -type MatrixOutboundMessageContent = MatrixRoomMessageContent | MediaEventContent; - type StoredReaction = { emoji: EmojiValue; messageID: string; @@ -136,13 +143,6 @@ type ResolvedAuth = { userID: string; }; -type ResolvedPersistenceConfig = { - keyPrefix: string; - session: Pick, "decrypt" | "encrypt" | "ttlMs">; - sync: Required, "persistIntervalMs">> & - Pick, "snapshotTtlMs">; -}; - type StoredSession = { accessToken?: string; authType: MatrixAuthConfig["type"]; @@ -157,55 +157,8 @@ type StoredSession = { username?: string; }; -type CursorKind = "room_messages" | "thread_relations" | "thread_list"; - -type CursorDirection = "forward" | "backward"; - -type CursorV1Payload = { - dir: CursorDirection; - kind: CursorKind; - roomID: string; - rootEventID?: string; - token: string; -}; - type DirectAccountData = Record; -type OutboundUpload = { - data: Blob; - fileName: string; - info?: { - h?: number; - mimetype?: string; - size?: number; - w?: number; - }; - msgtype: MatrixMediaMsgType; - type?: string; -}; - -type MatrixMediaMsgType = - | MsgType.Audio - | MsgType.File - | MsgType.Image - | MsgType.Video; - -type ParsedMatrixContent = { - markdown: string; - mentionsRoom: boolean; - mentionedUserIDs: Set; - text: string; -}; - -type RenderedMatrixMessage = { - body: string; - formattedBody?: string; - mentions?: { - room?: boolean; - user_ids?: string[]; - }; -}; - type MatrixRoomMetadata = { avatarURL?: string; canonicalAlias?: string; @@ -217,6 +170,11 @@ type MatrixRoomMetadata = { topic?: string; }; +type SentRoomMessage = { + response: Awaited>; + sentContent: MatrixOutboundMessageContent; +}; + // Intentionally unsupported in this adapter: postEphemeral, openModal, and native stream. export class MatrixAdapter implements Adapter { readonly name = "matrix"; @@ -256,7 +214,7 @@ export class MatrixAdapter implements Adapter { private shuttingDown = false; constructor(config: MatrixAdapterConfig) { - this.validateConfig(config); + validateConfig(config); this.baseURL = config.baseURL; this.auth = config.auth; this.userID = config.auth.userID ?? ""; @@ -383,28 +341,15 @@ export class MatrixAdapter implements Adapter { } encodeThreadId(platformData: MatrixThreadID): string { - const room = encodeURIComponent(platformData.roomID); - if (platformData.rootEventID) { - return `${MATRIX_PREFIX}:${room}:${encodeURIComponent(platformData.rootEventID)}`; - } - return `${MATRIX_PREFIX}:${room}`; + return encodeThreadId(platformData); } decodeThreadId(threadId: string): MatrixThreadID { - const parts = threadId.split(":"); - if (parts.length < 2 || parts[0] !== MATRIX_PREFIX) { - throw new Error(`Invalid Matrix thread ID: ${threadId}`); - } - - const roomID = decodeURIComponent(parts[1]); - const rootEventID = parts[2] ? decodeURIComponent(parts[2]) : undefined; - - return { roomID, rootEventID }; + return decodeThreadId(threadId); } channelIdFromThreadId(threadId: string): string { - const { roomID } = this.decodeThreadId(threadId); - return this.encodeThreadId({ roomID }); + return channelIdFromThreadId(threadId); } renderFormatted(content: FormattedContent): string { @@ -420,21 +365,27 @@ export class MatrixAdapter implements Adapter { message: AdapterPostableMessage ): Promise> { const { roomID, rootEventID } = this.decodeThreadId(threadId); + const replyEventID = extractReplyEventID(message); const contents = await this.toRoomMessageContents(message); const [firstContent, ...extraContents] = contents; if (!firstContent) { throw new Error("Cannot post an empty Matrix message."); } - const response = await this.sendRoomMessage(roomID, rootEventID, firstContent); + const { response, sentContent } = await this.sendRoomMessage( + roomID, + rootEventID, + replyEventID, + firstContent, + ); for (const content of extraContents) { - await this.sendRoomMessage(roomID, rootEventID, content); + await this.sendRoomMessage(roomID, rootEventID, null, content); } return { id: response.event_id, threadId, raw: this.resolveSentEvent(roomID, response.event_id, { - content: firstContent, + content: sentContent, roomID, sender: this.userID, }), @@ -455,7 +406,7 @@ export class MatrixAdapter implements Adapter { message: AdapterPostableMessage ): Promise> { const { roomID, rootEventID } = this.decodeThreadId(threadId); - const baseContent = this.toRoomMessageContent(message); + const baseContent = toRoomMessageContent(message); const newContent: MatrixTextMessageContent = { ...baseContent, "com.beeper.dont_render_edited": true, @@ -472,13 +423,18 @@ export class MatrixAdapter implements Adapter { body: `* ${baseContent.body}`, }; - const response = await this.sendRoomMessage(roomID, rootEventID, editContent); + const { response, sentContent } = await this.sendRoomMessage( + roomID, + rootEventID, + null, + editContent + ); return { id: response.event_id, threadId, raw: this.resolveSentEvent(roomID, response.event_id, { - content: editContent, + content: sentContent, roomID, sender: this.userID, }), @@ -613,7 +569,7 @@ export class MatrixAdapter implements Adapter { const direction = options.direction ?? "backward"; const limit = options.limit ?? 50; const cursor = options.cursor - ? this.decodeCursorV1( + ? decodeCursorV1( options.cursor, rootEventID ? "thread_relations" : "room_messages", roomID, @@ -634,7 +590,7 @@ export class MatrixAdapter implements Adapter { return { messages: response.events.map((event) => this.parseMessageInternal(event)), nextCursor: response.nextToken - ? this.encodeCursorV1({ + ? encodeCursorV1({ kind: "room_messages", dir: direction, token: response.nextToken, @@ -659,7 +615,7 @@ export class MatrixAdapter implements Adapter { this.parseMessageInternal(event, this.encodeThreadId({ roomID, rootEventID })) ), nextCursor: response.nextToken - ? this.encodeCursorV1({ + ? encodeCursorV1({ kind: "thread_relations", dir: direction, token: response.nextToken, @@ -734,9 +690,10 @@ export class MatrixAdapter implements Adapter { options: ListThreadsOptions = {} ): Promise> { const roomID = this.decodeThreadId(channelId).roomID; + const room = this.requireRoom(roomID); const limit = options.limit ?? 50; const cursor = options.cursor - ? this.decodeCursorV1(options.cursor, "thread_list", roomID, undefined, "backward") + ? decodeCursorV1(options.cursor, "thread_list", roomID, undefined, "backward") : null; const listResponse = await this.requireClient().createThreadListMessagesRequest( roomID, @@ -746,6 +703,7 @@ export class MatrixAdapter implements Adapter { ThreadFilterType.All ); const events = await this.mapRawEvents(listResponse.chunk ?? [], roomID); + room.processThreadRoots(events, true); const summaries: ThreadSummary[] = []; for (const rootEvent of events) { @@ -754,16 +712,32 @@ export class MatrixAdapter implements Adapter { continue; } - const bundled = rootEvent.getServerAggregatedRelation( - THREAD_RELATION_TYPE.name - ); - const latestTS = bundled?.latest_event?.origin_server_ts; + const localRootEvent = room.findEventById(rootID); + const summaryRootEvent = localRootEvent ?? rootEvent; + const bundled = + summaryRootEvent.getServerAggregatedRelation( + THREAD_RELATION_TYPE.name + ) ?? + rootEvent.getServerAggregatedRelation( + THREAD_RELATION_TYPE.name + ); + const roomThread = room.getThread(rootID); + const latestReply = roomThread?.replyToEvent; + let latestTS = latestReply?.getTs() ?? bundled?.latest_event?.origin_server_ts; + let replyCount = Math.max(roomThread?.length ?? 0, bundled?.count ?? 0); + + if (replyCount === 0 && typeof latestTS !== "number") { + const fallback = await this.fetchLatestThreadReplySummary(roomID, rootID); + replyCount = Math.max(replyCount, fallback.replyCount); + latestTS = latestTS ?? fallback.latestReplyTS; + } + const threadID = this.encodeThreadId({ roomID, rootEventID: rootID }); summaries.push({ id: threadID, - rootMessage: this.parseMessageInternal(rootEvent, threadID), - replyCount: bundled?.count ?? 0, + rootMessage: this.parseMessageInternal(summaryRootEvent, threadID), + replyCount, lastReplyAt: typeof latestTS === "number" ? new Date(latestTS) : undefined, }); } @@ -771,7 +745,7 @@ export class MatrixAdapter implements Adapter { return { threads: summaries, nextCursor: listResponse.end - ? this.encodeCursorV1({ + ? encodeCursorV1({ kind: "thread_list", dir: "backward", token: listResponse.end, @@ -781,6 +755,28 @@ export class MatrixAdapter implements Adapter { }; } + private async fetchLatestThreadReplySummary( + roomID: string, + rootEventID: string + ): Promise<{ replyCount: number; latestReplyTS?: number }> { + const response = await this.fetchThreadMessagesPage({ + roomID, + rootEventID, + includeRoot: false, + limit: 1, + direction: "backward", + fromToken: null, + }); + const latestReply = response.events.at(-1); + + return latestReply + ? { + replyCount: 1, + latestReplyTS: latestReply.getTs(), + } + : { replyCount: 0 }; + } + private parseMessageInternal( raw: MatrixEvent, overrideThreadID?: string @@ -794,7 +790,7 @@ export class MatrixAdapter implements Adapter { const content = raw.getContent(); const edited = this.extractEditedContent(raw); const effectiveContent = edited?.content ?? content; - const parsed = this.parseMatrixContent(effectiveContent); + const parsed = parseMatrixContent(effectiveContent); const sender = raw.getSender() ?? "unknown"; return new Message({ @@ -810,80 +806,15 @@ export class MatrixAdapter implements Adapter { }, attachments: this.extractAttachments(effectiveContent), raw, - isMention: this.isMentioned(effectiveContent, parsed), + isMention: isMentioned({ + content: effectiveContent, + parsed, + userID: this.userID, + userName: this.userName, + }), }); } - private encodeCursorV1(payload: CursorV1Payload): string { - return `${MATRIX_CURSOR_PREFIX}${Buffer.from( - JSON.stringify(payload), - "utf8" - ).toString("base64url")}`; - } - - private decodeCursorV1( - cursor: string, - expectedKind: CursorKind, - expectedRoomID: string, - expectedRootEventID?: string, - expectedDirection?: CursorDirection - ): CursorV1Payload { - if (!cursor.startsWith(MATRIX_CURSOR_PREFIX)) { - throw new Error("Invalid cursor format. Expected mxv1 cursor."); - } - - let parsed: unknown; - try { - parsed = JSON.parse( - Buffer.from(cursor.slice(MATRIX_CURSOR_PREFIX.length), "base64url").toString("utf8") - ); - } catch (error) { - throw new Error(`Invalid cursor format. ${String(error)}`); - } - - if (!isRecord(parsed)) { - throw new Error("Invalid cursor format. Cursor payload must be an object."); - } - - if (parsed.kind !== expectedKind) { - throw new Error(`Invalid cursor kind. Expected ${expectedKind}.`); - } - if (parsed.roomID !== expectedRoomID) { - throw new Error("Invalid cursor context. Room mismatch."); - } - if (parsed.dir !== "forward" && parsed.dir !== "backward") { - throw new Error("Invalid cursor format. Invalid direction."); - } - if (expectedDirection && parsed.dir !== expectedDirection) { - throw new Error(`Invalid cursor direction. Expected ${expectedDirection}.`); - } - if (typeof parsed.token !== "string" || parsed.token.length === 0) { - throw new Error("Invalid cursor format. Missing token."); - } - - const rootEventID = - typeof parsed.rootEventID === "string" ? parsed.rootEventID : undefined; - if (expectedRootEventID) { - if (rootEventID !== expectedRootEventID) { - throw new Error("Invalid cursor context. Thread mismatch."); - } - } else if (rootEventID) { - throw new Error("Invalid cursor context. Unexpected thread scope."); - } - - return { - dir: parsed.dir, - kind: expectedKind, - roomID: expectedRoomID, - rootEventID, - token: parsed.token, - }; - } - - private toSDKDirection(dir: CursorDirection): Direction { - return dir === "forward" ? Direction.Forward : Direction.Backward; - } - private async fetchRoomMessagesPage(args: { roomID: string; includeThreadReplies: boolean; @@ -895,7 +826,7 @@ export class MatrixAdapter implements Adapter { args.roomID, args.fromToken, args.limit, - this.toSDKDirection(args.direction) + toSDKDirection(args.direction) ); const messageChunk = (response.chunk ?? []).filter( (raw) => @@ -942,7 +873,7 @@ export class MatrixAdapter implements Adapter { THREAD_RELATION_TYPE.name, null, { - dir: this.toSDKDirection(args.direction), + dir: toSDKDirection(args.direction), from: args.fromToken ?? undefined, limit: relationLimit, } @@ -1290,7 +1221,7 @@ export class MatrixAdapter implements Adapter { this.auth.password ); - let userID = this.auth.userID ?? loginResponse.user_id; + const userID = this.auth.userID ?? loginResponse.user_id; if (!userID) { throw new Error("Password login succeeded but no user ID was returned."); } @@ -1298,7 +1229,6 @@ export class MatrixAdapter implements Adapter { let authDeviceID = normalizeOptionalString(loginResponse.device_id); if (!authDeviceID) { const whoami = await this.lookupWhoAmIFromAccessToken(loginResponse.access_token); - userID = userID ?? whoami.userID; authDeviceID = whoami.deviceID; } @@ -1504,37 +1434,73 @@ export class MatrixAdapter implements Adapter { private async sendRoomMessage( roomID: string, rootEventID: string | undefined, + replyEventID: string | null | undefined, content: MatrixOutboundMessageContent - ) { - const response = await this.withLoggedMatrixOperation( - "Matrix send message failed", - { + ): Promise { + const threadContent = applyThreadReplyMetadata(content, rootEventID, replyEventID); + + try { + const response = await this.performSendRoomMessage(roomID, rootEventID, threadContent); + void this.maybePersistSecretsBundle(); + return { + response, + sentContent: threadContent, + }; + } catch (error) { + if (isTooLargeMatrixError(error)) { + const splitContents = splitOversizedTextContent(threadContent); + if (splitContents.length > 0) { + this.logger.warn( + "Matrix message exceeded size limit; retrying as split plain-text chunks", + { + roomId: roomID, + rootEventId: rootEventID, + originalLength: typeof threadContent.body === "string" ? threadContent.body.length : undefined, + chunkCount: splitContents.length, + msgtype: threadContent.msgtype, + } + ); + + let firstSentMessage: SentRoomMessage | undefined; + for (const splitContent of splitContents) { + const chunkWithMeta = applyThreadReplyMetadata( + splitContent, + rootEventID, + replyEventID + ); + const response = await this.performSendRoomMessage(roomID, rootEventID, chunkWithMeta); + void this.maybePersistSecretsBundle(); + firstSentMessage ??= { + response, + sentContent: chunkWithMeta, + }; + } + + if (firstSentMessage) { + return firstSentMessage; + } + } + } + + this.logger.error("Matrix send message failed", { roomId: roomID, rootEventId: rootEventID, eventType: EventType.RoomMessage, msgtype: content.msgtype, - }, - async () => { - const client = this.requireClient(); - if (rootEventID) { - return client.sendEvent(roomID, rootEventID, EventType.RoomMessage, content); - } - - return client.sendEvent(roomID, EventType.RoomMessage, content); - } - ); - void this.maybePersistSecretsBundle(); - return response; + error, + }); + throw error; + } } private async toRoomMessageContents( message: AdapterPostableMessage ): Promise { - const textContent = this.toRoomMessageContent(message); - const attachments = this.extractAttachmentsFromMessage(message); + const textContent = toRoomMessageContent(message); + const attachments = extractAttachmentsFromMessage(message); const uploads = await this.collectUploads(message, attachments); - const linkLines = this.collectLinkOnlyAttachmentLines(attachments); - const textBody = this.mergeTextAndLinks(textContent, linkLines); + const linkLines = collectLinkOnlyAttachmentLines(attachments); + const textBody = mergeTextAndLinks(textContent, linkLines); const contents: MatrixOutboundMessageContent[] = []; if ((normalizeOptionalString(textBody.body) ?? "").length > 0) { @@ -1675,6 +1641,10 @@ export class MatrixAdapter implements Adapter { } private async maybePersistSecretsBundle(force = false): Promise { + if (!force && Date.now() - this.lastSecretsBundlePersistAt < 60_000) { + return; + } + if (!this.stateAdapter) { return; } @@ -1690,9 +1660,6 @@ export class MatrixAdapter implements Adapter { } const now = Date.now(); - if (!force && now - this.lastSecretsBundlePersistAt < 60_000) { - return; - } try { const bundle = await crypto.exportSecretsBundle(); @@ -1778,7 +1745,9 @@ export class MatrixAdapter implements Adapter { return; } this.processedTimelineEventIDs.add(eventID); - evictOldestEntries(this.processedTimelineEventIDs); + if (this.processedTimelineEventIDs.size > 10_000) { + evictOldestEntries(this.processedTimelineEventIDs); + } } const roomID = room?.roomId ?? event.getRoomId(); @@ -1860,7 +1829,7 @@ export class MatrixAdapter implements Adapter { raw: event, user: message.author, triggerId: event.getId(), - }); + }, undefined); } } @@ -1907,14 +1876,11 @@ export class MatrixAdapter implements Adapter { } private async joinRoomWithRetry(roomID: string, maxAttempts = 3): Promise { - let lastError: unknown; - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { try { await this.requireClient().joinRoom(roomID); return; } catch (error) { - lastError = error; if (!this.isRetryableJoinError(error) || attempt === maxAttempts) { throw error; } @@ -1932,8 +1898,6 @@ export class MatrixAdapter implements Adapter { ); } } - - throw lastError; } private shouldAcceptInvite( @@ -2021,7 +1985,9 @@ export class MatrixAdapter implements Adapter { userID: sender, }); - evictOldestEntries(this.reactionByEventID); + if (this.reactionByEventID.size > 10_000) { + evictOldestEntries(this.reactionByEventID); + } } this.requireChat().processReaction({ @@ -2162,240 +2128,6 @@ export class MatrixAdapter implements Adapter { return room.getMember(userId) ?? undefined; } - private parseMatrixContent(content: MatrixMessageContent): ParsedMatrixContent { - const mentionedUserIDs = this.extractMentionedUserIDs(content); - const mentionsRoom = this.extractRoomMention(content); - const formattedBody = normalizeOptionalString(content.formatted_body); - if (formattedBody) { - const htmlMarkdown = this.parseMatrixHTML(formattedBody); - for (const mentionedUserID of htmlMarkdown.mentionedUserIDs) { - mentionedUserIDs.add(mentionedUserID); - } - - if (htmlMarkdown.markdown.length > 0) { - return { - text: markdownToPlainText(htmlMarkdown.markdown), - markdown: htmlMarkdown.markdown, - mentionedUserIDs, - mentionsRoom, - }; - } - } - - const body = this.stripReplyFallbackFromBody( - normalizeOptionalString(content.body) ?? "" - ); - return { - text: body, - markdown: this.markdownForPlainText(body, content.msgtype), - mentionedUserIDs, - mentionsRoom, - }; - } - - private parseMatrixHTML( - html: string - ): { markdown: string; mentionedUserIDs: Set } { - const root = parseHTML(this.stripReplyFallbackFromHTML(html)); - const mentionedUserIDs = new Set(); - const markdown = this.normalizeMarkdownSpacing( - this.renderHTMLNodesToMarkdown(root.childNodes, mentionedUserIDs) - ); - return { - markdown, - mentionedUserIDs, - }; - } - - private renderHTMLNodesToMarkdown( - nodes: HTMLNode[], - mentionedUserIDs: Set - ): string { - return nodes - .map((node) => this.renderHTMLNodeToMarkdown(node, mentionedUserIDs)) - .join(""); - } - - private renderHTMLNodeToMarkdown( - node: HTMLNode, - mentionedUserIDs: Set - ): string { - if (node.nodeType === NodeType.TEXT_NODE) { - return node.text; - } - - if (!(node instanceof HTMLElement)) { - return ""; - } - - const tagName = node.tagName.toLowerCase(); - const children = this.renderHTMLNodesToMarkdown(node.childNodes, mentionedUserIDs); - - switch (tagName) { - case "mx-reply": - return ""; - case "html": - case "body": - case "span": - return children; - case "br": - return "\n"; - case "p": - case "div": - return children.trim() ? `${children.trim()}\n\n` : ""; - case "strong": - case "b": - return children ? `**${children}**` : ""; - case "em": - case "i": - return children ? `*${children}*` : ""; - case "del": - case "s": - return children ? `~~${children}~~` : ""; - case "code": - return node.parentNode instanceof HTMLElement && - node.parentNode.tagName.toLowerCase() === "pre" - ? children - : `\`${children}\``; - case "pre": { - const codeContent = children.replace(/\n+$/u, ""); - return codeContent ? `\n\`\`\`\n${codeContent}\n\`\`\`\n\n` : ""; - } - case "blockquote": { - const quoted = children.trim(); - if (!quoted) { - return ""; - } - return `${quoted - .split("\n") - .map((line) => `> ${line}`) - .join("\n")}\n\n`; - } - case "ul": - return `${node.childNodes - .map((child) => this.renderListItemToMarkdown(child, mentionedUserIDs, null)) - .filter(Boolean) - .join("\n")}\n\n`; - case "ol": - return `${node.childNodes - .map((child, index) => - this.renderListItemToMarkdown(child, mentionedUserIDs, index + 1) - ) - .filter(Boolean) - .join("\n")}\n\n`; - case "a": - return this.renderHTMLLinkToMarkdown(node, children, mentionedUserIDs); - case "img": - return normalizeOptionalString(node.getAttribute("alt")) ?? "image"; - default: - return children; - } - } - - private renderListItemToMarkdown( - node: HTMLNode, - mentionedUserIDs: Set, - ordinal: number | null - ): string { - if (!(node instanceof HTMLElement) || node.tagName.toLowerCase() !== "li") { - return ""; - } - const content = this.normalizeMarkdownSpacing( - this.renderHTMLNodesToMarkdown(node.childNodes, mentionedUserIDs) - ); - if (!content) { - return ""; - } - return `${ordinal === null ? "-" : `${ordinal}.`} ${content}`; - } - - private renderHTMLLinkToMarkdown( - node: HTMLElement, - children: string, - mentionedUserIDs: Set - ): string { - const href = normalizeOptionalString(node.getAttribute("href")); - const text = children || node.text; - if (!href) { - return text; - } - - const mentionedUserID = this.parseMatrixToUserID(href); - if (mentionedUserID) { - mentionedUserIDs.add(mentionedUserID); - return text || this.matrixMentionDisplayText(mentionedUserID); - } - - return `[${text || href}](${href})`; - } - - private parseMatrixToUserID(href: string): string | null { - let url: URL; - try { - url = new URL(href); - } catch { - return null; - } - - if (url.hostname !== "matrix.to") { - return null; - } - - const rawPath = url.hash.startsWith("#/") ? url.hash.slice(2) : url.hash; - const firstSegment = rawPath.split("/")[0]; - if (!firstSegment) { - return null; - } - - const identifier = decodeURIComponent(firstSegment); - return identifier.startsWith("@") ? identifier : null; - } - - private extractMentionedUserIDs(content: MatrixMessageContent): Set { - const mentions = new Set(); - const matrixMentions = content["m.mentions"]; - if (!isRecord(matrixMentions) || !Array.isArray(matrixMentions.user_ids)) { - return mentions; - } - - for (const userID of matrixMentions.user_ids) { - if (typeof userID === "string" && userID.length > 0) { - mentions.add(userID); - } - } - - return mentions; - } - - private extractRoomMention(content: MatrixMessageContent): boolean { - const matrixMentions = content["m.mentions"]; - return isRecord(matrixMentions) && matrixMentions.room === true; - } - - private stripReplyFallbackFromBody(body: string): string { - const lines = body.split("\n"); - let index = 0; - while (index < lines.length && lines[index]?.startsWith(">")) { - index += 1; - } - - if (index === 0 || index >= lines.length || lines[index] !== "") { - return body; - } - - return lines.slice(index + 1).join("\n"); - } - - private stripReplyFallbackFromHTML(html: string): string { - const root = parseHTML(html); - for (const child of [...root.childNodes]) { - if (child instanceof HTMLElement && child.tagName.toLowerCase() === "mx-reply") { - child.remove(); - } - } - return root.toString(); - } - private extractAttachments(content: MatrixMessageContent) { const url = typeof content.url === "string" ? content.url : undefined; if (!url) { @@ -2460,7 +2192,7 @@ export class MatrixAdapter implements Adapter { case MsgType.File: return "file"; default: { - const mediaType = this.messageTypeForMimeType(mimeType); + const mediaType = messageTypeForMimeType(mimeType); switch (mediaType) { case MsgType.Image: return "image"; @@ -2545,34 +2277,6 @@ export class MatrixAdapter implements Adapter { return relation?.rel_type === RelationType.Replace; } - private isMentioned(content: MatrixMessageContent, parsed: ParsedMatrixContent): boolean { - if (parsed.mentionsRoom) { - return true; - } - if (this.userID && parsed.mentionedUserIDs.has(this.userID)) { - return true; - } - - const formatted = - typeof content.formatted_body === "string" ? content.formatted_body : ""; - - const hasUserID = this.userID - ? parsed.text.includes(this.userID) || formatted.includes(this.userID) - : false; - const hasMatrixTo = this.userID - ? formatted.includes(`matrix.to/#/${encodeURIComponent(this.userID)}`) - : false; - - const usernameMention = this.userName.startsWith("@") - ? this.userName - : `@${this.userName}`; - - const hasUserName = - parsed.text.includes(usernameMention) || formatted.includes(usernameMention); - - return hasUserID || hasMatrixTo || hasUserName; - } - private parseSlashCommand( text: string ): { command: string; text: string } | null { @@ -2594,213 +2298,22 @@ export class MatrixAdapter implements Adapter { }; } - private toRoomMessageContent( - message: AdapterPostableMessage - ): MatrixTextMessageContent { - const rendered = this.renderTextMessage(message); - const content: MatrixTextMessageContent = { - body: rendered.body, - msgtype: MsgType.Text, - }; - if (rendered.formattedBody) { - content.format = "org.matrix.custom.html"; - content.formatted_body = rendered.formattedBody; - } - if (rendered.mentions) { - content["m.mentions"] = rendered.mentions; - } - - return content; - } - - private renderTextMessage(message: AdapterPostableMessage): RenderedMatrixMessage { - if (typeof message === "string") { - return this.renderPlainTextMessage(message); - } - - if (isCardElement(message)) { - return this.renderPlainTextMessage("[Card message]"); - } - - if (typeof message === "object" && message !== null) { - if ("raw" in message && typeof message.raw === "string") { - return this.renderPlainTextMessage(message.raw); - } - if ("markdown" in message && typeof message.markdown === "string") { - return this.renderMarkdownMessage(message.markdown); - } - if ("ast" in message) { - return this.renderMarkdownMessage(stringifyMarkdown(message.ast)); - } - if ("card" in message) { - return this.renderPlainTextMessage(message.fallbackText ?? "[Card message]"); - } - } - - return { body: "" }; - } - - private renderPlainTextMessage(text: string): RenderedMatrixMessage { - const rendered = this.replaceMentionPlaceholdersInPlainText(text); - if (rendered.mentionedUserIDs.size === 0) { - return { - body: rendered.body, - }; - } - - return { - body: rendered.body, - formattedBody: this.renderMarkdownToMatrixHTML(rendered.markdown), - mentions: this.buildMentionsContent(rendered.mentionedUserIDs), - }; - } - - private renderMarkdownMessage(markdown: string): RenderedMatrixMessage { - const rendered = this.replaceMentionPlaceholdersInMarkdown(markdown); - return { - body: markdownToPlainText(rendered.markdown), - formattedBody: this.renderMarkdownToMatrixHTML(rendered.markdown), - mentions: this.buildMentionsContent(rendered.mentionedUserIDs), - }; - } - - private replaceMentionPlaceholdersInPlainText(text: string): { - body: string; - markdown: string; - mentionedUserIDs: Set; - } { - const mentionedUserIDs = new Set(); - const pattern = /<@(@[^>\s]+:[^>\s]+)>/gu; - let body = ""; - let markdown = ""; - let lastIndex = 0; - - for (const match of text.matchAll(pattern)) { - const [token, userID] = match; - const index = match.index ?? 0; - const plainSegment = text.slice(lastIndex, index); - body += plainSegment; - markdown += escapeMarkdownText(plainSegment); - - const mentionText = this.matrixMentionDisplayText(userID); - body += mentionText; - markdown += `[${escapeMarkdownLinkText(mentionText)}](${this.matrixToUserLink(userID)})`; - mentionedUserIDs.add(userID); - lastIndex = index + token.length; - } - - const trailing = text.slice(lastIndex); - body += trailing; - markdown += escapeMarkdownText(trailing); - - return { body, markdown, mentionedUserIDs }; - } - - private replaceMentionPlaceholdersInMarkdown(markdown: string): { - markdown: string; - mentionedUserIDs: Set; - } { - const mentionedUserIDs = new Set(); - const transformed = markdown.replace( - /<@(@[^>\s]+:[^>\s]+)>/gu, - (_match, userID: string) => { - mentionedUserIDs.add(userID); - return `[${escapeMarkdownLinkText(this.matrixMentionDisplayText(userID))}](${this.matrixToUserLink( - userID - )})`; - } - ); - - return { - markdown: transformed, - mentionedUserIDs, - }; - } - - private renderMarkdownToMatrixHTML(markdown: string): string { - return marked.parse(markdown, { - async: false, - breaks: true, - gfm: true, - }); - } - - private buildMentionsContent( - mentionedUserIDs: Set - ): { room?: boolean; user_ids?: string[] } | undefined { - if (mentionedUserIDs.size === 0) { - return undefined; - } - - return { - user_ids: [...mentionedUserIDs], - }; - } - - private matrixToUserLink(userID: string): string { - return `https://matrix.to/#/${encodeURIComponent(userID)}`; - } - - private matrixMentionDisplayText(userID: string): string { - return `@${matrixLocalpart(userID)}`; - } - - private markdownForPlainText(text: string, msgtype?: string): string { - const escaped = escapeMarkdownText(text); - if (msgtype === "m.emote" && escaped.length > 0) { - return `*${escaped}*`; + private async performSendRoomMessage( + roomID: string, + rootEventID: string | undefined, + content: MatrixOutboundMessageContent + ) { + const client = this.requireClient(); + if (rootEventID) { + return client.sendEvent( + roomID, + rootEventID, + EventType.RoomMessage, + content as RoomMessageEventContent + ); } - return escaped; - } - private normalizeMarkdownSpacing(markdown: string): string { - return markdown.replace(/\n{3,}/gu, "\n\n").trim(); - } - - private mergeTextAndLinks( - content: MatrixTextMessageContent, - linkLines: string[] - ): MatrixTextMessageContent { - if (linkLines.length === 0) { - return content; - } - - const suffix = linkLines.join("\n"); - const body = content.body ?? ""; - const mergedBody = body ? `${body}\n\n${suffix}` : suffix; - if (!content.formatted_body) { - return { - ...content, - body: mergedBody, - }; - } - - const formattedSuffix = linkLines - .map((line) => `

${escapeHTML(line)}

`) - .join(""); - - return { - ...content, - body: mergedBody, - formatted_body: `${content.formatted_body}${formattedSuffix}`, - }; - } - - private collectLinkOnlyAttachmentLines(attachments: Attachment[]): string[] { - const lines: string[] = []; - for (const attachment of attachments) { - const hasLocalData = - Boolean(attachment.data) || typeof attachment.fetchData === "function"; - if (hasLocalData) { - continue; - } - if (!attachment.url) { - continue; - } - const label = attachment.name ?? attachment.type; - lines.push(`${label}: ${attachment.url}`); - } - return lines; + return client.sendEvent(roomID, EventType.RoomMessage, content); } private async collectUploads( @@ -2808,179 +2321,51 @@ export class MatrixAdapter implements Adapter { attachments: Attachment[] ): Promise { const uploads: OutboundUpload[] = []; - const files = this.extractFilesFromMessage(message); + const files = extractFilesFromMessage(message, this.logger); for (const file of files) { uploads.push({ - data: this.normalizeUploadData(file.data), + data: normalizeUploadData(file.data), fileName: file.filename, info: { mimetype: normalizeOptionalString(file.mimeType), - size: this.binarySize(file.data), + size: binarySize(file.data), }, - msgtype: this.messageTypeForMimeType(normalizeOptionalString(file.mimeType)), + msgtype: messageTypeForMimeType(normalizeOptionalString(file.mimeType)), }); } - for (const attachment of attachments) { - const data = await this.readAttachmentData(attachment); - if (!data) { - continue; + const attachmentUploads = await Promise.all( + attachments.map(async (attachment): Promise => { + const data = await readAttachmentData(attachment); + if (!data) { + return null; + } + const fileName = + normalizeOptionalString(attachment.name) ?? + defaultAttachmentName(attachment); + return { + data: normalizeUploadData(data), + fileName, + info: { + h: attachment.height, + mimetype: normalizeOptionalString(attachment.mimeType), + size: attachment.size ?? binarySize(data), + w: attachment.width, + }, + msgtype: messageTypeForAttachment(attachment), + type: attachment.type, + }; + }) + ); + for (const upload of attachmentUploads) { + if (upload) { + uploads.push(upload); } - const fileName = - normalizeOptionalString(attachment.name) ?? - this.defaultAttachmentName(attachment); - uploads.push({ - data: this.normalizeUploadData(data), - fileName, - info: { - h: attachment.height, - mimetype: normalizeOptionalString(attachment.mimeType), - size: attachment.size ?? this.binarySize(data), - w: attachment.width, - }, - msgtype: this.messageTypeForAttachment(attachment), - type: attachment.type, - }); } return uploads; } - private extractFilesFromMessage(message: AdapterPostableMessage): FileUpload[] { - if (typeof message !== "object" || message === null) { - return []; - } - if (!("files" in message) || !Array.isArray(message.files)) { - return []; - } - return message.files.flatMap((file): FileUpload[] => { - const normalized = this.normalizeFileUpload(file); - return normalized ? [normalized] : []; - }); - } - - private normalizeFileUpload(file: unknown): FileUpload | null { - if (!isRecord(file)) { - return null; - } - - const filename = normalizeOptionalString( - typeof file.filename === "string" ? file.filename : undefined - ); - if (!filename) { - return null; - } - - const data = this.normalizeFileUploadData(file.data); - if (!data) { - this.logger.warn("Skipping invalid Matrix file upload", { filename }); - return null; - } - - return { - filename, - data, - mimeType: - typeof file.mimeType === "string" ? normalizeOptionalString(file.mimeType) : undefined, - }; - } - - private normalizeFileUploadData(data: unknown): Buffer | Blob | ArrayBuffer | null { - if (Buffer.isBuffer(data) || data instanceof Blob || data instanceof ArrayBuffer) { - return data; - } - - if (data instanceof Uint8Array) { - return Buffer.from(data); - } - - return null; - } - - private extractAttachmentsFromMessage(message: AdapterPostableMessage): Attachment[] { - if (typeof message !== "object" || message === null) { - return []; - } - if (!("attachments" in message) || !Array.isArray(message.attachments)) { - return []; - } - return message.attachments.filter((a): a is Attachment => isRecord(a)); - } - - private async readAttachmentData( - attachment: Attachment - ): Promise { - if (typeof attachment.fetchData === "function") { - return attachment.fetchData(); - } - return attachment.data ?? null; - } - - private normalizeUploadData(data: Buffer | Blob | ArrayBuffer): Blob { - if (data instanceof Blob) { - return data; - } - if (this.isNodeBuffer(data)) { - return new Blob([new Uint8Array(data)]); - } - return new Blob([data]); - } - - private binarySize(data: Buffer | Blob | ArrayBuffer): number { - if (data instanceof ArrayBuffer) { - return data.byteLength; - } - if (this.isNodeBuffer(data)) { - return data.length; - } - return data.size; - } - - private isNodeBuffer(value: unknown): value is Buffer { - return typeof Buffer !== "undefined" && Buffer.isBuffer(value); - } - - private messageTypeForAttachment(attachment: Attachment): MatrixMediaMsgType { - switch (attachment.type) { - case "image": - return MsgType.Image; - case "video": - return MsgType.Video; - case "audio": - return MsgType.Audio; - default: - return this.messageTypeForMimeType(normalizeOptionalString(attachment.mimeType)); - } - } - - private messageTypeForMimeType(mimeType?: string): MatrixMediaMsgType { - if (!mimeType) { - return MsgType.File; - } - if (mimeType.startsWith("image/")) { - return MsgType.Image; - } - if (mimeType.startsWith("video/")) { - return MsgType.Video; - } - if (mimeType.startsWith("audio/")) { - return MsgType.Audio; - } - return MsgType.File; - } - - private defaultAttachmentName(attachment: Attachment): string { - switch (attachment.type) { - case "image": - return "image"; - case "video": - return "video"; - case "audio": - return "audio"; - default: - return "file"; - } - } private mustGetEventByID(roomID: string, eventID: string): MatrixEvent { const room = this.requireRoom(roomID); @@ -2995,7 +2380,7 @@ export class MatrixAdapter implements Adapter { roomID: string, eventID: string, fallback: { - content: MatrixRoomMessageContent; + content: MatrixOutboundMessageContent; roomID: string; sender?: string; } @@ -3069,19 +2454,15 @@ export class MatrixAdapter implements Adapter { return `${this.persistenceConfig.keyPrefix}:store`; } - private get sessionBasePrefix(): string { - return this.persistenceSessionPrefix; - } - private getSessionStorageKey(userID: string): string { - return `${this.sessionBasePrefix}:user:${encodeURIComponent(userID)}`; + return `${this.persistenceSessionPrefix}:user:${encodeURIComponent(userID)}`; } private getSessionUsernameTemporaryKey(): string | null { if (this.auth.type !== "password") { return null; } - return `${this.sessionBasePrefix}:username:${encodeURIComponent(this.auth.username)}`; + return `${this.persistenceSessionPrefix}:username:${encodeURIComponent(this.auth.username)}`; } private async loadPersistedSession(): Promise { @@ -3221,35 +2602,6 @@ export class MatrixAdapter implements Adapter { } } - private validateConfig(config: MatrixAdapterConfig): void { - if (!config.baseURL?.trim()) { - throw new Error("baseURL is required."); - } - if (config.persistence?.session?.ttlMs !== undefined && config.persistence.session.ttlMs <= 0) { - throw new Error("persistence.session.ttlMs must be a positive number."); - } - if ( - config.persistence?.sync?.persistIntervalMs !== undefined && - config.persistence.sync.persistIntervalMs <= 0 - ) { - throw new Error("persistence.sync.persistIntervalMs must be a positive number."); - } - if ( - config.persistence?.sync?.snapshotTtlMs !== undefined && - config.persistence.sync.snapshotTtlMs <= 0 - ) { - throw new Error("persistence.sync.snapshotTtlMs must be a positive number."); - } - if ( - (config.persistence?.session?.encrypt && !config.persistence?.session?.decrypt) || - (!config.persistence?.session?.encrypt && config.persistence?.session?.decrypt) - ) { - throw new Error( - "persistence.session.encrypt and persistence.session.decrypt must be provided together." - ); - } - } - private async resolveDeviceID(...candidates: Array): Promise { const configuredDeviceID = normalizeOptionalString(this.deviceID); if (configuredDeviceID) { @@ -3299,9 +2651,6 @@ export class MatrixAdapter implements Adapter { ]); for (const key of candidates) { - if (!key) { - continue; - } const value = await this.stateAdapter.get(key); const normalized = normalizeOptionalString(value); if (normalized) { @@ -3357,8 +2706,8 @@ export class MatrixAdapter implements Adapter { const setLevel = Reflect.get(matrixSDKLogger, "setLevel"); if (typeof setLevel === "function") { setLevel.call(matrixSDKLogger, numericLevel, false); + matrixSDKLogConfigured = true; } - matrixSDKLogConfigured = true; } } @@ -3373,37 +2722,7 @@ export function createMatrixAdapter(config?: MatrixAdapterConfig): MatrixAdapter }); } - const baseURL = process.env.MATRIX_BASE_URL; - if (!baseURL) { - throw new Error("baseURL is required. Set MATRIX_BASE_URL."); - } - - const recoveryKey = process.env.MATRIX_RECOVERY_KEY; - const inviteAutoJoinInviterAllowlist = parseEnvList( - process.env.MATRIX_INVITE_AUTOJOIN_ALLOWLIST - ); - const inviteAutoJoinEnabled = envBool( - process.env.MATRIX_INVITE_AUTOJOIN, - inviteAutoJoinInviterAllowlist.length > 0 - ); - - const auth = resolveAuthFromEnv(); - - return new MatrixAdapter({ - baseURL, - auth, - userName: process.env.MATRIX_BOT_USERNAME ?? "bot", - deviceID: normalizeOptionalString(process.env.MATRIX_DEVICE_ID), - commandPrefix: process.env.MATRIX_COMMAND_PREFIX, - recoveryKey, - inviteAutoJoin: inviteAutoJoinEnabled - ? { - inviterAllowlist: inviteAutoJoinInviterAllowlist, - } - : undefined, - matrixSDKLogLevel: - parseSDKLogLevel(process.env.MATRIX_SDK_LOG_LEVEL) ?? "error", - }); + return new MatrixAdapter(createMatrixAdapterConfigFromEnv()); } export type { @@ -3413,171 +2732,3 @@ export type { MatrixPersistenceSyncConfig, MatrixThreadID, } from "./types"; - -function normalizePersistenceConfig( - config?: MatrixPersistenceConfig -): ResolvedPersistenceConfig { - return { - keyPrefix: - normalizeOptionalString(config?.keyPrefix) ?? DEFAULT_PERSISTENCE_KEY_PREFIX, - session: { - decrypt: config?.session?.decrypt, - encrypt: config?.session?.encrypt, - ttlMs: config?.session?.ttlMs, - }, - sync: { - persistIntervalMs: - config?.sync?.persistIntervalMs ?? - DEFAULT_MATRIX_STORE_PERSIST_INTERVAL_MS, - snapshotTtlMs: config?.sync?.snapshotTtlMs, - }, - }; -} - -function resolveAuthFromEnv(): MatrixAuthConfig { - const username = process.env.MATRIX_USERNAME; - const password = process.env.MATRIX_PASSWORD; - - if (username && password) { - return { - type: "password", - username, - password, - userID: process.env.MATRIX_USER_ID, - }; - } - - const accessToken = process.env.MATRIX_ACCESS_TOKEN; - const userID = process.env.MATRIX_USER_ID; - - if (!accessToken) { - throw new Error( - "Set MATRIX_USERNAME+MATRIX_PASSWORD for password auth, or MATRIX_ACCESS_TOKEN for access token auth." - ); - } - - const auth: MatrixAccessTokenAuthConfig = { - type: "accessToken", - accessToken, - userID, - }; - - return auth; -} - -function envBool(value: string | undefined, fallback = false): boolean { - if (!value) { - return fallback; - } - - const normalized = value.trim().toLowerCase(); - return ( - normalized === "1" || - normalized === "true" || - normalized === "yes" || - normalized === "on" - ); -} - -function parseEnvList(value: string | undefined): string[] { - if (!value) { - return []; - } - - return value - .split(",") - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0); -} - -function generateDeviceID(length = 8): string { - const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - const bytes = randomBytes(length); - let out = "chatsdk_"; - - for (let i = 0; i < length; i += 1) { - out += alphabet[bytes[i] % alphabet.length]; - } - - return out; -} - -function normalizeOptionalString(value: string | null | undefined): string | undefined { - if (!value) { - return undefined; - } - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : undefined; -} - -function readStringValue(value: unknown): string | undefined { - return typeof value === "string" ? normalizeOptionalString(value) : undefined; -} - -function matrixLocalpart(userID: string): string { - return userID.startsWith("@") ? userID.slice(1).split(":")[0] ?? userID : userID; -} - -function escapeMarkdownText(value: string): string { - return value.replace(/([\\`*_{}\[\]()#+\-.!|>])/gu, "\\$1"); -} - -function escapeMarkdownLinkText(value: string): string { - return value.replace(/([\\\]])/gu, "\\$1"); -} - -function escapeHTML(value: string): string { - return value - .replaceAll("&", "&") - .replaceAll("<", "<") - .replaceAll(">", ">") - .replaceAll('"', """) - .replaceAll("'", "'"); -} - -function normalizeStringList(values: string[] | undefined): string[] { - if (!values || values.length === 0) { - return []; - } - - return values - .map((value) => normalizeOptionalString(value)) - .filter((value): value is string => Boolean(value)); -} - -function hasIndexedDB(): boolean { - return typeof globalThis.indexedDB !== "undefined" && globalThis.indexedDB !== null; -} - -function isSDKLogLevel(value: string): value is SDKLogLevel { - return value in MATRIX_SDK_LOG_LEVELS; -} - -function parseSDKLogLevel(value: string | undefined): SDKLogLevel | undefined { - if (!value) { - return undefined; - } - const normalized = value.trim().toLowerCase(); - return isSDKLogLevel(normalized) ? normalized : undefined; -} - -function evictOldestEntries( - collection: { size: number; keys(): Iterable; delete(key: string): unknown }, - maxSize = 10_000, - targetSize = 5_000 -): void { - if (collection.size <= maxSize) return; - const toDelete = collection.size - targetSize; - let deleted = 0; - // Map and Set iteration is insertion ordered, so keys() yields the oldest - // entries first for the collections used by this adapter. - for (const key of collection.keys()) { - if (deleted >= toDelete) break; - collection.delete(key); - deleted++; - } -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} diff --git a/src/messages/inbound.ts b/src/messages/inbound.ts new file mode 100644 index 0000000..938e123 --- /dev/null +++ b/src/messages/inbound.ts @@ -0,0 +1,319 @@ +import { markdownToPlainText } from "chat"; +import { MsgType } from "matrix-js-sdk"; +import { + HTMLElement, + NodeType, + parse as parseHTML, + type Node as HTMLNode, +} from "node-html-parser"; +import { + escapeMarkdownLinkText, + escapeMarkdownText, + isRecord, + matrixMentionDisplayText, + normalizeOptionalString, +} from "../shared/utils"; + +export type MatrixMessageContent = { + body?: string; + format?: string; + formatted_body?: string; + msgtype?: string; + "m.mentions"?: { + room?: boolean; + user_ids?: string[]; + }; + [key: string]: unknown; +}; + +export type ParsedMatrixContent = { + markdown: string; + mentionsRoom: boolean; + mentionedUserIDs: Set; + text: string; +}; + +export function parseMatrixContent(content: MatrixMessageContent): ParsedMatrixContent { + const mentionedUserIDs = extractMentionedUserIDs(content); + const mentionsRoom = extractRoomMention(content); + const formattedBody = normalizeOptionalString(content.formatted_body); + if (formattedBody) { + const htmlMarkdown = parseMatrixHTML(formattedBody); + for (const mentionedUserID of htmlMarkdown.mentionedUserIDs) { + mentionedUserIDs.add(mentionedUserID); + } + + if (htmlMarkdown.markdown.length > 0) { + return { + text: markdownToPlainText(htmlMarkdown.markdown), + markdown: htmlMarkdown.markdown, + mentionedUserIDs, + mentionsRoom, + }; + } + } + + const body = stripReplyFallbackFromBody( + normalizeOptionalString(content.body) ?? "" + ); + return { + text: body, + markdown: markdownForPlainText(body, content.msgtype), + mentionedUserIDs, + mentionsRoom, + }; +} + +export function isMentioned(args: { + content: MatrixMessageContent; + parsed: ParsedMatrixContent; + userID: string; + userName: string; +}): boolean { + const { content, parsed, userID, userName } = args; + if (parsed.mentionsRoom) { + return true; + } + if (userID && parsed.mentionedUserIDs.has(userID)) { + return true; + } + + const formatted = + typeof content.formatted_body === "string" ? content.formatted_body : ""; + + const hasUserID = userID + ? parsed.text.includes(userID) || formatted.includes(userID) + : false; + const hasMatrixTo = userID + ? formatted.includes(`matrix.to/#/${encodeURIComponent(userID)}`) + : false; + const normalizedUserName = userName.trim(); + const hasUserName = normalizedUserName + ? (() => { + const usernameMention = normalizedUserName.startsWith("@") + ? normalizedUserName + : `@${normalizedUserName}`; + return ( + parsed.text.includes(usernameMention) || + formatted.includes(usernameMention) + ); + })() + : false; + + return hasUserID || hasMatrixTo || hasUserName; +} + +function parseMatrixHTML( + html: string +): { markdown: string; mentionedUserIDs: Set } { + const root = parseHTML(html); + for (const child of [...root.childNodes]) { + if (child instanceof HTMLElement && child.tagName.toLowerCase() === "mx-reply") { + child.remove(); + } + } + const mentionedUserIDs = new Set(); + const markdown = normalizeMarkdownSpacing( + renderHTMLNodesToMarkdown(root.childNodes, mentionedUserIDs) + ); + return { + markdown, + mentionedUserIDs, + }; +} + +function renderHTMLNodesToMarkdown( + nodes: HTMLNode[], + mentionedUserIDs: Set +): string { + return nodes + .map((node) => renderHTMLNodeToMarkdown(node, mentionedUserIDs)) + .join(""); +} + +function renderHTMLNodeToMarkdown( + node: HTMLNode, + mentionedUserIDs: Set +): string { + if (node.nodeType === NodeType.TEXT_NODE) { + return node.text; + } + + if (!(node instanceof HTMLElement)) { + return ""; + } + + const tagName = node.tagName.toLowerCase(); + const children = renderHTMLNodesToMarkdown(node.childNodes, mentionedUserIDs); + + switch (tagName) { + case "mx-reply": + return ""; + case "html": + case "body": + case "span": + return children; + case "br": + return "\n"; + case "p": + case "div": + return children.trim() ? `${children.trim()}\n\n` : ""; + case "strong": + case "b": + return children ? `**${children}**` : ""; + case "em": + case "i": + return children ? `*${children}*` : ""; + case "del": + case "s": + return children ? `~~${children}~~` : ""; + case "code": + return node.parentNode instanceof HTMLElement && + node.parentNode.tagName.toLowerCase() === "pre" + ? children + : `\`${children}\``; + case "pre": { + const codeContent = children.replace(/\n+$/u, ""); + return codeContent ? `\n\`\`\`\n${codeContent}\n\`\`\`\n\n` : ""; + } + case "blockquote": { + const quoted = children.trim(); + if (!quoted) { + return ""; + } + return `${quoted + .split("\n") + .map((line) => `> ${line}`) + .join("\n")}\n\n`; + } + case "ul": + return `${node.childNodes + .map((child) => renderListItemToMarkdown(child, mentionedUserIDs, null)) + .filter(Boolean) + .join("\n")}\n\n`; + case "ol": + return `${node.childNodes + .map((child, index) => + renderListItemToMarkdown(child, mentionedUserIDs, index + 1) + ) + .filter(Boolean) + .join("\n")}\n\n`; + case "a": + return renderHTMLLinkToMarkdown(node, children, mentionedUserIDs); + case "img": + return normalizeOptionalString(node.getAttribute("alt")) ?? "image"; + default: + return children; + } +} + +function renderListItemToMarkdown( + node: HTMLNode, + mentionedUserIDs: Set, + ordinal: number | null +): string { + if (!(node instanceof HTMLElement) || node.tagName.toLowerCase() !== "li") { + return ""; + } + const content = normalizeMarkdownSpacing( + renderHTMLNodesToMarkdown(node.childNodes, mentionedUserIDs) + ); + if (!content) { + return ""; + } + return `${ordinal === null ? "-" : `${ordinal}.`} ${content}`; +} + +function renderHTMLLinkToMarkdown( + node: HTMLElement, + children: string, + mentionedUserIDs: Set +): string { + const href = normalizeOptionalString(node.getAttribute("href")); + const text = children || node.text; + if (!href) { + return text; + } + + const mentionedUserID = parseMatrixToUserID(href); + if (mentionedUserID) { + mentionedUserIDs.add(mentionedUserID); + return text || matrixMentionDisplayText(mentionedUserID); + } + + return `[${escapeMarkdownLinkText(text || href)}](${href})`; +} + +function parseMatrixToUserID(href: string): string | null { + let url: URL; + try { + url = new URL(href); + } catch { + return null; + } + + if (url.hostname !== "matrix.to") { + return null; + } + + const rawPath = url.hash.startsWith("#/") ? url.hash.slice(2) : url.hash; + const firstSegment = rawPath.split("/")[0]; + if (!firstSegment) { + return null; + } + + try { + const identifier = decodeURIComponent(firstSegment); + return identifier.startsWith("@") ? identifier : null; + } catch { + return null; + } +} + +function extractMentionedUserIDs(content: MatrixMessageContent): Set { + const mentions = new Set(); + const matrixMentions = content["m.mentions"]; + if (!isRecord(matrixMentions) || !Array.isArray(matrixMentions.user_ids)) { + return mentions; + } + + for (const userID of matrixMentions.user_ids) { + if (typeof userID === "string" && userID.length > 0) { + mentions.add(userID); + } + } + + return mentions; +} + +function extractRoomMention(content: MatrixMessageContent): boolean { + const matrixMentions = content["m.mentions"]; + return isRecord(matrixMentions) && matrixMentions.room === true; +} + +function stripReplyFallbackFromBody(body: string): string { + const lines = body.split("\n"); + let index = 0; + while (index < lines.length && lines[index]?.startsWith(">")) { + index += 1; + } + + if (index === 0 || index >= lines.length || lines[index] !== "") { + return body; + } + + return lines.slice(index + 1).join("\n"); +} + + +function markdownForPlainText(text: string, msgtype?: string): string { + const escaped = escapeMarkdownText(text); + if (msgtype === MsgType.Emote && escaped.length > 0) { + return `*${escaped}*`; + } + return escaped; +} + +function normalizeMarkdownSpacing(markdown: string): string { + return markdown.replace(/\n{3,}/gu, "\n\n").trim(); +} diff --git a/src/messages/outbound.ts b/src/messages/outbound.ts new file mode 100644 index 0000000..97241a9 --- /dev/null +++ b/src/messages/outbound.ts @@ -0,0 +1,593 @@ +import type { + AdapterPostableMessage, + Attachment, + EmojiValue, + FileUpload, + Logger, +} from "chat"; +import { + isCardElement, + markdownToPlainText, + stringifyMarkdown, +} from "chat"; +import { marked } from "marked"; +import { MatrixError } from "matrix-js-sdk/lib/http-api/errors"; +import { MsgType, RelationType } from "matrix-js-sdk"; +import type { + RoomMessageEventContent, + RoomMessageTextEventContent, +} from "matrix-js-sdk/lib/@types/events"; +import type { MediaEventContent } from "matrix-js-sdk/lib/@types/media"; +import { + escapeHTML, + escapeMarkdownLinkText, + escapeMarkdownText, + isRecord, + matrixMentionDisplayText, + normalizeOptionalString, +} from "../shared/utils"; + +const DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES = 12_000; + +export type MatrixTextMessageContent = RoomMessageTextEventContent & { + "com.beeper.dont_render_edited"?: boolean; +}; + +export type MatrixRoomMessageContent = RoomMessageEventContent & { + "com.beeper.dont_render_edited"?: boolean; + "m.new_content"?: RoomMessageEventContent & { + "com.beeper.dont_render_edited"?: boolean; + }; +}; + +export type MatrixOutboundMessageContent = MatrixRoomMessageContent | MediaEventContent; + +export type MatrixMediaMsgType = + | MsgType.Audio + | MsgType.File + | MsgType.Image + | MsgType.Video; + +type RenderedMatrixMessage = { + body: string; + formattedBody?: string; + mentions?: { + room?: boolean; + user_ids?: string[]; + }; +}; + +export type OutboundUpload = { + data: Blob; + fileName: string; + info?: { + h?: number; + mimetype?: string; + size?: number; + w?: number; + }; + msgtype: MatrixMediaMsgType; + type?: string; +}; + +export function extractReplyEventID( + message: AdapterPostableMessage +): string | undefined { + if (typeof message !== "object" || message === null || isCardElement(message)) { + return undefined; + } + + const replyEventID = (message as { matrixReplyToEventId?: unknown }).matrixReplyToEventId; + return typeof replyEventID === "string" && replyEventID.length > 0 + ? replyEventID + : undefined; +} + +export function applyThreadReplyMetadata( + content: MatrixOutboundMessageContent, + rootEventID: string | undefined, + replyEventID: string | null | undefined +): MatrixOutboundMessageContent { + const threadableContent = content as MatrixOutboundMessageContent & { + "m.relates_to"?: { + rel_type?: string; + "m.in_reply_to"?: { event_id: string }; + [key: string]: unknown; + }; + }; + + if (!rootEventID || threadableContent["m.relates_to"]?.rel_type) { + return threadableContent; + } + + if (replyEventID === null) { + return threadableContent; + } + + return { + ...threadableContent, + "m.relates_to": { + ...threadableContent["m.relates_to"], + "m.in_reply_to": { + event_id: replyEventID ?? rootEventID, + }, + }, + } as MatrixOutboundMessageContent; +} + +export function isTooLargeMatrixError(error: unknown): error is MatrixError { + return ( + error instanceof MatrixError && + (error.errcode === "M_TOO_LARGE" || error.httpStatus === 413) + ); +} + +export function splitOversizedTextContent( + content: MatrixOutboundMessageContent +): MatrixTextMessageContent[] { + if (!isSplittableTextContent(content)) { + return []; + } + + const body = content.body; + if (Buffer.byteLength(body, "utf8") <= DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES) { + return [ + { + body, + msgtype: content.msgtype, + }, + ]; + } + + const parts = splitTextByUtf8Bytes(body, DEFAULT_OVERSIZED_MESSAGE_CHUNK_BYTES); + + return parts.map((part) => ({ + body: part, + msgtype: content.msgtype, + })); +} + +export function toRoomMessageContent( + message: AdapterPostableMessage +): MatrixTextMessageContent { + const rendered = renderTextMessage(message); + const content: MatrixTextMessageContent = { + body: rendered.body, + msgtype: MsgType.Text, + }; + if (rendered.formattedBody) { + content.format = "org.matrix.custom.html"; + content.formatted_body = rendered.formattedBody; + } + if (rendered.mentions) { + content["m.mentions"] = rendered.mentions; + } + + return content; +} + +export function mergeTextAndLinks( + content: MatrixTextMessageContent, + linkLines: string[] +): MatrixTextMessageContent { + if (linkLines.length === 0) { + return content; + } + + const suffix = linkLines.join("\n"); + const body = content.body ?? ""; + const mergedBody = body ? `${body}\n\n${suffix}` : suffix; + if (!content.formatted_body) { + return { + ...content, + body: mergedBody, + }; + } + + const formattedSuffix = linkLines + .map((line) => `

${escapeHTML(line)}

`) + .join(""); + + return { + ...content, + body: mergedBody, + formatted_body: `${content.formatted_body}${formattedSuffix}`, + }; +} + +export function collectLinkOnlyAttachmentLines(attachments: Attachment[]): string[] { + const lines: string[] = []; + for (const attachment of attachments) { + const hasLocalData = + Boolean(attachment.data) || typeof attachment.fetchData === "function"; + if (hasLocalData) { + continue; + } + if (!attachment.url) { + continue; + } + const label = attachment.name ?? attachment.type; + lines.push(`${label}: ${attachment.url}`); + } + return lines; +} + +export function extractFilesFromMessage( + message: AdapterPostableMessage, + logger?: Logger +): FileUpload[] { + if (typeof message !== "object" || message === null) { + return []; + } + if (!("files" in message) || !Array.isArray(message.files)) { + return []; + } + return message.files.flatMap((file): FileUpload[] => { + const normalized = normalizeFileUpload(file, logger); + return normalized ? [normalized] : []; + }); +} + +export function normalizeFileUpload( + file: unknown, + logger?: Logger +): FileUpload | null { + if (!isRecord(file)) { + return null; + } + + const filename = normalizeOptionalString( + typeof file.filename === "string" ? file.filename : undefined + ); + if (!filename) { + return null; + } + + const data = normalizeFileUploadData(file.data); + if (!data) { + logger?.warn("Skipping invalid Matrix file upload", { filename }); + return null; + } + + return { + filename, + data, + mimeType: + typeof file.mimeType === "string" ? normalizeOptionalString(file.mimeType) : undefined, + }; +} + +export function normalizeFileUploadData( + data: unknown +): Buffer | Blob | ArrayBuffer | null { + if (Buffer.isBuffer(data) || data instanceof Blob || data instanceof ArrayBuffer) { + return data; + } + + if (data instanceof Uint8Array) { + return Buffer.from(data); + } + + return null; +} + +export function extractAttachmentsFromMessage( + message: AdapterPostableMessage +): Attachment[] { + if (typeof message !== "object" || message === null) { + return []; + } + if (!("attachments" in message) || !Array.isArray(message.attachments)) { + return []; + } + return message.attachments.filter((a): a is Attachment => isRecord(a)); +} + +export async function readAttachmentData( + attachment: Attachment +): Promise { + if (typeof attachment.fetchData === "function") { + return attachment.fetchData(); + } + return attachment.data ?? null; +} + +export function normalizeUploadData(data: Buffer | Blob | ArrayBuffer): Blob { + if (data instanceof Blob) { + return data; + } + if (isNodeBuffer(data)) { + return new Blob([new Uint8Array(data)]); + } + return new Blob([data]); +} + +export function binarySize(data: Buffer | Blob | ArrayBuffer): number { + if (data instanceof ArrayBuffer) { + return data.byteLength; + } + if (isNodeBuffer(data)) { + return data.length; + } + return data.size; +} + +export function messageTypeForAttachment( + attachment: Attachment +): MatrixMediaMsgType { + switch (attachment.type) { + case "image": + return MsgType.Image; + case "video": + return MsgType.Video; + case "audio": + return MsgType.Audio; + default: + return messageTypeForMimeType(normalizeOptionalString(attachment.mimeType)); + } +} + +export function messageTypeForMimeType(mimeType?: string): MatrixMediaMsgType { + if (!mimeType) { + return MsgType.File; + } + if (mimeType.startsWith("image/")) { + return MsgType.Image; + } + if (mimeType.startsWith("video/")) { + return MsgType.Video; + } + if (mimeType.startsWith("audio/")) { + return MsgType.Audio; + } + return MsgType.File; +} + +export function defaultAttachmentName(attachment: Attachment): string { + switch (attachment.type) { + case "image": + return "image"; + case "video": + return "video"; + case "audio": + return "audio"; + default: + return "file"; + } +} + +function renderTextMessage(message: AdapterPostableMessage): RenderedMatrixMessage { + if (typeof message === "string") { + return renderPlainTextMessage(message); + } + + if (isCardElement(message)) { + return renderPlainTextMessage("[Card message]"); + } + + if (typeof message === "object" && message !== null) { + if ("raw" in message && typeof message.raw === "string") { + return renderPlainTextMessage(message.raw); + } + if ("markdown" in message && typeof message.markdown === "string") { + return renderMarkdownMessage(message.markdown); + } + if ("ast" in message) { + return renderMarkdownMessage(stringifyMarkdown(message.ast)); + } + if ("card" in message) { + return renderPlainTextMessage(message.fallbackText ?? "[Card message]"); + } + } + + return { body: "" }; +} + +function renderPlainTextMessage(text: string): RenderedMatrixMessage { + const rendered = replaceMentionPlaceholdersInPlainText(text); + if (rendered.mentionedUserIDs.size === 0) { + return { + body: rendered.body, + }; + } + + return { + body: rendered.body, + formattedBody: renderMarkdownToMatrixHTML(rendered.markdown), + mentions: buildMentionsContent(rendered.mentionedUserIDs), + }; +} + +function renderMarkdownMessage(markdown: string): RenderedMatrixMessage { + const rendered = replaceMentionPlaceholdersInMarkdown(markdown); + return { + body: markdownToPlainText(rendered.markdown), + formattedBody: renderMarkdownToMatrixHTML(rendered.markdown), + mentions: buildMentionsContent(rendered.mentionedUserIDs), + }; +} + +function replaceMentionPlaceholdersInPlainText(text: string): { + body: string; + markdown: string; + mentionedUserIDs: Set; +} { + const mentionedUserIDs = new Set(); + const pattern = /<@(@[^>\s]+:[^>\s]+)>/gu; + let body = ""; + let markdown = ""; + let lastIndex = 0; + + for (const match of text.matchAll(pattern)) { + const [token, userID] = match; + const index = match.index ?? 0; + const plainSegment = text.slice(lastIndex, index); + body += plainSegment; + markdown += escapeMarkdownText(plainSegment); + + const mentionText = matrixMentionDisplayText(userID); + body += mentionText; + markdown += `[${escapeMarkdownLinkText(mentionText)}](${matrixToUserLink(userID)})`; + mentionedUserIDs.add(userID); + lastIndex = index + token.length; + } + + const trailing = text.slice(lastIndex); + body += trailing; + markdown += escapeMarkdownText(trailing); + + return { body, markdown, mentionedUserIDs }; +} + +function replaceMentionPlaceholdersInMarkdown(markdown: string): { + markdown: string; + mentionedUserIDs: Set; +} { + const mentionedUserIDs = new Set(); + const transformed = markdown.replace( + /<@(@[^>\s]+:[^>\s]+)>/gu, + (_match, userID: string) => { + mentionedUserIDs.add(userID); + return `[${escapeMarkdownLinkText(matrixMentionDisplayText(userID))}](${matrixToUserLink( + userID + )})`; + } + ); + + return { + markdown: transformed, + mentionedUserIDs, + }; +} + +function renderMarkdownToMatrixHTML(markdown: string): string { + return marked.parse(markdown, { + async: false, + breaks: true, + gfm: true, + }); +} + +function buildMentionsContent( + mentionedUserIDs: Set +): { room?: boolean; user_ids?: string[] } | undefined { + if (mentionedUserIDs.size === 0) { + return undefined; + } + + return { + user_ids: [...mentionedUserIDs], + }; +} + +function matrixToUserLink(userID: string): string { + return `https://matrix.to/#/${encodeURIComponent(userID)}`; +} + +function isSplittableTextContent( + content: MatrixOutboundMessageContent +): content is MatrixTextMessageContent { + if ("url" in content || "info" in content) { + return false; + } + + if (typeof content.body !== "string" || content.body.length <= 1) { + return false; + } + + if ("m.new_content" in content) { + return false; + } + + return ( + content.msgtype === MsgType.Text || + content.msgtype === MsgType.Notice || + content.msgtype === MsgType.Emote + ); +} + +function splitTextByUtf8Bytes(text: string, maxBytes: number): string[] { + const normalizedMaxBytes = Math.max(1, Math.floor(maxBytes)); + const chunks: string[] = []; + let remaining = text; + + while (remaining.length > 0) { + if (Buffer.byteLength(remaining, "utf8") <= normalizedMaxBytes) { + chunks.push(remaining); + remaining = ""; + break; + } + + const boundary = clampSurrogateBoundary( + remaining, + findSplitBoundary(remaining, normalizedMaxBytes) + ); + const head = remaining.slice(0, boundary); + const tail = remaining.slice(boundary); + + if (!head || head === remaining) { + break; + } + + chunks.push(head); + remaining = tail; + } + + if (remaining.length > 0 && chunks.at(-1) !== remaining) { + chunks.push(remaining); + } + + return chunks.filter((chunk) => chunk.length > 0); +} + +function findSplitBoundary(text: string, maxBytes: number): number { + let low = 1; + let high = text.length; + let best = 1; + + while (low <= high) { + const mid = Math.floor((low + high) / 2); + const candidate = text.slice(0, clampSurrogateBoundary(text, mid)); + if (Buffer.byteLength(candidate, "utf8") <= maxBytes) { + best = candidate.length; + low = mid + 1; + } else { + high = mid - 1; + } + } + + for (let i = best - 1; i > 0; i--) { + const ch = text[i]; + if (ch === "\n" || ch === " " || ch === "\t" || ch === "\r") { + return clampSurrogateBoundary(text, i + 1); + } + } + + return clampSurrogateBoundary(text, best); +} + +function clampSurrogateBoundary(text: string, boundary: number): number { + if (boundary <= 0 || boundary >= text.length) { + return boundary; + } + + const current = text.charCodeAt(boundary); + const previous = text.charCodeAt(boundary - 1); + if (isLowSurrogateCodeUnit(current) && isHighSurrogateCodeUnit(previous)) { + return boundary - 1; + } + + return boundary; +} + +function isHighSurrogateCodeUnit(code: number): boolean { + return code >= 0xd800 && code <= 0xdbff; +} + +function isLowSurrogateCodeUnit(code: number): boolean { + return code >= 0xdc00 && code <= 0xdfff; +} + +function isNodeBuffer(value: unknown): value is Buffer { + return typeof Buffer !== "undefined" && Buffer.isBuffer(value); +} diff --git a/src/shared/utils.ts b/src/shared/utils.ts new file mode 100644 index 0000000..9e7af7d --- /dev/null +++ b/src/shared/utils.ts @@ -0,0 +1,85 @@ +import { randomBytes } from "node:crypto"; + +export function generateDeviceID(length = 8): string { + const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + const bytes = randomBytes(length); + let out = "chatsdk_"; + + for (let i = 0; i < length; i += 1) { + out += alphabet[bytes[i] % alphabet.length]; + } + + return out; +} + +export function normalizeOptionalString( + value: string | null | undefined +): string | undefined { + if (!value) { + return undefined; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +export function readStringValue(value: unknown): string | undefined { + return typeof value === "string" ? normalizeOptionalString(value) : undefined; +} + +export function matrixLocalpart(userID: string): string { + return userID.startsWith("@") ? userID.slice(1).split(":")[0] ?? userID : userID; +} + +export function escapeMarkdownText(value: string): string { + return value.replace(/([\\`*_{}\[\]()#+\-.!|>])/gu, "\\$1"); +} + +export function escapeMarkdownLinkText(value: string): string { + return value.replace(/([\\\]])/gu, "\\$1"); +} + +export function escapeHTML(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +export function normalizeStringList(values: string[] | undefined): string[] { + if (!values || values.length === 0) { + return []; + } + + return values + .map((value) => normalizeOptionalString(value)) + .filter((value): value is string => Boolean(value)); +} + +export function hasIndexedDB(): boolean { + return typeof globalThis.indexedDB !== "undefined" && globalThis.indexedDB !== null; +} + +export function evictOldestEntries( + collection: { size: number; keys(): Iterable; delete(key: string): unknown }, + maxSize = 10_000, + targetSize = 5_000 +): void { + if (collection.size <= maxSize) return; + const toDelete = collection.size - targetSize; + let deleted = 0; + for (const key of collection.keys()) { + if (deleted >= toDelete) break; + collection.delete(key); + deleted++; + } +} + +export function matrixMentionDisplayText(userID: string): string { + return `@${matrixLocalpart(userID)}`; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +}