perf: pre-lex details inner content at tokenize time

Author: Algorithm5838

src/lib/components/chat/Messages/Markdown/MarkdownTokens.svelte

@@ -6,7 +6,7 @@
 	import fileSaver from 'file-saver';
 	const { saveAs } = fileSaver;
 
-	import { marked, type Token } from 'marked';
+	import { type Token } from 'marked';
 	import { copyToClipboard, unescapeHtml } from '$lib/utils';
 
 	import { WEBUI_BASE_URL } from '$lib/constants';
@@ -399,7 +399,7 @@
 							<div class="mb-1.5" slot="content">
 								<svelte:self
 									id={`${id}-${tokenIdx}-${detailIdx}-d`}
-									tokens={marked.lexer(decode(detailToken.text))}
+									tokens={detailToken.tokens ?? []}
 									attributes={detailToken?.attributes}
 									{done}
 									{editCodeBlock}
@@ -447,7 +447,7 @@
 				<div class=" mb-1.5" slot="content">
 					<svelte:self
 						id={`${id}-${tokenIdx}-d`}
-						tokens={marked.lexer(decode(token.text))}
+						tokens={token.tokens ?? []}
 						attributes={token?.attributes}
 						{done}
 						{editCodeBlock}

src/lib/utils/marked/extension.ts

@@ -1,3 +1,6 @@
+import { decode } from 'html-entities';
+import { type TokenizerThis } from 'marked';
+
 // Helper function to find matching closing tag
 function findMatchingClosingTag(src: string, openTag: string, closeTag: string): number {
 	let depth = 1;
@@ -26,7 +29,7 @@ function parseAttributes(tag: string): { [key: string]: string } {
 	return attributes;
 }
 
-function detailsTokenizer(src: string) {
+function detailsTokenizer(this: TokenizerThis, src: string) {
 	// Updated regex to capture attributes inside <details>
 	const detailsRegex = /^<details(\s+[^>]*)?>\n/;
 	const summaryRegex = /^<summary>(.*?)<\/summary>\n/;
@@ -49,11 +52,17 @@ function detailsTokenizer(src: string) {
 			content = content.slice(summaryMatch[0].length).trim();
 		}
 
+		const tokens: any[] = [];
+		if (attributes?.type !== 'tool_calls') {
+			this.lexer.blockTokens(decode(content), tokens);
+		}
+
 		return {
 			type: 'details',
 			raw: fullMatch,
 			summary: summary,
 			text: content,
+			tokens,
 			attributes: attributes // Include extracted attributes from <details>
 		};
 	}
@@ -89,6 +98,6 @@ function detailsExtension() {
 
 export default function (options = {}) {
 	return {
-		extensions: [detailsExtension(options)]
+		extensions: [detailsExtension()]
 	};
 }