From 0ed3a5d933c6a0ef4334e4b345488e2faae87ca7 Mon Sep 17 00:00:00 2001 From: Gautam Kumar Date: Wed, 15 Apr 2026 16:24:08 +0530 Subject: [PATCH] feat: implement memory controller, content extraction utilities, and validation schemas for text, link, and document processing. --- client/src/components/MainArea.tsx | 4 +- .../memories/memorie.controller.ts | 2 +- server/src/utils/content-extractors.ts | 172 +++++++----------- server/src/validations/memory.validation.ts | 1 + 4 files changed, 67 insertions(+), 112 deletions(-) diff --git a/client/src/components/MainArea.tsx b/client/src/components/MainArea.tsx index a67e7d5..e1072f3 100644 --- a/client/src/components/MainArea.tsx +++ b/client/src/components/MainArea.tsx @@ -69,7 +69,7 @@ const TABS: { id: Tab; label: string; icon: React.ReactNode }[] = [ }, ]; -const ACCEPTED_TYPES = '.pdf,.docx,.txt,.md'; +const ACCEPTED_TYPES = '.pdf,.docx,.txt,.md,.csv'; // ── Helpers ──────────────────────────────────────────────────────────────── // No manual auth headers needed; axios will attach the neura_token cookie automatically. @@ -318,7 +318,7 @@ const MainArea = () => { : 'Drop file here or click to browse'}

- PDF, DOCX, TXT, MD — max 10 MB + PDF, DOCX, TXT, MD, CSV — max 10 MB

{ // --------------------------------------------------------------------------- // Document text extraction // --------------------------------------------------------------------------- +// Document content extraction +// --------------------------------------------------------------------------- /** * Extracts text from an uploaded document buffer. * * Supported MIME types: - * - text/plain, text/markdown → decode UTF‑8 - * - application/pdf → basic text layer extraction + * - text/plain, text/markdown → decode UTF‑8 + * - text/csv → decode UTF-8 (tabular data) + * - application/pdf → pdfjs-dist (handles compressed streams, CIDFonts) * - application/vnd.openxmlformats‑officedocument.wordprocessingml.document - * → extract raw text from docx XML + * → mammoth (semantic HTML → plain text) */ export async function extractTextFromDocument( buffer: Buffer, @@ -135,6 +143,7 @@ export async function extractTextFromDocument( switch (mimetype) { case 'text/plain': case 'text/markdown': + case 'text/csv': return buffer.toString('utf-8'); case 'application/pdf': @@ -146,7 +155,7 @@ export async function extractTextFromDocument( default: throw new AppError( 415, - `Unsupported document type: ${mimetype}. Supported types: PDF, DOCX, TXT, MD.`, + `Unsupported document type: ${mimetype}. Supported: PDF, DOCX, TXT, MD, CSV.`, ); } } @@ -156,124 +165,69 @@ export async function extractTextFromDocument( // --------------------------------------------------------------------------- /** - * Very lightweight PDF text extraction. - * - * PDF files store text in stream objects between `BT` (Begin Text) and `ET` - * (End Text) markers. Text‑showing operators like `Tj`, `TJ`, `'`, and `"` - * carry the actual string content inside parentheses `(…)`. - * - * This extractor: - * 1. Converts the raw buffer to a latin‑1 string (PDFs are byte‑oriented). - * 2. Finds all `BT … ET` blocks. - * 3. Inside each block, captures strings enclosed in `(…)`. - * 4. Joins everything with spaces / newlines. + * Production-grade PDF text extraction using Mozilla's pdfjs-dist. * - * Limitations: - * - Only works with PDFs whose text layer is NOT compressed (FlateDecode, etc.). - * - Scanned / image‑only PDFs will return empty text. - * - Complex encodings (CIDFont, ToUnicode CMaps) are not decoded. - * - * For production‑grade extraction, swap this with `pdf-parse` or `pdfjs-dist`. + * Handles compressed text streams (FlateDecode), CIDFonts, ToUnicode CMaps, + * and multi-page documents. Only scanned/image-only PDFs will still fail + * (those require OCR via the Unstructured path). */ -function extractTextFromPdfBuffer(buffer: Buffer): string { - const raw = buffer.toString('latin1'); - - const textBlocks: string[] = []; - const btEtRegex = /BT\s([\s\S]*?)ET/g; - let blockMatch: RegExpExecArray | null; - - while ((blockMatch = btEtRegex.exec(raw)) !== null) { - const block = blockMatch[1]; - if (!block) continue; +async function extractTextFromPdfBuffer(buffer: Buffer): Promise { + try { + const data = new Uint8Array(buffer); + const doc = await getDocument({ data, useSystemFonts: true }).promise; + + const pageTexts: string[] = []; + + for (let i = 1; i <= doc.numPages; i++) { + const page = await doc.getPage(i); + const content = await page.getTextContent(); + const pageText = content.items + .filter((item: any) => 'str' in item) + .map((item: any) => item.str) + .join(' '); + pageTexts.push(pageText.trim()); + } - // Match strings inside parentheses — the text operands of Tj / TJ / ' / " - const stringRegex = /\(([^)]*)\)/g; - let strMatch: RegExpExecArray | null; + const text = pageTexts.filter(Boolean).join('\n\n').trim(); - while ((strMatch = stringRegex.exec(block)) !== null) { - const decoded = strMatch[1]; - if (decoded) { - textBlocks.push(decoded); - } + if (!text) { + throw new AppError( + 422, + 'Could not extract text from the PDF. The file may be scanned/image‑based. Try enabling Unstructured OCR or upload a text‑based PDF.', + ); } - } - - const text = textBlocks.join(' ').trim(); - if (!text) { - throw new AppError( - 422, - 'Could not extract text from the PDF. The file may be scanned/image‑based or use compressed text streams. Please provide a text‑based PDF.', - ); + return text; + } catch (err) { + if (err instanceof AppError) throw err; + const msg = err instanceof Error ? err.message : String(err); + throw new AppError(422, `PDF extraction failed: ${msg}`); } - - return text; } /** - * Very lightweight DOCX text extraction. + * Production-grade DOCX text extraction using mammoth. * - * A `.docx` file is a ZIP archive. The main document text lives inside - * `word/document.xml`. We locate that entry, extract it, strip XML tags, - * and return the raw text content. - * - * Limitations: - * - Only extracts from `word/document.xml` — headers, footers, footnotes, - * and embedded charts are ignored. - * - Images are ignored. - * - * For production use, swap with a library like `mammoth` or `docx-parser`. + * mammoth reads the full document structure (paragraphs, tables, lists, + * headers, footers) and converts it to plain text with proper formatting. */ -function extractTextFromDocxBuffer(buffer: Buffer): string { - // DOCX = ZIP. The ZIP local file header signature is PK\x03\x04. - // We scan for the `word/document.xml` entry, find its data, and strip XML. - const marker = 'word/document.xml'; - const idx = buffer.indexOf(marker); - - if (idx === -1) { - throw new AppError( - 422, - 'The uploaded DOCX file appears to be malformed — could not locate word/document.xml.', - ); - } - - // A quick‑and‑dirty approach: extract everything from the buffer as UTF‑8 - // and look for the XML content between ` { + try { + const result = await mammoth.extractRawText({ buffer }); + const text = result.value.trim(); - // Strip XML tags, decode entities, collapse whitespace - const text = xmlContent - .replace(/]*>/g, '\n') // paragraph boundaries → newlines - .replace(/<[^>]+>/g, '') - .replace(/&/g, '&') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/[ \t]+/g, ' ') - .replace(/\n{3,}/g, '\n\n') - .trim(); + if (!text) { + throw new AppError( + 422, + 'Could not extract text from the DOCX file. The document may be empty.', + ); + } - if (!text) { - throw new AppError( - 422, - 'Could not extract text from the DOCX file. The document may be empty.', - ); + return text; + } catch (err) { + if (err instanceof AppError) throw err; + const msg = err instanceof Error ? err.message : String(err); + throw new AppError(422, `DOCX extraction failed: ${msg}`); } - - return text; } + diff --git a/server/src/validations/memory.validation.ts b/server/src/validations/memory.validation.ts index 8bc335c..3d39d64 100644 --- a/server/src/validations/memory.validation.ts +++ b/server/src/validations/memory.validation.ts @@ -38,6 +38,7 @@ export const ALLOWED_DOCUMENT_MIMES = [ 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', // .docx 'text/plain', 'text/markdown', + 'text/csv', ] as const; /** Max file size in bytes (10 MB) */