diff --git a/.gitignore b/.gitignore index 0897c5e7..25126642 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ vite.config.ts.timestamp-* # IDE .idea + +static/mupdf-wasm.wasm diff --git a/bun.lock b/bun.lock index 355ea6b9..69bc975f 100644 --- a/bun.lock +++ b/bun.lock @@ -17,6 +17,7 @@ "clsx": "^2.1.1", "fflate": "^0.8.2", "lucide-svelte": "^0.554.0", + "mupdf": "^1.28.0", "music-metadata": "^11.10.1", "overlayscrollbars": "^2.12.0", "overlayscrollbars-svelte": "^0.5.5", @@ -642,6 +643,8 @@ "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "mupdf": ["mupdf@1.28.0", "", {}, "sha512-ACUnbpECaQ5JLq04pwd89lS+0IGMest5qL5tb08g9TAR7bDtfqflHEkb2Xm3o4rvC/szguLiV+WEbW9kstj8Sg=="], + "music-metadata": ["music-metadata@11.10.3", "", { "dependencies": { "@borewit/text-codec": "^0.2.0", "@tokenizer/token": "^0.3.0", "content-type": "^1.0.5", "debug": "^4.4.3", "file-type": "^21.1.1", "media-typer": "^1.1.0", "strtok3": "^10.3.4", "token-types": "^6.1.1", "uint8array-extras": "^1.5.0" } }, "sha512-j0g/x4cNNZW6I5gdcPAY+GFkJY9WHTpkFDMBJKQLxJQyvSfQbXm57fTE3haGFFuOzCgtsTd4Plwc49Sn9RacDQ=="], "mz": ["mz@2.7.0", "", { "dependencies": { "any-promise": "^1.0.0", "object-assign": "^4.0.1", "thenify-all": "^1.0.0" } }, "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q=="], diff --git a/package.json b/package.json index 46988826..4e627c28 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "format": "prettier --write .", - "lint": "prettier --check . && eslint ." + "lint": "prettier --check . && eslint .", + "postinstall": "cp node_modules/mupdf/dist/mupdf-wasm.wasm static/mupdf-wasm.wasm" }, "devDependencies": { "@inlang/paraglide-js": "^2.5.0", @@ -51,6 +52,7 @@ "clsx": "^2.1.1", "fflate": "^0.8.2", "lucide-svelte": "^0.554.0", + "mupdf": "^1.28.0", "music-metadata": "^11.10.3", "overlayscrollbars": "^2.12.0", "overlayscrollbars-svelte": "^0.5.5", diff --git a/src/lib/converters/converter.svelte.ts b/src/lib/converters/converter.svelte.ts index a2b7a12f..d888cbb6 100644 --- a/src/lib/converters/converter.svelte.ts +++ b/src/lib/converters/converter.svelte.ts @@ -93,3 +93,61 @@ export class Converter { return this.supportedFormats.map((f) => f.name); } } + +export interface ChainStep { + converter: Converter; + to: string; // intermediate format (last step uses the final target) +} + +export class ChainedConverter extends Converter { + public override name: string; + private steps: ChainStep[]; + + constructor(steps: ChainStep[]) { + super(0); + if (steps.length < 2) throw new Error("Chain requires at least 2 steps"); + this.steps = steps; + this.name = steps.map((s) => s.converter.name).join("+"); + this.clearTimeout(); + this.status = this.deriveStatus(); + this.supportedFormats = [ + ...steps[0].converter.supportedFormats.filter((f) => f.fromSupported), + ...steps[steps.length - 1].converter.supportedFormats.filter((f) => f.toSupported), + ]; + } + + private deriveStatus(): WorkerStatus { + const statuses = this.steps.map((s) => s.converter.status); + if (statuses.some((s) => s === "error")) return "error"; + if (statuses.every((s) => s === "ready")) return "ready"; + if (statuses.some((s) => s === "downloading")) return "downloading"; + return "not-ready"; + } + + public override async convert(input: VertFile, to: string): Promise { + this.status = this.deriveStatus(); + const { VertFile: VF } = await import("$lib/types"); + let current: VertFile = input; + + for (let i = 0; i < this.steps.length; i++) { + const { converter, to: stepTo } = this.steps[i]; + const isLast = i === this.steps.length - 1; + const target = isLast ? to : stepTo; + const result = await converter.convert(current, target); + if (!isLast) { + current = new VF( + new File([await result.file.arrayBuffer()], input.name.replace(/\.[^/.]+$/, target)), + target, + ); + } else { + current = result; + } + } + + return current; + } + + public override async cancel(input: VertFile): Promise { + await Promise.allSettled(this.steps.map((s) => s.converter.cancel(input))); + } +} diff --git a/src/lib/converters/index.ts b/src/lib/converters/index.ts index 11089ae9..4194fddb 100644 --- a/src/lib/converters/index.ts +++ b/src/lib/converters/index.ts @@ -4,6 +4,7 @@ import { FFmpegConverter } from "./ffmpeg.svelte"; import { PandocConverter } from "./pandoc.svelte"; import { VertdConverter } from "./vertd.svelte"; import { MagickConverter } from "./magick.svelte"; +import { MuPDFConverter } from "./mupdf.svelte"; import { DISABLE_ALL_EXTERNAL_REQUESTS } from "$lib/util/consts"; const getConverters = (): Converter[] => { @@ -17,6 +18,7 @@ const getConverters = (): Converter[] => { } converters.push(new PandocConverter()); + converters.push(new MuPDFConverter()); return converters; }; diff --git a/src/lib/converters/mupdf.svelte.ts b/src/lib/converters/mupdf.svelte.ts new file mode 100644 index 00000000..c744b48e --- /dev/null +++ b/src/lib/converters/mupdf.svelte.ts @@ -0,0 +1,53 @@ +import { VertFile } from "$lib/types"; +import { Converter, FormatInfo } from "./converter.svelte"; +import { browser } from "$app/environment"; +import MuPDFWorker from "$lib/workers/mupdf?worker&url"; + +export class MuPDFConverter extends Converter { + public name = "mupdf"; + + private activeConversions = new Map(); + + constructor() { + super(60); + if (!browser) return; + this.status = "ready"; + this.clearTimeout(); + } + + public async convert(file: VertFile, to: string): Promise { + const worker = new Worker(MuPDFWorker, { type: "module" }); + this.activeConversions.set(file.id, worker); + + worker.postMessage({ file: file.file, to, id: file.id }); + + const result = await new Promise<{ type: string; output?: Uint8Array; error?: string }>( + (resolve) => { + worker.onmessage = (e) => resolve(e.data); + worker.onerror = (e) => resolve({ type: "error", error: e.message }); + }, + ); + + worker.terminate(); + this.activeConversions.delete(file.id); + + if (result.type === "error") throw new Error(result.error); + + if (!to.startsWith(".")) to = `.${to}`; + return new VertFile(new File([result.output! as BlobPart], file.name), to); + } + + public async cancel(input: VertFile): Promise { + const worker = this.activeConversions.get(input.id); + if (worker) { + worker.terminate(); + this.activeConversions.delete(input.id); + } + } + + public supportedFormats = [ + new FormatInfo("pdf", true, false), + new FormatInfo("md", false, true), + new FormatInfo("txt", false, true), + ]; +} diff --git a/src/lib/types/file.svelte.ts b/src/lib/types/file.svelte.ts index 75615420..31f9578b 100644 --- a/src/lib/types/file.svelte.ts +++ b/src/lib/types/file.svelte.ts @@ -1,5 +1,6 @@ import { byNative, converters } from "$lib/converters"; import type { Converter } from "$lib/converters/converter.svelte"; +import { ChainedConverter } from "$lib/converters/converter.svelte"; import { m } from "$lib/paraglide/messages"; import { ToastManager } from "$lib/util/toast.svelte"; import type { Component } from "svelte"; @@ -65,7 +66,29 @@ export class VertFile { if (!theirFrom.isNative && !theirTo.isNative) return false; return true; }); - return converter; + if (converter) return converter; + + for (const a of this.converters) { + for (const fmt of a.supportedFormats) { + if (!fmt.toSupported) continue; + const b = converters.find((c) => { + const cFrom = c.supportedFormats.find( + (f) => f.name === fmt.name && f.fromSupported, + ); + const cTo = c.supportedFormats.find( + (f) => f.name === this.to && f.toSupported, + ); + return cFrom && cTo; + }); + if (b) + return new ChainedConverter([ + { converter: a, to: fmt.name }, + { converter: b, to: this.to }, + ]); + } + } + + return undefined; } public isLarge(): boolean { diff --git a/src/lib/workers/mupdf.ts b/src/lib/workers/mupdf.ts new file mode 100644 index 00000000..0be0af50 --- /dev/null +++ b/src/lib/workers/mupdf.ts @@ -0,0 +1,34 @@ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +(globalThis as any).$libmupdf_wasm_Module = { + locateFile: () => "/mupdf-wasm.wasm", +}; + +let mupdf: typeof import("mupdf") | null = null; + +self.onmessage = async (e: MessageEvent) => { + const { file, to, id } = e.data as { file: File; to: string; id: string }; + try { + if (!mupdf) mupdf = await import("mupdf"); + + const buf = new Uint8Array(await file.arrayBuffer()); + const doc = mupdf.Document.openDocument(buf, "application/pdf"); + const pageCount = doc.countPages(); + const parts: string[] = []; + + for (let i = 0; i < pageCount; i++) { + const page = doc.loadPage(i); + const st = page.toStructuredText("preserve-whitespace"); + parts.push(st.asText()); + page.destroy(); + } + doc.destroy(); + + self.postMessage({ + type: "finished", + output: new TextEncoder().encode(parts.join("\n\n")), + id, + }); + } catch (err) { + self.postMessage({ type: "error", error: String(err), id }); + } +}; diff --git a/vite.config.ts b/vite.config.ts index d6ebd1ee..f9e2fd0f 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -53,7 +53,7 @@ export default defineConfig(({ command }) => { format: "es", }, optimizeDeps: { - exclude: ["@ffmpeg/core-mt", "@ffmpeg/ffmpeg", "@ffmpeg/util"], + exclude: ["@ffmpeg/core-mt", "@ffmpeg/ffmpeg", "@ffmpeg/util", "mupdf"], }, css: { preprocessorOptions: {