Skip to content

fix: export text overlays reliably #1533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jayesh-durge wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: export text overlays reliably #1533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 85 additions & 1 deletion src/lib/ffmpeg.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { EditRecipe, ExportResult, BackgroundMusicOptions, ImageOverlayOptions } from "./types";
import { getPresetById } from "./presets";
import { buildTextFilter } from "./text-overlay";
import { ensureFontLoaded, getFontFamily } from "@/utils/fontLoader";

export class FFmpegLoadError extends Error {}

Expand All @@ -21,6 +22,7 @@ type WorkerExportRequest = {
musicOptions?: BackgroundMusicOptions;
overlayFile?: SerializedFile;
overlayOptions?: ImageOverlayOptions;
textOverlayFile?: SerializedFile;
};

type WorkerLoadResponse = { type: "ready" };
Expand Down Expand Up @@ -222,6 +224,19 @@ export async function exportVideo(
}

const sessionId = buildSessionId();
const { width: targetW, height: targetH } = getRecipeOutputSize(recipe);
const hasTextOverlays = (recipe.textOverlays || []).some((overlay) => overlay.text.trim());
const textOverlayFilePayload = hasTextOverlays
? await renderTextOverlayFile(recipe, targetW, targetH, sessionId)
: undefined;

if (hasTextOverlays && !textOverlayFilePayload) {
throw new Error("Text overlays could not be rendered for export.");
}

const exportRecipe = hasTextOverlays
? { ...recipe, textOverlays: [] }
: recipe;
const arrayBuffer = await file.arrayBuffer();
const filePayload: SerializedFile = {
name: file.name,
Expand Down Expand Up @@ -273,18 +288,20 @@ export async function exportVideo(
const transfers: Transferable[] = [arrayBuffer];
if (musicFilePayload) transfers.push(musicFilePayload.data);
if (overlayFilePayload) transfers.push(overlayFilePayload.data);
if (textOverlayFilePayload) transfers.push(textOverlayFilePayload.data);

ffmpegWorker.postMessage(
{
type: "export",
id: sessionId,
file: filePayload,
recipe,
recipe: exportRecipe,
videoDuration: await getVideoDuration(file),
musicFile: musicFilePayload,
musicOptions: sanitizedMusicOptions,
overlayFile: overlayFilePayload,
overlayOptions: sanitizedOverlayOptions,
textOverlayFile: textOverlayFilePayload,
} as WorkerExportRequest,
transfers
);
Expand Down Expand Up @@ -325,6 +342,73 @@ function buildSessionId(): string {
return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
}

function getRecipeOutputSize(recipe: EditRecipe): { width: number; height: number } {
let width: number;
let height: number;

if (recipe.preset === "custom") {
width = recipe.customWidth;
height = recipe.customHeight;
} else {
const preset = getPresetById(recipe.preset);
width = preset?.width ?? 1920;
height = preset?.height ?? 1080;
}

return {
width: Math.round(width / 2) * 2,
height: Math.round(height / 2) * 2,
};
}

async function renderTextOverlayFile(
recipe: EditRecipe,
targetW: number,
targetH: number,
sessionId: string
): Promise<SerializedFile | undefined> {
const textOverlays = (recipe.textOverlays || []).filter((overlay) => overlay.text.trim());
if (textOverlays.length === 0) return undefined;

await Promise.all(
textOverlays.map((overlay) => ensureFontLoaded(overlay.fontFamily, overlay.fontSize))
);

const canvas = document.createElement("canvas");
canvas.width = targetW;
canvas.height = targetH;
const ctx = canvas.getContext("2d");
if (!ctx) return undefined;

ctx.clearRect(0, 0, targetW, targetH);
ctx.textAlign = "center";
ctx.textBaseline = "middle";

textOverlays.forEach((overlay) => {
const x = (overlay.x / 100) * targetW;
const y = (overlay.y / 100) * targetH;
const weight =
overlay.fontWeight === "900" ? 900 : overlay.fontWeight === "bold" ? 700 : 400;

ctx.font = `${weight} ${overlay.fontSize}px ${getFontFamily(overlay.fontFamily)}`;
ctx.fillStyle = overlay.color;
ctx.shadowColor = "rgba(0, 0, 0, 0.8)";
ctx.shadowBlur = 8;
ctx.shadowOffsetX = 0;
ctx.shadowOffsetY = 2;
ctx.fillText(overlay.text, x, y);
});

const blob = await new Promise<Blob | null>((resolve) => canvas.toBlob(resolve, "image/png"));
if (!blob) return undefined;

return {
name: `text_overlay_${sessionId}.png`,
type: "image/png",
data: await blob.arrayBuffer(),
};
}

export function buildVideoFilter(recipe: EditRecipe, targetW: number, targetH: number): string {
const filters: string[] = [];

Expand Down
71 changes: 63 additions & 8 deletions src/lib/ffmpeg.worker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type ExportRequest = {
musicOptions?: BackgroundMusicOptions;
overlayFile?: SerializedFile;
overlayOptions?: ImageOverlayOptions;
textOverlayFile?: SerializedFile;
};

type LoadRequest = { type: "load" };
Expand Down Expand Up @@ -186,6 +187,8 @@ function buildArguments(
hasOverlay: boolean,
overlayInputName: string,
overlayOptions: ImageOverlayOptions | undefined,
hasTextOverlay: boolean,
textOverlayInputName: string,
hasOriginalAudio: boolean,
videoDuration: number
): string[] {
Expand All @@ -196,7 +199,8 @@ function buildArguments(
const af = afParts.join(",");

const musicIdx = 1;
const overlayIdx = hasMusicTrack ? 2 : 1;
const overlayIdx = 1 + (hasMusicTrack ? 1 : 0);
const textOverlayIdx = 1 + (hasMusicTrack ? 1 : 0) + (hasOverlay ? 1 : 0);

const args: string[] = [];
args.push("-i", inputName);
Expand All @@ -207,8 +211,11 @@ function buildArguments(
if (hasOverlay) {
args.push("-i", overlayInputName);
}
if (hasTextOverlay) {
args.push("-i", textOverlayInputName);
}

const needsFilterComplex = hasOverlay || hasMusicTrack;
const needsFilterComplex = hasOverlay || hasTextOverlay || hasMusicTrack;
const shouldKeepAudio = recipe.keepAudio && (hasOriginalAudio || hasMusicTrack);

if (needsFilterComplex) {
Expand Down Expand Up @@ -246,6 +253,12 @@ interface PositionCoords {
videoOut = "[vout]";
}

if (hasTextOverlay) {
filterParts.push(`[${textOverlayIdx}:v]format=rgba[textlayer]`);
filterParts.push(`${videoOut}[textlayer]overlay=0:0[vtext]`);
videoOut = "[vtext]";
}

let audioOut = "";
if (shouldKeepAudio) {
if (hasMusicTrack) {
Expand Down Expand Up @@ -429,6 +442,15 @@ async function runExport(request: ExportRequest): Promise<ResultPayload> {
});
}

const hasTextOverlay = !!request.textOverlayFile;
const textOverlayInputName = `text_overlay_${sessionId}.png`;
if (hasTextOverlay) {
cleanupFiles.add(textOverlayInputName);
await ffmpeg.writeFile(textOverlayInputName, serializeFileBuffer(request.textOverlayFile!), {
signal: activeExportAbortController?.signal,
});
}

const videoDuration = request.videoDuration;

const handleProgress = ({ progress }: { progress: number }) => {
Expand All @@ -442,10 +464,13 @@ async function runExport(request: ExportRequest): Promise<ResultPayload> {
try {
if (recipe.format === "gif") {
const vf = buildVideoFilter(recipe, targetW, targetH);
const vfWithPalette = vf ? `${vf},palettegen` : "palettegen";
const vfWithPaletteUse = vf
? `[0:v]${vf}[x];[x][1:v]paletteuse`
: "[0:v][1:v]paletteuse";
const baseVideo = vf ? `[0:v]${vf}[vbase]` : "[0:v]null[vbase]";
const gifVideoForPalette = hasTextOverlay
? `${baseVideo};[1:v]format=rgba[textlayer];[vbase][textlayer]overlay=0:0[gifbase];[gifbase]palettegen[paletteout]`
: `${baseVideo};[vbase]palettegen[paletteout]`;
const gifVideoWithPalette = hasTextOverlay
? `${baseVideo};[1:v]format=rgba[textlayer];[vbase][textlayer]overlay=0:0[gifbase];[gifbase][2:v]paletteuse[gifout]`
: `${baseVideo};[vbase][1:v]paletteuse[gifout]`;

const gifDurationArgs = recipe.speed !== 1
? (() => {
Expand All @@ -456,14 +481,38 @@ async function runExport(request: ExportRequest): Promise<ResultPayload> {
: [];

const pass1Code = await ffmpeg.exec(
["-i", inputName, "-vf", vfWithPalette, ...gifDurationArgs, "-y", paletteName],
[
"-i",
inputName,
...(hasTextOverlay ? ["-i", textOverlayInputName] : []),
"-filter_complex",
gifVideoForPalette,
"-map",
"[paletteout]",
...gifDurationArgs,
"-y",
paletteName,
],
undefined,
{ signal: activeExportAbortController?.signal }
);
if (pass1Code !== 0) throw new Error("GIF palette generation failed");

const pass2Code = await ffmpeg.exec(
["-i", inputName, "-i", paletteName, "-lavfi", vfWithPaletteUse, ...gifDurationArgs, "-y", outputName],
[
"-i",
inputName,
...(hasTextOverlay ? ["-i", textOverlayInputName] : []),
"-i",
paletteName,
"-filter_complex",
gifVideoWithPalette,
"-map",
"[gifout]",
...gifDurationArgs,
"-y",
outputName,
],
undefined,
{ signal: activeExportAbortController?.signal }
);
Expand Down Expand Up @@ -511,6 +560,8 @@ async function runExport(request: ExportRequest): Promise<ResultPayload> {
hasOverlay,
overlayInputName,
request.overlayOptions,
hasTextOverlay,
textOverlayInputName,
true,
videoDuration
);
Expand All @@ -534,6 +585,8 @@ async function runExport(request: ExportRequest): Promise<ResultPayload> {
hasOverlay,
overlayInputName,
request.overlayOptions,
hasTextOverlay,
textOverlayInputName,
false,
videoDuration
);
Expand All @@ -556,6 +609,8 @@ async function runExport(request: ExportRequest): Promise<ResultPayload> {
hasOverlay,
overlayInputName,
request.overlayOptions,
hasTextOverlay,
textOverlayInputName,
!missingAudioDetected,
videoDuration
);
Expand Down
38 changes: 38 additions & 0 deletions src/lib/tests/textOverlay.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import { describe, expect, it } from "vitest";
import { buildTextFilter } from "../text-overlay";
import { TextOverlay } from "../types";

function overlay(overrides: Partial<TextOverlay> = {}): TextOverlay {
return {
id: "text-1",
text: "Title",
x: 50,
y: 25,
fontSize: 48,
color: "#ffffff",
fontWeight: "bold",
fontFamily: "Arial",
...overrides,
};
}

describe("buildTextFilter", () => {
it("builds a drawtext filter without unsupported font options for built-in fonts", () => {
const filter = buildTextFilter(overlay(), 1920, 1080);

expect(filter).toBe("drawtext=text='Title':x=960:y=270:fontsize=48:fontcolor=#ffffff");
expect(filter).not.toContain("fontweight=");
expect(filter).not.toContain("fontfile='Arial'");
});

it("escapes drawtext separators inside overlay text", () => {
const filter = buildTextFilter(
overlay({ text: "It's 10:30, go\\now; 50%", x: 10, y: 20 }),
1000,
500
);

expect(filter).toContain("text='It\\'s 10\\:30\\, go\\\\now\\; 50\\%'");
expect(filter).toContain("x=100:y=100");
});
});
33 changes: 13 additions & 20 deletions src/lib/text-overlay.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,17 @@ export function getTextPercentPosition(
};
}

function escapeDrawtextValue(value: string): string {
return value
.replace(/\\/g, "\\\\")
.replace(/'/g, "\\'")
.replace(/:/g, "\\:")
.replace(/,/g, "\\,")
.replace(/;/g, "\\;")
.replace(/%/g, "\\%")
.replace(/\r?\n/g, "\\n");
}

/**
* Generates a drawText FFmpeg filter for a single text overlay.
* Escapes special characters and positions text on the output video.
Expand All @@ -69,34 +80,16 @@ export function buildTextFilter(
targetHeight: number
): string {
// Escape special characters for FFmpeg drawtext filter
const escapedText = overlay.text
.replace(/\\/g, "\\\\")
.replace(/'/g, "\\'")
.replace(/:/g, "\\:");
const escapedText = escapeDrawtextValue(overlay.text);

// Convert percentage position to pixel position
const pixelX = Math.round((overlay.x / 100) * targetWidth);
const pixelY = Math.round((overlay.y / 100) * targetHeight);

// Build font parameters
const fontWeightParam = overlay.fontWeight === "900"
? "bold"
: overlay.fontWeight === "bold"
? "bold"
: "normal";

// Get font file parameter for custom fonts (if available)
const fontFileParam = getFFmpegFontArg(overlay.fontFamily, overlay.fontPath);

// Build the drawtext filter with font support
let filter = `drawtext=text='${escapedText}':x=${pixelX}:y=${pixelY}:fontsize=${overlay.fontSize}:fontcolor=${overlay.color}:fontweight=${fontWeightParam}`;

// Add font family if specified
if (overlay.fontFamily) {
// Sanitize font name for FFmpeg
const safeFontName = overlay.fontFamily.replace(/[^a-zA-Z0-9-]/g, "");
filter += `:fontfile='${safeFontName}'`;
}
let filter = `drawtext=text='${escapedText}':x=${pixelX}:y=${pixelY}:fontsize=${overlay.fontSize}:fontcolor=${overlay.color}`;

// Add custom font file path if available
if (fontFileParam) {
Expand Down