From 803f1107433f304039d481c3e1694c0d878e2ce4 Mon Sep 17 00:00:00 2001 From: Gerry Saporito Date: Wed, 25 Feb 2026 09:38:14 -0800 Subject: [PATCH 1/5] transcription provider comparison --- package-lock.json | 263 ++++++++++++++++++ .../.env.sample | 8 + .../package.json | 21 ++ .../run.sh | 23 ++ .../src/config/env.ts | 6 + .../src/config/providers.ts | 43 +++ .../src/convert_to_readable_transcript.ts | 50 ++++ .../src/fetch_with_retry.ts | 24 ++ .../src/index.ts | 70 +++++ .../src/schemas/EnvSchema.ts | 7 + .../schemas/RecordingArtifactEventSchema.ts | 24 ++ .../schemas/TranscriptArtifactEventSchema.ts | 27 ++ .../src/schemas/TranscriptArtifactSchema.ts | 25 ++ .../src/schemas/TranscriptPartSchema.ts | 25 ++ ...transcription_provider_comparison_async.ts | 141 ++++++++++ .../tsconfig.json | 14 + 16 files changed, 771 insertions(+) create mode 100644 transcription_provider_comparison_async/.env.sample create mode 100644 transcription_provider_comparison_async/package.json create mode 100755 transcription_provider_comparison_async/run.sh create mode 100644 transcription_provider_comparison_async/src/config/env.ts create mode 100644 transcription_provider_comparison_async/src/config/providers.ts create mode 100644 transcription_provider_comparison_async/src/convert_to_readable_transcript.ts create mode 100644 transcription_provider_comparison_async/src/fetch_with_retry.ts create mode 100644 transcription_provider_comparison_async/src/index.ts create mode 100644 transcription_provider_comparison_async/src/schemas/EnvSchema.ts create mode 100644 transcription_provider_comparison_async/src/schemas/RecordingArtifactEventSchema.ts create mode 100644 transcription_provider_comparison_async/src/schemas/TranscriptArtifactEventSchema.ts create mode 100644 transcription_provider_comparison_async/src/schemas/TranscriptArtifactSchema.ts create mode 100644 transcription_provider_comparison_async/src/schemas/TranscriptPartSchema.ts create mode 100644 transcription_provider_comparison_async/src/transcription_provider_comparison_async.ts create mode 100644 transcription_provider_comparison_async/tsconfig.json diff --git a/package-lock.json b/package-lock.json index fa669d7..1ccdd00 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31467,6 +31467,10 @@ "dev": true, "license": "MIT" }, + "node_modules/transcription_provider_comparison_async": { + "resolved": "transcription_provider_comparison_async", + "link": true + }, "node_modules/turbo": { "version": "2.7.2", "dev": true, @@ -31511,6 +31515,265 @@ "resolved": "verify_requests_from_recall", "link": true }, + "transcription_provider_comparison": { + "version": "1.0.0", + "extraneous": true, + "license": "MIT", + "dependencies": { + "dotenv": "^17.2.3", + "http": "^0.0.1-security", + "zod": "^4.1.13" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + } + }, + "transcription_provider_comparison_async": { + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "dotenv": "^17.2.3", + "http": "^0.0.1-security", + "zod": "^4.1.13" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + } + }, + "transcription_provider_comparison_async/node_modules/@types/node": { + "version": "24.10.13", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.13.tgz", + "integrity": "sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "transcription_provider_comparison_async/node_modules/@types/node/node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + }, + "transcription_provider_comparison_async/node_modules/dotenv": { + "version": "17.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.3.1.tgz", + "integrity": "sha512-IO8C/dzEb6O3F9/twg6ZLXz164a2fhTnEWb95H23Dm4OuN+92NmEAlTrupP9VW6Jm3sO26tQlqyvyi4CsnY9GA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "transcription_provider_comparison_async/node_modules/http": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/http/-/http-0.0.1-security.tgz", + "integrity": "sha512-RnDvP10Ty9FxqOtPZuxtebw1j4L/WiqNMDtuc1YMH1XQm5TgDRaR1G9u8upL6KD1bXHSp9eSXo/ED+8Q7FAr+g==" + }, + "transcription_provider_comparison_async/node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping/node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping/node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/@tsconfig/node10": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", + "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "dev": true, + "license": "MIT" + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/acorn": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/acorn-walk": { + "version": "8.3.5", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", + "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/diff": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", + "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, + "transcription_provider_comparison_async/node_modules/ts-node/node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "transcription_provider_comparison_async/node_modules/zod": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", + "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "verify_requests_from_recall": { "version": "1.0.0", "license": "MIT", diff --git a/transcription_provider_comparison_async/.env.sample b/transcription_provider_comparison_async/.env.sample new file mode 100644 index 0000000..60186e4 --- /dev/null +++ b/transcription_provider_comparison_async/.env.sample @@ -0,0 +1,8 @@ +RECALL_API_KEY=RECALL_API_KEY +RECALL_REGION=RECALL_REGION # e.g. us-west-2, us-east-1, eu-central-1, ap-northeast-1 + +# Optional if using the run.sh script to launch a bot +MEETING_URL=MEETING_URL # e.g. any Zoom, Google Meet, Microsoft Teams, Webex, or GoToMeeting URL +NGROK_DOMAIN=NGROK_DOMAIN # Omit the protocol e.g. if your ngrok URL is https://1a8d23b7ab2d.ngrok-free.app, drop the protocol and set to 1a8d23b7ab2d.ngrok-free.app + +# Configure providers in src/config/providers.ts diff --git a/transcription_provider_comparison_async/package.json b/transcription_provider_comparison_async/package.json new file mode 100644 index 0000000..194f696 --- /dev/null +++ b/transcription_provider_comparison_async/package.json @@ -0,0 +1,21 @@ +{ + "name": "transcription_provider_comparison_async", + "version": "1.0.0", + "description": "Compare async transcription results across multiple providers (AssemblyAI, Deepgram, Gladia, etc.)", + "main": "index.ts", + "scripts": { + "dev": "ts-node src/index.ts" + }, + "author": "Gerry Saporito", + "license": "MIT", + "devDependencies": { + "@types/node": "^24.10.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + }, + "dependencies": { + "dotenv": "^17.2.3", + "http": "^0.0.1-security", + "zod": "^4.1.13" + } +} diff --git a/transcription_provider_comparison_async/run.sh b/transcription_provider_comparison_async/run.sh new file mode 100755 index 0000000..b3cc351 --- /dev/null +++ b/transcription_provider_comparison_async/run.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +DOTENV_FILE="${DOTENV_FILE:-.env}" +if [ -f "$DOTENV_FILE" ]; then + # shellcheck source=/dev/null + source "$DOTENV_FILE" +fi + +: "${RECALL_REGION:?RECALL_REGION is required (us-west-2, us-east-1, eu-central-1, ap-northeast-1)}" +: "${RECALL_API_KEY:?RECALL_API_KEY is required}" +: "${MEETING_URL:?MEETING_URL is required (Zoom/Meet URL)}" + +curl --request POST \ + --url https://${RECALL_REGION}.recall.ai/api/v1/bot/ \ + --header "Authorization: ${RECALL_API_KEY}" \ + --header "accept: application/json" \ + --header "content-type: application/json" \ + --data @- < { + const paragraph = words.map((w) => w.text).join(" ").trim(); + if (!paragraph) return null; + + const first = words.find((w) => w?.start_timestamp); + const last = [...words].reverse().find((w) => w?.end_timestamp); + + const start_relative = first?.start_timestamp?.relative ?? null; + const start_absolute = first?.start_timestamp?.absolute ?? null; + const end_relative = last?.end_timestamp?.relative ?? null; + const end_absolute = last?.end_timestamp?.absolute ?? null; + + const duration_seconds = + start_relative !== null && end_relative !== null + ? end_relative - start_relative + : (start_absolute && end_absolute ? (Date.parse(end_absolute) - Date.parse(start_absolute)) / 1000 : null); + + return { + speaker: participant.name, + paragraph, + start_timestamp: { relative: start_relative, absolute: start_absolute }, + end_timestamp: { relative: end_relative, absolute: end_absolute }, + duration_seconds, + }; + }) + .filter(Boolean); +} diff --git a/transcription_provider_comparison_async/src/fetch_with_retry.ts b/transcription_provider_comparison_async/src/fetch_with_retry.ts new file mode 100644 index 0000000..c913278 --- /dev/null +++ b/transcription_provider_comparison_async/src/fetch_with_retry.ts @@ -0,0 +1,24 @@ +/** + * Helper function to fetch with retry. + * Respects the Retry-After header. + */ +export async function fetch_with_retry(url: string, options: RequestInit, max_attempts: number = 5): Promise { + for (let attempt = 1; attempt <= max_attempts; attempt++) { + const response = await fetch(url, options); + if (response.status === 429) { + let retry_after = Number(response.headers.get("Retry-After")) || 0; + console.log(`Rate limit exceeded, retrying in ${retry_after} seconds`); + if (!retry_after) { + console.error("Retry-After header not found"); + retry_after = 0; + } + await new Promise((resolve) => setTimeout( + resolve, + 1000 * (retry_after + Math.ceil(Math.random() * 5)), + )); + continue; + } + return response; + } + throw new Error(`Max attempts (${max_attempts}) reached while fetching ${url}. options=${JSON.stringify(options)}`); +} diff --git a/transcription_provider_comparison_async/src/index.ts b/transcription_provider_comparison_async/src/index.ts new file mode 100644 index 0000000..47a1217 --- /dev/null +++ b/transcription_provider_comparison_async/src/index.ts @@ -0,0 +1,70 @@ +import http from "http"; +import z from "zod"; +import { env } from "./config/env"; +import { PROVIDER_CONFIGS, get_provider_name } from "./config/providers"; +import { RecordingArtifactEventSchema } from "./schemas/RecordingArtifactEventSchema"; +import { TranscriptArtifactEventSchema } from "./schemas/TranscriptArtifactEventSchema"; +import { create_async_transcripts_for_all_providers, save_provider_transcript } from "./transcription_provider_comparison_async"; + +const server = http.createServer(); + +server.on("request", async (req, res) => { + try { + if (req.method !== "POST") { + res.writeHead(405, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "Method not allowed" })); + return; + } + + const body_chunks: Buffer[] = []; + for await (const chunk of req) { + body_chunks.push(chunk); + } + const raw_body = Buffer.concat(body_chunks).toString("utf-8"); + const body = JSON.parse(raw_body); + + const result = z.discriminatedUnion("event", [ + RecordingArtifactEventSchema, + TranscriptArtifactEventSchema, + ]).safeParse(body); + + if (!result.success) { + console.log(`[Recording=${body?.data?.recording?.id ?? "N/A"}] Received unhandled webhook event: ${body?.event}`); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ success: true })); + return; + } + + const { data: msg } = result; + console.log(`[Recording=${msg.data.recording.id}] Received webhook event: ${msg.event}`); + + switch (msg.event) { + case "recording.done": { + await create_async_transcripts_for_all_providers({ recording_id: msg.data.recording.id }); + break; + } + case "transcript.done": { + const { provider_name } = await save_provider_transcript({ msg: body }); + console.log(`[Recording=${msg.data.recording.id}] Saved ${provider_name} transcript to output files`); + break; + } + case "transcript.failed": { + console.error(`[Recording=${msg.data.recording.id}] Transcript failed: ${msg.data.data.sub_code}`); + break; + } + default: { + console.log(`[Recording=${msg.data.recording.id}] Received event: ${msg.event}`); + } + } + } catch (error) { + console.error(`Error handling webhook event: ${req.method} ${req.url}`, error); + } + + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ success: true })); +}); + +server.listen(env.PORT, "0.0.0.0", () => { + console.log(`Server is running on port ${env.PORT}`); + console.log(`Configured providers: ${PROVIDER_CONFIGS.map(get_provider_name).join(", ")}`); +}); diff --git a/transcription_provider_comparison_async/src/schemas/EnvSchema.ts b/transcription_provider_comparison_async/src/schemas/EnvSchema.ts new file mode 100644 index 0000000..d41d583 --- /dev/null +++ b/transcription_provider_comparison_async/src/schemas/EnvSchema.ts @@ -0,0 +1,7 @@ +import { z } from "zod"; + +export const EnvSchema = z.object({ + PORT: z.coerce.number().default(4000), + RECALL_REGION: z.string(), + RECALL_API_KEY: z.string(), +}); diff --git a/transcription_provider_comparison_async/src/schemas/RecordingArtifactEventSchema.ts b/transcription_provider_comparison_async/src/schemas/RecordingArtifactEventSchema.ts new file mode 100644 index 0000000..8561805 --- /dev/null +++ b/transcription_provider_comparison_async/src/schemas/RecordingArtifactEventSchema.ts @@ -0,0 +1,24 @@ +import { z } from "zod"; + +export const RecordingArtifactEventSchema = z.object({ + event: z.enum([ + "recording.processing", + "recording.done", + "recording.failed", + "recording.deleted", + "recording.paused", + ]), + data: z.object({ + data: z.object({ + code: z.string(), + sub_code: z.string().nullable(), + updated_at: z.string(), + }), + recording: z.object({ + id: z.string(), + metadata: z.record(z.string(), z.string()), + }), + }), +}); + +export type RecordingArtifactEventType = z.infer; diff --git a/transcription_provider_comparison_async/src/schemas/TranscriptArtifactEventSchema.ts b/transcription_provider_comparison_async/src/schemas/TranscriptArtifactEventSchema.ts new file mode 100644 index 0000000..4e9bce0 --- /dev/null +++ b/transcription_provider_comparison_async/src/schemas/TranscriptArtifactEventSchema.ts @@ -0,0 +1,27 @@ +import { z } from "zod"; + +export const TranscriptArtifactEventSchema = z.object({ + event: z.enum([ + "transcript.processing", + "transcript.done", + "transcript.failed", + "transcript.deleted", + ]), + data: z.object({ + data: z.object({ + code: z.string(), + sub_code: z.string().nullable(), + updated_at: z.string(), + }), + transcript: z.object({ + id: z.string(), + metadata: z.record(z.string(), z.string()), + }), + recording: z.object({ + id: z.string(), + metadata: z.record(z.string(), z.string()), + }), + }), +}); + +export type TranscriptArtifactEventType = z.infer; diff --git a/transcription_provider_comparison_async/src/schemas/TranscriptArtifactSchema.ts b/transcription_provider_comparison_async/src/schemas/TranscriptArtifactSchema.ts new file mode 100644 index 0000000..428cc8e --- /dev/null +++ b/transcription_provider_comparison_async/src/schemas/TranscriptArtifactSchema.ts @@ -0,0 +1,25 @@ +import { z } from "zod"; + +export const TranscriptArtifactSchema = z.object({ + id: z.string(), + created_at: z.string(), + status: z.object({ + code: z.enum(["processing", "done", "failed", "deleted"]), + sub_code: z.string().nullable(), + updated_at: z.string(), + }), + data: z.object({ + download_url: z.string().url().nullable(), + provider_data_download_url: z.string().url().nullish(), + }), + diarization: z.object({ + use_separate_streams_when_available: z.boolean(), + }).nullable(), + provider: z.record(z.string(), z.any()), +}); + +export type TranscriptArtifactType = z.infer; + +export function get_provider_name(transcript: TranscriptArtifactType): string { + return Object.keys(transcript.provider)[0] ?? "unknown"; +} diff --git a/transcription_provider_comparison_async/src/schemas/TranscriptPartSchema.ts b/transcription_provider_comparison_async/src/schemas/TranscriptPartSchema.ts new file mode 100644 index 0000000..313ec64 --- /dev/null +++ b/transcription_provider_comparison_async/src/schemas/TranscriptPartSchema.ts @@ -0,0 +1,25 @@ +import { z } from "zod"; + +export const TranscriptPartSchema = z.object({ + participant: z.object({ + id: z.number().nullable(), + name: z.string().nullable(), + is_host: z.boolean().nullable(), + platform: z.string().nullable(), + extra_data: z.any().nullable(), + email: z.string().nullish(), + }), + words: z.object({ + text: z.string(), + start_timestamp: z.object({ + relative: z.number(), + absolute: z.string().nullish(), + }).nullish(), + end_timestamp: z.object({ + relative: z.number(), + absolute: z.string().nullish(), + }).nullish(), + }).array(), +}); + +export type TranscriptPartType = z.infer; diff --git a/transcription_provider_comparison_async/src/transcription_provider_comparison_async.ts b/transcription_provider_comparison_async/src/transcription_provider_comparison_async.ts new file mode 100644 index 0000000..f534ed7 --- /dev/null +++ b/transcription_provider_comparison_async/src/transcription_provider_comparison_async.ts @@ -0,0 +1,141 @@ +import fs from "fs"; +import path from "path"; +import { z } from "zod"; +import { env } from "./config/env"; +import { PROVIDER_CONFIGS, type ProviderConfig, get_provider_name } from "./config/providers"; +import { convert_to_readable_transcript } from "./convert_to_readable_transcript"; +import { fetch_with_retry } from "./fetch_with_retry"; +import { TranscriptArtifactEventSchema, type TranscriptArtifactEventType } from "./schemas/TranscriptArtifactEventSchema"; +import { TranscriptArtifactSchema, get_provider_name as get_transcript_provider_name } from "./schemas/TranscriptArtifactSchema"; +import { TranscriptPartSchema } from "./schemas/TranscriptPartSchema"; + +/** + * Create async transcript jobs for all configured providers. + * Each provider will generate a separate transcript for comparison. + */ +export async function create_async_transcripts_for_all_providers(args: { recording_id: string }) { + const { recording_id } = z.object({ recording_id: z.string() }).parse(args); + + const provider_names = PROVIDER_CONFIGS.map(get_provider_name); + console.log(`[Recording=${recording_id}] Creating transcripts for ${PROVIDER_CONFIGS.length} providers: ${provider_names.join(", ")}`); + + const results = await Promise.allSettled( + PROVIDER_CONFIGS.map((provider) => create_async_transcript({ recording_id, provider })), + ); + + const summary = results.map((result, i) => ({ + provider: get_provider_name(PROVIDER_CONFIGS[i]), + status: result.status, + transcript_id: result.status === "fulfilled" ? result.value.id : null, + error: result.status === "rejected" ? String(result.reason) : null, + })); + + console.log(`[Recording=${recording_id}] Transcript creation results:`, JSON.stringify(summary, null, 2)); + + return summary; +} + +/** + * Create an async transcript job for a specific provider. + */ +async function create_async_transcript(args: { recording_id: string; provider: ProviderConfig }) { + const { recording_id, provider } = args; + + const response = await fetch_with_retry( + `https://${env.RECALL_REGION}.recall.ai/api/v1/recording/${recording_id}/create_transcript/`, + { + method: "POST", + headers: { + "Authorization": `${env.RECALL_API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + provider, + diarization: { use_separate_streams_when_available: true }, + metadata: { + provider_name: get_provider_name(provider), + }, + }), + }, + ); + + if (!response.ok) { + const error_text = await response.text(); + throw new Error(`Failed to create transcript with ${get_provider_name(provider)}: ${error_text}`); + } + + return TranscriptArtifactSchema.parse(await response.json()); +} + +/** + * Retrieve and save the transcript for a specific provider. + */ +export async function save_provider_transcript(args: { msg: TranscriptArtifactEventType }) { + const { msg } = z.object({ msg: TranscriptArtifactEventSchema }).parse(args); + + const transcript = await retrieve_transcript({ transcript_id: msg.data.transcript.id }); + const provider_name = get_transcript_provider_name(transcript); + + if (!transcript.data.download_url) { + throw new Error(`[${provider_name}] Transcript download URL is null`); + } + + const transcript_parts = await retrieve_transcript_parts({ download_url: transcript.data.download_url }); + const readable_transcript = convert_to_readable_transcript({ transcript_parts }); + + const output_dir = path.join( + process.cwd(), + `output/recording-${msg.data.recording.id}/${provider_name}`, + ); + fs.mkdirSync(output_dir, { recursive: true }); + + fs.writeFileSync( + path.join(output_dir, "transcript.json"), + JSON.stringify(transcript_parts, null, 2), + ); + + fs.writeFileSync( + path.join(output_dir, "readable.txt"), + readable_transcript.map((t) => t ? `${t.speaker}: ${t.paragraph}` : "").join("\n"), + ); + + fs.writeFileSync( + path.join(output_dir, "metadata.json"), + JSON.stringify(transcript, null, 2), + ); + + return { provider_name, transcript_parts, readable_transcript }; +} + +/** + * Retrieve transcript artifact by ID. + */ +async function retrieve_transcript(args: { transcript_id: string }) { + const { transcript_id } = z.object({ transcript_id: z.string() }).parse(args); + + const response = await fetch_with_retry( + `https://${env.RECALL_REGION}.recall.ai/api/v1/transcript/${transcript_id}/`, + { + method: "GET", + headers: { + "Authorization": `${env.RECALL_API_KEY}`, + "Content-Type": "application/json", + }, + }, + ); + + if (!response.ok) throw new Error(await response.text()); + return TranscriptArtifactSchema.parse(await response.json()); +} + +/** + * Retrieve transcript parts from download URL. + */ +async function retrieve_transcript_parts(args: { download_url: string }) { + const { download_url } = z.object({ download_url: z.string() }).parse(args); + + const response = await fetch(download_url); + if (!response.ok) throw new Error(await response.text()); + + return TranscriptPartSchema.array().parse(await response.json()); +} diff --git a/transcription_provider_comparison_async/tsconfig.json b/transcription_provider_comparison_async/tsconfig.json new file mode 100644 index 0000000..3332414 --- /dev/null +++ b/transcription_provider_comparison_async/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"] +} From 28a3c9a0cb3b0010c88c2f78941ca7c7a91635d7 Mon Sep 17 00:00:00 2001 From: Gerry Saporito Date: Wed, 25 Feb 2026 09:41:00 -0800 Subject: [PATCH 2/5] save failed error code --- .../src/index.ts | 5 ++-- ...transcription_provider_comparison_async.ts | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/transcription_provider_comparison_async/src/index.ts b/transcription_provider_comparison_async/src/index.ts index 47a1217..c23d9e4 100644 --- a/transcription_provider_comparison_async/src/index.ts +++ b/transcription_provider_comparison_async/src/index.ts @@ -4,7 +4,7 @@ import { env } from "./config/env"; import { PROVIDER_CONFIGS, get_provider_name } from "./config/providers"; import { RecordingArtifactEventSchema } from "./schemas/RecordingArtifactEventSchema"; import { TranscriptArtifactEventSchema } from "./schemas/TranscriptArtifactEventSchema"; -import { create_async_transcripts_for_all_providers, save_provider_transcript } from "./transcription_provider_comparison_async"; +import { create_async_transcripts_for_all_providers, save_provider_transcript, save_failed_transcript } from "./transcription_provider_comparison_async"; const server = http.createServer(); @@ -49,7 +49,8 @@ server.on("request", async (req, res) => { break; } case "transcript.failed": { - console.error(`[Recording=${msg.data.recording.id}] Transcript failed: ${msg.data.data.sub_code}`); + const { provider_name, error } = await save_failed_transcript({ msg: body }); + console.error(`[Recording=${msg.data.recording.id}] Saved ${provider_name} failure: ${error.code} / ${error.sub_code}`); break; } default: { diff --git a/transcription_provider_comparison_async/src/transcription_provider_comparison_async.ts b/transcription_provider_comparison_async/src/transcription_provider_comparison_async.ts index f534ed7..f188139 100644 --- a/transcription_provider_comparison_async/src/transcription_provider_comparison_async.ts +++ b/transcription_provider_comparison_async/src/transcription_provider_comparison_async.ts @@ -107,6 +107,32 @@ export async function save_provider_transcript(args: { msg: TranscriptArtifactEv return { provider_name, transcript_parts, readable_transcript }; } +/** + * Save failed transcript information to output directory. + */ +export async function save_failed_transcript(args: { msg: TranscriptArtifactEventType }) { + const { msg } = z.object({ msg: TranscriptArtifactEventSchema }).parse(args); + + const transcript = await retrieve_transcript({ transcript_id: msg.data.transcript.id }); + const provider_name = get_transcript_provider_name(transcript); + + const output_dir = path.join( + process.cwd(), + `output/recording-${msg.data.recording.id}/${provider_name}`, + ); + fs.mkdirSync(output_dir, { recursive: true }); + + fs.writeFileSync( + path.join(output_dir, "error.json"), + JSON.stringify({ + transcript, + event: msg, + }, null, 2), + ); + + return { provider_name, error: msg.data.data }; +} + /** * Retrieve transcript artifact by ID. */ From 27e92f49c1e4abe46c5c5636f94b25ccd52c9294 Mon Sep 17 00:00:00 2001 From: Gerry Saporito Date: Wed, 25 Feb 2026 10:49:30 -0800 Subject: [PATCH 3/5] added readme --- .../README.md | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 transcription_provider_comparison_async/README.md diff --git a/transcription_provider_comparison_async/README.md b/transcription_provider_comparison_async/README.md new file mode 100644 index 0000000..2d2a962 --- /dev/null +++ b/transcription_provider_comparison_async/README.md @@ -0,0 +1,157 @@ +# Transcription Provider Comparison for Async Transcription + +A tool for comparing transcription quality and output across multiple third-party providers using Recall.ai's async transcription API. + +## What it does + +When a meeting recording completes, this tool automatically transcribes the same audio using every configured provider and saves the results side-by-side. This lets you evaluate how each of the transcription providers work for your use case, including: + +- **Transcription accuracy** across providers with the same source audio +- **Multilingual support** and code-switching capabilities + +This helps you identify which provider works best for your specific use case. + +## Output + +Results are organized by recording and provider: + +``` +output/recording-{id}/ +├── recallai_async/ +│ ├── transcript.json # Raw transcript data +│ ├── readable.txt # Human-readable format +│ └── metadata.json # Provider config and timing +├── assembly_ai_async/ +│ └── ... +├── deepgram_async/ +│ └── error.json # Saved if transcription failed +└── ... +``` + +## Supported Providers + +Configure in `src/config/providers.ts`: + +| Provider | Code Switching | Language Detection | +| -------------------- | ----------------- | ------------------ | +| `recallai_async` | ✅ | ✅ auto | +| `assembly_ai_async` | ✅ | ✅ universal model | +| `deepgram_async` | ✅ | ✅ nova-3 multi | +| `gladia_v2_async` | ✅ | ✅ | +| `speechmatics_async` | 🚧 language pairs | 🚧 | +| `rev_async` | ❌ | ❌ | +| `google_cloud_stt` | ❌ | ❌ | + +> Each provider requires an API key configured in the [Recall dashboard](https://us-west-2.recall.ai/dashboard/transcription). + +## Pre-requisites + +- [ngrok](https://ngrok.com/) +- [Node.js](https://nodejs.org/en/download) +- [NPM](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) + +## Quickstart + +**Before running, make sure you don't have any apps running on port 4000** + +### 1. Start ngrok + +In a terminal window, run: + +```bash +ngrok http 4000 +``` + +After it's running, copy the ngrok URL (e.g. `somehash.ngrok-free.app`). + +### 2. Set up environment variables + +Copy the `.env.sample` file and rename it to `.env`: + +```bash +cp .env.sample .env +``` + +Then fill out the variables in the `.env` file, including the ngrok domain from step 1 (Don't forget to omit the protocol (e.g. `https://`)) + +### 3. Configure providers + +Edit `src/config/providers.ts` to enable the providers you want to compare: + +```typescript +export const PROVIDER_CONFIGS = [ + { recallai_async: { language_code: "auto" } }, + { + assembly_ai_async: { + speech_model: "universal", + language_detection: true, + }, + }, + { deepgram_async: { model: "nova-3", language: "multi" } }, + // Uncomment or add more providers as needed +]; +``` + +### 4. Add your webhook URL to the Recall dashboard + +Go to the Recall.ai webhooks dashboard for your region and add your ngrok URL as a webhook: + +- [`us-east-1` webhooks dashboard](https://us-east-1.recall.ai/dashboard/webhooks) +- [`us-west-2` webhooks dashboard](https://us-west-2.recall.ai/dashboard/webhooks) +- [`eu-central-1` webhooks dashboard](https://eu-central-1.recall.ai/dashboard/webhooks) +- [`ap-northeast-1` webhooks dashboard](https://ap-northeast-1.recall.ai/dashboard/webhooks) + +Make sure to subscribe to the following webhook events: + +- `recording.done` +- `transcript.done` +- `transcript.failed` + +### 5. Start the server + +Open this directory in a new terminal and run: + +```bash +npm install +npm run dev +``` + +This will start a server on port 4000. + +### 6. Create a bot + +You can create a bot using the `run.sh` script or manually with curl. + +#### Option A: Using run.sh (recommended) + +In a new terminal, run the script: + +```bash +chmod +x run.sh +./run.sh +``` + +This will create a bot and paste the response in the terminal. + +#### Option B: Using curl + +```bash +curl --request POST \ + --url https://RECALL_REGION.recall.ai/api/v1/bot/ \ + --header 'Authorization: RECALL_API_KEY' \ + --header 'accept: application/json' \ + --header 'content-type: application/json' \ + --data '{ + "meeting_url": "YOUR_MEETING_URL" + }' +``` + +**Note:** + +- Replace `RECALL_REGION`, `RECALL_API_KEY`, and `YOUR_MEETING_URL` with your own values. + +## Resources + +- [Async Transcription](https://docs.recall.ai/docs/async-transcription) +- [Third-Party Providers](https://docs.recall.ai/docs/ai-transcription) +- [Multilingual Transcription](https://docs.recall.ai/docs/multilingual-transcription) From 36c43b7c67d653db469806cd49b6d515053bccb1 Mon Sep 17 00:00:00 2001 From: Gerry Saporito Date: Wed, 25 Feb 2026 10:50:34 -0800 Subject: [PATCH 4/5] readme --- .../README.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/transcription_provider_comparison_async/README.md b/transcription_provider_comparison_async/README.md index 2d2a962..dcbd73b 100644 --- a/transcription_provider_comparison_async/README.md +++ b/transcription_provider_comparison_async/README.md @@ -30,19 +30,9 @@ output/recording-{id}/ ## Supported Providers -Configure in `src/config/providers.ts`: - -| Provider | Code Switching | Language Detection | -| -------------------- | ----------------- | ------------------ | -| `recallai_async` | ✅ | ✅ auto | -| `assembly_ai_async` | ✅ | ✅ universal model | -| `deepgram_async` | ✅ | ✅ nova-3 multi | -| `gladia_v2_async` | ✅ | ✅ | -| `speechmatics_async` | 🚧 language pairs | 🚧 | -| `rev_async` | ❌ | ❌ | -| `google_cloud_stt` | ❌ | ❌ | - -> Each provider requires an API key configured in the [Recall dashboard](https://us-west-2.recall.ai/dashboard/transcription). +See the [Third-Party Transcription docs](https://docs.recall.ai/docs/ai-transcription) for the full list of supported providers and their configurations. For multilingual and code-switching support, see the [Multilingual Transcription docs](https://docs.recall.ai/docs/multilingual-transcription). + +Configure which providers to compare in `src/config/providers.ts`. Each provider requires an API key configured in the [Recall dashboard](https://us-west-2.recall.ai/dashboard/transcription). ## Pre-requisites From 3e5b8f0afe2e38e152a22accdfb082305c050b24 Mon Sep 17 00:00:00 2001 From: Gerry Saporito Date: Wed, 25 Feb 2026 11:20:01 -0800 Subject: [PATCH 5/5] Readme --- .../README.md | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/transcription_provider_comparison_async/README.md b/transcription_provider_comparison_async/README.md index dcbd73b..da28d71 100644 --- a/transcription_provider_comparison_async/README.md +++ b/transcription_provider_comparison_async/README.md @@ -4,12 +4,12 @@ A tool for comparing transcription quality and output across multiple third-part ## What it does -When a meeting recording completes, this tool automatically transcribes the same audio using every configured provider and saves the results side-by-side. This lets you evaluate how each of the transcription providers work for your use case, including: +When a meeting recording completes, this tool automatically transcribes the same audio using every configured provider and saves the results side-by-side. This lets you compare: - **Transcription accuracy** across providers with the same source audio - **Multilingual support** and code-switching capabilities -This helps you identify which provider works best for your specific use case. +You can also add your own specific configs to each provider to test out provider-specific features. ## Output @@ -18,7 +18,7 @@ Results are organized by recording and provider: ``` output/recording-{id}/ ├── recallai_async/ -│ ├── transcript.json # Raw transcript data +│ ├── transcript.json # Raw transcript data Recall will provide you │ ├── readable.txt # Human-readable format │ └── metadata.json # Provider config and timing ├── assembly_ai_async/ @@ -30,9 +30,19 @@ output/recording-{id}/ ## Supported Providers -See the [Third-Party Transcription docs](https://docs.recall.ai/docs/ai-transcription) for the full list of supported providers and their configurations. For multilingual and code-switching support, see the [Multilingual Transcription docs](https://docs.recall.ai/docs/multilingual-transcription). +Some initial notes: -Configure which providers to compare in `src/config/providers.ts`. Each provider requires an API key configured in the [Recall dashboard](https://us-west-2.recall.ai/dashboard/transcription). +- For the full list of supported providers and their configurations, see the [Third-Party Transcription docs](https://docs.recall.ai/docs/ai-transcription). +- For multilingual and code-switching support, see the [Multilingual Transcription docs](https://docs.recall.ai/docs/multilingual-transcription). + +You can configure which providers to compare in `src/config/providers.ts`. + +Each provider requires an API key configured in the Recall dashboard: + +- [`us-east-1` transcription dashboard](https://us-east-1.recall.ai/dashboard/transcription) +- [`us-west-2` transcription dashboard](https://us-west-2.recall.ai/dashboard/transcription) +- [`eu-central-1` transcription dashboard](https://eu-central-1.recall.ai/dashboard/transcription) +- [`ap-northeast-1` transcription dashboard](https://ap-northeast-1.recall.ai/dashboard/transcription) ## Pre-requisites @@ -82,6 +92,8 @@ export const PROVIDER_CONFIGS = [ ]; ``` +These configs are the same as the ones defined in the `provider` field in the [Create Async Transcript API reference](https://docs.recall.ai/reference/recording_create_transcript_create). + ### 4. Add your webhook URL to the Recall dashboard Go to the Recall.ai webhooks dashboard for your region and add your ngrok URL as a webhook: