diff --git a/.gitignore b/.gitignore index 593347ef2b0..d321b20680b 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ clean/ # Dart/Flutter .dart_tool/ **/pubspec.lock + +compare-report/ +compare-cache/ diff --git a/canary-matrix.json b/canary-matrix.json new file mode 100644 index 00000000000..21be318a2b3 --- /dev/null +++ b/canary-matrix.json @@ -0,0 +1,47 @@ +[ + { + "name": "Angular 19 Canary Matrix", + "variants": [ + { "id": "Local-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-19", "runtime": "nodejs24", "localBuild": true }, + { "id": "Source-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-19", "runtime": "nodejs24", "localBuild": false }, + { "id": "Source-Node22", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-19", "runtime": "nodejs22", "localBuild": false }, + { "id": "Source-NoABIU", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-19", "runtime": "nodejs", "localBuild": false } + ] + }, + { + "name": "Angular 20 Canary Matrix", + "variants": [ + { "id": "Local-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-20", "runtime": "nodejs24", "localBuild": true }, + { "id": "Source-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-20", "runtime": "nodejs24", "localBuild": false }, + { "id": "Source-Node22", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-20", "runtime": "nodejs22", "localBuild": false }, + { "id": "Source-NoABIU", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-20", "runtime": "nodejs", "localBuild": false } + ] + }, + { + "name": "Angular 21 Canary Matrix", + "variants": [ + { "id": "Local-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-21", "runtime": "nodejs24", "localBuild": true }, + { "id": "Source-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-21", "runtime": "nodejs24", "localBuild": false }, + { "id": "Source-Node22", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-21", "runtime": "nodejs22", "localBuild": false }, + { "id": "Source-NoABIU", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-21", "runtime": "nodejs", "localBuild": false } + ] + }, + { + "name": "Angular 22 Canary Matrix", + "variants": [ + { "id": "Local-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-22", "runtime": "nodejs24", "localBuild": true }, + { "id": "Source-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-22", "runtime": "nodejs24", "localBuild": false }, + { "id": "Source-Node22", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-22", "runtime": "nodejs22", "localBuild": false }, + { "id": "Source-NoABIU", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-22", "runtime": "nodejs", "localBuild": false } + ] + }, + { + "name": "Next.js 15.3 Canary Matrix", + "variants": [ + { "id": "Local-Node24", "path": "../firebase-apphosting-canary/apps/nextjs-reference/next-15.3", "runtime": "nodejs24", "localBuild": true }, + { "id": "Source-Node24", "path": "../firebase-apphosting-canary/apps/nextjs-reference/next-15.3", "runtime": "nodejs24", "localBuild": false }, + { "id": "Source-Node22", "path": "../firebase-apphosting-canary/apps/nextjs-reference/next-15.3", "runtime": "nodejs22", "localBuild": false }, + { "id": "Source-NoABIU", "path": "../firebase-apphosting-canary/apps/nextjs-reference/next-15.3", "runtime": "nodejs", "localBuild": false } + ] + } +] diff --git a/src/apphosting/compare/README.md b/src/apphosting/compare/README.md new file mode 100644 index 00000000000..2a8b6affa13 --- /dev/null +++ b/src/apphosting/compare/README.md @@ -0,0 +1,69 @@ +# App Hosting N-Way Matrix Comparison Tool + +This is an experimental internal CLI tool built for the Firebase App Hosting team to dynamically verify the compatibility and performance of different infrastructure backends or application builds. + +It takes an array of $N$ configurations (variants) and automatically deploys them, discovering their routes, and dumping an $O(N^2)$ Pair-wise Cartesian Matrix of differences. + +## Capabilities + +1. **Standard CLI Fidelity**: Deployments are executed by automatically constructing a temporary `firebase.json` and programmatically calling the `firebase deploy` CLI execution path. This guarantees that test deployments faithfully mirror the actual customer experience (including Secrets, AutoInit Env Vars, and custom headers). +2. **Local vs Remote Build Verification**: Can deploy locally built bundles (e.g. `localBuild: true`) side-by-side with remote Cloud Build source zips. +3. **Automated IAM & Secrets Management**: Intelligently creates a single mock secret in Secret Manager for each distinct codebase path, mapping the IAM `secretAccessor` roles simultaneously to all backends generated from that codebase. +4. **Dynamic Spidering**: Automatically crawls Next.js / Angular apps recursively starting from `/` to discover hidden dynamic routes, alongside statically parsing `.next/prerender-manifest.json`. +5. **Exact Diff Inspection**: Generates HTML dashboards, JSON summaries, and specifically dumps the raw HTTP HTML outputs of each variant so engineers can run local diffs. + +## Usage + +1. Create a `matrix-test.json` file to define your test cases: + +```json +[ + { + "name": "Node Matrix Test", + "variants": [ + { + "id": "Local-Node24", + "path": "../next-sample-1", + "localBuild": true, + "runtime": "nodejs24" + }, + { + "id": "Source-Node24", + "path": "../next-sample-1", + "localBuild": false, + "runtime": "nodejs24" + }, + { + "id": "Source-Node22", + "path": "../next-sample-1", + "localBuild": false, + "runtime": "nodejs22" + } + ] + } +] +``` + +2. Run the command: + +```bash +FIREBASE_CLI_EXPERIMENTS=apphosting firebase apphosting:compare-suite --project --suite-config matrix-test.json +``` + +## How to Inspect Diffs + +When the command completes, it generates reports in the `./compare-report//` directory. +Inside this folder, you will see a subfolder for each pairwise comparison, such as `Local-Node24-vs-Source-Node24/`. + +Inside the pair folder: +- **`index.html`**: A beautifully styled visual dashboard showing percentage differences and mismatches. +- **`summary.json`**: The structured data representation. +- **`backendA/` and `backendB/`**: These folders contain the raw HTTP HTML bodies retrieved during the crawl! + +To manually inspect the exact diffs, you can use standard diff tools on the generated files: + +```bash +diff compare-report/Node\ Matrix\ Test/Local-Node24-vs-Source-Node24/backendA/index.html compare-report/Node\ Matrix\ Test/Local-Node24-vs-Source-Node24/backendB/index.html +``` + +Or you can right-click the files in VSCode and select "Select for Compare" and "Compare with Selected". diff --git a/src/apphosting/compare/cache.ts b/src/apphosting/compare/cache.ts new file mode 100644 index 00000000000..5e0e1251eef --- /dev/null +++ b/src/apphosting/compare/cache.ts @@ -0,0 +1,174 @@ +import * as fs from "fs-extra"; +import * as path from "path"; +import * as crypto from "crypto"; +import { logger } from "../../logger"; + +export interface RouteResponse { + status: number; + headers: Record; + body: string; + isBinary: boolean; +} + +export interface VariantRecording { + id: string; + testCaseName: string; + timestamp: string; + url: string; + routes: Record; + localBuild?: boolean; + runtime?: string; +} + +export function isRouteResponse(obj: unknown): obj is RouteResponse { + if (typeof obj !== "object" || obj === null) return false; + const o = obj as Record; + return ( + typeof o.status === "number" && + typeof o.headers === "object" && o.headers !== null && + typeof o.body === "string" && + typeof o.isBinary === "boolean" + ); +} + +export function isVariantRecording(obj: unknown): obj is VariantRecording { + if (typeof obj !== "object" || obj === null) return false; + const o = obj as Record; + + if (!( + typeof o.id === "string" && + typeof o.testCaseName === "string" && + typeof o.timestamp === "string" && + typeof o.url === "string" && + typeof o.routes === "object" && o.routes !== null + )) { + return false; + } + + if (o.localBuild !== undefined && typeof o.localBuild !== "boolean") { + return false; + } + + if (o.runtime !== undefined && typeof o.runtime !== "string") { + return false; + } + + const routes = o.routes as Record; + for (const key of Object.keys(routes)) { + if (!isRouteResponse(routes[key])) { + return false; + } + } + + return true; +} + +function isErrnoException(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && "code" in error; +} + +const CACHE_DIR = path.resolve(process.cwd(), "compare-cache"); + +function getRecordingPath(testCaseName: string, variantId: string): string { + const tcHash = crypto.createHash("sha256").update(testCaseName).digest("hex").slice(0, 8); + const vHash = crypto.createHash("sha256").update(variantId).digest("hex").slice(0, 8); + const safeTestCase = `${testCaseName.replace(/[^a-zA-Z0-9_-]/g, "_")}_${tcHash}`; + const safeVariant = `${variantId.replace(/[^a-zA-Z0-9_-]/g, "_")}_${vHash}`; + const resolvedPath = path.resolve(path.join(CACHE_DIR, "recordings", safeTestCase, `${safeVariant}.json`)); + + if (!resolvedPath.startsWith(path.resolve(CACHE_DIR))) { + throw new Error("Path traversal restriction violation"); + } + return resolvedPath; +} + +/** + * Saves a variant recording to the cache atomically. + */ +export async function saveRecording(recording: VariantRecording): Promise { + const filePath = getRecordingPath(recording.testCaseName, recording.id); + const tempPath = filePath + ".tmp"; + + await fs.ensureDir(path.dirname(tempPath)); + await fs.writeJson(tempPath, recording, { spaces: 2 }); + await fs.rename(tempPath, filePath); + + logger.info(`Saved recording to cache: ${filePath}`); +} + +/** + * Loads a variant recording from the cache. + */ +export async function loadRecording(testCaseName: string, variantId: string): Promise { + const filePath = getRecordingPath(testCaseName, variantId); + if (!(await fs.pathExists(filePath))) { + throw new Error(`No recording found in cache for variant "${variantId}" under test case "${testCaseName}"`); + } + const data = await fs.readJson(filePath); + if (!isVariantRecording(data)) { + throw new Error(`Invalid recording format in cache for variant "${variantId}" under test case "${testCaseName}"`); + } + return data; +} + +/** + * Lists all recorded test cases and their variants. + */ +export async function listRecordings(): Promise> { + const recordingsDir = path.join(CACHE_DIR, "recordings"); + if (!(await fs.pathExists(recordingsDir))) { + return {}; + } + + const result: Record = {}; + try { + const testCases = await fs.readdir(recordingsDir); + for (const tc of testCases) { + const tcDir = path.join(recordingsDir, tc); + try { + const stat = await fs.stat(tcDir); + if (!stat.isDirectory()) { + continue; + } + const files = await fs.readdir(tcDir); + const jsonFiles = files.filter((file) => file.endsWith(".json")); + if (jsonFiles.length === 0) { + continue; + } + const variantIds: string[] = []; + let originalTestCaseName = ""; + for (const file of jsonFiles) { + try { + const data = await fs.readJson(path.join(tcDir, file)); + if (isVariantRecording(data)) { + originalTestCaseName = data.testCaseName; + variantIds.push(data.id); + } else { + logger.debug(`Cache file ${file} does not match VariantRecording schema`); + } + } catch (readErr) { + // If a file is partially written or corrupted, skip it + logger.debug(`Failed to read metadata for cache file: ${file}`, readErr); + } + } + if (originalTestCaseName && variantIds.length > 0) { + result[originalTestCaseName] = variantIds; + } + } catch (err: unknown) { + if (isErrnoException(err) && err.code === "ENOENT") { + continue; // Directory was concurrently deleted/moved + } + throw err; + } + } + } catch (err: unknown) { + if (isErrnoException(err) && err.code === "ENOENT") { + return {}; + } + throw err; + } + + return result; +} + + diff --git a/src/apphosting/compare/compare.spec.ts b/src/apphosting/compare/compare.spec.ts new file mode 100644 index 00000000000..aed79b61e76 --- /dev/null +++ b/src/apphosting/compare/compare.spec.ts @@ -0,0 +1,92 @@ +import { expect } from "chai"; +import * as nock from "nock"; +import { compareRoute } from "./compare"; + +describe("compareRoute", () => { + beforeEach(() => { + nock.cleanAll(); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + it("should match identical text pages", async () => { + nock("https://backend-a.com").get("/").reply(200, "hello world", { + "Content-Type": "text/html", + "Cache-Control": "public, max-age=3600", + }); + nock("https://backend-b.com").get("/").reply(200, "hello world", { + "Content-Type": "text/html", + "Cache-Control": "public, max-age=3600", + }); + + const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); + expect(res.statusMatch).to.be.true; + expect(res.headerMismatches).to.be.empty; + expect(res.bodySimilarity).to.equal(1.0); + expect(res.isBinary).to.be.false; + }); + + it("should flag status mismatches", async () => { + nock("https://backend-a.com").get("/").reply(200, "ok"); + nock("https://backend-b.com").get("/").reply(404, "not found"); + + const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); + expect(res.statusMatch).to.be.false; + }); + + it("should flag behavioral header mismatches but record dynamic ones as expected variations", async () => { + nock("https://backend-a.com").get("/").reply(200, "ok", { + "Cache-Control": "public, max-age=3600", + Date: "Wed, 17 Jun 2026 01:00:00 GMT", + }); + nock("https://backend-b.com").get("/").reply(200, "ok", { + "Cache-Control": "no-cache", + Date: "Wed, 17 Jun 2026 01:05:00 GMT", + }); + + const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); + expect(res.headerMismatches).to.have.lengthOf(1); + expect(res.headerMismatches[0].header.toLowerCase()).to.equal("cache-control"); + + expect(res.expectedHeaderVariations).to.have.lengthOf(1); + expect(res.expectedHeaderVariations[0].header.toLowerCase()).to.equal("date"); + }); + + it("should match identical binary pages and flag mismatching binary content", async () => { + const binA = Buffer.from([1, 2, 3, 4]); + const binB = Buffer.from([1, 2, 3, 4]); + const binC = Buffer.from([1, 2, 3, 5]); + + nock("https://backend-a.com").get("/img.png").reply(200, binA, { + "Content-Type": "image/png", + }); + nock("https://backend-b.com").get("/img.png").reply(200, binB, { + "Content-Type": "image/png", + }); + + const resMatch = await compareRoute( + "/img.png", + "https://backend-a.com", + "https://backend-b.com", + ); + expect(resMatch.isBinary).to.be.true; + expect(resMatch.bodySimilarity).to.equal(1.0); + + nock("https://backend-a.com").get("/img.png").reply(200, binA, { + "Content-Type": "image/png", + }); + nock("https://backend-b.com").get("/img.png").reply(200, binC, { + "Content-Type": "image/png", + }); + + const resMismatch = await compareRoute( + "/img.png", + "https://backend-a.com", + "https://backend-b.com", + ); + expect(resMismatch.isBinary).to.be.true; + expect(resMismatch.bodySimilarity).to.equal(0.0); + }); +}); diff --git a/src/apphosting/compare/compare.ts b/src/apphosting/compare/compare.ts new file mode 100644 index 00000000000..6b889e1b53a --- /dev/null +++ b/src/apphosting/compare/compare.ts @@ -0,0 +1,202 @@ +import fetch from "node-fetch"; +import * as crypto from "crypto"; +import { MyersDiffEngine } from "./distance"; + +export interface ComparisonResult { + route: string; + statusMatch: boolean; + statusA?: number; + statusB?: number; + headerMismatches: Array<{ header: string; valA: string; valB: string }>; + expectedHeaderVariations: Array<{ header: string; valA: string; valB: string }>; + bodySimilarity: number; // 0.0 to 1.0 + bodyDiff: string; + isBinary: boolean; + bodyA?: string; + bodyB?: string; +} + +const BEHAVIORAL_HEADERS = [ + "cache-control", + "content-security-policy", + "content-type", + "content-encoding", + "location", + "strict-transport-security", +]; + +const BINARY_CONTENT_TYPES = [ + "image/", + "application/pdf", + "application/zip", + "application/octet-stream", +]; + +function isBinaryContentType(contentType: string): boolean { + const normalized = contentType.toLowerCase(); + return BINARY_CONTENT_TYPES.some((type) => normalized.includes(type)); +} + +/** + * + */ +export async function compareRoute( + route: string, + urlA: string, + urlB: string, + options: { headers?: Record } = {}, +): Promise { + const fetchOptions = { + headers: options.headers || {}, + redirect: "manual" as const, + size: 2 * 1024 * 1024, + }; + + const [resA, resB] = await Promise.all([ + fetch(`${urlA}${route}`, fetchOptions), + fetch(`${urlB}${route}`, fetchOptions), + ]); + + const contentTypeA = resA.headers.get("content-type") || ""; + const contentTypeB = resB.headers.get("content-type") || ""; + const isBinaryA = isBinaryContentType(contentTypeA); + const isBinaryB = isBinaryContentType(contentTypeB); + + const headersA: Record = {}; + resA.headers.forEach((val, key) => { headersA[key.toLowerCase()] = val; }); + + const headersB: Record = {}; + resB.headers.forEach((val, key) => { headersB[key.toLowerCase()] = val; }); + + const responseA: RouteResponse = { + status: resA.status, + headers: headersA, + isBinary: isBinaryA || isBinaryB, + body: (isBinaryA || isBinaryB) ? (await resA.buffer()).toString("base64") : await resA.text(), + }; + + const responseB: RouteResponse = { + status: resB.status, + headers: headersB, + isBinary: isBinaryA || isBinaryB, + body: (isBinaryA || isBinaryB) ? (await resB.buffer()).toString("base64") : await resB.text(), + }; + + return await compareRouteResponses(route, responseA, responseB); +} + +export interface RouteResponse { + status: number; + headers: Record; + body: string; + isBinary: boolean; +} + +/** + * + */ +export async function compareRouteResponses( + route: string, + resA: RouteResponse, + resB: RouteResponse, +): Promise { + const result: ComparisonResult = { + route, + statusMatch: resA.status === resB.status, + statusA: resA.status, + statusB: resB.status, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 1.0, + bodyDiff: "", + isBinary: resA.isBinary || resB.isBinary, + }; + + // 1. Compare Headers + const normalizedHeadersA: Record = {}; + Object.entries(resA.headers).forEach(([k, v]) => { normalizedHeadersA[k.toLowerCase()] = v; }); + + const normalizedHeadersB: Record = {}; + Object.entries(resB.headers).forEach(([k, v]) => { normalizedHeadersB[k.toLowerCase()] = v; }); + + const allHeaderKeys = new Set([ + ...Object.keys(normalizedHeadersA), + ...Object.keys(normalizedHeadersB), + ]); + + for (const key of allHeaderKeys) { + const valA = normalizedHeadersA[key] || ""; + const valB = normalizedHeadersB[key] || ""; + if (valA !== valB) { + if (BEHAVIORAL_HEADERS.includes(key.toLowerCase())) { + result.headerMismatches.push({ header: key, valA, valB }); + } else { + result.expectedHeaderVariations.push({ header: key, valA, valB }); + } + } + } + + // 2. Compare Binary + if (result.isBinary) { + const bufA = Buffer.from(resA.body, "base64"); + const bufB = Buffer.from(resB.body, "base64"); + + const sizeA = bufA.length; + const sizeB = bufB.length; + + if (sizeA !== sizeB) { + result.bodySimilarity = 0.0; + result.bodyDiff = `Binary size mismatch: ${sizeA} bytes vs ${sizeB} bytes`; + } else { + const hashA = crypto.createHash("sha256").update(bufA).digest("hex"); + const hashB = crypto.createHash("sha256").update(bufB).digest("hex"); + if (hashA === hashB) { + result.bodySimilarity = 1.0; + } else { + result.bodySimilarity = 0.0; + result.bodyDiff = "Binary content hash mismatch"; + } + } + return result; + } + + // 3. Compare Text Body + let bodyA = resA.body; + let bodyB = resB.body; + + const contentType = (resA.headers["content-type"] || resA.headers["Content-Type"] || "").toLowerCase(); + if (contentType.includes("text/html")) { + try { + const prettier = require("prettier"); + const formattedA = await prettier.format(bodyA, { parser: "html" }); + const formattedB = await prettier.format(bodyB, { parser: "html" }); + bodyA = formattedA; + bodyB = formattedB; + } catch (e: any) { + // Fallback to advanced tag-based line splitting if prettier fails + const HTML_SPLIT_REGEX = /(<(script|style)\b[\s\S]*?<\/\2>||<[^'">]*(?:"[^"]*"[^'">]*|'[^']*'[^'">]*)*>)\s*(?=<)/gi; + bodyA = bodyA.replace(HTML_SPLIT_REGEX, "$1\n"); + bodyB = bodyB.replace(HTML_SPLIT_REGEX, "$1\n"); + } + } else if (contentType.includes("application/json") || route.endsWith(".json")) { + try { + bodyA = JSON.stringify(JSON.parse(bodyA), null, 2); + bodyB = JSON.stringify(JSON.parse(bodyB), null, 2); + } catch (e: any) { + // Fallback to raw text + } + } + + result.bodyA = bodyA; + result.bodyB = bodyB; + + if (bodyA !== bodyB) { + result.bodySimilarity = MyersDiffEngine.getSimilarity(bodyA, bodyB); + if (result.bodySimilarity < 1.0) { + result.bodyDiff = "HTML content mismatch"; + } + } + + return result; +} + diff --git a/src/apphosting/compare/crawler.spec.ts b/src/apphosting/compare/crawler.spec.ts new file mode 100644 index 00000000000..0713358b86d --- /dev/null +++ b/src/apphosting/compare/crawler.spec.ts @@ -0,0 +1,84 @@ +import { expect } from "chai"; +import * as nock from "nock"; +import { Crawler } from "./crawler"; + +describe("Crawler", () => { + beforeEach(() => { + nock.cleanAll(); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + it("should recursively crawl text/html links", async () => { + const base = "https://example.com"; + + nock(base) + .get("/") + .reply( + 200, + ` + + + About Us + Contact Us + External Link + + + `, + { "Content-Type": "text/html" }, + ); + + nock(base) + .get("/about") + .reply( + 200, + ` + + + Careers + + + `, + { "Content-Type": "text/html" }, + ); + + nock(base).get("/contact").reply(200, "Contact Details", { "Content-Type": "text/html" }); + + nock(base).get("/careers").reply(200, "Join our team", { "Content-Type": "text/html" }); + + const crawler = new Crawler(base); + await crawler.crawl(); + + const routes = crawler.getRoutes(); + expect(routes).to.deep.equal(["/", "/about", "/careers", "/contact"]); + }); + + it("should follow redirect links", async () => { + const base = "https://example.com"; + + nock(base) + .get("/") + .reply(200, `Redirect me`, { "Content-Type": "text/html" }); + + nock(base).get("/old-link").reply(302, undefined, { Location: "/new-link" }); + + nock(base).get("/new-link").reply(200, "Done!", { "Content-Type": "text/html" }); + + const crawler = new Crawler(base); + await crawler.crawl(); + + const routes = crawler.getRoutes(); + expect(routes).to.deep.equal(["/", "/new-link", "/old-link"]); + }); + + it("should canonicalize paths and sort queries", async () => { + const base = "https://example.com"; + const crawler = new Crawler(base); + + expect(crawler.canonicalizeRoute("/about/")).to.equal("/about"); + expect(crawler.canonicalizeRoute("/search?q=foo&a=bar")).to.equal("/search?a=bar&q=foo"); + expect(crawler.canonicalizeRoute("https://external.com/foo")).to.be.null; + }); +}); diff --git a/src/apphosting/compare/crawler.ts b/src/apphosting/compare/crawler.ts new file mode 100644 index 00000000000..36005011e33 --- /dev/null +++ b/src/apphosting/compare/crawler.ts @@ -0,0 +1,172 @@ +import fetch from "node-fetch"; + +function decodeHtmlEntities(str: string): string { + const entities: Record = { + "amp": "&", + "lt": "<", + "gt": ">", + "quot": '"', + "#39": "'" + }; + return str.replace(/&(amp|lt|gt|quot|#39);/gi, (match, entity) => { + return entities[entity.toLowerCase()] || match; + }); +} + +interface Destroyable { + destroy(): void; +} + +function isDestroyable(obj: unknown): obj is Destroyable { + return ( + typeof obj === "object" && + obj !== null && + "destroy" in obj && + typeof (obj as Record).destroy === "function" + ); +} + +export class Crawler { + private visited = new Set(); + private discoveredRoutes = new Set(); + + constructor( + private readonly baseUrl: string, + private readonly maxDepth = 3, + ) {} + + public getRoutes(): string[] { + return Array.from(this.discoveredRoutes).sort(); + } + + /** + * Crawls starting from root and recursively finds links. + */ + public async crawl(): Promise { + await this.crawlRoute("/"); + } + + private async crawlRoute(route: string, depth = 0): Promise { + const canonical = this.canonicalizeRoute(route); + if (!canonical || this.visited.has(canonical) || depth > this.maxDepth) { + return; + } + + this.visited.add(canonical); + this.discoveredRoutes.add(canonical); + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 15000); + + try { + const url = `${this.baseUrl}${canonical}`; + const res = await fetch(url, { + redirect: "manual" as const, + headers: { "User-Agent": "FirebaseCompareCrawler/1.0" }, + signal: controller.signal, + size: 2 * 1024 * 1024, + }); + + // Handle Redirects + if ([301, 302, 307, 308].includes(res.status)) { + if (res.body && isDestroyable(res.body)) { + res.body.destroy(); + } + const location = res.headers.get("location"); + if (location) { + const nextRoute = this.resolveRelative(canonical, location); + if (nextRoute) { + await this.crawlRoute(nextRoute, depth + 1); + } + } + return; + } + + // Only parse HTML responses + const contentType = res.headers.get("content-type") || ""; + if (!contentType.toLowerCase().includes("text/html")) { + if (res.body && isDestroyable(res.body)) { + res.body.destroy(); + } + return; + } + + const html = await res.text(); + const links = this.extractLinks(html, canonical); + + await Promise.all(links.map((link) => this.crawlRoute(link, depth + 1))); + } catch (err) { + // Ignore fetch failures for single routes during discovery + } finally { + clearTimeout(timeout); + } + } + + public canonicalizeRoute(route: string): string | null { + try { + const url = new URL(route, this.baseUrl); + if (url.origin !== new URL(this.baseUrl).origin) { + return null; + } + + let pathname = url.pathname; + if (pathname.endsWith("/") && pathname.length > 1) { + pathname = pathname.slice(0, -1); + } + + const params = Array.from(url.searchParams.entries()).sort((a, b) => + a[0].localeCompare(b[0]), + ); + const search = params.length > 0 ? "?" + params.map(([k, v]) => `${k}=${v}`).join("&") : ""; + + return `${pathname}${search}`; + } catch { + return null; + } + } + + private extractLinks(html: string, currentRoute: string): string[] { + const links: string[] = []; + const regex = /]*?\s+)?href\s*=\s*(?:(["'])(.*?)\1|([^\s>]+))/gi; + let match; + + while ((match = regex.exec(html)) !== null) { + const rawHref = match[2] !== undefined ? match[2] : match[3]; + if (!rawHref) continue; + + const href = decodeHtmlEntities(rawHref.trim()); + if (!href || href.startsWith("#")) { + continue; + } + + const schemeMatch = href.match(/^[a-z][a-z0-9+.-]*:/i); + if (schemeMatch) { + const scheme = schemeMatch[0].toLowerCase(); + if (scheme !== "http:" && scheme !== "https:") { + continue; + } + } + + const resolved = this.resolveRelative(currentRoute, href); + if (resolved) { + links.push(resolved); + } + } + + return links; + } + + private resolveRelative(currentRoute: string, href: string): string | null { + try { + const base = new URL(currentRoute, this.baseUrl); + const resolved = new URL(href, base.href); + if (resolved.origin !== new URL(this.baseUrl).origin) { + return null; + } + return resolved.pathname + resolved.search; + } catch { + return null; + } + } +} + diff --git a/src/apphosting/compare/discover.spec.ts b/src/apphosting/compare/discover.spec.ts new file mode 100644 index 00000000000..746095ecf98 --- /dev/null +++ b/src/apphosting/compare/discover.spec.ts @@ -0,0 +1,97 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; +import { discoverRoutes } from "./discover"; + +describe("discoverRoutes", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = path.join( + process.cwd(), + "scratch-test-discover-" + Math.random().toString(36).substring(7), + ); + fs.ensureDirSync(tempDir); + }); + + afterEach(() => { + fs.removeSync(tempDir); + }); + + it("should return root route by default if no manifests exist", async () => { + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/"]); + }); + + it("should discover routes from Next.js manifests", async () => { + const nextDir = path.join(tempDir, ".next"); + fs.ensureDirSync(nextDir); + + fs.writeJsonSync(path.join(nextDir, "prerender-manifest.json"), { + routes: { + "/about": {}, + "/blog/post-1": {}, + }, + }); + + fs.writeJsonSync(path.join(nextDir, "routes-manifest.json"), { + staticRoutes: [{ page: "/" }, { page: "/contact" }], + }); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/about", "/blog/post-1", "/contact"]); + }); + + it("should discover routes from sitemap.xml", async () => { + const xml = ` + + + + https://example.com/ + + + https://example.com/products?id=12 + + + `; + fs.writeFileSync(path.join(tempDir, "sitemap.xml"), xml, "utf-8"); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/products?id=12"]); + }); + + it("should discover routes from Next.js source app directory structure", async () => { + const appDir = path.join(tempDir, "src", "app"); + fs.ensureDirSync(path.join(appDir, "about")); + fs.ensureDirSync(path.join(appDir, "dashboard", "settings")); + fs.ensureDirSync(path.join(appDir, "blog", "[id]")); + fs.ensureDirSync(path.join(appDir, "(marketing)", "pricing")); + + fs.writeFileSync(path.join(appDir, "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "about", "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "dashboard", "settings", "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "blog", "[id]", "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "(marketing)", "pricing", "page.tsx"), "export default function P() {}"); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/about", "/blog/1", "/dashboard/settings", "/pricing"]); + }); + + it("should discover routes from Angular source TS routing modules", async () => { + const srcDir = path.join(tempDir, "src"); + fs.ensureDirSync(srcDir); + + const routingCode = ` + const routes: Routes = [ + { path: '', component: HomeComponent }, + { path: 'products', component: ProductsComponent }, + { path: 'user-profile', component: ProfileComponent }, + { path: '**', redirectTo: '' } + ]; + `; + fs.writeFileSync(path.join(srcDir, "app-routing.module.ts"), routingCode, "utf-8"); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/products", "/user-profile"]); + }); +}); diff --git a/src/apphosting/compare/discover.ts b/src/apphosting/compare/discover.ts new file mode 100644 index 00000000000..5050edf5818 --- /dev/null +++ b/src/apphosting/compare/discover.ts @@ -0,0 +1,201 @@ +import * as fs from "fs-extra"; +import * as path from "path"; +import { logger } from "../../logger"; + +/** + * Recursively scans a directory for files matching a pattern. + */ +async function scanDir( + dir: string, + fileCallback: (filePath: string) => void | Promise, +): Promise { + if (!(await fs.pathExists(dir))) return; + const files = await fs.readdir(dir); + for (const file of files) { + const fullPath = path.join(dir, file); + const stat = await fs.stat(fullPath); + if (stat.isDirectory()) { + if ( + file !== "node-modules" && + file !== "node_modules" && + file !== ".git" && + file !== ".next" && + file !== ".angular" && + file !== "dist" + ) { + await scanDir(fullPath, fileCallback); + } + } else { + await fileCallback(fullPath); + } + } +} + +/** + * Discovers Next.js source routes from src/app, app, src/pages, or pages directory + */ +export async function discoverSourceRoutes(appPath: string): Promise { + const routes = new Set(); + + // Check App Router (src/app or app) + const appDirs = [path.join(appPath, "src", "app"), path.join(appPath, "app")]; + for (const appDir of appDirs) { + if (await fs.pathExists(appDir)) { + await scanDir(appDir, (filePath) => { + const base = path.basename(filePath); + if (/^(page|route)\.[jt]sx?$/.test(base)) { + const relDir = path.relative(appDir, path.dirname(filePath)); + let route = relDir ? "/" + relDir : "/"; + // Normalize dynamic params: [id] -> 1, [...catchall] -> 1 + route = route.replace(/\[\.\.\.[^\]]+\]/g, "1"); + route = route.replace(/\[[^\]]+\]/g, "1"); + // Remove Next.js route groups like (marketing) + route = route.replace(/\/\([^)]+\)/g, ""); + routes.add(route === "" ? "/" : route); + } + }); + } + } + + // Check Pages Router (src/pages or pages) + const pagesDirs = [path.join(appPath, "src", "pages"), path.join(appPath, "pages")]; + for (const pagesDir of pagesDirs) { + if (await fs.pathExists(pagesDir)) { + await scanDir(pagesDir, (filePath) => { + const ext = path.extname(filePath); + if ([".js", ".jsx", ".ts", ".tsx"].includes(ext)) { + const base = path.basename(filePath, ext); + if (["_app", "_document", "_error", "api"].includes(base) || filePath.includes("/api/")) { + return; + } + const relPath = path.relative(pagesDir, filePath); + let route = "/" + relPath.substring(0, relPath.length - ext.length); + if (route.endsWith("/index")) { + route = route.substring(0, route.length - 6); + } + // Normalize dynamic parameters + route = route.replace(/\[\.\.\.[^\]]+\]/g, "1"); + route = route.replace(/\[[^\]]+\]/g, "1"); + routes.add(route === "" ? "/" : route); + } + }); + } + } + + return Array.from(routes); +} + +/** + * Scans source TS files for Angular style route paths like `path: 'about'` + */ +export async function discoverAngularRoutes(appPath: string): Promise { + const routes = new Set(); + const srcDir = path.join(appPath, "src"); + if (await fs.pathExists(srcDir)) { + await scanDir(srcDir, async (filePath) => { + if (filePath.endsWith(".ts") && !filePath.endsWith(".spec.ts")) { + try { + const content = await fs.readFile(filePath, "utf-8"); + // Match path: 'about' or path: "about" + const pathRegex = /path\s*:\s*(['"])(.*?)\1/g; + let match; + while ((match = pathRegex.exec(content)) !== null) { + const val = match[2].trim(); + // Skip wildcards and dynamic parameter placeholders + if (val && !val.includes("**") && !val.startsWith(":") && !val.includes("/")) { + routes.add("/" + val); + } + } + } catch { + // Ignore read errors + } + } + }); + } + return Array.from(routes); +} + +/** + * Discovers built routes in a project by checking Next.js manifests, source pages, and sitemaps. + */ +export async function discoverRoutes(appPath: string): Promise { + const routes = new Set(["/"]); + + // 1. Next.js Manifest Parsing (Local Build Output) + const nextDir = path.join(appPath, ".next"); + if (await fs.pathExists(nextDir)) { + logger.info("Next.js build directory detected. Parsing manifests..."); + try { + const prerenderManifestPath = path.join(nextDir, "prerender-manifest.json"); + if (await fs.pathExists(prerenderManifestPath)) { + const prerender = await fs.readJson(prerenderManifestPath); + if (prerender.routes) { + for (const route of Object.keys(prerender.routes)) { + routes.add(route); + } + } + } + + const routesManifestPath = path.join(nextDir, "routes-manifest.json"); + if (await fs.pathExists(routesManifestPath)) { + const manifests = await fs.readJson(routesManifestPath); + if (manifests.staticRoutes) { + for (const route of manifests.staticRoutes) { + routes.add(route.page); + } + } + } + } catch (err) { + logger.debug(`Error parsing Next.js manifests: ${err}`); + } + } + + // 2. Next.js Source Tree Traversal + try { + const srcRoutes = await discoverSourceRoutes(appPath); + for (const r of srcRoutes) { + routes.add(r); + } + } catch (err) { + logger.debug(`Error scanning Next.js source routes: ${err}`); + } + + // 3. Angular Routing Scanner + try { + const angularRoutes = await discoverAngularRoutes(appPath); + for (const r of angularRoutes) { + routes.add(r); + } + } catch (err) { + logger.debug(`Error scanning Angular source routes: ${err}`); + } + + // 4. Local Sitemap Parsing + const sitemapPaths = [ + path.join(appPath, "public", "sitemap.xml"), + path.join(appPath, "sitemap.xml"), + path.join(appPath, "dist", "sitemap.xml"), + ]; + + for (const sitemapPath of sitemapPaths) { + if (await fs.pathExists(sitemapPath)) { + logger.info(`Sitemap detected at ${sitemapPath}. Parsing...`); + try { + const xml = await fs.readFile(sitemapPath, "utf-8"); + const locMatches = xml.matchAll(/\s*(https?:\/\/[^\s<]+)\s*<\/loc>/gi); + for (const match of locMatches) { + try { + const url = new URL(match[1].trim()); + routes.add(url.pathname + url.search); + } catch { + // Ignore invalid URLs + } + } + } catch (err) { + logger.debug(`Error parsing sitemap ${sitemapPath}: ${err}`); + } + } + } + + return Array.from(routes).sort(); +} diff --git a/src/apphosting/compare/distance.spec.ts b/src/apphosting/compare/distance.spec.ts new file mode 100644 index 00000000000..86ee0be00a2 --- /dev/null +++ b/src/apphosting/compare/distance.spec.ts @@ -0,0 +1,36 @@ +import { expect } from "chai"; +import { MyersDiffEngine } from "./distance"; + +describe("MyersDiffEngine", () => { + describe("getSimilarity", () => { + it("should return 1.0 for exact string match", () => { + const a = "line 1\nline 2\nline 3"; + const b = "line 1\nline 2\nline 3"; + expect(MyersDiffEngine.getSimilarity(a, b)).to.equal(1.0); + }); + + it("should return 1.0 for both empty strings", () => { + expect(MyersDiffEngine.getSimilarity("", "")).to.equal(1.0); + }); + + it("should return 0.0 if one string is empty", () => { + expect(MyersDiffEngine.getSimilarity("hello", "")).to.equal(0.0); + expect(MyersDiffEngine.getSimilarity("", "world")).to.equal(0.0); + }); + + it("should return correct similarity for partial match", () => { + // 2 matched lines, total lines A = 3, total lines B = 3 + // Similarity = (2 * 2) / (3 + 3) = 4 / 6 = 0.6666... + const a = "line 1\nline 2\nline 3"; + const b = "line 1\nline 2\nline 4"; + const similarity = MyersDiffEngine.getSimilarity(a, b); + expect(similarity).to.be.closeTo(0.666, 0.001); + }); + + it("should return 0.0 for completely disjoint strings", () => { + const a = "foo\nbar"; + const b = "baz\nqux"; + expect(MyersDiffEngine.getSimilarity(a, b)).to.equal(0.0); + }); + }); +}); diff --git a/src/apphosting/compare/distance.ts b/src/apphosting/compare/distance.ts new file mode 100644 index 00000000000..60e4c7d3722 --- /dev/null +++ b/src/apphosting/compare/distance.ts @@ -0,0 +1,41 @@ +import { diffLines } from "diff"; +import * as crypto from "crypto"; + +export class MyersDiffEngine { + /** + * Calculates a similarity score between 0.0 and 1.0 based on line differences. + */ + public static getSimilarity(a: string, b: string): number { + if (a === b) return 1.0; + if (a.length === 0 && b.length === 0) return 1.0; + if (a.length === 0 || b.length === 0) return 0.0; + + // Fast-path: hash comparison + const hashA = crypto.createHash("sha256").update(a).digest("hex"); + const hashB = crypto.createHash("sha256").update(b).digest("hex"); + if (hashA === hashB) return 1.0; + + const changes = diffLines(a, b); + let matchedLines = 0; + let totalLinesA = 0; + let totalLinesB = 0; + + for (const change of changes) { + const count = change.count || 0; + if (!change.added && !change.removed) { + matchedLines += count; + totalLinesA += count; + totalLinesB += count; + } else if (change.added) { + totalLinesB += count; + } else if (change.removed) { + totalLinesA += count; + } + } + + const totalLines = totalLinesA + totalLinesB; + if (totalLines === 0) return 1.0; + + return (2 * matchedLines) / totalLines; + } +} diff --git a/src/apphosting/compare/lifecycle.spec.ts b/src/apphosting/compare/lifecycle.spec.ts new file mode 100644 index 00000000000..2107e9ea265 --- /dev/null +++ b/src/apphosting/compare/lifecycle.spec.ts @@ -0,0 +1,63 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; +import * as apphosting from "../../gcp/apphosting"; +import { validateProject, runGarbageCollection } from "./lifecycle"; + +describe("Lifecycle Manager", () => { + let listBackendsStub: sinon.SinonStub; + let updateBackendStub: sinon.SinonStub; + + beforeEach(() => { + listBackendsStub = sinon.stub(apphosting, "listBackends"); + updateBackendStub = sinon.stub(apphosting, "updateBackend"); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe("validateProject", () => { + it("should allow whitelisted projects", () => { + expect(() => validateProject("aryanf-test")).to.not.throw(); + expect(() => validateProject("pretend-public")).to.not.throw(); + }); + + it("should throw on non-whitelisted projects", () => { + expect(() => validateProject("my-secret-project")).to.throw( + /Invalid project ID "my-secret-project"/, + ); + }); + }); + + describe("runGarbageCollection", () => { + it("should unlock stale busy backends (> 2 hours)", async () => { + const now = Date.now(); + const threeHoursAgo = new Date(now - 3 * 60 * 60 * 1000).toISOString(); + const oneHourAgo = new Date(now - 1 * 60 * 60 * 1000).toISOString(); + + listBackendsStub.resolves({ + backends: [ + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-a", + labels: { status: "busy", type: "comparison-sandbox" }, + updateTime: threeHoursAgo, + }, + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-b", + labels: { status: "busy", type: "comparison-sandbox" }, + updateTime: oneHourAgo, + }, + ], + }); + + updateBackendStub.resolves({ name: "op-name" }); + + await runGarbageCollection("aryanf-test", "us-central1"); + + expect(updateBackendStub.callCount).to.equal(1); + const args = updateBackendStub.firstCall.args; + expect(args[2]).to.equal("compare-slot-1-a"); + expect(args[3].labels.status).to.equal("idle"); + }); + }); +}); diff --git a/src/apphosting/compare/lifecycle.ts b/src/apphosting/compare/lifecycle.ts new file mode 100644 index 00000000000..460c86ef053 --- /dev/null +++ b/src/apphosting/compare/lifecycle.ts @@ -0,0 +1,55 @@ +import { FirebaseError } from "../../error"; +import * as apphosting from "../../gcp/apphosting"; +import { logger } from "../../logger"; + +const ALLOWED_PROJECTS = [ + "aryanf-test", + "pretend-public", + ...(process.env.APP_HOSTING_COMPARE_ALLOWED_PROJECTS || "") + .split(",") + .map((p) => p.trim()) + .filter(Boolean), +]; + +/** + * + */ +export function validateProject(projectId: string): void { + if (!ALLOWED_PROJECTS.includes(projectId)) { + throw new FirebaseError( + `Invalid project ID "${projectId}". This tool can only run on projects: ${ALLOWED_PROJECTS.join(", ")}`, + ); + } +} + +/** + * Sweeps all slots in the project, resetting stale locks (busy for > 2 hours) to idle. + */ +export async function runGarbageCollection(projectId: string, location: string): Promise { + const existingBackends = await apphosting.listBackends(projectId, location); + const backendsList = existingBackends.backends || []; + const now = Date.now(); + const twoHours = 2 * 60 * 60 * 1000; + + for (const backend of backendsList) { + const nameParts = backend.name.split("/"); + const backendId = nameParts[nameParts.length - 1]; + + if (backendId.startsWith("compare-slot-")) { + const isBusy = backend.labels?.status === "busy"; + if (isBusy) { + const updateTime = new Date(backend.updateTime).getTime(); + if (now - updateTime > twoHours) { + logger.info(`Found stale lock on comparison slot backend ${backendId}. Unlocking...`); + try { + await apphosting.updateBackend(projectId, location, backendId, { + labels: { ...backend.labels, status: "idle" }, + }); + } catch (err) { + logger.debug(`Failed to unlock stale backend ${backendId}: ${err}`); + } + } + } + } + } +} diff --git a/src/apphosting/compare/reporter.spec.ts b/src/apphosting/compare/reporter.spec.ts new file mode 100644 index 00000000000..225030c7f60 --- /dev/null +++ b/src/apphosting/compare/reporter.spec.ts @@ -0,0 +1,69 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; +import { generateReport } from "./reporter"; +import { ComparisonResult } from "./compare"; + +describe("Report Generator", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = path.join( + process.cwd(), + "scratch-test-report-" + Math.random().toString(36).substring(7), + ); + fs.ensureDirSync(tempDir); + }); + + afterEach(() => { + fs.removeSync(tempDir); + }); + + it("should generate JSON and HTML reports", async () => { + const results: ComparisonResult[] = [ + { + route: "/", + statusMatch: true, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 1.0, + bodyDiff: "", + isBinary: false, + }, + { + route: "/about", + statusMatch: true, + headerMismatches: [{ header: "Cache-Control", valA: "max-age=0", valB: "no-cache" }], + expectedHeaderVariations: [], + bodySimilarity: 0.95, + bodyDiff: "HTML content mismatch", + isBinary: false, + }, + ]; + + await generateReport( + "aryanf-test", + "us-central1", + "compare-slot-1-a", + "compare-slot-1-b", + results, + tempDir, + ); + + const jsonPath = path.join(tempDir, "summary.json"); + const htmlPath = path.join(tempDir, "index.html"); + + expect(fs.existsSync(jsonPath)).to.be.true; + expect(fs.existsSync(htmlPath)).to.be.true; + + const data = fs.readJsonSync(jsonPath); + expect(data.summary.totalRoutes).to.equal(2); + expect(data.summary.matchingRoutes).to.equal(1); + expect(data.summary.mismatchingRoutes).to.equal(1); + expect(data.summary.overallSimilarity).to.be.closeTo(0.975, 0.001); + + const htmlContent = fs.readFileSync(htmlPath, "utf-8"); + expect(htmlContent).to.include("App Hosting Comparison Dashboard"); + expect(htmlContent).to.include("/about"); + }); +}); diff --git a/src/apphosting/compare/reporter.ts b/src/apphosting/compare/reporter.ts new file mode 100644 index 00000000000..5e96c4882d7 --- /dev/null +++ b/src/apphosting/compare/reporter.ts @@ -0,0 +1,361 @@ +import * as fs from "fs-extra"; +import * as path from "path"; +import * as clc from "colorette"; +import { ComparisonResult } from "./compare"; +import { logger } from "../../logger"; + +function escapeHtml(str: string): string { + return str + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +export interface ComparisonSummary { + projectId: string; + location: string; + backendA: string; + backendB: string; + timestamp: string; + totalRoutes: number; + matchingRoutes: number; + mismatchingRoutes: number; + overallSimilarity: number; +} + +/** + * + */ +export async function generateReport( + projectId: string, + location: string, + backendA: string, + backendB: string, + results: ComparisonResult[], + outputDir = "./compare-report", +): Promise { + const totalRoutes = results.length; + const matching = results.filter( + (r) => r.statusMatch && r.headerMismatches.length === 0 && r.bodySimilarity >= 0.99, + ); + const mismatches = results.filter( + (r) => !r.statusMatch || r.headerMismatches.length > 0 || r.bodySimilarity < 0.99, + ); + + const totalSimilarity = results.reduce((sum, r) => sum + r.bodySimilarity, 0); + const overallSimilarity = totalRoutes > 0 ? totalSimilarity / totalRoutes : 1.0; + + const summary: ComparisonSummary = { + projectId, + location, + backendA, + backendB, + timestamp: new Date().toISOString(), + totalRoutes, + matchingRoutes: matching.length, + mismatchingRoutes: mismatches.length, + overallSimilarity, + }; + + logger.info("\n=========================================="); + logger.info(" COMPARISON TEST SUMMARY"); + logger.info("=========================================="); + logger.info(`Project: ${projectId}`); + logger.info(`Location: ${location}`); + logger.info(`Backend A: ${backendA}`); + logger.info(`Backend B: ${backendB}`); + logger.info(`Total Routes: ${totalRoutes}`); + logger.info(`Passed: ${clc.green(String(matching.length))}`); + logger.info( + `Mismatched: ${mismatches.length > 0 ? clc.red(String(mismatches.length)) : clc.green("0")}`, + ); + logger.info(`Similarity: ${clc.cyan((overallSimilarity * 100).toFixed(2) + "%")}`); + logger.info("==========================================\n"); + + if (mismatches.length > 0) { + logger.warn(clc.bold(clc.red("Mismatched Routes:"))); + for (const m of mismatches) { + logger.warn(` - ${clc.bold(m.route)}`); + if (!m.statusMatch) { + logger.warn(` * Status Code Mismatch`); + } + if (m.headerMismatches.length > 0) { + logger.warn(` * ${m.headerMismatches.length} Behavioral Header Mismatch(es)`); + } + if (m.bodySimilarity < 0.99) { + logger.warn(` * Body Similarity: ${(m.bodySimilarity * 100).toFixed(2)}%`); + } + } + logger.info(""); + } + + await fs.ensureDir(outputDir); + + // Dump summary without the bodies to save space + const resultsWithoutBodies: ComparisonResult[] = results.map(r => { + const { bodyA, bodyB, ...rest } = r; + return rest; + }); + await fs.writeJson(path.join(outputDir, "summary.json"), { summary, results: resultsWithoutBodies }, { spaces: 2 }); + logger.info(`JSON report saved to: ${path.join(outputDir, "summary.json")}`); + + // Dump raw bodies for manual diffing + const routesDirA = path.join(outputDir, "backendA"); + const routesDirB = path.join(outputDir, "backendB"); + for (const r of results) { + if (r.bodyA !== undefined && r.bodyB !== undefined) { + // Map route /foo/bar to /foo/bar.html or /foo/bar/index.html + const safeRoute = r.route === "/" ? "index.html" : r.route.replace(/^\//, "") + ".html"; + await fs.outputFile(path.join(routesDirA, safeRoute), r.bodyA, "utf-8"); + await fs.outputFile(path.join(routesDirB, safeRoute), r.bodyB, "utf-8"); + } + } + logger.info(`Raw responses saved to ${routesDirA} and ${routesDirB} for manual diffing.`); + + const html = getHtmlTemplate(summary, resultsWithoutBodies); + await fs.outputFile(path.join(outputDir, "index.html"), html, "utf-8"); + logger.info( + `HTML Dashboard generated at: ${clc.underline(path.join(outputDir, "index.html"))}\n`, + ); +} + +function getHtmlTemplate(summary: ComparisonSummary, results: ComparisonResult[]): string { + const resultsRows = results + .map((r) => { + const isPass = r.statusMatch && r.headerMismatches.length === 0 && r.bodySimilarity >= 0.99; + const badgeClass = isPass ? "badge-success" : "badge-error"; + const badgeText = isPass ? "PASS" : "FAIL"; + + const headersList = r.headerMismatches + .map((m) => `
  • ${escapeHtml(m.header)}: "${escapeHtml(m.valA)}" vs "${escapeHtml(m.valB)}"
    • `) + .join(""); + + const variationsList = r.expectedHeaderVariations + .map((m) => `
  • ${escapeHtml(m.header)}: "${escapeHtml(m.valA)}" vs "${escapeHtml(m.valB)}"
    • `) + .join(""); + + return ` + + ${escapeHtml(r.route)} + ${badgeText} + ${r.statusMatch ? "Match" : "Mismatch"} + ${(r.bodySimilarity * 100).toFixed(1)}% + + ${r.headerMismatches.length > 0 ? `
    Mismatches:
      ${headersList}
    ` : "Match"} + ${r.expectedHeaderVariations.length > 0 ? `
    Variations:
      ${variationsList}
    ` : ""} + + + `; + }) + .join(""); + + return ` + + + + + + App Hosting Comparison Report + + + + + +
    +

    App Hosting Comparison Dashboard

    +
    + Ran on ${new Date(summary.timestamp).toLocaleString()} for project ${summary.projectId} (${summary.location}) +
    +
    + +
    +
    +
    Total Checked Routes
    +
    ${summary.totalRoutes}
    +
    +
    +
    Passed Routes
    +
    ${summary.matchingRoutes}
    +
    +
    +
    Mismatched Routes
    +
    ${summary.mismatchingRoutes}
    +
    +
    +
    Overall Similarity
    +
    ${(summary.overallSimilarity * 100).toFixed(2)}%
    +
    +
    + +
    + + + + + + + + + + + + ${resultsRows} + +
    RouteParityHTTP StatusBody ParityHeader Assertions
    +
    + + + + `; +} diff --git a/src/apphosting/compare/secrets.spec.ts b/src/apphosting/compare/secrets.spec.ts new file mode 100644 index 00000000000..8843eeed194 --- /dev/null +++ b/src/apphosting/compare/secrets.spec.ts @@ -0,0 +1,87 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; +import * as fs from "fs-extra"; +import { AppHostingYamlConfig } from "../yaml"; +import * as apphosting from "../../gcp/apphosting"; +import * as csm from "../../gcp/secretManager"; +import * as secretsHelper from "../secrets"; +import * as projectNumberHelper from "../../getProjectNumber"; +import { setupSandboxSecrets, cleanupSandboxSecrets } from "./secrets"; + +describe("Sandbox Secrets Manager", () => { + let pathExistsStub: sinon.SinonStub; + let loadConfigStub: sinon.SinonStub; + let getProjectNumberStub: sinon.SinonStub; + let getBackendStub: sinon.SinonStub; + let serviceAccountsStub: sinon.SinonStub; + let secretExistsStub: sinon.SinonStub; + let createSecretStub: sinon.SinonStub; + let addVersionStub: sinon.SinonStub; + let deleteSecretStub: sinon.SinonStub; + let grantSecretAccessStub: sinon.SinonStub; + + beforeEach(() => { + pathExistsStub = sinon.stub(fs, "pathExists"); + loadConfigStub = sinon.stub(AppHostingYamlConfig, "loadFromFile"); + getProjectNumberStub = sinon.stub(projectNumberHelper, "getProjectNumber"); + getBackendStub = sinon.stub(apphosting, "getBackend"); + serviceAccountsStub = sinon.stub(secretsHelper, "serviceAccountsForBackend"); + secretExistsStub = sinon.stub(csm, "secretExists"); + createSecretStub = sinon.stub(csm, "createSecret"); + addVersionStub = sinon.stub(csm, "addVersion"); + deleteSecretStub = sinon.stub(csm, "deleteSecret"); + grantSecretAccessStub = sinon.stub(secretsHelper, "grantSecretAccess"); + }); + + afterEach(() => { + sinon.restore(); + }); + + it("should setup sandbox secrets for yaml configuration", async () => { + pathExistsStub.resolves(true); + + const mockYaml = new AppHostingYamlConfig(); + mockYaml.env = { + API_KEY: { secret: "my-production-api-key", availability: ["RUNTIME"] }, + }; + loadConfigStub.resolves(mockYaml); + getProjectNumberStub.resolves("12345"); + + getBackendStub.resolves({ name: "backend-resource" }); + serviceAccountsStub.resolves({ + buildServiceAccount: "build-sa@google.com", + runServiceAccount: "run-sa@google.com", + }); + + secretExistsStub.resolves(false); + createSecretStub.resolves(); + addVersionStub.resolves(); + grantSecretAccessStub.resolves(); + + const mappings = await setupSandboxSecrets("aryanf-test", "us-central1", "/app/path", 1, [ + "compare-slot-1-a", + "compare-slot-1-b", + ]); + + expect(mappings).to.have.lengthOf(1); + expect(mappings[0].originalName).to.equal("my-production-api-key"); + expect(mappings[0].mockSecretName).to.equal("cmp-sec-1-my-production-api-key"); + expect(mappings[0].mockValue).to.equal("mock-value-for-API_KEY-slot-1"); + + expect(createSecretStub.callCount).to.equal(1); + expect(addVersionStub.callCount).to.equal(1); + expect(grantSecretAccessStub.callCount).to.equal(1); + }); + + it("should delete secrets on cleanup", async () => { + deleteSecretStub.resolves(); + + const mappings = [ + { originalName: "my-key", mockSecretName: "cmp-sec-1-my-key", mockValue: "val" }, + ]; + + await cleanupSandboxSecrets("aryanf-test", mappings); + expect(deleteSecretStub.callCount).to.equal(1); + expect(deleteSecretStub.firstCall.args[1]).to.equal("cmp-sec-1-my-key"); + }); +}); diff --git a/src/apphosting/compare/secrets.ts b/src/apphosting/compare/secrets.ts new file mode 100644 index 00000000000..e1123abe308 --- /dev/null +++ b/src/apphosting/compare/secrets.ts @@ -0,0 +1,102 @@ +import * as path from "path"; +import * as fs from "fs-extra"; +import * as csm from "../../gcp/secretManager"; +import * as apphosting from "../../gcp/apphosting"; +import { AppHostingYamlConfig } from "../yaml"; +import { getProjectNumber } from "../../getProjectNumber"; +import { serviceAccountsForBackend, grantSecretAccess, toMulti } from "../secrets"; +import { logger } from "../../logger"; + +export interface SecretMapping { + originalName: string; + mockSecretName: string; + mockValue: string; +} + +/** + * + */ +export async function setupSandboxSecrets( + projectId: string, + location: string, + appPath: string, + slotIndex: number, + backendIds: string[], +): Promise { + const yamlPath = path.join(appPath, "apphosting.yaml"); + if (!(await fs.pathExists(yamlPath))) { + return []; + } + + const config = await AppHostingYamlConfig.loadFromFile(yamlPath); + const secretEntries = Object.entries(config.env).filter(([, val]) => val.secret !== undefined); + if (secretEntries.length === 0) { + return []; + } + + const projectNumber = await getProjectNumber({ projectId }); + const mappings: SecretMapping[] = []; + + // Fetch all backends to extract their service accounts + const backends = await Promise.all( + backendIds.map((id) => apphosting.getBackend(projectId, location, id)), + ); + + const multiAccountsList = await Promise.all( + backends.map(async (b) => toMulti(await serviceAccountsForBackend(projectNumber, b))), + ); + + // Combine build/run service accounts from all backends + const combinedAccounts = { + buildServiceAccounts: Array.from( + new Set(multiAccountsList.flatMap((a) => a.buildServiceAccounts)), + ), + runServiceAccounts: Array.from(new Set(multiAccountsList.flatMap((a) => a.runServiceAccounts))), + }; + + for (const [envName, envVal] of secretEntries) { + const originalSecretName = envVal.secret!; + // Clean and build mock secret name + const cleanName = originalSecretName.replace(/[^a-zA-Z0-9_-]/g, "").toLowerCase(); + const mockSecretName = `cmp-sec-${slotIndex}-${cleanName}`.substring(0, 255); + const mockValue = `mock-value-for-${envName}-slot-${slotIndex}`; + + logger.info(`Setting up sandboxed secret for ${envName}: ${mockSecretName}...`); + + const exists = await csm.secretExists(projectId, mockSecretName); + if (!exists) { + await csm.createSecret(projectId, mockSecretName, { + "created-by": "apphosting-compare-tool", + slot: String(slotIndex), + }); + } + + await csm.addVersion(projectId, mockSecretName, mockValue); + await grantSecretAccess(projectId, projectNumber, mockSecretName, combinedAccounts); + + mappings.push({ + originalName: originalSecretName, + mockSecretName, + mockValue, + }); + } + + return mappings; +} + +/** + * + */ +export async function cleanupSandboxSecrets( + projectId: string, + mappings: SecretMapping[], +): Promise { + if (mappings.length === 0) return; + + logger.info("Cleaning up sandboxed secrets in Secret Manager..."); + await Promise.allSettled( + mappings.map((m) => + csm.deleteSecret(projectId, m.mockSecretName).catch((e) => logger.debug(e)), + ), + ); +} diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts new file mode 100644 index 00000000000..e2ff55ee30f --- /dev/null +++ b/src/apphosting/compare/server.ts @@ -0,0 +1,1550 @@ +import * as express from "express"; +import * as http from "http"; +import { logger } from "../../logger"; +import * as cache from "./cache"; +import * as compare from "./compare"; +import * as diff from "diff"; +import { CompareResponse, MatrixResponse, DashboardComparisonResult, VariantMetadata } from "./types"; + +export function startServer(port: number): Promise { + return new Promise((resolve, reject) => { + const app = express(); + + app.use(express.json()); + + // API: List all recordings in compare-cache + app.get("/api/recordings", async (req, res) => { + try { + const recordings = await cache.listRecordings(); + res.json(recordings); + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).json({ error: errMsg }); + } + }); + + app.get("/api/compare", async (req, res) => { + const { testCase, variantA, variantB } = req.query; + if (typeof testCase !== "string" || typeof variantA !== "string" || typeof variantB !== "string") { + res.status(400).json({ error: "Missing or invalid query parameters: testCase, variantA, and variantB must be strings." }); + return; + } + + try { + let recA: cache.VariantRecording; + let recB: cache.VariantRecording; + if (testCase === "GLOBAL") { + if (!variantA.includes("/") || !variantB.includes("/")) { + throw new Error("Invalid variant query parameters for GLOBAL testCase"); + } + const [tcA, varA] = variantA.split("/"); + const [tcB, varB] = variantB.split("/"); + recA = await cache.loadRecording(tcA, varA); + recB = await cache.loadRecording(tcB, varB); + } else { + recA = await cache.loadRecording(testCase, variantA); + recB = await cache.loadRecording(testCase, variantB); + } + + const allRoutes = Array.from(new Set([ + ...Object.keys(recA.routes), + ...Object.keys(recB.routes) + ])).sort(); + + const results: DashboardComparisonResult[] = []; + for (const route of allRoutes) { + const resA = recA.routes[route]; + const resB = recB.routes[route]; + + if (!resA || !resB) { + results.push({ + route, + statusMatch: false, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 0.0, + bodyDiff: `Route missing on one variant: ${!resA ? recA.id : recB.id}`, + isBinary: false, + bodyA: resA?.body, + bodyB: resB?.body, + }); + continue; + } + + const compResult = await compare.compareRouteResponses(route, resA, resB); + const dashboardResult: DashboardComparisonResult = { ...compResult }; + + if (!resA.isBinary && !resB.isBinary) { + const changes = diff.diffLines(dashboardResult.bodyA || "", dashboardResult.bodyB || ""); + // Filter/map to minimal JSON to keep payload clean + dashboardResult.diffChanges = changes.map((c: any) => ({ + value: c.value, + added: !!c.added, + removed: !!c.removed + })); + } + + results.push(dashboardResult); + } + + const responsePayload: CompareResponse = { + testCase, + variantA: recA.id, + variantB: recB.id, + urlA: recA.url, + urlB: recB.url, + results + }; + res.json(responsePayload); + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).json({ error: errMsg }); + } + }); + + app.get("/api/matrix", async (req, res) => { + const { testCase } = req.query; + if (typeof testCase !== "string") { + res.status(400).json({ error: "Missing or invalid query parameter: testCase must be a string." }); + return; + } + + try { + const recordings = await cache.listRecordings(); + let variantsList: string[] = []; + const recMap: Record = {}; + + if (testCase === "GLOBAL") { + for (const tc of Object.keys(recordings)) { + for (const v of recordings[tc]) { + const id = `${tc}/${v}`; + variantsList.push(id); + recMap[id] = await cache.loadRecording(tc, v); + } + } + } else { + variantsList = recordings[testCase] || []; + for (const v of variantsList) { + recMap[v] = await cache.loadRecording(testCase, v); + } + } + + if (variantsList.length === 0) { + const emptyPayload: MatrixResponse = { testCase, variants: [], variantsMetadata: {}, matrix: {} }; + res.json(emptyPayload); + return; + } + + const variantsMetadata: Record = {}; + for (const v of variantsList) { + variantsMetadata[v] = { + id: recMap[v].id, + localBuild: !!recMap[v].localBuild, + runtime: recMap[v].runtime || "default" + }; + } + + const matrix: Record> = {}; + + for (const vA of variantsList) { + matrix[vA] = matrix[vA] || {}; + for (const vB of variantsList) { + matrix[vB] = matrix[vB] || {}; + + if (vA === vB) { + matrix[vA][vB] = 1.0; + continue; + } + + if (matrix[vA][vB] !== undefined) { + continue; // Already computed symmetrical pair + } + + // Compute average body similarity across all shared routes + const recA = recMap[vA]; + const recB = recMap[vB]; + const allRoutes = Array.from(new Set([ + ...Object.keys(recA.routes), + ...Object.keys(recB.routes) + ])); + + let totalSimilarity = 0; + let countedRoutes = 0; + + for (const route of allRoutes) { + const resA = recA.routes[route]; + const resB = recB.routes[route]; + + if (!resA || !resB) { + countedRoutes++; // Missing routes act as 0% similarity penalty + continue; + } + + const compResult = await compare.compareRouteResponses(route, resA, resB); + totalSimilarity += compResult.bodySimilarity; + countedRoutes++; + } + + const score = countedRoutes > 0 ? (totalSimilarity / countedRoutes) : 0.0; + matrix[vA][vB] = score; + matrix[vB][vA] = score; + } + } + + const responsePayload: MatrixResponse = { + testCase, + variants: variantsList, + variantsMetadata, + matrix + }; + res.json(responsePayload); + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).json({ error: errMsg }); + } + }); + + app.get("/api/render", async (req, res) => { + const { testCase, variant, route } = req.query; + if (typeof testCase !== "string" || typeof variant !== "string" || typeof route !== "string") { + res.status(400).type("text/plain").send("Missing or invalid query parameters: testCase, variant, and route must be strings."); + return; + } + try { + let tc = testCase; + let varId = variant; + if (tc === "GLOBAL") { + const parts = varId.split("/"); + if (parts.length >= 2) { + tc = parts[0]; + varId = parts.slice(1).join("/"); + } + } + const rec = await cache.loadRecording(tc, varId); + const resData = rec.routes[route]; + if (!resData) { + res.status(404).type("text/plain").send("Route not found in cache"); + return; + } + if (resData.isBinary) { + res.setHeader("Content-Type", "application/octet-stream"); + res.send(Buffer.from(resData.body, "base64")); + } else { + res.setHeader("Content-Type", "text/html"); + // Inject tag to fix relative assets + let html = resData.body; + if (!html.includes("/i.test(html)) { + html = html.replace(//i, ``); + } else { + html = `` + html; + } + } + res.send(html); + } + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).type("text/plain").send(errMsg); + } + }); + + // Serve Single Page Application dashboard + app.get("/", (req, res) => { + res.send(getDashboardHtml()); + }); + + const server = http.createServer(app); + server.on("error", reject); + server.listen(port, () => { + logger.info(`\n🚀 Parity Visualization Dashboard running at http://localhost:${port}`); + logger.info("Press Ctrl+C to stop the server.\n"); + }); + + const cleanUp = () => { + server.close(() => { + resolve(); + }); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + }); +} + +function getDashboardHtml(): string { + return ` + + + + + + Parity Comparison Dashboard + + + + + + + + + +
    +

    🚀 Firebase App Hosting Parity Dashboard

    +
    Connected
    +
    + +
    + +
    +
    +
    Test Cases
    +
    + +
    +
    + + +
    + + +
    + + +
    +
    + Understanding Parity Metrics + [Click to Expand] +
    + +
    + + + + + + + + + + + +
    + + + + +
    Select a test case and variants to start comparing.
    +
    +
    +
    + + + + + `; +} diff --git a/src/apphosting/compare/slots.spec.ts b/src/apphosting/compare/slots.spec.ts new file mode 100644 index 00000000000..5d0b5669982 --- /dev/null +++ b/src/apphosting/compare/slots.spec.ts @@ -0,0 +1,85 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; + +import * as apphosting from "../../gcp/apphosting"; +import * as apps from "../../management/apps"; +import * as backendHelper from "../backend"; +import * as poller from "../../operation-poller"; +import { acquireComparisonSlot, releaseComparisonSlot } from "./slots"; + +describe("Comparison Slots Manager", () => { + let listBackendsStub: sinon.SinonStub; + let patchBackendStub: sinon.SinonStub; + let createBackendStub: sinon.SinonStub; + let listFirebaseAppsStub: sinon.SinonStub; + let createWebAppStub: sinon.SinonStub; + let pollOperationStub: sinon.SinonStub; + + beforeEach(() => { + listBackendsStub = sinon.stub(apphosting, "listBackends"); + patchBackendStub = sinon.stub(apphosting.client, "patch").resolves({ body: { name: "op-name" } } as any); + createBackendStub = sinon.stub(backendHelper, "createBackend"); + listFirebaseAppsStub = sinon.stub(apps, "listFirebaseApps"); + createWebAppStub = sinon.stub(apps, "createWebApp"); + pollOperationStub = sinon.stub(poller, "pollOperation").resolves(); + }); + + afterEach(() => { + sinon.restore(); + }); + + it("should acquire an existing idle slot", async () => { + listBackendsStub.resolves({ + backends: [ + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-0", + labels: { status: "idle", type: "comparison-sandbox" }, + }, + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-1", + labels: { status: "idle", type: "comparison-sandbox" }, + }, + ], + }); + listFirebaseAppsStub.resolves([{ appId: "web-app-123", displayName: "existing-app" }]); + + const slot = await acquireComparisonSlot("aryanf-test", "us-central1", 2); + expect(slot.index).to.equal(1); + expect(slot.backendIds[0]).to.equal("compare-slot-1-0"); + expect(slot.backendIds[1]).to.equal("compare-slot-1-1"); + + expect(patchBackendStub.callCount).to.equal(2); // Updates labels twice + expect(createBackendStub.callCount).to.equal(0); + }); + + it("should provision a slot if it doesn't exist and project is below limit", async () => { + listBackendsStub.resolves({ backends: [] }); + listFirebaseAppsStub.resolves([{ appId: "web-app-123", displayName: "existing-app" }]); + createBackendStub.resolves({ name: "backend-resource" }); + + const slot = await acquireComparisonSlot("aryanf-test", "us-central1", 2); + expect(slot.index).to.equal(1); + expect(createBackendStub.callCount).to.equal(2); + }); + + it("should throw if all slots are locked/busy", async () => { + const busyBackends = []; + for (let i = 1; i <= 10; i++) { + busyBackends.push( + { + name: `projects/test/locations/us-central1/backends/compare-slot-${i}-0`, + labels: { status: "busy", type: "comparison-sandbox" }, + }, + { + name: `projects/test/locations/us-central1/backends/compare-slot-${i}-1`, + labels: { status: "busy", type: "comparison-sandbox" }, + }, + ); + } + listBackendsStub.resolves({ backends: busyBackends }); + + await expect(acquireComparisonSlot("aryanf-test", "us-central1", 2)).to.be.rejectedWith( + "All 10 comparison slots are currently in use or project backend limits exceeded", + ); + }); +}); diff --git a/src/apphosting/compare/slots.ts b/src/apphosting/compare/slots.ts new file mode 100644 index 00000000000..9d3f0c09f7a --- /dev/null +++ b/src/apphosting/compare/slots.ts @@ -0,0 +1,159 @@ +import * as apphosting from "../../gcp/apphosting"; +import * as poller from "../../operation-poller"; +import { createBackend } from "../backend"; +import { listFirebaseApps, createWebApp, AppPlatform } from "../../management/apps"; + +import { logger } from "../../logger"; +import { FirebaseError } from "../../error"; +import { apphostingOrigin } from "../../api"; + +export interface ComparisonSlot { + index: number; + backendIds: string[]; +} + +const MAX_SLOTS = 10; +const apphostingPollerOptions = { + apiOrigin: apphostingOrigin(), + apiVersion: apphosting.API_VERSION, + masterTimeout: 25 * 60 * 1_000, + maxBackoff: 10_000, +}; + +async function updateBackendLabels( + projectId: string, + location: string, + backendId: string, + labels: Record, +): Promise { + const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; + const res = await apphosting.client.patch( + name, + { name, labels }, + { queryParams: { updateMask: "labels" } }, + ); + + await poller.pollOperation({ + ...apphostingPollerOptions, + pollerName: `update-labels-${projectId}-${location}-${backendId}`, + operationResourceName: res.body.name, + }); +} + +/** + * + */ +export async function getOrCreateSharedWebAppId(projectId: string): Promise { + const apps = await listFirebaseApps(projectId, AppPlatform.WEB); + if (apps.length > 0) { + return apps[0].appId; + } + + logger.info( + "No existing Web Apps found. Provisioning a shared Web App for comparison slot runners...", + ); + const createdApp = await createWebApp(projectId, { displayName: "firebase-compare-shared-app" }); + return createdApp.appId; +} + +/** + * + */ +export async function acquireComparisonSlot( + projectId: string, + location: string, + numVariants: number, +): Promise { + const existingBackends = await apphosting.listBackends(projectId, location); + const backendsList = existingBackends.backends || []; + + for (let i = 1; i <= MAX_SLOTS; i++) { + const slotBackendIds: string[] = []; + let isLocked = false; + + for (let v = 0; v < numVariants; v++) { + const backendId = `compare-slot-${i}-${v}`; + slotBackendIds.push(backendId); + const backend = backendsList.find((b) => b.name.endsWith(backendId)); + if (backend?.labels?.status === "busy") { + isLocked = true; + } + } + + if (!isLocked) { + const webAppId = await getOrCreateSharedWebAppId(projectId); + + // Check how many we need to create + const missingCount = slotBackendIds.filter( + (id) => !backendsList.find((b) => b.name.endsWith(id)), + ).length; + + if (backendsList.length + missingCount > 30) { + continue; // Quota limit hit, check next slot + } + + logger.info(`Acquiring Comparison Slot ${i} for ${numVariants} variants...`); + const updatePromises: Promise[] = []; + + for (const backendId of slotBackendIds) { + const backend = backendsList.find((b) => b.name.endsWith(backendId)); + if (!backend) { + logger.info(`Provisioning backend ${backendId}...`); + await createBackend(projectId, location, backendId, null, undefined, webAppId); + updatePromises.push( + updateBackendLabels(projectId, location, backendId, { + status: "busy", + type: "comparison-sandbox", + }), + ); + } else { + updatePromises.push( + updateBackendLabels(projectId, location, backendId, { + ...backend.labels, + status: "busy", + }), + ); + } + } + + await Promise.all(updatePromises); + return { index: i, backendIds: slotBackendIds }; + } + } + + throw new FirebaseError( + "All 10 comparison slots are currently in use or project backend limits exceeded. Please wait and try again.", + ); +} + +/** + * + */ +export async function releaseComparisonSlot( + projectId: string, + location: string, + slotIndex: number, + numVariants: number, +): Promise { + logger.info(`Releasing Comparison Slot ${slotIndex}...`); + + const existingBackends = await apphosting.listBackends(projectId, location); + const backendsList = existingBackends.backends || []; + + const updatePromises: Promise[] = []; + + for (let v = 0; v < numVariants; v++) { + const backendId = `compare-slot-${slotIndex}-${v}`; + const backend = backendsList.find((b) => b.name.endsWith(backendId)); + if (backend) { + updatePromises.push( + updateBackendLabels(projectId, location, backendId, { + ...backend.labels, + status: "idle", + }), + ); + } + } + + await Promise.allSettled(updatePromises); +} diff --git a/src/apphosting/compare/suite.spec.ts b/src/apphosting/compare/suite.spec.ts new file mode 100644 index 00000000000..bb6d9f057b0 --- /dev/null +++ b/src/apphosting/compare/suite.spec.ts @@ -0,0 +1,155 @@ +import * as path from "path"; +import * as fs from "fs-extra"; +import { expect } from "chai"; +import * as sinon from "sinon"; +import * as childProcess from "child_process"; +import * as apphosting from "../../gcp/apphosting"; +import * as projectNumberHelper from "../../getProjectNumber"; +import * as secretsManager from "./secrets"; +import * as discoverManager from "./discover"; +import { Crawler } from "./crawler"; +import * as compareManager from "./compare"; +import * as poller from "../../operation-poller"; +import * as reporterManager from "./reporter"; +import * as fetchModule from "node-fetch"; +import * as cache from "./cache"; +import { runCompareSuite } from "./suite"; +import * as utils from "../../utils"; + +describe("runCompareSuite Orchestrator", () => { + let tempDir: string; + let execStub: sinon.SinonStub; + let getProjectNumberStub: sinon.SinonStub; + let setupSecretsStub: sinon.SinonStub; + let cleanupSecretsStub: sinon.SinonStub; + let discoverRoutesStub: sinon.SinonStub; + let compareRouteResponsesStub: sinon.SinonStub; + let generateReportStub: sinon.SinonStub; + let getBackendStub: sinon.SinonStub; + let crawlStub: sinon.SinonStub; + let getRoutesStub: sinon.SinonStub; + let pollOperationStub: sinon.SinonStub; + let saveRecordingStub: sinon.SinonStub; + let loadRecordingStub: sinon.SinonStub; + let fetchStub: sinon.SinonStub; + + beforeEach(() => { + tempDir = path.join(process.cwd(), "scratch-test-suite-" + Math.random().toString(36).substring(7)); + fs.ensureDirSync(tempDir); + + execStub = sinon.stub(childProcess, "exec").yields(null, { stdout: "success", stderr: "" }); + pollOperationStub = sinon.stub(poller, "pollOperation").resolves(); + getProjectNumberStub = sinon.stub(projectNumberHelper, "getProjectNumber").resolves("12345"); + setupSecretsStub = sinon.stub(secretsManager, "setupSandboxSecrets").resolves([]); + cleanupSecretsStub = sinon.stub(secretsManager, "cleanupSandboxSecrets").resolves(); + discoverRoutesStub = sinon.stub(discoverManager, "discoverRoutes").resolves(["/"]); + sinon.stub(utils, "sleep").resolves(); + + compareRouteResponsesStub = sinon.stub(compareManager, "compareRouteResponses").resolves({ + route: "/", + statusMatch: true, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 1.0, + bodyDiff: "", + isBinary: false + } as any); + + generateReportStub = sinon.stub(reporterManager, "generateReport").resolves(); + getBackendStub = sinon.stub(apphosting, "getBackend").resolves({ uri: "https://my-backend.com" } as any); + + crawlStub = sinon.stub(Crawler.prototype, "crawl").resolves(); + getRoutesStub = sinon.stub(Crawler.prototype, "getRoutes").returns(["/about"]); + + saveRecordingStub = sinon.stub(cache, "saveRecording").resolves(); + loadRecordingStub = sinon.stub(cache, "loadRecording").resolves({ + id: "mock", + testCaseName: "mock", + timestamp: "mock", + url: "mock", + routes: {} + }); + + fetchStub = sinon.stub(fetchModule, "default").resolves({ + status: 200, + headers: { + get: (k: string) => k === "content-type" ? "text/html" : "", + forEach: (fn: (v: string, k: string) => void) => fn("text/html", "content-type"), + }, + buffer: async () => Buffer.from("mock body"), + text: async () => "mock body", + } as any); + }); + + afterEach(() => { + sinon.restore(); + fs.removeSync(tempDir); + }); + + it("should coordinate the full deployment, crawling, comparison, and reporting pipeline", async () => { + const backendIds = ["compare-slot-1-0", "compare-slot-1-1"]; + await runCompareSuite( + "aryanf-test", + "us-central1", + backendIds, + 1, + "Test-Case-A", + [ + { path: tempDir }, + { path: tempDir } + ] + ); + + expect(setupSecretsStub.callCount).to.equal(1); + expect(execStub.callCount).to.equal(2); // Deploys twice + expect(discoverRoutesStub.callCount).to.equal(2); + expect(crawlStub.callCount).to.equal(2); + + expect(compareRouteResponsesStub.callCount).to.equal(2); // for "/" and "/about" + expect(generateReportStub.callCount).to.equal(1); + expect(cleanupSecretsStub.callCount).to.equal(1); + }); + + it("should support running with local builds enabled", async () => { + const backendIds = ["compare-slot-1-0", "compare-slot-1-1"]; + await runCompareSuite( + "aryanf-test", + "us-central1", + backendIds, + 1, + "Test-Case-B", + [ + { path: tempDir, localBuild: false }, + { path: tempDir, localBuild: true } + ] + ); + + expect(execStub.callCount).to.equal(2); + // Verifies one of them was deployed with localBuild experiment prefix + const firstCallCmd = execStub.firstCall.args[0]; + const secondCallCmd = execStub.secondCall.args[0]; + + expect(firstCallCmd).to.not.include("FIREBASE_CLI_EXPERIMENTS=apphostinglocalbuilds"); + expect(secondCallCmd).to.include("FIREBASE_CLI_EXPERIMENTS=apphostinglocalbuilds"); + }); + + it("should support runtime version patching for backends", async () => { + const patchStub = sinon.stub(apphosting.client, "patch").resolves({ body: { name: "operation-123" } } as any); + const backendIds = ["compare-slot-1-0", "compare-slot-1-1"]; + + await runCompareSuite( + "aryanf-test", + "us-central1", + backendIds, + 1, + "Test-Case-C", + [ + { path: tempDir, runtime: "nodejs20" }, + { path: tempDir, runtime: "nodejs22" } + ] + ); + + expect(patchStub.callCount).to.equal(2); // Patch both runtimes + expect(execStub.callCount).to.equal(2); + }); +}); diff --git a/src/apphosting/compare/suite.ts b/src/apphosting/compare/suite.ts new file mode 100644 index 00000000000..7f96e574b78 --- /dev/null +++ b/src/apphosting/compare/suite.ts @@ -0,0 +1,366 @@ +import * as path from "path"; +import * as fs from "fs-extra"; +import * as apphosting from "../../gcp/apphosting"; +import { getProjectNumber } from "../../getProjectNumber"; +import { apphostingOrigin } from "../../api"; +import * as secrets from "./secrets"; +import * as slots from "./slots"; +import * as lifecycle from "./lifecycle"; +import * as discover from "./discover"; +import { Crawler } from "./crawler"; +import * as compare from "./compare"; +import * as reporter from "./reporter"; +import * as poller from "../../operation-poller"; +import { logger } from "../../logger"; +import { FirebaseError } from "../../error"; +import { sleep } from "../../utils"; + + + +const apphostingPollerOptions: Omit = { + apiOrigin: apphostingOrigin(), + apiVersion: "v1beta", + backoff: 200, + maxBackoff: 10000, + masterTimeout: 120000, // 2 minutes +}; + +import * as cp from "child_process"; +import * as util from "util"; + +export const createdConfigs = new Set(); + +interface Destroyable { + destroy(): void; +} + +function isDestroyable(obj: unknown): obj is Destroyable { + return ( + typeof obj === "object" && + obj !== null && + "destroy" in obj && + typeof (obj as Record).destroy === "function" + ); +} + +async function deployToBackend( + projectId: string, + location: string, + backendId: string, + appPath: string, + bucketName: string, // Kept for backwards compatibility but unused + useLocalBuild: boolean, + runtimeVersion?: string, +): Promise { + if (runtimeVersion) { + logger.info(`Patching runtime version for backend ${backendId} to ${runtimeVersion}...`); + const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; + const op = await apphosting.client.patch<{ name: string; runtime: { value: string } }, apphosting.Operation>( + name, + { name, runtime: { value: runtimeVersion } }, + { queryParams: { updateMask: "runtime" } }, + ); + await poller.pollOperation({ + ...apphostingPollerOptions, + pollerName: `update-runtime-${backendId}`, + operationResourceName: op.body.name, + }); + } + + const tempConfigName = `firebase-compare-${backendId}.json`; + const configPath = path.join(appPath, tempConfigName); + createdConfigs.add(configPath); + + const firebaseJson = { + apphosting: [ + { + source: ".", + backendId: backendId, + localBuild: useLocalBuild + } + ] + }; + + await fs.writeJson(configPath, firebaseJson, { spaces: 2 }); + + try { + logger.info(`Triggering CLI deploy for backend ${backendId} (localBuild: ${useLocalBuild})...`); + // Run exactly the same deployment path as a customer + const experimentPrefix = useLocalBuild ? "FIREBASE_CLI_EXPERIMENTS=apphostinglocalbuilds " : ""; + let binPath = path.resolve(__dirname, "../../bin/firebase.js"); + if (!fs.existsSync(binPath)) { + binPath = path.resolve(__dirname, "../../../lib/bin/firebase.js"); + } + + const cmd = `${experimentPrefix}node "${binPath}" deploy --only apphosting:${backendId} --project ${projectId} --config ${tempConfigName} --non-interactive --allow-local-build-secrets`; + + const execAsync = util.promisify(cp.exec); + const { stdout, stderr } = await execAsync(cmd, { cwd: appPath, maxBuffer: 1024 * 1024 * 100 }); + logger.debug(`Deploy output for ${backendId}:\n${stdout}`); + } catch (err: unknown) { + const execErr = err as { stdout?: string; stderr?: string }; + logger.error(`Deploy for ${backendId} failed!\nSTDOUT:\n${execErr.stdout || ""}\nSTDERR:\n${execErr.stderr || ""}`); + throw new FirebaseError(`Failed to deploy variant to ${backendId}.`, { original: err instanceof Error ? err : undefined }); + } finally { + await fs.remove(configPath); + createdConfigs.delete(configPath); + } +} + +export interface VariantConfig { + id?: string; + path: string; + localBuild?: boolean; + runtime?: string; +} + +/** + * + */ +import * as cache from "./cache"; +import fetch from "node-fetch"; + +async function recordVariant( + testCaseName: string, + variantId: string, + url: string, + appPath: string, +): Promise { + const discoveredStaticRoutes = await discover.discoverRoutes(appPath); + const allRoutesSet = new Set(discoveredStaticRoutes); + + logger.info(`Crawling Variant ${variantId} at ${url} for dynamic link discovery...`); + const crawler = new Crawler(url); + await crawler.crawl(); + crawler.getRoutes().forEach((r) => allRoutesSet.add(r)); + + const routes: Record = {}; + + for (const route of allRoutesSet) { + logger.debug(`Recording route ${route}...`); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 15000); + try { + const res = await fetch(`${url}${route}`, { + redirect: "manual" as const, + headers: { "User-Agent": "FirebaseCompareCrawler/1.0" }, + signal: controller.signal, + size: 2 * 1024 * 1024, + }); + + const contentType = res.headers.get("content-type") || ""; + const isBinary = isBinaryContentType(contentType); + + const headers: Record = {}; + res.headers.forEach((val, key) => { headers[key.toLowerCase()] = val; }); + + let body = ""; + const contentLength = parseInt(res.headers.get("content-length") || "0", 10); + if (contentLength > 2 * 1024 * 1024) { + body = `(omitted - size ${contentLength} bytes exceeds 2MB limit)`; + if (res.body && isDestroyable(res.body)) { + res.body.destroy(); + } + } else { + const buffer = await res.buffer(); + if (buffer.length > 2 * 1024 * 1024) { + body = `(omitted - size ${buffer.length} bytes exceeds 2MB limit)`; + } else { + body = isBinary ? buffer.toString("base64") : buffer.toString("utf-8"); + } + } + + routes[route] = { + status: res.status, + headers, + isBinary, + body, + }; + } catch (err) { + logger.warn(`Failed to record route ${route}: ${err}`); + } finally { + clearTimeout(timeout); + } + } + + return { + id: variantId, + testCaseName, + timestamp: new Date().toISOString(), + url, + routes, + }; +} + +function isBinaryContentType(contentType: string): boolean { + const normalized = contentType.toLowerCase(); + return [ + "image/", + "application/pdf", + "application/zip", + "application/octet-stream", + ].some((type) => normalized.includes(type)); +} + +export async function runCompareSuite( + projectId: string, + location: string, + backendIds: string[], + slotIndex: number, + testCaseName: string, + variants: VariantConfig[], + options: { + outputDir?: string; + recordOnly?: boolean; + compareOnly?: boolean; + } = {}, +): Promise { + const recordings: cache.VariantRecording[] = []; + + if (!options.compareOnly) { + // === RECORD PHASE === + const projectNumber = await getProjectNumber({ projectId }); + let secretsMappings: secrets.SecretMapping[][] = []; + + const cleanUp = async () => { + logger.warn("\nInterrupted. Deleting mock secrets..."); + for (const mapping of secretsMappings) { + await secrets.cleanupSandboxSecrets(projectId, mapping); + } + process.exit(1); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + + try { + // Setup secrets + const uniquePaths = Array.from(new Set(variants.map((v) => v.path))); + // Setup secrets sequentially to avoid concurrent creation conflicts in Secret Manager + secretsMappings = []; + for (const uniquePath of uniquePaths) { + const pathBackendIds = variants + .map((v, i) => (v.path === uniquePath ? backendIds[i] : null)) + .filter((id): id is string => id !== null); + + const mappings = await secrets.setupSandboxSecrets( + projectId, + location, + uniquePath, + slotIndex, + pathBackendIds + ); + secretsMappings.push(mappings); + } + + // Deploy variants sequentially + for (let i = 0; i < variants.length; i++) { + const v = variants[i]; + await deployToBackend( + projectId, + location, + backendIds[i], + v.path, + "", // bucketName + !!v.localBuild, + v.runtime, + ); + } + + logger.info("All rollouts completed successfully!"); + + logger.info("Waiting 30 seconds for Firebase Hosting routing propagation to complete..."); + await sleep(30000); + + // Retrieve URLs and Record + + const backendDataList = await Promise.all( + backendIds.map((id) => apphosting.getBackend(projectId, location, id)), + ); + const urls = backendDataList.map((b) => { + if (!b.uri) { + throw new FirebaseError(`Backend ${b.name} has no valid URI. Deployment may have failed or is still provisioning.`); + } + return b.uri.startsWith("http") ? b.uri : `https://${b.uri}`; + }); + + for (let i = 0; i < variants.length; i++) { + const v = variants[i]; + const url = urls[i]; + const record = await recordVariant(testCaseName, v.id || String(i), url, v.path); + record.localBuild = !!v.localBuild; + record.runtime = v.runtime; + await cache.saveRecording(record); + recordings.push(record); + } + } finally { + process.off("SIGINT", cleanUp); + process.off("SIGTERM", cleanUp); + for (const mapping of secretsMappings) { + await secrets.cleanupSandboxSecrets(projectId, mapping); + } + } + } else { + // === LOAD RECORDINGS FROM CACHE === + logger.info(`Loading cached recordings for test case "${testCaseName}"...`); + for (let i = 0; i < variants.length; i++) { + const v = variants[i]; + const record = await cache.loadRecording(testCaseName, v.id || String(i)); + recordings.push(record); + } + } + + if (options.recordOnly) { + logger.info("Record phase complete. Skipping comparison as requested."); + return; + } + + // === COMPARE PHASE === + logger.info("Starting pairwise comparison of recorded variants..."); + for (let i = 0; i < recordings.length; i++) { + for (let j = i + 1; j < recordings.length; j++) { + const recA = recordings[i]; + const recB = recordings[j]; + logger.info(`\nGenerating Comparison Report: ${recA.id} vs ${recB.id}...`); + + const allRoutes = Array.from(new Set([ + ...Object.keys(recA.routes), + ...Object.keys(recB.routes) + ])).sort(); + + const results: compare.ComparisonResult[] = []; + for (const route of allRoutes) { + const resA = recA.routes[route]; + const resB = recB.routes[route]; + + if (!resA || !resB) { + results.push({ + route, + statusMatch: false, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 0.0, + bodyDiff: `Route missing on one variant: ${!resA ? recA.id : recB.id}`, + isBinary: false + }); + continue; + } + + const res = await compare.compareRouteResponses(route, resA, resB); + results.push(res); + } + + const pairOutputDir = options.outputDir + ? path.join(options.outputDir, `${recA.id}-vs-${recB.id}`) + : undefined; + + await reporter.generateReport( + projectId, + location, + recA.id, + recB.id, + results, + pairOutputDir, + ); + } + } +} diff --git a/src/apphosting/compare/types.ts b/src/apphosting/compare/types.ts new file mode 100644 index 00000000000..e9eec4b0bbd --- /dev/null +++ b/src/apphosting/compare/types.ts @@ -0,0 +1,42 @@ +import { ComparisonResult } from "./compare"; + +export interface ComparisonSlot { + index: number; + backendIds: string[]; +} + +export interface SecretMapping { + originalName: string; + mockSecretName: string; + mockValue: string; +} + +export interface DashboardComparisonResult extends ComparisonResult { + diffChanges?: Array<{ + value: string; + added: boolean; + removed: boolean; + }>; +} + +export interface CompareResponse { + testCase: string; + variantA: string; + variantB: string; + urlA: string; + urlB: string; + results: DashboardComparisonResult[]; +} + +export interface VariantMetadata { + id: string; + localBuild: boolean; + runtime: string; +} + +export interface MatrixResponse { + testCase: string; + variants: string[]; + variantsMetadata: Record; + matrix: Record>; +} diff --git a/src/commands/apphosting-compare-suite.ts b/src/commands/apphosting-compare-suite.ts new file mode 100644 index 00000000000..aa8ea9ea166 --- /dev/null +++ b/src/commands/apphosting-compare-suite.ts @@ -0,0 +1,144 @@ +import { Command } from "../command"; +import { Options } from "../options"; +import { needProjectId } from "../projectUtils"; +import { requireAuth } from "../requireAuth"; +import * as suiteModule from "../apphosting/compare/suite"; +import * as lifecycle from "../apphosting/compare/lifecycle"; +import * as slots from "../apphosting/compare/slots"; +import { FirebaseError } from "../error"; +import * as fs from "fs-extra"; +import * as path from "path"; +import { logger } from "../logger"; + +export const command = new Command("apphosting:compare-suite") + .description("Autonomously run a suite of comparison tests on multiple App Hosting codebases") + .option( + "--location ", + "the primary region of the App Hosting backends to use", + "us-central1", + ) + .option("--suite-config ", "path to comparison suite JSON configuration file") + .option( + "--output-dir ", + "directory to output comparison report files", + "./compare-report", + ) + .option("--record-only", "only deploy variants and record their output, skipping diffing") + .option("--compare-only", "run comparisons based on previously cached recordings, skipping deployment") + .option("--serve", "spin up the localhost comparison viewer dashboard") + .option("--port ", "port to run the localhost comparison viewer on", "3000") + .before(requireAuth) + .action(async (options: Options) => { + if (options.serve) { + const { startServer } = require("../apphosting/compare/server"); + const port = parseInt(options.port as string, 10) || 3000; + await startServer(port); + return; + } + + const projectId = needProjectId(options); + const location = options.location as string; + const configPath = options.suiteConfig as string; + + if (!configPath) { + throw new FirebaseError( + "Must specify --suite-config file containing the list of apps to compare.", + ); + } + + if (!(await fs.pathExists(configPath))) { + throw new FirebaseError(`Suite config file does not exist at ${configPath}`); + } + + const suite = await fs.readJson(configPath); + if (!Array.isArray(suite)) { + throw new FirebaseError("Suite config must be a JSON array of test cases."); + } + + lifecycle.validateProject(projectId); + + // === OPTION A: COMPARE ONLY (NO SLOTS / DEPLOYMENTS) === + if (options.compareOnly) { + logger.info(`Starting comparison run for ${suite.length} test cases using cache...`); + for (const testCase of suite) { + logger.info(`\nComparing test case: ${testCase.name || "Unnamed Test"}`); + const caseOutputDir = path.join(options.outputDir as string, testCase.name || "unnamed"); + + try { + await suiteModule.runCompareSuite(projectId, location, [], 0, testCase.name, testCase.variants, { + outputDir: caseOutputDir, + compareOnly: true, + }); + } catch (err: any) { + logger.error(`Parity run for ${testCase.name} failed: ${err.message}`); + } + } + return; + } + + // === OPTION B: RECORD (Requires locking GCM Slot) === + await lifecycle.runGarbageCollection(projectId, location); + + // Compute max variants to acquire a slot large enough + if (suite.length === 0) { + throw new FirebaseError("Suite config must contain at least one test case."); + } + const maxVariants = Math.max(...suite.map((tc: any) => tc.variants?.length || 0)); + if (maxVariants < 2) { + throw new FirebaseError("All test cases must have at least 2 variants."); + } + + const slot = await slots.acquireComparisonSlot(projectId, location, maxVariants); + logger.info(`Acquired Comparison Slot ${slot.index} globally for the suite run.`); + + const cleanUp = async () => { + logger.warn("\nInterrupted. Restoring comparison slot lock and cleaning up temp files..."); + for (const configPath of suiteModule.createdConfigs) { + try { + await fs.remove(configPath); + } catch (e) {} + } + await slots.releaseComparisonSlot(projectId, location, slot.index, maxVariants); + process.exit(1); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + + try { + logger.info(`Starting suite of ${suite.length} comparison tests...`); + for (const testCase of suite) { + logger.info(`\nRunning test case: ${testCase.name || "Unnamed Test"}`); + const caseOutputDir = path.join(options.outputDir as string, testCase.name || "unnamed"); + + if (!testCase.variants || testCase.variants.length < 2) { + logger.error(`Skipping test case ${testCase.name}: must have at least 2 configurations.`); + continue; + } + + // Slice slot.backendIds to match the current testCase's variant count + const caseBackendIds = slot.backendIds.slice(0, testCase.variants.length); + + try { + await suiteModule.runCompareSuite( + projectId, + location, + caseBackendIds, + slot.index, + testCase.name, + testCase.variants, + { + outputDir: caseOutputDir, + recordOnly: !!options.recordOnly, + compareOnly: false, + } + ); + } catch (err: any) { + logger.error(`Matrix execution for ${testCase.name} failed: ${err.message}`); + } + } + } finally { + process.off("SIGINT", cleanUp); + process.off("SIGTERM", cleanUp); + await slots.releaseComparisonSlot(projectId, location, slot.index, maxVariants); + } + }); diff --git a/src/commands/apphosting-compare.ts b/src/commands/apphosting-compare.ts new file mode 100644 index 00000000000..083a75b76a0 --- /dev/null +++ b/src/commands/apphosting-compare.ts @@ -0,0 +1,98 @@ +import { Command } from "../command"; +import { Options } from "../options"; +import { needProjectId } from "../projectUtils"; +import { requireAuth } from "../requireAuth"; +import * as suiteModule from "../apphosting/compare/suite"; +import * as lifecycle from "../apphosting/compare/lifecycle"; +import * as slots from "../apphosting/compare/slots"; +import * as fs from "fs-extra"; +import { FirebaseError } from "../error"; +import { logger } from "../logger"; + +export const command = new Command("apphosting:compare") + .description( + "Autonomously deploy and compare two versions/configurations of a Firebase App Hosting codebase", + ) + .option( + "--location ", + "the primary region of the App Hosting backends to use", + "us-central1", + ) + .option( + "--path-a ", + "path to directory containing codebase version A (defaults to current directory)", + ".", + ) + .option( + "--path-b ", + "path to directory containing codebase version B (defaults to path-a or current directory)", + ) + .option("--local-build-a", "compile and deploy version A using a local build") + .option("--local-build-b", "compile and deploy version B using a local build") + .option( + "--runtime-a ", + "specify the ABIU runtime version for backend A (e.g. nodejs22)", + ) + .option( + "--runtime-b ", + "specify the ABIU runtime version for backend B (e.g. nodejs22)", + ) + .option( + "--output-dir ", + "directory to output comparison report files", + "./compare-report", + ) + .before(requireAuth) + .action(async (options: Options) => { + const projectId = needProjectId(options); + const location = options.location as string; + const pathA = (options.pathA as string) || "."; + const pathB = (options.pathB as string) || pathA; + + lifecycle.validateProject(projectId); + await lifecycle.runGarbageCollection(projectId, location); + + const slot = await slots.acquireComparisonSlot(projectId, location, 2); + + const cleanUp = async () => { + logger.warn("\nInterrupted. Restoring comparison slot lock and cleaning up temp files..."); + for (const configPath of suiteModule.createdConfigs) { + try { + await fs.remove(configPath); + } catch (e) {} + } + await slots.releaseComparisonSlot(projectId, location, slot.index, 2); + process.exit(1); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + + try { + await suiteModule.runCompareSuite( + projectId, + location, + slot.backendIds, + slot.index, + "Single-Comparison-Run", + [ + { + path: pathA, + localBuild: !!options.localBuildA, + runtime: options.runtimeA as string | undefined, + }, + { + path: pathB, + localBuild: !!options.localBuildB, + runtime: options.runtimeB as string | undefined, + }, + ], + { + outputDir: options.outputDir as string, + }, + ); + } finally { + process.off("SIGINT", cleanUp); + process.off("SIGTERM", cleanUp); + await slots.releaseComparisonSlot(projectId, location, slot.index, 2); + } + }); diff --git a/src/commands/index.ts b/src/commands/index.ts index ea736e9144d..8ec41bb6145 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -210,6 +210,8 @@ export function load(client: CLIClient): CLIClient { client.apphosting.secrets.access = loadCommand("apphosting-secrets-access"); client.apphosting.rollouts = {}; client.apphosting.rollouts.create = loadCommand("apphosting-rollouts-create"); + client.apphosting.compare = loadCommand("apphosting-compare"); + client.apphosting["compare-suite"] = loadCommand("apphosting-compare-suite"); client.apphosting.config = {}; if (experiments.isEnabled("internaltesting")) { client.apphosting.builds = {}; diff --git a/src/gcp/apphosting.ts b/src/gcp/apphosting.ts index b9f3c98d69d..84f767ef797 100644 --- a/src/gcp/apphosting.ts +++ b/src/gcp/apphosting.ts @@ -811,3 +811,28 @@ export async function getNextRolloutId( const highest = Math.max(highestId(builds.builds), highestId(rollouts.rollouts)); return `build-${year}-${month}-${day}-${String(highest + 1).padStart(3, "0")}`; } + +/** + * Update a backend configuration (e.g. labels). + */ +export async function updateBackend( + projectId: string, + location: string, + backendId: string, + backend: DeepOmit, BackendOutputOnlyFields | "name">, +): Promise { + const fieldMasks = proto.fieldMasks(backend); + const queryParams = { + updateMask: fieldMasks.join(","), + }; + const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; + const res = await client.patch( + name, + { ...backend, name }, + { + queryParams, + }, + ); + return res.body; +} +