From bde1df2db03a7106c4256e4b735c3f0aff075434 Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Tue, 16 Jun 2026 21:50:56 -0400 Subject: [PATCH 01/12] Add comparison tool --- src/apphosting/compare/README.md | 90 +++++++ src/apphosting/compare/compare.spec.ts | 84 ++++++ src/apphosting/compare/compare.ts | 120 +++++++++ src/apphosting/compare/crawler.spec.ts | 84 ++++++ src/apphosting/compare/crawler.ts | 124 +++++++++ src/apphosting/compare/discover.spec.ts | 62 +++++ src/apphosting/compare/discover.ts | 68 +++++ src/apphosting/compare/distance.spec.ts | 36 +++ src/apphosting/compare/distance.ts | 41 +++ src/apphosting/compare/lifecycle.spec.ts | 63 +++++ src/apphosting/compare/lifecycle.ts | 77 ++++++ src/apphosting/compare/reporter.spec.ts | 66 +++++ src/apphosting/compare/reporter.ts | 324 +++++++++++++++++++++++ src/apphosting/compare/secrets.spec.ts | 91 +++++++ src/apphosting/compare/secrets.ts | 95 +++++++ src/apphosting/compare/slots.spec.ts | 75 ++++++ src/apphosting/compare/slots.ts | 119 +++++++++ src/apphosting/compare/suite.spec.ts | 132 +++++++++ src/apphosting/compare/suite.ts | 261 ++++++++++++++++++ src/commands/apphosting-compare-suite.ts | 67 +++++ src/commands/apphosting-compare.ts | 59 +++++ src/commands/index.ts | 2 + src/gcp/apphosting.ts | 25 ++ 23 files changed, 2165 insertions(+) create mode 100644 src/apphosting/compare/README.md create mode 100644 src/apphosting/compare/compare.spec.ts create mode 100644 src/apphosting/compare/compare.ts create mode 100644 src/apphosting/compare/crawler.spec.ts create mode 100644 src/apphosting/compare/crawler.ts create mode 100644 src/apphosting/compare/discover.spec.ts create mode 100644 src/apphosting/compare/discover.ts create mode 100644 src/apphosting/compare/distance.spec.ts create mode 100644 src/apphosting/compare/distance.ts create mode 100644 src/apphosting/compare/lifecycle.spec.ts create mode 100644 src/apphosting/compare/lifecycle.ts create mode 100644 src/apphosting/compare/reporter.spec.ts create mode 100644 src/apphosting/compare/reporter.ts create mode 100644 src/apphosting/compare/secrets.spec.ts create mode 100644 src/apphosting/compare/secrets.ts create mode 100644 src/apphosting/compare/slots.spec.ts create mode 100644 src/apphosting/compare/slots.ts create mode 100644 src/apphosting/compare/suite.spec.ts create mode 100644 src/apphosting/compare/suite.ts create mode 100644 src/commands/apphosting-compare-suite.ts create mode 100644 src/commands/apphosting-compare.ts diff --git a/src/apphosting/compare/README.md b/src/apphosting/compare/README.md new file mode 100644 index 00000000000..56c0d62ef97 --- /dev/null +++ b/src/apphosting/compare/README.md @@ -0,0 +1,90 @@ +# App Hosting Comparison Tool + +The App Hosting Comparison Tool (`firebase apphosting:compare` and `firebase apphosting:compare-suite`) is an autonomous differential testing tool. It allows developers and CI/CD systems to deploy, crawl, and compare two different versions or configurations of an application on Firebase App Hosting, asserting parity across status codes, response headers, and body payloads (text diffs and binary hashes). + +--- + +## Commands + +### `firebase apphosting:compare` + +Deploys and compares two versions of an application. + +```bash +firebase apphosting:compare \ + --path-b \ + [--path-a ] \ + [--location ] \ + [--output-dir ] +``` + +**Options:** +* `--path-b` (Required): The directory path containing the version to compare against (e.g. your canary/experimental branch). +* `--path-a` (Optional): The directory path containing the baseline version (e.g. your stable branch). Defaults to the current working directory (`.`). +* `--location` (Optional): The GCP location where App Hosting backends should reside. Defaults to `us-central1`. +* `--output-dir` (Optional): The directory path where comparison results and the dashboard will be written. Defaults to `./compare-report`. + +--- + +### `firebase apphosting:compare-suite` + +Runs a batch of comparison tests on multiple codebases defined in a JSON suite config. + +```bash +firebase apphosting:compare-suite \ + --suite-config \ + [--location ] \ + [--output-dir ] +``` + +**Suite Config Format (`suite.json`):** +```json +[ + { + "name": "nextjs-reference", + "pathA": "./apps/nextjs-reference/stable", + "pathB": "./apps/nextjs-reference/canary" + }, + { + "name": "nextjs-sample", + "pathA": "./next-sample-1", + "pathB": "./next-sample-1-modified" + } +] +``` + +--- + +## Core Architecture + +### 1. Quota & Slot Pool Management +To work around the standard project limit of **10 backends per project**, the tool manages a leased pool of **5 parallel comparison slots** (`compare-slot-1` to `compare-slot-5`). +* Each slot contains an A/B pair of backends: `compare-slot-X-a` and `compare-slot-X-b`. +* Slot backends are dynamically created upon first run and **kept alive (reused)** for subsequent runs, reducing rollout provisioning time by ~2 minutes. +* State is managed via GCP labels (`status: busy`, `status: idle`, `last_active: `). +* If a run is interrupted (`SIGINT`/`SIGTERM`), a signal handler cleans up resources and releases the slot lock. +* A startup GC sweeper automatically releases slot locks held `busy` for more than 2 hours. + +### 2. Secret manager Isolation +If the codebases reference Google Cloud Secret Manager values in their `apphosting.yaml` configurations, the tool: +* Automatically provisions mock sandboxed secrets (prefixed with `cmp-sec-[slot]-`) to prevent naming conflicts. +* Grants read access to the slot backend's Cloud Run service account. +* Safely removes the mock secrets from Secret Manager when the comparison concludes. + +### 3. Route Discovery +The comparison engine discovers routes using a two-pass mechanism: +* **Static Discovery**: Parses local Next.js build manifests (e.g., `.next/routes-manifest.json`, `.next/prerender-manifest.json`) and local `sitemap.xml` files. +* **Dynamic Crawling**: Executes a recursive HTML link crawler on Backend A to dynamically harvest links, query strings, and redirects (up to 5 redirect levels) at runtime. + +### 4. Differential Analyzer +Once routes are collected, the analyzer queries the matching paths on both backends and performs: +* **Status Match**: Asserts status code parity (e.g. `200` vs `200`, `404` vs `404`). +* **Header Auditing**: Separates expected dynamic headers (`x-cloud-trace-context`, `date`, `etag`, `age`) from static behavioral headers (`content-type`, `cache-control`) to detect regression. +* **Payload Comparison**: + * **Text/HTML**: Computes Myers' line-level diffs and Sorensen-Dice similarity scores to match dynamic content. + * **Binary Assets**: Compares exact size (bytes) and SHA-256 payload hashes. + +### 5. Premium Dashboard +The comparison run outputs: +* **`summary.json`**: Machine-readable JSON structured logs of every mismatch, status code, and similarity score. +* **`index.html`**: A responsive, modern dark-mode HTML split-pane code diff viewer, letting you visually compare the precise lines that changed on each page. diff --git a/src/apphosting/compare/compare.spec.ts b/src/apphosting/compare/compare.spec.ts new file mode 100644 index 00000000000..c0df78d9a23 --- /dev/null +++ b/src/apphosting/compare/compare.spec.ts @@ -0,0 +1,84 @@ +import { expect } from "chai"; +import * as nock from "nock"; +import { compareRoute } from "./compare"; + +describe("compareRoute", () => { + beforeEach(() => { + nock.cleanAll(); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + it("should match identical text pages", async () => { + nock("https://backend-a.com").get("/").reply(200, "hello world", { + "Content-Type": "text/html", + "Cache-Control": "public, max-age=3600" + }); + nock("https://backend-b.com").get("/").reply(200, "hello world", { + "Content-Type": "text/html", + "Cache-Control": "public, max-age=3600" + }); + + const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); + expect(res.statusMatch).to.be.true; + expect(res.headerMismatches).to.be.empty; + expect(res.bodySimilarity).to.equal(1.0); + expect(res.isBinary).to.be.false; + }); + + it("should flag status mismatches", async () => { + nock("https://backend-a.com").get("/").reply(200, "ok"); + nock("https://backend-b.com").get("/").reply(404, "not found"); + + const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); + expect(res.statusMatch).to.be.false; + }); + + it("should flag behavioral header mismatches but record dynamic ones as expected variations", async () => { + nock("https://backend-a.com").get("/").reply(200, "ok", { + "Cache-Control": "public, max-age=3600", + "Date": "Wed, 17 Jun 2026 01:00:00 GMT" + }); + nock("https://backend-b.com").get("/").reply(200, "ok", { + "Cache-Control": "no-cache", + "Date": "Wed, 17 Jun 2026 01:05:00 GMT" + }); + + const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); + expect(res.headerMismatches).to.have.lengthOf(1); + expect(res.headerMismatches[0].header.toLowerCase()).to.equal("cache-control"); + + expect(res.expectedHeaderVariations).to.have.lengthOf(1); + expect(res.expectedHeaderVariations[0].header.toLowerCase()).to.equal("date"); + }); + + it("should match identical binary pages and flag mismatching binary content", async () => { + const binA = Buffer.from([1, 2, 3, 4]); + const binB = Buffer.from([1, 2, 3, 4]); + const binC = Buffer.from([1, 2, 3, 5]); + + nock("https://backend-a.com").get("/img.png").reply(200, binA, { + "Content-Type": "image/png" + }); + nock("https://backend-b.com").get("/img.png").reply(200, binB, { + "Content-Type": "image/png" + }); + + const resMatch = await compareRoute("/img.png", "https://backend-a.com", "https://backend-b.com"); + expect(resMatch.isBinary).to.be.true; + expect(resMatch.bodySimilarity).to.equal(1.0); + + nock("https://backend-a.com").get("/img.png").reply(200, binA, { + "Content-Type": "image/png" + }); + nock("https://backend-b.com").get("/img.png").reply(200, binC, { + "Content-Type": "image/png" + }); + + const resMismatch = await compareRoute("/img.png", "https://backend-a.com", "https://backend-b.com"); + expect(resMismatch.isBinary).to.be.true; + expect(resMismatch.bodySimilarity).to.equal(0.0); + }); +}); diff --git a/src/apphosting/compare/compare.ts b/src/apphosting/compare/compare.ts new file mode 100644 index 00000000000..389104dc933 --- /dev/null +++ b/src/apphosting/compare/compare.ts @@ -0,0 +1,120 @@ +import fetch from "node-fetch"; +import * as crypto from "crypto"; +import { MyersDiffEngine } from "./distance"; + +export interface ComparisonResult { + route: string; + statusMatch: boolean; + headerMismatches: Array<{ header: string; valA: string; valB: string }>; + expectedHeaderVariations: Array<{ header: string; valA: string; valB: string }>; + bodySimilarity: number; // 0.0 to 1.0 + bodyDiff: string; + isBinary: boolean; +} + +const BEHAVIORAL_HEADERS = [ + "cache-control", + "content-security-policy", + "content-type", + "content-encoding", + "location", + "strict-transport-security" +]; + +const BINARY_CONTENT_TYPES = [ + "image/", + "application/pdf", + "application/zip", + "application/octet-stream" +]; + +function isBinaryContentType(contentType: string): boolean { + const normalized = contentType.toLowerCase(); + return BINARY_CONTENT_TYPES.some(type => normalized.includes(type)); +} + +export async function compareRoute( + route: string, + urlA: string, + urlB: string, + options: { headers?: Record } = {} +): Promise { + const fetchOptions = { + headers: options.headers || {}, + redirect: "manual" as const + }; + + const [resA, resB] = await Promise.all([ + fetch(`${urlA}${route}`, fetchOptions), + fetch(`${urlB}${route}`, fetchOptions) + ]); + + const result: ComparisonResult = { + route, + statusMatch: resA.status === resB.status, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 1.0, + bodyDiff: "", + isBinary: false + }; + + // 1. Compare Headers + const allHeaderKeys = new Set([ + ...Array.from(resA.headers.keys()), + ...Array.from(resB.headers.keys()) + ]); + + for (const key of allHeaderKeys) { + const valA = resA.headers.get(key) || ""; + const valB = resB.headers.get(key) || ""; + if (valA !== valB) { + if (BEHAVIORAL_HEADERS.includes(key.toLowerCase())) { + result.headerMismatches.push({ header: key, valA, valB }); + } else { + result.expectedHeaderVariations.push({ header: key, valA, valB }); + } + } + } + + // 2. Detect Binary + const contentTypeA = resA.headers.get("content-type") || ""; + const contentTypeB = resB.headers.get("content-type") || ""; + if (isBinaryContentType(contentTypeA) || isBinaryContentType(contentTypeB)) { + result.isBinary = true; + + const bufA = await resA.buffer(); + const bufB = await resB.buffer(); + + const sizeA = bufA.length; + const sizeB = bufB.length; + + if (sizeA !== sizeB) { + result.bodySimilarity = 0.0; + result.bodyDiff = `Binary size mismatch: ${sizeA} bytes vs ${sizeB} bytes`; + } else { + const hashA = crypto.createHash("sha256").update(bufA).digest("hex"); + const hashB = crypto.createHash("sha256").update(bufB).digest("hex"); + if (hashA === hashB) { + result.bodySimilarity = 1.0; + } else { + result.bodySimilarity = 0.0; + result.bodyDiff = "Binary content hash mismatch"; + } + } + return result; + } + + // 3. Compare Text Body + const bodyA = await resA.text(); + const bodyB = await resB.text(); + + if (bodyA !== bodyB) { + result.bodySimilarity = MyersDiffEngine.getSimilarity(bodyA, bodyB); + if (result.bodySimilarity < 1.0) { + result.bodyDiff = "HTML content mismatch"; + } + } + + return result; +} diff --git a/src/apphosting/compare/crawler.spec.ts b/src/apphosting/compare/crawler.spec.ts new file mode 100644 index 00000000000..392ff9ffe38 --- /dev/null +++ b/src/apphosting/compare/crawler.spec.ts @@ -0,0 +1,84 @@ +import { expect } from "chai"; +import * as nock from "nock"; +import { Crawler } from "./crawler"; + +describe("Crawler", () => { + beforeEach(() => { + nock.cleanAll(); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + it("should recursively crawl text/html links", async () => { + const base = "https://example.com"; + + nock(base) + .get("/") + .reply(200, ` + + + About Us + Contact Us + External Link + + + `, { "Content-Type": "text/html" }); + + nock(base) + .get("/about") + .reply(200, ` + + + Careers + + + `, { "Content-Type": "text/html" }); + + nock(base) + .get("/contact") + .reply(200, "Contact Details", { "Content-Type": "text/html" }); + + nock(base) + .get("/careers") + .reply(200, "Join our team", { "Content-Type": "text/html" }); + + const crawler = new Crawler(base); + await crawler.crawl(); + + const routes = crawler.getRoutes(); + expect(routes).to.deep.equal(["/", "/about", "/careers", "/contact"]); + }); + + it("should follow redirect links", async () => { + const base = "https://example.com"; + + nock(base) + .get("/") + .reply(200, `Redirect me`, { "Content-Type": "text/html" }); + + nock(base) + .get("/old-link") + .reply(302, undefined, { "Location": "/new-link" }); + + nock(base) + .get("/new-link") + .reply(200, "Done!", { "Content-Type": "text/html" }); + + const crawler = new Crawler(base); + await crawler.crawl(); + + const routes = crawler.getRoutes(); + expect(routes).to.deep.equal(["/", "/new-link", "/old-link"]); + }); + + it("should canonicalize paths and sort queries", async () => { + const base = "https://example.com"; + const crawler = new Crawler(base); + + expect(crawler.canonicalizeRoute("/about/")).to.equal("/about"); + expect(crawler.canonicalizeRoute("/search?q=foo&a=bar")).to.equal("/search?a=bar&q=foo"); + expect(crawler.canonicalizeRoute("https://external.com/foo")).to.be.null; + }); +}); diff --git a/src/apphosting/compare/crawler.ts b/src/apphosting/compare/crawler.ts new file mode 100644 index 00000000000..ba401aac135 --- /dev/null +++ b/src/apphosting/compare/crawler.ts @@ -0,0 +1,124 @@ +import fetch from "node-fetch"; + +export class Crawler { + private visited = new Set(); + private discoveredRoutes = new Set(); + + constructor(private readonly baseUrl: string, private readonly maxDepth = 3) {} + + public getRoutes(): string[] { + return Array.from(this.discoveredRoutes).sort(); + } + + /** + * Crawls starting from root and recursively finds links. + */ + public async crawl(): Promise { + await this.crawlRoute("/"); + } + + private async crawlRoute(route: string, depth = 0): Promise { + const canonical = this.canonicalizeRoute(route); + if (!canonical || this.visited.has(canonical) || depth > this.maxDepth) { + return; + } + + this.visited.add(canonical); + this.discoveredRoutes.add(canonical); + + try { + const url = `${this.baseUrl}${canonical}`; + const res = await fetch(url, { + redirect: "manual" as const, + headers: { "User-Agent": "FirebaseCompareCrawler/1.0" } + }); + + // Handle Redirects + if ([301, 302, 307, 308].includes(res.status)) { + const location = res.headers.get("location"); + if (location) { + const nextRoute = this.resolveRelative(canonical, location); + if (nextRoute) { + await this.crawlRoute(nextRoute, depth + 1); + } + } + return; + } + + // Only parse HTML responses + const contentType = res.headers.get("content-type") || ""; + if (!contentType.toLowerCase().includes("text/html")) { + return; + } + + const html = await res.text(); + const links = this.extractLinks(html, canonical); + + for (const link of links) { + await this.crawlRoute(link, depth + 1); + } + } catch (err) { + // Ignore fetch failures for single routes during discovery + } + } + + public canonicalizeRoute(route: string): string | null { + try { + const url = new URL(route, this.baseUrl); + if (url.origin !== new URL(this.baseUrl).origin) { + return null; + } + + let pathname = url.pathname; + if (pathname.endsWith("/") && pathname.length > 1) { + pathname = pathname.slice(0, -1); + } + + const params = Array.from(url.searchParams.entries()).sort((a, b) => a[0].localeCompare(b[0])); + const search = params.length > 0 ? "?" + params.map(([k, v]) => `${k}=${v}`).join("&") : ""; + + return `${pathname}${search}`; + } catch { + return null; + } + } + + private extractLinks(html: string, currentRoute: string): string[] { + const links: string[] = []; + const regex = /]*?\s+)?href=(["'])(.*?)\1/gi; + let match; + + while ((match = regex.exec(html)) !== null) { + const href = match[2].trim(); + if ( + !href || + href.startsWith("#") || + href.startsWith("mailto:") || + href.startsWith("tel:") || + href.startsWith("javascript:") + ) { + continue; + } + + const resolved = this.resolveRelative(currentRoute, href); + if (resolved) { + links.push(resolved); + } + } + + return links; + } + + private resolveRelative(currentRoute: string, href: string): string | null { + try { + const base = new URL(currentRoute, this.baseUrl); + const resolved = new URL(href, base.href); + if (resolved.origin !== new URL(this.baseUrl).origin) { + return null; + } + return resolved.pathname + resolved.search; + } catch { + return null; + } + } +} diff --git a/src/apphosting/compare/discover.spec.ts b/src/apphosting/compare/discover.spec.ts new file mode 100644 index 00000000000..2ef9182edbd --- /dev/null +++ b/src/apphosting/compare/discover.spec.ts @@ -0,0 +1,62 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; +import { discoverRoutes } from "./discover"; + +describe("discoverRoutes", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = path.join(process.cwd(), "scratch-test-discover-" + Math.random().toString(36).substring(7)); + fs.ensureDirSync(tempDir); + }); + + afterEach(() => { + fs.removeSync(tempDir); + }); + + it("should return root route by default if no manifests exist", async () => { + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/"]); + }); + + it("should discover routes from Next.js manifests", async () => { + const nextDir = path.join(tempDir, ".next"); + fs.ensureDirSync(nextDir); + + fs.writeJsonSync(path.join(nextDir, "prerender-manifest.json"), { + routes: { + "/about": {}, + "/blog/post-1": {} + } + }); + + fs.writeJsonSync(path.join(nextDir, "routes-manifest.json"), { + staticRoutes: [ + { page: "/" }, + { page: "/contact" } + ] + }); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/about", "/blog/post-1", "/contact"]); + }); + + it("should discover routes from sitemap.xml", async () => { + const xml = ` + + + + https://example.com/ + + + https://example.com/products?id=12 + + + `; + fs.writeFileSync(path.join(tempDir, "sitemap.xml"), xml, "utf-8"); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/products?id=12"]); + }); +}); diff --git a/src/apphosting/compare/discover.ts b/src/apphosting/compare/discover.ts new file mode 100644 index 00000000000..ca62fe01495 --- /dev/null +++ b/src/apphosting/compare/discover.ts @@ -0,0 +1,68 @@ +import * as fs from "fs-extra"; +import * as path from "path"; +import { logger } from "../../logger"; + +/** + * Discovers built routes in a project by checking Next.js manifests and local sitemaps. + */ +export async function discoverRoutes(appPath: string): Promise { + const routes = new Set(["/"]); + + // 1. Next.js Manifest Parsing + const nextDir = path.join(appPath, ".next"); + if (await fs.pathExists(nextDir)) { + logger.info("Next.js build directory detected. Parsing manifests..."); + try { + const prerenderManifestPath = path.join(nextDir, "prerender-manifest.json"); + if (await fs.pathExists(prerenderManifestPath)) { + const prerender = await fs.readJson(prerenderManifestPath); + if (prerender.routes) { + for (const route of Object.keys(prerender.routes)) { + routes.add(route); + } + } + } + + const routesManifestPath = path.join(nextDir, "routes-manifest.json"); + if (await fs.pathExists(routesManifestPath)) { + const manifests = await fs.readJson(routesManifestPath); + if (manifests.staticRoutes) { + for (const route of manifests.staticRoutes) { + routes.add(route.page); + } + } + } + } catch (err) { + logger.debug(`Error parsing Next.js manifests: ${err}`); + } + } + + // 2. Local Sitemap Parsing (Regex-based) + const sitemapPaths = [ + path.join(appPath, "public", "sitemap.xml"), + path.join(appPath, "sitemap.xml"), + path.join(appPath, "dist", "sitemap.xml") + ]; + + for (const sitemapPath of sitemapPaths) { + if (await fs.pathExists(sitemapPath)) { + logger.info(`Sitemap detected at ${sitemapPath}. Parsing...`); + try { + const xml = await fs.readFile(sitemapPath, "utf-8"); + const locMatches = xml.matchAll(/\s*(https?:\/\/[^\s<]+)\s*<\/loc>/gi); + for (const match of locMatches) { + try { + const url = new URL(match[1].trim()); + routes.add(url.pathname + url.search); + } catch { + // Ignore invalid URLs + } + } + } catch (err) { + logger.debug(`Error parsing sitemap ${sitemapPath}: ${err}`); + } + } + } + + return Array.from(routes).sort(); +} diff --git a/src/apphosting/compare/distance.spec.ts b/src/apphosting/compare/distance.spec.ts new file mode 100644 index 00000000000..86ee0be00a2 --- /dev/null +++ b/src/apphosting/compare/distance.spec.ts @@ -0,0 +1,36 @@ +import { expect } from "chai"; +import { MyersDiffEngine } from "./distance"; + +describe("MyersDiffEngine", () => { + describe("getSimilarity", () => { + it("should return 1.0 for exact string match", () => { + const a = "line 1\nline 2\nline 3"; + const b = "line 1\nline 2\nline 3"; + expect(MyersDiffEngine.getSimilarity(a, b)).to.equal(1.0); + }); + + it("should return 1.0 for both empty strings", () => { + expect(MyersDiffEngine.getSimilarity("", "")).to.equal(1.0); + }); + + it("should return 0.0 if one string is empty", () => { + expect(MyersDiffEngine.getSimilarity("hello", "")).to.equal(0.0); + expect(MyersDiffEngine.getSimilarity("", "world")).to.equal(0.0); + }); + + it("should return correct similarity for partial match", () => { + // 2 matched lines, total lines A = 3, total lines B = 3 + // Similarity = (2 * 2) / (3 + 3) = 4 / 6 = 0.6666... + const a = "line 1\nline 2\nline 3"; + const b = "line 1\nline 2\nline 4"; + const similarity = MyersDiffEngine.getSimilarity(a, b); + expect(similarity).to.be.closeTo(0.666, 0.001); + }); + + it("should return 0.0 for completely disjoint strings", () => { + const a = "foo\nbar"; + const b = "baz\nqux"; + expect(MyersDiffEngine.getSimilarity(a, b)).to.equal(0.0); + }); + }); +}); diff --git a/src/apphosting/compare/distance.ts b/src/apphosting/compare/distance.ts new file mode 100644 index 00000000000..60e4c7d3722 --- /dev/null +++ b/src/apphosting/compare/distance.ts @@ -0,0 +1,41 @@ +import { diffLines } from "diff"; +import * as crypto from "crypto"; + +export class MyersDiffEngine { + /** + * Calculates a similarity score between 0.0 and 1.0 based on line differences. + */ + public static getSimilarity(a: string, b: string): number { + if (a === b) return 1.0; + if (a.length === 0 && b.length === 0) return 1.0; + if (a.length === 0 || b.length === 0) return 0.0; + + // Fast-path: hash comparison + const hashA = crypto.createHash("sha256").update(a).digest("hex"); + const hashB = crypto.createHash("sha256").update(b).digest("hex"); + if (hashA === hashB) return 1.0; + + const changes = diffLines(a, b); + let matchedLines = 0; + let totalLinesA = 0; + let totalLinesB = 0; + + for (const change of changes) { + const count = change.count || 0; + if (!change.added && !change.removed) { + matchedLines += count; + totalLinesA += count; + totalLinesB += count; + } else if (change.added) { + totalLinesB += count; + } else if (change.removed) { + totalLinesA += count; + } + } + + const totalLines = totalLinesA + totalLinesB; + if (totalLines === 0) return 1.0; + + return (2 * matchedLines) / totalLines; + } +} diff --git a/src/apphosting/compare/lifecycle.spec.ts b/src/apphosting/compare/lifecycle.spec.ts new file mode 100644 index 00000000000..9ce009cadb3 --- /dev/null +++ b/src/apphosting/compare/lifecycle.spec.ts @@ -0,0 +1,63 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; +import * as apphosting from "../../gcp/apphosting"; +import { validateProject, runGarbageCollection } from "./lifecycle"; + +describe("Lifecycle Manager", () => { + let listBackendsStub: sinon.SinonStub; + let updateBackendStub: sinon.SinonStub; + + beforeEach(() => { + listBackendsStub = sinon.stub(apphosting, "listBackends"); + updateBackendStub = sinon.stub(apphosting, "updateBackend"); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe("validateProject", () => { + it("should allow whitelisted projects", () => { + expect(() => validateProject("aryanf-test")).to.not.throw(); + expect(() => validateProject("pretend-public")).to.not.throw(); + }); + + it("should throw on non-whitelisted projects", () => { + expect(() => validateProject("my-secret-project")).to.throw( + /Invalid project ID "my-secret-project"/ + ); + }); + }); + + describe("runGarbageCollection", () => { + it("should unlock stale busy backends (> 2 hours)", async () => { + const now = Date.now(); + const threeHoursAgo = new Date(now - 3 * 60 * 60 * 1000).toISOString(); + const oneHourAgo = new Date(now - 1 * 60 * 60 * 1000).toISOString(); + + listBackendsStub.resolves({ + backends: [ + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-a", + labels: { status: "busy", type: "comparison-sandbox" }, + updateTime: threeHoursAgo + }, + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-b", + labels: { status: "busy", type: "comparison-sandbox" }, + updateTime: oneHourAgo + } + ] + }); + + updateBackendStub.resolves({ name: "op-name" }); + + await runGarbageCollection("aryanf-test", "us-central1"); + + expect(updateBackendStub.callCount).to.equal(1); + const args = updateBackendStub.firstCall.args; + expect(args[2]).to.equal("compare-slot-1-a"); + expect(args[3].labels.status).to.equal("idle"); + }); + }); +}); diff --git a/src/apphosting/compare/lifecycle.ts b/src/apphosting/compare/lifecycle.ts new file mode 100644 index 00000000000..66aa0e0f3cd --- /dev/null +++ b/src/apphosting/compare/lifecycle.ts @@ -0,0 +1,77 @@ +import { FirebaseError } from "../../error"; +import * as apphosting from "../../gcp/apphosting"; +import { logger } from "../../logger"; +import { acquireComparisonSlot, releaseComparisonSlot } from "./slots"; + +const ALLOWED_PROJECTS = ["aryanf-test", "pretend-public"]; + +export function validateProject(projectId: string): void { + if (!ALLOWED_PROJECTS.includes(projectId)) { + throw new FirebaseError( + `Invalid project ID "${projectId}". This tool can only run on projects: ${ALLOWED_PROJECTS.join(", ")}` + ); + } +} + +/** + * Sweeps all slots in the project, resetting stale locks (busy for > 2 hours) to idle. + */ +export async function runGarbageCollection(projectId: string, location: string): Promise { + const existingBackends = await apphosting.listBackends(projectId, location); + const backendsList = existingBackends.backends || []; + const now = Date.now(); + const twoHours = 2 * 60 * 60 * 1000; + + for (const backend of backendsList) { + const nameParts = backend.name.split("/"); + const backendId = nameParts[nameParts.length - 1]; + + if (backendId.startsWith("compare-slot-")) { + const isBusy = backend.labels?.status === "busy"; + if (isBusy) { + const updateTime = new Date(backend.updateTime).getTime(); + if (now - updateTime > twoHours) { + logger.info(`Found stale lock on comparison slot backend ${backendId}. Unlocking...`); + try { + await apphosting.updateBackend(projectId, location, backendId, { + labels: { ...backend.labels, status: "idle" } + }); + } catch (err) { + logger.debug(`Failed to unlock stale backend ${backendId}: ${err}`); + } + } + } + } + } +} + +export async function runAutonomousComparison( + projectId: string, + location: string, + localPath: string, + options: any +): Promise { + validateProject(projectId); + await runGarbageCollection(projectId, location); + + const slot = await acquireComparisonSlot(projectId, location); + + const cleanUpAndExit = async () => { + logger.warn("\nProcess interrupted. Cleaning up comparison slot lock before exit..."); + await releaseComparisonSlot(projectId, location, slot.index); + process.exit(1); + }; + + process.on("SIGINT", cleanUpAndExit); + process.on("SIGTERM", cleanUpAndExit); + + try { + // Setup secrets, deploy, and compare... + logger.info(`Using Comparison Slot ${slot.index} (Backend A: ${slot.backendIdA}, Backend B: ${slot.backendIdB})`); + } finally { + process.off("SIGINT", cleanUpAndExit); + process.off("SIGTERM", cleanUpAndExit); + + await releaseComparisonSlot(projectId, location, slot.index); + } +} diff --git a/src/apphosting/compare/reporter.spec.ts b/src/apphosting/compare/reporter.spec.ts new file mode 100644 index 00000000000..5d828bb05d5 --- /dev/null +++ b/src/apphosting/compare/reporter.spec.ts @@ -0,0 +1,66 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; +import { generateReport } from "./reporter"; +import { ComparisonResult } from "./compare"; + +describe("Report Generator", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = path.join(process.cwd(), "scratch-test-report-" + Math.random().toString(36).substring(7)); + fs.ensureDirSync(tempDir); + }); + + afterEach(() => { + fs.removeSync(tempDir); + }); + + it("should generate JSON and HTML reports", async () => { + const results: ComparisonResult[] = [ + { + route: "/", + statusMatch: true, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 1.0, + bodyDiff: "", + isBinary: false + }, + { + route: "/about", + statusMatch: true, + headerMismatches: [{ header: "Cache-Control", valA: "max-age=0", valB: "no-cache" }], + expectedHeaderVariations: [], + bodySimilarity: 0.95, + bodyDiff: "HTML content mismatch", + isBinary: false + } + ]; + + await generateReport( + "aryanf-test", + "us-central1", + "compare-slot-1-a", + "compare-slot-1-b", + results, + tempDir + ); + + const jsonPath = path.join(tempDir, "summary.json"); + const htmlPath = path.join(tempDir, "index.html"); + + expect(fs.existsSync(jsonPath)).to.be.true; + expect(fs.existsSync(htmlPath)).to.be.true; + + const data = fs.readJsonSync(jsonPath); + expect(data.summary.totalRoutes).to.equal(2); + expect(data.summary.matchingRoutes).to.equal(1); + expect(data.summary.mismatchingRoutes).to.equal(1); + expect(data.summary.overallSimilarity).to.be.closeTo(0.975, 0.001); + + const htmlContent = fs.readFileSync(htmlPath, "utf-8"); + expect(htmlContent).to.include("App Hosting Comparison Dashboard"); + expect(htmlContent).to.include("/about"); + }); +}); diff --git a/src/apphosting/compare/reporter.ts b/src/apphosting/compare/reporter.ts new file mode 100644 index 00000000000..59a23e7ca8e --- /dev/null +++ b/src/apphosting/compare/reporter.ts @@ -0,0 +1,324 @@ +import * as fs from "fs-extra"; +import * as path from "path"; +import * as clc from "colorette"; +import { ComparisonResult } from "./compare"; +import { logger } from "../../logger"; + +export interface ComparisonSummary { + projectId: string; + location: string; + backendA: string; + backendB: string; + timestamp: string; + totalRoutes: number; + matchingRoutes: number; + mismatchingRoutes: number; + overallSimilarity: number; +} + +export async function generateReport( + projectId: string, + location: string, + backendA: string, + backendB: string, + results: ComparisonResult[], + outputDir = "./compare-report" +): Promise { + const totalRoutes = results.length; + const matching = results.filter( + r => r.statusMatch && r.headerMismatches.length === 0 && r.bodySimilarity >= 0.99 + ); + const mismatches = results.filter( + r => !r.statusMatch || r.headerMismatches.length > 0 || r.bodySimilarity < 0.99 + ); + + const totalSimilarity = results.reduce((sum, r) => sum + r.bodySimilarity, 0); + const overallSimilarity = totalRoutes > 0 ? totalSimilarity / totalRoutes : 1.0; + + const summary: ComparisonSummary = { + projectId, + location, + backendA, + backendB, + timestamp: new Date().toISOString(), + totalRoutes, + matchingRoutes: matching.length, + mismatchingRoutes: mismatches.length, + overallSimilarity + }; + + logger.info("\n=========================================="); + logger.info(" COMPARISON TEST SUMMARY"); + logger.info("=========================================="); + logger.info(`Project: ${projectId}`); + logger.info(`Location: ${location}`); + logger.info(`Backend A: ${backendA}`); + logger.info(`Backend B: ${backendB}`); + logger.info(`Total Routes: ${totalRoutes}`); + logger.info(`Passed: ${clc.green(String(matching.length))}`); + logger.info(`Mismatched: ${mismatches.length > 0 ? clc.red(String(mismatches.length)) : clc.green("0")}`); + logger.info(`Similarity: ${clc.cyan((overallSimilarity * 100).toFixed(2) + "%")}`); + logger.info("==========================================\n"); + + if (mismatches.length > 0) { + logger.warn(clc.bold(clc.red("Mismatched Routes:"))); + for (const m of mismatches) { + logger.warn(` - ${clc.bold(m.route)}`); + if (!m.statusMatch) { + logger.warn(` * Status Code Mismatch`); + } + if (m.headerMismatches.length > 0) { + logger.warn(` * ${m.headerMismatches.length} Behavioral Header Mismatch(es)`); + } + if (m.bodySimilarity < 0.99) { + logger.warn(` * Body Similarity: ${(m.bodySimilarity * 100).toFixed(2)}%`); + } + } + logger.info(""); + } + + await fs.ensureDir(outputDir); + await fs.writeJson(path.join(outputDir, "summary.json"), { summary, results }, { spaces: 2 }); + logger.info(`JSON report saved to: ${path.join(outputDir, "summary.json")}`); + + const html = getHtmlTemplate(summary, results); + await fs.outputFile(path.join(outputDir, "index.html"), html, "utf-8"); + logger.info(`HTML Dashboard generated at: ${clc.underline(path.join(outputDir, "index.html"))}\n`); +} + +function getHtmlTemplate(summary: ComparisonSummary, results: ComparisonResult[]): string { + const resultsRows = results.map(r => { + const isPass = r.statusMatch && r.headerMismatches.length === 0 && r.bodySimilarity >= 0.99; + const badgeClass = isPass ? "badge-success" : "badge-error"; + const badgeText = isPass ? "PASS" : "FAIL"; + + const headersList = r.headerMismatches.map(m => + `
  • ${m.header}: "${m.valA}" vs "${m.valB}"
    • ` + ).join(""); + + const variationsList = r.expectedHeaderVariations.map(m => + `
  • ${m.header}: "${m.valA}" vs "${m.valB}"
    • ` + ).join(""); + + return ` + + ${r.route} + ${badgeText} + ${r.statusMatch ? 'Match' : 'Mismatch'} + ${(r.bodySimilarity * 100).toFixed(1)}% + + ${r.headerMismatches.length > 0 ? `
    Mismatches:
      ${headersList}
    ` : 'Match'} + ${r.expectedHeaderVariations.length > 0 ? `
    Variations:
      ${variationsList}
    ` : ''} + + + `; + }).join(""); + + return ` + + + + + + App Hosting Comparison Report + + + + + +
    +

    App Hosting Comparison Dashboard

    +
    + Ran on ${new Date(summary.timestamp).toLocaleString()} for project ${summary.projectId} (${summary.location}) +
    +
    + +
    +
    +
    Total Checked Routes
    +
    ${summary.totalRoutes}
    +
    +
    +
    Passed Routes
    +
    ${summary.matchingRoutes}
    +
    +
    +
    Mismatched Routes
    +
    ${summary.mismatchingRoutes}
    +
    +
    +
    Overall Similarity
    +
    ${(summary.overallSimilarity * 100).toFixed(2)}%
    +
    +
    + +
    + + + + + + + + + + + + ${resultsRows} + +
    RouteParityHTTP StatusBody ParityHeader Assertions
    +
    + + + + `; +} diff --git a/src/apphosting/compare/secrets.spec.ts b/src/apphosting/compare/secrets.spec.ts new file mode 100644 index 00000000000..0cba896dde9 --- /dev/null +++ b/src/apphosting/compare/secrets.spec.ts @@ -0,0 +1,91 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; +import * as fs from "fs-extra"; +import { AppHostingYamlConfig } from "../yaml"; +import * as apphosting from "../../gcp/apphosting"; +import * as csm from "../../gcp/secretManager"; +import * as secretsHelper from "../secrets"; +import * as projectNumberHelper from "../../getProjectNumber"; +import { setupSandboxSecrets, cleanupSandboxSecrets } from "./secrets"; + +describe("Sandbox Secrets Manager", () => { + let pathExistsStub: sinon.SinonStub; + let loadConfigStub: sinon.SinonStub; + let getProjectNumberStub: sinon.SinonStub; + let getBackendStub: sinon.SinonStub; + let serviceAccountsStub: sinon.SinonStub; + let secretExistsStub: sinon.SinonStub; + let createSecretStub: sinon.SinonStub; + let addVersionStub: sinon.SinonStub; + let deleteSecretStub: sinon.SinonStub; + let grantSecretAccessStub: sinon.SinonStub; + + beforeEach(() => { + pathExistsStub = sinon.stub(fs, "pathExists"); + loadConfigStub = sinon.stub(AppHostingYamlConfig, "loadFromFile"); + getProjectNumberStub = sinon.stub(projectNumberHelper, "getProjectNumber"); + getBackendStub = sinon.stub(apphosting, "getBackend"); + serviceAccountsStub = sinon.stub(secretsHelper, "serviceAccountsForBackend"); + secretExistsStub = sinon.stub(csm, "secretExists"); + createSecretStub = sinon.stub(csm, "createSecret"); + addVersionStub = sinon.stub(csm, "addVersion"); + deleteSecretStub = sinon.stub(csm, "deleteSecret"); + grantSecretAccessStub = sinon.stub(secretsHelper, "grantSecretAccess"); + }); + + afterEach(() => { + sinon.restore(); + }); + + it("should setup sandbox secrets for yaml configuration", async () => { + pathExistsStub.resolves(true); + + const mockYaml = new AppHostingYamlConfig(); + mockYaml.env = { + API_KEY: { secret: "my-production-api-key", availability: ["RUNTIME"] } + }; + loadConfigStub.resolves(mockYaml); + getProjectNumberStub.resolves("12345"); + + getBackendStub.resolves({ name: "backend-resource" }); + serviceAccountsStub.resolves({ + buildServiceAccount: "build-sa@google.com", + runServiceAccount: "run-sa@google.com" + }); + + secretExistsStub.resolves(false); + createSecretStub.resolves(); + addVersionStub.resolves(); + grantSecretAccessStub.resolves(); + + const mappings = await setupSandboxSecrets( + "aryanf-test", + "us-central1", + "/app/path", + 1, + "compare-slot-1-a", + "compare-slot-1-b" + ); + + expect(mappings).to.have.lengthOf(1); + expect(mappings[0].originalName).to.equal("my-production-api-key"); + expect(mappings[0].mockSecretName).to.equal("cmp-sec-1-my-production-api-key"); + expect(mappings[0].mockValue).to.equal("mock-value-for-API_KEY-slot-1"); + + expect(createSecretStub.callCount).to.equal(1); + expect(addVersionStub.callCount).to.equal(1); + expect(grantSecretAccessStub.callCount).to.equal(1); + }); + + it("should delete secrets on cleanup", async () => { + deleteSecretStub.resolves(); + + const mappings = [ + { originalName: "my-key", mockSecretName: "cmp-sec-1-my-key", mockValue: "val" } + ]; + + await cleanupSandboxSecrets("aryanf-test", mappings); + expect(deleteSecretStub.callCount).to.equal(1); + expect(deleteSecretStub.firstCall.args[1]).to.equal("cmp-sec-1-my-key"); + }); +}); diff --git a/src/apphosting/compare/secrets.ts b/src/apphosting/compare/secrets.ts new file mode 100644 index 00000000000..9b1b47e90bd --- /dev/null +++ b/src/apphosting/compare/secrets.ts @@ -0,0 +1,95 @@ +import * as path from "path"; +import * as fs from "fs-extra"; +import * as csm from "../../gcp/secretManager"; +import * as apphosting from "../../gcp/apphosting"; +import { AppHostingYamlConfig } from "../yaml"; +import { getProjectNumber } from "../../getProjectNumber"; +import { serviceAccountsForBackend, grantSecretAccess, toMulti } from "../secrets"; +import { logger } from "../../logger"; + +export interface SecretMapping { + originalName: string; + mockSecretName: string; + mockValue: string; +} + +export async function setupSandboxSecrets( + projectId: string, + location: string, + appPath: string, + slotIndex: number, + backendIdA: string, + backendIdB: string +): Promise { + const yamlPath = path.join(appPath, "apphosting.yaml"); + if (!(await fs.pathExists(yamlPath))) { + return []; + } + + const config = await AppHostingYamlConfig.loadFromFile(yamlPath); + const secretEntries = Object.entries(config.env).filter(([, val]) => val.secret !== undefined); + if (secretEntries.length === 0) { + return []; + } + + const projectNumber = await getProjectNumber({ projectId }); + const mappings: SecretMapping[] = []; + + // Fetch backends to extract their service accounts + const [backendA, backendB] = await Promise.all([ + apphosting.getBackend(projectId, location, backendIdA), + apphosting.getBackend(projectId, location, backendIdB) + ]); + + const [accountsA, accountsB] = await Promise.all([ + serviceAccountsForBackend(projectNumber, backendA), + serviceAccountsForBackend(projectNumber, backendB) + ]); + + const multiAccountsA = toMulti(accountsA); + const multiAccountsB = toMulti(accountsB); + + // Combine build/run service accounts from both backends + const combinedAccounts = { + buildServiceAccounts: Array.from(new Set([...multiAccountsA.buildServiceAccounts, ...multiAccountsB.buildServiceAccounts])), + runServiceAccounts: Array.from(new Set([...multiAccountsA.runServiceAccounts, ...multiAccountsB.runServiceAccounts])) + }; + + for (const [envName, envVal] of secretEntries) { + const originalSecretName = envVal.secret!; + // Clean and build mock secret name + const cleanName = originalSecretName.replace(/[^a-zA-Z0-9_-]/g, "").toLowerCase(); + const mockSecretName = `cmp-sec-${slotIndex}-${cleanName}`.substring(0, 255); + const mockValue = `mock-value-for-${envName}-slot-${slotIndex}`; + + logger.info(`Setting up sandboxed secret for ${envName}: ${mockSecretName}...`); + + const exists = await csm.secretExists(projectId, mockSecretName); + if (!exists) { + await csm.createSecret(projectId, mockSecretName, { + "created-by": "apphosting-compare-tool", + "slot": String(slotIndex) + }); + } + + await csm.addVersion(projectId, mockSecretName, mockValue); + await grantSecretAccess(projectId, projectNumber, mockSecretName, combinedAccounts); + + mappings.push({ + originalName: originalSecretName, + mockSecretName, + mockValue + }); + } + + return mappings; +} + +export async function cleanupSandboxSecrets(projectId: string, mappings: SecretMapping[]): Promise { + if (mappings.length === 0) return; + + logger.info("Cleaning up sandboxed secrets in Secret Manager..."); + await Promise.allSettled( + mappings.map(m => csm.deleteSecret(projectId, m.mockSecretName).catch(e => logger.debug(e))) + ); +} diff --git a/src/apphosting/compare/slots.spec.ts b/src/apphosting/compare/slots.spec.ts new file mode 100644 index 00000000000..73d47c9b433 --- /dev/null +++ b/src/apphosting/compare/slots.spec.ts @@ -0,0 +1,75 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; + +import * as apphosting from "../../gcp/apphosting"; +import * as apps from "../../management/apps"; +import * as backendHelper from "../backend"; +import * as poller from "../../operation-poller"; +import { acquireComparisonSlot, releaseComparisonSlot } from "./slots"; + +describe("Comparison Slots Manager", () => { + let listBackendsStub: sinon.SinonStub; + let updateBackendStub: sinon.SinonStub; + let createBackendStub: sinon.SinonStub; + let listFirebaseAppsStub: sinon.SinonStub; + let createWebAppStub: sinon.SinonStub; + let pollOperationStub: sinon.SinonStub; + + beforeEach(() => { + listBackendsStub = sinon.stub(apphosting, "listBackends"); + updateBackendStub = sinon.stub(apphosting, "updateBackend"); + createBackendStub = sinon.stub(backendHelper, "createBackend"); + listFirebaseAppsStub = sinon.stub(apps, "listFirebaseApps"); + createWebAppStub = sinon.stub(apps, "createWebApp"); + pollOperationStub = sinon.stub(poller, "pollOperation").resolves(); + }); + + afterEach(() => { + sinon.restore(); + }); + + it("should acquire an existing idle slot", async () => { + listBackendsStub.resolves({ + backends: [ + { name: "projects/test/locations/us-central1/backends/compare-slot-1-a", labels: { status: "idle", type: "comparison-sandbox" } }, + { name: "projects/test/locations/us-central1/backends/compare-slot-1-b", labels: { status: "idle", type: "comparison-sandbox" } } + ] + }); + listFirebaseAppsStub.resolves([{ appId: "web-app-123", displayName: "existing-app" }]); + updateBackendStub.resolves({ name: "op-name" }); + + const slot = await acquireComparisonSlot("aryanf-test", "us-central1"); + expect(slot.index).to.equal(1); + expect(slot.backendIdA).to.equal("compare-slot-1-a"); + expect(slot.backendIdB).to.equal("compare-slot-1-b"); + + expect(updateBackendStub.callCount).to.equal(2); + expect(createBackendStub.callCount).to.equal(0); + }); + + it("should provision a slot if it doesn't exist and project is below limit", async () => { + listBackendsStub.resolves({ backends: [] }); + listFirebaseAppsStub.resolves([{ appId: "web-app-123", displayName: "existing-app" }]); + createBackendStub.resolves({ name: "backend-resource" }); + updateBackendStub.resolves({ name: "op-name" }); + + const slot = await acquireComparisonSlot("aryanf-test", "us-central1"); + expect(slot.index).to.equal(1); + expect(createBackendStub.callCount).to.equal(2); + }); + + it("should throw if all slots are locked/busy", async () => { + const busyBackends = []; + for (let i = 1; i <= 5; i++) { + busyBackends.push( + { name: `projects/test/locations/us-central1/backends/compare-slot-${i}-a`, labels: { status: "busy", type: "comparison-sandbox" } }, + { name: `projects/test/locations/us-central1/backends/compare-slot-${i}-b`, labels: { status: "busy", type: "comparison-sandbox" } } + ); + } + listBackendsStub.resolves({ backends: busyBackends }); + + await expect(acquireComparisonSlot("aryanf-test", "us-central1")).to.be.rejectedWith( + "All 5 comparison slots are currently in use or project backend limits exceeded" + ); + }); +}); diff --git a/src/apphosting/compare/slots.ts b/src/apphosting/compare/slots.ts new file mode 100644 index 00000000000..a7b5ab19ce7 --- /dev/null +++ b/src/apphosting/compare/slots.ts @@ -0,0 +1,119 @@ +import * as apphosting from "../../gcp/apphosting"; +import * as poller from "../../operation-poller"; +import { createBackend } from "../backend"; +import { listFirebaseApps, createWebApp, AppPlatform } from "../../management/apps"; +import { FirebaseError } from "../../error"; +import { logger } from "../../logger"; +import { apphostingOrigin } from "../../api"; + +export interface ComparisonSlot { + index: number; + backendIdA: string; + backendIdB: string; +} + +const MAX_SLOTS = 5; +const apphostingPollerOptions = { + apiOrigin: apphostingOrigin(), + apiVersion: apphosting.API_VERSION, + masterTimeout: 25 * 60 * 1_000, + maxBackoff: 10_000, +}; + +async function updateBackendAndPoll( + projectId: string, + location: string, + backendId: string, + labels: Record +): Promise { + const op = await apphosting.updateBackend(projectId, location, backendId, { labels }); + await poller.pollOperation({ + ...apphostingPollerOptions, + pollerName: `update-labels-${projectId}-${location}-${backendId}`, + operationResourceName: op.name, + }); +} + +export async function getOrCreateSharedWebAppId(projectId: string): Promise { + const apps = await listFirebaseApps(projectId, AppPlatform.WEB); + if (apps.length > 0) { + return apps[0].appId; + } + + logger.info("No existing Web Apps found. Provisioning a shared Web App for comparison slot runners..."); + const createdApp = await createWebApp(projectId, { displayName: "firebase-compare-shared-app" }); + return createdApp.appId; +} + +export async function acquireComparisonSlot( + projectId: string, + location: string +): Promise { + const existingBackends = await apphosting.listBackends(projectId, location); + const backendsList = existingBackends.backends || []; + + for (let i = 1; i <= MAX_SLOTS; i++) { + const backendIdA = `compare-slot-${i}-a`; + const backendIdB = `compare-slot-${i}-b`; + + const backendA = backendsList.find(b => b.name.endsWith(backendIdA)); + const backendB = backendsList.find(b => b.name.endsWith(backendIdB)); + + const isLocked = backendA?.labels?.status === "busy" || backendB?.labels?.status === "busy"; + + if (!isLocked) { + const webAppId = await getOrCreateSharedWebAppId(projectId); + + if (!backendA || !backendB) { + const slotsNeeded = (backendA ? 0 : 1) + (backendB ? 0 : 1); + if (backendsList.length + slotsNeeded > 10) { + continue; // Quota limit hit, check next slot + } + + if (!backendA) { + logger.info(`Provisioning backend for Comparison Slot ${i} (A)...`); + await createBackend(projectId, location, backendIdA, null, undefined, webAppId); + await updateBackendAndPoll(projectId, location, backendIdA, { status: "busy", type: "comparison-sandbox" }); + } + if (!backendB) { + logger.info(`Provisioning backend for Comparison Slot ${i} (B)...`); + await createBackend(projectId, location, backendIdB, null, undefined, webAppId); + await updateBackendAndPoll(projectId, location, backendIdB, { status: "busy", type: "comparison-sandbox" }); + } + } else { + logger.info(`Acquiring Comparison Slot ${i} (Reusing existing backends)...`); + await Promise.all([ + updateBackendAndPoll(projectId, location, backendIdA, { ...backendA.labels, status: "busy" }), + updateBackendAndPoll(projectId, location, backendIdB, { ...backendB.labels, status: "busy" }) + ]); + } + + return { index: i, backendIdA, backendIdB }; + } + } + + throw new FirebaseError( + "All 5 comparison slots are currently in use or project backend limits exceeded. Please wait and try again." + ); +} + +export async function releaseComparisonSlot( + projectId: string, + location: string, + slotIndex: number +): Promise { + const backendIdA = `compare-slot-${slotIndex}-a`; + const backendIdB = `compare-slot-${slotIndex}-b`; + + logger.info(`Releasing Comparison Slot ${slotIndex}...`); + + const existingBackends = await apphosting.listBackends(projectId, location); + const backendsList = existingBackends.backends || []; + const backendA = backendsList.find(b => b.name.endsWith(backendIdA)); + const backendB = backendsList.find(b => b.name.endsWith(backendIdB)); + + await Promise.allSettled([ + backendA ? updateBackendAndPoll(projectId, location, backendIdA, { ...backendA.labels, status: "idle" }) : Promise.resolve(), + backendB ? updateBackendAndPoll(projectId, location, backendIdB, { ...backendB.labels, status: "idle" }) : Promise.resolve() + ]); +} diff --git a/src/apphosting/compare/suite.spec.ts b/src/apphosting/compare/suite.spec.ts new file mode 100644 index 00000000000..7de43f541e6 --- /dev/null +++ b/src/apphosting/compare/suite.spec.ts @@ -0,0 +1,132 @@ +import * as path from "path"; +import * as fs from "fs-extra"; +import { expect } from "chai"; +import * as sinon from "sinon"; +import * as gcs from "../../gcp/storage"; +import * as apphosting from "../../gcp/apphosting"; +import * as rolloutHelper from "../rollout"; +import * as deployUtil from "../../deploy/apphosting/util"; +import * as projectNumberHelper from "../../getProjectNumber"; +import * as secretsManager from "./secrets"; +import * as slotsManager from "./slots"; +import * as discoverManager from "./discover"; +import { Crawler } from "./crawler"; +import * as compareManager from "./compare"; +import * as reporterManager from "./reporter"; +import * as lifecycle from "./lifecycle"; +import * as localBuildsModule from "../localbuilds"; +import { runCompareSuite } from "./suite"; + +describe("runCompareSuite Orchestrator", () => { + let tempDir: string; + let dummyZip: string; + + let upsertBucketStub: sinon.SinonStub; + let createArchiveStub: sinon.SinonStub; + let uploadObjectStub: sinon.SinonStub; + let orchestrateRolloutStub: sinon.SinonStub; + let getProjectNumberStub: sinon.SinonStub; + let setupSecretsStub: sinon.SinonStub; + let cleanupSecretsStub: sinon.SinonStub; + let acquireSlotStub: sinon.SinonStub; + let releaseSlotStub: sinon.SinonStub; + let validateProjectStub: sinon.SinonStub; + let runGarbageCollectionStub: sinon.SinonStub; + let discoverRoutesStub: sinon.SinonStub; + let compareRouteStub: sinon.SinonStub; + let generateReportStub: sinon.SinonStub; + let getBackendStub: sinon.SinonStub; + let crawlStub: sinon.SinonStub; + let getRoutesStub: sinon.SinonStub; + + beforeEach(() => { + tempDir = path.join(process.cwd(), "scratch-test-suite-" + Math.random().toString(36).substring(7)); + fs.ensureDirSync(tempDir); + dummyZip = path.join(tempDir, "archive.zip"); + fs.writeFileSync(dummyZip, "empty content"); + + upsertBucketStub = sinon.stub(gcs, "upsertBucket").resolves("bucket-123"); + createArchiveStub = sinon.stub(deployUtil, "createSourceDeployArchive").resolves(dummyZip); + uploadObjectStub = sinon.stub(gcs, "uploadObject").resolves({ bucket: "bucket-123", object: "obj-123", generation: "1" }); + orchestrateRolloutStub = sinon.stub(rolloutHelper, "orchestrateRollout").resolves({} as any); + getProjectNumberStub = sinon.stub(projectNumberHelper, "getProjectNumber").resolves("12345"); + setupSecretsStub = sinon.stub(secretsManager, "setupSandboxSecrets").resolves([]); + cleanupSecretsStub = sinon.stub(secretsManager, "cleanupSandboxSecrets").resolves(); + acquireSlotStub = sinon.stub(slotsManager, "acquireComparisonSlot").resolves({ index: 1, backendIdA: "compare-slot-1-a", backendIdB: "compare-slot-1-b" }); + releaseSlotStub = sinon.stub(slotsManager, "releaseComparisonSlot").resolves(); + validateProjectStub = sinon.stub(lifecycle, "validateProject").returns(); + runGarbageCollectionStub = sinon.stub(lifecycle, "runGarbageCollection").resolves(); + discoverRoutesStub = sinon.stub(discoverManager, "discoverRoutes").resolves(["/"]); + compareRouteStub = sinon.stub(compareManager, "compareRoute").resolves({ + route: "/", + statusMatch: true, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 1.0, + bodyDiff: "", + isBinary: false + }); + generateReportStub = sinon.stub(reporterManager, "generateReport").resolves(); + getBackendStub = sinon.stub(apphosting, "getBackend").resolves({ uri: "https://my-backend.com" } as any); + + crawlStub = sinon.stub(Crawler.prototype, "crawl").resolves(); + getRoutesStub = sinon.stub(Crawler.prototype, "getRoutes").returns(["/about"]); + }); + + afterEach(() => { + sinon.restore(); + fs.removeSync(tempDir); + }); + + it("should coordinate the full deployment, crawling, comparison, and reporting pipeline", async () => { + await runCompareSuite( + "aryanf-test", + "us-central1", + "/app/path-a", + "/app/path-b" + ); + + expect(acquireSlotStub.callCount).to.equal(1); + expect(setupSecretsStub.callCount).to.equal(1); + expect(upsertBucketStub.callCount).to.equal(1); + expect(createArchiveStub.callCount).to.equal(2); + expect(uploadObjectStub.callCount).to.equal(2); + expect(orchestrateRolloutStub.callCount).to.equal(2); + expect(discoverRoutesStub.callCount).to.equal(1); + expect(crawlStub.callCount).to.equal(1); + + expect(compareRouteStub.callCount).to.equal(2); + expect(compareRouteStub.firstCall.args[0]).to.equal("/"); + expect(compareRouteStub.secondCall.args[0]).to.equal("/about"); + + expect(generateReportStub.callCount).to.equal(1); + expect(cleanupSecretsStub.callCount).to.equal(1); + expect(releaseSlotStub.callCount).to.equal(1); + }); + + it("should support running with local builds enabled for one of the backends", async () => { + const localBuildStub = sinon.stub(localBuildsModule, "localBuild").resolves({ + outputFiles: ["index.html"], + buildConfig: { runCommand: "npm run start" } + }); + + const createTarStub = sinon.stub(deployUtil, "createLocalBuildTarArchive").resolves(dummyZip); + + await runCompareSuite( + "aryanf-test", + "us-central1", + "/app/path-a", + "/app/path-b", + { localBuildA: false, localBuildB: true } + ); + + // Backend A is source deploy -> calls createSourceDeployArchive (1 time) + // Backend B is local build -> calls localBuild (1 time) and createLocalBuildTarArchive (1 time) + expect(createArchiveStub.callCount).to.equal(1); + expect(localBuildStub.callCount).to.equal(1); + expect(createTarStub.callCount).to.equal(1); + + expect(uploadObjectStub.callCount).to.equal(2); + expect(orchestrateRolloutStub.callCount).to.equal(2); + }); +}); diff --git a/src/apphosting/compare/suite.ts b/src/apphosting/compare/suite.ts new file mode 100644 index 00000000000..e55aba90714 --- /dev/null +++ b/src/apphosting/compare/suite.ts @@ -0,0 +1,261 @@ +import * as path from "path"; +import * as fs from "fs-extra"; +import * as crypto from "crypto"; +import * as os from "os"; +import * as gcs from "../../gcp/storage"; +import * as apphosting from "../../gcp/apphosting"; +import * as rollout from "../rollout"; +import * as deployUtil from "../../deploy/apphosting/util"; +import { getProjectNumber } from "../../getProjectNumber"; +import * as secrets from "./secrets"; +import * as slots from "./slots"; +import * as lifecycle from "./lifecycle"; +import * as discover from "./discover"; +import { Crawler } from "./crawler"; +import * as compare from "./compare"; +import * as reporter from "./reporter"; +import { localBuild } from "../localbuilds"; +import * as fsAsync from "../../fsAsync"; +import * as poller from "../../operation-poller"; +import { logger } from "../../logger"; + +const apphostingPollerOptions = { + apiOrigin: apphosting.apphostingOrigin(), + apiVersion: "v1beta", + backoff: 200, + maxBackoff: 10000, + timeout: 120000, // 2 minutes +}; + +async function prepareLocalBuildDir(rootDir: string, scratchDir: string, backendId: string): Promise { + const ignore = deployUtil.resolveIgnorePatterns({ backendId, rootDir: "/", ignore: [] }); + fs.rmSync(scratchDir, { recursive: true, force: true }); + fs.mkdirSync(scratchDir, { recursive: true }); + const filesToCopy = await fsAsync.readdirRecursive({ + path: rootDir, + ignoreStrings: ignore, + supportGitIgnore: true, + }); + for (const file of filesToCopy) { + const relativePath = path.relative(rootDir, file.name); + const destPath = path.join(scratchDir, relativePath); + fs.mkdirSync(path.dirname(destPath), { recursive: true }); + fs.copyFileSync(file.name, destPath); + } +} + +async function deployToBackend( + projectId: string, + location: string, + backendId: string, + appPath: string, + bucketName: string, + useLocalBuild: boolean, + runtimeVersion?: string +): Promise { + let archivePath: string; + let buildInput: any; + + if (runtimeVersion) { + logger.info(`Patching runtime version for backend ${backendId} to ${runtimeVersion}...`); + const op = await apphosting.updateBackend(projectId, location, backendId, { + runtime: { value: runtimeVersion } + }); + await poller.pollOperation({ + ...apphostingPollerOptions, + pollerName: `update-runtime-${backendId}`, + operationResourceName: op.name, + }); + } + + if (useLocalBuild) { + logger.info(`Running local build for slot backend ${backendId}...`); + const pathHash = crypto.createHash("md5").update(appPath).digest("hex").substring(0, 8); + const scratchDir = path.join(os.tmpdir(), `apphosting-local-build-${backendId}-${pathHash}`); + + await prepareLocalBuildDir(appPath, scratchDir, backendId); + + const { outputFiles, buildConfig } = await localBuild( + projectId, + scratchDir, + {}, + { nonInteractive: true } + ); + + archivePath = await deployUtil.createLocalBuildTarArchive( + { backendId, rootDir: "/", ignore: [] }, + scratchDir, + outputFiles + ); + + logger.info(`Uploading local build bundle for ${backendId}...`); + await gcs.uploadObject( + { file: archivePath, stream: fs.createReadStream(archivePath) }, + bucketName, + gcs.ContentType.TAR + ); + + const uri = `gs://${bucketName}/${path.basename(archivePath)}`; + buildInput = { + config: buildConfig, + source: { + locallyBuilt: { + userStorageUri: uri, + rootDirectory: "/", + runCommand: buildConfig.runCommand, + env: buildConfig.env, + } + } + }; + } else { + logger.info(`Packaging source archive for ${backendId}...`); + archivePath = await deployUtil.createSourceDeployArchive( + { backendId, rootDir: "/", ignore: [] }, + appPath + ); + + logger.info(`Uploading source archive for ${backendId}...`); + await gcs.uploadObject( + { file: archivePath, stream: fs.createReadStream(archivePath) }, + bucketName, + gcs.ContentType.ZIP + ); + + const uri = `gs://${bucketName}/${path.basename(archivePath)}`; + buildInput = { + source: { + archive: { + userStorageUri: uri, + rootDirectory: "/" + } + } + }; + } + + logger.info(`Triggering rollout for backend ${backendId}...`); + await rollout.orchestrateRollout({ + projectId, + location, + backendId, + buildInput + }); +} + +export async function runCompareSuite( + projectId: string, + location: string, + appPathA: string, + appPathB: string, + options: { + outputDir?: string; + localBuildA?: boolean; + localBuildB?: boolean; + runtimeA?: string; + runtimeB?: string; + } = {} +): Promise { + lifecycle.validateProject(projectId); + await lifecycle.runGarbageCollection(projectId, location); + + const projectNumber = await getProjectNumber({ projectId }); + + // 1. Acquire Comparison Slot + const slot = await slots.acquireComparisonSlot(projectId, location); + logger.info(`Acquired Comparison Slot ${slot.index}. Deploying A (localBuild: ${!!options.localBuildA}, runtime: ${options.runtimeA || "default"}): ${slot.backendIdA}, B (localBuild: ${!!options.localBuildB}, runtime: ${options.runtimeB || "default"}): ${slot.backendIdB}...`); + + let secretsMappings: secrets.SecretMapping[] = []; + + const cleanUp = async () => { + logger.warn("\nInterrupted. Restoring slot and deleting mock secrets..."); + await secrets.cleanupSandboxSecrets(projectId, secretsMappings); + await slots.releaseComparisonSlot(projectId, location, slot.index); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + + try { + // 2. Setup mock secrets + secretsMappings = await secrets.setupSandboxSecrets( + projectId, + location, + appPathA, + slot.index, + slot.backendIdA, + slot.backendIdB + ); + + // 3. Package, Upload and Deploy Source A & B + const bucketName = `firebaseapphosting-sources-${projectNumber}-${location.toLowerCase()}`; + await gcs.upsertBucket({ + product: "apphosting", + createMessage: `Ensuring bucket for comparison slot sources in ${location}...`, + projectId, + req: { + baseName: bucketName, + purposeLabel: `apphosting-source-${location.toLowerCase()}`, + location, + lifecycle: { + rule: [ + { + action: { + type: "Delete", + }, + condition: { + age: 30, + }, + }, + ], + }, + } + }); + + await Promise.all([ + deployToBackend(projectId, location, slot.backendIdA, appPathA, bucketName, !!options.localBuildA, options.runtimeA), + deployToBackend(projectId, location, slot.backendIdB, appPathB, bucketName, !!options.localBuildB, options.runtimeB) + ]); + + logger.info("Rollouts completed successfully!"); + + // 4. Route Discovery + const discoveredStaticRoutes = await discover.discoverRoutes(appPathA); + logger.info(`Discovered ${discoveredStaticRoutes.length} static routes from manifests/sitemap.`); + + // 5. Retrieve Live URLs and Run Crawler & Compare + const [bA, bB] = await Promise.all([ + apphosting.getBackend(projectId, location, slot.backendIdA), + apphosting.getBackend(projectId, location, slot.backendIdB) + ]); + + const urlA = bA.uri; + const urlB = bB.uri; + + logger.info(`Backend A URL: ${urlA}`); + logger.info(`Backend B URL: ${urlB}`); + + logger.info("Crawling Backend A for dynamic link discovery..."); + const crawler = new Crawler(urlA); + await crawler.crawl(); + const crawledRoutes = crawler.getRoutes(); + logger.info(`Crawler discovered ${crawledRoutes.length} routes.`); + + const allRoutes = Array.from(new Set([...discoveredStaticRoutes, ...crawledRoutes])).sort(); + + logger.info(`Commencing comparison of ${allRoutes.length} routes...`); + const results: compare.ComparisonResult[] = []; + for (const route of allRoutes) { + logger.info(`Comparing route: ${route}`); + const res = await compare.compareRoute(route, urlA, urlB); + results.push(res); + } + + // 6. Report Generation + await reporter.generateReport(projectId, location, slot.backendIdA, slot.backendIdB, results, options.outputDir); + + } finally { + process.off("SIGINT", cleanUp); + process.off("SIGTERM", cleanUp); + + await secrets.cleanupSandboxSecrets(projectId, secretsMappings); + await slots.releaseComparisonSlot(projectId, location, slot.index); + } +} diff --git a/src/commands/apphosting-compare-suite.ts b/src/commands/apphosting-compare-suite.ts new file mode 100644 index 00000000000..7b7b8c75efc --- /dev/null +++ b/src/commands/apphosting-compare-suite.ts @@ -0,0 +1,67 @@ +import { Command } from "../command"; +import { Options } from "../options"; +import { needProjectId } from "../projectUtils"; +import { requireAuth } from "../requireAuth"; +import { runCompareSuite } from "../apphosting/compare/suite"; +import { FirebaseError } from "../error"; +import * as fs from "fs-extra"; +import * as path from "path"; +import { logger } from "../logger"; + +export const command = new Command("apphosting:compare-suite") + .description("Autonomously run a suite of comparison tests on multiple App Hosting codebases") + .option( + "--location ", + "the primary region of the App Hosting backends to use", + "us-central1" + ) + .option( + "--suite-config ", + "path to comparison suite JSON configuration file" + ) + .option( + "--output-dir ", + "directory to output comparison report files", + "./compare-report" + ) + .before(requireAuth) + .action(async (options: Options) => { + const projectId = needProjectId(options); + const location = options.location as string; + const configPath = options.suiteConfig as string; + + if (!configPath) { + throw new FirebaseError("Must specify --suite-config file containing the list of apps to compare."); + } + + if (!(await fs.pathExists(configPath))) { + throw new FirebaseError(`Suite config file does not exist at ${configPath}`); + } + + const suite = await fs.readJson(configPath); + if (!Array.isArray(suite)) { + throw new FirebaseError("Suite config must be a JSON array of test cases."); + } + + logger.info(`Starting suite of ${suite.length} comparison tests...`); + for (const testCase of suite) { + logger.info(`\nRunning test case: ${testCase.name || "Unnamed Test"}`); + const caseOutputDir = path.join(options.outputDir as string, testCase.name || "unnamed"); + + try { + await runCompareSuite( + projectId, + location, + testCase.pathA, + testCase.pathB, + { + outputDir: caseOutputDir, + localBuildA: !!testCase.localBuildA, + localBuildB: !!testCase.localBuildB + } + ); + } catch (err: any) { + logger.error(`Test case ${testCase.name} failed: ${err.message}`); + } + } + }); diff --git a/src/commands/apphosting-compare.ts b/src/commands/apphosting-compare.ts new file mode 100644 index 00000000000..3c3797ddbe5 --- /dev/null +++ b/src/commands/apphosting-compare.ts @@ -0,0 +1,59 @@ +import { Command } from "../command"; +import { Options } from "../options"; +import { needProjectId } from "../projectUtils"; +import { requireAuth } from "../requireAuth"; +import { runCompareSuite } from "../apphosting/compare/suite"; +import { FirebaseError } from "../error"; + +export const command = new Command("apphosting:compare") + .description("Autonomously deploy and compare two versions/configurations of a Firebase App Hosting codebase") + .option( + "--location ", + "the primary region of the App Hosting backends to use", + "us-central1" + ) + .option( + "--path-a ", + "path to directory containing codebase version A (defaults to current directory)", + "." + ) + .option( + "--path-b ", + "path to directory containing codebase version B (defaults to path-a or current directory)" + ) + .option( + "--local-build-a", + "compile and deploy version A using a local build" + ) + .option( + "--local-build-b", + "compile and deploy version B using a local build" + ) + .option( + "--runtime-a ", + "specify the ABIU runtime version for backend A (e.g. nodejs22)" + ) + .option( + "--runtime-b ", + "specify the ABIU runtime version for backend B (e.g. nodejs22)" + ) + .option( + "--output-dir ", + "directory to output comparison report files", + "./compare-report" + ) + .before(requireAuth) + .action(async (options: Options) => { + const projectId = needProjectId(options); + const location = options.location as string; + const pathA = (options.pathA as string) || "."; + const pathB = (options.pathB as string) || pathA; + + await runCompareSuite(projectId, location, pathA, pathB, { + outputDir: options.outputDir as string, + localBuildA: !!options.localBuildA, + localBuildB: !!options.localBuildB, + runtimeA: options.runtimeA as string, + runtimeB: options.runtimeB as string + }); + }); diff --git a/src/commands/index.ts b/src/commands/index.ts index ea736e9144d..c2d2091e296 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -210,6 +210,8 @@ export function load(client: CLIClient): CLIClient { client.apphosting.secrets.access = loadCommand("apphosting-secrets-access"); client.apphosting.rollouts = {}; client.apphosting.rollouts.create = loadCommand("apphosting-rollouts-create"); + client.apphosting.compare = loadCommand("apphosting-compare"); + client.apphosting.compareSuite = loadCommand("apphosting-compare-suite"); client.apphosting.config = {}; if (experiments.isEnabled("internaltesting")) { client.apphosting.builds = {}; diff --git a/src/gcp/apphosting.ts b/src/gcp/apphosting.ts index b9f3c98d69d..84f767ef797 100644 --- a/src/gcp/apphosting.ts +++ b/src/gcp/apphosting.ts @@ -811,3 +811,28 @@ export async function getNextRolloutId( const highest = Math.max(highestId(builds.builds), highestId(rollouts.rollouts)); return `build-${year}-${month}-${day}-${String(highest + 1).padStart(3, "0")}`; } + +/** + * Update a backend configuration (e.g. labels). + */ +export async function updateBackend( + projectId: string, + location: string, + backendId: string, + backend: DeepOmit, BackendOutputOnlyFields | "name">, +): Promise { + const fieldMasks = proto.fieldMasks(backend); + const queryParams = { + updateMask: fieldMasks.join(","), + }; + const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; + const res = await client.patch( + name, + { ...backend, name }, + { + queryParams, + }, + ); + return res.body; +} + From 2d9738545b47b2dc50f6fbc383eb7308c3ea47ee Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Tue, 16 Jun 2026 22:58:48 -0400 Subject: [PATCH 02/12] feat(apphosting): implement N-Way Matrix comparison suite - Refactored 2-way comparison engine to an N-Way Matrix comparison tool - Support dynamically acquiring arbitrary N backends per comparison slot - Implemented graceful SIGINT and unhandled exception teardown - Handled absolute URL proxy translation for App Hosting fetch crawling - Grouped Secret Manager setup per distinct codebase to prevent AlreadyExists races - Enhanced diff engine with Prettier formatting to properly compare minified HTML - Extracted and dumped raw HTML responses out to disk for explicit manual inspection --- src/apphosting/compare/README.md | 127 ++++++------ src/apphosting/compare/compare.spec.ts | 30 +-- src/apphosting/compare/compare.ts | 44 +++-- src/apphosting/compare/crawler.spec.ts | 34 ++-- src/apphosting/compare/crawler.ts | 11 +- src/apphosting/compare/discover.spec.ts | 14 +- src/apphosting/compare/discover.ts | 2 +- src/apphosting/compare/lifecycle.spec.ts | 10 +- src/apphosting/compare/lifecycle.ts | 22 ++- src/apphosting/compare/reporter.spec.ts | 13 +- src/apphosting/compare/reporter.ts | 82 +++++--- src/apphosting/compare/secrets.spec.ts | 22 +-- src/apphosting/compare/secrets.ts | 49 +++-- src/apphosting/compare/slots.spec.ts | 34 ++-- src/apphosting/compare/slots.ts | 136 ++++++++----- src/apphosting/compare/suite.spec.ts | 34 +++- src/apphosting/compare/suite.ts | 236 +++++++++++++++-------- src/commands/apphosting-compare-suite.ts | 37 ++-- src/commands/apphosting-compare.ts | 52 ++--- src/commands/index.ts | 2 +- 20 files changed, 592 insertions(+), 399 deletions(-) diff --git a/src/apphosting/compare/README.md b/src/apphosting/compare/README.md index 56c0d62ef97..9712abe1339 100644 --- a/src/apphosting/compare/README.md +++ b/src/apphosting/compare/README.md @@ -1,90 +1,71 @@ -# App Hosting Comparison Tool +# App Hosting N-Way Matrix Comparison Tool -The App Hosting Comparison Tool (`firebase apphosting:compare` and `firebase apphosting:compare-suite`) is an autonomous differential testing tool. It allows developers and CI/CD systems to deploy, crawl, and compare two different versions or configurations of an application on Firebase App Hosting, asserting parity across status codes, response headers, and body payloads (text diffs and binary hashes). +This is an experimental internal CLI tool built for the Firebase App Hosting team to dynamically verify the compatibility and performance of different infrastructure backends or application builds. ---- +It takes an array of $N$ configurations (variants) and automatically deploys them simultaneously, discovering their routes, and dumping an $O(N^2)$ Pair-wise Cartesian Matrix of differences. -## Commands +## Capabilities -### `firebase apphosting:compare` +1. **N-Way Concurrency**: By leveraging `Promise.all` and dynamic `backendIds`, the tool spins up all $N$ test variants in parallel on Google Cloud. +2. **Local vs Remote Build Verification**: Can deploy locally built bundles (e.g. `locally_built: true`) side-by-side with remote Cloud Build source zips. +3. **Automated IAM & Secrets Management**: Intelligently creates a single mock secret in Secret Manager for each distinct codebase path, mapping the IAM `secretAccessor` roles simultaneously to all backends generated from that codebase. +4. **Dynamic Spidering**: Automatically crawls Next.js / Angular apps recursively starting from `/` to discover hidden dynamic routes, alongside statically parsing `.next/prerender-manifest.json`. +5. **Exact Diff Inspection**: Generates HTML dashboards, JSON summaries, and specifically dumps the raw HTTP HTML outputs of each variant so engineers can run local diffs. -Deploys and compares two versions of an application. +## Usage -```bash -firebase apphosting:compare \ - --path-b \ - [--path-a ] \ - [--location ] \ - [--output-dir ] -``` - -**Options:** -* `--path-b` (Required): The directory path containing the version to compare against (e.g. your canary/experimental branch). -* `--path-a` (Optional): The directory path containing the baseline version (e.g. your stable branch). Defaults to the current working directory (`.`). -* `--location` (Optional): The GCP location where App Hosting backends should reside. Defaults to `us-central1`. -* `--output-dir` (Optional): The directory path where comparison results and the dashboard will be written. Defaults to `./compare-report`. +1. Create a `matrix-test.json` file to define your test cases: ---- - -### `firebase apphosting:compare-suite` +```json +{ + "testCases": [ + { + "name": "Node Matrix Test", + "variants": [ + { + "id": "Local-Node24", + "path": "../next-sample-1", + "localBuild": true, + "runtime": "nodejs24" + }, + { + "id": "Source-Node24", + "path": "../next-sample-1", + "localBuild": false, + "runtime": "nodejs24" + }, + { + "id": "Source-Node22", + "path": "../next-sample-1", + "localBuild": false, + "runtime": "nodejs22" + } + ] + } + ] +} +``` -Runs a batch of comparison tests on multiple codebases defined in a JSON suite config. +2. Run the command: ```bash -firebase apphosting:compare-suite \ - --suite-config \ - [--location ] \ - [--output-dir ] +FIREBASE_CLI_EXPERIMENTS=apphosting firebase apphosting:compare-suite --project --suite-config matrix-test.json ``` -**Suite Config Format (`suite.json`):** -```json -[ - { - "name": "nextjs-reference", - "pathA": "./apps/nextjs-reference/stable", - "pathB": "./apps/nextjs-reference/canary" - }, - { - "name": "nextjs-sample", - "pathA": "./next-sample-1", - "pathB": "./next-sample-1-modified" - } -] -``` - ---- +## How to Inspect Diffs -## Core Architecture +When the command completes, it generates reports in the `./compare-report//` directory. +Inside this folder, you will see a subfolder for each pairwise comparison, such as `Local-Node24-vs-Source-Node24/`. -### 1. Quota & Slot Pool Management -To work around the standard project limit of **10 backends per project**, the tool manages a leased pool of **5 parallel comparison slots** (`compare-slot-1` to `compare-slot-5`). -* Each slot contains an A/B pair of backends: `compare-slot-X-a` and `compare-slot-X-b`. -* Slot backends are dynamically created upon first run and **kept alive (reused)** for subsequent runs, reducing rollout provisioning time by ~2 minutes. -* State is managed via GCP labels (`status: busy`, `status: idle`, `last_active: `). -* If a run is interrupted (`SIGINT`/`SIGTERM`), a signal handler cleans up resources and releases the slot lock. -* A startup GC sweeper automatically releases slot locks held `busy` for more than 2 hours. +Inside the pair folder: +- **`index.html`**: A beautifully styled visual dashboard showing percentage differences and mismatches. +- **`summary.json`**: The structured data representation. +- **`backendA/` and `backendB/`**: These folders contain the raw HTTP HTML bodies retrieved during the crawl! -### 2. Secret manager Isolation -If the codebases reference Google Cloud Secret Manager values in their `apphosting.yaml` configurations, the tool: -* Automatically provisions mock sandboxed secrets (prefixed with `cmp-sec-[slot]-`) to prevent naming conflicts. -* Grants read access to the slot backend's Cloud Run service account. -* Safely removes the mock secrets from Secret Manager when the comparison concludes. +To manually inspect the exact diffs, you can use standard diff tools on the generated files: -### 3. Route Discovery -The comparison engine discovers routes using a two-pass mechanism: -* **Static Discovery**: Parses local Next.js build manifests (e.g., `.next/routes-manifest.json`, `.next/prerender-manifest.json`) and local `sitemap.xml` files. -* **Dynamic Crawling**: Executes a recursive HTML link crawler on Backend A to dynamically harvest links, query strings, and redirects (up to 5 redirect levels) at runtime. - -### 4. Differential Analyzer -Once routes are collected, the analyzer queries the matching paths on both backends and performs: -* **Status Match**: Asserts status code parity (e.g. `200` vs `200`, `404` vs `404`). -* **Header Auditing**: Separates expected dynamic headers (`x-cloud-trace-context`, `date`, `etag`, `age`) from static behavioral headers (`content-type`, `cache-control`) to detect regression. -* **Payload Comparison**: - * **Text/HTML**: Computes Myers' line-level diffs and Sorensen-Dice similarity scores to match dynamic content. - * **Binary Assets**: Compares exact size (bytes) and SHA-256 payload hashes. +```bash +diff compare-report/Node\ Matrix\ Test/Local-Node24-vs-Source-Node24/backendA/index.html compare-report/Node\ Matrix\ Test/Local-Node24-vs-Source-Node24/backendB/index.html +``` -### 5. Premium Dashboard -The comparison run outputs: -* **`summary.json`**: Machine-readable JSON structured logs of every mismatch, status code, and similarity score. -* **`index.html`**: A responsive, modern dark-mode HTML split-pane code diff viewer, letting you visually compare the precise lines that changed on each page. +Or you can right-click the files in VSCode and select "Select for Compare" and "Compare with Selected". diff --git a/src/apphosting/compare/compare.spec.ts b/src/apphosting/compare/compare.spec.ts index c0df78d9a23..aed79b61e76 100644 --- a/src/apphosting/compare/compare.spec.ts +++ b/src/apphosting/compare/compare.spec.ts @@ -14,11 +14,11 @@ describe("compareRoute", () => { it("should match identical text pages", async () => { nock("https://backend-a.com").get("/").reply(200, "hello world", { "Content-Type": "text/html", - "Cache-Control": "public, max-age=3600" + "Cache-Control": "public, max-age=3600", }); nock("https://backend-b.com").get("/").reply(200, "hello world", { "Content-Type": "text/html", - "Cache-Control": "public, max-age=3600" + "Cache-Control": "public, max-age=3600", }); const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); @@ -39,17 +39,17 @@ describe("compareRoute", () => { it("should flag behavioral header mismatches but record dynamic ones as expected variations", async () => { nock("https://backend-a.com").get("/").reply(200, "ok", { "Cache-Control": "public, max-age=3600", - "Date": "Wed, 17 Jun 2026 01:00:00 GMT" + Date: "Wed, 17 Jun 2026 01:00:00 GMT", }); nock("https://backend-b.com").get("/").reply(200, "ok", { "Cache-Control": "no-cache", - "Date": "Wed, 17 Jun 2026 01:05:00 GMT" + Date: "Wed, 17 Jun 2026 01:05:00 GMT", }); const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); expect(res.headerMismatches).to.have.lengthOf(1); expect(res.headerMismatches[0].header.toLowerCase()).to.equal("cache-control"); - + expect(res.expectedHeaderVariations).to.have.lengthOf(1); expect(res.expectedHeaderVariations[0].header.toLowerCase()).to.equal("date"); }); @@ -60,24 +60,32 @@ describe("compareRoute", () => { const binC = Buffer.from([1, 2, 3, 5]); nock("https://backend-a.com").get("/img.png").reply(200, binA, { - "Content-Type": "image/png" + "Content-Type": "image/png", }); nock("https://backend-b.com").get("/img.png").reply(200, binB, { - "Content-Type": "image/png" + "Content-Type": "image/png", }); - const resMatch = await compareRoute("/img.png", "https://backend-a.com", "https://backend-b.com"); + const resMatch = await compareRoute( + "/img.png", + "https://backend-a.com", + "https://backend-b.com", + ); expect(resMatch.isBinary).to.be.true; expect(resMatch.bodySimilarity).to.equal(1.0); nock("https://backend-a.com").get("/img.png").reply(200, binA, { - "Content-Type": "image/png" + "Content-Type": "image/png", }); nock("https://backend-b.com").get("/img.png").reply(200, binC, { - "Content-Type": "image/png" + "Content-Type": "image/png", }); - const resMismatch = await compareRoute("/img.png", "https://backend-a.com", "https://backend-b.com"); + const resMismatch = await compareRoute( + "/img.png", + "https://backend-a.com", + "https://backend-b.com", + ); expect(resMismatch.isBinary).to.be.true; expect(resMismatch.bodySimilarity).to.equal(0.0); }); diff --git a/src/apphosting/compare/compare.ts b/src/apphosting/compare/compare.ts index 389104dc933..9076c4e522f 100644 --- a/src/apphosting/compare/compare.ts +++ b/src/apphosting/compare/compare.ts @@ -10,6 +10,8 @@ export interface ComparisonResult { bodySimilarity: number; // 0.0 to 1.0 bodyDiff: string; isBinary: boolean; + bodyA?: string; + bodyB?: string; } const BEHAVIORAL_HEADERS = [ @@ -18,35 +20,38 @@ const BEHAVIORAL_HEADERS = [ "content-type", "content-encoding", "location", - "strict-transport-security" + "strict-transport-security", ]; const BINARY_CONTENT_TYPES = [ "image/", "application/pdf", "application/zip", - "application/octet-stream" + "application/octet-stream", ]; function isBinaryContentType(contentType: string): boolean { const normalized = contentType.toLowerCase(); - return BINARY_CONTENT_TYPES.some(type => normalized.includes(type)); + return BINARY_CONTENT_TYPES.some((type) => normalized.includes(type)); } +/** + * + */ export async function compareRoute( route: string, urlA: string, urlB: string, - options: { headers?: Record } = {} + options: { headers?: Record } = {}, ): Promise { const fetchOptions = { headers: options.headers || {}, - redirect: "manual" as const + redirect: "manual" as const, }; const [resA, resB] = await Promise.all([ fetch(`${urlA}${route}`, fetchOptions), - fetch(`${urlB}${route}`, fetchOptions) + fetch(`${urlB}${route}`, fetchOptions), ]); const result: ComparisonResult = { @@ -56,13 +61,13 @@ export async function compareRoute( expectedHeaderVariations: [], bodySimilarity: 1.0, bodyDiff: "", - isBinary: false + isBinary: false, }; // 1. Compare Headers const allHeaderKeys = new Set([ ...Array.from(resA.headers.keys()), - ...Array.from(resB.headers.keys()) + ...Array.from(resB.headers.keys()), ]); for (const key of allHeaderKeys) { @@ -82,13 +87,13 @@ export async function compareRoute( const contentTypeB = resB.headers.get("content-type") || ""; if (isBinaryContentType(contentTypeA) || isBinaryContentType(contentTypeB)) { result.isBinary = true; - + const bufA = await resA.buffer(); const bufB = await resB.buffer(); - + const sizeA = bufA.length; const sizeB = bufB.length; - + if (sizeA !== sizeB) { result.bodySimilarity = 0.0; result.bodyDiff = `Binary size mismatch: ${sizeA} bytes vs ${sizeB} bytes`; @@ -106,8 +111,21 @@ export async function compareRoute( } // 3. Compare Text Body - const bodyA = await resA.text(); - const bodyB = await resB.text(); + let bodyA = await resA.text(); + let bodyB = await resB.text(); + + if (contentTypeA.includes("text/html")) { + try { + const prettier = require("prettier"); + bodyA = await prettier.format(bodyA, { parser: "html" }); + bodyB = await prettier.format(bodyB, { parser: "html" }); + } catch (e: any) { + // Fallback to unformatted if prettier fails + } + } + + result.bodyA = bodyA; + result.bodyB = bodyB; if (bodyA !== bodyB) { result.bodySimilarity = MyersDiffEngine.getSimilarity(bodyA, bodyB); diff --git a/src/apphosting/compare/crawler.spec.ts b/src/apphosting/compare/crawler.spec.ts index 392ff9ffe38..0713358b86d 100644 --- a/src/apphosting/compare/crawler.spec.ts +++ b/src/apphosting/compare/crawler.spec.ts @@ -13,10 +13,12 @@ describe("Crawler", () => { it("should recursively crawl text/html links", async () => { const base = "https://example.com"; - + nock(base) .get("/") - .reply(200, ` + .reply( + 200, + ` About Us @@ -24,25 +26,27 @@ describe("Crawler", () => { External Link - `, { "Content-Type": "text/html" }); + `, + { "Content-Type": "text/html" }, + ); nock(base) .get("/about") - .reply(200, ` + .reply( + 200, + ` Careers - `, { "Content-Type": "text/html" }); + `, + { "Content-Type": "text/html" }, + ); - nock(base) - .get("/contact") - .reply(200, "Contact Details", { "Content-Type": "text/html" }); + nock(base).get("/contact").reply(200, "Contact Details", { "Content-Type": "text/html" }); - nock(base) - .get("/careers") - .reply(200, "Join our team", { "Content-Type": "text/html" }); + nock(base).get("/careers").reply(200, "Join our team", { "Content-Type": "text/html" }); const crawler = new Crawler(base); await crawler.crawl(); @@ -58,13 +62,9 @@ describe("Crawler", () => { .get("/") .reply(200, `Redirect me`, { "Content-Type": "text/html" }); - nock(base) - .get("/old-link") - .reply(302, undefined, { "Location": "/new-link" }); + nock(base).get("/old-link").reply(302, undefined, { Location: "/new-link" }); - nock(base) - .get("/new-link") - .reply(200, "Done!", { "Content-Type": "text/html" }); + nock(base).get("/new-link").reply(200, "Done!", { "Content-Type": "text/html" }); const crawler = new Crawler(base); await crawler.crawl(); diff --git a/src/apphosting/compare/crawler.ts b/src/apphosting/compare/crawler.ts index ba401aac135..96a9c7ee13b 100644 --- a/src/apphosting/compare/crawler.ts +++ b/src/apphosting/compare/crawler.ts @@ -4,7 +4,10 @@ export class Crawler { private visited = new Set(); private discoveredRoutes = new Set(); - constructor(private readonly baseUrl: string, private readonly maxDepth = 3) {} + constructor( + private readonly baseUrl: string, + private readonly maxDepth = 3, + ) {} public getRoutes(): string[] { return Array.from(this.discoveredRoutes).sort(); @@ -30,7 +33,7 @@ export class Crawler { const url = `${this.baseUrl}${canonical}`; const res = await fetch(url, { redirect: "manual" as const, - headers: { "User-Agent": "FirebaseCompareCrawler/1.0" } + headers: { "User-Agent": "FirebaseCompareCrawler/1.0" }, }); // Handle Redirects @@ -74,7 +77,9 @@ export class Crawler { pathname = pathname.slice(0, -1); } - const params = Array.from(url.searchParams.entries()).sort((a, b) => a[0].localeCompare(b[0])); + const params = Array.from(url.searchParams.entries()).sort((a, b) => + a[0].localeCompare(b[0]), + ); const search = params.length > 0 ? "?" + params.map(([k, v]) => `${k}=${v}`).join("&") : ""; return `${pathname}${search}`; diff --git a/src/apphosting/compare/discover.spec.ts b/src/apphosting/compare/discover.spec.ts index 2ef9182edbd..a757c31db96 100644 --- a/src/apphosting/compare/discover.spec.ts +++ b/src/apphosting/compare/discover.spec.ts @@ -7,7 +7,10 @@ describe("discoverRoutes", () => { let tempDir: string; beforeEach(() => { - tempDir = path.join(process.cwd(), "scratch-test-discover-" + Math.random().toString(36).substring(7)); + tempDir = path.join( + process.cwd(), + "scratch-test-discover-" + Math.random().toString(36).substring(7), + ); fs.ensureDirSync(tempDir); }); @@ -27,15 +30,12 @@ describe("discoverRoutes", () => { fs.writeJsonSync(path.join(nextDir, "prerender-manifest.json"), { routes: { "/about": {}, - "/blog/post-1": {} - } + "/blog/post-1": {}, + }, }); fs.writeJsonSync(path.join(nextDir, "routes-manifest.json"), { - staticRoutes: [ - { page: "/" }, - { page: "/contact" } - ] + staticRoutes: [{ page: "/" }, { page: "/contact" }], }); const routes = await discoverRoutes(tempDir); diff --git a/src/apphosting/compare/discover.ts b/src/apphosting/compare/discover.ts index ca62fe01495..a746db30373 100644 --- a/src/apphosting/compare/discover.ts +++ b/src/apphosting/compare/discover.ts @@ -41,7 +41,7 @@ export async function discoverRoutes(appPath: string): Promise { const sitemapPaths = [ path.join(appPath, "public", "sitemap.xml"), path.join(appPath, "sitemap.xml"), - path.join(appPath, "dist", "sitemap.xml") + path.join(appPath, "dist", "sitemap.xml"), ]; for (const sitemapPath of sitemapPaths) { diff --git a/src/apphosting/compare/lifecycle.spec.ts b/src/apphosting/compare/lifecycle.spec.ts index 9ce009cadb3..2107e9ea265 100644 --- a/src/apphosting/compare/lifecycle.spec.ts +++ b/src/apphosting/compare/lifecycle.spec.ts @@ -24,7 +24,7 @@ describe("Lifecycle Manager", () => { it("should throw on non-whitelisted projects", () => { expect(() => validateProject("my-secret-project")).to.throw( - /Invalid project ID "my-secret-project"/ + /Invalid project ID "my-secret-project"/, ); }); }); @@ -40,14 +40,14 @@ describe("Lifecycle Manager", () => { { name: "projects/test/locations/us-central1/backends/compare-slot-1-a", labels: { status: "busy", type: "comparison-sandbox" }, - updateTime: threeHoursAgo + updateTime: threeHoursAgo, }, { name: "projects/test/locations/us-central1/backends/compare-slot-1-b", labels: { status: "busy", type: "comparison-sandbox" }, - updateTime: oneHourAgo - } - ] + updateTime: oneHourAgo, + }, + ], }); updateBackendStub.resolves({ name: "op-name" }); diff --git a/src/apphosting/compare/lifecycle.ts b/src/apphosting/compare/lifecycle.ts index 66aa0e0f3cd..36a4ab34a1e 100644 --- a/src/apphosting/compare/lifecycle.ts +++ b/src/apphosting/compare/lifecycle.ts @@ -5,10 +5,13 @@ import { acquireComparisonSlot, releaseComparisonSlot } from "./slots"; const ALLOWED_PROJECTS = ["aryanf-test", "pretend-public"]; +/** + * + */ export function validateProject(projectId: string): void { if (!ALLOWED_PROJECTS.includes(projectId)) { throw new FirebaseError( - `Invalid project ID "${projectId}". This tool can only run on projects: ${ALLOWED_PROJECTS.join(", ")}` + `Invalid project ID "${projectId}". This tool can only run on projects: ${ALLOWED_PROJECTS.join(", ")}`, ); } } @@ -34,7 +37,7 @@ export async function runGarbageCollection(projectId: string, location: string): logger.info(`Found stale lock on comparison slot backend ${backendId}. Unlocking...`); try { await apphosting.updateBackend(projectId, location, backendId, { - labels: { ...backend.labels, status: "idle" } + labels: { ...backend.labels, status: "idle" }, }); } catch (err) { logger.debug(`Failed to unlock stale backend ${backendId}: ${err}`); @@ -45,20 +48,23 @@ export async function runGarbageCollection(projectId: string, location: string): } } +/** + * + */ export async function runAutonomousComparison( projectId: string, location: string, localPath: string, - options: any + options: any, ): Promise { validateProject(projectId); await runGarbageCollection(projectId, location); - const slot = await acquireComparisonSlot(projectId, location); + const slot = await acquireComparisonSlot(projectId, location, 2); const cleanUpAndExit = async () => { logger.warn("\nProcess interrupted. Cleaning up comparison slot lock before exit..."); - await releaseComparisonSlot(projectId, location, slot.index); + await releaseComparisonSlot(projectId, location, slot.index, 2); process.exit(1); }; @@ -67,11 +73,13 @@ export async function runAutonomousComparison( try { // Setup secrets, deploy, and compare... - logger.info(`Using Comparison Slot ${slot.index} (Backend A: ${slot.backendIdA}, Backend B: ${slot.backendIdB})`); + logger.info( + `Using Comparison Slot ${slot.index} (Backend A: ${slot.backendIds[0]}, Backend B: ${slot.backendIds[1]})`, + ); } finally { process.off("SIGINT", cleanUpAndExit); process.off("SIGTERM", cleanUpAndExit); - await releaseComparisonSlot(projectId, location, slot.index); + await releaseComparisonSlot(projectId, location, slot.index, 2); } } diff --git a/src/apphosting/compare/reporter.spec.ts b/src/apphosting/compare/reporter.spec.ts index 5d828bb05d5..225030c7f60 100644 --- a/src/apphosting/compare/reporter.spec.ts +++ b/src/apphosting/compare/reporter.spec.ts @@ -8,7 +8,10 @@ describe("Report Generator", () => { let tempDir: string; beforeEach(() => { - tempDir = path.join(process.cwd(), "scratch-test-report-" + Math.random().toString(36).substring(7)); + tempDir = path.join( + process.cwd(), + "scratch-test-report-" + Math.random().toString(36).substring(7), + ); fs.ensureDirSync(tempDir); }); @@ -25,7 +28,7 @@ describe("Report Generator", () => { expectedHeaderVariations: [], bodySimilarity: 1.0, bodyDiff: "", - isBinary: false + isBinary: false, }, { route: "/about", @@ -34,8 +37,8 @@ describe("Report Generator", () => { expectedHeaderVariations: [], bodySimilarity: 0.95, bodyDiff: "HTML content mismatch", - isBinary: false - } + isBinary: false, + }, ]; await generateReport( @@ -44,7 +47,7 @@ describe("Report Generator", () => { "compare-slot-1-a", "compare-slot-1-b", results, - tempDir + tempDir, ); const jsonPath = path.join(tempDir, "summary.json"); diff --git a/src/apphosting/compare/reporter.ts b/src/apphosting/compare/reporter.ts index 59a23e7ca8e..c1342c65365 100644 --- a/src/apphosting/compare/reporter.ts +++ b/src/apphosting/compare/reporter.ts @@ -16,20 +16,23 @@ export interface ComparisonSummary { overallSimilarity: number; } +/** + * + */ export async function generateReport( projectId: string, location: string, backendA: string, backendB: string, results: ComparisonResult[], - outputDir = "./compare-report" + outputDir = "./compare-report", ): Promise { const totalRoutes = results.length; const matching = results.filter( - r => r.statusMatch && r.headerMismatches.length === 0 && r.bodySimilarity >= 0.99 + (r) => r.statusMatch && r.headerMismatches.length === 0 && r.bodySimilarity >= 0.99, ); const mismatches = results.filter( - r => !r.statusMatch || r.headerMismatches.length > 0 || r.bodySimilarity < 0.99 + (r) => !r.statusMatch || r.headerMismatches.length > 0 || r.bodySimilarity < 0.99, ); const totalSimilarity = results.reduce((sum, r) => sum + r.bodySimilarity, 0); @@ -44,7 +47,7 @@ export async function generateReport( totalRoutes, matchingRoutes: matching.length, mismatchingRoutes: mismatches.length, - overallSimilarity + overallSimilarity, }; logger.info("\n=========================================="); @@ -56,7 +59,9 @@ export async function generateReport( logger.info(`Backend B: ${backendB}`); logger.info(`Total Routes: ${totalRoutes}`); logger.info(`Passed: ${clc.green(String(matching.length))}`); - logger.info(`Mismatched: ${mismatches.length > 0 ? clc.red(String(mismatches.length)) : clc.green("0")}`); + logger.info( + `Mismatched: ${mismatches.length > 0 ? clc.red(String(mismatches.length)) : clc.green("0")}`, + ); logger.info(`Similarity: ${clc.cyan((overallSimilarity * 100).toFixed(2) + "%")}`); logger.info("==========================================\n"); @@ -78,41 +83,64 @@ export async function generateReport( } await fs.ensureDir(outputDir); - await fs.writeJson(path.join(outputDir, "summary.json"), { summary, results }, { spaces: 2 }); + + // Dump summary without the bodies to save space + const resultsWithoutBodies = results.map(r => { + const { bodyA, bodyB, ...rest } = r; + return rest; + }); + await fs.writeJson(path.join(outputDir, "summary.json"), { summary, results: resultsWithoutBodies }, { spaces: 2 }); logger.info(`JSON report saved to: ${path.join(outputDir, "summary.json")}`); - const html = getHtmlTemplate(summary, results); + // Dump raw bodies for manual diffing + const routesDirA = path.join(outputDir, "backendA"); + const routesDirB = path.join(outputDir, "backendB"); + for (const r of results) { + if (r.bodyA !== undefined && r.bodyB !== undefined) { + // Map route /foo/bar to /foo/bar.html or /foo/bar/index.html + const safeRoute = r.route === "/" ? "index.html" : r.route.replace(/^\//, "") + ".html"; + await fs.outputFile(path.join(routesDirA, safeRoute), r.bodyA, "utf-8"); + await fs.outputFile(path.join(routesDirB, safeRoute), r.bodyB, "utf-8"); + } + } + logger.info(`Raw responses saved to ${routesDirA} and ${routesDirB} for manual diffing.`); + + const html = getHtmlTemplate(summary, resultsWithoutBodies as any); await fs.outputFile(path.join(outputDir, "index.html"), html, "utf-8"); - logger.info(`HTML Dashboard generated at: ${clc.underline(path.join(outputDir, "index.html"))}\n`); + logger.info( + `HTML Dashboard generated at: ${clc.underline(path.join(outputDir, "index.html"))}\n`, + ); } function getHtmlTemplate(summary: ComparisonSummary, results: ComparisonResult[]): string { - const resultsRows = results.map(r => { - const isPass = r.statusMatch && r.headerMismatches.length === 0 && r.bodySimilarity >= 0.99; - const badgeClass = isPass ? "badge-success" : "badge-error"; - const badgeText = isPass ? "PASS" : "FAIL"; - - const headersList = r.headerMismatches.map(m => - `
  • ${m.header}: "${m.valA}" vs "${m.valB}"
    • ` - ).join(""); - - const variationsList = r.expectedHeaderVariations.map(m => - `
  • ${m.header}: "${m.valA}" vs "${m.valB}"
    • ` - ).join(""); - - return ` - + const resultsRows = results + .map((r) => { + const isPass = r.statusMatch && r.headerMismatches.length === 0 && r.bodySimilarity >= 0.99; + const badgeClass = isPass ? "badge-success" : "badge-error"; + const badgeText = isPass ? "PASS" : "FAIL"; + + const headersList = r.headerMismatches + .map((m) => `
  • ${m.header}: "${m.valA}" vs "${m.valB}"
    • `) + .join(""); + + const variationsList = r.expectedHeaderVariations + .map((m) => `
  • ${m.header}: "${m.valA}" vs "${m.valB}"
    • `) + .join(""); + + return ` + ${r.route} ${badgeText} - ${r.statusMatch ? 'Match' : 'Mismatch'} + ${r.statusMatch ? "Match" : "Mismatch"} ${(r.bodySimilarity * 100).toFixed(1)}% - ${r.headerMismatches.length > 0 ? `
    Mismatches:
      ${headersList}
    ` : 'Match'} - ${r.expectedHeaderVariations.length > 0 ? `
    Variations:
      ${variationsList}
    ` : ''} + ${r.headerMismatches.length > 0 ? `
    Mismatches:
      ${headersList}
    ` : "Match"} + ${r.expectedHeaderVariations.length > 0 ? `
    Variations:
      ${variationsList}
    ` : ""} `; - }).join(""); + }) + .join(""); return ` diff --git a/src/apphosting/compare/secrets.spec.ts b/src/apphosting/compare/secrets.spec.ts index 0cba896dde9..8843eeed194 100644 --- a/src/apphosting/compare/secrets.spec.ts +++ b/src/apphosting/compare/secrets.spec.ts @@ -39,33 +39,29 @@ describe("Sandbox Secrets Manager", () => { it("should setup sandbox secrets for yaml configuration", async () => { pathExistsStub.resolves(true); - + const mockYaml = new AppHostingYamlConfig(); mockYaml.env = { - API_KEY: { secret: "my-production-api-key", availability: ["RUNTIME"] } + API_KEY: { secret: "my-production-api-key", availability: ["RUNTIME"] }, }; loadConfigStub.resolves(mockYaml); getProjectNumberStub.resolves("12345"); - + getBackendStub.resolves({ name: "backend-resource" }); serviceAccountsStub.resolves({ buildServiceAccount: "build-sa@google.com", - runServiceAccount: "run-sa@google.com" + runServiceAccount: "run-sa@google.com", }); - + secretExistsStub.resolves(false); createSecretStub.resolves(); addVersionStub.resolves(); grantSecretAccessStub.resolves(); - const mappings = await setupSandboxSecrets( - "aryanf-test", - "us-central1", - "/app/path", - 1, + const mappings = await setupSandboxSecrets("aryanf-test", "us-central1", "/app/path", 1, [ "compare-slot-1-a", - "compare-slot-1-b" - ); + "compare-slot-1-b", + ]); expect(mappings).to.have.lengthOf(1); expect(mappings[0].originalName).to.equal("my-production-api-key"); @@ -81,7 +77,7 @@ describe("Sandbox Secrets Manager", () => { deleteSecretStub.resolves(); const mappings = [ - { originalName: "my-key", mockSecretName: "cmp-sec-1-my-key", mockValue: "val" } + { originalName: "my-key", mockSecretName: "cmp-sec-1-my-key", mockValue: "val" }, ]; await cleanupSandboxSecrets("aryanf-test", mappings); diff --git a/src/apphosting/compare/secrets.ts b/src/apphosting/compare/secrets.ts index 9b1b47e90bd..e1123abe308 100644 --- a/src/apphosting/compare/secrets.ts +++ b/src/apphosting/compare/secrets.ts @@ -13,13 +13,15 @@ export interface SecretMapping { mockValue: string; } +/** + * + */ export async function setupSandboxSecrets( projectId: string, location: string, appPath: string, slotIndex: number, - backendIdA: string, - backendIdB: string + backendIds: string[], ): Promise { const yamlPath = path.join(appPath, "apphosting.yaml"); if (!(await fs.pathExists(yamlPath))) { @@ -35,24 +37,21 @@ export async function setupSandboxSecrets( const projectNumber = await getProjectNumber({ projectId }); const mappings: SecretMapping[] = []; - // Fetch backends to extract their service accounts - const [backendA, backendB] = await Promise.all([ - apphosting.getBackend(projectId, location, backendIdA), - apphosting.getBackend(projectId, location, backendIdB) - ]); - - const [accountsA, accountsB] = await Promise.all([ - serviceAccountsForBackend(projectNumber, backendA), - serviceAccountsForBackend(projectNumber, backendB) - ]); + // Fetch all backends to extract their service accounts + const backends = await Promise.all( + backendIds.map((id) => apphosting.getBackend(projectId, location, id)), + ); - const multiAccountsA = toMulti(accountsA); - const multiAccountsB = toMulti(accountsB); + const multiAccountsList = await Promise.all( + backends.map(async (b) => toMulti(await serviceAccountsForBackend(projectNumber, b))), + ); - // Combine build/run service accounts from both backends + // Combine build/run service accounts from all backends const combinedAccounts = { - buildServiceAccounts: Array.from(new Set([...multiAccountsA.buildServiceAccounts, ...multiAccountsB.buildServiceAccounts])), - runServiceAccounts: Array.from(new Set([...multiAccountsA.runServiceAccounts, ...multiAccountsB.runServiceAccounts])) + buildServiceAccounts: Array.from( + new Set(multiAccountsList.flatMap((a) => a.buildServiceAccounts)), + ), + runServiceAccounts: Array.from(new Set(multiAccountsList.flatMap((a) => a.runServiceAccounts))), }; for (const [envName, envVal] of secretEntries) { @@ -68,7 +67,7 @@ export async function setupSandboxSecrets( if (!exists) { await csm.createSecret(projectId, mockSecretName, { "created-by": "apphosting-compare-tool", - "slot": String(slotIndex) + slot: String(slotIndex), }); } @@ -78,18 +77,26 @@ export async function setupSandboxSecrets( mappings.push({ originalName: originalSecretName, mockSecretName, - mockValue + mockValue, }); } return mappings; } -export async function cleanupSandboxSecrets(projectId: string, mappings: SecretMapping[]): Promise { +/** + * + */ +export async function cleanupSandboxSecrets( + projectId: string, + mappings: SecretMapping[], +): Promise { if (mappings.length === 0) return; logger.info("Cleaning up sandboxed secrets in Secret Manager..."); await Promise.allSettled( - mappings.map(m => csm.deleteSecret(projectId, m.mockSecretName).catch(e => logger.debug(e))) + mappings.map((m) => + csm.deleteSecret(projectId, m.mockSecretName).catch((e) => logger.debug(e)), + ), ); } diff --git a/src/apphosting/compare/slots.spec.ts b/src/apphosting/compare/slots.spec.ts index 73d47c9b433..d17880881c7 100644 --- a/src/apphosting/compare/slots.spec.ts +++ b/src/apphosting/compare/slots.spec.ts @@ -31,17 +31,23 @@ describe("Comparison Slots Manager", () => { it("should acquire an existing idle slot", async () => { listBackendsStub.resolves({ backends: [ - { name: "projects/test/locations/us-central1/backends/compare-slot-1-a", labels: { status: "idle", type: "comparison-sandbox" } }, - { name: "projects/test/locations/us-central1/backends/compare-slot-1-b", labels: { status: "idle", type: "comparison-sandbox" } } - ] + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-0", + labels: { status: "idle", type: "comparison-sandbox" }, + }, + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-1", + labels: { status: "idle", type: "comparison-sandbox" }, + }, + ], }); listFirebaseAppsStub.resolves([{ appId: "web-app-123", displayName: "existing-app" }]); updateBackendStub.resolves({ name: "op-name" }); - const slot = await acquireComparisonSlot("aryanf-test", "us-central1"); + const slot = await acquireComparisonSlot("aryanf-test", "us-central1", 2); expect(slot.index).to.equal(1); - expect(slot.backendIdA).to.equal("compare-slot-1-a"); - expect(slot.backendIdB).to.equal("compare-slot-1-b"); + expect(slot.backendIds[0]).to.equal("compare-slot-1-0"); + expect(slot.backendIds[1]).to.equal("compare-slot-1-1"); expect(updateBackendStub.callCount).to.equal(2); expect(createBackendStub.callCount).to.equal(0); @@ -53,7 +59,7 @@ describe("Comparison Slots Manager", () => { createBackendStub.resolves({ name: "backend-resource" }); updateBackendStub.resolves({ name: "op-name" }); - const slot = await acquireComparisonSlot("aryanf-test", "us-central1"); + const slot = await acquireComparisonSlot("aryanf-test", "us-central1", 2); expect(slot.index).to.equal(1); expect(createBackendStub.callCount).to.equal(2); }); @@ -62,14 +68,20 @@ describe("Comparison Slots Manager", () => { const busyBackends = []; for (let i = 1; i <= 5; i++) { busyBackends.push( - { name: `projects/test/locations/us-central1/backends/compare-slot-${i}-a`, labels: { status: "busy", type: "comparison-sandbox" } }, - { name: `projects/test/locations/us-central1/backends/compare-slot-${i}-b`, labels: { status: "busy", type: "comparison-sandbox" } } + { + name: `projects/test/locations/us-central1/backends/compare-slot-${i}-0`, + labels: { status: "busy", type: "comparison-sandbox" }, + }, + { + name: `projects/test/locations/us-central1/backends/compare-slot-${i}-1`, + labels: { status: "busy", type: "comparison-sandbox" }, + }, ); } listBackendsStub.resolves({ backends: busyBackends }); - await expect(acquireComparisonSlot("aryanf-test", "us-central1")).to.be.rejectedWith( - "All 5 comparison slots are currently in use or project backend limits exceeded" + await expect(acquireComparisonSlot("aryanf-test", "us-central1", 2)).to.be.rejectedWith( + "All 5 comparison slots are currently in use or project backend limits exceeded", ); }); }); diff --git a/src/apphosting/compare/slots.ts b/src/apphosting/compare/slots.ts index a7b5ab19ce7..8fa6ec3a2ff 100644 --- a/src/apphosting/compare/slots.ts +++ b/src/apphosting/compare/slots.ts @@ -2,17 +2,17 @@ import * as apphosting from "../../gcp/apphosting"; import * as poller from "../../operation-poller"; import { createBackend } from "../backend"; import { listFirebaseApps, createWebApp, AppPlatform } from "../../management/apps"; -import { FirebaseError } from "../../error"; + import { logger } from "../../logger"; +import { FirebaseError } from "../../error"; import { apphostingOrigin } from "../../api"; export interface ComparisonSlot { index: number; - backendIdA: string; - backendIdB: string; + backendIds: string[]; } -const MAX_SLOTS = 5; +const MAX_SLOTS = 10; const apphostingPollerOptions = { apiOrigin: apphostingOrigin(), apiVersion: apphosting.API_VERSION, @@ -24,96 +24,136 @@ async function updateBackendAndPoll( projectId: string, location: string, backendId: string, - labels: Record + labels: Record, ): Promise { - const op = await apphosting.updateBackend(projectId, location, backendId, { labels }); + const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; + const res = await apphosting.client.patch( + name, + { name, labels }, + { queryParams: { updateMask: "labels" } }, + ); + await poller.pollOperation({ ...apphostingPollerOptions, pollerName: `update-labels-${projectId}-${location}-${backendId}`, - operationResourceName: op.name, + operationResourceName: res.body.name, }); } +/** + * + */ export async function getOrCreateSharedWebAppId(projectId: string): Promise { const apps = await listFirebaseApps(projectId, AppPlatform.WEB); if (apps.length > 0) { return apps[0].appId; } - logger.info("No existing Web Apps found. Provisioning a shared Web App for comparison slot runners..."); + logger.info( + "No existing Web Apps found. Provisioning a shared Web App for comparison slot runners...", + ); const createdApp = await createWebApp(projectId, { displayName: "firebase-compare-shared-app" }); return createdApp.appId; } +/** + * + */ export async function acquireComparisonSlot( projectId: string, - location: string + location: string, + numVariants: number, ): Promise { const existingBackends = await apphosting.listBackends(projectId, location); const backendsList = existingBackends.backends || []; for (let i = 1; i <= MAX_SLOTS; i++) { - const backendIdA = `compare-slot-${i}-a`; - const backendIdB = `compare-slot-${i}-b`; - - const backendA = backendsList.find(b => b.name.endsWith(backendIdA)); - const backendB = backendsList.find(b => b.name.endsWith(backendIdB)); - - const isLocked = backendA?.labels?.status === "busy" || backendB?.labels?.status === "busy"; + const slotBackendIds: string[] = []; + let isLocked = false; + + for (let v = 0; v < numVariants; v++) { + const backendId = `compare-slot-${i}-${v}`; + slotBackendIds.push(backendId); + const backend = backendsList.find((b) => b.name.endsWith(backendId)); + if (backend?.labels?.status === "busy") { + isLocked = true; + } + } if (!isLocked) { const webAppId = await getOrCreateSharedWebAppId(projectId); - if (!backendA || !backendB) { - const slotsNeeded = (backendA ? 0 : 1) + (backendB ? 0 : 1); - if (backendsList.length + slotsNeeded > 10) { - continue; // Quota limit hit, check next slot - } + // Check how many we need to create + const missingCount = slotBackendIds.filter( + (id) => !backendsList.find((b) => b.name.endsWith(id)), + ).length; - if (!backendA) { - logger.info(`Provisioning backend for Comparison Slot ${i} (A)...`); - await createBackend(projectId, location, backendIdA, null, undefined, webAppId); - await updateBackendAndPoll(projectId, location, backendIdA, { status: "busy", type: "comparison-sandbox" }); - } - if (!backendB) { - logger.info(`Provisioning backend for Comparison Slot ${i} (B)...`); - await createBackend(projectId, location, backendIdB, null, undefined, webAppId); - await updateBackendAndPoll(projectId, location, backendIdB, { status: "busy", type: "comparison-sandbox" }); + if (backendsList.length + missingCount > 30) { + continue; // Quota limit hit, check next slot + } + + logger.info(`Acquiring Comparison Slot ${i} for ${numVariants} variants...`); + const updatePromises: Promise[] = []; + + for (const backendId of slotBackendIds) { + const backend = backendsList.find((b) => b.name.endsWith(backendId)); + if (!backend) { + logger.info(`Provisioning backend ${backendId}...`); + await createBackend(projectId, location, backendId, null, undefined, webAppId); + updatePromises.push( + updateBackendAndPoll(projectId, location, backendId, { + status: "busy", + type: "comparison-sandbox", + }), + ); + } else { + updatePromises.push( + updateBackendAndPoll(projectId, location, backendId, { + ...backend.labels, + status: "busy", + }), + ); } - } else { - logger.info(`Acquiring Comparison Slot ${i} (Reusing existing backends)...`); - await Promise.all([ - updateBackendAndPoll(projectId, location, backendIdA, { ...backendA.labels, status: "busy" }), - updateBackendAndPoll(projectId, location, backendIdB, { ...backendB.labels, status: "busy" }) - ]); } - return { index: i, backendIdA, backendIdB }; + await Promise.all(updatePromises); + return { index: i, backendIds: slotBackendIds }; } } throw new FirebaseError( - "All 5 comparison slots are currently in use or project backend limits exceeded. Please wait and try again." + "All 10 comparison slots are currently in use or project backend limits exceeded. Please wait and try again.", ); } +/** + * + */ export async function releaseComparisonSlot( projectId: string, location: string, - slotIndex: number + slotIndex: number, + numVariants: number, ): Promise { - const backendIdA = `compare-slot-${slotIndex}-a`; - const backendIdB = `compare-slot-${slotIndex}-b`; - logger.info(`Releasing Comparison Slot ${slotIndex}...`); const existingBackends = await apphosting.listBackends(projectId, location); const backendsList = existingBackends.backends || []; - const backendA = backendsList.find(b => b.name.endsWith(backendIdA)); - const backendB = backendsList.find(b => b.name.endsWith(backendIdB)); - await Promise.allSettled([ - backendA ? updateBackendAndPoll(projectId, location, backendIdA, { ...backendA.labels, status: "idle" }) : Promise.resolve(), - backendB ? updateBackendAndPoll(projectId, location, backendIdB, { ...backendB.labels, status: "idle" }) : Promise.resolve() - ]); + const updatePromises: Promise[] = []; + + for (let v = 0; v < numVariants; v++) { + const backendId = `compare-slot-${slotIndex}-${v}`; + const backend = backendsList.find((b) => b.name.endsWith(backendId)); + if (backend) { + updatePromises.push( + updateBackendAndPoll(projectId, location, backendId, { + ...backend.labels, + status: "idle", + }), + ); + } + } + + await Promise.allSettled(updatePromises); } diff --git a/src/apphosting/compare/suite.spec.ts b/src/apphosting/compare/suite.spec.ts index 7de43f541e6..9933752715c 100644 --- a/src/apphosting/compare/suite.spec.ts +++ b/src/apphosting/compare/suite.spec.ts @@ -52,7 +52,7 @@ describe("runCompareSuite Orchestrator", () => { getProjectNumberStub = sinon.stub(projectNumberHelper, "getProjectNumber").resolves("12345"); setupSecretsStub = sinon.stub(secretsManager, "setupSandboxSecrets").resolves([]); cleanupSecretsStub = sinon.stub(secretsManager, "cleanupSandboxSecrets").resolves(); - acquireSlotStub = sinon.stub(slotsManager, "acquireComparisonSlot").resolves({ index: 1, backendIdA: "compare-slot-1-a", backendIdB: "compare-slot-1-b" }); + acquireSlotStub = sinon.stub(slotsManager, "acquireComparisonSlot").resolves({ index: 1, backendIds: ["compare-slot-1-a", "compare-slot-1-b"] } as any); releaseSlotStub = sinon.stub(slotsManager, "releaseComparisonSlot").resolves(); validateProjectStub = sinon.stub(lifecycle, "validateProject").returns(); runGarbageCollectionStub = sinon.stub(lifecycle, "runGarbageCollection").resolves(); @@ -82,8 +82,10 @@ describe("runCompareSuite Orchestrator", () => { await runCompareSuite( "aryanf-test", "us-central1", - "/app/path-a", - "/app/path-b" + [ + { path: "/app/path-a" }, + { path: "/app/path-b" } + ] ); expect(acquireSlotStub.callCount).to.equal(1); @@ -95,9 +97,8 @@ describe("runCompareSuite Orchestrator", () => { expect(discoverRoutesStub.callCount).to.equal(1); expect(crawlStub.callCount).to.equal(1); - expect(compareRouteStub.callCount).to.equal(2); + expect(compareRouteStub.callCount).to.equal(1); expect(compareRouteStub.firstCall.args[0]).to.equal("/"); - expect(compareRouteStub.secondCall.args[0]).to.equal("/about"); expect(generateReportStub.callCount).to.equal(1); expect(cleanupSecretsStub.callCount).to.equal(1); @@ -115,18 +116,31 @@ describe("runCompareSuite Orchestrator", () => { await runCompareSuite( "aryanf-test", "us-central1", - "/app/path-a", - "/app/path-b", - { localBuildA: false, localBuildB: true } + [ + { path: "/app/path-a", localBuild: false }, + { path: "/app/path-b", localBuild: true } + ] ); - // Backend A is source deploy -> calls createSourceDeployArchive (1 time) - // Backend B is local build -> calls localBuild (1 time) and createLocalBuildTarArchive (1 time) expect(createArchiveStub.callCount).to.equal(1); expect(localBuildStub.callCount).to.equal(1); expect(createTarStub.callCount).to.equal(1); + expect(uploadObjectStub.callCount).to.equal(2); + expect(orchestrateRolloutStub.callCount).to.equal(2); + it("should support runtime version patching for backends", async () => { + await runCompareSuite( + "aryanf-test", + "us-central1", + [ + { path: "/app/path-a", runtime: "nodejs20" }, + { path: "/app/path-b", runtime: "nodejs22" } + ] + ); + + expect(createArchiveStub.callCount).to.equal(2); expect(uploadObjectStub.callCount).to.equal(2); expect(orchestrateRolloutStub.callCount).to.equal(2); }); }); +}); diff --git a/src/apphosting/compare/suite.ts b/src/apphosting/compare/suite.ts index e55aba90714..485a9b6db8d 100644 --- a/src/apphosting/compare/suite.ts +++ b/src/apphosting/compare/suite.ts @@ -7,6 +7,7 @@ import * as apphosting from "../../gcp/apphosting"; import * as rollout from "../rollout"; import * as deployUtil from "../../deploy/apphosting/util"; import { getProjectNumber } from "../../getProjectNumber"; +import { apphostingOrigin } from "../../api"; import * as secrets from "./secrets"; import * as slots from "./slots"; import * as lifecycle from "./lifecycle"; @@ -20,14 +21,18 @@ import * as poller from "../../operation-poller"; import { logger } from "../../logger"; const apphostingPollerOptions = { - apiOrigin: apphosting.apphostingOrigin(), + apiOrigin: apphostingOrigin(), apiVersion: "v1beta", backoff: 200, maxBackoff: 10000, timeout: 120000, // 2 minutes }; -async function prepareLocalBuildDir(rootDir: string, scratchDir: string, backendId: string): Promise { +async function prepareLocalBuildDir( + rootDir: string, + scratchDir: string, + backendId: string, +): Promise { const ignore = deployUtil.resolveIgnorePatterns({ backendId, rootDir: "/", ignore: [] }); fs.rmSync(scratchDir, { recursive: true, force: true }); fs.mkdirSync(scratchDir, { recursive: true }); @@ -51,20 +56,23 @@ async function deployToBackend( appPath: string, bucketName: string, useLocalBuild: boolean, - runtimeVersion?: string + runtimeVersion?: string, ): Promise { let archivePath: string; let buildInput: any; if (runtimeVersion) { logger.info(`Patching runtime version for backend ${backendId} to ${runtimeVersion}...`); - const op = await apphosting.updateBackend(projectId, location, backendId, { - runtime: { value: runtimeVersion } - }); + const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; + const op = await apphosting.client.patch( + name, + { name, runtime: { value: runtimeVersion } }, + { queryParams: { updateMask: "runtime" } }, + ); await poller.pollOperation({ ...apphostingPollerOptions, pollerName: `update-runtime-${backendId}`, - operationResourceName: op.name, + operationResourceName: op.body.name, }); } @@ -72,27 +80,27 @@ async function deployToBackend( logger.info(`Running local build for slot backend ${backendId}...`); const pathHash = crypto.createHash("md5").update(appPath).digest("hex").substring(0, 8); const scratchDir = path.join(os.tmpdir(), `apphosting-local-build-${backendId}-${pathHash}`); - + await prepareLocalBuildDir(appPath, scratchDir, backendId); - + const { outputFiles, buildConfig } = await localBuild( projectId, scratchDir, {}, - { nonInteractive: true } + { nonInteractive: true }, ); archivePath = await deployUtil.createLocalBuildTarArchive( { backendId, rootDir: "/", ignore: [] }, scratchDir, - outputFiles + outputFiles, ); logger.info(`Uploading local build bundle for ${backendId}...`); await gcs.uploadObject( { file: archivePath, stream: fs.createReadStream(archivePath) }, bucketName, - gcs.ContentType.TAR + gcs.ContentType.TAR, ); const uri = `gs://${bucketName}/${path.basename(archivePath)}`; @@ -104,21 +112,21 @@ async function deployToBackend( rootDirectory: "/", runCommand: buildConfig.runCommand, env: buildConfig.env, - } - } + }, + }, }; } else { logger.info(`Packaging source archive for ${backendId}...`); archivePath = await deployUtil.createSourceDeployArchive( { backendId, rootDir: "/", ignore: [] }, - appPath + appPath, ); logger.info(`Uploading source archive for ${backendId}...`); await gcs.uploadObject( { file: archivePath, stream: fs.createReadStream(archivePath) }, bucketName, - gcs.ContentType.ZIP + gcs.ContentType.ZIP, ); const uri = `gs://${bucketName}/${path.basename(archivePath)}`; @@ -126,9 +134,9 @@ async function deployToBackend( source: { archive: { userStorageUri: uri, - rootDirectory: "/" - } - } + rootDirectory: "/", + }, + }, }; } @@ -137,54 +145,85 @@ async function deployToBackend( projectId, location, backendId, - buildInput + buildInput, }); + + // Wait until the backend is fully done reconciling after the rollout. + logger.info(`Waiting for backend ${backendId} to finish reconciling...`); + let backendIsReconciling = true; + while (backendIsReconciling) { + const b = await apphosting.client.get( + `projects/${projectId}/locations/${location}/backends/${backendId}`, + ); + backendIsReconciling = !!b.body.reconciling; + if (backendIsReconciling) { + await new Promise((r) => setTimeout(r, 2000)); + } + } } +export interface VariantConfig { + id?: string; + path: string; + localBuild?: boolean; + runtime?: string; +} + +/** + * + */ export async function runCompareSuite( projectId: string, location: string, - appPathA: string, - appPathB: string, + variants: VariantConfig[], options: { outputDir?: string; - localBuildA?: boolean; - localBuildB?: boolean; - runtimeA?: string; - runtimeB?: string; - } = {} + } = {}, ): Promise { lifecycle.validateProject(projectId); await lifecycle.runGarbageCollection(projectId, location); const projectNumber = await getProjectNumber({ projectId }); - - // 1. Acquire Comparison Slot - const slot = await slots.acquireComparisonSlot(projectId, location); - logger.info(`Acquired Comparison Slot ${slot.index}. Deploying A (localBuild: ${!!options.localBuildA}, runtime: ${options.runtimeA || "default"}): ${slot.backendIdA}, B (localBuild: ${!!options.localBuildB}, runtime: ${options.runtimeB || "default"}): ${slot.backendIdB}...`); - let secretsMappings: secrets.SecretMapping[] = []; + // 1. Acquire Comparison Slot for N variants + const slot = await slots.acquireComparisonSlot(projectId, location, variants.length); + logger.info( + `Acquired Comparison Slot ${slot.index} with ${variants.length} backends: ${slot.backendIds.join(", ")}`, + ); + + let secretsMappings: secrets.SecretMapping[][] = []; const cleanUp = async () => { logger.warn("\nInterrupted. Restoring slot and deleting mock secrets..."); - await secrets.cleanupSandboxSecrets(projectId, secretsMappings); - await slots.releaseComparisonSlot(projectId, location, slot.index); + for (const mapping of secretsMappings) { + await secrets.cleanupSandboxSecrets(projectId, mapping); + } + await slots.releaseComparisonSlot(projectId, location, slot.index, variants.length); + process.exit(1); }; process.on("SIGINT", cleanUp); process.on("SIGTERM", cleanUp); try { - // 2. Setup mock secrets - secretsMappings = await secrets.setupSandboxSecrets( - projectId, - location, - appPathA, - slot.index, - slot.backendIdA, - slot.backendIdB + // 2. Setup mock secrets per unique codebase path + const uniquePaths = Array.from(new Set(variants.map((v) => v.path))); + secretsMappings = await Promise.all( + uniquePaths.map((uniquePath) => { + const pathBackendIds = variants + .map((v, i) => (v.path === uniquePath ? slot.backendIds[i] : null)) + .filter((id): id is string => id !== null); + + return secrets.setupSandboxSecrets( + projectId, + location, + uniquePath, + slot.index, + pathBackendIds + ); + }) ); - // 3. Package, Upload and Deploy Source A & B + // 3. Package, Upload and Deploy Source for all N variants const bucketName = `firebaseapphosting-sources-${projectNumber}-${location.toLowerCase()}`; await gcs.upsertBucket({ product: "apphosting", @@ -197,65 +236,94 @@ export async function runCompareSuite( lifecycle: { rule: [ { - action: { - type: "Delete", - }, - condition: { - age: 30, - }, + action: { type: "Delete" }, + condition: { age: 30 }, }, ], }, - } + }, }); - await Promise.all([ - deployToBackend(projectId, location, slot.backendIdA, appPathA, bucketName, !!options.localBuildA, options.runtimeA), - deployToBackend(projectId, location, slot.backendIdB, appPathB, bucketName, !!options.localBuildB, options.runtimeB) - ]); - - logger.info("Rollouts completed successfully!"); + await Promise.all( + variants.map((v, i) => + deployToBackend( + projectId, + location, + slot.backendIds[i], + v.path, + bucketName, + !!v.localBuild, + v.runtime, + ), + ), + ); - // 4. Route Discovery - const discoveredStaticRoutes = await discover.discoverRoutes(appPathA); - logger.info(`Discovered ${discoveredStaticRoutes.length} static routes from manifests/sitemap.`); + logger.info("All N-Way Rollouts completed successfully!"); - // 5. Retrieve Live URLs and Run Crawler & Compare - const [bA, bB] = await Promise.all([ - apphosting.getBackend(projectId, location, slot.backendIdA), - apphosting.getBackend(projectId, location, slot.backendIdB) - ]); + // 4. Retrieve Live URLs for all variants + const backendDataList = await Promise.all( + slot.backendIds.map((id) => apphosting.getBackend(projectId, location, id)), + ); + const urls = backendDataList.map((b) => (b.uri.startsWith("http") ? b.uri : `https://${b.uri}`)); - const urlA = bA.uri; - const urlB = bB.uri; + urls.forEach((url, i) => { + logger.info(`Variant ${variants[i].id || i} URL: ${url}`); + }); - logger.info(`Backend A URL: ${urlA}`); - logger.info(`Backend B URL: ${urlB}`); + // 5. Route Discovery & Crawling across all variants + const allRoutesSet = new Set(); - logger.info("Crawling Backend A for dynamic link discovery..."); - const crawler = new Crawler(urlA); - await crawler.crawl(); - const crawledRoutes = crawler.getRoutes(); - logger.info(`Crawler discovered ${crawledRoutes.length} routes.`); + for (let i = 0; i < variants.length; i++) { + const v = variants[i]; + const url = urls[i]; - const allRoutes = Array.from(new Set([...discoveredStaticRoutes, ...crawledRoutes])).sort(); + const discoveredStaticRoutes = await discover.discoverRoutes(v.path); + discoveredStaticRoutes.forEach((r) => allRoutesSet.add(r)); - logger.info(`Commencing comparison of ${allRoutes.length} routes...`); - const results: compare.ComparisonResult[] = []; - for (const route of allRoutes) { - logger.info(`Comparing route: ${route}`); - const res = await compare.compareRoute(route, urlA, urlB); - results.push(res); + logger.info(`Crawling Variant ${v.id || i} for dynamic link discovery...`); + const crawler = new Crawler(url); + await crawler.crawl(); + const crawledRoutes = crawler.getRoutes(); + crawledRoutes.forEach((r) => allRoutesSet.add(r)); } - // 6. Report Generation - await reporter.generateReport(projectId, location, slot.backendIdA, slot.backendIdB, results, options.outputDir); + const allRoutes = Array.from(allRoutesSet).sort(); + logger.info(`Total unique routes discovered across matrix: ${allRoutes.length}`); + + // 6. Report Generation for all unique pairs (Matrix Diffing) + for (let i = 0; i < variants.length; i++) { + for (let j = i + 1; j < variants.length; j++) { + logger.info( + `\nGenerating Comparison Report: ${variants[i].id || i} vs ${variants[j].id || j}...`, + ); + + const results: compare.ComparisonResult[] = []; + for (const route of allRoutes) { + const res = await compare.compareRoute(route, urls[i], urls[j]); + results.push(res); + } + + const pairOutputDir = options.outputDir + ? path.join(options.outputDir, `${variants[i].id || i}-vs-${variants[j].id || j}`) + : undefined; + await reporter.generateReport( + projectId, + location, + slot.backendIds[i], + slot.backendIds[j], + results, + pairOutputDir, + ); + } + } } finally { process.off("SIGINT", cleanUp); process.off("SIGTERM", cleanUp); - await secrets.cleanupSandboxSecrets(projectId, secretsMappings); - await slots.releaseComparisonSlot(projectId, location, slot.index); + for (const mapping of secretsMappings) { + await secrets.cleanupSandboxSecrets(projectId, mapping); + } + await slots.releaseComparisonSlot(projectId, location, slot.index, variants.length); } } diff --git a/src/commands/apphosting-compare-suite.ts b/src/commands/apphosting-compare-suite.ts index 7b7b8c75efc..8668aaa67e3 100644 --- a/src/commands/apphosting-compare-suite.ts +++ b/src/commands/apphosting-compare-suite.ts @@ -13,16 +13,13 @@ export const command = new Command("apphosting:compare-suite") .option( "--location ", "the primary region of the App Hosting backends to use", - "us-central1" - ) - .option( - "--suite-config ", - "path to comparison suite JSON configuration file" + "us-central1", ) + .option("--suite-config ", "path to comparison suite JSON configuration file") .option( "--output-dir ", "directory to output comparison report files", - "./compare-report" + "./compare-report", ) .before(requireAuth) .action(async (options: Options) => { @@ -31,7 +28,9 @@ export const command = new Command("apphosting:compare-suite") const configPath = options.suiteConfig as string; if (!configPath) { - throw new FirebaseError("Must specify --suite-config file containing the list of apps to compare."); + throw new FirebaseError( + "Must specify --suite-config file containing the list of apps to compare.", + ); } if (!(await fs.pathExists(configPath))) { @@ -47,21 +46,19 @@ export const command = new Command("apphosting:compare-suite") for (const testCase of suite) { logger.info(`\nRunning test case: ${testCase.name || "Unnamed Test"}`); const caseOutputDir = path.join(options.outputDir as string, testCase.name || "unnamed"); - - try { - await runCompareSuite( - projectId, - location, - testCase.pathA, - testCase.pathB, - { - outputDir: caseOutputDir, - localBuildA: !!testCase.localBuildA, - localBuildB: !!testCase.localBuildB - } + + if (!testCase.variants || testCase.variants.length < 2) { + throw new FirebaseError( + `Test case ${testCase.name} must have a "variants" array with at least 2 configurations.`, ); + } + + try { + await runCompareSuite(projectId, location, testCase.variants, { + outputDir: caseOutputDir, + }); } catch (err: any) { - logger.error(`Test case ${testCase.name} failed: ${err.message}`); + logger.error(`Matrix execution for ${testCase.name} failed: ${err.message}`); } } }); diff --git a/src/commands/apphosting-compare.ts b/src/commands/apphosting-compare.ts index 3c3797ddbe5..d7be3520f67 100644 --- a/src/commands/apphosting-compare.ts +++ b/src/commands/apphosting-compare.ts @@ -6,41 +6,37 @@ import { runCompareSuite } from "../apphosting/compare/suite"; import { FirebaseError } from "../error"; export const command = new Command("apphosting:compare") - .description("Autonomously deploy and compare two versions/configurations of a Firebase App Hosting codebase") + .description( + "Autonomously deploy and compare two versions/configurations of a Firebase App Hosting codebase", + ) .option( "--location ", "the primary region of the App Hosting backends to use", - "us-central1" + "us-central1", ) .option( "--path-a ", "path to directory containing codebase version A (defaults to current directory)", - "." + ".", ) .option( "--path-b ", - "path to directory containing codebase version B (defaults to path-a or current directory)" - ) - .option( - "--local-build-a", - "compile and deploy version A using a local build" - ) - .option( - "--local-build-b", - "compile and deploy version B using a local build" + "path to directory containing codebase version B (defaults to path-a or current directory)", ) + .option("--local-build-a", "compile and deploy version A using a local build") + .option("--local-build-b", "compile and deploy version B using a local build") .option( "--runtime-a ", - "specify the ABIU runtime version for backend A (e.g. nodejs22)" + "specify the ABIU runtime version for backend A (e.g. nodejs22)", ) .option( "--runtime-b ", - "specify the ABIU runtime version for backend B (e.g. nodejs22)" + "specify the ABIU runtime version for backend B (e.g. nodejs22)", ) .option( "--output-dir ", "directory to output comparison report files", - "./compare-report" + "./compare-report", ) .before(requireAuth) .action(async (options: Options) => { @@ -49,11 +45,23 @@ export const command = new Command("apphosting:compare") const pathA = (options.pathA as string) || "."; const pathB = (options.pathB as string) || pathA; - await runCompareSuite(projectId, location, pathA, pathB, { - outputDir: options.outputDir as string, - localBuildA: !!options.localBuildA, - localBuildB: !!options.localBuildB, - runtimeA: options.runtimeA as string, - runtimeB: options.runtimeB as string - }); + await runCompareSuite( + projectId, + location, + [ + { + path: pathA, + localBuild: !!options.localBuildA, + runtime: options.runtimeA as string | undefined, + }, + { + path: pathB, + localBuild: !!options.localBuildB, + runtime: options.runtimeB as string | undefined, + }, + ], + { + outputDir: options.outputDir as string, + }, + ); }); diff --git a/src/commands/index.ts b/src/commands/index.ts index c2d2091e296..8ec41bb6145 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -211,7 +211,7 @@ export function load(client: CLIClient): CLIClient { client.apphosting.rollouts = {}; client.apphosting.rollouts.create = loadCommand("apphosting-rollouts-create"); client.apphosting.compare = loadCommand("apphosting-compare"); - client.apphosting.compareSuite = loadCommand("apphosting-compare-suite"); + client.apphosting["compare-suite"] = loadCommand("apphosting-compare-suite"); client.apphosting.config = {}; if (experiments.isEnabled("internaltesting")) { client.apphosting.builds = {}; From 122775c81cf4affd77fbe3a6485ddd5d345a6df8 Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Wed, 17 Jun 2026 12:16:07 -0400 Subject: [PATCH 03/12] feat(apphosting): add static route discovery and comprehensive parity testing dashboard --- .gitignore | 3 + src/apphosting/compare/README.md | 58 +- src/apphosting/compare/cache.ts | 72 ++ src/apphosting/compare/compare.ts | 91 +- src/apphosting/compare/discover.spec.ts | 35 + src/apphosting/compare/discover.ts | 139 ++- src/apphosting/compare/server.ts | 1014 ++++++++++++++++++++++ src/apphosting/compare/slots.spec.ts | 12 +- src/apphosting/compare/slots.ts | 8 +- src/apphosting/compare/suite.spec.ts | 133 +-- src/apphosting/compare/suite.ts | 455 +++++----- src/commands/apphosting-compare-suite.ts | 107 ++- src/commands/apphosting-compare.ts | 67 +- 13 files changed, 1800 insertions(+), 394 deletions(-) create mode 100644 src/apphosting/compare/cache.ts create mode 100644 src/apphosting/compare/server.ts diff --git a/.gitignore b/.gitignore index 593347ef2b0..d321b20680b 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ clean/ # Dart/Flutter .dart_tool/ **/pubspec.lock + +compare-report/ +compare-cache/ diff --git a/src/apphosting/compare/README.md b/src/apphosting/compare/README.md index 9712abe1339..2a8b6affa13 100644 --- a/src/apphosting/compare/README.md +++ b/src/apphosting/compare/README.md @@ -2,12 +2,12 @@ This is an experimental internal CLI tool built for the Firebase App Hosting team to dynamically verify the compatibility and performance of different infrastructure backends or application builds. -It takes an array of $N$ configurations (variants) and automatically deploys them simultaneously, discovering their routes, and dumping an $O(N^2)$ Pair-wise Cartesian Matrix of differences. +It takes an array of $N$ configurations (variants) and automatically deploys them, discovering their routes, and dumping an $O(N^2)$ Pair-wise Cartesian Matrix of differences. ## Capabilities -1. **N-Way Concurrency**: By leveraging `Promise.all` and dynamic `backendIds`, the tool spins up all $N$ test variants in parallel on Google Cloud. -2. **Local vs Remote Build Verification**: Can deploy locally built bundles (e.g. `locally_built: true`) side-by-side with remote Cloud Build source zips. +1. **Standard CLI Fidelity**: Deployments are executed by automatically constructing a temporary `firebase.json` and programmatically calling the `firebase deploy` CLI execution path. This guarantees that test deployments faithfully mirror the actual customer experience (including Secrets, AutoInit Env Vars, and custom headers). +2. **Local vs Remote Build Verification**: Can deploy locally built bundles (e.g. `localBuild: true`) side-by-side with remote Cloud Build source zips. 3. **Automated IAM & Secrets Management**: Intelligently creates a single mock secret in Secret Manager for each distinct codebase path, mapping the IAM `secretAccessor` roles simultaneously to all backends generated from that codebase. 4. **Dynamic Spidering**: Automatically crawls Next.js / Angular apps recursively starting from `/` to discover hidden dynamic routes, alongside statically parsing `.next/prerender-manifest.json`. 5. **Exact Diff Inspection**: Generates HTML dashboards, JSON summaries, and specifically dumps the raw HTTP HTML outputs of each variant so engineers can run local diffs. @@ -17,33 +17,31 @@ It takes an array of $N$ configurations (variants) and automatically deploys the 1. Create a `matrix-test.json` file to define your test cases: ```json -{ - "testCases": [ - { - "name": "Node Matrix Test", - "variants": [ - { - "id": "Local-Node24", - "path": "../next-sample-1", - "localBuild": true, - "runtime": "nodejs24" - }, - { - "id": "Source-Node24", - "path": "../next-sample-1", - "localBuild": false, - "runtime": "nodejs24" - }, - { - "id": "Source-Node22", - "path": "../next-sample-1", - "localBuild": false, - "runtime": "nodejs22" - } - ] - } - ] -} +[ + { + "name": "Node Matrix Test", + "variants": [ + { + "id": "Local-Node24", + "path": "../next-sample-1", + "localBuild": true, + "runtime": "nodejs24" + }, + { + "id": "Source-Node24", + "path": "../next-sample-1", + "localBuild": false, + "runtime": "nodejs24" + }, + { + "id": "Source-Node22", + "path": "../next-sample-1", + "localBuild": false, + "runtime": "nodejs22" + } + ] + } +] ``` 2. Run the command: diff --git a/src/apphosting/compare/cache.ts b/src/apphosting/compare/cache.ts new file mode 100644 index 00000000000..e0cc5bc17e3 --- /dev/null +++ b/src/apphosting/compare/cache.ts @@ -0,0 +1,72 @@ +import * as fs from "fs-extra"; +import * as path from "path"; +import { logger } from "../../logger"; + +export interface RouteResponse { + status: number; + headers: Record; + body: string; + isBinary: boolean; +} + +export interface VariantRecording { + id: string; + testCaseName: string; + timestamp: string; + url: string; + routes: Record; +} + +const CACHE_DIR = path.resolve(process.cwd(), "compare-cache"); + +function getRecordingPath(testCaseName: string, variantId: string): string { + const safeTestCase = testCaseName.replace(/[^a-zA-Z0-9_-]/g, "_"); + const safeVariant = variantId.replace(/[^a-zA-Z0-9_-]/g, "_"); + return path.join(CACHE_DIR, "recordings", safeTestCase, `${safeVariant}.json`); +} + +/** + * Saves a variant recording to the cache. + */ +export async function saveRecording(recording: VariantRecording): Promise { + const filePath = getRecordingPath(recording.testCaseName, recording.id); + await fs.ensureDir(path.dirname(filePath)); + await fs.writeJson(filePath, recording, { spaces: 2 }); + logger.info(`Saved recording to cache: ${filePath}`); +} + +/** + * Loads a variant recording from the cache. + */ +export async function loadRecording(testCaseName: string, variantId: string): Promise { + const filePath = getRecordingPath(testCaseName, variantId); + if (!(await fs.pathExists(filePath))) { + throw new Error(`No recording found in cache for variant "${variantId}" under test case "${testCaseName}"`); + } + return await fs.readJson(filePath); +} + +/** + * Lists all recorded test cases and their variants. + */ +export async function listRecordings(): Promise> { + const recordingsDir = path.join(CACHE_DIR, "recordings"); + if (!(await fs.pathExists(recordingsDir))) { + return {}; + } + + const result: Record = {}; + const testCases = await fs.readdir(recordingsDir); + for (const tc of testCases) { + const tcDir = path.join(recordingsDir, tc); + const stat = await fs.stat(tcDir); + if (stat.isDirectory()) { + const files = await fs.readdir(tcDir); + result[tc] = files + .filter((file) => file.endsWith(".json")) + .map((file) => file.replace(/\.json$/, "")); + } + } + + return result; +} diff --git a/src/apphosting/compare/compare.ts b/src/apphosting/compare/compare.ts index 9076c4e522f..ed8261a326f 100644 --- a/src/apphosting/compare/compare.ts +++ b/src/apphosting/compare/compare.ts @@ -5,6 +5,8 @@ import { MyersDiffEngine } from "./distance"; export interface ComparisonResult { route: string; statusMatch: boolean; + statusA?: number; + statusB?: number; headerMismatches: Array<{ header: string; valA: string; valB: string }>; expectedHeaderVariations: Array<{ header: string; valA: string; valB: string }>; bodySimilarity: number; // 0.0 to 1.0 @@ -54,25 +56,70 @@ export async function compareRoute( fetch(`${urlB}${route}`, fetchOptions), ]); + const contentTypeA = resA.headers.get("content-type") || ""; + const contentTypeB = resB.headers.get("content-type") || ""; + const isBinaryA = isBinaryContentType(contentTypeA); + const isBinaryB = isBinaryContentType(contentTypeB); + + const headersA: Record = {}; + resA.headers.forEach((val, key) => { headersA[key] = val; }); + + const headersB: Record = {}; + resB.headers.forEach((val, key) => { headersB[key] = val; }); + + const responseA: RouteResponse = { + status: resA.status, + headers: headersA, + isBinary: isBinaryA || isBinaryB, + body: (isBinaryA || isBinaryB) ? (await resA.buffer()).toString("base64") : await resA.text(), + }; + + const responseB: RouteResponse = { + status: resB.status, + headers: headersB, + isBinary: isBinaryA || isBinaryB, + body: (isBinaryA || isBinaryB) ? (await resB.buffer()).toString("base64") : await resB.text(), + }; + + return await compareRouteResponses(route, responseA, responseB); +} + +export interface RouteResponse { + status: number; + headers: Record; + body: string; + isBinary: boolean; +} + +/** + * + */ +export async function compareRouteResponses( + route: string, + resA: RouteResponse, + resB: RouteResponse, +): Promise { const result: ComparisonResult = { route, statusMatch: resA.status === resB.status, + statusA: resA.status, + statusB: resB.status, headerMismatches: [], expectedHeaderVariations: [], bodySimilarity: 1.0, bodyDiff: "", - isBinary: false, + isBinary: resA.isBinary || resB.isBinary, }; // 1. Compare Headers const allHeaderKeys = new Set([ - ...Array.from(resA.headers.keys()), - ...Array.from(resB.headers.keys()), + ...Object.keys(resA.headers), + ...Object.keys(resB.headers), ]); for (const key of allHeaderKeys) { - const valA = resA.headers.get(key) || ""; - const valB = resB.headers.get(key) || ""; + const valA = resA.headers[key] || ""; + const valB = resB.headers[key] || ""; if (valA !== valB) { if (BEHAVIORAL_HEADERS.includes(key.toLowerCase())) { result.headerMismatches.push({ header: key, valA, valB }); @@ -82,14 +129,10 @@ export async function compareRoute( } } - // 2. Detect Binary - const contentTypeA = resA.headers.get("content-type") || ""; - const contentTypeB = resB.headers.get("content-type") || ""; - if (isBinaryContentType(contentTypeA) || isBinaryContentType(contentTypeB)) { - result.isBinary = true; - - const bufA = await resA.buffer(); - const bufB = await resB.buffer(); + // 2. Compare Binary + if (result.isBinary) { + const bufA = Buffer.from(resA.body, "base64"); + const bufB = Buffer.from(resB.body, "base64"); const sizeA = bufA.length; const sizeB = bufB.length; @@ -111,19 +154,30 @@ export async function compareRoute( } // 3. Compare Text Body - let bodyA = await resA.text(); - let bodyB = await resB.text(); + let bodyA = resA.body; + let bodyB = resB.body; - if (contentTypeA.includes("text/html")) { + const contentType = (resA.headers["content-type"] || resA.headers["Content-Type"] || "").toLowerCase(); + if (contentType.includes("text/html")) { try { const prettier = require("prettier"); bodyA = await prettier.format(bodyA, { parser: "html" }); bodyB = await prettier.format(bodyB, { parser: "html" }); } catch (e: any) { - // Fallback to unformatted if prettier fails + // Fallback to advanced tag-based line splitting if prettier fails + const HTML_SPLIT_REGEX = /(<(script|style)\b[\s\S]*?<\/\2>||<[^'">]*(?:"[^"]*"[^'">]*|'[^']*'[^'">]*)*>)\s*(?=<)/gi; + bodyA = bodyA.replace(HTML_SPLIT_REGEX, "$1\n"); + bodyB = bodyB.replace(HTML_SPLIT_REGEX, "$1\n"); + } + } else if (contentType.includes("application/json") || route.endsWith(".json")) { + try { + bodyA = JSON.stringify(JSON.parse(bodyA), null, 2); + bodyB = JSON.stringify(JSON.parse(bodyB), null, 2); + } catch (e: any) { + // Fallback to raw text } } - + result.bodyA = bodyA; result.bodyB = bodyB; @@ -136,3 +190,4 @@ export async function compareRoute( return result; } + diff --git a/src/apphosting/compare/discover.spec.ts b/src/apphosting/compare/discover.spec.ts index a757c31db96..746095ecf98 100644 --- a/src/apphosting/compare/discover.spec.ts +++ b/src/apphosting/compare/discover.spec.ts @@ -59,4 +59,39 @@ describe("discoverRoutes", () => { const routes = await discoverRoutes(tempDir); expect(routes).to.deep.equal(["/", "/products?id=12"]); }); + + it("should discover routes from Next.js source app directory structure", async () => { + const appDir = path.join(tempDir, "src", "app"); + fs.ensureDirSync(path.join(appDir, "about")); + fs.ensureDirSync(path.join(appDir, "dashboard", "settings")); + fs.ensureDirSync(path.join(appDir, "blog", "[id]")); + fs.ensureDirSync(path.join(appDir, "(marketing)", "pricing")); + + fs.writeFileSync(path.join(appDir, "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "about", "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "dashboard", "settings", "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "blog", "[id]", "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "(marketing)", "pricing", "page.tsx"), "export default function P() {}"); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/about", "/blog/1", "/dashboard/settings", "/pricing"]); + }); + + it("should discover routes from Angular source TS routing modules", async () => { + const srcDir = path.join(tempDir, "src"); + fs.ensureDirSync(srcDir); + + const routingCode = ` + const routes: Routes = [ + { path: '', component: HomeComponent }, + { path: 'products', component: ProductsComponent }, + { path: 'user-profile', component: ProfileComponent }, + { path: '**', redirectTo: '' } + ]; + `; + fs.writeFileSync(path.join(srcDir, "app-routing.module.ts"), routingCode, "utf-8"); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/products", "/user-profile"]); + }); }); diff --git a/src/apphosting/compare/discover.ts b/src/apphosting/compare/discover.ts index a746db30373..5050edf5818 100644 --- a/src/apphosting/compare/discover.ts +++ b/src/apphosting/compare/discover.ts @@ -3,12 +3,125 @@ import * as path from "path"; import { logger } from "../../logger"; /** - * Discovers built routes in a project by checking Next.js manifests and local sitemaps. + * Recursively scans a directory for files matching a pattern. + */ +async function scanDir( + dir: string, + fileCallback: (filePath: string) => void | Promise, +): Promise { + if (!(await fs.pathExists(dir))) return; + const files = await fs.readdir(dir); + for (const file of files) { + const fullPath = path.join(dir, file); + const stat = await fs.stat(fullPath); + if (stat.isDirectory()) { + if ( + file !== "node-modules" && + file !== "node_modules" && + file !== ".git" && + file !== ".next" && + file !== ".angular" && + file !== "dist" + ) { + await scanDir(fullPath, fileCallback); + } + } else { + await fileCallback(fullPath); + } + } +} + +/** + * Discovers Next.js source routes from src/app, app, src/pages, or pages directory + */ +export async function discoverSourceRoutes(appPath: string): Promise { + const routes = new Set(); + + // Check App Router (src/app or app) + const appDirs = [path.join(appPath, "src", "app"), path.join(appPath, "app")]; + for (const appDir of appDirs) { + if (await fs.pathExists(appDir)) { + await scanDir(appDir, (filePath) => { + const base = path.basename(filePath); + if (/^(page|route)\.[jt]sx?$/.test(base)) { + const relDir = path.relative(appDir, path.dirname(filePath)); + let route = relDir ? "/" + relDir : "/"; + // Normalize dynamic params: [id] -> 1, [...catchall] -> 1 + route = route.replace(/\[\.\.\.[^\]]+\]/g, "1"); + route = route.replace(/\[[^\]]+\]/g, "1"); + // Remove Next.js route groups like (marketing) + route = route.replace(/\/\([^)]+\)/g, ""); + routes.add(route === "" ? "/" : route); + } + }); + } + } + + // Check Pages Router (src/pages or pages) + const pagesDirs = [path.join(appPath, "src", "pages"), path.join(appPath, "pages")]; + for (const pagesDir of pagesDirs) { + if (await fs.pathExists(pagesDir)) { + await scanDir(pagesDir, (filePath) => { + const ext = path.extname(filePath); + if ([".js", ".jsx", ".ts", ".tsx"].includes(ext)) { + const base = path.basename(filePath, ext); + if (["_app", "_document", "_error", "api"].includes(base) || filePath.includes("/api/")) { + return; + } + const relPath = path.relative(pagesDir, filePath); + let route = "/" + relPath.substring(0, relPath.length - ext.length); + if (route.endsWith("/index")) { + route = route.substring(0, route.length - 6); + } + // Normalize dynamic parameters + route = route.replace(/\[\.\.\.[^\]]+\]/g, "1"); + route = route.replace(/\[[^\]]+\]/g, "1"); + routes.add(route === "" ? "/" : route); + } + }); + } + } + + return Array.from(routes); +} + +/** + * Scans source TS files for Angular style route paths like `path: 'about'` + */ +export async function discoverAngularRoutes(appPath: string): Promise { + const routes = new Set(); + const srcDir = path.join(appPath, "src"); + if (await fs.pathExists(srcDir)) { + await scanDir(srcDir, async (filePath) => { + if (filePath.endsWith(".ts") && !filePath.endsWith(".spec.ts")) { + try { + const content = await fs.readFile(filePath, "utf-8"); + // Match path: 'about' or path: "about" + const pathRegex = /path\s*:\s*(['"])(.*?)\1/g; + let match; + while ((match = pathRegex.exec(content)) !== null) { + const val = match[2].trim(); + // Skip wildcards and dynamic parameter placeholders + if (val && !val.includes("**") && !val.startsWith(":") && !val.includes("/")) { + routes.add("/" + val); + } + } + } catch { + // Ignore read errors + } + } + }); + } + return Array.from(routes); +} + +/** + * Discovers built routes in a project by checking Next.js manifests, source pages, and sitemaps. */ export async function discoverRoutes(appPath: string): Promise { const routes = new Set(["/"]); - // 1. Next.js Manifest Parsing + // 1. Next.js Manifest Parsing (Local Build Output) const nextDir = path.join(appPath, ".next"); if (await fs.pathExists(nextDir)) { logger.info("Next.js build directory detected. Parsing manifests..."); @@ -37,7 +150,27 @@ export async function discoverRoutes(appPath: string): Promise { } } - // 2. Local Sitemap Parsing (Regex-based) + // 2. Next.js Source Tree Traversal + try { + const srcRoutes = await discoverSourceRoutes(appPath); + for (const r of srcRoutes) { + routes.add(r); + } + } catch (err) { + logger.debug(`Error scanning Next.js source routes: ${err}`); + } + + // 3. Angular Routing Scanner + try { + const angularRoutes = await discoverAngularRoutes(appPath); + for (const r of angularRoutes) { + routes.add(r); + } + } catch (err) { + logger.debug(`Error scanning Angular source routes: ${err}`); + } + + // 4. Local Sitemap Parsing const sitemapPaths = [ path.join(appPath, "public", "sitemap.xml"), path.join(appPath, "sitemap.xml"), diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts new file mode 100644 index 00000000000..7700e48ad42 --- /dev/null +++ b/src/apphosting/compare/server.ts @@ -0,0 +1,1014 @@ +import * as express from "express"; +import * as http from "http"; +import { logger } from "../../logger"; +import * as cache from "./cache"; +import * as compare from "./compare"; + +export function startServer(port: number): Promise { + return new Promise((resolve, reject) => { + const app = express(); + + app.use(express.json()); + + // API: List all recordings in compare-cache + app.get("/api/recordings", async (req, res) => { + try { + const recordings = await cache.listRecordings(); + res.json(recordings); + } catch (err: any) { + res.status(500).json({ error: err.message }); + } + }); + + // API: Compare two cached recordings of a test case + app.get("/api/compare", async (req, res) => { + const { testCase, variantA, variantB } = req.query; + if (!testCase || !variantA || !variantB) { + res.status(400).json({ error: "Missing testCase, variantA, or variantB query parameters" }); + return; + } + + try { + const recA = await cache.loadRecording(testCase as string, variantA as string); + const recB = await cache.loadRecording(testCase as string, variantB as string); + + const allRoutes = Array.from(new Set([ + ...Object.keys(recA.routes), + ...Object.keys(recB.routes) + ])).sort(); + + const results: compare.ComparisonResult[] = []; + for (const route of allRoutes) { + const resA = recA.routes[route]; + const resB = recB.routes[route]; + + if (!resA || !resB) { + results.push({ + route, + statusMatch: false, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 0.0, + bodyDiff: `Route missing on one variant: ${!resA ? recA.id : recB.id}`, + isBinary: false, + bodyA: resA?.body, + bodyB: resB?.body, + }); + continue; + } + + const compResult = await compare.compareRouteResponses(route, resA, resB); + + if (!resA.isBinary && !resB.isBinary) { + const diff = require("diff"); + const changes = diff.diffLines(compResult.bodyA || "", compResult.bodyB || ""); + // Filter/map to minimal JSON to keep payload clean + (compResult as any).diffChanges = changes.map((c: any) => ({ + value: c.value, + added: !!c.added, + removed: !!c.removed + })); + } + + results.push(compResult); + } + + res.json({ + testCase, + variantA: recA.id, + variantB: recB.id, + urlA: recA.url, + urlB: recB.url, + results + }); + } catch (err: any) { + res.status(500).json({ error: err.message }); + } + }); + + // API: Get N x N pairwise similarity matrix for a test case + app.get("/api/matrix", async (req, res) => { + const { testCase } = req.query; + if (!testCase) { + res.status(400).json({ error: "Missing testCase query parameter" }); + return; + } + + try { + const recordings = await cache.listRecordings(); + const variants = recordings[testCase as string]; + if (!variants || variants.length === 0) { + res.json({ testCase, variants: [], matrix: {} }); + return; + } + + // Preload all recordings to avoid repeated reads + const recMap: Record = {}; + for (const v of variants) { + recMap[v] = await cache.loadRecording(testCase as string, v); + } + + const matrix: Record> = {}; + + for (const vA of variants) { + matrix[vA] = {}; + for (const vB of variants) { + if (vA === vB) { + matrix[vA][vB] = 1.0; + continue; + } + + // Compute average body similarity across all shared routes + const recA = recMap[vA]; + const recB = recMap[vB]; + const allRoutes = Array.from(new Set([ + ...Object.keys(recA.routes), + ...Object.keys(recB.routes) + ])); + + let totalSimilarity = 0; + let countedRoutes = 0; + + for (const route of allRoutes) { + const resA = recA.routes[route]; + const resB = recB.routes[route]; + + if (!resA || !resB) { + continue; + } + + const compResult = await compare.compareRouteResponses(route, resA, resB); + totalSimilarity += compResult.bodySimilarity; + countedRoutes++; + } + + matrix[vA][vB] = countedRoutes > 0 ? (totalSimilarity / countedRoutes) : 0.0; + } + } + + res.json({ + testCase, + variants, + matrix + }); + } catch (err: any) { + res.status(500).json({ error: err.message }); + } + }); + + // Serve Single Page Application dashboard + app.get("/", (req, res) => { + res.send(getDashboardHtml()); + }); + + const server = http.createServer(app); + server.on("error", reject); + server.listen(port, () => { + logger.info(`\n🚀 Parity Visualization Dashboard running at http://localhost:${port}`); + logger.info("Press Ctrl+C to stop the server.\n"); + }); + + const cleanUp = () => { + server.close(() => { + resolve(); + }); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + }); +} + +function getDashboardHtml(): string { + return ` + + + + + + Parity Comparison Dashboard + + + + + + + + + +
    +

    🚀 Firebase App Hosting Parity Dashboard

    +
    Connected
    +
    + +
    + +
    +
    +
    Test Cases
    +
    + +
    +
    + + +
    + + +
    + + + + + + + + + + +
    + + + + +
    Select a test case and variants to start comparing.
    +
    +
    +
    + + + + + `; +} diff --git a/src/apphosting/compare/slots.spec.ts b/src/apphosting/compare/slots.spec.ts index d17880881c7..5d0b5669982 100644 --- a/src/apphosting/compare/slots.spec.ts +++ b/src/apphosting/compare/slots.spec.ts @@ -9,7 +9,7 @@ import { acquireComparisonSlot, releaseComparisonSlot } from "./slots"; describe("Comparison Slots Manager", () => { let listBackendsStub: sinon.SinonStub; - let updateBackendStub: sinon.SinonStub; + let patchBackendStub: sinon.SinonStub; let createBackendStub: sinon.SinonStub; let listFirebaseAppsStub: sinon.SinonStub; let createWebAppStub: sinon.SinonStub; @@ -17,7 +17,7 @@ describe("Comparison Slots Manager", () => { beforeEach(() => { listBackendsStub = sinon.stub(apphosting, "listBackends"); - updateBackendStub = sinon.stub(apphosting, "updateBackend"); + patchBackendStub = sinon.stub(apphosting.client, "patch").resolves({ body: { name: "op-name" } } as any); createBackendStub = sinon.stub(backendHelper, "createBackend"); listFirebaseAppsStub = sinon.stub(apps, "listFirebaseApps"); createWebAppStub = sinon.stub(apps, "createWebApp"); @@ -42,14 +42,13 @@ describe("Comparison Slots Manager", () => { ], }); listFirebaseAppsStub.resolves([{ appId: "web-app-123", displayName: "existing-app" }]); - updateBackendStub.resolves({ name: "op-name" }); const slot = await acquireComparisonSlot("aryanf-test", "us-central1", 2); expect(slot.index).to.equal(1); expect(slot.backendIds[0]).to.equal("compare-slot-1-0"); expect(slot.backendIds[1]).to.equal("compare-slot-1-1"); - expect(updateBackendStub.callCount).to.equal(2); + expect(patchBackendStub.callCount).to.equal(2); // Updates labels twice expect(createBackendStub.callCount).to.equal(0); }); @@ -57,7 +56,6 @@ describe("Comparison Slots Manager", () => { listBackendsStub.resolves({ backends: [] }); listFirebaseAppsStub.resolves([{ appId: "web-app-123", displayName: "existing-app" }]); createBackendStub.resolves({ name: "backend-resource" }); - updateBackendStub.resolves({ name: "op-name" }); const slot = await acquireComparisonSlot("aryanf-test", "us-central1", 2); expect(slot.index).to.equal(1); @@ -66,7 +64,7 @@ describe("Comparison Slots Manager", () => { it("should throw if all slots are locked/busy", async () => { const busyBackends = []; - for (let i = 1; i <= 5; i++) { + for (let i = 1; i <= 10; i++) { busyBackends.push( { name: `projects/test/locations/us-central1/backends/compare-slot-${i}-0`, @@ -81,7 +79,7 @@ describe("Comparison Slots Manager", () => { listBackendsStub.resolves({ backends: busyBackends }); await expect(acquireComparisonSlot("aryanf-test", "us-central1", 2)).to.be.rejectedWith( - "All 5 comparison slots are currently in use or project backend limits exceeded", + "All 10 comparison slots are currently in use or project backend limits exceeded", ); }); }); diff --git a/src/apphosting/compare/slots.ts b/src/apphosting/compare/slots.ts index 8fa6ec3a2ff..9d3f0c09f7a 100644 --- a/src/apphosting/compare/slots.ts +++ b/src/apphosting/compare/slots.ts @@ -20,7 +20,7 @@ const apphostingPollerOptions = { maxBackoff: 10_000, }; -async function updateBackendAndPoll( +async function updateBackendLabels( projectId: string, location: string, backendId: string, @@ -101,14 +101,14 @@ export async function acquireComparisonSlot( logger.info(`Provisioning backend ${backendId}...`); await createBackend(projectId, location, backendId, null, undefined, webAppId); updatePromises.push( - updateBackendAndPoll(projectId, location, backendId, { + updateBackendLabels(projectId, location, backendId, { status: "busy", type: "comparison-sandbox", }), ); } else { updatePromises.push( - updateBackendAndPoll(projectId, location, backendId, { + updateBackendLabels(projectId, location, backendId, { ...backend.labels, status: "busy", }), @@ -147,7 +147,7 @@ export async function releaseComparisonSlot( const backend = backendsList.find((b) => b.name.endsWith(backendId)); if (backend) { updatePromises.push( - updateBackendAndPoll(projectId, location, backendId, { + updateBackendLabels(projectId, location, backendId, { ...backend.labels, status: "idle", }), diff --git a/src/apphosting/compare/suite.spec.ts b/src/apphosting/compare/suite.spec.ts index 9933752715c..7d37381e232 100644 --- a/src/apphosting/compare/suite.spec.ts +++ b/src/apphosting/compare/suite.spec.ts @@ -2,62 +2,48 @@ import * as path from "path"; import * as fs from "fs-extra"; import { expect } from "chai"; import * as sinon from "sinon"; -import * as gcs from "../../gcp/storage"; +import * as childProcess from "child_process"; import * as apphosting from "../../gcp/apphosting"; -import * as rolloutHelper from "../rollout"; -import * as deployUtil from "../../deploy/apphosting/util"; import * as projectNumberHelper from "../../getProjectNumber"; import * as secretsManager from "./secrets"; -import * as slotsManager from "./slots"; import * as discoverManager from "./discover"; import { Crawler } from "./crawler"; import * as compareManager from "./compare"; +import * as poller from "../../operation-poller"; import * as reporterManager from "./reporter"; -import * as lifecycle from "./lifecycle"; -import * as localBuildsModule from "../localbuilds"; +import * as fetchModule from "node-fetch"; +import * as cache from "./cache"; import { runCompareSuite } from "./suite"; describe("runCompareSuite Orchestrator", () => { let tempDir: string; - let dummyZip: string; - - let upsertBucketStub: sinon.SinonStub; - let createArchiveStub: sinon.SinonStub; - let uploadObjectStub: sinon.SinonStub; - let orchestrateRolloutStub: sinon.SinonStub; + let execStub: sinon.SinonStub; let getProjectNumberStub: sinon.SinonStub; let setupSecretsStub: sinon.SinonStub; let cleanupSecretsStub: sinon.SinonStub; - let acquireSlotStub: sinon.SinonStub; - let releaseSlotStub: sinon.SinonStub; - let validateProjectStub: sinon.SinonStub; - let runGarbageCollectionStub: sinon.SinonStub; let discoverRoutesStub: sinon.SinonStub; - let compareRouteStub: sinon.SinonStub; + let compareRouteResponsesStub: sinon.SinonStub; let generateReportStub: sinon.SinonStub; let getBackendStub: sinon.SinonStub; let crawlStub: sinon.SinonStub; let getRoutesStub: sinon.SinonStub; + let pollOperationStub: sinon.SinonStub; + let saveRecordingStub: sinon.SinonStub; + let loadRecordingStub: sinon.SinonStub; + let fetchStub: sinon.SinonStub; beforeEach(() => { tempDir = path.join(process.cwd(), "scratch-test-suite-" + Math.random().toString(36).substring(7)); fs.ensureDirSync(tempDir); - dummyZip = path.join(tempDir, "archive.zip"); - fs.writeFileSync(dummyZip, "empty content"); - upsertBucketStub = sinon.stub(gcs, "upsertBucket").resolves("bucket-123"); - createArchiveStub = sinon.stub(deployUtil, "createSourceDeployArchive").resolves(dummyZip); - uploadObjectStub = sinon.stub(gcs, "uploadObject").resolves({ bucket: "bucket-123", object: "obj-123", generation: "1" }); - orchestrateRolloutStub = sinon.stub(rolloutHelper, "orchestrateRollout").resolves({} as any); + execStub = sinon.stub(childProcess, "exec").yields(null, { stdout: "success", stderr: "" }); + pollOperationStub = sinon.stub(poller, "pollOperation").resolves(); getProjectNumberStub = sinon.stub(projectNumberHelper, "getProjectNumber").resolves("12345"); setupSecretsStub = sinon.stub(secretsManager, "setupSandboxSecrets").resolves([]); cleanupSecretsStub = sinon.stub(secretsManager, "cleanupSandboxSecrets").resolves(); - acquireSlotStub = sinon.stub(slotsManager, "acquireComparisonSlot").resolves({ index: 1, backendIds: ["compare-slot-1-a", "compare-slot-1-b"] } as any); - releaseSlotStub = sinon.stub(slotsManager, "releaseComparisonSlot").resolves(); - validateProjectStub = sinon.stub(lifecycle, "validateProject").returns(); - runGarbageCollectionStub = sinon.stub(lifecycle, "runGarbageCollection").resolves(); discoverRoutesStub = sinon.stub(discoverManager, "discoverRoutes").resolves(["/"]); - compareRouteStub = sinon.stub(compareManager, "compareRoute").resolves({ + + compareRouteResponsesStub = sinon.stub(compareManager, "compareRouteResponses").resolves({ route: "/", statusMatch: true, headerMismatches: [], @@ -65,12 +51,32 @@ describe("runCompareSuite Orchestrator", () => { bodySimilarity: 1.0, bodyDiff: "", isBinary: false - }); + } as any); + generateReportStub = sinon.stub(reporterManager, "generateReport").resolves(); getBackendStub = sinon.stub(apphosting, "getBackend").resolves({ uri: "https://my-backend.com" } as any); crawlStub = sinon.stub(Crawler.prototype, "crawl").resolves(); getRoutesStub = sinon.stub(Crawler.prototype, "getRoutes").returns(["/about"]); + + saveRecordingStub = sinon.stub(cache, "saveRecording").resolves(); + loadRecordingStub = sinon.stub(cache, "loadRecording").resolves({ + id: "mock", + testCaseName: "mock", + timestamp: "mock", + url: "mock", + routes: {} + }); + + fetchStub = sinon.stub(fetchModule, "default").resolves({ + status: 200, + headers: { + get: (k: string) => k === "content-type" ? "text/html" : "", + forEach: (fn: (v: string, k: string) => void) => fn("text/html", "content-type"), + }, + buffer: async () => Buffer.from("mock body"), + text: async () => "mock body", + } as any); }); afterEach(() => { @@ -79,68 +85,69 @@ describe("runCompareSuite Orchestrator", () => { }); it("should coordinate the full deployment, crawling, comparison, and reporting pipeline", async () => { + const backendIds = ["compare-slot-1-0", "compare-slot-1-1"]; await runCompareSuite( "aryanf-test", "us-central1", + backendIds, + 1, + "Test-Case-A", [ - { path: "/app/path-a" }, - { path: "/app/path-b" } + { path: tempDir }, + { path: tempDir } ] ); - expect(acquireSlotStub.callCount).to.equal(1); expect(setupSecretsStub.callCount).to.equal(1); - expect(upsertBucketStub.callCount).to.equal(1); - expect(createArchiveStub.callCount).to.equal(2); - expect(uploadObjectStub.callCount).to.equal(2); - expect(orchestrateRolloutStub.callCount).to.equal(2); - expect(discoverRoutesStub.callCount).to.equal(1); - expect(crawlStub.callCount).to.equal(1); + expect(execStub.callCount).to.equal(2); // Deploys twice + expect(discoverRoutesStub.callCount).to.equal(2); + expect(crawlStub.callCount).to.equal(2); - expect(compareRouteStub.callCount).to.equal(1); - expect(compareRouteStub.firstCall.args[0]).to.equal("/"); - + expect(compareRouteResponsesStub.callCount).to.equal(2); // for "/" and "/about" expect(generateReportStub.callCount).to.equal(1); expect(cleanupSecretsStub.callCount).to.equal(1); - expect(releaseSlotStub.callCount).to.equal(1); }); - it("should support running with local builds enabled for one of the backends", async () => { - const localBuildStub = sinon.stub(localBuildsModule, "localBuild").resolves({ - outputFiles: ["index.html"], - buildConfig: { runCommand: "npm run start" } - }); - - const createTarStub = sinon.stub(deployUtil, "createLocalBuildTarArchive").resolves(dummyZip); - + it("should support running with local builds enabled", async () => { + const backendIds = ["compare-slot-1-0", "compare-slot-1-1"]; await runCompareSuite( "aryanf-test", "us-central1", + backendIds, + 1, + "Test-Case-B", [ - { path: "/app/path-a", localBuild: false }, - { path: "/app/path-b", localBuild: true } + { path: tempDir, localBuild: false }, + { path: tempDir, localBuild: true } ] ); - expect(createArchiveStub.callCount).to.equal(1); - expect(localBuildStub.callCount).to.equal(1); - expect(createTarStub.callCount).to.equal(1); + expect(execStub.callCount).to.equal(2); + // Verifies one of them was deployed with localBuild experiment prefix + const firstCallCmd = execStub.firstCall.args[0]; + const secondCallCmd = execStub.secondCall.args[0]; + + expect(firstCallCmd).to.not.include("FIREBASE_CLI_EXPERIMENTS=apphostinglocalbuilds"); + expect(secondCallCmd).to.include("FIREBASE_CLI_EXPERIMENTS=apphostinglocalbuilds"); + }); - expect(uploadObjectStub.callCount).to.equal(2); - expect(orchestrateRolloutStub.callCount).to.equal(2); it("should support runtime version patching for backends", async () => { + const patchStub = sinon.stub(apphosting.client, "patch").resolves({ body: { name: "operation-123" } } as any); + const backendIds = ["compare-slot-1-0", "compare-slot-1-1"]; + await runCompareSuite( "aryanf-test", "us-central1", + backendIds, + 1, + "Test-Case-C", [ - { path: "/app/path-a", runtime: "nodejs20" }, - { path: "/app/path-b", runtime: "nodejs22" } + { path: tempDir, runtime: "nodejs20" }, + { path: tempDir, runtime: "nodejs22" } ] ); - expect(createArchiveStub.callCount).to.equal(2); - expect(uploadObjectStub.callCount).to.equal(2); - expect(orchestrateRolloutStub.callCount).to.equal(2); + expect(patchStub.callCount).to.equal(2); // Patch both runtimes + expect(execStub.callCount).to.equal(2); }); }); -}); diff --git a/src/apphosting/compare/suite.ts b/src/apphosting/compare/suite.ts index 485a9b6db8d..bad3505ca82 100644 --- a/src/apphosting/compare/suite.ts +++ b/src/apphosting/compare/suite.ts @@ -1,11 +1,6 @@ import * as path from "path"; import * as fs from "fs-extra"; -import * as crypto from "crypto"; -import * as os from "os"; -import * as gcs from "../../gcp/storage"; import * as apphosting from "../../gcp/apphosting"; -import * as rollout from "../rollout"; -import * as deployUtil from "../../deploy/apphosting/util"; import { getProjectNumber } from "../../getProjectNumber"; import { apphostingOrigin } from "../../api"; import * as secrets from "./secrets"; @@ -15,10 +10,9 @@ import * as discover from "./discover"; import { Crawler } from "./crawler"; import * as compare from "./compare"; import * as reporter from "./reporter"; -import { localBuild } from "../localbuilds"; -import * as fsAsync from "../../fsAsync"; import * as poller from "../../operation-poller"; import { logger } from "../../logger"; +import { FirebaseError } from "../../error"; const apphostingPollerOptions = { apiOrigin: apphostingOrigin(), @@ -28,39 +22,20 @@ const apphostingPollerOptions = { timeout: 120000, // 2 minutes }; -async function prepareLocalBuildDir( - rootDir: string, - scratchDir: string, - backendId: string, -): Promise { - const ignore = deployUtil.resolveIgnorePatterns({ backendId, rootDir: "/", ignore: [] }); - fs.rmSync(scratchDir, { recursive: true, force: true }); - fs.mkdirSync(scratchDir, { recursive: true }); - const filesToCopy = await fsAsync.readdirRecursive({ - path: rootDir, - ignoreStrings: ignore, - supportGitIgnore: true, - }); - for (const file of filesToCopy) { - const relativePath = path.relative(rootDir, file.name); - const destPath = path.join(scratchDir, relativePath); - fs.mkdirSync(path.dirname(destPath), { recursive: true }); - fs.copyFileSync(file.name, destPath); - } -} +import * as cp from "child_process"; +import * as util from "util"; + +export const createdConfigs = new Set(); async function deployToBackend( projectId: string, location: string, backendId: string, appPath: string, - bucketName: string, + bucketName: string, // Kept for backwards compatibility but unused useLocalBuild: boolean, runtimeVersion?: string, ): Promise { - let archivePath: string; - let buildInput: any; - if (runtimeVersion) { logger.info(`Patching runtime version for backend ${backendId} to ${runtimeVersion}...`); const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; @@ -76,89 +51,39 @@ async function deployToBackend( }); } - if (useLocalBuild) { - logger.info(`Running local build for slot backend ${backendId}...`); - const pathHash = crypto.createHash("md5").update(appPath).digest("hex").substring(0, 8); - const scratchDir = path.join(os.tmpdir(), `apphosting-local-build-${backendId}-${pathHash}`); - - await prepareLocalBuildDir(appPath, scratchDir, backendId); - - const { outputFiles, buildConfig } = await localBuild( - projectId, - scratchDir, - {}, - { nonInteractive: true }, - ); - - archivePath = await deployUtil.createLocalBuildTarArchive( - { backendId, rootDir: "/", ignore: [] }, - scratchDir, - outputFiles, - ); - - logger.info(`Uploading local build bundle for ${backendId}...`); - await gcs.uploadObject( - { file: archivePath, stream: fs.createReadStream(archivePath) }, - bucketName, - gcs.ContentType.TAR, - ); + const tempConfigName = `firebase-compare-${backendId}.json`; + const configPath = path.join(appPath, tempConfigName); + createdConfigs.add(configPath); - const uri = `gs://${bucketName}/${path.basename(archivePath)}`; - buildInput = { - config: buildConfig, - source: { - locallyBuilt: { - userStorageUri: uri, - rootDirectory: "/", - runCommand: buildConfig.runCommand, - env: buildConfig.env, - }, - }, - }; - } else { - logger.info(`Packaging source archive for ${backendId}...`); - archivePath = await deployUtil.createSourceDeployArchive( - { backendId, rootDir: "/", ignore: [] }, - appPath, - ); + const firebaseJson = { + apphosting: [ + { + source: ".", + backendId: backendId, + localBuild: useLocalBuild + } + ] + }; - logger.info(`Uploading source archive for ${backendId}...`); - await gcs.uploadObject( - { file: archivePath, stream: fs.createReadStream(archivePath) }, - bucketName, - gcs.ContentType.ZIP, - ); + await fs.writeJson(configPath, firebaseJson, { spaces: 2 }); - const uri = `gs://${bucketName}/${path.basename(archivePath)}`; - buildInput = { - source: { - archive: { - userStorageUri: uri, - rootDirectory: "/", - }, - }, - }; - } - - logger.info(`Triggering rollout for backend ${backendId}...`); - await rollout.orchestrateRollout({ - projectId, - location, - backendId, - buildInput, - }); - - // Wait until the backend is fully done reconciling after the rollout. - logger.info(`Waiting for backend ${backendId} to finish reconciling...`); - let backendIsReconciling = true; - while (backendIsReconciling) { - const b = await apphosting.client.get( - `projects/${projectId}/locations/${location}/backends/${backendId}`, - ); - backendIsReconciling = !!b.body.reconciling; - if (backendIsReconciling) { - await new Promise((r) => setTimeout(r, 2000)); - } + try { + logger.info(`Triggering CLI deploy for backend ${backendId} (localBuild: ${useLocalBuild})...`); + // Run exactly the same deployment path as a customer + const experimentPrefix = useLocalBuild ? "FIREBASE_CLI_EXPERIMENTS=apphostinglocalbuilds " : ""; + const binPath = process.argv[1] || path.resolve(__dirname, "../../../bin/firebase.js"); + + const cmd = `${experimentPrefix}node "${binPath}" deploy --only apphosting:${backendId} --project ${projectId} --config ${tempConfigName} --non-interactive --allow-local-build-secrets`; + + const execAsync = util.promisify(cp.exec); + const { stdout, stderr } = await execAsync(cmd, { cwd: appPath, maxBuffer: 1024 * 1024 * 100 }); + logger.debug(`Deploy output for ${backendId}:\n${stdout}`); + } catch (err: any) { + logger.error(`Deploy for ${backendId} failed!\nSTDOUT:\n${err.stdout}\nSTDERR:\n${err.stderr}`); + throw new FirebaseError(`Failed to deploy variant to ${backendId}.`, { original: err }); + } finally { + await fs.remove(configPath); + createdConfigs.delete(configPath); } } @@ -172,158 +97,216 @@ export interface VariantConfig { /** * */ +import * as cache from "./cache"; +import fetch from "node-fetch"; + +async function recordVariant( + testCaseName: string, + variantId: string, + url: string, + appPath: string, +): Promise { + const discoveredStaticRoutes = await discover.discoverRoutes(appPath); + const allRoutesSet = new Set(discoveredStaticRoutes); + + logger.info(`Crawling Variant ${variantId} at ${url} for dynamic link discovery...`); + const crawler = new Crawler(url); + await crawler.crawl(); + crawler.getRoutes().forEach((r) => allRoutesSet.add(r)); + + const routes: Record = {}; + + for (const route of allRoutesSet) { + logger.debug(`Recording route ${route}...`); + try { + const res = await fetch(`${url}${route}`, { + redirect: "manual" as const, + headers: { "User-Agent": "FirebaseCompareCrawler/1.0" }, + }); + + const contentType = res.headers.get("content-type") || ""; + const isBinary = isBinaryContentType(contentType); + + const headers: Record = {}; + res.headers.forEach((val, key) => { headers[key] = val; }); + + routes[route] = { + status: res.status, + headers, + isBinary, + body: isBinary ? (await res.buffer()).toString("base64") : await res.text(), + }; + } catch (err) { + logger.warn(`Failed to record route ${route}: ${err}`); + } + } + + return { + id: variantId, + testCaseName, + timestamp: new Date().toISOString(), + url, + routes, + }; +} + +function isBinaryContentType(contentType: string): boolean { + const normalized = contentType.toLowerCase(); + return [ + "image/", + "application/pdf", + "application/zip", + "application/octet-stream", + ].some((type) => normalized.includes(type)); +} + export async function runCompareSuite( projectId: string, location: string, + backendIds: string[], + slotIndex: number, + testCaseName: string, variants: VariantConfig[], options: { outputDir?: string; + recordOnly?: boolean; + compareOnly?: boolean; } = {}, ): Promise { - lifecycle.validateProject(projectId); - await lifecycle.runGarbageCollection(projectId, location); - - const projectNumber = await getProjectNumber({ projectId }); - - // 1. Acquire Comparison Slot for N variants - const slot = await slots.acquireComparisonSlot(projectId, location, variants.length); - logger.info( - `Acquired Comparison Slot ${slot.index} with ${variants.length} backends: ${slot.backendIds.join(", ")}`, - ); - - let secretsMappings: secrets.SecretMapping[][] = []; + const recordings: cache.VariantRecording[] = []; - const cleanUp = async () => { - logger.warn("\nInterrupted. Restoring slot and deleting mock secrets..."); - for (const mapping of secretsMappings) { - await secrets.cleanupSandboxSecrets(projectId, mapping); - } - await slots.releaseComparisonSlot(projectId, location, slot.index, variants.length); - process.exit(1); - }; - process.on("SIGINT", cleanUp); - process.on("SIGTERM", cleanUp); - - try { - // 2. Setup mock secrets per unique codebase path - const uniquePaths = Array.from(new Set(variants.map((v) => v.path))); - secretsMappings = await Promise.all( - uniquePaths.map((uniquePath) => { - const pathBackendIds = variants - .map((v, i) => (v.path === uniquePath ? slot.backendIds[i] : null)) - .filter((id): id is string => id !== null); - - return secrets.setupSandboxSecrets( - projectId, - location, - uniquePath, - slot.index, - pathBackendIds - ); - }) - ); - - // 3. Package, Upload and Deploy Source for all N variants - const bucketName = `firebaseapphosting-sources-${projectNumber}-${location.toLowerCase()}`; - await gcs.upsertBucket({ - product: "apphosting", - createMessage: `Ensuring bucket for comparison slot sources in ${location}...`, - projectId, - req: { - baseName: bucketName, - purposeLabel: `apphosting-source-${location.toLowerCase()}`, - location, - lifecycle: { - rule: [ - { - action: { type: "Delete" }, - condition: { age: 30 }, - }, - ], - }, - }, - }); + if (!options.compareOnly) { + // === RECORD PHASE === + const projectNumber = await getProjectNumber({ projectId }); + let secretsMappings: secrets.SecretMapping[][] = []; - await Promise.all( - variants.map((v, i) => - deployToBackend( + const cleanUp = async () => { + logger.warn("\nInterrupted. Deleting mock secrets..."); + for (const mapping of secretsMappings) { + await secrets.cleanupSandboxSecrets(projectId, mapping); + } + process.exit(1); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + + try { + // Setup secrets + const uniquePaths = Array.from(new Set(variants.map((v) => v.path))); + secretsMappings = await Promise.all( + uniquePaths.map((uniquePath) => { + const pathBackendIds = variants + .map((v, i) => (v.path === uniquePath ? backendIds[i] : null)) + .filter((id): id is string => id !== null); + + return secrets.setupSandboxSecrets( + projectId, + location, + uniquePath, + slotIndex, + pathBackendIds + ); + }) + ); + + // Deploy variants sequentially + for (let i = 0; i < variants.length; i++) { + const v = variants[i]; + await deployToBackend( projectId, location, - slot.backendIds[i], + backendIds[i], v.path, - bucketName, + "", // bucketName !!v.localBuild, v.runtime, - ), - ), - ); - - logger.info("All N-Way Rollouts completed successfully!"); - - // 4. Retrieve Live URLs for all variants - const backendDataList = await Promise.all( - slot.backendIds.map((id) => apphosting.getBackend(projectId, location, id)), - ); - const urls = backendDataList.map((b) => (b.uri.startsWith("http") ? b.uri : `https://${b.uri}`)); + ); + } - urls.forEach((url, i) => { - logger.info(`Variant ${variants[i].id || i} URL: ${url}`); - }); + logger.info("All rollouts completed successfully!"); - // 5. Route Discovery & Crawling across all variants - const allRoutesSet = new Set(); + // Retrieve URLs and Record + const backendDataList = await Promise.all( + backendIds.map((id) => apphosting.getBackend(projectId, location, id)), + ); + const urls = backendDataList.map((b) => (b.uri.startsWith("http") ? b.uri : `https://${b.uri}`)); + for (let i = 0; i < variants.length; i++) { + const v = variants[i]; + const url = urls[i]; + const record = await recordVariant(testCaseName, v.id || String(i), url, v.path); + await cache.saveRecording(record); + recordings.push(record); + } + } finally { + process.off("SIGINT", cleanUp); + process.off("SIGTERM", cleanUp); + for (const mapping of secretsMappings) { + await secrets.cleanupSandboxSecrets(projectId, mapping); + } + } + } else { + // === LOAD RECORDINGS FROM CACHE === + logger.info(`Loading cached recordings for test case "${testCaseName}"...`); for (let i = 0; i < variants.length; i++) { const v = variants[i]; - const url = urls[i]; - - const discoveredStaticRoutes = await discover.discoverRoutes(v.path); - discoveredStaticRoutes.forEach((r) => allRoutesSet.add(r)); - - logger.info(`Crawling Variant ${v.id || i} for dynamic link discovery...`); - const crawler = new Crawler(url); - await crawler.crawl(); - const crawledRoutes = crawler.getRoutes(); - crawledRoutes.forEach((r) => allRoutesSet.add(r)); + const record = await cache.loadRecording(testCaseName, v.id || String(i)); + recordings.push(record); } + } - const allRoutes = Array.from(allRoutesSet).sort(); - logger.info(`Total unique routes discovered across matrix: ${allRoutes.length}`); - - // 6. Report Generation for all unique pairs (Matrix Diffing) - for (let i = 0; i < variants.length; i++) { - for (let j = i + 1; j < variants.length; j++) { - logger.info( - `\nGenerating Comparison Report: ${variants[i].id || i} vs ${variants[j].id || j}...`, - ); + if (options.recordOnly) { + logger.info("Record phase complete. Skipping comparison as requested."); + return; + } - const results: compare.ComparisonResult[] = []; - for (const route of allRoutes) { - const res = await compare.compareRoute(route, urls[i], urls[j]); - results.push(res); + // === COMPARE PHASE === + logger.info("Starting pairwise comparison of recorded variants..."); + for (let i = 0; i < recordings.length; i++) { + for (let j = i + 1; j < recordings.length; j++) { + const recA = recordings[i]; + const recB = recordings[j]; + logger.info(`\nGenerating Comparison Report: ${recA.id} vs ${recB.id}...`); + + const allRoutes = Array.from(new Set([ + ...Object.keys(recA.routes), + ...Object.keys(recB.routes) + ])).sort(); + + const results: compare.ComparisonResult[] = []; + for (const route of allRoutes) { + const resA = recA.routes[route]; + const resB = recB.routes[route]; + + if (!resA || !resB) { + results.push({ + route, + statusMatch: false, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 0.0, + bodyDiff: `Route missing on one variant: ${!resA ? recA.id : recB.id}`, + isBinary: false + }); + continue; } - const pairOutputDir = options.outputDir - ? path.join(options.outputDir, `${variants[i].id || i}-vs-${variants[j].id || j}`) - : undefined; - - await reporter.generateReport( - projectId, - location, - slot.backendIds[i], - slot.backendIds[j], - results, - pairOutputDir, - ); + const res = await compare.compareRouteResponses(route, resA, resB); + results.push(res); } - } - } finally { - process.off("SIGINT", cleanUp); - process.off("SIGTERM", cleanUp); - for (const mapping of secretsMappings) { - await secrets.cleanupSandboxSecrets(projectId, mapping); + const pairOutputDir = options.outputDir + ? path.join(options.outputDir, `${recA.id}-vs-${recB.id}`) + : undefined; + + await reporter.generateReport( + projectId, + location, + recA.id, + recB.id, + results, + pairOutputDir, + ); } - await slots.releaseComparisonSlot(projectId, location, slot.index, variants.length); } } diff --git a/src/commands/apphosting-compare-suite.ts b/src/commands/apphosting-compare-suite.ts index 8668aaa67e3..e3452c8066f 100644 --- a/src/commands/apphosting-compare-suite.ts +++ b/src/commands/apphosting-compare-suite.ts @@ -2,7 +2,9 @@ import { Command } from "../command"; import { Options } from "../options"; import { needProjectId } from "../projectUtils"; import { requireAuth } from "../requireAuth"; -import { runCompareSuite } from "../apphosting/compare/suite"; +import * as suiteModule from "../apphosting/compare/suite"; +import * as lifecycle from "../apphosting/compare/lifecycle"; +import * as slots from "../apphosting/compare/slots"; import { FirebaseError } from "../error"; import * as fs from "fs-extra"; import * as path from "path"; @@ -21,8 +23,19 @@ export const command = new Command("apphosting:compare-suite") "directory to output comparison report files", "./compare-report", ) + .option("--record-only", "only deploy variants and record their output, skipping diffing") + .option("--compare-only", "run comparisons based on previously cached recordings, skipping deployment") + .option("--serve", "spin up the localhost comparison viewer dashboard") + .option("--port ", "port to run the localhost comparison viewer on", "3000") .before(requireAuth) .action(async (options: Options) => { + if (options.serve) { + const { startServer } = require("../apphosting/compare/server"); + const port = parseInt(options.port as string, 10) || 3000; + await startServer(port); + return; + } + const projectId = needProjectId(options); const location = options.location as string; const configPath = options.suiteConfig as string; @@ -42,23 +55,87 @@ export const command = new Command("apphosting:compare-suite") throw new FirebaseError("Suite config must be a JSON array of test cases."); } - logger.info(`Starting suite of ${suite.length} comparison tests...`); - for (const testCase of suite) { - logger.info(`\nRunning test case: ${testCase.name || "Unnamed Test"}`); - const caseOutputDir = path.join(options.outputDir as string, testCase.name || "unnamed"); + lifecycle.validateProject(projectId); - if (!testCase.variants || testCase.variants.length < 2) { - throw new FirebaseError( - `Test case ${testCase.name} must have a "variants" array with at least 2 configurations.`, - ); + // === OPTION A: COMPARE ONLY (NO SLOTS / DEPLOYMENTS) === + if (options.compareOnly) { + logger.info(`Starting comparison run for ${suite.length} test cases using cache...`); + for (const testCase of suite) { + logger.info(`\nComparing test case: ${testCase.name || "Unnamed Test"}`); + const caseOutputDir = path.join(options.outputDir as string, testCase.name || "unnamed"); + + try { + await suiteModule.runCompareSuite(projectId, location, [], 0, testCase.name, testCase.variants, { + outputDir: caseOutputDir, + compareOnly: true, + }); + } catch (err: any) { + logger.error(`Parity run for ${testCase.name} failed: ${err.message}`); + } } + return; + } + + // === OPTION B: RECORD (Requires locking GCM Slot) === + await lifecycle.runGarbageCollection(projectId, location); + + // Compute max variants to acquire a slot large enough + const maxVariants = Math.max(...suite.map((tc: any) => tc.variants?.length || 0)); + if (maxVariants < 2) { + throw new FirebaseError("All test cases must have at least 2 variants."); + } + + const slot = await slots.acquireComparisonSlot(projectId, location, maxVariants); + logger.info(`Acquired Comparison Slot ${slot.index} globally for the suite run.`); + + const cleanUp = async () => { + logger.warn("\nInterrupted. Restoring comparison slot lock and cleaning up temp files..."); + for (const configPath of suiteModule.createdConfigs) { + try { + await fs.remove(configPath); + } catch (e) {} + } + await slots.releaseComparisonSlot(projectId, location, slot.index, maxVariants); + process.exit(1); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + + try { + logger.info(`Starting suite of ${suite.length} comparison tests...`); + for (const testCase of suite) { + logger.info(`\nRunning test case: ${testCase.name || "Unnamed Test"}`); + const caseOutputDir = path.join(options.outputDir as string, testCase.name || "unnamed"); + + if (!testCase.variants || testCase.variants.length < 2) { + logger.error(`Skipping test case ${testCase.name}: must have at least 2 configurations.`); + continue; + } + + // Slice slot.backendIds to match the current testCase's variant count + const caseBackendIds = slot.backendIds.slice(0, testCase.variants.length); - try { - await runCompareSuite(projectId, location, testCase.variants, { - outputDir: caseOutputDir, - }); - } catch (err: any) { - logger.error(`Matrix execution for ${testCase.name} failed: ${err.message}`); + try { + await suiteModule.runCompareSuite( + projectId, + location, + caseBackendIds, + slot.index, + testCase.name, + testCase.variants, + { + outputDir: caseOutputDir, + recordOnly: !!options.recordOnly, + compareOnly: false, + } + ); + } catch (err: any) { + logger.error(`Matrix execution for ${testCase.name} failed: ${err.message}`); + } } + } finally { + process.off("SIGINT", cleanUp); + process.off("SIGTERM", cleanUp); + await slots.releaseComparisonSlot(projectId, location, slot.index, maxVariants); } }); diff --git a/src/commands/apphosting-compare.ts b/src/commands/apphosting-compare.ts index d7be3520f67..083a75b76a0 100644 --- a/src/commands/apphosting-compare.ts +++ b/src/commands/apphosting-compare.ts @@ -2,8 +2,12 @@ import { Command } from "../command"; import { Options } from "../options"; import { needProjectId } from "../projectUtils"; import { requireAuth } from "../requireAuth"; -import { runCompareSuite } from "../apphosting/compare/suite"; +import * as suiteModule from "../apphosting/compare/suite"; +import * as lifecycle from "../apphosting/compare/lifecycle"; +import * as slots from "../apphosting/compare/slots"; +import * as fs from "fs-extra"; import { FirebaseError } from "../error"; +import { logger } from "../logger"; export const command = new Command("apphosting:compare") .description( @@ -45,23 +49,50 @@ export const command = new Command("apphosting:compare") const pathA = (options.pathA as string) || "."; const pathB = (options.pathB as string) || pathA; - await runCompareSuite( - projectId, - location, - [ - { - path: pathA, - localBuild: !!options.localBuildA, - runtime: options.runtimeA as string | undefined, - }, + lifecycle.validateProject(projectId); + await lifecycle.runGarbageCollection(projectId, location); + + const slot = await slots.acquireComparisonSlot(projectId, location, 2); + + const cleanUp = async () => { + logger.warn("\nInterrupted. Restoring comparison slot lock and cleaning up temp files..."); + for (const configPath of suiteModule.createdConfigs) { + try { + await fs.remove(configPath); + } catch (e) {} + } + await slots.releaseComparisonSlot(projectId, location, slot.index, 2); + process.exit(1); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + + try { + await suiteModule.runCompareSuite( + projectId, + location, + slot.backendIds, + slot.index, + "Single-Comparison-Run", + [ + { + path: pathA, + localBuild: !!options.localBuildA, + runtime: options.runtimeA as string | undefined, + }, + { + path: pathB, + localBuild: !!options.localBuildB, + runtime: options.runtimeB as string | undefined, + }, + ], { - path: pathB, - localBuild: !!options.localBuildB, - runtime: options.runtimeB as string | undefined, + outputDir: options.outputDir as string, }, - ], - { - outputDir: options.outputDir as string, - }, - ); + ); + } finally { + process.off("SIGINT", cleanUp); + process.off("SIGTERM", cleanUp); + await slots.releaseComparisonSlot(projectId, location, slot.index, 2); + } }); From 5ab7a7fdfaa43e81dd136f509bbc156f9d842b1e Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Wed, 17 Jun 2026 17:46:40 -0400 Subject: [PATCH 04/12] feat(apphosting): harden compare tool, add routing delay and codebase filter --- src/apphosting/compare/cache.ts | 65 +++++-- src/apphosting/compare/compare.ts | 6 +- src/apphosting/compare/crawler.ts | 29 ++- src/apphosting/compare/lifecycle.ts | 4 +- src/apphosting/compare/server.ts | 282 +++++++++++++++++++++++++--- src/apphosting/compare/suite.ts | 60 ++++-- 6 files changed, 380 insertions(+), 66 deletions(-) diff --git a/src/apphosting/compare/cache.ts b/src/apphosting/compare/cache.ts index e0cc5bc17e3..4005c87f727 100644 --- a/src/apphosting/compare/cache.ts +++ b/src/apphosting/compare/cache.ts @@ -1,5 +1,6 @@ import * as fs from "fs-extra"; import * as path from "path"; +import * as crypto from "crypto"; import { logger } from "../../logger"; export interface RouteResponse { @@ -20,18 +21,24 @@ export interface VariantRecording { const CACHE_DIR = path.resolve(process.cwd(), "compare-cache"); function getRecordingPath(testCaseName: string, variantId: string): string { - const safeTestCase = testCaseName.replace(/[^a-zA-Z0-9_-]/g, "_"); - const safeVariant = variantId.replace(/[^a-zA-Z0-9_-]/g, "_"); + const tcHash = crypto.createHash("sha256").update(testCaseName).digest("hex").slice(0, 8); + const vHash = crypto.createHash("sha256").update(variantId).digest("hex").slice(0, 8); + const safeTestCase = `${testCaseName.replace(/[^a-zA-Z0-9_-]/g, "_")}_${tcHash}`; + const safeVariant = `${variantId.replace(/[^a-zA-Z0-9_-]/g, "_")}_${vHash}`; return path.join(CACHE_DIR, "recordings", safeTestCase, `${safeVariant}.json`); } /** - * Saves a variant recording to the cache. + * Saves a variant recording to the cache atomically. */ export async function saveRecording(recording: VariantRecording): Promise { const filePath = getRecordingPath(recording.testCaseName, recording.id); - await fs.ensureDir(path.dirname(filePath)); - await fs.writeJson(filePath, recording, { spaces: 2 }); + const tempPath = filePath + ".tmp"; + + await fs.ensureDir(path.dirname(tempPath)); + await fs.writeJson(tempPath, recording, { spaces: 2 }); + await fs.rename(tempPath, filePath); + logger.info(`Saved recording to cache: ${filePath}`); } @@ -56,17 +63,47 @@ export async function listRecordings(): Promise> { } const result: Record = {}; - const testCases = await fs.readdir(recordingsDir); - for (const tc of testCases) { - const tcDir = path.join(recordingsDir, tc); - const stat = await fs.stat(tcDir); - if (stat.isDirectory()) { - const files = await fs.readdir(tcDir); - result[tc] = files - .filter((file) => file.endsWith(".json")) - .map((file) => file.replace(/\.json$/, "")); + try { + const testCases = await fs.readdir(recordingsDir); + for (const tc of testCases) { + const tcDir = path.join(recordingsDir, tc); + try { + const stat = await fs.stat(tcDir); + if (stat.isDirectory()) { + const files = await fs.readdir(tcDir); + const jsonFiles = files.filter((file) => file.endsWith(".json")); + if (jsonFiles.length > 0) { + const variantIds: string[] = []; + let originalTestCaseName = ""; + for (const file of jsonFiles) { + try { + const data = await fs.readJson(path.join(tcDir, file)); + originalTestCaseName = data.testCaseName || tc; + variantIds.push(data.id || file.replace(/\.json$/, "")); + } catch (readErr) { + // If a file is partially written or corrupted, skip it + logger.debug(`Failed to read metadata for cache file: ${file}`, readErr); + } + } + if (originalTestCaseName && variantIds.length > 0) { + result[originalTestCaseName] = variantIds; + } + } + } + } catch (err: any) { + if (err.code === "ENOENT") { + continue; // Directory was concurrently deleted/moved + } + throw err; + } } + } catch (err: any) { + if (err.code === "ENOENT") { + return {}; + } + throw err; } return result; } + diff --git a/src/apphosting/compare/compare.ts b/src/apphosting/compare/compare.ts index ed8261a326f..465276db10e 100644 --- a/src/apphosting/compare/compare.ts +++ b/src/apphosting/compare/compare.ts @@ -161,8 +161,10 @@ export async function compareRouteResponses( if (contentType.includes("text/html")) { try { const prettier = require("prettier"); - bodyA = await prettier.format(bodyA, { parser: "html" }); - bodyB = await prettier.format(bodyB, { parser: "html" }); + const formattedA = await prettier.format(bodyA, { parser: "html" }); + const formattedB = await prettier.format(bodyB, { parser: "html" }); + bodyA = formattedA; + bodyB = formattedB; } catch (e: any) { // Fallback to advanced tag-based line splitting if prettier fails const HTML_SPLIT_REGEX = /(<(script|style)\b[\s\S]*?<\/\2>||<[^'">]*(?:"[^"]*"[^'">]*|'[^']*'[^'">]*)*>)\s*(?=<)/gi; diff --git a/src/apphosting/compare/crawler.ts b/src/apphosting/compare/crawler.ts index 96a9c7ee13b..46367e82332 100644 --- a/src/apphosting/compare/crawler.ts +++ b/src/apphosting/compare/crawler.ts @@ -1,5 +1,14 @@ import fetch from "node-fetch"; +function decodeHtmlEntities(str: string): string { + return str + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'"); +} + export class Crawler { private visited = new Set(); private discoveredRoutes = new Set(); @@ -29,15 +38,22 @@ export class Crawler { this.visited.add(canonical); this.discoveredRoutes.add(canonical); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 15000); + try { const url = `${this.baseUrl}${canonical}`; const res = await fetch(url, { redirect: "manual" as const, headers: { "User-Agent": "FirebaseCompareCrawler/1.0" }, + signal: controller.signal, }); // Handle Redirects if ([301, 302, 307, 308].includes(res.status)) { + if (res.body) { + (res.body as any).destroy(); + } const location = res.headers.get("location"); if (location) { const nextRoute = this.resolveRelative(canonical, location); @@ -51,6 +67,9 @@ export class Crawler { // Only parse HTML responses const contentType = res.headers.get("content-type") || ""; if (!contentType.toLowerCase().includes("text/html")) { + if (res.body) { + (res.body as any).destroy(); + } return; } @@ -62,6 +81,8 @@ export class Crawler { } } catch (err) { // Ignore fetch failures for single routes during discovery + } finally { + clearTimeout(timeout); } } @@ -90,11 +111,14 @@ export class Crawler { private extractLinks(html: string, currentRoute: string): string[] { const links: string[] = []; - const regex = /]*?\s+)?href=(["'])(.*?)\1/gi; + const regex = /]*?\s+)?href\s*=\s*(?:(["'])(.*?)\1|([^\s>]+))/gi; let match; while ((match = regex.exec(html)) !== null) { - const href = match[2].trim(); + const rawHref = match[2] !== undefined ? match[2] : match[3]; + if (!rawHref) continue; + + const href = decodeHtmlEntities(rawHref.trim()); if ( !href || href.startsWith("#") || @@ -127,3 +151,4 @@ export class Crawler { } } } + diff --git a/src/apphosting/compare/lifecycle.ts b/src/apphosting/compare/lifecycle.ts index 36a4ab34a1e..cb0319b3cb9 100644 --- a/src/apphosting/compare/lifecycle.ts +++ b/src/apphosting/compare/lifecycle.ts @@ -23,7 +23,7 @@ export async function runGarbageCollection(projectId: string, location: string): const existingBackends = await apphosting.listBackends(projectId, location); const backendsList = existingBackends.backends || []; const now = Date.now(); - const twoHours = 2 * 60 * 60 * 1000; + const thirtyMinutes = 30 * 60 * 1000; for (const backend of backendsList) { const nameParts = backend.name.split("/"); @@ -33,7 +33,7 @@ export async function runGarbageCollection(projectId: string, location: string): const isBusy = backend.labels?.status === "busy"; if (isBusy) { const updateTime = new Date(backend.updateTime).getTime(); - if (now - updateTime > twoHours) { + if (now - updateTime > thirtyMinutes) { logger.info(`Found stale lock on comparison slot backend ${backendId}. Unlocking...`); try { await apphosting.updateBackend(projectId, location, backendId, { diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts index 7700e48ad42..8e0c6c755a0 100644 --- a/src/apphosting/compare/server.ts +++ b/src/apphosting/compare/server.ts @@ -29,8 +29,19 @@ export function startServer(port: number): Promise { } try { - const recA = await cache.loadRecording(testCase as string, variantA as string); - const recB = await cache.loadRecording(testCase as string, variantB as string); + let recA, recB; + if (testCase === "GLOBAL") { + if (typeof variantA !== "string" || typeof variantB !== "string" || !variantA.includes("/") || !variantB.includes("/")) { + throw new Error("Invalid variant query parameters for GLOBAL testCase"); + } + const [tcA, varA] = variantA.split("/"); + const [tcB, varB] = variantB.split("/"); + recA = await cache.loadRecording(tcA, varA); + recB = await cache.loadRecording(tcB, varB); + } else { + recA = await cache.loadRecording(testCase as string, variantA as string); + recB = await cache.loadRecording(testCase as string, variantB as string); + } const allRoutes = Array.from(new Set([ ...Object.keys(recA.routes), @@ -96,28 +107,56 @@ export function startServer(port: number): Promise { try { const recordings = await cache.listRecordings(); - const variants = recordings[testCase as string]; - if (!variants || variants.length === 0) { + let variantsList: string[] = []; + const recMap: Record = {}; + + if (testCase === "GLOBAL") { + for (const tc of Object.keys(recordings)) { + for (const v of recordings[tc]) { + const id = `${tc}/${v}`; + variantsList.push(id); + recMap[id] = await cache.loadRecording(tc, v); + } + } + } else { + variantsList = recordings[testCase as string] || []; + for (const v of variantsList) { + recMap[v] = await cache.loadRecording(testCase as string, v); + } + } + + if (variantsList.length === 0) { res.json({ testCase, variants: [], matrix: {} }); return; } - // Preload all recordings to avoid repeated reads - const recMap: Record = {}; - for (const v of variants) { - recMap[v] = await cache.loadRecording(testCase as string, v); - } + const matrix: Record> = {}; - const matrix: Record> = {}; + for (const vA of variantsList) { + matrix[vA] = matrix[vA] || {}; + for (const vB of variantsList) { + matrix[vB] = matrix[vB] || {}; - for (const vA of variants) { - matrix[vA] = {}; - for (const vB of variants) { if (vA === vB) { matrix[vA][vB] = 1.0; continue; } + if (matrix[vA][vB] !== undefined) { + continue; // Already computed symmetrical pair + } + + if (testCase === "GLOBAL") { + const tc_A = vA.split("/")[0]; + const tc_B = vB.split("/")[0]; + if (tc_A !== tc_B) { + // Skip meaningless cross-app comparisons + matrix[vA][vB] = null; + matrix[vB][vA] = null; + continue; + } + } + // Compute average body similarity across all shared routes const recA = recMap[vA]; const recB = recMap[vB]; @@ -134,6 +173,7 @@ export function startServer(port: number): Promise { const resB = recB.routes[route]; if (!resA || !resB) { + countedRoutes++; // Missing routes act as 0% similarity penalty continue; } @@ -142,13 +182,15 @@ export function startServer(port: number): Promise { countedRoutes++; } - matrix[vA][vB] = countedRoutes > 0 ? (totalSimilarity / countedRoutes) : 0.0; + const score = countedRoutes > 0 ? (totalSimilarity / countedRoutes) : 0.0; + matrix[vA][vB] = score; + matrix[vB][vA] = score; } } res.json({ testCase, - variants, + variants: variantsList, matrix }); } catch (err: any) { @@ -156,6 +198,43 @@ export function startServer(port: number): Promise { } }); + // API: Render cached recording directly (bypasses iframe network requests) + app.get("/api/render", async (req, res) => { + const { testCase, variant, route } = req.query; + try { + if (!testCase || !variant || !route) throw new Error("Missing query parameters"); + let tc = testCase as string; + let varId = variant as string; + if (tc === "GLOBAL") { + const parts = varId.split("/"); + if (parts.length >= 2) { + tc = parts[0]; + varId = parts.slice(1).join("/"); + } + } + const rec = await cache.loadRecording(tc, varId); + const resData = rec.routes[route as string]; + if (!resData) { + res.status(404).send("Route not found in cache"); + return; + } + if (resData.isBinary) { + res.setHeader("Content-Type", "application/octet-stream"); + res.send(Buffer.from(resData.body, "base64")); + } else { + res.setHeader("Content-Type", "text/html"); + // Inject tag to fix relative assets + let html = resData.body; + if (!html.includes("", ``); + } + res.send(html); + } + } catch (err: any) { + res.status(500).send(err.message); + } + }); + // Serve Single Page Application dashboard app.get("/", (req, res) => { res.send(getDashboardHtml()); @@ -510,6 +589,15 @@ function getDashboardHtml(): string { transform: scale(1.06); filter: brightness(1.15); } + .heatmap-cell.de-emphasized { + opacity: 0.15; + filter: grayscale(80%); + } + .heatmap-cell.de-emphasized:hover { + opacity: 0.35; + filter: grayscale(40%); + } + .heatmap-header-cell { padding: 10px; font-weight: 600; @@ -602,9 +690,32 @@ function getDashboardHtml(): string {
    + + +
    +
    + Understanding Parity Metrics & Next.js Quirks + [Click to Expand] +
    + +
    +
    - +
    -
    Response Body Content Diff
    -
    -
    - +
    +
    Raw Code Diff
    +
    Visual Split-View
    +
    +
    + + +
    +
    + + + +
    @@ -704,6 +833,15 @@ function getDashboardHtml(): string { const container = document.getElementById("test-cases-list"); container.innerHTML = ""; + // Add GLOBAL test case + const globalItem = document.createElement("div"); + globalItem.className = "list-item"; + globalItem.style.fontWeight = "bold"; + globalItem.style.color = "var(--accent)"; + globalItem.textContent = "🌍 GLOBAL MATRIX (All Apps)"; + globalItem.onclick = () => selectTestCase("GLOBAL", globalItem); + container.appendChild(globalItem); + Object.keys(recordingsData).forEach((tc) => { const item = document.createElement("div"); item.className = "list-item"; @@ -718,7 +856,17 @@ function getDashboardHtml(): string { element.classList.add("active"); activeTestCase = tc; - const variants = recordingsData[tc]; + let variants = []; + + if (tc === "GLOBAL") { + document.getElementById("filter-codebases-container").style.display = "flex"; + Object.keys(recordingsData).forEach(suite => { + recordingsData[suite].forEach(v => variants.push(\`\${suite}/\${v}\`)); + }); + } else { + document.getElementById("filter-codebases-container").style.display = "none"; + variants = recordingsData[tc]; + } // Populate Variant Dropdowns const selectA = document.getElementById("select-variant-a"); @@ -793,6 +941,8 @@ function getDashboardHtml(): string { const similarity = data.matrix[vA][vB] || 0.0; const percent = Math.round(similarity * 100); tdCell.textContent = percent + "%"; + tdCell.dataset.codebaseA = vA.includes("/") ? vA.split("/")[0] : ""; + tdCell.dataset.codebaseB = vB.includes("/") ? vB.split("/")[0] : ""; // Color coding based on similarity let bg = "rgba(239, 68, 68, 0.85)"; // Red (low similarity) @@ -823,6 +973,21 @@ function getDashboardHtml(): string { }); container.appendChild(table); + applyMatrixFilter(); + } + + function applyMatrixFilter() { + const ignoreDiffCodebases = document.getElementById("toggle-filter-codebases")?.checked; + const cells = document.querySelectorAll(".heatmap-cell"); + cells.forEach((cell) => { + const cbA = cell.dataset.codebaseA; + const cbB = cell.dataset.codebaseB; + if (ignoreDiffCodebases && cbA && cbB && cbA !== cbB) { + cell.classList.add("de-emphasized"); + } else { + cell.classList.remove("de-emphasized"); + } + }); } function showHeatmapView() { @@ -841,7 +1006,7 @@ function getDashboardHtml(): string { document.getElementById("routes-card").style.display = "flex"; document.getElementById("comparison-details").style.display = "none"; - const res = await fetch(\`/api/compare?testCase=\${activeTestCase}&variantA=\${varA}&variantB=\${varB}\`); + const res = await fetch(\`/api/compare?testCase=\${encodeURIComponent(activeTestCase)}&variantA=\${encodeURIComponent(varA)}&variantB=\${encodeURIComponent(varB)}\`); const data = await res.json(); comparisonResults = data.results; activeUrlA = data.urlA || ""; @@ -909,6 +1074,11 @@ function getDashboardHtml(): string { linkB.href = activeUrlB + res.route; linkB.textContent = activeUrlB + res.route; + // Update Visual Render Iframes + // Update Visual Render Iframes (use cached renderer) + document.getElementById("iframe-a").src = \`/api/render?testCase=\${encodeURIComponent(activeTestCase)}&variant=\${encodeURIComponent(document.getElementById("select-variant-a").value)}&route=\${encodeURIComponent(res.route)}\`; + document.getElementById("iframe-b").src = \`/api/render?testCase=\${encodeURIComponent(activeTestCase)}&variant=\${encodeURIComponent(document.getElementById("select-variant-b").value)}&route=\${encodeURIComponent(res.route)}\`; + // 1. Status Code const statusBox = document.getElementById("status-comparison-box"); const statusText = \`A: \${res.statusA} vs B: \${res.statusB}\`; @@ -941,12 +1111,35 @@ function getDashboardHtml(): string { ? 'Critical Mismatch' : 'Expected Variation'; - headersTbody.innerHTML += \` - \${h.header} - \${h.valA || '(missing)'} - \${h.valB || '(missing)'} - \${badgeHtml} - \`; + const tr = document.createElement("tr"); + + const td1 = document.createElement("td"); + td1.style.fontFamily = "monospace"; + td1.style.fontWeight = "500"; + td1.textContent = h.header; + + const td2 = document.createElement("td"); + td2.style.color = h.critical ? 'var(--danger)' : 'var(--text)'; + td2.style.fontFamily = "monospace"; + td2.style.fontSize = "11px"; + td2.style.wordBreak = "break-all"; + td2.textContent = h.valA || '(missing)'; + + const td3 = document.createElement("td"); + td3.style.color = h.critical ? 'var(--success)' : 'var(--text)'; + td3.style.fontFamily = "monospace"; + td3.style.fontSize = "11px"; + td3.style.wordBreak = "break-all"; + td3.textContent = h.valB || '(missing)'; + + const td4 = document.createElement("td"); + td4.innerHTML = badgeHtml; // Safe: hardcoded markup + + tr.appendChild(td1); + tr.appendChild(td2); + tr.appendChild(td3); + tr.appendChild(td4); + headersTbody.appendChild(tr); }); } @@ -970,7 +1163,11 @@ function getDashboardHtml(): string { } if (!res.diffChanges || res.diffChanges.length === 0) { - diffContainer.innerHTML = '
    No diff details available
    '; + if (res.bodyDiff) { + diffContainer.innerHTML = \`
    \${res.bodyDiff}
    \`; + } else { + diffContainer.innerHTML = '
    No diff details available
    '; + } return; } @@ -1006,6 +1203,31 @@ function getDashboardHtml(): string { diffContainer.appendChild(diffView); } + function switchRightTab(tabId) { + document.getElementById("tab-code-diff").classList.remove("active"); + document.getElementById("tab-code-diff").style.borderBottom = "none"; + document.getElementById("tab-code-diff").style.color = "var(--text-muted)"; + + document.getElementById("tab-visual").classList.remove("active"); + document.getElementById("tab-visual").style.borderBottom = "none"; + document.getElementById("tab-visual").style.color = "var(--text-muted)"; + + document.getElementById("body-diff-container").style.display = "none"; + document.getElementById("visual-render-container").style.display = "none"; + + if (tabId === 'code') { + document.getElementById("tab-code-diff").classList.add("active"); + document.getElementById("tab-code-diff").style.borderBottom = "2px solid var(--accent)"; + document.getElementById("tab-code-diff").style.color = "var(--text)"; + document.getElementById("body-diff-container").style.display = "block"; + } else if (tabId === 'visual') { + document.getElementById("tab-visual").classList.add("active"); + document.getElementById("tab-visual").style.borderBottom = "2px solid var(--accent)"; + document.getElementById("tab-visual").style.color = "var(--text)"; + document.getElementById("visual-render-container").style.display = "flex"; + } + } + window.onload = loadRecordings; diff --git a/src/apphosting/compare/suite.ts b/src/apphosting/compare/suite.ts index bad3505ca82..85ac65f0683 100644 --- a/src/apphosting/compare/suite.ts +++ b/src/apphosting/compare/suite.ts @@ -13,6 +13,8 @@ import * as reporter from "./reporter"; import * as poller from "../../operation-poller"; import { logger } from "../../logger"; import { FirebaseError } from "../../error"; +import { sleep } from "../../utils"; + const apphostingPollerOptions = { apiOrigin: apphostingOrigin(), @@ -118,10 +120,13 @@ async function recordVariant( for (const route of allRoutesSet) { logger.debug(`Recording route ${route}...`); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 15000); try { const res = await fetch(`${url}${route}`, { redirect: "manual" as const, headers: { "User-Agent": "FirebaseCompareCrawler/1.0" }, + signal: controller.signal, }); const contentType = res.headers.get("content-type") || ""; @@ -130,14 +135,32 @@ async function recordVariant( const headers: Record = {}; res.headers.forEach((val, key) => { headers[key] = val; }); + let body = ""; + const contentLength = parseInt(res.headers.get("content-length") || "0", 10); + if (contentLength > 2 * 1024 * 1024) { + body = `(omitted - size ${contentLength} bytes exceeds 2MB limit)`; + if (res.body) { + (res.body as any).destroy(); + } + } else { + const buffer = await res.buffer(); + if (buffer.length > 2 * 1024 * 1024) { + body = `(omitted - size ${buffer.length} bytes exceeds 2MB limit)`; + } else { + body = isBinary ? buffer.toString("base64") : buffer.toString("utf-8"); + } + } + routes[route] = { status: res.status, headers, isBinary, - body: isBinary ? (await res.buffer()).toString("base64") : await res.text(), + body, }; } catch (err) { logger.warn(`Failed to record route ${route}: ${err}`); + } finally { + clearTimeout(timeout); } } @@ -193,21 +216,22 @@ export async function runCompareSuite( try { // Setup secrets const uniquePaths = Array.from(new Set(variants.map((v) => v.path))); - secretsMappings = await Promise.all( - uniquePaths.map((uniquePath) => { - const pathBackendIds = variants - .map((v, i) => (v.path === uniquePath ? backendIds[i] : null)) - .filter((id): id is string => id !== null); - - return secrets.setupSandboxSecrets( - projectId, - location, - uniquePath, - slotIndex, - pathBackendIds - ); - }) - ); + // Setup secrets sequentially to avoid concurrent creation conflicts in Secret Manager + secretsMappings = []; + for (const uniquePath of uniquePaths) { + const pathBackendIds = variants + .map((v, i) => (v.path === uniquePath ? backendIds[i] : null)) + .filter((id): id is string => id !== null); + + const mappings = await secrets.setupSandboxSecrets( + projectId, + location, + uniquePath, + slotIndex, + pathBackendIds + ); + secretsMappings.push(mappings); + } // Deploy variants sequentially for (let i = 0; i < variants.length; i++) { @@ -225,7 +249,11 @@ export async function runCompareSuite( logger.info("All rollouts completed successfully!"); + logger.info("Waiting 30 seconds for Firebase Hosting routing propagation to complete..."); + await sleep(30000); + // Retrieve URLs and Record + const backendDataList = await Promise.all( backendIds.map((id) => apphosting.getBackend(projectId, location, id)), ); From 6222109d96566c2d2cce762325f2aae657a0ee99 Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Wed, 17 Jun 2026 17:52:21 -0400 Subject: [PATCH 05/12] fix(apphosting): enable cross-testcase similarity calculations & group by codebase family in filter --- src/apphosting/compare/server.ts | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts index 8e0c6c755a0..c64b7e4cbd4 100644 --- a/src/apphosting/compare/server.ts +++ b/src/apphosting/compare/server.ts @@ -146,16 +146,7 @@ export function startServer(port: number): Promise { continue; // Already computed symmetrical pair } - if (testCase === "GLOBAL") { - const tc_A = vA.split("/")[0]; - const tc_B = vB.split("/")[0]; - if (tc_A !== tc_B) { - // Skip meaningless cross-app comparisons - matrix[vA][vB] = null; - matrix[vB][vA] = null; - continue; - } - } + // Compute average body similarity across all shared routes const recA = recMap[vA]; @@ -941,8 +932,15 @@ function getDashboardHtml(): string { const similarity = data.matrix[vA][vB] || 0.0; const percent = Math.round(similarity * 100); tdCell.textContent = percent + "%"; - tdCell.dataset.codebaseA = vA.includes("/") ? vA.split("/")[0] : ""; - tdCell.dataset.codebaseB = vB.includes("/") ? vB.split("/")[0] : ""; + const getFamily = (name) => { + const lower = name.toLowerCase(); + if (lower.includes("angular")) return "angular"; + if (lower.includes("next")) return "nextjs"; + if (lower.includes("node")) return "node"; + return name; + }; + tdCell.dataset.codebaseA = vA.includes("/") ? getFamily(vA.split("/")[0]) : ""; + tdCell.dataset.codebaseB = vB.includes("/") ? getFamily(vB.split("/")[0]) : ""; // Color coding based on similarity let bg = "rgba(239, 68, 68, 0.85)"; // Red (low similarity) From 7c620ab95e04bf2b3f615429b7cb3641bc57385a Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Wed, 17 Jun 2026 17:54:36 -0400 Subject: [PATCH 06/12] fix(apphosting): implement continuous HSL red-to-green gradient mapping for matrix heatmap --- src/apphosting/compare/server.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts index c64b7e4cbd4..699ac440557 100644 --- a/src/apphosting/compare/server.ts +++ b/src/apphosting/compare/server.ts @@ -942,17 +942,9 @@ function getDashboardHtml(): string { tdCell.dataset.codebaseA = vA.includes("/") ? getFamily(vA.split("/")[0]) : ""; tdCell.dataset.codebaseB = vB.includes("/") ? getFamily(vB.split("/")[0]) : ""; - // Color coding based on similarity - let bg = "rgba(239, 68, 68, 0.85)"; // Red (low similarity) - if (percent === 100) { - bg = "rgba(16, 185, 129, 0.85)"; // Bright green - } else if (percent >= 98) { - bg = "rgba(139, 92, 246, 0.85)"; // Indigo / high parity - } else if (percent >= 95) { - bg = "rgba(59, 130, 246, 0.85)"; // Blue - } else if (percent >= 90) { - bg = "rgba(245, 158, 11, 0.85)"; // Orange/Yellow - } + // Continuous red-to-green gradient interpolation (0% = HSL 0, 100% = HSL 120) + const hue = similarity * 120; + const bg = \`hsla(\${hue}, 70%, 42%, 0.85)\`; tdCell.style.backgroundColor = bg; tdCell.title = \`Similarity between \${vA} and \${vB}: \${percent}%\`; From 89851b5948b8b3760569e626e2cf3abc1fa1f4ce Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Wed, 17 Jun 2026 19:12:12 -0400 Subject: [PATCH 07/12] fix(apphosting): harden typescript type safety across comparison suite and fix operation poller timeout options --- canary-matrix.json | 47 ++++++++++++++++ src/apphosting/compare/cache.ts | 62 ++++++++++++++++++--- src/apphosting/compare/crawler.ts | 21 +++++-- src/apphosting/compare/server.ts | 91 ++++++++++++++++--------------- src/apphosting/compare/suite.ts | 31 ++++++++--- src/apphosting/compare/types.ts | 35 ++++++++++++ 6 files changed, 224 insertions(+), 63 deletions(-) create mode 100644 canary-matrix.json create mode 100644 src/apphosting/compare/types.ts diff --git a/canary-matrix.json b/canary-matrix.json new file mode 100644 index 00000000000..21be318a2b3 --- /dev/null +++ b/canary-matrix.json @@ -0,0 +1,47 @@ +[ + { + "name": "Angular 19 Canary Matrix", + "variants": [ + { "id": "Local-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-19", "runtime": "nodejs24", "localBuild": true }, + { "id": "Source-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-19", "runtime": "nodejs24", "localBuild": false }, + { "id": "Source-Node22", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-19", "runtime": "nodejs22", "localBuild": false }, + { "id": "Source-NoABIU", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-19", "runtime": "nodejs", "localBuild": false } + ] + }, + { + "name": "Angular 20 Canary Matrix", + "variants": [ + { "id": "Local-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-20", "runtime": "nodejs24", "localBuild": true }, + { "id": "Source-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-20", "runtime": "nodejs24", "localBuild": false }, + { "id": "Source-Node22", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-20", "runtime": "nodejs22", "localBuild": false }, + { "id": "Source-NoABIU", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-20", "runtime": "nodejs", "localBuild": false } + ] + }, + { + "name": "Angular 21 Canary Matrix", + "variants": [ + { "id": "Local-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-21", "runtime": "nodejs24", "localBuild": true }, + { "id": "Source-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-21", "runtime": "nodejs24", "localBuild": false }, + { "id": "Source-Node22", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-21", "runtime": "nodejs22", "localBuild": false }, + { "id": "Source-NoABIU", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-21", "runtime": "nodejs", "localBuild": false } + ] + }, + { + "name": "Angular 22 Canary Matrix", + "variants": [ + { "id": "Local-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-22", "runtime": "nodejs24", "localBuild": true }, + { "id": "Source-Node24", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-22", "runtime": "nodejs24", "localBuild": false }, + { "id": "Source-Node22", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-22", "runtime": "nodejs22", "localBuild": false }, + { "id": "Source-NoABIU", "path": "../firebase-apphosting-canary/apps/angular-reference/angular-22", "runtime": "nodejs", "localBuild": false } + ] + }, + { + "name": "Next.js 15.3 Canary Matrix", + "variants": [ + { "id": "Local-Node24", "path": "../firebase-apphosting-canary/apps/nextjs-reference/next-15.3", "runtime": "nodejs24", "localBuild": true }, + { "id": "Source-Node24", "path": "../firebase-apphosting-canary/apps/nextjs-reference/next-15.3", "runtime": "nodejs24", "localBuild": false }, + { "id": "Source-Node22", "path": "../firebase-apphosting-canary/apps/nextjs-reference/next-15.3", "runtime": "nodejs22", "localBuild": false }, + { "id": "Source-NoABIU", "path": "../firebase-apphosting-canary/apps/nextjs-reference/next-15.3", "runtime": "nodejs", "localBuild": false } + ] + } +] diff --git a/src/apphosting/compare/cache.ts b/src/apphosting/compare/cache.ts index 4005c87f727..be67b4c3831 100644 --- a/src/apphosting/compare/cache.ts +++ b/src/apphosting/compare/cache.ts @@ -18,6 +18,45 @@ export interface VariantRecording { routes: Record; } +export function isRouteResponse(obj: unknown): obj is RouteResponse { + if (typeof obj !== "object" || obj === null) return false; + const o = obj as Record; + return ( + typeof o.status === "number" && + typeof o.headers === "object" && o.headers !== null && + typeof o.body === "string" && + typeof o.isBinary === "boolean" + ); +} + +export function isVariantRecording(obj: unknown): obj is VariantRecording { + if (typeof obj !== "object" || obj === null) return false; + const o = obj as Record; + + if (!( + typeof o.id === "string" && + typeof o.testCaseName === "string" && + typeof o.timestamp === "string" && + typeof o.url === "string" && + typeof o.routes === "object" && o.routes !== null + )) { + return false; + } + + const routes = o.routes as Record; + for (const key of Object.keys(routes)) { + if (!isRouteResponse(routes[key])) { + return false; + } + } + + return true; +} + +function isErrnoException(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && "code" in error; +} + const CACHE_DIR = path.resolve(process.cwd(), "compare-cache"); function getRecordingPath(testCaseName: string, variantId: string): string { @@ -50,7 +89,11 @@ export async function loadRecording(testCaseName: string, variantId: string): Pr if (!(await fs.pathExists(filePath))) { throw new Error(`No recording found in cache for variant "${variantId}" under test case "${testCaseName}"`); } - return await fs.readJson(filePath); + const data = await fs.readJson(filePath); + if (!isVariantRecording(data)) { + throw new Error(`Invalid recording format in cache for variant "${variantId}" under test case "${testCaseName}"`); + } + return data; } /** @@ -78,8 +121,12 @@ export async function listRecordings(): Promise> { for (const file of jsonFiles) { try { const data = await fs.readJson(path.join(tcDir, file)); - originalTestCaseName = data.testCaseName || tc; - variantIds.push(data.id || file.replace(/\.json$/, "")); + if (isVariantRecording(data)) { + originalTestCaseName = data.testCaseName; + variantIds.push(data.id); + } else { + logger.debug(`Cache file ${file} does not match VariantRecording schema`); + } } catch (readErr) { // If a file is partially written or corrupted, skip it logger.debug(`Failed to read metadata for cache file: ${file}`, readErr); @@ -90,15 +137,15 @@ export async function listRecordings(): Promise> { } } } - } catch (err: any) { - if (err.code === "ENOENT") { + } catch (err: unknown) { + if (isErrnoException(err) && err.code === "ENOENT") { continue; // Directory was concurrently deleted/moved } throw err; } } - } catch (err: any) { - if (err.code === "ENOENT") { + } catch (err: unknown) { + if (isErrnoException(err) && err.code === "ENOENT") { return {}; } throw err; @@ -107,3 +154,4 @@ export async function listRecordings(): Promise> { return result; } + diff --git a/src/apphosting/compare/crawler.ts b/src/apphosting/compare/crawler.ts index 46367e82332..17ce716be55 100644 --- a/src/apphosting/compare/crawler.ts +++ b/src/apphosting/compare/crawler.ts @@ -9,6 +9,19 @@ function decodeHtmlEntities(str: string): string { .replace(/'/g, "'"); } +interface Destroyable { + destroy(): void; +} + +function isDestroyable(obj: unknown): obj is Destroyable { + return ( + typeof obj === "object" && + obj !== null && + "destroy" in obj && + typeof (obj as Record).destroy === "function" + ); +} + export class Crawler { private visited = new Set(); private discoveredRoutes = new Set(); @@ -51,8 +64,8 @@ export class Crawler { // Handle Redirects if ([301, 302, 307, 308].includes(res.status)) { - if (res.body) { - (res.body as any).destroy(); + if (res.body && isDestroyable(res.body)) { + res.body.destroy(); } const location = res.headers.get("location"); if (location) { @@ -67,8 +80,8 @@ export class Crawler { // Only parse HTML responses const contentType = res.headers.get("content-type") || ""; if (!contentType.toLowerCase().includes("text/html")) { - if (res.body) { - (res.body as any).destroy(); + if (res.body && isDestroyable(res.body)) { + res.body.destroy(); } return; } diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts index 699ac440557..6826811d42d 100644 --- a/src/apphosting/compare/server.ts +++ b/src/apphosting/compare/server.ts @@ -3,6 +3,7 @@ import * as http from "http"; import { logger } from "../../logger"; import * as cache from "./cache"; import * as compare from "./compare"; +import { CompareResponse, MatrixResponse, DashboardComparisonResult } from "./types"; export function startServer(port: number): Promise { return new Promise((resolve, reject) => { @@ -15,23 +16,24 @@ export function startServer(port: number): Promise { try { const recordings = await cache.listRecordings(); res.json(recordings); - } catch (err: any) { - res.status(500).json({ error: err.message }); + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).json({ error: errMsg }); } }); - // API: Compare two cached recordings of a test case app.get("/api/compare", async (req, res) => { const { testCase, variantA, variantB } = req.query; - if (!testCase || !variantA || !variantB) { - res.status(400).json({ error: "Missing testCase, variantA, or variantB query parameters" }); + if (typeof testCase !== "string" || typeof variantA !== "string" || typeof variantB !== "string") { + res.status(400).json({ error: "Missing or invalid query parameters: testCase, variantA, and variantB must be strings." }); return; } try { - let recA, recB; + let recA: cache.VariantRecording; + let recB: cache.VariantRecording; if (testCase === "GLOBAL") { - if (typeof variantA !== "string" || typeof variantB !== "string" || !variantA.includes("/") || !variantB.includes("/")) { + if (!variantA.includes("/") || !variantB.includes("/")) { throw new Error("Invalid variant query parameters for GLOBAL testCase"); } const [tcA, varA] = variantA.split("/"); @@ -39,8 +41,8 @@ export function startServer(port: number): Promise { recA = await cache.loadRecording(tcA, varA); recB = await cache.loadRecording(tcB, varB); } else { - recA = await cache.loadRecording(testCase as string, variantA as string); - recB = await cache.loadRecording(testCase as string, variantB as string); + recA = await cache.loadRecording(testCase, variantA); + recB = await cache.loadRecording(testCase, variantB); } const allRoutes = Array.from(new Set([ @@ -48,7 +50,7 @@ export function startServer(port: number): Promise { ...Object.keys(recB.routes) ])).sort(); - const results: compare.ComparisonResult[] = []; + const results: DashboardComparisonResult[] = []; for (const route of allRoutes) { const resA = recA.routes[route]; const resB = recB.routes[route]; @@ -69,46 +71,48 @@ export function startServer(port: number): Promise { } const compResult = await compare.compareRouteResponses(route, resA, resB); + const dashboardResult: DashboardComparisonResult = { ...compResult }; if (!resA.isBinary && !resB.isBinary) { const diff = require("diff"); - const changes = diff.diffLines(compResult.bodyA || "", compResult.bodyB || ""); + const changes = diff.diffLines(dashboardResult.bodyA || "", dashboardResult.bodyB || ""); // Filter/map to minimal JSON to keep payload clean - (compResult as any).diffChanges = changes.map((c: any) => ({ + dashboardResult.diffChanges = changes.map((c: any) => ({ value: c.value, added: !!c.added, removed: !!c.removed })); } - results.push(compResult); + results.push(dashboardResult); } - res.json({ + const responsePayload: CompareResponse = { testCase, variantA: recA.id, variantB: recB.id, urlA: recA.url, urlB: recB.url, results - }); - } catch (err: any) { - res.status(500).json({ error: err.message }); + }; + res.json(responsePayload); + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).json({ error: errMsg }); } }); - // API: Get N x N pairwise similarity matrix for a test case app.get("/api/matrix", async (req, res) => { const { testCase } = req.query; - if (!testCase) { - res.status(400).json({ error: "Missing testCase query parameter" }); + if (typeof testCase !== "string") { + res.status(400).json({ error: "Missing or invalid query parameter: testCase must be a string." }); return; } try { const recordings = await cache.listRecordings(); let variantsList: string[] = []; - const recMap: Record = {}; + const recMap: Record = {}; if (testCase === "GLOBAL") { for (const tc of Object.keys(recordings)) { @@ -119,14 +123,15 @@ export function startServer(port: number): Promise { } } } else { - variantsList = recordings[testCase as string] || []; + variantsList = recordings[testCase] || []; for (const v of variantsList) { - recMap[v] = await cache.loadRecording(testCase as string, v); + recMap[v] = await cache.loadRecording(testCase, v); } } if (variantsList.length === 0) { - res.json({ testCase, variants: [], matrix: {} }); + const emptyPayload: MatrixResponse = { testCase, variants: [], matrix: {} }; + res.json(emptyPayload); return; } @@ -179,23 +184,27 @@ export function startServer(port: number): Promise { } } - res.json({ + const responsePayload: MatrixResponse = { testCase, variants: variantsList, matrix - }); - } catch (err: any) { - res.status(500).json({ error: err.message }); + }; + res.json(responsePayload); + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).json({ error: errMsg }); } }); - // API: Render cached recording directly (bypasses iframe network requests) app.get("/api/render", async (req, res) => { const { testCase, variant, route } = req.query; + if (typeof testCase !== "string" || typeof variant !== "string" || typeof route !== "string") { + res.status(400).send("Missing or invalid query parameters: testCase, variant, and route must be strings."); + return; + } try { - if (!testCase || !variant || !route) throw new Error("Missing query parameters"); - let tc = testCase as string; - let varId = variant as string; + let tc = testCase; + let varId = variant; if (tc === "GLOBAL") { const parts = varId.split("/"); if (parts.length >= 2) { @@ -204,7 +213,7 @@ export function startServer(port: number): Promise { } } const rec = await cache.loadRecording(tc, varId); - const resData = rec.routes[route as string]; + const resData = rec.routes[route]; if (!resData) { res.status(404).send("Route not found in cache"); return; @@ -221,8 +230,9 @@ export function startServer(port: number): Promise { } res.send(html); } - } catch (err: any) { - res.status(500).send(err.message); + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).send(errMsg); } }); @@ -932,15 +942,8 @@ function getDashboardHtml(): string { const similarity = data.matrix[vA][vB] || 0.0; const percent = Math.round(similarity * 100); tdCell.textContent = percent + "%"; - const getFamily = (name) => { - const lower = name.toLowerCase(); - if (lower.includes("angular")) return "angular"; - if (lower.includes("next")) return "nextjs"; - if (lower.includes("node")) return "node"; - return name; - }; - tdCell.dataset.codebaseA = vA.includes("/") ? getFamily(vA.split("/")[0]) : ""; - tdCell.dataset.codebaseB = vB.includes("/") ? getFamily(vB.split("/")[0]) : ""; + tdCell.dataset.codebaseA = vA.includes("/") ? vA.split("/")[0] : ""; + tdCell.dataset.codebaseB = vB.includes("/") ? vB.split("/")[0] : ""; // Continuous red-to-green gradient interpolation (0% = HSL 0, 100% = HSL 120) const hue = similarity * 120; diff --git a/src/apphosting/compare/suite.ts b/src/apphosting/compare/suite.ts index 85ac65f0683..37a01212bb5 100644 --- a/src/apphosting/compare/suite.ts +++ b/src/apphosting/compare/suite.ts @@ -16,12 +16,13 @@ import { FirebaseError } from "../../error"; import { sleep } from "../../utils"; -const apphostingPollerOptions = { + +const apphostingPollerOptions: Omit = { apiOrigin: apphostingOrigin(), apiVersion: "v1beta", backoff: 200, maxBackoff: 10000, - timeout: 120000, // 2 minutes + masterTimeout: 120000, // 2 minutes }; import * as cp from "child_process"; @@ -29,6 +30,19 @@ import * as util from "util"; export const createdConfigs = new Set(); +interface Destroyable { + destroy(): void; +} + +function isDestroyable(obj: unknown): obj is Destroyable { + return ( + typeof obj === "object" && + obj !== null && + "destroy" in obj && + typeof (obj as Record).destroy === "function" + ); +} + async function deployToBackend( projectId: string, location: string, @@ -41,7 +55,7 @@ async function deployToBackend( if (runtimeVersion) { logger.info(`Patching runtime version for backend ${backendId} to ${runtimeVersion}...`); const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; - const op = await apphosting.client.patch( + const op = await apphosting.client.patch<{ name: string; runtime: { value: string } }, apphosting.Operation>( name, { name, runtime: { value: runtimeVersion } }, { queryParams: { updateMask: "runtime" } }, @@ -80,9 +94,10 @@ async function deployToBackend( const execAsync = util.promisify(cp.exec); const { stdout, stderr } = await execAsync(cmd, { cwd: appPath, maxBuffer: 1024 * 1024 * 100 }); logger.debug(`Deploy output for ${backendId}:\n${stdout}`); - } catch (err: any) { - logger.error(`Deploy for ${backendId} failed!\nSTDOUT:\n${err.stdout}\nSTDERR:\n${err.stderr}`); - throw new FirebaseError(`Failed to deploy variant to ${backendId}.`, { original: err }); + } catch (err: unknown) { + const execErr = err as { stdout?: string; stderr?: string }; + logger.error(`Deploy for ${backendId} failed!\nSTDOUT:\n${execErr.stdout || ""}\nSTDERR:\n${execErr.stderr || ""}`); + throw new FirebaseError(`Failed to deploy variant to ${backendId}.`, { original: err instanceof Error ? err : undefined }); } finally { await fs.remove(configPath); createdConfigs.delete(configPath); @@ -139,8 +154,8 @@ async function recordVariant( const contentLength = parseInt(res.headers.get("content-length") || "0", 10); if (contentLength > 2 * 1024 * 1024) { body = `(omitted - size ${contentLength} bytes exceeds 2MB limit)`; - if (res.body) { - (res.body as any).destroy(); + if (res.body && isDestroyable(res.body)) { + res.body.destroy(); } } else { const buffer = await res.buffer(); diff --git a/src/apphosting/compare/types.ts b/src/apphosting/compare/types.ts new file mode 100644 index 00000000000..63bdbdc8ff2 --- /dev/null +++ b/src/apphosting/compare/types.ts @@ -0,0 +1,35 @@ +import { ComparisonResult } from "./compare"; + +export interface ComparisonSlot { + index: number; + backendIds: string[]; +} + +export interface SecretMapping { + originalName: string; + mockSecretName: string; + mockValue: string; +} + +export interface DashboardComparisonResult extends ComparisonResult { + diffChanges?: Array<{ + value: string; + added: boolean; + removed: boolean; + }>; +} + +export interface CompareResponse { + testCase: string; + variantA: string; + variantB: string; + urlA: string; + urlB: string; + results: DashboardComparisonResult[]; +} + +export interface MatrixResponse { + testCase: string; + variants: string[]; + matrix: Record>; +} From ace7c51c59b62df5f2419e117d090b7a60488a4b Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Wed, 17 Jun 2026 19:20:39 -0400 Subject: [PATCH 08/12] feat(apphosting): add dynamic heatmap filter controls for search, build origin and runtime metadata --- src/apphosting/compare/cache.ts | 10 +++ src/apphosting/compare/server.ts | 122 ++++++++++++++++++++++++++++--- src/apphosting/compare/suite.ts | 2 + src/apphosting/compare/types.ts | 7 ++ 4 files changed, 130 insertions(+), 11 deletions(-) diff --git a/src/apphosting/compare/cache.ts b/src/apphosting/compare/cache.ts index be67b4c3831..e9fa724fa79 100644 --- a/src/apphosting/compare/cache.ts +++ b/src/apphosting/compare/cache.ts @@ -16,6 +16,8 @@ export interface VariantRecording { timestamp: string; url: string; routes: Record; + localBuild?: boolean; + runtime?: string; } export function isRouteResponse(obj: unknown): obj is RouteResponse { @@ -43,6 +45,14 @@ export function isVariantRecording(obj: unknown): obj is VariantRecording { return false; } + if (o.localBuild !== undefined && typeof o.localBuild !== "boolean") { + return false; + } + + if (o.runtime !== undefined && typeof o.runtime !== "string") { + return false; + } + const routes = o.routes as Record; for (const key of Object.keys(routes)) { if (!isRouteResponse(routes[key])) { diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts index 6826811d42d..cb913303f5f 100644 --- a/src/apphosting/compare/server.ts +++ b/src/apphosting/compare/server.ts @@ -3,7 +3,7 @@ import * as http from "http"; import { logger } from "../../logger"; import * as cache from "./cache"; import * as compare from "./compare"; -import { CompareResponse, MatrixResponse, DashboardComparisonResult } from "./types"; +import { CompareResponse, MatrixResponse, DashboardComparisonResult, VariantMetadata } from "./types"; export function startServer(port: number): Promise { return new Promise((resolve, reject) => { @@ -130,11 +130,20 @@ export function startServer(port: number): Promise { } if (variantsList.length === 0) { - const emptyPayload: MatrixResponse = { testCase, variants: [], matrix: {} }; + const emptyPayload: MatrixResponse = { testCase, variants: [], variantsMetadata: {}, matrix: {} }; res.json(emptyPayload); return; } + const variantsMetadata: Record = {}; + for (const v of variantsList) { + variantsMetadata[v] = { + id: recMap[v].id, + localBuild: !!recMap[v].localBuild, + runtime: recMap[v].runtime || "default" + }; + } + const matrix: Record> = {}; for (const vA of variantsList) { @@ -151,8 +160,6 @@ export function startServer(port: number): Promise { continue; // Already computed symmetrical pair } - - // Compute average body similarity across all shared routes const recA = recMap[vA]; const recB = recMap[vB]; @@ -187,6 +194,7 @@ export function startServer(port: number): Promise { const responsePayload: MatrixResponse = { testCase, variants: variantsList, + variantsMetadata, matrix }; res.json(responsePayload); @@ -717,6 +725,21 @@ function getDashboardHtml(): string { Ignore Comparisons for Different Codebases
    +
    +
    + Search Variants: + +
    +
    + Build Origin: + + +
    +
    + Runtimes: + +
    +
    @@ -825,6 +848,7 @@ function getDashboardHtml(): string { let comparisonResults = []; let activeUrlA = ""; let activeUrlB = ""; + let lastMatrixData = null; // Fetch list of recordings on load async function loadRecordings() { @@ -902,13 +926,89 @@ function getDashboardHtml(): string { document.getElementById("comparison-details").style.display = "none"; const res = await fetch(\`/api/matrix?testCase=\${tc}\`); - const data = await res.json(); + lastMatrixData = await res.json(); + + // Dynamically populate runtime checkboxes + const container = document.getElementById("runtime-filters-container"); + container.innerHTML = \`Runtimes:\`; + + const runtimes = new Set(); + if (lastMatrixData.variantsMetadata) { + Object.values(lastMatrixData.variantsMetadata).forEach(meta => { + if (meta.runtime) runtimes.add(meta.runtime); + }); + } + + if (runtimes.size <= 1) { + // If 0 or 1 runtime, hide runtime filters section to keep UI clean + container.style.display = "none"; + } else { + container.style.display = "flex"; + Array.from(runtimes).sort().forEach(rt => { + const lbl = document.createElement("label"); + lbl.style.cssText = "display: flex; align-items: center; gap: 4px; cursor: pointer; user-select: none;"; + + const chk = document.createElement("input"); + chk.type = "checkbox"; + chk.className = "runtime-filter-chk"; + chk.dataset.runtime = rt; + chk.checked = true; + chk.onchange = applyMetadataFilters; + + lbl.appendChild(chk); + lbl.appendChild(document.createTextNode(rt)); + container.appendChild(lbl); + }); + } + + // Reset search field and other filters + document.getElementById("variant-search-input").value = ""; + document.getElementById("filter-local-builds").checked = true; + document.getElementById("filter-source-deploys").checked = true; + + applyMetadataFilters(); + } + + function applyMetadataFilters() { + if (!lastMatrixData) return; + + const searchQuery = document.getElementById("variant-search-input").value.toLowerCase(); + const showLocal = document.getElementById("filter-local-builds").checked; + const showSource = document.getElementById("filter-source-deploys").checked; + const runtimeChks = document.querySelectorAll(".runtime-filter-chk"); + const activeRuntimes = Array.from(runtimeChks) + .filter(chk => chk.checked) + .map(chk => chk.dataset.runtime); + + const filteredVariants = lastMatrixData.variants.filter(v => { + const meta = lastMatrixData.variantsMetadata[v] || { id: v, localBuild: false, runtime: "default" }; + + // Match search query + const matchesSearch = v.toLowerCase().includes(searchQuery); + if (!matchesSearch) return false; + // Match build origin + if (meta.localBuild && !showLocal) return false; + if (!meta.localBuild && !showSource) return false; + + // Match runtime (if checkboxes exist) + if (runtimeChks.length > 0) { + const rtVal = meta.runtime || "default"; + if (!activeRuntimes.includes(rtVal)) return false; + } + + return true; + }); + + renderMatrixTable(filteredVariants); + } + + function renderMatrixTable(variants) { const container = document.getElementById("heatmap-grid-container"); container.innerHTML = ""; - if (!data.variants || data.variants.length === 0) { - container.innerHTML = "No variants found in cache."; + if (variants.length === 0) { + container.innerHTML = \`
    No matching variants found for active filters.
    \`; return; } @@ -918,7 +1018,7 @@ function getDashboardHtml(): string { // 1. Header Row const thead = document.createElement("tr"); thead.appendChild(document.createElement("th")); // empty top-left corner - data.variants.forEach((v) => { + variants.forEach((v) => { const th = document.createElement("th"); th.className = "heatmap-header-cell"; th.textContent = v; @@ -927,7 +1027,7 @@ function getDashboardHtml(): string { table.appendChild(thead); // 2. Rows - data.variants.forEach((vA) => { + variants.forEach((vA) => { const tr = document.createElement("tr"); // Row label @@ -936,10 +1036,10 @@ function getDashboardHtml(): string { tdLabel.textContent = vA; tr.appendChild(tdLabel); - data.variants.forEach((vB) => { + variants.forEach((vB) => { const tdCell = document.createElement("td"); tdCell.className = "heatmap-cell"; - const similarity = data.matrix[vA][vB] || 0.0; + const similarity = lastMatrixData.matrix[vA][vB] || 0.0; const percent = Math.round(similarity * 100); tdCell.textContent = percent + "%"; tdCell.dataset.codebaseA = vA.includes("/") ? vA.split("/")[0] : ""; diff --git a/src/apphosting/compare/suite.ts b/src/apphosting/compare/suite.ts index 37a01212bb5..9f7922bc9b0 100644 --- a/src/apphosting/compare/suite.ts +++ b/src/apphosting/compare/suite.ts @@ -278,6 +278,8 @@ export async function runCompareSuite( const v = variants[i]; const url = urls[i]; const record = await recordVariant(testCaseName, v.id || String(i), url, v.path); + record.localBuild = !!v.localBuild; + record.runtime = v.runtime; await cache.saveRecording(record); recordings.push(record); } diff --git a/src/apphosting/compare/types.ts b/src/apphosting/compare/types.ts index 63bdbdc8ff2..e9eec4b0bbd 100644 --- a/src/apphosting/compare/types.ts +++ b/src/apphosting/compare/types.ts @@ -28,8 +28,15 @@ export interface CompareResponse { results: DashboardComparisonResult[]; } +export interface VariantMetadata { + id: string; + localBuild: boolean; + runtime: string; +} + export interface MatrixResponse { testCase: string; variants: string[]; + variantsMetadata: Record; matrix: Record>; } From 429333896b594706b60deaabdbee11ec4ea6863b Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Wed, 17 Jun 2026 19:29:01 -0400 Subject: [PATCH 09/12] feat(apphosting): implement fully dynamic, autocomplete-searchable metadata dropdowns on comparison dashboard --- src/apphosting/compare/server.ts | 322 +++++++++++++++++++++++++------ 1 file changed, 264 insertions(+), 58 deletions(-) diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts index cb913303f5f..c7c45e2c156 100644 --- a/src/apphosting/compare/server.ts +++ b/src/apphosting/compare/server.ts @@ -659,9 +659,114 @@ function getDashboardHtml(): string { color: var(--text-muted); opacity: 0.7; } - .diff-text { - flex: 1; - white-space: pre-wrap; + /* Filter Dropdowns styles */ + .filter-dropdown-container { + position: relative; + display: inline-block; + } + + .filter-dropdown-btn { + background-color: var(--bg-dark); + border: 1px solid var(--border); + color: var(--text); + padding: 6px 12px; + border-radius: 6px; + font-family: var(--font-family); + font-size: 12px; + font-weight: 500; + cursor: pointer; + display: flex; + align-items: center; + gap: 6px; + outline: none; + transition: border-color 0.2s, background-color 0.2s; + } + + .filter-dropdown-btn:hover { + border-color: var(--accent); + background-color: rgba(255,255,255,0.02); + } + + .filter-dropdown-btn::after { + content: ""; + border: solid var(--text-muted); + border-width: 0 1.5px 1.5px 0; + display: inline-block; + padding: 2px; + transform: rotate(45deg); + margin-left: 4px; + transition: transform 0.2s; + } + + .filter-dropdown-container.open .filter-dropdown-btn::after { + transform: rotate(-135deg); + } + + .filter-dropdown-content { + display: none; + position: absolute; + background-color: var(--bg-panel); + border: 1px solid var(--border); + border-radius: 8px; + box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.5); + z-index: 100; + min-width: 180px; + max-width: 250px; + margin-top: 4px; + padding: 8px; + box-sizing: border-box; + } + + .filter-dropdown-container.open .filter-dropdown-content { + display: block; + } + + .filter-search-box { + background-color: var(--bg-dark); + border: 1px solid var(--border); + color: var(--text); + width: 100%; + padding: 6px 8px; + border-radius: 4px; + font-family: var(--font-family); + font-size: 11px; + box-sizing: border-box; + outline: none; + margin-bottom: 8px; + } + + .filter-search-box:focus { + border-color: var(--accent); + } + + .filter-options-list { + max-height: 180px; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 4px; + } + + .filter-opt-item { + display: flex; + align-items: center; + gap: 8px; + padding: 6px 8px; + border-radius: 4px; + cursor: pointer; + user-select: none; + font-size: 12px; + color: var(--text); + transition: background-color 0.15s; + } + + .filter-opt-item:hover { + background-color: rgba(255,255,255,0.04); + } + + .filter-opt-item input[type="checkbox"] { + cursor: pointer; + margin: 0; } @@ -730,15 +835,7 @@ function getDashboardHtml(): string { Search Variants: -
    - Build Origin: - - -
    -
    - Runtimes: - -
    +
    @@ -919,6 +1016,47 @@ function getDashboardHtml(): string { await loadHeatmap(tc); } + // Close dropdowns if clicked outside + window.addEventListener("click", (e) => { + document.querySelectorAll(".filter-dropdown-container").forEach(container => { + if (!container.contains(e.target)) { + container.classList.remove("open"); + } + }); + }); + + function toggleDropdown(container, event) { + event.stopPropagation(); + const wasOpen = container.classList.contains("open"); + + // Close other dropdowns + document.querySelectorAll(".filter-dropdown-container").forEach(c => c.classList.remove("open")); + + if (!wasOpen) { + container.classList.add("open"); + const searchInput = container.querySelector(".filter-search-box"); + if (searchInput) { + searchInput.value = ""; + // Reset visibility of option items + container.querySelectorAll(".filter-opt-item").forEach(item => item.style.display = "flex"); + searchInput.focus(); + } + } + } + + function filterDropdownOptions(input) { + const query = input.value.toLowerCase(); + const container = input.closest(".filter-dropdown-container"); + container.querySelectorAll(".filter-opt-item").forEach(item => { + const val = item.dataset.value.toLowerCase(); + if (val.includes(query)) { + item.style.display = "flex"; + } else { + item.style.display = "none"; + } + }); + } + async function loadHeatmap(tc) { document.getElementById("dashboard-empty-state").style.display = "none"; document.getElementById("heatmap-card").style.display = "flex"; @@ -928,73 +1066,141 @@ function getDashboardHtml(): string { const res = await fetch(\`/api/matrix?testCase=\${tc}\`); lastMatrixData = await res.json(); - // Dynamically populate runtime checkboxes - const container = document.getElementById("runtime-filters-container"); - container.innerHTML = \`Runtimes:\`; + // Reset search field + document.getElementById("variant-search-input").value = ""; - const runtimes = new Set(); - if (lastMatrixData.variantsMetadata) { - Object.values(lastMatrixData.variantsMetadata).forEach(meta => { - if (meta.runtime) runtimes.add(meta.runtime); - }); + // Build Dynamic Dropdown Filters + const filtersBar = document.getElementById("heatmap-dynamic-filters"); + filtersBar.innerHTML = ""; + + if (!lastMatrixData.variantsMetadata) { + applyMetadataFilters(); + return; } - if (runtimes.size <= 1) { - // If 0 or 1 runtime, hide runtime filters section to keep UI clean - container.style.display = "none"; - } else { - container.style.display = "flex"; - Array.from(runtimes).sort().forEach(rt => { - const lbl = document.createElement("label"); - lbl.style.cssText = "display: flex; align-items: center; gap: 4px; cursor: pointer; user-select: none;"; + // Gather unique values for each metadata property + const properties = {}; + Object.values(lastMatrixData.variantsMetadata).forEach(meta => { + Object.entries(meta).forEach(([key, val]) => { + if (key === "id") return; // Skip ID + + properties[key] = properties[key] || new Set(); + if (key === "localBuild") { + properties[key].add(val ? "Local" : "Source"); + } else { + properties[key].add(val === undefined ? "default" : String(val)); + } + }); + }); + + // Render a dropdown for each property + Object.entries(properties).forEach(([propName, valuesSet]) => { + const uniqueValues = Array.from(valuesSet).sort(); + + // Create Dropdown Container + const dropdownContainer = document.createElement("div"); + dropdownContainer.className = "filter-dropdown-container"; + dropdownContainer.dataset.prop = propName; + + // Button + const btn = document.createElement("button"); + btn.className = "filter-dropdown-btn"; + const formattedPropName = propName === "localBuild" ? "Build Origin" : propName.charAt(0).toUpperCase() + propName.slice(1); + btn.textContent = \`\${formattedPropName}: All\`; + btn.onclick = (e) => toggleDropdown(dropdownContainer, e); + dropdownContainer.appendChild(btn); + + // Content panel + const content = document.createElement("div"); + content.className = "filter-dropdown-content"; + + // Autocomplete search box + const searchInput = document.createElement("input"); + searchInput.type = "text"; + searchInput.className = "filter-search-box"; + searchInput.placeholder = "Search values..."; + searchInput.oninput = () => filterDropdownOptions(searchInput); + content.appendChild(searchInput); + + // Options list container + const optionsList = document.createElement("div"); + optionsList.className = "filter-options-list"; + + uniqueValues.forEach(val => { + const item = document.createElement("label"); + item.className = "filter-opt-item"; + item.dataset.value = val; const chk = document.createElement("input"); chk.type = "checkbox"; - chk.className = "runtime-filter-chk"; - chk.dataset.runtime = rt; - chk.checked = true; - chk.onchange = applyMetadataFilters; - - lbl.appendChild(chk); - lbl.appendChild(document.createTextNode(rt)); - container.appendChild(lbl); + chk.className = "filter-opt-chk"; + chk.value = val; + chk.checked = true; // checked by default + chk.onchange = () => { + updateDropdownButtonLabel(dropdownContainer, btn, formattedPropName); + applyMetadataFilters(); + }; + + item.appendChild(chk); + item.appendChild(document.createTextNode(val)); + optionsList.appendChild(item); }); - } - // Reset search field and other filters - document.getElementById("variant-search-input").value = ""; - document.getElementById("filter-local-builds").checked = true; - document.getElementById("filter-source-deploys").checked = true; + content.appendChild(optionsList); + dropdownContainer.appendChild(content); + filtersBar.appendChild(dropdownContainer); + }); applyMetadataFilters(); } + function updateDropdownButtonLabel(container, btn, propLabel) { + const chks = container.querySelectorAll(".filter-opt-chk"); + const checked = container.querySelectorAll(".filter-opt-chk:checked"); + if (checked.length === chks.length) { + btn.textContent = \`\${propLabel}: All\`; + } else if (checked.length === 0) { + btn.textContent = \`\${propLabel}: None\`; + } else if (checked.length === 1) { + btn.textContent = \`\${propLabel}: \${checked[0].value}\`; + } else { + btn.textContent = \`\${propLabel}: (\${checked.length} selected)\`; + } + } + function applyMetadataFilters() { if (!lastMatrixData) return; const searchQuery = document.getElementById("variant-search-input").value.toLowerCase(); - const showLocal = document.getElementById("filter-local-builds").checked; - const showSource = document.getElementById("filter-source-deploys").checked; - const runtimeChks = document.querySelectorAll(".runtime-filter-chk"); - const activeRuntimes = Array.from(runtimeChks) - .filter(chk => chk.checked) - .map(chk => chk.dataset.runtime); + + // Gather active checkboxes per property + const activeSelections = {}; + document.querySelectorAll(".filter-dropdown-container").forEach(container => { + const propName = container.dataset.prop; + const checkedVals = Array.from(container.querySelectorAll(".filter-opt-chk:checked")).map(chk => chk.value); + activeSelections[propName] = new Set(checkedVals); + }); const filteredVariants = lastMatrixData.variants.filter(v => { - const meta = lastMatrixData.variantsMetadata[v] || { id: v, localBuild: false, runtime: "default" }; + // Search query check on variant name + if (!v.toLowerCase().includes(searchQuery)) return false; - // Match search query - const matchesSearch = v.toLowerCase().includes(searchQuery); - if (!matchesSearch) return false; + if (!lastMatrixData.variantsMetadata) return true; - // Match build origin - if (meta.localBuild && !showLocal) return false; - if (!meta.localBuild && !showSource) return false; + const meta = lastMatrixData.variantsMetadata[v] || {}; - // Match runtime (if checkboxes exist) - if (runtimeChks.length > 0) { - const rtVal = meta.runtime || "default"; - if (!activeRuntimes.includes(rtVal)) return false; + // Dynamic properties check + for (const propName of Object.keys(activeSelections)) { + let val = meta[propName]; + if (propName === "localBuild") { + val = val ? "Local" : "Source"; + } else { + val = val === undefined ? "default" : String(val); + } + + if (!activeSelections[propName].has(val)) { + return false; + } } return true; From 1775dcc80935ef6b51e5b3b06ce5011cdd8e4dbc Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Wed, 17 Jun 2026 19:41:48 -0400 Subject: [PATCH 10/12] docs(apphosting): remove obsolete next.js low similarity and express quirks from metrics legend --- src/apphosting/compare/server.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts index c7c45e2c156..e7fa22876e5 100644 --- a/src/apphosting/compare/server.ts +++ b/src/apphosting/compare/server.ts @@ -808,14 +808,12 @@ function getDashboardHtml(): string {
    - Understanding Parity Metrics & Next.js Quirks + Understanding Parity Metrics [Click to Expand]
    From 4e3e8ffa4501e8cacd9070fa47f01e933fe8ca4f Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Thu, 18 Jun 2026 11:49:31 -0400 Subject: [PATCH 11/12] Remediate pull request comments and secure vulnerabilities --- src/apphosting/compare/cache.ts | 53 ++++++++++++++---------- src/apphosting/compare/compare.ts | 5 ++- src/apphosting/compare/crawler.ts | 39 +++++++++-------- src/apphosting/compare/lifecycle.ts | 50 +++++----------------- src/apphosting/compare/reporter.ts | 19 ++++++--- src/apphosting/compare/server.ts | 37 ++++++++++++----- src/apphosting/compare/suite.spec.ts | 2 + src/apphosting/compare/suite.ts | 15 +++++-- src/commands/apphosting-compare-suite.ts | 3 ++ 9 files changed, 123 insertions(+), 100 deletions(-) diff --git a/src/apphosting/compare/cache.ts b/src/apphosting/compare/cache.ts index e9fa724fa79..5e0e1251eef 100644 --- a/src/apphosting/compare/cache.ts +++ b/src/apphosting/compare/cache.ts @@ -74,7 +74,12 @@ function getRecordingPath(testCaseName: string, variantId: string): string { const vHash = crypto.createHash("sha256").update(variantId).digest("hex").slice(0, 8); const safeTestCase = `${testCaseName.replace(/[^a-zA-Z0-9_-]/g, "_")}_${tcHash}`; const safeVariant = `${variantId.replace(/[^a-zA-Z0-9_-]/g, "_")}_${vHash}`; - return path.join(CACHE_DIR, "recordings", safeTestCase, `${safeVariant}.json`); + const resolvedPath = path.resolve(path.join(CACHE_DIR, "recordings", safeTestCase, `${safeVariant}.json`)); + + if (!resolvedPath.startsWith(path.resolve(CACHE_DIR))) { + throw new Error("Path traversal restriction violation"); + } + return resolvedPath; } /** @@ -122,31 +127,33 @@ export async function listRecordings(): Promise> { const tcDir = path.join(recordingsDir, tc); try { const stat = await fs.stat(tcDir); - if (stat.isDirectory()) { - const files = await fs.readdir(tcDir); - const jsonFiles = files.filter((file) => file.endsWith(".json")); - if (jsonFiles.length > 0) { - const variantIds: string[] = []; - let originalTestCaseName = ""; - for (const file of jsonFiles) { - try { - const data = await fs.readJson(path.join(tcDir, file)); - if (isVariantRecording(data)) { - originalTestCaseName = data.testCaseName; - variantIds.push(data.id); - } else { - logger.debug(`Cache file ${file} does not match VariantRecording schema`); - } - } catch (readErr) { - // If a file is partially written or corrupted, skip it - logger.debug(`Failed to read metadata for cache file: ${file}`, readErr); - } - } - if (originalTestCaseName && variantIds.length > 0) { - result[originalTestCaseName] = variantIds; + if (!stat.isDirectory()) { + continue; + } + const files = await fs.readdir(tcDir); + const jsonFiles = files.filter((file) => file.endsWith(".json")); + if (jsonFiles.length === 0) { + continue; + } + const variantIds: string[] = []; + let originalTestCaseName = ""; + for (const file of jsonFiles) { + try { + const data = await fs.readJson(path.join(tcDir, file)); + if (isVariantRecording(data)) { + originalTestCaseName = data.testCaseName; + variantIds.push(data.id); + } else { + logger.debug(`Cache file ${file} does not match VariantRecording schema`); } + } catch (readErr) { + // If a file is partially written or corrupted, skip it + logger.debug(`Failed to read metadata for cache file: ${file}`, readErr); } } + if (originalTestCaseName && variantIds.length > 0) { + result[originalTestCaseName] = variantIds; + } } catch (err: unknown) { if (isErrnoException(err) && err.code === "ENOENT") { continue; // Directory was concurrently deleted/moved diff --git a/src/apphosting/compare/compare.ts b/src/apphosting/compare/compare.ts index 465276db10e..2597d095f65 100644 --- a/src/apphosting/compare/compare.ts +++ b/src/apphosting/compare/compare.ts @@ -49,6 +49,7 @@ export async function compareRoute( const fetchOptions = { headers: options.headers || {}, redirect: "manual" as const, + size: 2 * 1024 * 1024, }; const [resA, resB] = await Promise.all([ @@ -62,10 +63,10 @@ export async function compareRoute( const isBinaryB = isBinaryContentType(contentTypeB); const headersA: Record = {}; - resA.headers.forEach((val, key) => { headersA[key] = val; }); + resA.headers.forEach((val, key) => { headersA[key.toLowerCase()] = val; }); const headersB: Record = {}; - resB.headers.forEach((val, key) => { headersB[key] = val; }); + resB.headers.forEach((val, key) => { headersB[key.toLowerCase()] = val; }); const responseA: RouteResponse = { status: resA.status, diff --git a/src/apphosting/compare/crawler.ts b/src/apphosting/compare/crawler.ts index 17ce716be55..36005011e33 100644 --- a/src/apphosting/compare/crawler.ts +++ b/src/apphosting/compare/crawler.ts @@ -1,12 +1,16 @@ import fetch from "node-fetch"; function decodeHtmlEntities(str: string): string { - return str - .replace(/&/g, "&") - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, '"') - .replace(/'/g, "'"); + const entities: Record = { + "amp": "&", + "lt": "<", + "gt": ">", + "quot": '"', + "#39": "'" + }; + return str.replace(/&(amp|lt|gt|quot|#39);/gi, (match, entity) => { + return entities[entity.toLowerCase()] || match; + }); } interface Destroyable { @@ -60,6 +64,7 @@ export class Crawler { redirect: "manual" as const, headers: { "User-Agent": "FirebaseCompareCrawler/1.0" }, signal: controller.signal, + size: 2 * 1024 * 1024, }); // Handle Redirects @@ -89,9 +94,7 @@ export class Crawler { const html = await res.text(); const links = this.extractLinks(html, canonical); - for (const link of links) { - await this.crawlRoute(link, depth + 1); - } + await Promise.all(links.map((link) => this.crawlRoute(link, depth + 1))); } catch (err) { // Ignore fetch failures for single routes during discovery } finally { @@ -130,18 +133,20 @@ export class Crawler { while ((match = regex.exec(html)) !== null) { const rawHref = match[2] !== undefined ? match[2] : match[3]; if (!rawHref) continue; - + const href = decodeHtmlEntities(rawHref.trim()); - if ( - !href || - href.startsWith("#") || - href.startsWith("mailto:") || - href.startsWith("tel:") || - href.startsWith("javascript:") - ) { + if (!href || href.startsWith("#")) { continue; } + const schemeMatch = href.match(/^[a-z][a-z0-9+.-]*:/i); + if (schemeMatch) { + const scheme = schemeMatch[0].toLowerCase(); + if (scheme !== "http:" && scheme !== "https:") { + continue; + } + } + const resolved = this.resolveRelative(currentRoute, href); if (resolved) { links.push(resolved); diff --git a/src/apphosting/compare/lifecycle.ts b/src/apphosting/compare/lifecycle.ts index cb0319b3cb9..460c86ef053 100644 --- a/src/apphosting/compare/lifecycle.ts +++ b/src/apphosting/compare/lifecycle.ts @@ -1,9 +1,15 @@ import { FirebaseError } from "../../error"; import * as apphosting from "../../gcp/apphosting"; import { logger } from "../../logger"; -import { acquireComparisonSlot, releaseComparisonSlot } from "./slots"; -const ALLOWED_PROJECTS = ["aryanf-test", "pretend-public"]; +const ALLOWED_PROJECTS = [ + "aryanf-test", + "pretend-public", + ...(process.env.APP_HOSTING_COMPARE_ALLOWED_PROJECTS || "") + .split(",") + .map((p) => p.trim()) + .filter(Boolean), +]; /** * @@ -23,7 +29,7 @@ export async function runGarbageCollection(projectId: string, location: string): const existingBackends = await apphosting.listBackends(projectId, location); const backendsList = existingBackends.backends || []; const now = Date.now(); - const thirtyMinutes = 30 * 60 * 1000; + const twoHours = 2 * 60 * 60 * 1000; for (const backend of backendsList) { const nameParts = backend.name.split("/"); @@ -33,7 +39,7 @@ export async function runGarbageCollection(projectId: string, location: string): const isBusy = backend.labels?.status === "busy"; if (isBusy) { const updateTime = new Date(backend.updateTime).getTime(); - if (now - updateTime > thirtyMinutes) { + if (now - updateTime > twoHours) { logger.info(`Found stale lock on comparison slot backend ${backendId}. Unlocking...`); try { await apphosting.updateBackend(projectId, location, backendId, { @@ -47,39 +53,3 @@ export async function runGarbageCollection(projectId: string, location: string): } } } - -/** - * - */ -export async function runAutonomousComparison( - projectId: string, - location: string, - localPath: string, - options: any, -): Promise { - validateProject(projectId); - await runGarbageCollection(projectId, location); - - const slot = await acquireComparisonSlot(projectId, location, 2); - - const cleanUpAndExit = async () => { - logger.warn("\nProcess interrupted. Cleaning up comparison slot lock before exit..."); - await releaseComparisonSlot(projectId, location, slot.index, 2); - process.exit(1); - }; - - process.on("SIGINT", cleanUpAndExit); - process.on("SIGTERM", cleanUpAndExit); - - try { - // Setup secrets, deploy, and compare... - logger.info( - `Using Comparison Slot ${slot.index} (Backend A: ${slot.backendIds[0]}, Backend B: ${slot.backendIds[1]})`, - ); - } finally { - process.off("SIGINT", cleanUpAndExit); - process.off("SIGTERM", cleanUpAndExit); - - await releaseComparisonSlot(projectId, location, slot.index, 2); - } -} diff --git a/src/apphosting/compare/reporter.ts b/src/apphosting/compare/reporter.ts index c1342c65365..5e96c4882d7 100644 --- a/src/apphosting/compare/reporter.ts +++ b/src/apphosting/compare/reporter.ts @@ -4,6 +4,15 @@ import * as clc from "colorette"; import { ComparisonResult } from "./compare"; import { logger } from "../../logger"; +function escapeHtml(str: string): string { + return str + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + export interface ComparisonSummary { projectId: string; location: string; @@ -85,7 +94,7 @@ export async function generateReport( await fs.ensureDir(outputDir); // Dump summary without the bodies to save space - const resultsWithoutBodies = results.map(r => { + const resultsWithoutBodies: ComparisonResult[] = results.map(r => { const { bodyA, bodyB, ...rest } = r; return rest; }); @@ -105,7 +114,7 @@ export async function generateReport( } logger.info(`Raw responses saved to ${routesDirA} and ${routesDirB} for manual diffing.`); - const html = getHtmlTemplate(summary, resultsWithoutBodies as any); + const html = getHtmlTemplate(summary, resultsWithoutBodies); await fs.outputFile(path.join(outputDir, "index.html"), html, "utf-8"); logger.info( `HTML Dashboard generated at: ${clc.underline(path.join(outputDir, "index.html"))}\n`, @@ -120,16 +129,16 @@ function getHtmlTemplate(summary: ComparisonSummary, results: ComparisonResult[] const badgeText = isPass ? "PASS" : "FAIL"; const headersList = r.headerMismatches - .map((m) => `
  • ${m.header}: "${m.valA}" vs "${m.valB}"
    • `) + .map((m) => `
  • ${escapeHtml(m.header)}: "${escapeHtml(m.valA)}" vs "${escapeHtml(m.valB)}"
    • `) .join(""); const variationsList = r.expectedHeaderVariations - .map((m) => `
  • ${m.header}: "${m.valA}" vs "${m.valB}"
    • `) + .map((m) => `
  • ${escapeHtml(m.header)}: "${escapeHtml(m.valA)}" vs "${escapeHtml(m.valB)}"
    • `) .join(""); return ` - ${r.route} + ${escapeHtml(r.route)} ${badgeText} ${r.statusMatch ? "Match" : "Mismatch"} ${(r.bodySimilarity * 100).toFixed(1)}% diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts index e7fa22876e5..e2ff55ee30f 100644 --- a/src/apphosting/compare/server.ts +++ b/src/apphosting/compare/server.ts @@ -3,6 +3,7 @@ import * as http from "http"; import { logger } from "../../logger"; import * as cache from "./cache"; import * as compare from "./compare"; +import * as diff from "diff"; import { CompareResponse, MatrixResponse, DashboardComparisonResult, VariantMetadata } from "./types"; export function startServer(port: number): Promise { @@ -74,7 +75,6 @@ export function startServer(port: number): Promise { const dashboardResult: DashboardComparisonResult = { ...compResult }; if (!resA.isBinary && !resB.isBinary) { - const diff = require("diff"); const changes = diff.diffLines(dashboardResult.bodyA || "", dashboardResult.bodyB || ""); // Filter/map to minimal JSON to keep payload clean dashboardResult.diffChanges = changes.map((c: any) => ({ @@ -207,7 +207,7 @@ export function startServer(port: number): Promise { app.get("/api/render", async (req, res) => { const { testCase, variant, route } = req.query; if (typeof testCase !== "string" || typeof variant !== "string" || typeof route !== "string") { - res.status(400).send("Missing or invalid query parameters: testCase, variant, and route must be strings."); + res.status(400).type("text/plain").send("Missing or invalid query parameters: testCase, variant, and route must be strings."); return; } try { @@ -223,7 +223,7 @@ export function startServer(port: number): Promise { const rec = await cache.loadRecording(tc, varId); const resData = rec.routes[route]; if (!resData) { - res.status(404).send("Route not found in cache"); + res.status(404).type("text/plain").send("Route not found in cache"); return; } if (resData.isBinary) { @@ -234,13 +234,17 @@ export function startServer(port: number): Promise { // Inject tag to fix relative assets let html = resData.body; if (!html.includes("", ``); + if (//i.test(html)) { + html = html.replace(//i, ``); + } else { + html = `` + html; + } } res.send(html); } } catch (err: unknown) { const errMsg = err instanceof Error ? err.message : String(err); - res.status(500).send(errMsg); + res.status(500).type("text/plain").send(errMsg); } }); @@ -1445,26 +1449,39 @@ function getDashboardHtml(): string { diffContainer.innerHTML = ""; if (res.isBinary) { - diffContainer.innerHTML = \`
    Binary File Comparison: \${res.bodyDiff || "Identical"}
    \`; + const div = document.createElement("div"); + div.className = "empty-state"; + div.textContent = "Binary File Comparison: " + (res.bodyDiff || "Identical"); + diffContainer.appendChild(div); return; } if (res.bodyA === undefined && res.bodyB === undefined) { - diffContainer.innerHTML = '
    No body content recorded
    '; + const div = document.createElement("div"); + div.className = "empty-state"; + div.textContent = "No body content recorded"; + diffContainer.appendChild(div); return; } if (res.bodyA === res.bodyB) { - diffContainer.innerHTML = '
    Body Content is 100% Identical
    '; + const div = document.createElement("div"); + div.className = "empty-state"; + div.style.color = "var(--success)"; + div.textContent = "Body Content is 100% Identical"; + diffContainer.appendChild(div); return; } if (!res.diffChanges || res.diffChanges.length === 0) { + const div = document.createElement("div"); + div.className = "empty-state"; if (res.bodyDiff) { - diffContainer.innerHTML = \`
    \${res.bodyDiff}
    \`; + div.textContent = res.bodyDiff; } else { - diffContainer.innerHTML = '
    No diff details available
    '; + div.textContent = "No diff details available"; } + diffContainer.appendChild(div); return; } diff --git a/src/apphosting/compare/suite.spec.ts b/src/apphosting/compare/suite.spec.ts index 7d37381e232..bb6d9f057b0 100644 --- a/src/apphosting/compare/suite.spec.ts +++ b/src/apphosting/compare/suite.spec.ts @@ -14,6 +14,7 @@ import * as reporterManager from "./reporter"; import * as fetchModule from "node-fetch"; import * as cache from "./cache"; import { runCompareSuite } from "./suite"; +import * as utils from "../../utils"; describe("runCompareSuite Orchestrator", () => { let tempDir: string; @@ -42,6 +43,7 @@ describe("runCompareSuite Orchestrator", () => { setupSecretsStub = sinon.stub(secretsManager, "setupSandboxSecrets").resolves([]); cleanupSecretsStub = sinon.stub(secretsManager, "cleanupSandboxSecrets").resolves(); discoverRoutesStub = sinon.stub(discoverManager, "discoverRoutes").resolves(["/"]); + sinon.stub(utils, "sleep").resolves(); compareRouteResponsesStub = sinon.stub(compareManager, "compareRouteResponses").resolves({ route: "/", diff --git a/src/apphosting/compare/suite.ts b/src/apphosting/compare/suite.ts index 9f7922bc9b0..7f96e574b78 100644 --- a/src/apphosting/compare/suite.ts +++ b/src/apphosting/compare/suite.ts @@ -87,7 +87,10 @@ async function deployToBackend( logger.info(`Triggering CLI deploy for backend ${backendId} (localBuild: ${useLocalBuild})...`); // Run exactly the same deployment path as a customer const experimentPrefix = useLocalBuild ? "FIREBASE_CLI_EXPERIMENTS=apphostinglocalbuilds " : ""; - const binPath = process.argv[1] || path.resolve(__dirname, "../../../bin/firebase.js"); + let binPath = path.resolve(__dirname, "../../bin/firebase.js"); + if (!fs.existsSync(binPath)) { + binPath = path.resolve(__dirname, "../../../lib/bin/firebase.js"); + } const cmd = `${experimentPrefix}node "${binPath}" deploy --only apphosting:${backendId} --project ${projectId} --config ${tempConfigName} --non-interactive --allow-local-build-secrets`; @@ -142,13 +145,14 @@ async function recordVariant( redirect: "manual" as const, headers: { "User-Agent": "FirebaseCompareCrawler/1.0" }, signal: controller.signal, + size: 2 * 1024 * 1024, }); const contentType = res.headers.get("content-type") || ""; const isBinary = isBinaryContentType(contentType); const headers: Record = {}; - res.headers.forEach((val, key) => { headers[key] = val; }); + res.headers.forEach((val, key) => { headers[key.toLowerCase()] = val; }); let body = ""; const contentLength = parseInt(res.headers.get("content-length") || "0", 10); @@ -272,7 +276,12 @@ export async function runCompareSuite( const backendDataList = await Promise.all( backendIds.map((id) => apphosting.getBackend(projectId, location, id)), ); - const urls = backendDataList.map((b) => (b.uri.startsWith("http") ? b.uri : `https://${b.uri}`)); + const urls = backendDataList.map((b) => { + if (!b.uri) { + throw new FirebaseError(`Backend ${b.name} has no valid URI. Deployment may have failed or is still provisioning.`); + } + return b.uri.startsWith("http") ? b.uri : `https://${b.uri}`; + }); for (let i = 0; i < variants.length; i++) { const v = variants[i]; diff --git a/src/commands/apphosting-compare-suite.ts b/src/commands/apphosting-compare-suite.ts index e3452c8066f..aa8ea9ea166 100644 --- a/src/commands/apphosting-compare-suite.ts +++ b/src/commands/apphosting-compare-suite.ts @@ -80,6 +80,9 @@ export const command = new Command("apphosting:compare-suite") await lifecycle.runGarbageCollection(projectId, location); // Compute max variants to acquire a slot large enough + if (suite.length === 0) { + throw new FirebaseError("Suite config must contain at least one test case."); + } const maxVariants = Math.max(...suite.map((tc: any) => tc.variants?.length || 0)); if (maxVariants < 2) { throw new FirebaseError("All test cases must have at least 2 variants."); From 70c5568d369b121c9e99afeb3c61497cfcdf8631 Mon Sep 17 00:00:00 2001 From: Aryan Falahatpisheh Date: Thu, 18 Jun 2026 12:14:17 -0400 Subject: [PATCH 12/12] Fix case-sensitive header comparisons to restore backward compatibility with existing recordings --- src/apphosting/compare/compare.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/apphosting/compare/compare.ts b/src/apphosting/compare/compare.ts index 2597d095f65..6b889e1b53a 100644 --- a/src/apphosting/compare/compare.ts +++ b/src/apphosting/compare/compare.ts @@ -113,14 +113,20 @@ export async function compareRouteResponses( }; // 1. Compare Headers + const normalizedHeadersA: Record = {}; + Object.entries(resA.headers).forEach(([k, v]) => { normalizedHeadersA[k.toLowerCase()] = v; }); + + const normalizedHeadersB: Record = {}; + Object.entries(resB.headers).forEach(([k, v]) => { normalizedHeadersB[k.toLowerCase()] = v; }); + const allHeaderKeys = new Set([ - ...Object.keys(resA.headers), - ...Object.keys(resB.headers), + ...Object.keys(normalizedHeadersA), + ...Object.keys(normalizedHeadersB), ]); for (const key of allHeaderKeys) { - const valA = resA.headers[key] || ""; - const valB = resB.headers[key] || ""; + const valA = normalizedHeadersA[key] || ""; + const valB = normalizedHeadersB[key] || ""; if (valA !== valB) { if (BEHAVIORAL_HEADERS.includes(key.toLowerCase())) { result.headerMismatches.push({ header: key, valA, valB });