feat(brand-evolution): per-target design-token extraction + wayback CDX fix

.changeset/brand-evolution.md

@@ -0,0 +1,19 @@
+---
+'@tangle-network/browser-agent-driver': minor
+---
+
+feat(jobs+reports): brand-kit / design-system extraction at every audit target
+
+Comparative-audit jobs can now extract the full deterministic design-token bundle (colors, font families, type scale, logos, font files, brand metadata, detected libraries) at every target — including every wayback snapshot. New `brand-evolution` report template renders a per-URL chronological view of palette and typography drift, with snapshot-to-snapshot deltas (colors added/removed, font family swaps, brand-meta changes, library adoption).
+
+**Spec:** add `audit.extractTokens: true` to a `JobSpec`. Each per-target output dir gets a `tokens.json` alongside `report.json`.
+
+**CLI:** `bad reports generate --template brand-evolution --job <id>`
+
+**AI SDK tools:** two new tools — `fetchTokens` (returns the per-target token summaries, optionally filtered to one URL's chronological series) and `diffTokens` (deterministic delta between two token summaries in the same job). `renderTemplate` now accepts `template: 'brand-evolution'`.
+
+The token extractor is the existing `extractDesignTokens` (no LLM, ~10s per target). Same deterministic-data / LLM-narrates contract as the rest of the reports surface — every callout in the brand-evolution report comes from a pure function of `tokens.json`.
+
+Verified end-to-end on `https://stripe.com/` 2014 → 2019 → 2024 wayback snapshots: pulled out the Whitney → Camphor → sohne-var typeface progression and the matching primary-color shifts (`#008cdd` → `#6772e5` → `#635bff`).
+
++12 new tests across `reports-tokens` and the queue/tools touch-ups. Total: 1460 passing.

.changeset/wayback-collapse-fix.md

@@ -0,0 +1,13 @@
+---
+'@tangle-network/browser-agent-driver': patch
+---
+
+fix(discover/wayback): use CDX `collapse=timestamp:6` instead of `limit` so longitudinal jobs span the requested window
+
+Symptom: a job with `since: 2012-01-01, until: 2024-01-01, snapshotsPerUrl: 4` against a popular site returned four snapshots all clustered in 2012-2013 instead of evenly across 2012-2024.
+
+Cause: the CDX call passed `limit: max(count*4, 50)`, which caps how many captures CDX returns *before* `sampleEvenly` runs. For sites with thousands of captures (Stripe, Linear, GitHub, etc.) the first 50 in chronological order are all from the start of the window, so even sampling could only produce early-window snapshots.
+
+Fix: drop `limit`, use `collapse=timestamp:6` (one capture per month). The row count is now bounded by the window length in months, which keeps payloads sane while ensuring captures are spread across the whole window.
+
+Verified: `discoverWaybackSnapshots('https://stripe.com/', { count: 5, since: '2012-01-01', until: '2024-01-01' })` now returns snapshots at 2012-02, 2015-03, 2018-03, 2021-02, 2024-01.

src/cli-jobs.ts

@@ -152,7 +152,7 @@ async function cmdCreate(opts: ParsedArgs): Promise<void> {
  * we can deterministically locate `report.json` after the audit returns.
  */
 async function buildAuditFn(_spec: JobSpec): Promise<AuditFn> {
-  const { runDesignAudit } = await import('./cli-design-audit.js')
+  const { runDesignAudit, extractDesignTokens } = await import('./cli-design-audit.js')
   let counter = 0
   return async (target, opts) => {
     const url = target.snapshotUrl ?? target.url
@@ -186,11 +186,29 @@ async function buildAuditFn(_spec: JobSpec): Promise<AuditFn> {
     const page = data.pages?.[0]
     const rollupScore = page?.auditResultV2?.rollup?.score ?? page?.rollup?.score ?? page?.score
     const pageType = page?.auditResultV2?.classification?.type ?? page?.classification?.type
+
+    let tokensPath: string | undefined
+    if (opts?.extractTokens) {
+      try {
+        const tokensDir = path.join(outputDir, 'tokens')
+        const { tokens } = await extractDesignTokens({ url, headless: opts?.headless ?? true, outputDir: tokensDir })
+        tokensPath = path.resolve(tokensDir, 'tokens.json')
+        // extractDesignTokens persists its own files; ensure tokens.json exists at the canonical path.
+        if (!fs.existsSync(tokensPath)) {
+          fs.writeFileSync(tokensPath, JSON.stringify(tokens, null, 2))
+        }
+      } catch (err) {
+        // Token extraction is additive — never let it fail the parent audit.
+        console.warn(`  ${chalk.dim('tokens:')} extraction failed for ${url}: ${(err as Error).message}`)
+      }
+    }
+
     return {
       runId: outputDir, // The output dir is the de-facto runId for jobs.
       resultPath: reportJson,
       rollupScore,
       pageType,
+      tokensPath,
     }
   }
 }

src/cli-reports.ts

@@ -23,6 +23,7 @@ import {
   renderLeaderboard,
   renderLongitudinal,
   renderBatchComparison,
+  renderBrandEvolution,
   renderJobHeader,
   narrateReport,
 } from './reports/index.js'
@@ -56,7 +57,7 @@ function parseArgs(argv: string[]): ReportArgs {
   return out
 }
 
-const TEMPLATES = new Set(['leaderboard', 'longitudinal', 'batch-comparison'])
+const TEMPLATES = new Set(['leaderboard', 'longitudinal', 'batch-comparison', 'brand-evolution'])
 
 export async function runReportsCli(args: string[]): Promise<void> {
   const sub = args[0]
@@ -76,6 +77,8 @@ export async function runReportsCli(args: string[]): Promise<void> {
     body = renderLeaderboard(rows, { topN: opts.top, byType: opts.byType, buckets: opts.buckets })
   } else if (opts.template === 'longitudinal') {
     body = renderLongitudinal(rows)
+  } else if (opts.template === 'brand-evolution') {
+    body = renderBrandEvolution(job)
   } else {
     body = renderBatchComparison(rows)
   }

src/discover/wayback.ts

@@ -111,7 +111,12 @@ export async function discoverWaybackSnapshots(url: string, opts: WaybackOptions
   const params = new URLSearchParams({
     url,
     output: 'json',
-    limit: String(Math.max(count * 4, 50)), // overcollect, then sample evenly
+    // collapse=timestamp:6 dedupes to one capture per month (yyyymm = 6 chars).
+    // Without this, CDX returns every capture in the window — which for popular
+    // sites is tens of thousands and silently skews `sampleEvenly` if combined
+    // with a `limit`. With the collapse, the row count is bounded by the
+    // window length in months, so we don't need a limit.
+    collapse: 'timestamp:6',
   })
   if (opts.since) params.set('from', isoToCdxStamp(opts.since))
   if (opts.until) params.set('to', isoToCdxStamp(opts.until, true))

src/jobs/queue.ts

@@ -21,6 +21,7 @@ export interface AuditFn {
     rollupScore?: number
     pageType?: string
     costUSD?: number
+    tokensPath?: string
   }>
 }
 
@@ -87,6 +88,7 @@ async function runOne(
       rollupScore: out.rollupScore,
       pageType: out.pageType,
       costUSD: out.costUSD,
+      tokensPath: out.tokensPath,
     }
   } catch (err) {
     const error = err as Error

src/jobs/types.ts

@@ -41,6 +41,12 @@ export interface AuditOptions {
   regulatoryContext?: RegulatoryContextTag[]
   headless?: boolean
   skipEthics?: boolean
+  /**
+   * Layer 8 add-on: also run the deterministic brand/design-token extractor at
+   * every target. Adds ~10s/target (no LLM). Output lands at
+   * `<resultPath dir>/tokens.json` and is surfaced via `JobResultEntry.tokensPath`.
+   */
+  extractTokens?: boolean
 }
 
 export interface JobSpec {
@@ -81,6 +87,8 @@ export interface JobResultEntry extends JobTarget {
   rollupScore?: number
   /** Page-type classification. */
   pageType?: string
+  /** Path to the tokens.json from the brand-kit extractor (when extractTokens=true). */
+  tokensPath?: string
 }
 
 export interface Job {

src/reports/index.ts

@@ -15,13 +15,17 @@ export {
   renderLeaderboard,
   renderLongitudinal,
   renderBatchComparison,
+  renderBrandEvolution,
   renderJobHeader,
 } from './templates.js'
 export type {
   LeaderboardRenderOpts,
   LongitudinalRenderOpts,
   BatchComparisonRenderOpts,
+  BrandEvolutionRenderOpts,
 } from './templates.js'
+export { aggregateTokens, diffTokens, groupByUrl } from './tokens.js'
+export type { TokenSummary, TokenDiff, TokenSeries } from './tokens.js'
 export { buildReportTools } from './tools.js'
 export type { ReportToolsContext, ReportToolSet } from './tools.js'
 export { narrateReport } from './narrate.js'

src/reports/templates.ts

@@ -9,6 +9,7 @@
 import type { Job } from '../jobs/types.js'
 import type { AggregateRow, LongitudinalRow } from './types.js'
 import { leaderboard, longitudinalFor, tierBuckets, compareRuns } from './aggregate.js'
+import { aggregateTokens, diffTokens, groupByUrl, type TokenSummary } from './tokens.js'
 
 export interface LeaderboardRenderOpts {
   title?: string
@@ -136,6 +137,73 @@ export function renderBatchComparison(rows: AggregateRow[], opts: BatchCompariso
   return lines.join('\n')
 }
 
+export interface BrandEvolutionRenderOpts {
+  title?: string
+  /** Max colors to render per snapshot (top by usage count). Default 6. */
+  topColors?: number
+}
+
+/**
+ * Render a per-URL brand-kit evolution: for each URL, list the snapshots in
+ * chronological order with their distinct color palette, font families, and
+ * detected libraries. Between consecutive snapshots, surface the delta
+ * (colors added/removed, families added/removed, brand-meta changes).
+ */
+export function renderBrandEvolution(job: Job, opts: BrandEvolutionRenderOpts = {}): string {
+  const summaries = aggregateTokens(job)
+  const lines: string[] = []
+  lines.push(`# ${opts.title ?? 'Brand & Design-System Evolution'}`)
+  lines.push('')
+  lines.push(`Generated: ${new Date().toISOString()}`)
+  lines.push('')
+
+  if (summaries.length === 0) {
+    lines.push(`_No tokens.json files were produced. Run the job with \`audit.extractTokens: true\` to enable._`)
+    return lines.join('\n')
+  }
+
+  const topColors = opts.topColors ?? 6
+  const series = groupByUrl(summaries)
+  for (const { url, snapshots } of series) {
+    lines.push(`## ${escapeMd(url)}`)
+    lines.push('')
+    for (let i = 0; i < snapshots.length; i++) {
+      const s = snapshots[i]
+      const captured = s.capturedAt ? s.capturedAt.slice(0, 10) : 'live'
+      lines.push(`### ${captured}`)
+      lines.push('')
+      const swatches = s.colors.slice(0, topColors).map(c => `\`${c.hex}\` ×${c.count}`).join(' · ')
+      lines.push(`**Top colors**: ${swatches || '_none extracted_'}`)
+      const families = s.fontFamilies.map(f => `${f.family} (${f.classification})`).join(', ')
+      lines.push(`**Font families**: ${families || '_none_'}`)
+      lines.push(`**Type-scale entries**: ${s.typeScaleEntries}`)
+      if (s.detectedLibraries.length > 0) lines.push(`**Detected libraries**: ${s.detectedLibraries.join(', ')}`)
+      if (s.brand?.themeColor) lines.push(`**Theme color**: \`${s.brand.themeColor}\``)
+      if (s.logos.length > 0) lines.push(`**Logos**: ${s.logos.length}`)
+      lines.push('')
+
+      // Snapshot-to-snapshot delta against the previous snapshot in the series.
+      const prev = snapshots[i - 1]
+      if (prev) {
+        const d = diffTokens(prev, s)
+        const callouts: string[] = []
+        if (d.colorsAdded.length > 0) callouts.push(`+${d.colorsAdded.length} new colors`)
+        if (d.colorsRemoved.length > 0) callouts.push(`−${d.colorsRemoved.length} removed`)
+        if (d.familiesAdded.length > 0) callouts.push(`+ ${d.familiesAdded.join(', ')}`)
+        if (d.familiesRemoved.length > 0) callouts.push(`− ${d.familiesRemoved.join(', ')}`)
+        if (d.librariesAdded.length > 0) callouts.push(`adopted ${d.librariesAdded.join(', ')}`)
+        if (d.librariesRemoved.length > 0) callouts.push(`dropped ${d.librariesRemoved.join(', ')}`)
+        if (d.brandChanges.length > 0) callouts.push(`brand meta: ${d.brandChanges.map(c => c.field).join(', ')} changed`)
+        if (callouts.length > 0) {
+          lines.push(`_Δ vs ${prev.capturedAt?.slice(0, 10) ?? 'previous'}: ${callouts.join(' · ')}_`)
+          lines.push('')
+        }
+      }
+    }
+  }
+  return lines.join('\n')
+}
+
 export function renderJobHeader(job: Job): string {
   const ok = job.results.filter(r => r.status === 'ok').length
   const fail = job.results.filter(r => r.status === 'failed').length

src/reports/tokens.ts

@@ -0,0 +1,132 @@
+/**
+ * Brand-kit / design-token aggregation across a job's targets.
+ *
+ * Reads each per-target `tokens.json` (produced when AuditOptions.extractTokens
+ * was true) and projects to a flat row shape so longitudinal evolution and
+ * batch comparison templates can render without re-implementing extraction.
+ *
+ * No LLM. Pure function of on-disk data — same contract as aggregate.ts.
+ */
+
+import * as fs from 'node:fs'
+import type { Job } from '../jobs/types.js'
+import type { DesignTokens, ColorToken, FontFamily } from '../types.js'
+
+export interface TokenSummary {
+  /** Seed URL (groups snapshots of the same site). */
+  url: string
+  /** Snapshot URL when wayback. */
+  snapshotUrl?: string
+  /** ISO datetime of capture. */
+  capturedAt?: string
+  /** Per-target runId (== outputDir for jobs). */
+  runId: string
+  /** Resolved on-disk path to tokens.json. */
+  tokensPath?: string
+  /** Top-level brand metadata (title, theme color, favicon, og image). */
+  brand: DesignTokens['brand']
+  /** All distinct colors, sorted desc by usage count. */
+  colors: ColorToken[]
+  /** Distinct typography families with classification + weight set. */
+  fontFamilies: FontFamily[]
+  /** Type-scale entry count (proxy for typographic complexity). */
+  typeScaleEntries: number
+  /** Logo asset URLs (svg + raster). */
+  logos: string[]
+  /** Loaded font-file URLs. */
+  fontFiles: string[]
+  /** Detected libraries (e.g. ['tailwind','radix-ui']). */
+  detectedLibraries: string[]
+}
+
+/** Read each ok result's tokens.json and project to TokenSummary. */
+export function aggregateTokens(job: Job): TokenSummary[] {
+  const out: TokenSummary[] = []
+  for (const r of job.results) {
+    if (r.status !== 'ok' || !r.tokensPath || !fs.existsSync(r.tokensPath)) continue
+    try {
+      const tokens = JSON.parse(fs.readFileSync(r.tokensPath, 'utf-8')) as DesignTokens
+      out.push({
+        url: r.url,
+        snapshotUrl: r.snapshotUrl,
+        capturedAt: r.capturedAt,
+        runId: r.runId ?? '',
+        tokensPath: r.tokensPath,
+        brand: tokens.brand ?? {},
+        colors: (tokens.colors ?? []).slice().sort((a, b) => (b.count ?? 0) - (a.count ?? 0)),
+        fontFamilies: tokens.typography?.families ?? [],
+        typeScaleEntries: tokens.typography?.scale?.length ?? 0,
+        logos: (tokens.logos ?? []).map(l => l.src ?? '').filter(Boolean),
+        fontFiles: (tokens.fontFiles ?? []).map(f => f.src).filter(Boolean),
+        detectedLibraries: tokens.detectedLibraries ?? [],
+      })
+    } catch {
+      // Skip corrupted token files.
+    }
+  }
+  return out
+}
+
+/**
+ * Diff between two TokenSummary records. Useful for "this URL evolved from
+ * 4 colors → 12 colors and dropped Helvetica for Inter" callouts.
+ */
+export interface TokenDiff {
+  colorsAdded: string[]
+  colorsRemoved: string[]
+  colorsCommon: number
+  familiesAdded: string[]
+  familiesRemoved: string[]
+  brandChanges: Array<{ field: keyof DesignTokens['brand']; before: string | undefined; after: string | undefined }>
+  librariesAdded: string[]
+  librariesRemoved: string[]
+}
+
+export function diffTokens(a: TokenSummary, b: TokenSummary): TokenDiff {
+  const aHex = new Set(a.colors.map(c => c.hex.toLowerCase()))
+  const bHex = new Set(b.colors.map(c => c.hex.toLowerCase()))
+  const colorsAdded = [...bHex].filter(h => !aHex.has(h))
+  const colorsRemoved = [...aHex].filter(h => !bHex.has(h))
+  const colorsCommon = [...aHex].filter(h => bHex.has(h)).length
+
+  const aFam = new Set(a.fontFamilies.map(f => f.family))
+  const bFam = new Set(b.fontFamilies.map(f => f.family))
+  const familiesAdded = [...bFam].filter(f => !aFam.has(f))
+  const familiesRemoved = [...aFam].filter(f => !bFam.has(f))
+
+  const brandFields: Array<keyof DesignTokens['brand']> = ['title', 'description', 'themeColor', 'favicon', 'ogImage']
+  const brandChanges = brandFields
+    .filter(f => (a.brand?.[f] ?? '') !== (b.brand?.[f] ?? ''))
+    .map(f => ({ field: f, before: a.brand?.[f], after: b.brand?.[f] }))
+
+  const aLib = new Set(a.detectedLibraries)
+  const bLib = new Set(b.detectedLibraries)
+  const librariesAdded = [...bLib].filter(l => !aLib.has(l))
+  const librariesRemoved = [...aLib].filter(l => !bLib.has(l))
+
+  return { colorsAdded, colorsRemoved, colorsCommon, familiesAdded, familiesRemoved, brandChanges, librariesAdded, librariesRemoved }
+}
+
+/**
+ * Group token summaries by URL and return a chronological evolution series.
+ * Returns one entry per URL; each carries the sequence of TokenSummary rows
+ * sorted by capturedAt (or insertion order when capturedAt is missing).
+ */
+export interface TokenSeries {
+  url: string
+  snapshots: TokenSummary[]
+}
+
+export function groupByUrl(summaries: TokenSummary[]): TokenSeries[] {
+  const map = new Map<string, TokenSummary[]>()
+  for (const s of summaries) {
+    if (!map.has(s.url)) map.set(s.url, [])
+    map.get(s.url)!.push(s)
+  }
+  const out: TokenSeries[] = []
+  for (const [url, snapshots] of map.entries()) {
+    snapshots.sort((a, b) => (a.capturedAt ?? '').localeCompare(b.capturedAt ?? ''))
+    out.push({ url, snapshots })
+  }
+  return out
+}