From a5c53e83a20b4ade5c97367db9e60c20879d4690 Mon Sep 17 00:00:00 2001 From: f1shy-dev <56125930+f1shy-dev@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:22:12 +0000 Subject: [PATCH 1/2] Enable React trace analysis and fix large trace parsing by streaming gz traces and optimizing lazy Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- src/drivers/devtools.ts | 244 +++++++++++++++++++++++++---------- src/drivers/nextjs-bundle.ts | 3 +- tests/kernel-e2e.test.ts | 112 +++++++++++++++- 3 files changed, 282 insertions(+), 77 deletions(-) diff --git a/src/drivers/devtools.ts b/src/drivers/devtools.ts index e5b59bc..7712363 100644 --- a/src/drivers/devtools.ts +++ b/src/drivers/devtools.ts @@ -96,7 +96,7 @@ function parseTracePayload(payload: unknown): ParsedTrace { async function parseTraceFile(filePath: string): Promise { const stat = statSync(filePath); const isGzip = filePath.endsWith(".gz"); - const useStreaming = isGzip ? stat.size > 40 * 1024 * 1024 : stat.size > 256 * 1024 * 1024; + const useStreaming = isGzip || stat.size > 256 * 1024 * 1024; if (!useStreaming) { const text = await readMaybeGzipText(filePath); @@ -479,6 +479,19 @@ function parseDetail(detail: unknown): Record { } } +function normalizeUserTimingName(name: string | undefined) { + return name?.replace(/^[\u200B-\u200D\uFEFF]+/, ""); +} + +function getRenderMeasureKind(name: string | undefined, track: string | undefined) { + if (track === "Components" || track === "Components ⚛") { + return name === "Mount" || name === "Unmount" ? "lifecycle" : "component"; + } + if (track === "React Profile" || track === "React Profile ⚛") return "component"; + if (track === "Blocking") return "scheduler"; + return "measure"; +} + function buildRenderMeasures(trace: ParsedTrace) { const begins = new Map(); trace.traceEvents.forEach((event) => { @@ -499,14 +512,20 @@ function buildRenderMeasures(trace: ParsedTrace) { const properties = Array.isArray(detail.devtools?.properties) ? detail.devtools.properties : []; + const measureName = normalizeUserTimingName(begin?.name); + const track = getNestedString(detail.devtools?.track); + const kind = getRenderMeasureKind(measureName, track); return { renderMeasureId: `render:${index}`, eventId: `evt:${index}`, traceId, - componentName: begin?.name ?? undefined, + kind, + measureName, + componentName: kind === "component" ? measureName : undefined, tsUs: event.ts, durationMs: (event.dur ?? 0) / 1000, - track: getNestedString(detail.devtools?.track), + track, + trackGroup: getNestedString(detail.devtools?.trackGroup), tooltipText: getNestedString(detail.devtools?.tooltipText), properties, propKeys: properties @@ -518,7 +537,7 @@ function buildRenderMeasures(trace: ParsedTrace) { }, }; }) - .filter((row) => row.componentName); + .filter((row) => row.measureName); } function buildScripts(trace: ParsedTrace) { @@ -1556,7 +1575,8 @@ function buildRenderComponentHotspots( ) { const groups = new Map(); renders.forEach((row) => { - const componentName = row.componentName ?? "(unknown)"; + const componentName = row.componentName; + if (!componentName) return; if (!groups.has(componentName)) { groups.set(componentName, { componentName, @@ -1591,9 +1611,14 @@ function buildInteractionRenders( .flatMap((interaction) => { const groups = new Map(); renders - .filter((row) => row.tsUs >= interaction.startTsUs && row.tsUs <= interaction.endTsUs) + .filter( + (row) => + row.componentName && + row.tsUs >= interaction.startTsUs && + row.tsUs <= interaction.endTsUs, + ) .forEach((row) => { - const componentName = row.componentName ?? "(unknown)"; + const componentName = row.componentName!; if (!groups.has(componentName)) { groups.set(componentName, { interactionId: interaction.interactionId, @@ -1710,64 +1735,126 @@ function buildRequestBodies(trace: ParsedTrace, requests: ReturnType(key: string, fallback: T): Promise => { - try { - return await session.layers.get(key); - } catch (error) { - errors.push(`${key}: ${error instanceof Error ? error.message : String(error)}`); - return fallback; - } - }; - try { - const trace = await getLayer("devtools/trace", null); - const facts = await getLayer("devtools/facts.events", []); - const indexes = await getLayer("devtools/indexes.basic", null); - const processes = await getLayer("devtools/dims.processes", []); - const screenshots = await getLayer("devtools/dims.screenshots", []); - const requests = await getLayer("devtools/dims.requests", []); - const requestBodies = await getLayer("devtools/dims.requestBodies", []); - const interactions = await getLayer("devtools/dims.interactions", []); - const scripts = await getLayer("devtools/dims.scripts", []); - const sourceMaps = await getLayer("code/dims.sourceMaps", []); - const sources = await getLayer("code/dims.sources", []); - const frames = await getLayer("devtools/dims.frames", []); - const workers = await getLayer("devtools/dims.workers", []); - const layoutShifts = await getLayer("devtools/dims.layoutShifts", []); - const layoutShiftClusters = await getLayer("devtools/views.layoutShiftClusters", []); - const softNavigations = await getLayer("devtools/dims.softNavigations", []); - const framePipeline = await getLayer("devtools/views.framePipeline", []); - const codeHotspots = await getLayer("devtools/views.codeHotspots", []); - const cpuHotspots = await getLayer("devtools/views.cpuHotspots", []); - const cpuSamples = await getLayer("devtools/facts.cpuSamples", []); - const traceEvents = trace?.traceEvents ?? []; - const { minTs, maxTs } = - traceEvents.length > 0 ? getTraceBounds(traceEvents) : { minTs: 0, maxTs: 0 }; + const trace = await session.layers.get("devtools/trace"); + const traceEvents = trace.traceEvents; + const categories = new Set(); + const threads = new Set(); + const processes = new Set(); + const requests = new Set(); + const interactions = new Set(); + const scripts = new Set(); + const frames = new Set(); + const workers = new Set(); + const workerThreads = new Set(); + const softNavigations = new Set(); + const codeHotspots = new Set(); + const cpuHotspots = new Set(); + const { threadNames, processNames } = buildThreadMetadata(traceEvents); + let screenshots = 0; + let networkBodies = 0; + let layoutShifts = 0; + let frameReports = 0; + let cpuSamples = 0; + + traceEvents.forEach((event, index) => { + splitCategories(event.cat).forEach((category) => categories.add(category)); + threads.add(getThreadKey(event.pid, event.tid)); + processes.add(event.pid); + if (event.name === "Screenshot" && typeof event.args?.snapshot === "string") screenshots += 1; + const data = isRecord(event.args?.data) + ? (event.args!.data as Record) + : undefined; + const requestId = getNestedString(data?.requestId); + if (requestId) { + requests.add(requestId); + networkBodies += findEmbeddedBlobs(data).filter((blob) => blob.path !== "$").length; + } + const interactionId = + typeof data?.interactionId === "number" && Number.isFinite(data.interactionId) + ? String(data.interactionId) + : undefined; + if (event.name === "EventTiming" && data) { + interactions.add( + interactionId && interactionId !== "0" + ? interactionId + : `${getNestedString(data.type) ?? "unknown"}@${event.ts}`, + ); + } + const scriptId = canonicalId(data?.scriptId); + if (scriptId) scripts.add(scriptId); + const frameId = + getNestedString(data?.frame) ?? + getNestedString((event.args as any)?.frame) ?? + getNestedString((event.args as any)?.beginData?.frame); + if (frameId) frames.add(frameId); + const workerId = canonicalId(data?.workerId ?? (event.args as any)?.workerId); + if (workerId) workers.add(workerId); + if (event.name === "LayoutShift" && data) layoutShifts += 1; + if (event.name.startsWith("SoftNavigation")) { + softNavigations.add( + canonicalId((event.args as any)?.context?.softNavContextId) ?? `soft-nav:${index}`, + ); + } + if (event.name === "PipelineReporter" && isRecord((event.args as any)?.frame_reporter)) { + frameReports += 1; + } + if ((event.name === "FunctionCall" || event.name === "EvaluateScript") && event.ph === "X") { + const url = + getNestedString(data?.url) ?? getNestedString((event.args as any)?.url) ?? "(unknown)"; + const functionName = getNestedString(data?.functionName) ?? event.name; + codeHotspots.add(`${scriptId ?? "?"}|${url}|${functionName}`); + } + if (event.name === "ProfileChunk") { + const sampleIds = Array.isArray(data?.cpuProfile?.samples) ? data.cpuProfile.samples : []; + for (const sampleNodeId of sampleIds) { + if (typeof sampleNodeId !== "number") continue; + cpuSamples += 1; + cpuHotspots.add(sampleNodeId); + } + } + }); + for (const [threadKey, threadName] of threadNames) { + const [pidText] = threadKey.split(":"); + if (/worker/i.test(threadName) || /worker/i.test(processNames.get(Number(pidText)) ?? "")) { + workerThreads.add(threadKey); + } + } + const sourceMaps = Array.isArray(trace.metadata.sourceMaps) + ? (trace.metadata.sourceMaps as SourceMapEntry[]) + : []; + const sources = sourceMaps.reduce( + (count, entry) => + count + (Array.isArray(entry.sourceMap?.sources) ? entry.sourceMap.sources.length : 0), + 0, + ); + const layoutShiftClusters = buildLayoutShiftClusters(trace, buildLayoutShifts(trace)); + const { minTs, maxTs } = getTraceBounds(traceEvents); return { - ...(errors.length > 0 ? { error: errors.join("; ") } : {}), totalEvents: traceEvents.length, durationMs: traceEvents.length > 0 ? (maxTs - minTs) / 1000 : 0, - categories: indexes?.byCategory?.size ?? 0, - threads: indexes?.byThread?.size ?? 0, - processes: processes.length, - screenshots: screenshots.length, - networkRequests: requests.length, - networkBodies: requestBodies.length, - interactions: interactions.length, - scripts: scripts.length, + categories: categories.size, + threads: threads.size, + processes: processes.size, + screenshots, + networkRequests: requests.size, + networkBodies, + interactions: interactions.size, + scripts: scripts.size, sourceMaps: sourceMaps.length, - sources: sources.length, - frames: frames.length, - workers: workers.length, - layoutShifts: layoutShifts.length, + sources, + frames: frames.size, + workers: + workers.size + + [...workerThreads].filter((threadKey) => !workers.has(`thread:${threadKey}`)).length, + layoutShifts, layoutShiftClusters: layoutShiftClusters.length, - softNavigations: softNavigations.length, - frameReports: framePipeline.length, - codeHotspots: codeHotspots.length, - cpuHotspots: cpuHotspots.length, - cpuSamples: cpuSamples.length, - facts: facts.length, + softNavigations: softNavigations.size, + frameReports, + codeHotspots: codeHotspots.size, + cpuHotspots: cpuHotspots.size, + cpuSamples, + facts: traceEvents.length, }; } catch (error) { return { @@ -2435,25 +2522,37 @@ const requestBodiesCollection: FileCollectionProvider = { }; async function buildCapabilityMap(session: DatasetSession): Promise { - const [trace, screenshots, scripts, sourceMaps, sources, requestBodies] = await Promise.all([ - session.layers.get("devtools/trace"), - session.layers.get("devtools/dims.screenshots"), - session.layers.get("devtools/dims.scripts"), - session.layers.get("code/dims.sourceMaps"), - session.layers.get("code/dims.sources"), - session.layers.get("devtools/dims.requestBodies"), - ]); + const trace = await session.layers.get("devtools/trace"); const events = trace.traceEvents; + const sourceMaps = Array.isArray(trace.metadata.sourceMaps) + ? (trace.metadata.sourceMaps as SourceMapEntry[]) + : []; return { - screenshots: screenshots.length > 0, + screenshots: events.some( + (event) => event.name === "Screenshot" && typeof event.args?.snapshot === "string", + ), cpuProfile: events.some((event) => event.name === "ProfileChunk"), eventTiming: events.some((event) => event.name === "EventTiming"), framePipeline: events.some((event) => event.name === "PipelineReporter"), networkTiming: events.some((event) => event.name === "ResourceSendRequest"), - networkBodies: requestBodies.length > 0, - inlineScriptSource: scripts.some((row) => row.hasSourceText), + networkBodies: events.some((event) => { + const data = isRecord(event.args?.data) + ? (event.args!.data as Record) + : undefined; + return ( + !!getNestedString(data?.requestId) && + findEmbeddedBlobs(data).some((blob) => blob.path !== "$") + ); + }), + inlineScriptSource: events.some( + (event) => getNestedString(event.args?.data?.sourceText) !== undefined, + ), sourceMaps: sourceMaps.length > 0, - sourceContents: sources.some((row) => row.hasContent), + sourceContents: sourceMaps.some((entry) => + Array.isArray(entry.sourceMap?.sourcesContent) + ? entry.sourceMap.sourcesContent.some((content) => typeof content === "string") + : false, + ), renderUserTiming: events.some((event) => splitCategories(event.cat).includes("blink.user_timing"), ), @@ -3138,9 +3237,12 @@ export class DevtoolsDriver implements SourceDriver { description: "Parsed render measures derived from blink.user_timing and UserTiming::Measure", columns: [ { name: "renderMeasureId", type: "string" }, + { name: "kind", type: "string" }, + { name: "measureName", type: "string" }, { name: "componentName", type: "string" }, { name: "durationMs", type: "number", unit: "ms" }, { name: "track", type: "string" }, + { name: "trackGroup", type: "string" }, ], async rows(sessionRef, options) { const rows = await sessionRef.layers.get("devtools/views.renderMeasures"); diff --git a/src/drivers/nextjs-bundle.ts b/src/drivers/nextjs-bundle.ts index fa491f5..1024079 100644 --- a/src/drivers/nextjs-bundle.ts +++ b/src/drivers/nextjs-bundle.ts @@ -982,7 +982,8 @@ export class NextjsBundleDriver implements SourceDriver { session.registerNamespace("nextbundle", { report: { summary: async () => session.getReport("nextbundle.summary")!.run(session), - route: async (args) => session.getReport("nextbundle.route")!.run(session, args as Record), + route: async (args: Record) => + session.getReport("nextbundle.route")!.run(session, args), }, sources: async () => session.queryTable("nextbundle.dims.sources"), modules: async () => session.queryTable("nextbundle.dims.modules"), diff --git a/tests/kernel-e2e.test.ts b/tests/kernel-e2e.test.ts index 50b4ad0..978055c 100644 --- a/tests/kernel-e2e.test.ts +++ b/tests/kernel-e2e.test.ts @@ -563,6 +563,110 @@ describe("dataset kernel e2e", () => { expect(requestBodiesPayload.rows[0].mediaType).toBe("application/json"); }); + it("separates React component timings from scheduler and lifecycle measures", async () => { + const trace = sampleTrace(); + trace.traceEvents.push( + { + cat: "blink.user_timing", + name: "Update", + ph: "b", + pid: 1, + tid: 1, + ts: 3000, + args: { + traceId: 200, + detail: JSON.stringify({ + devtools: { track: "Blocking", trackGroup: "Scheduler ⚛" }, + }), + }, + }, + { + cat: "devtools.timeline", + name: "UserTiming::Measure", + ph: "X", + pid: 1, + tid: 1, + ts: 3010, + dur: 10, + args: { sampleTraceId: 200 }, + }, + { + cat: "blink.user_timing", + name: "\u200BBusyList", + ph: "b", + pid: 1, + tid: 1, + ts: 3020, + args: { + traceId: 201, + detail: JSON.stringify({ devtools: { track: "Components ⚛" } }), + }, + }, + { + cat: "devtools.timeline", + name: "UserTiming::Measure", + ph: "X", + pid: 1, + tid: 1, + ts: 3030, + dur: 20, + args: { sampleTraceId: 201 }, + }, + { + cat: "blink.user_timing", + name: "Mount", + ph: "b", + pid: 1, + tid: 1, + ts: 3040, + args: { + traceId: 202, + detail: JSON.stringify({ devtools: { track: "Components ⚛" } }), + }, + }, + { + cat: "devtools.timeline", + name: "UserTiming::Measure", + ph: "X", + pid: 1, + tid: 1, + ts: 3050, + dur: 30, + args: { sampleTraceId: 202 }, + }, + ); + const sessionId = await loadSession(createTraceFile(trace)); + + const renderMeasures = await handleRequest( + new Request( + `http://trace-server/sessions/${sessionId}/tables/${encodeURIComponent("devtools.views.renderMeasures")}/query`, + { method: "POST" }, + ), + ).then(parseJson); + expect(renderMeasures.rows.find((row: any) => row.measureName === "Update").kind).toBe( + "scheduler", + ); + expect(renderMeasures.rows.find((row: any) => row.measureName === "BusyList")).toMatchObject({ + kind: "component", + componentName: "BusyList", + }); + expect(renderMeasures.rows.find((row: any) => row.measureName === "Mount").kind).toBe( + "lifecycle", + ); + + const componentHotspots = await handleRequest( + new Request( + `http://trace-server/sessions/${sessionId}/tables/${encodeURIComponent("devtools.views.renderComponentHotspots")}/query`, + { method: "POST" }, + ), + ).then(parseJson); + expect(componentHotspots.rows.map((row: any) => row.componentName)).toEqual([ + "VirtualItem", + "ChatBlock", + "BusyList", + ]); + }); + it("validates report ids and keeps summary resilient", async () => { const file = createTraceFile(sampleTrace()); const sessionId = await loadSession(file); @@ -648,9 +752,7 @@ describe("dataset kernel e2e", () => { expect(session).toBeTruthy(); const originalGet = session.layers.get.bind(session.layers); session.layers.get = async (key: string, signal?: AbortSignal) => { - if (key === "devtools/views.framePipeline") { - throw new Error("frame pipeline exploded"); - } + if (key !== "devtools/trace") throw new Error(`summary loaded derived layer: ${key}`); return originalGet(key, signal); }; @@ -662,8 +764,8 @@ describe("dataset kernel e2e", () => { expect(summaryResponse.status).toBe(200); const summaryPayload = await parseJson(summaryResponse); expect(summaryPayload.result.totalEvents).toBe(28); - expect(summaryPayload.result.frameReports).toBe(0); - expect(summaryPayload.result.error).toContain("frame pipeline exploded"); + expect(summaryPayload.result.frameReports).toBe(1); + expect(summaryPayload.result.error).toBeUndefined(); }); it("supports querying through the ds runtime", async () => { From 6375ce4285881fda8c6b15c85e63ab4190fc4057 Mon Sep 17 00:00:00 2001 From: f1shy-dev <56125930+f1shy-dev@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:32:33 +0000 Subject: [PATCH 2/2] Align summary and capability scans with extractable trace data Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- src/drivers/devtools.ts | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/drivers/devtools.ts b/src/drivers/devtools.ts index 7712363..8472fc6 100644 --- a/src/drivers/devtools.ts +++ b/src/drivers/devtools.ts @@ -1814,9 +1814,12 @@ async function buildSummary(session: DatasetSession) { } } }); - for (const [threadKey, threadName] of threadNames) { + for (const threadKey of threads) { const [pidText] = threadKey.split(":"); - if (/worker/i.test(threadName) || /worker/i.test(processNames.get(Number(pidText)) ?? "")) { + if ( + /worker/i.test(threadNames.get(threadKey) ?? "") || + /worker/i.test(processNames.get(Number(pidText)) ?? "") + ) { workerThreads.add(threadKey); } } @@ -2545,19 +2548,24 @@ async function buildCapabilityMap(session: DatasetSession): Promise getNestedString(event.args?.data?.sourceText) !== undefined, + (event) => + canonicalId(event.args?.data?.scriptId) !== undefined && + getNestedString(event.args?.data?.sourceText) !== undefined, ), sourceMaps: sourceMaps.length > 0, - sourceContents: sourceMaps.some((entry) => - Array.isArray(entry.sourceMap?.sourcesContent) - ? entry.sourceMap.sourcesContent.some((content) => typeof content === "string") - : false, + sourceContents: sourceMaps.some( + (entry) => + Array.isArray(entry.sourceMap?.sources) && + Array.isArray(entry.sourceMap?.sourcesContent) && + entry.sourceMap.sources.some( + (_source, index) => typeof entry.sourceMap?.sourcesContent?.[index] === "string", + ), ), renderUserTiming: events.some((event) => splitCategories(event.cat).includes("blink.user_timing"), ), layoutShift: events.some((event) => event.name === "LayoutShift"), - softNavigation: events.some((event) => event.name === "SoftNavigation"), + softNavigation: events.some((event) => event.name.startsWith("SoftNavigation")), }; }