From 8eae8de3d7cfa9843986a0f931720adaea14550e Mon Sep 17 00:00:00 2001 From: shadowusr Date: Tue, 12 May 2026 13:10:12 +0300 Subject: [PATCH 1/4] feat: implement inspect-result tool --- packages/mcp/test/server.e2e.test.ts | 1 + packages/tools/src/index.ts | 2 + .../tools/src/tools/inspect-result/index.ts | 98 +++++ .../tools/src/tools/inspect-result/schema.ts | 35 ++ .../tools/src/tools/test-results/index.ts | 81 +---- .../tools/src/tools/test-results/types.ts | 39 +- packages/tools/src/utils/html-report.ts | 2 +- packages/tools/src/utils/test-result-view.ts | 343 ++++++++++++++++++ .../tools/test/tools/inspect-result.test.ts | 140 +++++++ .../tools/test/tools/test-results.test.ts | 3 +- 10 files changed, 642 insertions(+), 102 deletions(-) create mode 100644 packages/tools/src/tools/inspect-result/index.ts create mode 100644 packages/tools/src/tools/inspect-result/schema.ts create mode 100644 packages/tools/src/utils/test-result-view.ts create mode 100644 packages/tools/test/tools/inspect-result.test.ts diff --git a/packages/mcp/test/server.e2e.test.ts b/packages/mcp/test/server.e2e.test.ts index aefaf6d..547cd73 100644 --- a/packages/mcp/test/server.e2e.test.ts +++ b/packages/mcp/test/server.e2e.test.ts @@ -22,6 +22,7 @@ const EXPECTED_TOOL_NAMES = [ "close-tab", // report tools "test-results", + "inspect-result", // session tools "launch", "attach", diff --git a/packages/tools/src/index.ts b/packages/tools/src/index.ts index 1eb49ce..ff0990c 100644 --- a/packages/tools/src/index.ts +++ b/packages/tools/src/index.ts @@ -15,6 +15,7 @@ import { closeTab } from "./tools/close-tab.js"; import { launchBrowser, launchBrowserWithOptions } from "./tools/launch-browser.js"; import { attachToBrowser } from "./tools/attach-to-browser.js"; import { closeBrowser } from "./tools/close-browser.js"; +import { inspectResult } from "./tools/inspect-result/index.js"; import { testResults } from "./tools/test-results/index.js"; // This function just ensures that every item on a type level is a Tool. AFAIK the only way to do this. @@ -40,6 +41,7 @@ export const tools = typeCheckedTools([ attachToBrowser, closeBrowser, testResults, + inspectResult, ]); export { launchBrowserWithOptions, ToolKind }; diff --git a/packages/tools/src/tools/inspect-result/index.ts b/packages/tools/src/tools/inspect-result/index.ts new file mode 100644 index 0000000..7473c78 --- /dev/null +++ b/packages/tools/src/tools/inspect-result/index.ts @@ -0,0 +1,98 @@ +import { readResultsFromReport, type ReporterTestResult } from "html-reporter/experimental/sdk"; +import { createErrorResponse, createSimpleResponse } from "../../responses/index.js"; +import { StandaloneTool, ToolKind } from "../../types.js"; +import { downloadReportIfNeeded } from "../../utils/html-report.js"; +import { toTestResultView } from "../../utils/test-result-view.js"; +import { inspectResultSchema } from "./schema.js"; + +export { inspectResultSchema } from "./schema.js"; + +function getLatestAttempt(results: readonly T[]): T { + return results.reduce((latest, result) => { + if ( + result.attempt > latest.attempt || + (result.attempt === latest.attempt && (result.timestamp ?? 0) >= (latest.timestamp ?? 0)) + ) { + return result; + } + + return latest; + }); +} + +function getInspectResultSelectionError( + results: readonly ReporterTestResult[], + args: { name: string; browser: string; attempt?: number }, +): string { + const resultsWithMatchingName = results.filter(result => result.fullName === args.name); + + if (!resultsWithMatchingName.length) { + return `No test result found with name "${args.name}". You can check what tests are available with the "test-results" tool.`; + } + + const resultsWithMatchingBrowser = resultsWithMatchingName.filter(result => result.browserId === args.browser); + if (!resultsWithMatchingBrowser.length) { + const availableBrowsers = [...new Set(resultsWithMatchingName.map(result => result.browserId))].sort(); + + return `No test result found with name "${args.name}" in browser "${args.browser}". Available browsers: ${availableBrowsers.join(", ")}.`; + } + + const availableAttempts = [...new Set(results.map(result => result.attempt))].sort(); + + return `No attempt ${args.attempt} found for test "${args.name}" in browser "${args.browser}". Available attempts: ${availableAttempts.join(", ")}.`; +} + +function findTestResult( + results: readonly ReporterTestResult[], + args: { name: string; browser: string; attempt?: number }, +): ReporterTestResult { + const matchingAttempts = results.filter( + result => result.fullName === args.name && result.browserId === args.browser, + ); + if (args.attempt === undefined) { + return getLatestAttempt(matchingAttempts); + } + + const matchingAttempt = matchingAttempts.find(result => result.attempt === args.attempt); + + if (!matchingAttempt) { + throw new Error(getInspectResultSelectionError(results, args)); + } + + return matchingAttempt; +} + +const inspectResultCb: StandaloneTool["cb"] = async args => { + try { + const reportPath = await downloadReportIfNeeded(args.report); + const results = await readResultsFromReport(reportPath); + const result = findTestResult(results, args); + const view = toTestResultView(result, undefined, { includeBase64: args.includeBase64 }); + + return createSimpleResponse(JSON.stringify(view, null, args.pretty ? 2 : undefined)); + } catch (error) { + console.error("Error inspecting test result from report:", error); + + return createErrorResponse( + "Error inspecting test result from report", + error instanceof Error ? error : undefined, + ); + } +}; + +export const inspectResult: StandaloneTool = { + kind: ToolKind.Standalone, + name: "inspect-result", + description: "Read a Testplane HTML report and inspect one test result attempt as JSON", + schema: inspectResultSchema, + cb: inspectResultCb, + cli: { + positional: ["report"], + section: "Reports", + examples: [ + 'testplane-cli inspect-result /path/to/html-report --name "checkout submits order" --browser chrome', + 'testplane-cli inspect-result /path/to/html-report --name "checkout submits order" --browser chrome --attempt 0', + 'testplane-cli inspect-result /path/to/html-report --name "checkout submits order" --browser chrome --no-pretty', + ], + }, +}; diff --git a/packages/tools/src/tools/inspect-result/schema.ts b/packages/tools/src/tools/inspect-result/schema.ts new file mode 100644 index 0000000..2e9075c --- /dev/null +++ b/packages/tools/src/tools/inspect-result/schema.ts @@ -0,0 +1,35 @@ +import { z } from "zod"; + +export const inspectResultSchema = { + report: z + .string() + .min(1) + .transform(value => value.trim()) + .describe("Path or URL to a Testplane HTML report directory, report HTML file, or databaseUrls.json"), + name: z + .string() + .min(1) + .transform(value => value.trim()) + .describe("Full test name to inspect"), + browser: z + .string() + .min(1) + .transform(value => value.trim()) + .describe("Browser id to inspect"), + attempt: z + .number() + .int() + .min(0, "--attempt must be >= 0") + .optional() + .describe("Attempt index to inspect. Defaults to the latest attempt for the selected test and browser"), + pretty: z.boolean().default(true).describe("Pretty-print JSON output. Use --no-pretty for compact JSON"), + includeBase64: z + .boolean() + .default(false) + .describe( + "Include base64 image payloads in result images and errors. Omitted by default to keep output compact", + ), +}; + +export const inspectResultObjectSchema = z.object(inspectResultSchema); +export type InspectResultArgs = z.output; diff --git a/packages/tools/src/tools/test-results/index.ts b/packages/tools/src/tools/test-results/index.ts index b421432..83796b4 100644 --- a/packages/tools/src/tools/test-results/index.ts +++ b/packages/tools/src/tools/test-results/index.ts @@ -3,12 +3,12 @@ import { randomUUID } from "node:crypto"; import { mkdir, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { stripVTControlCharacters } from "node:util"; import { StandaloneTool, ToolKind } from "../../types.js"; import { createErrorResponse, createSimpleResponse } from "../../responses/index.js"; -import { getImageError, getImageStateName, isMutedResult, downloadReportIfNeeded } from "../../utils/html-report.js"; -import { formatDuration, formatError, formatFileSize, formatTimestamp } from "../../utils/formatters.js"; +import { downloadReportIfNeeded } from "../../utils/html-report.js"; +import { formatDuration, formatFileSize, formatTimestamp } from "../../utils/formatters.js"; import { stringify } from "../../utils/strings.js"; +import { toTestResultView } from "../../utils/test-result-view.js"; import { filterTestResults, getResultStatusTags } from "./filters.js"; import { testResultsSchema } from "./schema.js"; import { @@ -61,25 +61,6 @@ function getStatusCounts(results: readonly ReporterTestResult[]): StatusCounts { return statusCounts; } -function getFailedImageInfo(result: ReporterTestResult): string | null { - const failedImage = result.imagesInfo?.find(imageInfo => { - const status = imageInfo.status; - - return status === "error" || status === "fail"; - }); - - if (!failedImage) { - return null; - } - - const stateName = getImageStateName(failedImage); - const state = stateName ? `state: ${stateName}` : null; - const error = formatError(getImageError(failedImage)); - const message = [state, error].filter((part): part is string => part !== null).join("; "); - - return message || null; -} - function formatStatusCounts(statusCounts: StatusCounts): string { return TEST_RESULT_STATUSES.map(status => [status, statusCounts[status]] as const) .map(([status, count]) => `${status}: ${count}`) @@ -100,56 +81,6 @@ function formatActiveFilters(filters: FilterOptions): string | null { return activeFilters.length ? activeFilters.join("; ") : null; } -function getErrorLine(result: ReporterTestResult): string | null { - const status = result.status; - const shouldMentionMissingError = status === "error" || status === "fail" || isMutedResult(result); - - const error = formatError(result.error) ?? getFailedImageInfo(result); - - return error ?? (shouldMentionMissingError ? "No error message" : null); -} - -/** Picks fields from a test result and returns a view object */ -export function toTestResultView(result: ReporterTestResult, fields: TestResultField[]): TestResultView { - const view: TestResultView = {}; - - for (const field of fields) { - switch (field) { - case "name": - view.name = result.fullName; - break; - case "status": - view.status = isMutedResult(result) ? "muted" : result.status; - break; - case "browser": - view.browser = result.browserId; - break; - case "attempt": - view.attempt = result.attempt; - break; - case "duration": - view.duration = result.duration ?? null; - break; - case "file": - view.file = result.file ?? null; - break; - case "error": { - const errorLine = getErrorLine(result); - view.error = errorLine === null ? null : stripVTControlCharacters(errorLine); - break; - } - case "meta": - view.meta = result.meta ?? null; - break; - case "skipOrMuteReason": - view.skipOrMuteReason = result.skipReason ?? null; - break; - } - } - - return view; -} - function formatTestResultField(result: TestResultView, field: TestResultField): string | null { switch (field) { case "skipOrMuteReason": @@ -182,7 +113,7 @@ function formatTestResult(result: TestResultView, index: number, fields: readonl .filter(Boolean) .join(" | "); const title = fields.includes("name") ? (result.name ?? null) : null; - const errorLine = fields.includes("error") ? result.error : null; + const errorLine = fields.includes("error") && typeof result.error === "string" ? result.error : null; const firstLine = details || title || errorLine || "(no output fields)"; return [`${index}. ${firstLine}`, details && title ? ` ${title}` : null, errorLine ? ` ${errorLine}` : null] @@ -298,7 +229,9 @@ const testResultsCb: StandaloneTool["cb"] = async args const matchedResults = filterTestResults(finalResults, args); const counts = getTestResultsCounts(results.length, finalResults, matchedResults); - const resultViews = matchedResults.map(result => toTestResultView(result, args.fields)); + const resultViews = matchedResults.map(result => + toTestResultView(result, args.fields, { errorFormat: "line" }), + ); if (args.saveJson) { const savedReport = await saveTestResultsJsonReport(args.report, counts, args.fields, resultViews); diff --git a/packages/tools/src/tools/test-results/types.ts b/packages/tools/src/tools/test-results/types.ts index 6708d5c..18fc930 100644 --- a/packages/tools/src/tools/test-results/types.ts +++ b/packages/tools/src/tools/test-results/types.ts @@ -1,17 +1,18 @@ -import type { ReporterTestResult } from "html-reporter/experimental/sdk"; +import type { TestResultField, TestResultView } from "../../utils/test-result-view.js"; + +export { + TEST_RESULT_VIEW_FIELDS as TEST_RESULT_FIELDS, + DETAILED_TEST_RESULT_FIELDS, +} from "../../utils/test-result-view.js"; +export type { + DetailedTestResultField, + ReporterImageInfo, + TestResultField, + TestResultView, + TestStepView, +} from "../../utils/test-result-view.js"; export const TEST_RESULT_STATUSES = ["passed", "failed", "muted", "retried", "skipped"] as const; -export const TEST_RESULT_FIELDS = [ - "status", - "browser", - "attempt", - "duration", - "file", - "name", - "error", - "meta", - "skipOrMuteReason", -] as const; export const DEFAULT_TEST_RESULT_FIELDS = [ "status", "browser", @@ -22,9 +23,7 @@ export const DEFAULT_TEST_RESULT_FIELDS = [ "error", ] as const; -export type ReporterImageInfo = NonNullable[number]; export type TestResultStatus = (typeof TEST_RESULT_STATUSES)[number]; -export type TestResultField = (typeof TEST_RESULT_FIELDS)[number]; export type StatusCounts = Record; export interface RegexFilter { @@ -65,18 +64,6 @@ export interface PaginationOptions { offset: number; } -export interface TestResultView { - status?: string; - browser?: string; - attempt?: number; - duration?: number | null; - file?: string | null; - name?: string; - error?: string | null; - meta?: unknown | null; - skipOrMuteReason?: string | null; -} - export interface TestResultsCounts { totalTests: number; totalAttempts: number; diff --git a/packages/tools/src/utils/html-report.ts b/packages/tools/src/utils/html-report.ts index d9ccbd4..932fd6b 100644 --- a/packages/tools/src/utils/html-report.ts +++ b/packages/tools/src/utils/html-report.ts @@ -1,6 +1,6 @@ import { downloadReport, type ReporterTestResult } from "html-reporter/experimental/sdk"; import { DEFAULT_REMOTE_RESOURCE_CACHE_ROOT, resolveCachedRemoteResource } from "./remote-resource-cache.js"; -import type { ReporterImageInfo } from "../tools/test-results/types.js"; +import type { ReporterImageInfo } from "./test-result-view.js"; const DATABASE_URLS_FILE = "databaseUrls.json"; diff --git a/packages/tools/src/utils/test-result-view.ts b/packages/tools/src/utils/test-result-view.ts new file mode 100644 index 0000000..db9eebf --- /dev/null +++ b/packages/tools/src/utils/test-result-view.ts @@ -0,0 +1,343 @@ +import { Buffer } from "node:buffer"; +import { stripVTControlCharacters } from "node:util"; +import type { ReporterTestResult } from "html-reporter/experimental/sdk"; +import { formatError } from "./formatters.js"; +import { getImageError, getImageStateName, isMutedResult } from "./html-report.js"; + +export const TEST_RESULT_VIEW_FIELDS = [ + "status", + "browser", + "attempt", + "duration", + "file", + "name", + "error", + "meta", + "skipOrMuteReason", +] as const; +export const DETAILED_TEST_RESULT_FIELDS = [ + ...TEST_RESULT_VIEW_FIELDS, + "id", + "description", + "url", + "timestamp", + "sessionId", + "imageDir", + "errorDetails", + "steps", + "images", + "attachments", +] as const; + +export type ReporterImageInfo = NonNullable[number]; +type ReporterTestError = NonNullable; +type ReporterErrorDetails = NonNullable; +type ReporterAttachment = NonNullable[number]; +type ReporterTestStep = NonNullable[number]; +export type TestResultField = (typeof TEST_RESULT_VIEW_FIELDS)[number]; +export type DetailedTestResultField = (typeof DETAILED_TEST_RESULT_FIELDS)[number]; + +type SanitizedBufferPayload = + | string + | { + bufferOmitted: true; + byteLength: number; + }; +type SanitizedObject = { + [K in keyof T as K extends "base64" ? never : K]: SanitizedPayload; +} & (T extends { base64?: string } + ? { + base64?: string; + base64Omitted?: true; + } + : Record); +type SanitizedPayload = T extends Buffer + ? SanitizedBufferPayload + : T extends string + ? string + : T extends number | boolean | null | undefined + ? T + : T extends readonly (infer Item)[] + ? SanitizedPayload[] + : T extends object + ? SanitizedObject + : T; +type SanitizedTestError = SanitizedPayload; +type SanitizedImageInfo = SanitizedPayload; +type SanitizedErrorDetails = SanitizedPayload; +type AttachmentView = ReporterAttachment extends infer Attachment + ? Attachment extends ReporterAttachment + ? SanitizedPayload> & { type: string } + : never + : never; + +export interface TestResultView { + status?: string; + browser?: string; + attempt?: number; + duration?: number | null; + file?: string | null; + name?: string; + error?: string | SanitizedTestError | null; + meta?: unknown | null; + skipOrMuteReason?: string | null; + id?: string; + description?: string | null; + url?: string | null; + timestamp?: number | null; + sessionId?: string | null; + imageDir?: string | null; + errorDetails?: SanitizedErrorDetails | null; + steps?: TestStepView[]; + images?: SanitizedImageInfo[]; + attachments?: AttachmentView[]; +} + +export interface TestStepView { + name: string; + args: string[]; + duration: number; + timestamp: number; + isFailed?: boolean; + isGroup?: boolean; + repeat?: number; + children?: TestStepView[]; +} + +export interface TestResultViewOptions { + errorFormat?: "full" | "line"; + includeBase64?: boolean; +} + +function getFailedImageInfo(result: ReporterTestResult): string | null { + const failedImage = result.imagesInfo?.find(imageInfo => { + const status = imageInfo.status; + + return status === "error" || status === "fail"; + }); + + if (!failedImage) { + return null; + } + + const stateName = getImageStateName(failedImage); + const state = stateName ? `state: ${stateName}` : null; + const error = formatError(getImageError(failedImage)); + const message = [state, error].filter((part): part is string => part !== null).join("; "); + + return message || null; +} + +function getErrorLine(result: ReporterTestResult): string | null { + const status = result.status; + const shouldMentionMissingError = status === "error" || status === "fail" || isMutedResult(result); + + const error = formatError(result.error) ?? getFailedImageInfo(result); + + return error ?? (shouldMentionMissingError ? "No error message" : null); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function sanitizeBinaryPayloads(value: T, includeBase64: boolean): SanitizedPayload { + if (Array.isArray(value)) { + return value.map(item => sanitizeBinaryPayloads(item, includeBase64)) as SanitizedPayload; + } + + if (Buffer.isBuffer(value)) { + return ( + includeBase64 + ? value.toString("base64") + : { + bufferOmitted: true, + byteLength: value.byteLength, + } + ) as SanitizedPayload; + } + + if (!isRecord(value)) { + return value as SanitizedPayload; + } + + const result: Record = {}; + + for (const [key, nestedValue] of Object.entries(value)) { + if (key === "base64" && !includeBase64) { + result.base64Omitted = true; + continue; + } + + result[key] = sanitizeBinaryPayloads(nestedValue, includeBase64); + } + + return result as SanitizedPayload; +} + +function stripAnsiStrings(value: T): T { + if (typeof value === "string") { + return stripVTControlCharacters(value) as T; + } + + if (Array.isArray(value)) { + return value.map(stripAnsiStrings) as T; + } + + if (!isRecord(value)) { + return value; + } + + return Object.fromEntries( + Object.entries(value).map(([key, nestedValue]) => [key, stripAnsiStrings(nestedValue)]), + ) as T; +} + +function sanitizePayload(value: T, includeBase64: boolean): SanitizedPayload { + return stripAnsiStrings(sanitizeBinaryPayloads(value, includeBase64)); +} + +function sanitizeErrorPayload(value: ReporterTestError, includeBase64: boolean): SanitizedTestError; +function sanitizeErrorPayload(value: ReporterImageInfo, includeBase64: boolean): SanitizedImageInfo; +function sanitizeErrorPayload( + value: ReporterTestError | ReporterImageInfo, + includeBase64: boolean, +): SanitizedTestError | SanitizedImageInfo { + return sanitizePayload(value, includeBase64); +} + +function sanitizeErrorDetails(details: ReporterErrorDetails, includeBase64: boolean): SanitizedErrorDetails { + return sanitizePayload(details, includeBase64); +} + +function toTestStepView(step: ReporterTestStep): TestStepView { + const view: TestStepView = { + name: step.n, + args: step.a, + duration: step.d, + timestamp: step.ts, + }; + + if (step.f !== undefined) { + view.isFailed = step.f; + } + if (step.g !== undefined) { + view.isGroup = step.g; + } + if (step.r !== undefined) { + view.repeat = step.r; + } + if (step.c?.length) { + view.children = step.c.map(toTestStepView); + } + + return view; +} + +function getAttachmentType(type: unknown): string { + switch (type) { + case 0: + return "snapshot"; + case 1: + return "badges"; + case 2: + return "tags"; + default: + return typeof type === "string" ? type : `unknown:${String(type)}`; + } +} + +function toAttachmentView(attachment: ReporterAttachment, includeBase64: boolean): AttachmentView { + const { type, ...payload } = attachment; + + return { + ...sanitizePayload(payload, includeBase64), + type: getAttachmentType(type), + } as AttachmentView; +} + +/** Picks fields from a test result and returns a view object. */ +export function toTestResultView( + result: ReporterTestResult, + fields?: readonly DetailedTestResultField[], + options: TestResultViewOptions = {}, +): TestResultView { + const view: TestResultView = {}; + const outputFields = fields ?? DETAILED_TEST_RESULT_FIELDS; + const includeBase64 = options.includeBase64 ?? false; + const errorFormat = options.errorFormat ?? "full"; + + for (const field of outputFields) { + switch (field) { + case "name": + view.name = result.fullName; + break; + case "status": + view.status = isMutedResult(result) ? "muted" : result.status; + break; + case "browser": + view.browser = result.browserId; + break; + case "attempt": + view.attempt = result.attempt; + break; + case "duration": + view.duration = result.duration ?? null; + break; + case "file": + view.file = result.file ?? null; + break; + case "error": + if (errorFormat === "line") { + const errorLine = getErrorLine(result); + view.error = errorLine === null ? null : stripVTControlCharacters(errorLine); + } else { + view.error = result.error == null ? null : sanitizeErrorPayload(result.error, includeBase64); + } + break; + case "meta": + view.meta = result.meta ?? null; + break; + case "skipOrMuteReason": + view.skipOrMuteReason = result.skipReason ?? null; + break; + case "id": + view.id = result.id; + break; + case "description": + view.description = result.description ?? null; + break; + case "url": + view.url = result.url ?? null; + break; + case "timestamp": + view.timestamp = result.timestamp ?? null; + break; + case "sessionId": + view.sessionId = result.sessionId ?? null; + break; + case "imageDir": + view.imageDir = result.imageDir ?? null; + break; + case "errorDetails": + view.errorDetails = + result.errorDetails == null ? null : sanitizeErrorDetails(result.errorDetails, includeBase64); + break; + case "steps": + view.steps = (result.history ?? []).map(toTestStepView); + break; + case "images": + view.images = (result.imagesInfo ?? []).map(imageInfo => + sanitizeErrorPayload(imageInfo, includeBase64), + ); + break; + case "attachments": + view.attachments = (result.attachments ?? []).map(attachment => + toAttachmentView(attachment, includeBase64), + ); + break; + } + } + + return view; +} diff --git a/packages/tools/test/tools/inspect-result.test.ts b/packages/tools/test/tools/inspect-result.test.ts new file mode 100644 index 0000000..4a9f467 --- /dev/null +++ b/packages/tools/test/tools/inspect-result.test.ts @@ -0,0 +1,140 @@ +import { fileURLToPath } from "node:url"; +import type { ReporterTestResult } from "html-reporter/experimental/sdk"; +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { inspectResult } from "../../src/tools/inspect-result/index.js"; +import { inspectResultObjectSchema } from "../../src/tools/inspect-result/schema.js"; +import { toTestResultView } from "../../src/utils/test-result-view.js"; + +const SAMPLE_REPORT = fileURLToPath(new URL("../fixtures/sample-html-report", import.meta.url)); + +type InspectResultInput = z.input; + +function getTextContent(result: { content: unknown }): string { + const content = result.content as Array<{ text: string }>; + + return content.map(item => item.text).join("\n"); +} + +function parseInspectResultArgs(args: Partial = {}) { + return inspectResultObjectSchema.parse({ + report: SAMPLE_REPORT, + name: "failed describe test with image comparison diff", + browser: "chrome", + ...args, + }); +} + +describe("tools/inspect-result", () => { + it("prints detailed JSON for the latest selected test result attempt", async () => { + const result = await inspectResult.cb(parseInspectResultArgs()); + const text = getTextContent(result); + const details = JSON.parse(text); + + expect(result.isError).toBeFalsy(); + expect(text).toContain('\n "status": "fail"'); + expect(details).toMatchObject({ + name: "failed describe test with image comparison diff", + status: "fail", + browser: "chrome", + attempt: 1, + file: "failed-describe.testplane.js", + error: { + name: "AssertViewError", + message: "image comparison failed", + }, + }); + expect(details.error.stack).toContain("TestRunner.finishRun"); + expect(details.steps[0]).toMatchObject({ + name: "setWindowSize", + args: ["1280", "1024"], + }); + expect(details.images[0]).toMatchObject({ + status: "fail", + stateName: "header", + differentPixels: 25730, + }); + expect(details.attachments.map((attachment: { type: string }) => attachment.type)).toEqual([ + "tags", + "snapshot", + "badges", + ]); + expect(details).not.toHaveProperty("history"); + expect(details).not.toHaveProperty("imagesInfo"); + }); + + it("prints compact JSON and honors an explicit attempt", async () => { + const result = await inspectResult.cb(parseInspectResultArgs({ attempt: 0, pretty: false })); + const text = getTextContent(result); + const details = JSON.parse(text); + + expect(result.isError).toBeFalsy(); + expect(text).not.toContain("\n"); + expect(details).toMatchObject({ + name: "failed describe test with image comparison diff", + browser: "chrome", + attempt: 0, + status: "fail", + }); + }); + + it("omits base64 payloads and strips ANSI control characters from detailed result views", () => { + const reporterResult = { + fullName: "test", + browserId: "chrome", + attempt: 0, + status: "error", + duration: 1, + file: "testplane.js", + error: { + name: "Error", + message: "\u001B[31mWith screenshot\u001B[39m", + stack: "Error: \u001B[31mWith screenshot\u001B[39m", + screenshot: { + base64: "secret-error-base64", + size: { width: 1, height: 1 }, + }, + }, + errorDetails: { + title: "\u001B[31mDetails\u001B[39m", + filePath: "details.json", + }, + history: [], + id: "id", + imageDir: "images/test", + imagesInfo: [ + { + status: "error", + error: { + message: "\u001B[31mImage error\u001B[39m", + }, + actualImg: { + base64: "secret-image-base64", + size: { width: 1, height: 1 }, + }, + }, + ], + meta: {}, + sessionId: "session-id", + skipReason: undefined, + timestamp: 1, + url: "https://example.com", + description: undefined, + attachments: [], + } as unknown as ReporterTestResult; + + const withoutBase64 = JSON.stringify(toTestResultView(reporterResult)); + const withBase64 = JSON.stringify(toTestResultView(reporterResult, undefined, { includeBase64: true })); + + expect(withoutBase64).not.toContain("secret-error-base64"); + expect(withoutBase64).not.toContain("secret-image-base64"); + expect(withoutBase64).not.toContain("\u001B"); + expect(withoutBase64).toContain("With screenshot"); + expect(withoutBase64).toContain("Image error"); + expect(withoutBase64).toContain("base64Omitted"); + expect(withBase64).toContain("secret-error-base64"); + expect(withBase64).toContain("secret-image-base64"); + expect(withBase64).not.toContain("\u001B"); + }); +}); diff --git a/packages/tools/test/tools/test-results.test.ts b/packages/tools/test/tools/test-results.test.ts index 69211ee..cfbf6eb 100644 --- a/packages/tools/test/tools/test-results.test.ts +++ b/packages/tools/test/tools/test-results.test.ts @@ -6,8 +6,9 @@ import { readResultsFromReport, type ReporterTestResult } from "html-reporter/ex import { describe, expect, it, beforeAll } from "vitest"; import { z } from "zod"; -import { testResults, getFinalTestResults, toTestResultView } from "../../src/tools/test-results/index.js"; +import { testResults, getFinalTestResults } from "../../src/tools/test-results/index.js"; import { testResultsObjectSchema } from "../../src/tools/test-results/schema.js"; +import { toTestResultView } from "../../src/utils/test-result-view.js"; const SAMPLE_REPORT = fileURLToPath(new URL("../fixtures/sample-html-report", import.meta.url)); const EXPECTED_COUNTS_LINE = "Total tests: 9; total attempts: 17; matched tests:"; From eba68987ce279c39cce910ab1bf415398c5d0094 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Tue, 12 May 2026 15:38:58 +0300 Subject: [PATCH 2/4] fix: update html-reporter for enhanced error messages on download failure --- packages/cli/package.json | 2 +- packages/mcp/package.json | 2 +- packages/tools/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index be09817..39d9a57 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -28,7 +28,7 @@ "@testplane/testing-library": "^1.0.5", "commander": "^14.0.3", "debug": "^4.3.4", - "html-reporter": "^11.10.0-rc.1", + "html-reporter": "^11.10.0-rc.2", "testplane": "^8.44.1-rc.1", "zod": "^3.22.4" }, diff --git a/packages/mcp/package.json b/packages/mcp/package.json index 3d5cf92..67ca387 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -28,7 +28,7 @@ "@modelcontextprotocol/sdk": "^1.11.2", "@testplane/testing-library": "^1.0.5", "commander": "^13.1.0", - "html-reporter": "^11.10.0-rc.1", + "html-reporter": "^11.10.0-rc.2", "testplane": "^8.44.1-rc.1", "zod": "^3.22.4" }, diff --git a/packages/tools/package.json b/packages/tools/package.json index a2d8d11..dcbaa97 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -28,7 +28,7 @@ "license": "ISC", "dependencies": { "@testplane/testing-library": "^1.0.5", - "html-reporter": "^11.10.0-rc.1", + "html-reporter": "^11.10.0-rc.2", "lodash.escaperegexp": "^4.1.2", "testplane": "^8.44.1-rc.1", "zod": "^3.22.4" From ab926f11450ff08474f57485783cd9764a1d2299 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Wed, 13 May 2026 17:58:06 +0300 Subject: [PATCH 3/4] fix: return all attempts in inspect-result tool by default --- .../tools/src/tools/inspect-result/index.ts | 47 ++++++++++--------- .../tools/src/tools/inspect-result/schema.ts | 4 +- .../tools/test/tools/inspect-result.test.ts | 20 ++++---- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/packages/tools/src/tools/inspect-result/index.ts b/packages/tools/src/tools/inspect-result/index.ts index 7473c78..d467ee3 100644 --- a/packages/tools/src/tools/inspect-result/index.ts +++ b/packages/tools/src/tools/inspect-result/index.ts @@ -7,19 +7,6 @@ import { inspectResultSchema } from "./schema.js"; export { inspectResultSchema } from "./schema.js"; -function getLatestAttempt(results: readonly T[]): T { - return results.reduce((latest, result) => { - if ( - result.attempt > latest.attempt || - (result.attempt === latest.attempt && (result.timestamp ?? 0) >= (latest.timestamp ?? 0)) - ) { - return result; - } - - return latest; - }); -} - function getInspectResultSelectionError( results: readonly ReporterTestResult[], args: { name: string; browser: string; attempt?: number }, @@ -37,20 +24,35 @@ function getInspectResultSelectionError( return `No test result found with name "${args.name}" in browser "${args.browser}". Available browsers: ${availableBrowsers.join(", ")}.`; } - const availableAttempts = [...new Set(results.map(result => result.attempt))].sort(); + const availableAttempts = [...new Set(resultsWithMatchingBrowser.map(result => result.attempt))].sort(); return `No attempt ${args.attempt} found for test "${args.name}" in browser "${args.browser}". Available attempts: ${availableAttempts.join(", ")}.`; } -function findTestResult( +function sortAttempts(results: readonly T[]): T[] { + return [...results].sort((left, right) => { + if (left.attempt !== right.attempt) { + return left.attempt - right.attempt; + } + + return (left.timestamp ?? 0) - (right.timestamp ?? 0); + }); +} + +function findTestResults( results: readonly ReporterTestResult[], args: { name: string; browser: string; attempt?: number }, -): ReporterTestResult { +): ReporterTestResult[] { const matchingAttempts = results.filter( result => result.fullName === args.name && result.browserId === args.browser, ); + + if (!matchingAttempts.length) { + throw new Error(getInspectResultSelectionError(results, args)); + } + if (args.attempt === undefined) { - return getLatestAttempt(matchingAttempts); + return sortAttempts(matchingAttempts); } const matchingAttempt = matchingAttempts.find(result => result.attempt === args.attempt); @@ -59,15 +61,18 @@ function findTestResult( throw new Error(getInspectResultSelectionError(results, args)); } - return matchingAttempt; + return [matchingAttempt]; } const inspectResultCb: StandaloneTool["cb"] = async args => { try { const reportPath = await downloadReportIfNeeded(args.report); const results = await readResultsFromReport(reportPath); - const result = findTestResult(results, args); - const view = toTestResultView(result, undefined, { includeBase64: args.includeBase64 }); + const selectedResults = findTestResults(results, args); + const views = selectedResults.map(result => + toTestResultView(result, undefined, { includeBase64: args.includeBase64 }), + ); + const view = args.attempt === undefined ? views : views[0]; return createSimpleResponse(JSON.stringify(view, null, args.pretty ? 2 : undefined)); } catch (error) { @@ -83,7 +88,7 @@ const inspectResultCb: StandaloneTool["cb"] = async export const inspectResult: StandaloneTool = { kind: ToolKind.Standalone, name: "inspect-result", - description: "Read a Testplane HTML report and inspect one test result attempt as JSON", + description: "Read a Testplane HTML report and inspect test result attempt details as JSON", schema: inspectResultSchema, cb: inspectResultCb, cli: { diff --git a/packages/tools/src/tools/inspect-result/schema.ts b/packages/tools/src/tools/inspect-result/schema.ts index 2e9075c..a085078 100644 --- a/packages/tools/src/tools/inspect-result/schema.ts +++ b/packages/tools/src/tools/inspect-result/schema.ts @@ -21,7 +21,9 @@ export const inspectResultSchema = { .int() .min(0, "--attempt must be >= 0") .optional() - .describe("Attempt index to inspect. Defaults to the latest attempt for the selected test and browser"), + .describe( + "Attempt index to inspect. When omitted, all attempts for the selected test and browser are returned", + ), pretty: z.boolean().default(true).describe("Pretty-print JSON output. Use --no-pretty for compact JSON"), includeBase64: z .boolean() diff --git a/packages/tools/test/tools/inspect-result.test.ts b/packages/tools/test/tools/inspect-result.test.ts index 4a9f467..9bab54d 100644 --- a/packages/tools/test/tools/inspect-result.test.ts +++ b/packages/tools/test/tools/inspect-result.test.ts @@ -27,14 +27,16 @@ function parseInspectResultArgs(args: Partial = {}) { } describe("tools/inspect-result", () => { - it("prints detailed JSON for the latest selected test result attempt", async () => { + it("prints detailed JSON for all selected test result attempts by default", async () => { const result = await inspectResult.cb(parseInspectResultArgs()); const text = getTextContent(result); const details = JSON.parse(text); expect(result.isError).toBeFalsy(); - expect(text).toContain('\n "status": "fail"'); - expect(details).toMatchObject({ + expect(text).toContain('\n "status": "fail"'); + expect(details).toHaveLength(2); + expect(details.map((attempt: { attempt: number }) => attempt.attempt)).toEqual([0, 1]); + expect(details[1]).toMatchObject({ name: "failed describe test with image comparison diff", status: "fail", browser: "chrome", @@ -45,23 +47,23 @@ describe("tools/inspect-result", () => { message: "image comparison failed", }, }); - expect(details.error.stack).toContain("TestRunner.finishRun"); - expect(details.steps[0]).toMatchObject({ + expect(details[1].error.stack).toContain("TestRunner.finishRun"); + expect(details[1].steps[0]).toMatchObject({ name: "setWindowSize", args: ["1280", "1024"], }); - expect(details.images[0]).toMatchObject({ + expect(details[1].images[0]).toMatchObject({ status: "fail", stateName: "header", differentPixels: 25730, }); - expect(details.attachments.map((attachment: { type: string }) => attachment.type)).toEqual([ + expect(details[1].attachments.map((attachment: { type: string }) => attachment.type)).toEqual([ "tags", "snapshot", "badges", ]); - expect(details).not.toHaveProperty("history"); - expect(details).not.toHaveProperty("imagesInfo"); + expect(details[1]).not.toHaveProperty("history"); + expect(details[1]).not.toHaveProperty("imagesInfo"); }); it("prints compact JSON and honors an explicit attempt", async () => { From 81e032c164c9b74fe0dfdc7e0cd2d4ee8c3bf136 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Sun, 17 May 2026 22:04:39 +0300 Subject: [PATCH 4/4] fix: fix remaining review issues --- packages/tools/src/tools/inspect-result/index.ts | 10 +++++----- packages/tools/src/utils/test-result-view.ts | 6 +++--- packages/tools/test/tools/inspect-result.test.ts | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/tools/src/tools/inspect-result/index.ts b/packages/tools/src/tools/inspect-result/index.ts index d467ee3..0621ecf 100644 --- a/packages/tools/src/tools/inspect-result/index.ts +++ b/packages/tools/src/tools/inspect-result/index.ts @@ -14,17 +14,17 @@ function getInspectResultSelectionError( const resultsWithMatchingName = results.filter(result => result.fullName === args.name); if (!resultsWithMatchingName.length) { - return `No test result found with name "${args.name}". You can check what tests are available with the "test-results" tool.`; + return `Test with name "${args.name}" wasn't found in this report. You can check what tests are available with the "test-results" tool.`; } const resultsWithMatchingBrowser = resultsWithMatchingName.filter(result => result.browserId === args.browser); if (!resultsWithMatchingBrowser.length) { const availableBrowsers = [...new Set(resultsWithMatchingName.map(result => result.browserId))].sort(); - return `No test result found with name "${args.name}" in browser "${args.browser}". Available browsers: ${availableBrowsers.join(", ")}.`; + return `Test with name "${args.name}" wasn't run in browser "${args.browser}", it was run only in: ${availableBrowsers.join(", ")}.`; } - const availableAttempts = [...new Set(resultsWithMatchingBrowser.map(result => result.attempt))].sort(); + const availableAttempts = resultsWithMatchingBrowser.map(result => result.attempt).sort(); return `No attempt ${args.attempt} found for test "${args.name}" in browser "${args.browser}". Available attempts: ${availableAttempts.join(", ")}.`; } @@ -70,11 +70,11 @@ const inspectResultCb: StandaloneTool["cb"] = async const results = await readResultsFromReport(reportPath); const selectedResults = findTestResults(results, args); const views = selectedResults.map(result => - toTestResultView(result, undefined, { includeBase64: args.includeBase64 }), + toTestResultView(result, [], { includeBase64: args.includeBase64 }), ); const view = args.attempt === undefined ? views : views[0]; - return createSimpleResponse(JSON.stringify(view, null, args.pretty ? 2 : undefined)); + return createSimpleResponse(JSON.stringify(view, null, args.pretty ? 2 : 0)); } catch (error) { console.error("Error inspecting test result from report:", error); diff --git a/packages/tools/src/utils/test-result-view.ts b/packages/tools/src/utils/test-result-view.ts index db9eebf..c7561aa 100644 --- a/packages/tools/src/utils/test-result-view.ts +++ b/packages/tools/src/utils/test-result-view.ts @@ -134,7 +134,7 @@ function getErrorLine(result: ReporterTestResult): string | null { const error = formatError(result.error) ?? getFailedImageInfo(result); - return error ?? (shouldMentionMissingError ? "No error message" : null); + return !error && shouldMentionMissingError ? "No error message" : error; } function isRecord(value: unknown): value is Record { @@ -259,11 +259,11 @@ function toAttachmentView(attachment: ReporterAttachment, includeBase64: boolean /** Picks fields from a test result and returns a view object. */ export function toTestResultView( result: ReporterTestResult, - fields?: readonly DetailedTestResultField[], + fields: readonly DetailedTestResultField[], options: TestResultViewOptions = {}, ): TestResultView { const view: TestResultView = {}; - const outputFields = fields ?? DETAILED_TEST_RESULT_FIELDS; + const outputFields = fields && fields.length > 0 ? fields : DETAILED_TEST_RESULT_FIELDS; const includeBase64 = options.includeBase64 ?? false; const errorFormat = options.errorFormat ?? "full"; diff --git a/packages/tools/test/tools/inspect-result.test.ts b/packages/tools/test/tools/inspect-result.test.ts index 9bab54d..7ecb89c 100644 --- a/packages/tools/test/tools/inspect-result.test.ts +++ b/packages/tools/test/tools/inspect-result.test.ts @@ -126,8 +126,8 @@ describe("tools/inspect-result", () => { attachments: [], } as unknown as ReporterTestResult; - const withoutBase64 = JSON.stringify(toTestResultView(reporterResult)); - const withBase64 = JSON.stringify(toTestResultView(reporterResult, undefined, { includeBase64: true })); + const withoutBase64 = JSON.stringify(toTestResultView(reporterResult, [])); + const withBase64 = JSON.stringify(toTestResultView(reporterResult, [], { includeBase64: true })); expect(withoutBase64).not.toContain("secret-error-base64"); expect(withoutBase64).not.toContain("secret-image-base64");