diff --git a/package-lock.json b/package-lock.json index 0baea115..0fd6b84b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { - "@salesforce/agents": "^1.2.0", + "@salesforce/agents": "^1.6.4", "@salesforce/core": "^8.28.3", "@salesforce/kit": "^3.2.6", "@salesforce/types": "^1.7.1", @@ -2255,27 +2255,27 @@ } }, "node_modules/@salesforce/agents": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@salesforce/agents/-/agents-1.2.0.tgz", - "integrity": "sha512-gyF1xzJcEp3MuS8Apf1eUGUvy41zq7HQqKPEhDIU6aUiowoewW/RI9LcPB0K7LPrGCIiARq4TYKlxf452HASqA==", + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/@salesforce/agents/-/agents-1.6.4.tgz", + "integrity": "sha512-+E9Ljtr+LiDHEfgRnqpFV4crLsfd07bTmU4Gi6hVLQmLG4T/cBM15nNnkfsMMhMs5t50cY7hpAP/PWkutC8eQw==", "license": "Apache-2.0", "dependencies": { - "@salesforce/core": "^8.28.3", + "@salesforce/core": "^8.29.1", "@salesforce/kit": "^3.2.6", - "@salesforce/source-deploy-retrieve": "^12.32.7", + "@salesforce/source-deploy-retrieve": "^12.35.8", "@salesforce/types": "^1.7.1", - "fast-xml-parser": "^5.6.0", + "fast-xml-parser": "^5.7.3", "nock": "^13.5.6", - "yaml": "^2.8.3" + "yaml": "^2.8.4" }, "engines": { "node": ">=18.0.0" } }, "node_modules/@salesforce/core": { - "version": "8.28.3", - "resolved": "https://registry.npmjs.org/@salesforce/core/-/core-8.28.3.tgz", - "integrity": "sha512-DDAeHVwDO8wUlqEGwfp8Vuu7Vp7K+hpubKu6baWkHAXiO1u7ZbQkvwCbpPz9JiYEXVUBvP11JtBQ7zOUIPShlQ==", + "version": "8.29.1", + "resolved": "https://registry.npmjs.org/@salesforce/core/-/core-8.29.1.tgz", + "integrity": "sha512-Pdo9yjKU6Lgm3CJOVJPTM4PzUHhTVMX43K5XmClsBEt75fs8mHNFALniHZwnZbVf+TSMfGsUygYAyTX6ShnNQA==", "license": "BSD-3-Clause", "dependencies": { "@jsforce/jsforce-node": "^3.10.13", @@ -2312,17 +2312,17 @@ } }, "node_modules/@salesforce/source-deploy-retrieve": { - "version": "12.32.9", - "resolved": "https://registry.npmjs.org/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.32.9.tgz", - "integrity": "sha512-DRIiyt9dP+QzBs+Lonw7UeB+LvtHmEyrVbYibXvRtKkTAj/zuBGjvKHm06/ql1TymXFotwE54j6xLi1rxS5Zfw==", + "version": "12.35.9", + "resolved": "https://registry.npmjs.org/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.35.9.tgz", + "integrity": "sha512-Qud+1lHS0u+LM/AAHPN9Y/6W6hBZ8nvw6/TxJRzn/BuvRSz74YBwvmVhmn5pvpZdNrYmKRMPxsK47l2VmazSsw==", "license": "Apache-2.0", "dependencies": { - "@salesforce/core": "^8.27.1", + "@salesforce/core": "^8.29.0", "@salesforce/kit": "^3.2.4", "@salesforce/ts-types": "^2.0.12", "@salesforce/types": "^1.6.0", "fast-levenshtein": "^3.0.0", - "fast-xml-parser": "^5.5.11", + "fast-xml-parser": "^5.7.3", "got": "^11.8.6", "graceful-fs": "^4.2.11", "ignore": "^5.3.2", @@ -4071,9 +4071,9 @@ } }, "node_modules/basic-ftp": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.0.tgz", - "integrity": "sha512-5K9eNNn7ywHPsYnFwjKgYH8Hf8B5emh7JKcPaVjjrMJFQQwGpwowEnZNEtHs7DfR7hCZsmaK3VA4HUK0YarT+w==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.1.tgz", + "integrity": "sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -5633,9 +5633,9 @@ "license": "BSD-3-Clause" }, "node_modules/fast-xml-builder": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.5.tgz", - "integrity": "sha512-4TJn/8FKLeslLAH3dnohXqE3QSoxkhvaMzepOIZytwJXZO69Bfz0HBdDHzOTOon6G59Zrk6VQ2bEiv1t61rfkA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", + "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", "funding": [ { "type": "github", @@ -5644,13 +5644,14 @@ ], "license": "MIT", "dependencies": { - "path-expression-matcher": "^1.1.3" + "path-expression-matcher": "^1.5.0", + "xml-naming": "^0.1.0" } }, "node_modules/fast-xml-parser": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.1.tgz", - "integrity": "sha512-8Cc3f8GUGUULg34pBch/KGyPLglS+OFs05deyOlY7fL2MTagYPKrVQNmR1fLF/yJ9PH5ZSTd3YDF6pnmeZU+zA==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.3.tgz", + "integrity": "sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==", "funding": [ { "type": "github", @@ -5660,7 +5661,7 @@ "license": "MIT", "dependencies": { "@nodable/entities": "^2.1.0", - "fast-xml-builder": "^1.1.5", + "fast-xml-builder": "^1.1.7", "path-expression-matcher": "^1.5.0", "strnum": "^2.2.3" }, @@ -6532,9 +6533,9 @@ "optional": true }, "node_modules/ip-address": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", - "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", "license": "MIT", "engines": { "node": ">= 12" @@ -10797,12 +10798,12 @@ } }, "node_modules/socks": { - "version": "2.8.7", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", - "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz", + "integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==", "license": "MIT", "dependencies": { - "ip-address": "^10.0.1", + "ip-address": "^10.1.1", "smart-buffer": "^4.2.0" }, "engines": { @@ -12137,6 +12138,21 @@ "node": ">=18" } }, + "node_modules/xml-naming": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", + "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", @@ -12190,9 +12206,9 @@ "license": "ISC" }, "node_modules/yaml": { - "version": "2.8.3", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", - "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.4.tgz", + "integrity": "sha512-ml/JPOj9fOQK8RNnWojA67GbZ0ApXAUlN2UQclwv2eVgTgn7O9gg9o7paZWKMp4g0H3nTLtS9LVzhkpOFIKzog==", "license": "ISC", "bin": { "yaml": "bin.mjs" diff --git a/package.json b/package.json index 553c2639..663c4ea0 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "main": "lib/src/extension.js", "dependencies": { - "@salesforce/agents": "^1.2.0", + "@salesforce/agents": "^1.6.4", "@salesforce/core": "^8.28.3", "@salesforce/kit": "^3.2.6", "@salesforce/types": "^1.7.1", diff --git a/src/types/MetadataDefinitions.ts b/src/types/MetadataDefinitions.ts index bc4d37a1..7c0cb3e5 100644 --- a/src/types/MetadataDefinitions.ts +++ b/src/types/MetadataDefinitions.ts @@ -22,3 +22,21 @@ export type AgentTestCase = { { name: 'expectedOutcome'; expectedValue: string } ]; }; + +// until @salesforce/types has AiTestingDefinition +export type AgentforceStudioTestCaseMetadata = { + number: string; + inputs: { utterance: string }; +}; + +// until @salesforce/types has AiTestingDefinition +export type AiTestingDefinition = { + AiTestingDefinition: { + description: string; + name: string; + subjectType: 'AGENT'; + subjectName: string; + subjectVersion: string; + testCase: AgentforceStudioTestCaseMetadata | AgentforceStudioTestCaseMetadata[]; + }; +}; diff --git a/src/types/TestNodes.ts b/src/types/TestNodes.ts index c014885c..e4b2792b 100644 --- a/src/types/TestNodes.ts +++ b/src/types/TestNodes.ts @@ -1,5 +1,5 @@ import * as vscode from 'vscode'; -import { TestStatus } from '@salesforce/agents'; +import { TestStatus, type TestRunnerType } from '@salesforce/agents'; import { getTestOutlineProvider } from '../views/testOutlineProvider'; /** @@ -66,8 +66,13 @@ export abstract class TestNode extends vscode.TreeItem { * has children AgentTestNode for individual test cases */ export class AgentTestGroupNode extends TestNode { + /** The actual metadata API name, which may differ from the display label when there's a naming conflict. */ + public testDefinitionName: string; + public runnerType: TestRunnerType = 'testing-center'; + constructor(label: string, location?: vscode.Location) { super(label, vscode.TreeItemCollapsibleState.Expanded, location ?? null); + this.testDefinitionName = label; } public contextValue = 'agentTestGroup'; diff --git a/src/views/testOutlineProvider.ts b/src/views/testOutlineProvider.ts index f8416cb8..9997ea1e 100644 --- a/src/views/testOutlineProvider.ts +++ b/src/views/testOutlineProvider.ts @@ -18,50 +18,107 @@ import { basename } from 'path'; import * as vscode from 'vscode'; import * as xml from 'fast-xml-parser'; import { Commands } from '../enums/commands'; -import { AgentTestGroupNode, AgentTestNode, AiEvaluationDefinition, TestNode } from '../types'; +import { AgentTestGroupNode, AgentTestNode, AiEvaluationDefinition, AiTestingDefinition, TestNode } from '../types'; +import type { TestRunnerType } from '@salesforce/agents'; const NO_TESTS_MESSAGE = 'no tests found'; const NO_TESTS_DESCRIPTION = 'no test description'; const AGENT_TESTS = 'AgentTests'; + +const buildTestGroupNode = ( + definitionUri: vscode.Uri, + definitionApiName: string, + label: string, + runnerType: TestRunnerType, + testCases: Array<{ number: string; inputs: { utterance: string } }>, + fileContent: string +): AgentTestGroupNode => { + const testDefinitionNode = new AgentTestGroupNode( + label, + new vscode.Location(definitionUri, new vscode.Position(0, 0)) + ); + testDefinitionNode.testDefinitionName = definitionApiName; + testDefinitionNode.runnerType = runnerType; + const splitContent = fileContent.split(EOL); + testCases.forEach(test => { + const line = splitContent.findIndex(l => l.includes(`${test.number}`)); + const testcaseNode = new AgentTestNode( + `#${test.number}`, + new vscode.Location(definitionUri, new vscode.Position(line < 0 ? 0 : line, 8)) + ); + testcaseNode.parentName = label; + testcaseNode.description = test.inputs.utterance; + testDefinitionNode.children.push(testcaseNode); + }); + testDefinitionNode.children.sort((a, b) => a.name.localeCompare(b.name)); + return testDefinitionNode; +}; + export const parseAgentTestsFromProject = async (): Promise> => { - const aiTestDefs = await vscode.workspace.findFiles('**/*.aiEvaluationDefinition-meta.xml'); - //from the aiTestDef files, parse the xml using fast-xml-parser, find the testSetName that it points to - const aggregator = new Map(); + const [aiEvalDefs, aiTestingDefs] = await Promise.all([ + vscode.workspace.findFiles('**/*.aiEvaluationDefinition-meta.xml'), + vscode.workspace.findFiles('**/*.aiTestingDefinition-meta.xml') + ]); const parser = new xml.XMLParser(); - await Promise.all( - aiTestDefs.map(async definition => { + + // Parse both sets independently, keyed by API name + const evalNodes = new Map(); + const agentforceStudioNodes = new Map(); + + await Promise.all([ + ...aiEvalDefs.map(async definition => { const fileContent = (await vscode.workspace.fs.readFile(definition)).toString(); const testDefinition = parser.parse(fileContent) as AiEvaluationDefinition; const definitionApiName = basename(definition.fsPath, '.aiEvaluationDefinition-meta.xml'); - - const testDefinitionNode = new AgentTestGroupNode( + const rawTestCases = testDefinition.AiEvaluationDefinition?.testCase; + if (!rawTestCases) return; + const testCases = Array.isArray(rawTestCases) ? rawTestCases : [rawTestCases]; + evalNodes.set( definitionApiName, - new vscode.Location(definition, new vscode.Position(0, 0)) + buildTestGroupNode(definition, definitionApiName, definitionApiName, 'testing-center', testCases, fileContent) + ); + }), + ...aiTestingDefs.map(async definition => { + const fileContent = (await vscode.workspace.fs.readFile(definition)).toString(); + const testDefinition = parser.parse(fileContent) as AiTestingDefinition; + const definitionApiName = basename(definition.fsPath, '.aiTestingDefinition-meta.xml'); + const rawTestCases = testDefinition.AiTestingDefinition?.testCase; + if (!rawTestCases) return; + const testCases = Array.isArray(rawTestCases) ? rawTestCases : [rawTestCases]; + agentforceStudioNodes.set( + definitionApiName, + buildTestGroupNode( + definition, + definitionApiName, + definitionApiName, + 'agentforce-studio', + testCases, + fileContent + ) ); - - const splitContent = fileContent.split(EOL); - - (Array.isArray(testDefinition.AiEvaluationDefinition.testCase) - ? // xml parser will not parse single node into array - testDefinition.AiEvaluationDefinition.testCase - : [testDefinition.AiEvaluationDefinition.testCase] - ).map(test => { - const line = splitContent.findIndex(l => l.includes(`${test.number} a.name.localeCompare(b.name)); - - aggregator.set(testDefinitionNode.name, testDefinitionNode); }) - ); + ]); + + // Apply disambiguation labels for names that appear in both sets + const conflicts = new Set([...evalNodes.keys()].filter(k => agentforceStudioNodes.has(k))); + for (const name of conflicts) { + const evalNode = evalNodes.get(name)!; + const afsNode = agentforceStudioNodes.get(name)!; + const evalLabel = `${name} (testing-center)`; + const afsLabel = `${name} (agentforce-studio)`; + evalNode.label = evalLabel; + evalNode.name = evalLabel; + evalNode.children.forEach(c => (c.parentName = evalLabel)); + afsNode.label = afsLabel; + afsNode.name = afsLabel; + afsNode.children.forEach(c => (c.parentName = afsLabel)); + } + + // Merge into a single map keyed by the (possibly suffixed) label + const aggregator = new Map(); + for (const node of [...evalNodes.values(), ...agentforceStudioNodes.values()]) { + aggregator.set(node.name, node); + } return aggregator; }; diff --git a/src/views/testRunner.ts b/src/views/testRunner.ts index 78023a8d..d479af43 100644 --- a/src/views/testRunner.ts +++ b/src/views/testRunner.ts @@ -16,8 +16,18 @@ import * as vscode from 'vscode'; import { AgentTestOutlineProvider } from './testOutlineProvider'; -import { AgentTester, TestStatus, AgentTestResultsResponse, humanFriendlyName, metric } from '@salesforce/agents'; -import { Lifecycle, ConfigAggregator } from '@salesforce/core'; +import { + AgentTester, + AgentforceStudioTester, + createAgentTester, + TestStatus, + AgentTestResultsResponse, + AgentforceStudioTestResultsResponse, + AgentforceStudioTestStatusResponse, + humanFriendlyName, + metric +} from '@salesforce/agents'; +import { Lifecycle } from '@salesforce/core'; import { Duration } from '@salesforce/kit'; import type { AgentTestGroupNode, TestNode } from '../types'; import { CoreExtensionService } from '../services/coreExtensionService'; @@ -25,16 +35,72 @@ import { AgentTestNode } from '../types'; import { formatJson } from '../utils/jsonFormatter'; type AgentTestResults = AgentTestResultsResponse & { id: string }; +type AgentforceStudioTestResults = AgentforceStudioTestResultsResponse & { id: string }; + +type ParsedAgentforceStudioScorer = + | { kind: 'latency'; passing: boolean; latencyMs: number; reasoning: string } + | { kind: 'quality'; passing: boolean; score: number; reasoning: string } + | { kind: 'assertion'; passing: boolean; expectedValue: string; actualValue: string } + | { kind: 'unknown'; passing: boolean; raw: string }; + +function parseAgentforceStudioScorer(scorerResponse: string): ParsedAgentforceStudioScorer { + try { + const p = JSON.parse(scorerResponse) as { + status?: string; + score?: number; + reasoning?: string; + latencyMs?: number; + actualValue?: string; + expectedValue?: string; + }; + // Quality/latency scorers carry a status field + if (p.status !== undefined) { + const passing = p.status.toUpperCase() === 'PASS'; + if (p.latencyMs !== undefined) { + return { kind: 'latency', passing, latencyMs: p.latencyMs, reasoning: p.reasoning ?? '' }; + } + return { kind: 'quality', passing, score: p.score ?? 0, reasoning: p.reasoning ?? '' }; + } + // Assertion scorers compare actual vs expected + if (p.actualValue !== undefined || p.expectedValue !== undefined) { + const passing = p.actualValue !== undefined && p.actualValue === p.expectedValue; + return { kind: 'assertion', passing, expectedValue: p.expectedValue ?? '', actualValue: p.actualValue ?? '' }; + } + return { kind: 'unknown', passing: false, raw: scorerResponse }; + } catch { + return { kind: 'unknown', passing: false, raw: scorerResponse }; + } +} export class AgentTestRunner { private testGroupNameToResult = new Map(); + private agentforceStudioTestGroupNameToResult = new Map(); constructor(private testOutline: AgentTestOutlineProvider) {} public displayTestDetails(test: TestNode) { const channelService = CoreExtensionService.getTestChannelService(); channelService.showChannelOutput(); channelService.clear(); - if (test.parentName == '') { + + // Check Agentforce Studio results first, then fall back to standard results + const agentforceStudioResult = + this.agentforceStudioTestGroupNameToResult.get(test.name) ?? + this.agentforceStudioTestGroupNameToResult.get(test.parentName); + if (agentforceStudioResult) { + if (!test.parentName) { + channelService.appendLine(`Job Id: ${agentforceStudioResult.id}`); + this.printAgentforceStudioTestSummary(agentforceStudioResult); + return; + } + const testInfo = structuredClone(agentforceStudioResult); + if (test instanceof AgentTestNode) { + testInfo.testCases = testInfo.testCases.filter(f => `#${f.testNumber}` === test.name); + } + this.displayAgentforceStudioTestCases(testInfo); + return; + } + + if (!test.parentName) { // this is the parent test group, so we only show the test summary, test id const result = this.testGroupNameToResult.get(test.name); if (result) { @@ -58,7 +124,7 @@ export class AgentTestRunner { // filter to selected test case testInfo.testCases = testInfo.testCases.filter(f => `#${f.testNumber}` === test.name); } - testInfo.testCases.map(tc => { + testInfo.testCases.forEach(tc => { channelService.appendLine('════════════════════════════════════════════════════════════════════════'); channelService.appendLine(`CASE #${tc.testNumber} - ${testInfo.subjectName}`); channelService.appendLine('════════════════════════════════════════════════════════════════════════'); @@ -69,7 +135,7 @@ export class AgentTestRunner { // this is the output for topics/action/output validation (actual v expected) // filter out other metrics from it .filter(f => !metric.includes(f.name as (typeof metric)[number])) - .map(tr => { + .forEach(tr => { channelService.appendLine( `❯ ${humanFriendlyName(tr.name).toUpperCase()}: ${tr.result} ${tr.result === 'PASS' ? '✅' : '❌'}` ); @@ -98,7 +164,7 @@ export class AgentTestRunner { if (metricResults.length > 0) { channelService.appendLine(`❯ METRICS (Value/Threshold)`); channelService.appendLine('────────────────────────────────────────────────────────────────────────'); - metricResults.map(tr => { + metricResults.forEach(tr => { if (tr.name === 'output_latency_milliseconds') { channelService.appendLine( `⏱️ : ${humanFriendlyName(tr.name).toUpperCase()} (${tr.score}ms) ${tr.metricExplainability ? `: ${tr.metricExplainability}` : ''}` @@ -127,43 +193,41 @@ export class AgentTestRunner { const hrstart = process.hrtime(); telemetryService?.sendCommandEvent(commandName, hrstart, { commandName }); try { - const configAggregator = await ConfigAggregator.create(); const lifecycle = await Lifecycle.getInstance(); channelService.clear(); channelService.showChannelOutput(); - const tester = new AgentTester(await CoreExtensionService.getDefaultConnection()); - channelService.appendLine(`Starting ${test.name} tests: ${new Date().toLocaleString()}`); + const connection = await CoreExtensionService.getDefaultConnection(); + const { runner, type } = await createAgentTester(connection, { explicitType: test.runnerType }); + channelService.appendLine(`Starting ${test.testDefinitionName} tests: ${new Date().toLocaleString()}`); + vscode.window.withProgress( { location: vscode.ProgressLocation.Notification, - title: `Running ${test.name}...` + title: `Running ${test.testDefinitionName}...` }, async progress => { return new Promise((resolve, reject) => { lifecycle.on( 'AGENT_TEST_POLLING_EVENT', async (data: { - status: TestStatus; + status: TestStatus | AgentforceStudioTestStatusResponse['status']; completedTestCases: number; totalTestCases: number; failingTestCases: number; passingTestCases: number; }) => { - switch (data.status) { - case 'NEW': - case 'IN_PROGRESS': - // every time IN_PROGRESS is returned, 10 is added, is possible to 100% progress bar and tests not be done - progress.report({ increment: 10, message: `Status: In Progress` }); - break; - case 'ERROR': - case 'TERMINATED': - progress.report({ increment: 100, message: `Status: ${data.status}` }); - setTimeout(() => reject(), 1500); - break; - case 'COMPLETED': - progress.report({ increment: 100, message: `Status: ${data.status}` }); - setTimeout(() => resolve({}), 1500); - break; + const statusUpper = data.status.toUpperCase(); + if (['NEW', 'IN_PROGRESS', 'CREATED'].includes(statusUpper)) { + // every time IN_PROGRESS is returned, 10 is added, is possible to 100% progress bar and tests not be done + progress.report({ increment: 10, message: `Status: In Progress` }); + } else if (['ERROR', 'TERMINATED', 'FAILED'].includes(statusUpper)) { + progress.report({ increment: 100, message: `Status: ${data.status}` }); + setTimeout(() => reject(), 1500); + } else if (['COMPLETED', 'SUCCESS'].includes(statusUpper)) { + progress.report({ increment: 100, message: `Status: ${data.status}` }); + setTimeout(() => resolve({}), 1500); + } else { + channelService.appendLine(`Warning: Unexpected test status: ${data.status}`); } } ); @@ -171,33 +235,11 @@ export class AgentTestRunner { } ); - const response = await tester.start(test.name); - // begin in-progress - this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); - - const result = { - ...(await tester.poll(response.runId, { timeout: Duration.minutes(100) })), - id: response.runId - }; - result.id = response.runId; - channelService.appendLine(`Job Id: ${result.id}`); - - this.testGroupNameToResult.set(test.name, result); - this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); - let hasFailure = false; - result.testCases.map(tc => { - hasFailure = tc.testResults.some(tr => tr.result === 'FAILURE'); - - // update the children, accordingly if they passed/failed - this.testOutline - .getTestGroup(test.name) - ?.getChildren() - .find(child => child.name === `#${tc.testNumber}`) - ?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); - }); - // update the parent's icon - this.testOutline.getTestGroup(test.name)?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); - this.printTestSummary(result); + if (type === 'agentforce-studio') { + await this.runAgentforceStudioTest(test, runner as AgentforceStudioTester, channelService); + } else { + await this.runTestingCenterTest(test, runner as AgentTester, channelService); + } } catch (e) { const error = e as Error; void Lifecycle.getInstance().emit('AGENT_TEST_POLLING_EVENT', { status: 'ERROR' }); @@ -207,18 +249,138 @@ export class AgentTestRunner { } } + private async runTestingCenterTest( + test: AgentTestGroupNode, + tester: AgentTester, + channelService: ReturnType + ): Promise { + const response = await tester.start(test.testDefinitionName); + channelService.appendLine(`Job Id: ${response.runId}`); + this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); + + const result = { + ...(await tester.poll(response.runId, { timeout: Duration.minutes(100) })), + id: response.runId + }; + + this.testGroupNameToResult.set(test.name, result); + this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); + let hasFailure = false; + result.testCases.forEach(tc => { + const tcFailed = tc.testResults.some(tr => tr.result === 'FAILURE'); + if (tcFailed) hasFailure = true; + this.testOutline + .getTestGroup(test.name) + ?.getChildren() + .find(child => child.name === `#${tc.testNumber}`) + ?.updateOutcome(tcFailed ? 'ERROR' : 'COMPLETED'); + }); + this.testOutline.getTestGroup(test.name)?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); + this.printTestSummary(result); + } + + private async runAgentforceStudioTest( + test: AgentTestGroupNode, + tester: AgentforceStudioTester, + channelService: ReturnType + ): Promise { + const response = await tester.start(test.testDefinitionName); + channelService.appendLine(`Job Id: ${response.runId}`); + this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); + + const result: AgentforceStudioTestResults = { + ...(await tester.poll(response.runId, { timeout: Duration.minutes(100) })), + id: response.runId + }; + + this.agentforceStudioTestGroupNameToResult.set(test.name, result); + this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); + + let hasFailure = false; + result.testCases.forEach(tc => { + // A test case with no scorer results has not been evaluated and counts as a failure + const tcFailed = + tc.testScorerResults.length === 0 || + tc.testScorerResults.some(s => !parseAgentforceStudioScorer(s.scorerResponse).passing); + if (tcFailed) hasFailure = true; + this.testOutline + .getTestGroup(test.name) + ?.getChildren() + .find(child => child.name === `#${tc.testNumber}`) + ?.updateOutcome(tcFailed ? 'ERROR' : 'COMPLETED'); + }); + this.testOutline.getTestGroup(test.name)?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); + this.printAgentforceStudioTestSummary(result); + } + + private displayAgentforceStudioTestCases(testInfo: AgentforceStudioTestResults): void { + const channelService = CoreExtensionService.getTestChannelService(); + + testInfo.testCases.forEach(tc => { + channelService.appendLine('════════════════════════════════════════════════════════════════════════'); + channelService.appendLine(`CASE #${tc.testNumber}`); + channelService.appendLine('════════════════════════════════════════════════════════════════════════'); + channelService.appendLine(''); + let passing = 0; + tc.testScorerResults.forEach(scorer => { + const parsed = parseAgentforceStudioScorer(scorer.scorerResponse); + if (parsed.passing) passing++; + const icon = parsed.passing ? '✅' : '❌'; + const verdict = parsed.passing ? 'PASS' : 'FAILURE'; + channelService.appendLine(`❯ ${scorer.scorerName.toUpperCase()}: ${verdict} ${icon}`); + channelService.appendLine('────────────────────────────────────────────────────────────────────────'); + if (parsed.kind === 'assertion') { + channelService.appendLine(`EXPECTED : ${parsed.expectedValue.replaceAll('\n', '')}`); + channelService.appendLine(`ACTUAL : ${parsed.actualValue.replaceAll('\n', '')}`); + } else if (parsed.kind === 'latency') { + channelService.appendLine(`LATENCY : ${parsed.latencyMs}ms`); + if (parsed.reasoning) { + channelService.appendLine(`REASONING: ${parsed.reasoning}`); + } + } else if (parsed.kind === 'quality') { + channelService.appendLine(`SCORE : ${parsed.score}`); + if (parsed.reasoning) { + channelService.appendLine(`REASONING: ${parsed.reasoning}`); + } + } else { + channelService.appendLine(`RESPONSE : ${parsed.raw}`); + } + channelService.appendLine(''); + }); + + const failing = tc.testScorerResults.length - passing; + channelService.appendLine('────────────────────────────────────────────────────────────────────────'); + channelService.appendLine( + `TEST CASE SUMMARY: ${tc.testScorerResults.length} tests run | ✅ ${passing} passed | ❌ ${failing} failed` + ); + }); + } + private printTestSummary(result: AgentTestResults) { const channelService = CoreExtensionService.getTestChannelService(); channelService.appendLine(result.status); channelService.appendLine(''); channelService.appendLine('Test Results'); const total = result.testCases.length; - channelService.appendLine( - `Passing: ${result.testCases.filter(tc => tc.testResults.every(tr => tr.result === 'PASS')).length}/${total}` - ); - channelService.appendLine( - `Failing: ${result.testCases.filter(tc => tc.testResults.some(tr => tr.result === 'FAILURE')).length}/${total}` - ); + const passing = result.testCases.filter(tc => tc.testResults.every(tr => tr.result === 'PASS')).length; + channelService.appendLine(`Passing: ${passing}/${total}`); + channelService.appendLine(`Failing: ${total - passing}/${total}`); + channelService.appendLine(''); + channelService.appendLine(`Select a test case in the Test View panel for more information`); + } + + private printAgentforceStudioTestSummary(result: AgentforceStudioTestResults): void { + const channelService = CoreExtensionService.getTestChannelService(); + const tcPassing = (tc: (typeof result.testCases)[number]): boolean => + tc.testScorerResults.length > 0 && + tc.testScorerResults.every(s => parseAgentforceStudioScorer(s.scorerResponse).passing); + channelService.appendLine(result.status); + channelService.appendLine(''); + channelService.appendLine('Test Results'); + const total = result.testCases.length; + const passing = result.testCases.filter(tcPassing).length; + channelService.appendLine(`Passing: ${passing}/${total}`); + channelService.appendLine(`Failing: ${total - passing}/${total}`); channelService.appendLine(''); channelService.appendLine(`Select a test case in the Test View panel for more information`); } diff --git a/test/views/testOutlineProvider.test.ts b/test/views/testOutlineProvider.test.ts index 49510baf..35724e78 100644 --- a/test/views/testOutlineProvider.test.ts +++ b/test/views/testOutlineProvider.test.ts @@ -205,7 +205,10 @@ describe('parseAgentTestsFromProject', () => { fsPath: '/test/MyAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValue([mockUri]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([mockUri]) // aiEvaluationDefinition + .mockResolvedValueOnce([]); // aiTestingDefinition jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); const result = await parseAgentTestsFromProject(); @@ -248,7 +251,10 @@ describe('parseAgentTestsFromProject', () => { fsPath: '/test/MultiTestAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValue([mockUri]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([mockUri]) // aiEvaluationDefinition + .mockResolvedValueOnce([]); // aiTestingDefinition jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); const result = await parseAgentTestsFromProject(); @@ -292,7 +298,10 @@ describe('parseAgentTestsFromProject', () => { fsPath: '/test/Agent2.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValue([mockUri1, mockUri2]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([mockUri1, mockUri2]) // aiEvaluationDefinition + .mockResolvedValueOnce([]); // aiTestingDefinition jest .spyOn(vscode.workspace.fs, 'readFile') .mockResolvedValueOnce(Buffer.from(mockXmlContent1)) @@ -318,13 +327,16 @@ describe('parseAgentTestsFromProject', () => { fsPath: '/test/BrokenAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValue([mockUri]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([mockUri]) // aiEvaluationDefinition + .mockResolvedValueOnce([]); // aiTestingDefinition jest.spyOn(vscode.workspace.fs, 'readFile').mockRejectedValue(new Error('read failure')); await expect(parseAgentTestsFromProject()).rejects.toThrow('read failure'); }); - it('should throw when XML is missing test cases', async () => { + it('should skip definition files with no test cases', async () => { const mockUri = { fsPath: '/test/EmptyAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; @@ -333,9 +345,201 @@ describe('parseAgentTestsFromProject', () => { `; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValue([mockUri]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([mockUri]) // aiEvaluationDefinition + .mockResolvedValueOnce([]); // aiTestingDefinition + jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); + + const result = await parseAgentTestsFromProject(); + expect(result.size).toBe(0); + }); + + it('should parse single Agentforce Studio test case from AiTestingDefinition XML file', async () => { + const mockXmlContent = ` + + + 1 + + Agentforce Studio utterance 1 + + + GeneralFAQ + topic_sequence_match + + +`; + + const mockUri = { + fsPath: '/test/MyAgentforceStudioAgent.aiTestingDefinition-meta.xml' + } as vscode.Uri; + + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([]) // aiEvaluationDefinition + .mockResolvedValueOnce([mockUri]); // aiTestingDefinition + jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); + + const result = await parseAgentTestsFromProject(); + + expect(result.size).toBe(1); + expect(result.has('MyAgentforceStudioAgent')).toBe(true); + + const testGroup = result.get('MyAgentforceStudioAgent'); + expect(testGroup).toBeDefined(); + expect(testGroup!.name).toBe('MyAgentforceStudioAgent'); + expect(testGroup!.children.length).toBe(1); + expect(testGroup!.children[0].name).toBe('#1'); + expect((testGroup!.children[0] as any).description).toBe('Agentforce Studio utterance 1'); + }); + + it('should parse multiple Agentforce Studio test cases from AiTestingDefinition XML file', async () => { + const mockXmlContent = ` + + + 2 + + Agentforce Studio utterance 2 + + + + 1 + + Agentforce Studio utterance 1 + + +`; + + const mockUri = { + fsPath: '/test/MultiAgentforceStudioAgent.aiTestingDefinition-meta.xml' + } as vscode.Uri; + + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([]) // aiEvaluationDefinition + .mockResolvedValueOnce([mockUri]); // aiTestingDefinition jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); - await expect(parseAgentTestsFromProject()).rejects.toThrow(); + const result = await parseAgentTestsFromProject(); + + expect(result.size).toBe(1); + const testGroup = result.get('MultiAgentforceStudioAgent'); + expect(testGroup).toBeDefined(); + expect(testGroup!.children.length).toBe(2); + expect(testGroup!.children[0].name).toBe('#1'); + expect(testGroup!.children[1].name).toBe('#2'); + }); + + it('should parse both AiEvaluationDefinition and AiTestingDefinition files together', async () => { + const evalXml = ` + + + 1 + + Eval utterance + + +`; + + const agentforceStudioXml = ` + + + 1 + + Agentforce Studio utterance + + +`; + + const evalUri = { fsPath: '/test/EvalAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; + const agentforceStudioUri = { + fsPath: '/test/AgentforceStudioAgent.aiTestingDefinition-meta.xml' + } as vscode.Uri; + + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([evalUri]) // aiEvaluationDefinition + .mockResolvedValueOnce([agentforceStudioUri]); // aiTestingDefinition + jest + .spyOn(vscode.workspace.fs, 'readFile') + .mockResolvedValueOnce(Buffer.from(evalXml)) + .mockResolvedValueOnce(Buffer.from(agentforceStudioXml)); + + const result = await parseAgentTestsFromProject(); + + expect(result.size).toBe(2); + expect(result.has('EvalAgent')).toBe(true); + expect(result.has('AgentforceStudioAgent')).toBe(true); + }); + + it('should apply disambiguation suffixes when both types share the same API name', async () => { + const sharedXml = (root: string) => ` +<${root} xmlns="http://soap.sforce.com/2006/04/metadata"> + + 1 + + utterance + + +`; + + const evalUri = { fsPath: '/test/WillieTest.aiEvaluationDefinition-meta.xml' } as vscode.Uri; + const agentforceStudioUri = { fsPath: '/test/WillieTest.aiTestingDefinition-meta.xml' } as vscode.Uri; + + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([evalUri]) + .mockResolvedValueOnce([agentforceStudioUri]); + jest + .spyOn(vscode.workspace.fs, 'readFile') + .mockResolvedValueOnce(Buffer.from(sharedXml('AiEvaluationDefinition'))) + .mockResolvedValueOnce(Buffer.from(sharedXml('AiTestingDefinition'))); + + const result = await parseAgentTestsFromProject(); + + expect(result.size).toBe(2); + expect(result.has('WillieTest (testing-center)')).toBe(true); + expect(result.has('WillieTest (agentforce-studio)')).toBe(true); + // underlying API name is preserved + expect(result.get('WillieTest (testing-center)')!.testDefinitionName).toBe('WillieTest'); + expect(result.get('WillieTest (agentforce-studio)')!.testDefinitionName).toBe('WillieTest'); + // runner types are correct + expect(result.get('WillieTest (testing-center)')!.runnerType).toBe('testing-center'); + expect(result.get('WillieTest (agentforce-studio)')!.runnerType).toBe('agentforce-studio'); + // children parentName matches the suffixed label + expect(result.get('WillieTest (testing-center)')!.children[0].parentName).toBe('WillieTest (testing-center)'); + expect(result.get('WillieTest (agentforce-studio)')!.children[0].parentName).toBe('WillieTest (agentforce-studio)'); + }); + + it('should not apply suffixes when API names are unique across both types', async () => { + const evalXml = ` + + 1u +`; + const agentforceStudioXml = ` + + 1u +`; + + const evalUri = { fsPath: '/test/UniqueEval.aiEvaluationDefinition-meta.xml' } as vscode.Uri; + const agentforceStudioUri = { + fsPath: '/test/UniqueAgentforceStudio.aiTestingDefinition-meta.xml' + } as vscode.Uri; + + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([evalUri]) + .mockResolvedValueOnce([agentforceStudioUri]); + jest + .spyOn(vscode.workspace.fs, 'readFile') + .mockResolvedValueOnce(Buffer.from(evalXml)) + .mockResolvedValueOnce(Buffer.from(agentforceStudioXml)); + + const result = await parseAgentTestsFromProject(); + + expect(result.has('UniqueEval')).toBe(true); + expect(result.has('UniqueAgentforceStudio')).toBe(true); + expect(result.get('UniqueEval')!.testDefinitionName).toBe('UniqueEval'); + expect(result.get('UniqueAgentforceStudio')!.testDefinitionName).toBe('UniqueAgentforceStudio'); }); });