From 700824d55189bf194ceb0c4e54b1dc2960b2f8bf Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Wed, 29 Apr 2026 15:45:26 -0600 Subject: [PATCH 1/7] feat: add NGT support --- src/types/MetadataDefinitions.ts | 16 ++ src/types/TestNodes.ts | 7 +- src/views/testOutlineProvider.ts | 114 +++++++---- src/views/testRunner.ts | 253 ++++++++++++++++++++----- test/views/testOutlineProvider.test.ts | 206 +++++++++++++++++++- 5 files changed, 509 insertions(+), 87 deletions(-) diff --git a/src/types/MetadataDefinitions.ts b/src/types/MetadataDefinitions.ts index bc4d37a1..28305045 100644 --- a/src/types/MetadataDefinitions.ts +++ b/src/types/MetadataDefinitions.ts @@ -22,3 +22,19 @@ export type AgentTestCase = { { name: 'expectedOutcome'; expectedValue: string } ]; }; + +export type NGTTestCaseMetadata = { + number: string; + inputs: { utterance: string }; +}; + +export type AiTestingDefinition = { + AiTestingDefinition: { + description: string; + name: string; + subjectType: 'AGENT'; + subjectName: string; + subjectVersion: string; + testCase: NGTTestCaseMetadata | NGTTestCaseMetadata[]; + }; +}; diff --git a/src/types/TestNodes.ts b/src/types/TestNodes.ts index c014885c..e4b2792b 100644 --- a/src/types/TestNodes.ts +++ b/src/types/TestNodes.ts @@ -1,5 +1,5 @@ import * as vscode from 'vscode'; -import { TestStatus } from '@salesforce/agents'; +import { TestStatus, type TestRunnerType } from '@salesforce/agents'; import { getTestOutlineProvider } from '../views/testOutlineProvider'; /** @@ -66,8 +66,13 @@ export abstract class TestNode extends vscode.TreeItem { * has children AgentTestNode for individual test cases */ export class AgentTestGroupNode extends TestNode { + /** The actual metadata API name, which may differ from the display label when there's a naming conflict. */ + public testDefinitionName: string; + public runnerType: TestRunnerType = 'testing-center'; + constructor(label: string, location?: vscode.Location) { super(label, vscode.TreeItemCollapsibleState.Expanded, location ?? null); + this.testDefinitionName = label; } public contextValue = 'agentTestGroup'; diff --git a/src/views/testOutlineProvider.ts b/src/views/testOutlineProvider.ts index f8416cb8..ff1b1900 100644 --- a/src/views/testOutlineProvider.ts +++ b/src/views/testOutlineProvider.ts @@ -18,50 +18,100 @@ import { basename } from 'path'; import * as vscode from 'vscode'; import * as xml from 'fast-xml-parser'; import { Commands } from '../enums/commands'; -import { AgentTestGroupNode, AgentTestNode, AiEvaluationDefinition, TestNode } from '../types'; +import { AgentTestGroupNode, AgentTestNode, AiEvaluationDefinition, AiTestingDefinition, TestNode } from '../types'; +import type { TestRunnerType } from '@salesforce/agents'; const NO_TESTS_MESSAGE = 'no tests found'; const NO_TESTS_DESCRIPTION = 'no test description'; const AGENT_TESTS = 'AgentTests'; + +const buildTestGroupNode = ( + definitionUri: vscode.Uri, + definitionApiName: string, + label: string, + runnerType: TestRunnerType, + testCases: Array<{ number: string; inputs: { utterance: string } }>, + fileContent: string +): AgentTestGroupNode => { + const testDefinitionNode = new AgentTestGroupNode( + label, + new vscode.Location(definitionUri, new vscode.Position(0, 0)) + ); + testDefinitionNode.testDefinitionName = definitionApiName; + testDefinitionNode.runnerType = runnerType; + const splitContent = fileContent.split(EOL); + testCases.map(test => { + const line = splitContent.findIndex(l => l.includes(`${test.number} a.name.localeCompare(b.name)); + return testDefinitionNode; +}; + export const parseAgentTestsFromProject = async (): Promise> => { - const aiTestDefs = await vscode.workspace.findFiles('**/*.aiEvaluationDefinition-meta.xml'); - //from the aiTestDef files, parse the xml using fast-xml-parser, find the testSetName that it points to - const aggregator = new Map(); + const [aiEvalDefs, aiTestingDefs] = await Promise.all([ + vscode.workspace.findFiles('**/*.aiEvaluationDefinition-meta.xml'), + vscode.workspace.findFiles('**/*.aiTestingDefinition-meta.xml') + ]); const parser = new xml.XMLParser(); - await Promise.all( - aiTestDefs.map(async definition => { + + // Parse both sets independently, keyed by API name + const evalNodes = new Map(); + const ngtNodes = new Map(); + + await Promise.all([ + ...aiEvalDefs.map(async definition => { const fileContent = (await vscode.workspace.fs.readFile(definition)).toString(); const testDefinition = parser.parse(fileContent) as AiEvaluationDefinition; const definitionApiName = basename(definition.fsPath, '.aiEvaluationDefinition-meta.xml'); - - const testDefinitionNode = new AgentTestGroupNode( + const testCases = Array.isArray(testDefinition.AiEvaluationDefinition.testCase) + ? testDefinition.AiEvaluationDefinition.testCase + : [testDefinition.AiEvaluationDefinition.testCase]; + evalNodes.set( definitionApiName, - new vscode.Location(definition, new vscode.Position(0, 0)) + buildTestGroupNode(definition, definitionApiName, definitionApiName, 'testing-center', testCases, fileContent) + ); + }), + ...aiTestingDefs.map(async definition => { + const fileContent = (await vscode.workspace.fs.readFile(definition)).toString(); + const testDefinition = parser.parse(fileContent) as AiTestingDefinition; + const definitionApiName = basename(definition.fsPath, '.aiTestingDefinition-meta.xml'); + const testCases = Array.isArray(testDefinition.AiTestingDefinition.testCase) + ? testDefinition.AiTestingDefinition.testCase + : [testDefinition.AiTestingDefinition.testCase]; + ngtNodes.set( + definitionApiName, + buildTestGroupNode(definition, definitionApiName, definitionApiName, 'agentforce-studio', testCases, fileContent) ); - - const splitContent = fileContent.split(EOL); - - (Array.isArray(testDefinition.AiEvaluationDefinition.testCase) - ? // xml parser will not parse single node into array - testDefinition.AiEvaluationDefinition.testCase - : [testDefinition.AiEvaluationDefinition.testCase] - ).map(test => { - const line = splitContent.findIndex(l => l.includes(`${test.number} a.name.localeCompare(b.name)); - - aggregator.set(testDefinitionNode.name, testDefinitionNode); }) - ); + ]); + + // Apply disambiguation labels for names that appear in both sets + const conflicts = new Set([...evalNodes.keys()].filter(k => ngtNodes.has(k))); + for (const name of conflicts) { + const evalNode = evalNodes.get(name)!; + const ngtNode = ngtNodes.get(name)!; + const evalLabel = `${name} (testing-center)`; + const ngtLabel = `${name} (agentforce-studio)`; + evalNode.label = evalLabel; + evalNode.name = evalLabel; + evalNode.children.forEach(c => (c.parentName = evalLabel)); + ngtNode.label = ngtLabel; + ngtNode.name = ngtLabel; + ngtNode.children.forEach(c => (c.parentName = ngtLabel)); + } + + // Merge into a single map keyed by the (possibly suffixed) label + const aggregator = new Map(); + for (const node of [...evalNodes.values(), ...ngtNodes.values()]) { + aggregator.set(node.name, node); + } return aggregator; }; diff --git a/src/views/testRunner.ts b/src/views/testRunner.ts index 78023a8d..c9aab833 100644 --- a/src/views/testRunner.ts +++ b/src/views/testRunner.ts @@ -16,8 +16,17 @@ import * as vscode from 'vscode'; import { AgentTestOutlineProvider } from './testOutlineProvider'; -import { AgentTester, TestStatus, AgentTestResultsResponse, humanFriendlyName, metric } from '@salesforce/agents'; -import { Lifecycle, ConfigAggregator } from '@salesforce/core'; +import { + AgentTester, + AgentTesterNGT, + createAgentTester, + TestStatus, + AgentTestResultsResponse, + AgentTestNGTResultsResponse, + humanFriendlyName, + metric +} from '@salesforce/agents'; +import { Lifecycle } from '@salesforce/core'; import { Duration } from '@salesforce/kit'; import type { AgentTestGroupNode, TestNode } from '../types'; import { CoreExtensionService } from '../services/coreExtensionService'; @@ -25,15 +34,70 @@ import { AgentTestNode } from '../types'; import { formatJson } from '../utils/jsonFormatter'; type AgentTestResults = AgentTestResultsResponse & { id: string }; +type NGTTestResults = AgentTestNGTResultsResponse & { id: string }; + +type ParsedNGTScorer = + | { kind: 'latency'; passing: boolean; latencyMs: number; reasoning: string } + | { kind: 'quality'; passing: boolean; score: number; reasoning: string } + | { kind: 'assertion'; passing: boolean; expectedValue: string; actualValue: string } + | { kind: 'unknown'; passing: boolean; raw: string }; + +function parseNGTScorer(scorerResponse: string): ParsedNGTScorer { + try { + const p = JSON.parse(scorerResponse) as { + status?: string; + score?: number; + reasoning?: string; + latencyMs?: number; + actualValue?: string; + expectedValue?: string; + }; + // Quality/latency scorers carry a status field + if (p.status !== undefined) { + const passing = p.status.toUpperCase() === 'PASS'; + if (p.latencyMs !== undefined) { + return { kind: 'latency', passing, latencyMs: p.latencyMs, reasoning: p.reasoning ?? '' }; + } + return { kind: 'quality', passing, score: p.score ?? 0, reasoning: p.reasoning ?? '' }; + } + // Assertion scorers compare actual vs expected + if (p.actualValue !== undefined || p.expectedValue !== undefined) { + const passing = p.actualValue !== undefined && p.actualValue === p.expectedValue; + return { kind: 'assertion', passing, expectedValue: p.expectedValue ?? '', actualValue: p.actualValue ?? '' }; + } + return { kind: 'unknown', passing: false, raw: scorerResponse }; + } catch { + return { kind: 'unknown', passing: false, raw: scorerResponse }; + } +} export class AgentTestRunner { private testGroupNameToResult = new Map(); + private ngtTestGroupNameToResult = new Map(); constructor(private testOutline: AgentTestOutlineProvider) {} public displayTestDetails(test: TestNode) { const channelService = CoreExtensionService.getTestChannelService(); channelService.showChannelOutput(); channelService.clear(); + + // Check NGT results first, then fall back to standard results + const ngtResult = + this.ngtTestGroupNameToResult.get(test.name) ?? this.ngtTestGroupNameToResult.get(test.parentName); + if (ngtResult) { + if (test.parentName == '') { + channelService.appendLine(`Job Id: ${ngtResult.id}`); + this.printNGTTestSummary(ngtResult); + return; + } + const testInfo = structuredClone(ngtResult); + if (test instanceof AgentTestNode) { + testInfo.testCases = testInfo.testCases.filter(f => `#${f.testNumber}` === test.name); + } + this.displayNGTTestCases(testInfo); + return; + } + if (test.parentName == '') { // this is the parent test group, so we only show the test summary, test id const result = this.testGroupNameToResult.get(test.name); @@ -127,43 +191,39 @@ export class AgentTestRunner { const hrstart = process.hrtime(); telemetryService?.sendCommandEvent(commandName, hrstart, { commandName }); try { - const configAggregator = await ConfigAggregator.create(); const lifecycle = await Lifecycle.getInstance(); channelService.clear(); channelService.showChannelOutput(); - const tester = new AgentTester(await CoreExtensionService.getDefaultConnection()); - channelService.appendLine(`Starting ${test.name} tests: ${new Date().toLocaleString()}`); + const connection = await CoreExtensionService.getDefaultConnection(); + const { runner, type } = await createAgentTester(connection, { explicitType: test.runnerType }); + channelService.appendLine(`Starting ${test.testDefinitionName} tests: ${new Date().toLocaleString()}`); + vscode.window.withProgress( { location: vscode.ProgressLocation.Notification, - title: `Running ${test.name}...` + title: `Running ${test.testDefinitionName}...` }, async progress => { return new Promise((resolve, reject) => { lifecycle.on( 'AGENT_TEST_POLLING_EVENT', async (data: { - status: TestStatus; + status: TestStatus | string; completedTestCases: number; totalTestCases: number; failingTestCases: number; passingTestCases: number; }) => { - switch (data.status) { - case 'NEW': - case 'IN_PROGRESS': - // every time IN_PROGRESS is returned, 10 is added, is possible to 100% progress bar and tests not be done - progress.report({ increment: 10, message: `Status: In Progress` }); - break; - case 'ERROR': - case 'TERMINATED': - progress.report({ increment: 100, message: `Status: ${data.status}` }); - setTimeout(() => reject(), 1500); - break; - case 'COMPLETED': - progress.report({ increment: 100, message: `Status: ${data.status}` }); - setTimeout(() => resolve({}), 1500); - break; + const statusUpper = data.status.toUpperCase(); + if (statusUpper === 'NEW' || statusUpper === 'IN_PROGRESS') { + // every time IN_PROGRESS is returned, 10 is added, is possible to 100% progress bar and tests not be done + progress.report({ increment: 10, message: `Status: In Progress` }); + } else if (statusUpper === 'ERROR' || statusUpper === 'TERMINATED') { + progress.report({ increment: 100, message: `Status: ${data.status}` }); + setTimeout(() => reject(), 1500); + } else if (statusUpper === 'COMPLETED' || statusUpper === 'SUCCESS') { + progress.report({ increment: 100, message: `Status: ${data.status}` }); + setTimeout(() => resolve({}), 1500); } } ); @@ -171,33 +231,11 @@ export class AgentTestRunner { } ); - const response = await tester.start(test.name); - // begin in-progress - this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); - - const result = { - ...(await tester.poll(response.runId, { timeout: Duration.minutes(100) })), - id: response.runId - }; - result.id = response.runId; - channelService.appendLine(`Job Id: ${result.id}`); - - this.testGroupNameToResult.set(test.name, result); - this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); - let hasFailure = false; - result.testCases.map(tc => { - hasFailure = tc.testResults.some(tr => tr.result === 'FAILURE'); - - // update the children, accordingly if they passed/failed - this.testOutline - .getTestGroup(test.name) - ?.getChildren() - .find(child => child.name === `#${tc.testNumber}`) - ?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); - }); - // update the parent's icon - this.testOutline.getTestGroup(test.name)?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); - this.printTestSummary(result); + if (type === 'agentforce-studio') { + await this.runAgentforceStudioTest(test, runner as AgentTesterNGT, channelService); + } else { + await this.runTestingCenterTest(test, runner as AgentTester, channelService); + } } catch (e) { const error = e as Error; void Lifecycle.getInstance().emit('AGENT_TEST_POLLING_EVENT', { status: 'ERROR' }); @@ -207,6 +245,109 @@ export class AgentTestRunner { } } + private async runTestingCenterTest( + test: AgentTestGroupNode, + tester: AgentTester, + channelService: ReturnType + ): Promise { + const response = await tester.start(test.testDefinitionName); + channelService.appendLine(`Job Id: ${response.runId}`); + this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); + + const result = { + ...(await tester.poll(response.runId, { timeout: Duration.minutes(100) })), + id: response.runId + }; + + this.testGroupNameToResult.set(test.name, result); + this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); + let hasFailure = false; + result.testCases.map(tc => { + hasFailure = tc.testResults.some(tr => tr.result === 'FAILURE'); + this.testOutline + .getTestGroup(test.name) + ?.getChildren() + .find(child => child.name === `#${tc.testNumber}`) + ?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); + }); + this.testOutline.getTestGroup(test.name)?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); + this.printTestSummary(result); + } + + private async runAgentforceStudioTest( + test: AgentTestGroupNode, + tester: AgentTesterNGT, + channelService: ReturnType + ): Promise { + const response = await tester.start(test.testDefinitionName); + channelService.appendLine(`Job Id: ${response.runId}`); + this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); + + const result: NGTTestResults = { + ...(await tester.poll(response.runId, { timeout: Duration.minutes(100) })), + id: response.runId + }; + + this.ngtTestGroupNameToResult.set(test.name, result); + this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); + + let hasFailure = false; + result.testCases.map(tc => { + const tcFailed = + tc.testScorerResults.length === 0 || tc.testScorerResults.some(s => !parseNGTScorer(s.scorerResponse).passing); + if (tcFailed) hasFailure = true; + this.testOutline + .getTestGroup(test.name) + ?.getChildren() + .find(child => child.name === `#${tc.testNumber}`) + ?.updateOutcome(tcFailed ? 'ERROR' : 'COMPLETED'); + }); + this.testOutline.getTestGroup(test.name)?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); + this.printNGTTestSummary(result); + } + + private displayNGTTestCases(testInfo: NGTTestResults): void { + const channelService = CoreExtensionService.getTestChannelService(); + + testInfo.testCases.map(tc => { + channelService.appendLine('════════════════════════════════════════════════════════════════════════'); + channelService.appendLine(`CASE #${tc.testNumber}`); + channelService.appendLine('════════════════════════════════════════════════════════════════════════'); + channelService.appendLine(''); + tc.testScorerResults.map(scorer => { + const parsed = parseNGTScorer(scorer.scorerResponse); + const icon = parsed.passing ? '✅' : '❌'; + const verdict = parsed.passing ? 'PASS' : 'FAILURE'; + channelService.appendLine(`❯ ${scorer.scorerName.toUpperCase()}: ${verdict} ${icon}`); + channelService.appendLine('────────────────────────────────────────────────────────────────────────'); + if (parsed.kind === 'assertion') { + channelService.appendLine(`EXPECTED : ${parsed.expectedValue.replaceAll('\n', '')}`); + channelService.appendLine(`ACTUAL : ${parsed.actualValue.replaceAll('\n', '')}`); + } else if (parsed.kind === 'latency') { + channelService.appendLine(`LATENCY : ${parsed.latencyMs}ms`); + if (parsed.reasoning) { + channelService.appendLine(`REASONING: ${parsed.reasoning}`); + } + } else if (parsed.kind === 'quality') { + channelService.appendLine(`SCORE : ${parsed.score}`); + if (parsed.reasoning) { + channelService.appendLine(`REASONING: ${parsed.reasoning}`); + } + } else { + channelService.appendLine(`RESPONSE : ${parsed.raw}`); + } + channelService.appendLine(''); + }); + + const passing = tc.testScorerResults.filter(s => parseNGTScorer(s.scorerResponse).passing).length; + const failing = tc.testScorerResults.length - passing; + channelService.appendLine('────────────────────────────────────────────────────────────────────────'); + channelService.appendLine( + `TEST CASE SUMMARY: ${tc.testScorerResults.length} tests run | ✅ ${passing} passed | ❌ ${failing} failed` + ); + }); + } + private printTestSummary(result: AgentTestResults) { const channelService = CoreExtensionService.getTestChannelService(); channelService.appendLine(result.status); @@ -222,4 +363,18 @@ export class AgentTestRunner { channelService.appendLine(''); channelService.appendLine(`Select a test case in the Test View panel for more information`); } + + private printNGTTestSummary(result: NGTTestResults): void { + const channelService = CoreExtensionService.getTestChannelService(); + const tcPassing = (tc: (typeof result.testCases)[number]): boolean => + tc.testScorerResults.length > 0 && tc.testScorerResults.every(s => parseNGTScorer(s.scorerResponse).passing); + channelService.appendLine(result.status); + channelService.appendLine(''); + channelService.appendLine('Test Results'); + const total = result.testCases.length; + channelService.appendLine(`Passing: ${result.testCases.filter(tcPassing).length}/${total}`); + channelService.appendLine(`Failing: ${result.testCases.filter(tc => !tcPassing(tc)).length}/${total}`); + channelService.appendLine(''); + channelService.appendLine(`Select a test case in the Test View panel for more information`); + } } diff --git a/test/views/testOutlineProvider.test.ts b/test/views/testOutlineProvider.test.ts index 49510baf..6528fda6 100644 --- a/test/views/testOutlineProvider.test.ts +++ b/test/views/testOutlineProvider.test.ts @@ -205,7 +205,10 @@ describe('parseAgentTestsFromProject', () => { fsPath: '/test/MyAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValue([mockUri]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([mockUri]) // aiEvaluationDefinition + .mockResolvedValueOnce([]); // aiTestingDefinition jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); const result = await parseAgentTestsFromProject(); @@ -248,7 +251,10 @@ describe('parseAgentTestsFromProject', () => { fsPath: '/test/MultiTestAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValue([mockUri]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([mockUri]) // aiEvaluationDefinition + .mockResolvedValueOnce([]); // aiTestingDefinition jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); const result = await parseAgentTestsFromProject(); @@ -292,7 +298,10 @@ describe('parseAgentTestsFromProject', () => { fsPath: '/test/Agent2.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValue([mockUri1, mockUri2]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([mockUri1, mockUri2]) // aiEvaluationDefinition + .mockResolvedValueOnce([]); // aiTestingDefinition jest .spyOn(vscode.workspace.fs, 'readFile') .mockResolvedValueOnce(Buffer.from(mockXmlContent1)) @@ -318,7 +327,10 @@ describe('parseAgentTestsFromProject', () => { fsPath: '/test/BrokenAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValue([mockUri]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([mockUri]) // aiEvaluationDefinition + .mockResolvedValueOnce([]); // aiTestingDefinition jest.spyOn(vscode.workspace.fs, 'readFile').mockRejectedValue(new Error('read failure')); await expect(parseAgentTestsFromProject()).rejects.toThrow('read failure'); @@ -333,9 +345,193 @@ describe('parseAgentTestsFromProject', () => { `; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValue([mockUri]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([mockUri]) // aiEvaluationDefinition + .mockResolvedValueOnce([]); // aiTestingDefinition jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); await expect(parseAgentTestsFromProject()).rejects.toThrow(); }); + + it('should parse single NGT test case from AiTestingDefinition XML file', async () => { + const mockXmlContent = ` + + + 1 + + NGT utterance 1 + + + GeneralFAQ + topic_sequence_match + + +`; + + const mockUri = { + fsPath: '/test/MyNGTAgent.aiTestingDefinition-meta.xml' + } as vscode.Uri; + + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([]) // aiEvaluationDefinition + .mockResolvedValueOnce([mockUri]); // aiTestingDefinition + jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); + + const result = await parseAgentTestsFromProject(); + + expect(result.size).toBe(1); + expect(result.has('MyNGTAgent')).toBe(true); + + const testGroup = result.get('MyNGTAgent'); + expect(testGroup).toBeDefined(); + expect(testGroup!.name).toBe('MyNGTAgent'); + expect(testGroup!.children.length).toBe(1); + expect(testGroup!.children[0].name).toBe('#1'); + expect((testGroup!.children[0] as any).description).toBe('NGT utterance 1'); + }); + + it('should parse multiple NGT test cases from AiTestingDefinition XML file', async () => { + const mockXmlContent = ` + + + 2 + + NGT utterance 2 + + + + 1 + + NGT utterance 1 + + +`; + + const mockUri = { + fsPath: '/test/MultiNGTAgent.aiTestingDefinition-meta.xml' + } as vscode.Uri; + + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([]) // aiEvaluationDefinition + .mockResolvedValueOnce([mockUri]); // aiTestingDefinition + jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); + + const result = await parseAgentTestsFromProject(); + + expect(result.size).toBe(1); + const testGroup = result.get('MultiNGTAgent'); + expect(testGroup).toBeDefined(); + expect(testGroup!.children.length).toBe(2); + expect(testGroup!.children[0].name).toBe('#1'); + expect(testGroup!.children[1].name).toBe('#2'); + }); + + it('should parse both AiEvaluationDefinition and AiTestingDefinition files together', async () => { + const evalXml = ` + + + 1 + + Eval utterance + + +`; + + const ngtXml = ` + + + 1 + + NGT utterance + + +`; + + const evalUri = { fsPath: '/test/EvalAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; + const ngtUri = { fsPath: '/test/NGTAgent.aiTestingDefinition-meta.xml' } as vscode.Uri; + + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([evalUri]) // aiEvaluationDefinition + .mockResolvedValueOnce([ngtUri]); // aiTestingDefinition + jest + .spyOn(vscode.workspace.fs, 'readFile') + .mockResolvedValueOnce(Buffer.from(evalXml)) + .mockResolvedValueOnce(Buffer.from(ngtXml)); + + const result = await parseAgentTestsFromProject(); + + expect(result.size).toBe(2); + expect(result.has('EvalAgent')).toBe(true); + expect(result.has('NGTAgent')).toBe(true); + }); + + it('should apply disambiguation suffixes when both types share the same API name', async () => { + const sharedXml = (root: string) => ` +<${root} xmlns="http://soap.sforce.com/2006/04/metadata"> + + 1 + + utterance + + +`; + + const evalUri = { fsPath: '/test/WillieTest.aiEvaluationDefinition-meta.xml' } as vscode.Uri; + const ngtUri = { fsPath: '/test/WillieTest.aiTestingDefinition-meta.xml' } as vscode.Uri; + + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([evalUri]) + .mockResolvedValueOnce([ngtUri]); + jest + .spyOn(vscode.workspace.fs, 'readFile') + .mockResolvedValueOnce(Buffer.from(sharedXml('AiEvaluationDefinition'))) + .mockResolvedValueOnce(Buffer.from(sharedXml('AiTestingDefinition'))); + + const result = await parseAgentTestsFromProject(); + + expect(result.size).toBe(2); + expect(result.has('WillieTest (testing-center)')).toBe(true); + expect(result.has('WillieTest (agentforce-studio)')).toBe(true); + // underlying API name is preserved + expect(result.get('WillieTest (testing-center)')!.testDefinitionName).toBe('WillieTest'); + expect(result.get('WillieTest (agentforce-studio)')!.testDefinitionName).toBe('WillieTest'); + // runner types are correct + expect(result.get('WillieTest (testing-center)')!.runnerType).toBe('testing-center'); + expect(result.get('WillieTest (agentforce-studio)')!.runnerType).toBe('agentforce-studio'); + // children parentName matches the suffixed label + expect(result.get('WillieTest (testing-center)')!.children[0].parentName).toBe('WillieTest (testing-center)'); + expect(result.get('WillieTest (agentforce-studio)')!.children[0].parentName).toBe('WillieTest (agentforce-studio)'); + }); + + it('should not apply suffixes when API names are unique across both types', async () => { + const evalXml = ` + + 1u +`; + const ngtXml = ` + + 1u +`; + + const evalUri = { fsPath: '/test/UniqueEval.aiEvaluationDefinition-meta.xml' } as vscode.Uri; + const ngtUri = { fsPath: '/test/UniqueNGT.aiTestingDefinition-meta.xml' } as vscode.Uri; + + jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValueOnce([evalUri]).mockResolvedValueOnce([ngtUri]); + jest + .spyOn(vscode.workspace.fs, 'readFile') + .mockResolvedValueOnce(Buffer.from(evalXml)) + .mockResolvedValueOnce(Buffer.from(ngtXml)); + + const result = await parseAgentTestsFromProject(); + + expect(result.has('UniqueEval')).toBe(true); + expect(result.has('UniqueNGT')).toBe(true); + expect(result.get('UniqueEval')!.testDefinitionName).toBe('UniqueEval'); + expect(result.get('UniqueNGT')!.testDefinitionName).toBe('UniqueNGT'); + }); }); From 3e5ad35dddc34b36e34502fb645e5adb59a91e06 Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Thu, 30 Apr 2026 15:53:44 -0600 Subject: [PATCH 2/7] refactor: remove NGT references, update test states --- src/types/MetadataDefinitions.ts | 6 ++- src/views/testOutlineProvider.ts | 27 +++++++----- src/views/testRunner.ts | 53 ++++++++++++----------- test/views/testOutlineProvider.test.ts | 59 ++++++++++++++------------ 4 files changed, 82 insertions(+), 63 deletions(-) diff --git a/src/types/MetadataDefinitions.ts b/src/types/MetadataDefinitions.ts index 28305045..7c0cb3e5 100644 --- a/src/types/MetadataDefinitions.ts +++ b/src/types/MetadataDefinitions.ts @@ -23,11 +23,13 @@ export type AgentTestCase = { ]; }; -export type NGTTestCaseMetadata = { +// until @salesforce/types has AiTestingDefinition +export type AgentforceStudioTestCaseMetadata = { number: string; inputs: { utterance: string }; }; +// until @salesforce/types has AiTestingDefinition export type AiTestingDefinition = { AiTestingDefinition: { description: string; @@ -35,6 +37,6 @@ export type AiTestingDefinition = { subjectType: 'AGENT'; subjectName: string; subjectVersion: string; - testCase: NGTTestCaseMetadata | NGTTestCaseMetadata[]; + testCase: AgentforceStudioTestCaseMetadata | AgentforceStudioTestCaseMetadata[]; }; }; diff --git a/src/views/testOutlineProvider.ts b/src/views/testOutlineProvider.ts index ff1b1900..b99c34fa 100644 --- a/src/views/testOutlineProvider.ts +++ b/src/views/testOutlineProvider.ts @@ -63,7 +63,7 @@ export const parseAgentTestsFromProject = async (): Promise(); - const ngtNodes = new Map(); + const agentforceStudioNodes = new Map(); await Promise.all([ ...aiEvalDefs.map(async definition => { @@ -85,31 +85,38 @@ export const parseAgentTestsFromProject = async (): Promise ngtNodes.has(k))); + const conflicts = new Set([...evalNodes.keys()].filter(k => agentforceStudioNodes.has(k))); for (const name of conflicts) { const evalNode = evalNodes.get(name)!; - const ngtNode = ngtNodes.get(name)!; + const afsNode = agentforceStudioNodes.get(name)!; const evalLabel = `${name} (testing-center)`; - const ngtLabel = `${name} (agentforce-studio)`; + const afsLabel = `${name} (agentforce-studio)`; evalNode.label = evalLabel; evalNode.name = evalLabel; evalNode.children.forEach(c => (c.parentName = evalLabel)); - ngtNode.label = ngtLabel; - ngtNode.name = ngtLabel; - ngtNode.children.forEach(c => (c.parentName = ngtLabel)); + afsNode.label = afsLabel; + afsNode.name = afsLabel; + afsNode.children.forEach(c => (c.parentName = afsLabel)); } // Merge into a single map keyed by the (possibly suffixed) label const aggregator = new Map(); - for (const node of [...evalNodes.values(), ...ngtNodes.values()]) { + for (const node of [...evalNodes.values(), ...agentforceStudioNodes.values()]) { aggregator.set(node.name, node); } diff --git a/src/views/testRunner.ts b/src/views/testRunner.ts index c9aab833..aafce640 100644 --- a/src/views/testRunner.ts +++ b/src/views/testRunner.ts @@ -18,11 +18,11 @@ import * as vscode from 'vscode'; import { AgentTestOutlineProvider } from './testOutlineProvider'; import { AgentTester, - AgentTesterNGT, + AgentforceStudioTester, createAgentTester, TestStatus, AgentTestResultsResponse, - AgentTestNGTResultsResponse, + AgentforceStudioTestResultsResponse, humanFriendlyName, metric } from '@salesforce/agents'; @@ -34,15 +34,15 @@ import { AgentTestNode } from '../types'; import { formatJson } from '../utils/jsonFormatter'; type AgentTestResults = AgentTestResultsResponse & { id: string }; -type NGTTestResults = AgentTestNGTResultsResponse & { id: string }; +type AgentforceStudioTestResults = AgentforceStudioTestResultsResponse & { id: string }; -type ParsedNGTScorer = +type ParsedAgentforceStudioScorer = | { kind: 'latency'; passing: boolean; latencyMs: number; reasoning: string } | { kind: 'quality'; passing: boolean; score: number; reasoning: string } | { kind: 'assertion'; passing: boolean; expectedValue: string; actualValue: string } | { kind: 'unknown'; passing: boolean; raw: string }; -function parseNGTScorer(scorerResponse: string): ParsedNGTScorer { +function parseAgentforceStudioScorer(scorerResponse: string): ParsedAgentforceStudioScorer { try { const p = JSON.parse(scorerResponse) as { status?: string; @@ -73,7 +73,7 @@ function parseNGTScorer(scorerResponse: string): ParsedNGTScorer { export class AgentTestRunner { private testGroupNameToResult = new Map(); - private ngtTestGroupNameToResult = new Map(); + private agentforceStudioTestGroupNameToResult = new Map(); constructor(private testOutline: AgentTestOutlineProvider) {} public displayTestDetails(test: TestNode) { @@ -81,20 +81,21 @@ export class AgentTestRunner { channelService.showChannelOutput(); channelService.clear(); - // Check NGT results first, then fall back to standard results - const ngtResult = - this.ngtTestGroupNameToResult.get(test.name) ?? this.ngtTestGroupNameToResult.get(test.parentName); - if (ngtResult) { + // Check Agentforce Studio results first, then fall back to standard results + const agentforceStudioResult = + this.agentforceStudioTestGroupNameToResult.get(test.name) ?? + this.agentforceStudioTestGroupNameToResult.get(test.parentName); + if (agentforceStudioResult) { if (test.parentName == '') { - channelService.appendLine(`Job Id: ${ngtResult.id}`); - this.printNGTTestSummary(ngtResult); + channelService.appendLine(`Job Id: ${agentforceStudioResult.id}`); + this.printAgentforceStudioTestSummary(agentforceStudioResult); return; } - const testInfo = structuredClone(ngtResult); + const testInfo = structuredClone(agentforceStudioResult); if (test instanceof AgentTestNode) { testInfo.testCases = testInfo.testCases.filter(f => `#${f.testNumber}` === test.name); } - this.displayNGTTestCases(testInfo); + this.displayAgentforceStudioTestCases(testInfo); return; } @@ -232,7 +233,7 @@ export class AgentTestRunner { ); if (type === 'agentforce-studio') { - await this.runAgentforceStudioTest(test, runner as AgentTesterNGT, channelService); + await this.runAgentforceStudioTest(test, runner as AgentforceStudioTester, channelService); } else { await this.runTestingCenterTest(test, runner as AgentTester, channelService); } @@ -276,25 +277,26 @@ export class AgentTestRunner { private async runAgentforceStudioTest( test: AgentTestGroupNode, - tester: AgentTesterNGT, + tester: AgentforceStudioTester, channelService: ReturnType ): Promise { const response = await tester.start(test.testDefinitionName); channelService.appendLine(`Job Id: ${response.runId}`); this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); - const result: NGTTestResults = { + const result: AgentforceStudioTestResults = { ...(await tester.poll(response.runId, { timeout: Duration.minutes(100) })), id: response.runId }; - this.ngtTestGroupNameToResult.set(test.name, result); + this.agentforceStudioTestGroupNameToResult.set(test.name, result); this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); let hasFailure = false; result.testCases.map(tc => { const tcFailed = - tc.testScorerResults.length === 0 || tc.testScorerResults.some(s => !parseNGTScorer(s.scorerResponse).passing); + tc.testScorerResults.length === 0 || + tc.testScorerResults.some(s => !parseAgentforceStudioScorer(s.scorerResponse).passing); if (tcFailed) hasFailure = true; this.testOutline .getTestGroup(test.name) @@ -303,10 +305,10 @@ export class AgentTestRunner { ?.updateOutcome(tcFailed ? 'ERROR' : 'COMPLETED'); }); this.testOutline.getTestGroup(test.name)?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); - this.printNGTTestSummary(result); + this.printAgentforceStudioTestSummary(result); } - private displayNGTTestCases(testInfo: NGTTestResults): void { + private displayAgentforceStudioTestCases(testInfo: AgentforceStudioTestResults): void { const channelService = CoreExtensionService.getTestChannelService(); testInfo.testCases.map(tc => { @@ -315,7 +317,7 @@ export class AgentTestRunner { channelService.appendLine('════════════════════════════════════════════════════════════════════════'); channelService.appendLine(''); tc.testScorerResults.map(scorer => { - const parsed = parseNGTScorer(scorer.scorerResponse); + const parsed = parseAgentforceStudioScorer(scorer.scorerResponse); const icon = parsed.passing ? '✅' : '❌'; const verdict = parsed.passing ? 'PASS' : 'FAILURE'; channelService.appendLine(`❯ ${scorer.scorerName.toUpperCase()}: ${verdict} ${icon}`); @@ -339,7 +341,7 @@ export class AgentTestRunner { channelService.appendLine(''); }); - const passing = tc.testScorerResults.filter(s => parseNGTScorer(s.scorerResponse).passing).length; + const passing = tc.testScorerResults.filter(s => parseAgentforceStudioScorer(s.scorerResponse).passing).length; const failing = tc.testScorerResults.length - passing; channelService.appendLine('────────────────────────────────────────────────────────────────────────'); channelService.appendLine( @@ -364,10 +366,11 @@ export class AgentTestRunner { channelService.appendLine(`Select a test case in the Test View panel for more information`); } - private printNGTTestSummary(result: NGTTestResults): void { + private printAgentforceStudioTestSummary(result: AgentforceStudioTestResults): void { const channelService = CoreExtensionService.getTestChannelService(); const tcPassing = (tc: (typeof result.testCases)[number]): boolean => - tc.testScorerResults.length > 0 && tc.testScorerResults.every(s => parseNGTScorer(s.scorerResponse).passing); + tc.testScorerResults.length > 0 && + tc.testScorerResults.every(s => parseAgentforceStudioScorer(s.scorerResponse).passing); channelService.appendLine(result.status); channelService.appendLine(''); channelService.appendLine('Test Results'); diff --git a/test/views/testOutlineProvider.test.ts b/test/views/testOutlineProvider.test.ts index 6528fda6..5f8028ba 100644 --- a/test/views/testOutlineProvider.test.ts +++ b/test/views/testOutlineProvider.test.ts @@ -354,13 +354,13 @@ describe('parseAgentTestsFromProject', () => { await expect(parseAgentTestsFromProject()).rejects.toThrow(); }); - it('should parse single NGT test case from AiTestingDefinition XML file', async () => { + it('should parse single Agentforce Studio test case from AiTestingDefinition XML file', async () => { const mockXmlContent = ` 1 - NGT utterance 1 + Agentforce Studio utterance 1 GeneralFAQ @@ -370,7 +370,7 @@ describe('parseAgentTestsFromProject', () => { `; const mockUri = { - fsPath: '/test/MyNGTAgent.aiTestingDefinition-meta.xml' + fsPath: '/test/MyAgentforceStudioAgent.aiTestingDefinition-meta.xml' } as vscode.Uri; jest @@ -382,35 +382,35 @@ describe('parseAgentTestsFromProject', () => { const result = await parseAgentTestsFromProject(); expect(result.size).toBe(1); - expect(result.has('MyNGTAgent')).toBe(true); + expect(result.has('MyAgentforceStudioAgent')).toBe(true); - const testGroup = result.get('MyNGTAgent'); + const testGroup = result.get('MyAgentforceStudioAgent'); expect(testGroup).toBeDefined(); - expect(testGroup!.name).toBe('MyNGTAgent'); + expect(testGroup!.name).toBe('MyAgentforceStudioAgent'); expect(testGroup!.children.length).toBe(1); expect(testGroup!.children[0].name).toBe('#1'); - expect((testGroup!.children[0] as any).description).toBe('NGT utterance 1'); + expect((testGroup!.children[0] as any).description).toBe('Agentforce Studio utterance 1'); }); - it('should parse multiple NGT test cases from AiTestingDefinition XML file', async () => { + it('should parse multiple Agentforce Studio test cases from AiTestingDefinition XML file', async () => { const mockXmlContent = ` 2 - NGT utterance 2 + Agentforce Studio utterance 2 1 - NGT utterance 1 + Agentforce Studio utterance 1 `; const mockUri = { - fsPath: '/test/MultiNGTAgent.aiTestingDefinition-meta.xml' + fsPath: '/test/MultiAgentforceStudioAgent.aiTestingDefinition-meta.xml' } as vscode.Uri; jest @@ -422,7 +422,7 @@ describe('parseAgentTestsFromProject', () => { const result = await parseAgentTestsFromProject(); expect(result.size).toBe(1); - const testGroup = result.get('MultiNGTAgent'); + const testGroup = result.get('MultiAgentforceStudioAgent'); expect(testGroup).toBeDefined(); expect(testGroup!.children.length).toBe(2); expect(testGroup!.children[0].name).toBe('#1'); @@ -440,33 +440,35 @@ describe('parseAgentTestsFromProject', () => { `; - const ngtXml = ` + const agentforceStudioXml = ` 1 - NGT utterance + Agentforce Studio utterance `; const evalUri = { fsPath: '/test/EvalAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - const ngtUri = { fsPath: '/test/NGTAgent.aiTestingDefinition-meta.xml' } as vscode.Uri; + const agentforceStudioUri = { + fsPath: '/test/AgentforceStudioAgent.aiTestingDefinition-meta.xml' + } as vscode.Uri; jest .spyOn(vscode.workspace, 'findFiles') .mockResolvedValueOnce([evalUri]) // aiEvaluationDefinition - .mockResolvedValueOnce([ngtUri]); // aiTestingDefinition + .mockResolvedValueOnce([agentforceStudioUri]); // aiTestingDefinition jest .spyOn(vscode.workspace.fs, 'readFile') .mockResolvedValueOnce(Buffer.from(evalXml)) - .mockResolvedValueOnce(Buffer.from(ngtXml)); + .mockResolvedValueOnce(Buffer.from(agentforceStudioXml)); const result = await parseAgentTestsFromProject(); expect(result.size).toBe(2); expect(result.has('EvalAgent')).toBe(true); - expect(result.has('NGTAgent')).toBe(true); + expect(result.has('AgentforceStudioAgent')).toBe(true); }); it('should apply disambiguation suffixes when both types share the same API name', async () => { @@ -481,12 +483,12 @@ describe('parseAgentTestsFromProject', () => { `; const evalUri = { fsPath: '/test/WillieTest.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - const ngtUri = { fsPath: '/test/WillieTest.aiTestingDefinition-meta.xml' } as vscode.Uri; + const agentforceStudioUri = { fsPath: '/test/WillieTest.aiTestingDefinition-meta.xml' } as vscode.Uri; jest .spyOn(vscode.workspace, 'findFiles') .mockResolvedValueOnce([evalUri]) - .mockResolvedValueOnce([ngtUri]); + .mockResolvedValueOnce([agentforceStudioUri]); jest .spyOn(vscode.workspace.fs, 'readFile') .mockResolvedValueOnce(Buffer.from(sharedXml('AiEvaluationDefinition'))) @@ -513,25 +515,30 @@ describe('parseAgentTestsFromProject', () => { 1u `; - const ngtXml = ` + const agentforceStudioXml = ` 1u `; const evalUri = { fsPath: '/test/UniqueEval.aiEvaluationDefinition-meta.xml' } as vscode.Uri; - const ngtUri = { fsPath: '/test/UniqueNGT.aiTestingDefinition-meta.xml' } as vscode.Uri; + const agentforceStudioUri = { + fsPath: '/test/UniqueAgentforceStudio.aiTestingDefinition-meta.xml' + } as vscode.Uri; - jest.spyOn(vscode.workspace, 'findFiles').mockResolvedValueOnce([evalUri]).mockResolvedValueOnce([ngtUri]); + jest + .spyOn(vscode.workspace, 'findFiles') + .mockResolvedValueOnce([evalUri]) + .mockResolvedValueOnce([agentforceStudioUri]); jest .spyOn(vscode.workspace.fs, 'readFile') .mockResolvedValueOnce(Buffer.from(evalXml)) - .mockResolvedValueOnce(Buffer.from(ngtXml)); + .mockResolvedValueOnce(Buffer.from(agentforceStudioXml)); const result = await parseAgentTestsFromProject(); expect(result.has('UniqueEval')).toBe(true); - expect(result.has('UniqueNGT')).toBe(true); + expect(result.has('UniqueAgentforceStudio')).toBe(true); expect(result.get('UniqueEval')!.testDefinitionName).toBe('UniqueEval'); - expect(result.get('UniqueNGT')!.testDefinitionName).toBe('UniqueNGT'); + expect(result.get('UniqueAgentforceStudio')!.testDefinitionName).toBe('UniqueAgentforceStudio'); }); }); From 67541492b28cf3347cdcc30a37ab5cd231f2bfac Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Fri, 8 May 2026 08:17:34 -0600 Subject: [PATCH 3/7] chore: bump agents --- package-lock.json | 90 ++++++++++++++++++++++++++++------------------- package.json | 2 +- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index e5fb0386..bcd8de3d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { - "@salesforce/agents": "^1.1.2", + "@salesforce/agents": "^1.6.0", "@salesforce/core": "^8.28.3", "@salesforce/kit": "^3.2.6", "@salesforce/types": "^1.7.1", @@ -2255,27 +2255,27 @@ } }, "node_modules/@salesforce/agents": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@salesforce/agents/-/agents-1.1.2.tgz", - "integrity": "sha512-p7isCk2WoV0t1skRoTjYeead+GOoF2I7VPo+K6YYt+h6S+v8vJTdBc8NNhnmUJcz386FIK5jc0g7bSz8lCQ0tQ==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@salesforce/agents/-/agents-1.6.0.tgz", + "integrity": "sha512-3ziyrozhmO0SBu6anSZ2BaOWKu9QNPYxWR0jLIfqwhC4Fydle//eCjob1+F06aGbBGcAzaje4iM0pvMAJbWv6w==", "license": "Apache-2.0", "dependencies": { - "@salesforce/core": "^8.28.3", + "@salesforce/core": "^8.29.0", "@salesforce/kit": "^3.2.6", - "@salesforce/source-deploy-retrieve": "^12.32.7", + "@salesforce/source-deploy-retrieve": "^12.35.1", "@salesforce/types": "^1.7.1", - "fast-xml-parser": "^5.6.0", + "fast-xml-parser": "^5.7.2", "nock": "^13.5.6", - "yaml": "^2.8.3" + "yaml": "^2.8.4" }, "engines": { "node": ">=18.0.0" } }, "node_modules/@salesforce/core": { - "version": "8.28.3", - "resolved": "https://registry.npmjs.org/@salesforce/core/-/core-8.28.3.tgz", - "integrity": "sha512-DDAeHVwDO8wUlqEGwfp8Vuu7Vp7K+hpubKu6baWkHAXiO1u7ZbQkvwCbpPz9JiYEXVUBvP11JtBQ7zOUIPShlQ==", + "version": "8.29.0", + "resolved": "https://registry.npmjs.org/@salesforce/core/-/core-8.29.0.tgz", + "integrity": "sha512-q6xDNLPbbZW1n4X4YK1iM8jZvwvJRiwbJxdeF5iHuETxmMka16FoCVi+WziK/Rh5EP0yW08FYyiynwPlgz5RBw==", "license": "BSD-3-Clause", "dependencies": { "@jsforce/jsforce-node": "^3.10.13", @@ -2312,17 +2312,17 @@ } }, "node_modules/@salesforce/source-deploy-retrieve": { - "version": "12.32.9", - "resolved": "https://registry.npmjs.org/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.32.9.tgz", - "integrity": "sha512-DRIiyt9dP+QzBs+Lonw7UeB+LvtHmEyrVbYibXvRtKkTAj/zuBGjvKHm06/ql1TymXFotwE54j6xLi1rxS5Zfw==", + "version": "12.35.4", + "resolved": "https://registry.npmjs.org/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.35.4.tgz", + "integrity": "sha512-Wuz+qD11ek6DfHNk2gH7shfxjjk98nSGRh/0kY5a4dJz2lslDJIHFIiMoocT7O1Wl0i6qAS85NOek9Z3xWteGw==", "license": "Apache-2.0", "dependencies": { - "@salesforce/core": "^8.27.1", + "@salesforce/core": "^8.29.0", "@salesforce/kit": "^3.2.4", "@salesforce/ts-types": "^2.0.12", "@salesforce/types": "^1.6.0", "fast-levenshtein": "^3.0.0", - "fast-xml-parser": "^5.5.11", + "fast-xml-parser": "^5.7.2", "got": "^11.8.6", "graceful-fs": "^4.2.11", "ignore": "^5.3.2", @@ -4071,9 +4071,9 @@ } }, "node_modules/basic-ftp": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.0.tgz", - "integrity": "sha512-5K9eNNn7ywHPsYnFwjKgYH8Hf8B5emh7JKcPaVjjrMJFQQwGpwowEnZNEtHs7DfR7hCZsmaK3VA4HUK0YarT+w==", + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.1.tgz", + "integrity": "sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -5633,9 +5633,9 @@ "license": "BSD-3-Clause" }, "node_modules/fast-xml-builder": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.5.tgz", - "integrity": "sha512-4TJn/8FKLeslLAH3dnohXqE3QSoxkhvaMzepOIZytwJXZO69Bfz0HBdDHzOTOon6G59Zrk6VQ2bEiv1t61rfkA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", + "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", "funding": [ { "type": "github", @@ -5644,13 +5644,14 @@ ], "license": "MIT", "dependencies": { - "path-expression-matcher": "^1.1.3" + "path-expression-matcher": "^1.5.0", + "xml-naming": "^0.1.0" } }, "node_modules/fast-xml-parser": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.1.tgz", - "integrity": "sha512-8Cc3f8GUGUULg34pBch/KGyPLglS+OFs05deyOlY7fL2MTagYPKrVQNmR1fLF/yJ9PH5ZSTd3YDF6pnmeZU+zA==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.3.tgz", + "integrity": "sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==", "funding": [ { "type": "github", @@ -5660,7 +5661,7 @@ "license": "MIT", "dependencies": { "@nodable/entities": "^2.1.0", - "fast-xml-builder": "^1.1.5", + "fast-xml-builder": "^1.1.7", "path-expression-matcher": "^1.5.0", "strnum": "^2.2.3" }, @@ -6532,9 +6533,9 @@ "optional": true }, "node_modules/ip-address": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", - "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", "license": "MIT", "engines": { "node": ">= 12" @@ -10797,12 +10798,12 @@ } }, "node_modules/socks": { - "version": "2.8.7", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", - "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "version": "2.8.8", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.8.tgz", + "integrity": "sha512-NlGELfPrgX2f1TAAcz0WawlLn+0r3FyhhCRpFFK2CemXenPYvzMWWZINv3eDNo9ucdwme7oCHRY0Jnbs4aIkog==", "license": "MIT", "dependencies": { - "ip-address": "^10.0.1", + "ip-address": "^10.1.1", "smart-buffer": "^4.2.0" }, "engines": { @@ -12137,6 +12138,21 @@ "node": ">=18" } }, + "node_modules/xml-naming": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", + "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", @@ -12190,9 +12206,9 @@ "license": "ISC" }, "node_modules/yaml": { - "version": "2.8.3", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", - "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.4.tgz", + "integrity": "sha512-ml/JPOj9fOQK8RNnWojA67GbZ0ApXAUlN2UQclwv2eVgTgn7O9gg9o7paZWKMp4g0H3nTLtS9LVzhkpOFIKzog==", "license": "ISC", "bin": { "yaml": "bin.mjs" diff --git a/package.json b/package.json index 5302cdc0..83a82e89 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "main": "lib/src/extension.js", "dependencies": { - "@salesforce/agents": "^1.1.2", + "@salesforce/agents": "^1.6.0", "@salesforce/core": "^8.28.3", "@salesforce/kit": "^3.2.6", "@salesforce/types": "^1.7.1", From c572da410bd96a851f34e9e6fc5c5185d04c81fc Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Mon, 11 May 2026 16:22:23 -0600 Subject: [PATCH 4/7] fix: address PR review comments - Guard findIndex -1 result to prevent invalid Position(-1, 8) - Replace TestStatus | string with TestStatus | AgentforceStudioTestStatusResponse['status'] in polling event type - Use !test.parentName instead of == '' for consistency - Add comment explaining why empty testScorerResults counts as failure --- src/views/testOutlineProvider.ts | 2 +- src/views/testRunner.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/views/testOutlineProvider.ts b/src/views/testOutlineProvider.ts index b99c34fa..568badd2 100644 --- a/src/views/testOutlineProvider.ts +++ b/src/views/testOutlineProvider.ts @@ -44,7 +44,7 @@ const buildTestGroupNode = ( const line = splitContent.findIndex(l => l.includes(`${test.number} { + // A test case with no scorer results has not been evaluated and counts as a failure const tcFailed = tc.testScorerResults.length === 0 || tc.testScorerResults.some(s => !parseAgentforceStudioScorer(s.scorerResponse).passing); From 0bc9bf88bcfd1405010d2ec6c7a97be997efc51b Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Mon, 11 May 2026 16:25:08 -0600 Subject: [PATCH 5/7] refactor: simplify and fix edge cases from branch review - Handle CREATED and FAILED AgentforceStudio statuses in polling event listener to prevent progress notification hanging indefinitely - Fix hasFailure accumulation bug in runTestingCenterTest (was overwriting on each iteration instead of accumulating) - Guard against missing testCase element in XML parsing - Replace map() with forEach() for all side-effect-only loops - Eliminate double-filter in printTestSummary and printAgentforceStudioTestSummary (use total - passing) - Eliminate double parse of scorer responses in displayAgentforceStudioTestCases (accumulate during render loop) --- src/views/testOutlineProvider.ts | 14 +++++------ src/views/testRunner.ts | 40 ++++++++++++++++---------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/views/testOutlineProvider.ts b/src/views/testOutlineProvider.ts index 568badd2..1e473c10 100644 --- a/src/views/testOutlineProvider.ts +++ b/src/views/testOutlineProvider.ts @@ -40,7 +40,7 @@ const buildTestGroupNode = ( testDefinitionNode.testDefinitionName = definitionApiName; testDefinitionNode.runnerType = runnerType; const splitContent = fileContent.split(EOL); - testCases.map(test => { + testCases.forEach(test => { const line = splitContent.findIndex(l => l.includes(`${test.number} `#${f.testNumber}` === test.name); } - testInfo.testCases.map(tc => { + testInfo.testCases.forEach(tc => { channelService.appendLine('════════════════════════════════════════════════════════════════════════'); channelService.appendLine(`CASE #${tc.testNumber} - ${testInfo.subjectName}`); channelService.appendLine('════════════════════════════════════════════════════════════════════════'); @@ -135,7 +135,7 @@ export class AgentTestRunner { // this is the output for topics/action/output validation (actual v expected) // filter out other metrics from it .filter(f => !metric.includes(f.name as (typeof metric)[number])) - .map(tr => { + .forEach(tr => { channelService.appendLine( `❯ ${humanFriendlyName(tr.name).toUpperCase()}: ${tr.result} ${tr.result === 'PASS' ? '✅' : '❌'}` ); @@ -164,7 +164,7 @@ export class AgentTestRunner { if (metricResults.length > 0) { channelService.appendLine(`❯ METRICS (Value/Threshold)`); channelService.appendLine('────────────────────────────────────────────────────────────────────────'); - metricResults.map(tr => { + metricResults.forEach(tr => { if (tr.name === 'output_latency_milliseconds') { channelService.appendLine( `⏱️ : ${humanFriendlyName(tr.name).toUpperCase()} (${tr.score}ms) ${tr.metricExplainability ? `: ${tr.metricExplainability}` : ''}` @@ -217,10 +217,10 @@ export class AgentTestRunner { passingTestCases: number; }) => { const statusUpper = data.status.toUpperCase(); - if (statusUpper === 'NEW' || statusUpper === 'IN_PROGRESS') { + if (statusUpper === 'NEW' || statusUpper === 'IN_PROGRESS' || statusUpper === 'CREATED') { // every time IN_PROGRESS is returned, 10 is added, is possible to 100% progress bar and tests not be done progress.report({ increment: 10, message: `Status: In Progress` }); - } else if (statusUpper === 'ERROR' || statusUpper === 'TERMINATED') { + } else if (statusUpper === 'ERROR' || statusUpper === 'TERMINATED' || statusUpper === 'FAILED') { progress.report({ increment: 100, message: `Status: ${data.status}` }); setTimeout(() => reject(), 1500); } else if (statusUpper === 'COMPLETED' || statusUpper === 'SUCCESS') { @@ -264,13 +264,14 @@ export class AgentTestRunner { this.testGroupNameToResult.set(test.name, result); this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); let hasFailure = false; - result.testCases.map(tc => { - hasFailure = tc.testResults.some(tr => tr.result === 'FAILURE'); + result.testCases.forEach(tc => { + const tcFailed = tc.testResults.some(tr => tr.result === 'FAILURE'); + if (tcFailed) hasFailure = true; this.testOutline .getTestGroup(test.name) ?.getChildren() .find(child => child.name === `#${tc.testNumber}`) - ?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); + ?.updateOutcome(tcFailed ? 'ERROR' : 'COMPLETED'); }); this.testOutline.getTestGroup(test.name)?.updateOutcome(hasFailure ? 'ERROR' : 'COMPLETED'); this.printTestSummary(result); @@ -294,7 +295,7 @@ export class AgentTestRunner { this.testOutline.getTestGroup(test.name)?.updateOutcome('IN_PROGRESS', true); let hasFailure = false; - result.testCases.map(tc => { + result.testCases.forEach(tc => { // A test case with no scorer results has not been evaluated and counts as a failure const tcFailed = tc.testScorerResults.length === 0 || @@ -313,13 +314,15 @@ export class AgentTestRunner { private displayAgentforceStudioTestCases(testInfo: AgentforceStudioTestResults): void { const channelService = CoreExtensionService.getTestChannelService(); - testInfo.testCases.map(tc => { + testInfo.testCases.forEach(tc => { channelService.appendLine('════════════════════════════════════════════════════════════════════════'); channelService.appendLine(`CASE #${tc.testNumber}`); channelService.appendLine('════════════════════════════════════════════════════════════════════════'); channelService.appendLine(''); - tc.testScorerResults.map(scorer => { + let passing = 0; + tc.testScorerResults.forEach(scorer => { const parsed = parseAgentforceStudioScorer(scorer.scorerResponse); + if (parsed.passing) passing++; const icon = parsed.passing ? '✅' : '❌'; const verdict = parsed.passing ? 'PASS' : 'FAILURE'; channelService.appendLine(`❯ ${scorer.scorerName.toUpperCase()}: ${verdict} ${icon}`); @@ -343,7 +346,6 @@ export class AgentTestRunner { channelService.appendLine(''); }); - const passing = tc.testScorerResults.filter(s => parseAgentforceStudioScorer(s.scorerResponse).passing).length; const failing = tc.testScorerResults.length - passing; channelService.appendLine('────────────────────────────────────────────────────────────────────────'); channelService.appendLine( @@ -358,12 +360,9 @@ export class AgentTestRunner { channelService.appendLine(''); channelService.appendLine('Test Results'); const total = result.testCases.length; - channelService.appendLine( - `Passing: ${result.testCases.filter(tc => tc.testResults.every(tr => tr.result === 'PASS')).length}/${total}` - ); - channelService.appendLine( - `Failing: ${result.testCases.filter(tc => tc.testResults.some(tr => tr.result === 'FAILURE')).length}/${total}` - ); + const passing = result.testCases.filter(tc => tc.testResults.every(tr => tr.result === 'PASS')).length; + channelService.appendLine(`Passing: ${passing}/${total}`); + channelService.appendLine(`Failing: ${total - passing}/${total}`); channelService.appendLine(''); channelService.appendLine(`Select a test case in the Test View panel for more information`); } @@ -377,8 +376,9 @@ export class AgentTestRunner { channelService.appendLine(''); channelService.appendLine('Test Results'); const total = result.testCases.length; - channelService.appendLine(`Passing: ${result.testCases.filter(tcPassing).length}/${total}`); - channelService.appendLine(`Failing: ${result.testCases.filter(tc => !tcPassing(tc)).length}/${total}`); + const passing = result.testCases.filter(tcPassing).length; + channelService.appendLine(`Passing: ${passing}/${total}`); + channelService.appendLine(`Failing: ${total - passing}/${total}`); channelService.appendLine(''); channelService.appendLine(`Select a test case in the Test View panel for more information`); } From ccd91f538d699c72662d240c2cccf8c625ee81bf Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Mon, 11 May 2026 16:26:41 -0600 Subject: [PATCH 6/7] test: update test to match graceful skip for missing testCase element --- test/views/testOutlineProvider.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/views/testOutlineProvider.test.ts b/test/views/testOutlineProvider.test.ts index 5f8028ba..35724e78 100644 --- a/test/views/testOutlineProvider.test.ts +++ b/test/views/testOutlineProvider.test.ts @@ -336,7 +336,7 @@ describe('parseAgentTestsFromProject', () => { await expect(parseAgentTestsFromProject()).rejects.toThrow('read failure'); }); - it('should throw when XML is missing test cases', async () => { + it('should skip definition files with no test cases', async () => { const mockUri = { fsPath: '/test/EmptyAgent.aiEvaluationDefinition-meta.xml' } as vscode.Uri; @@ -351,7 +351,8 @@ describe('parseAgentTestsFromProject', () => { .mockResolvedValueOnce([]); // aiTestingDefinition jest.spyOn(vscode.workspace.fs, 'readFile').mockResolvedValue(Buffer.from(mockXmlContent)); - await expect(parseAgentTestsFromProject()).rejects.toThrow(); + const result = await parseAgentTestsFromProject(); + expect(result.size).toBe(0); }); it('should parse single Agentforce Studio test case from AiTestingDefinition XML file', async () => { From e2eeb103213e62ab47c668518225221383f777c5 Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Tue, 12 May 2026 08:23:35 -0600 Subject: [PATCH 7/7] chore: review again --- src/views/testOutlineProvider.ts | 2 +- src/views/testRunner.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/views/testOutlineProvider.ts b/src/views/testOutlineProvider.ts index 1e473c10..9997ea1e 100644 --- a/src/views/testOutlineProvider.ts +++ b/src/views/testOutlineProvider.ts @@ -41,7 +41,7 @@ const buildTestGroupNode = ( testDefinitionNode.runnerType = runnerType; const splitContent = fileContent.split(EOL); testCases.forEach(test => { - const line = splitContent.findIndex(l => l.includes(`${test.number} l.includes(`${test.number}`)); const testcaseNode = new AgentTestNode( `#${test.number}`, new vscode.Location(definitionUri, new vscode.Position(line < 0 ? 0 : line, 8)) diff --git a/src/views/testRunner.ts b/src/views/testRunner.ts index f2ff2e40..d479af43 100644 --- a/src/views/testRunner.ts +++ b/src/views/testRunner.ts @@ -217,15 +217,17 @@ export class AgentTestRunner { passingTestCases: number; }) => { const statusUpper = data.status.toUpperCase(); - if (statusUpper === 'NEW' || statusUpper === 'IN_PROGRESS' || statusUpper === 'CREATED') { + if (['NEW', 'IN_PROGRESS', 'CREATED'].includes(statusUpper)) { // every time IN_PROGRESS is returned, 10 is added, is possible to 100% progress bar and tests not be done progress.report({ increment: 10, message: `Status: In Progress` }); - } else if (statusUpper === 'ERROR' || statusUpper === 'TERMINATED' || statusUpper === 'FAILED') { + } else if (['ERROR', 'TERMINATED', 'FAILED'].includes(statusUpper)) { progress.report({ increment: 100, message: `Status: ${data.status}` }); setTimeout(() => reject(), 1500); - } else if (statusUpper === 'COMPLETED' || statusUpper === 'SUCCESS') { + } else if (['COMPLETED', 'SUCCESS'].includes(statusUpper)) { progress.report({ increment: 100, message: `Status: ${data.status}` }); setTimeout(() => resolve({}), 1500); + } else { + channelService.appendLine(`Warning: Unexpected test status: ${data.status}`); } } );