diff --git a/.eslintrc b/.eslintrc index fc74d48..09780ca 100644 --- a/.eslintrc +++ b/.eslintrc @@ -102,6 +102,7 @@ "Promise": true, "Proxy": true, "Set": true, + "Map": true, "Reflect": true, "element": "readonly", "by": "readonly", diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 38ade3e..ecc9e65 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -9,6 +9,7 @@ const path = require('path'); const AccessibilityAutomation = require('../src/accessibilityAutomation'); const eventHelper = require('../src/utils/eventHelper'); const OrchestrationUtils = require('../src/testorchestration/orchestrationUtils'); +const TestMap = require('../src/utils/testMap'); const localTunnel = new LocalTunnel(); const testObservability = new TestObservability(); const accessibilityAutomation = new AccessibilityAutomation(); @@ -19,6 +20,8 @@ const _tests = {}; const _testCasesData = {}; let currentTestUUID = ''; let workerList = {}; +let testRunner = ''; +let testEventPromises = []; eventHelper.eventEmitter.on(EVENTS.LOG_INIT, (loggingData) => { const testCaseStartedId = loggingData.message.replace('TEST-OBSERVABILITY-PID-TESTCASE-MAPPING-', '').slice(1, -1); @@ -65,8 +68,8 @@ module.exports = { promises.push(testObservability.processTestReportFile(JSON.parse(JSON.stringify(modulesWithEnv[testSetting][testFile])))); } } - await Promise.all(promises); + done(); } catch (error) { CrashReporter.uploadCrashReport(error.message, error.stack); @@ -92,7 +95,7 @@ module.exports = { const gherkinDocument = reportData?.gherkinDocument.find((document) => document.uri === pickleData.uri); const featureData = gherkinDocument.feature; const uniqueId = uuidv4(); - process.env.TEST_OPS_TEST_UUID = uniqueId; + process.env.TEST_RUN_UUID = uniqueId; Object.values(workerList).forEach((worker) => { worker.process.on('message', async (data) => { @@ -253,15 +256,37 @@ module.exports = { eventBroadcaster.on('ScreenshotCreated', async (args) => { if (!helper.isTestObservabilitySession()) {return} - handleScreenshotUpload({args: args, uuid: process.env.TEST_OPS_TEST_UUID}); + handleScreenshotUpload({args: args, uuid: process.env.TEST_RUN_UUID}); }); eventBroadcaster.on('TestRunStarted', async (test) => { + process.env.VALID_ALLY_PLATFORM = accessibilityAutomation.validateA11yCaps(browser); await accessibilityAutomation.beforeEachExecution(test); + if (testRunner !== 'cucumber'){ + const uuid = TestMap.storeTestDetails(test); + process.env.TEST_RUN_UUID = uuid; + testEventPromises.push(testObservability.sendTestRunEvent('TestRunStarted', test, uuid)); + } }); eventBroadcaster.on('TestRunFinished', async (test) => { - await accessibilityAutomation.afterEachExecution(test); + const uuid = process.env.TEST_RUN_UUID || TestMap.getUUID(test); + if (TestMap.hasTestFinished(uuid)) { + Logger.debug(`Test with UUID ${uuid} already marked as finished, skipping duplicate TestRunFinished event`); + + return; + } + try { + await accessibilityAutomation.afterEachExecution(test, uuid); + if (testRunner !== 'cucumber'){ + testEventPromises.push(testObservability.sendTestRunEvent('TestRunFinished', test, uuid)); + TestMap.markTestFinished(uuid); + } + + } catch (error) { + Logger.error(`Error in TestRunFinished event: ${error.message}`); + TestMap.markTestFinished(uuid); + } }); }, @@ -272,10 +297,12 @@ module.exports = { }, async before(settings, testEnvSettings) { + testRunner = settings.test_runner.type; + const pluginSettings = settings['@nightwatch/browserstack']; if (!settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options'] = {}; } - + process.env.BROWSERSTACK_APP_AUTOMATE = helper.checkTestEnvironmentForAppAutomate(testEnvSettings); // Plugin identifier settings.desiredCapabilities['bstack:options']['browserstackSDK'] = `nightwatch-plugin/${helper.getAgentVersion()}`; @@ -295,6 +322,7 @@ module.exports = { try { testObservability.configure(settings); + accessibilityAutomation.configure(settings); if (helper.isTestObservabilitySession()) { if (settings.reporter_options) { if (settings.reporter_options['save_command_result_value'] !== true){ @@ -312,7 +340,7 @@ module.exports = { settings.test_runner.options['require'] = path.resolve(__dirname, 'observabilityLogPatcherHook.js'); } settings.globals['customReporterCallbackTimeout'] = CUSTOM_REPORTER_CALLBACK_TIMEOUT; - if (testObservability._user && testObservability._key) { + if (helper.isTestHubBuild(pluginSettings, true)) { await testObservability.launchTestSession(); } if (process.env.BROWSERSTACK_RERUN === 'true' && process.env.BROWSERSTACK_RERUN_TESTS && process.env.BROWSERSTACK_RERUN_TESTS!=='null') { @@ -323,7 +351,17 @@ module.exports = { } catch (error) { Logger.error(`Could not configure or launch test reporting and analytics - ${error}`); } - + + try { + // In parallel mode, env-specific settings are passed to beforeChildProcess hook instead of before hook, + if (helper.isAccessibilitySession() && !settings.parallel_mode) { + accessibilityAutomation.setAccessibilityCapabilities(settings); + accessibilityAutomation.commandWrapper(); + helper.patchBrowserTerminateCommand(); + } + } catch (err){ + Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); + } // Initialize and configure test orchestration try { if (helper.isTestObservabilitySession()) { @@ -408,22 +446,6 @@ module.exports = { Logger.error(`Could not configure test orchestration - ${error}`); } - try { - accessibilityAutomation.configure(settings); - if (helper.isAccessibilitySession()) { - if (accessibilityAutomation._user && accessibilityAutomation._key) { - const [jwtToken, testRunId] = await accessibilityAutomation.createAccessibilityTestRun(); - process.env.BS_A11Y_JWT = jwtToken; - process.env.BS_A11Y_TEST_RUN_ID = testRunId; - if (helper.isAccessibilitySession()) { - accessibilityAutomation.setAccessibilityCapabilities(settings); - } - } - } - } catch (error) { - Logger.error(`Could not configure or launch accessibility automation - ${error}`); - } - addProductMapAndbuildUuidCapability(settings); }, @@ -443,46 +465,40 @@ module.exports = { } catch (error) { Logger.error(`Error collecting build data for test orchestration: ${error}`); } - - if (helper.isTestObservabilitySession()) { + + if (helper.isTestHubBuild()) { process.env.NIGHTWATCH_RERUN_FAILED = nightwatchRerun; process.env.NIGHTWATCH_RERUN_REPORT_FILE = nightwatchRerunFile; if (process.env.BROWSERSTACK_RERUN === 'true' && process.env.BROWSERSTACK_RERUN_TESTS) { await helper.deleteRerunFile(); } try { + if (testEventPromises.length > 0) { + await Promise.all(testEventPromises); + testEventPromises.length = 0; // Clear the array + } await testObservability.stopBuildUpstream(); - if (process.env.BS_TESTOPS_BUILD_HASHED_ID) { - Logger.info(`\nVisit https://automation.browserstack.com/builds/${process.env.BS_TESTOPS_BUILD_HASHED_ID} to view build report, insights, and many more debugging information all at one place!\n`); + if (process.env.BROWSERSTACK_TESTHUB_UUID) { + Logger.info(`\nVisit https://automation.browserstack.com/builds/${process.env.BROWSERSTACK_TESTHUB_UUID} to view build report, insights, and many more debugging information all at one place!\n`); } } catch (error) { Logger.error(`Something went wrong in stopping build session for test reporting and analytics - ${error}`); } process.exit(); } - if (helper.isAccessibilitySession()){ - try { - await accessibilityAutomation.stopAccessibilityTestRun(); - } catch (error) { - Logger.error(`Exception in stop accessibility test run: ${error}`); - } - - } }, async beforeEach(settings) { browser.getAccessibilityResults = () => { return accessibilityAutomation.getAccessibilityResults() }; browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAccessibilityResultsSummary() }; - // await accessibilityAutomation.beforeEachExecution(browser); }, // This will be run after each test suite is finished async afterEach(settings) { - // await accessibilityAutomation.afterEachExecution(browser); }, beforeChildProcess(settings) { - + if (!settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options'] = {}; } @@ -514,12 +530,23 @@ module.exports = { try { if (helper.isAccessibilitySession()) { accessibilityAutomation.setAccessibilityCapabilities(settings); + accessibilityAutomation.commandWrapper(); + helper.patchBrowserTerminateCommand(); } } catch (err){ Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); } addProductMapAndbuildUuidCapability(settings); + }, + + async afterChildProcess() { + + await helper.shutDownRequestHandler(); + if (testEventPromises.length > 0) { + await Promise.all(testEventPromises); + testEventPromises.length = 0; // Clear the array + } } }; @@ -569,10 +596,10 @@ const addProductMapAndbuildUuidCapability = (settings) => { if (settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options']['buildProductMap'] = buildProductMap; - settings.desiredCapabilities['bstack:options']['testhubBuildUuid'] = process.env.BS_TESTOPS_BUILD_HASHED_ID ? process.env.BS_TESTOPS_BUILD_HASHED_ID : '' ; + settings.desiredCapabilities['bstack:options']['testhubBuildUuid'] = process.env.BROWSERSTACK_TESTHUB_UUID ? process.env.BROWSERSTACK_TESTHUB_UUID : '' ; } else { settings.desiredCapabilities['browserstack.buildProductMap'] = buildProductMap; - settings.desiredCapabilities['browserstack.testhubBuildUuid'] = process.env.BS_TESTOPS_BUILD_HASHED_ID ? process.env.BS_TESTOPS_BUILD_HASHED_ID : '' ; + settings.desiredCapabilities['browserstack.testhubBuildUuid'] = process.env.BROWSERSTACK_TESTHUB_UUID ? process.env.BROWSERSTACK_TESTHUB_UUID : '' ; } } catch (error) { Logger.debug(`Error while sending productmap and build capabilities ${error}`); diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index af97fa0..d8f75c1 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -1,28 +1,13 @@ const path = require('path'); const helper = require('./utils/helper'); -const {makeRequest} = require('./utils/requestHelper'); const Logger = require('./utils/logger'); -const {ACCESSIBILITY_URL} = require('./utils/constants'); const util = require('util'); +const AccessibilityScripts = require('./scripts/accessibilityScripts'); class AccessibilityAutomation { + static pendingAllyReq = 0; configure(settings = {}) { this._settings = settings['@nightwatch/browserstack'] || {}; - - if (this._settings.accessibility) { - process.env.BROWSERSTACK_ACCESSIBILITY = - String(this._settings.accessibility).toLowerCase() === 'true'; - } - if (process.argv.includes('--disable-accessibility')) { - process.env.BROWSERSTACK_ACCESSIBILITY = false; - - return; - } - process.env.BROWSERSTACK_INFRA = true; - if (settings && settings.webdriver && settings.webdriver.host && settings.webdriver.host.indexOf('browserstack') === -1){ - process.env.BROWSERSTACK_INFRA = false; - } - this._testRunner = settings.test_runner; this._bstackOptions = {}; if ( @@ -37,165 +22,11 @@ class AccessibilityAutomation { this._user = helper.getUserName(settings, this._settings); this._key = helper.getAccessKey(settings, this._settings); } - } - - async createAccessibilityTestRun() { - const userName = this._user; - const accessKey = this._key; - - if (helper.isUndefined(userName) || helper.isUndefined(accessKey)) { - Logger.error( - 'Exception while creating test run for BrowserStack Accessibility Automation: Missing authentication token' - ); - - return [null, null]; - } - - try { - let accessibilityOptions; - if (helper.isUndefined(this._settings.accessibilityOptions)) { - accessibilityOptions = {}; - } else { - accessibilityOptions = this._settings.accessibilityOptions; - } - - const fromProduct = { - accessibility: true - }; - - const data = { - projectName: helper.getProjectName(this._settings, this._bstackOptions, fromProduct), - buildName: - helper.getBuildName(this._settings, this._bstackOptions, fromProduct) || - path.basename(path.resolve(process.cwd())), - startTime: new Date().toISOString(), - description: accessibilityOptions.buildDescription || '', - source: { - frameworkName: helper.getFrameworkName(this._testRunner), - frameworkVersion: helper.getPackageVersion('nightwatch'), - sdkVersion: helper.getAgentVersion() - }, - settings: accessibilityOptions, - versionControl: await helper.getGitMetaData(), - ciInfo: helper.getCiInfo(), - hostInfo: helper.getHostInfo(), - browserstackAutomation: helper.isBrowserstackInfra() - }; - const config = { - auth: { - user: userName, - pass: accessKey - }, - headers: { - 'Content-Type': 'application/json' - } - }; - - const response = await makeRequest('POST', 'test_runs', data, config, ACCESSIBILITY_URL); - const responseData = response.data.data || {}; - - accessibilityOptions.scannerVersion = responseData.scannerVersion; - process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS = JSON.stringify(accessibilityOptions); - - return [responseData.accessibilityToken, responseData.id]; - } catch (error) { - process.env.BROWSERSTACK_ACCESSIBILITY = 'false'; - if (error.response) { - Logger.error( - `Exception while creating test run for BrowserStack Accessibility Automation: ${ - error.response.status - } ${error.response.statusText} ${JSON.stringify(error.response.data)}` - ); - } else { - if (error.message === 'Invalid configuration passed.') { - Logger.error( - `Exception while creating test run for BrowserStack Accessibility Automation: ${ - error.message || error.stack - }` - ); - for (const errorkey of error.errors) { - Logger.error(errorkey.message); - } - process.env.BROWSERSTACK_ACCESSIBILITY = 'false'; - } else { - Logger.error( - `Exception while creating test run for BrowserStack Accessibility Automation: ${ - error.message || error.stack - }` - ); - } - } - - return [null, null]; - } - } - - async stopAccessibilityTestRun() { - if ( - helper.isUndefined(process.env.BS_A11Y_JWT) || - typeof process.env.BS_A11Y_JWT !== 'string' - ) { - return { - status: 'error', - message: 'Build creation had failed.' - }; - } - - const data = {endTime: new Date().toISOString()}; - const config = { - headers: { - Authorization: `Bearer ${process.env.BS_A11Y_JWT}`, - 'Content-Type': 'application/json' - } - }; - const options = { - ...config, - ...{ - body: data, - auth: null, - json: true - } - }; - try { - const response = await makeRequest( - 'PUT', - 'test_runs/stop', - options, - config, - ACCESSIBILITY_URL - ); - if (response.data.error) { - throw new Error('Invalid request: ' + response.data.error); - } else { - Logger.info( - `BrowserStack Accessibility Automation Test Run marked as completed at ${new Date().toISOString()}` - ); - - return {status: 'success', message: ''}; - } - } catch (error) { - if (error.response) { - Logger.error( - `Exception while marking completion of BrowserStack Accessibility Automation Test Run: ${ - error.response.status - } ${error.response.statusText} ${JSON.stringify(error.response.data)}` - ); - } else { - Logger.error( - `Exception while marking completion of BrowserStack Accessibility Automation Test Run: ${ - error.message || util.format(error) - }` - ); - } - - return { - status: 'error', - message: - error.message || - (error.response ? `${error.response.status}:${error.response.statusText}` : error) - }; - } + const accessibilityOptions = helper.isUndefined(this._settings.accessibilityOptions) + ? {} + : this._settings.accessibilityOptions; + process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS = JSON.stringify(accessibilityOptions); } setAccessibilityCapabilities(settings) { @@ -223,25 +54,21 @@ class AccessibilityAutomation { if (this._bstackOptions) { this._bstackOptions.accessibility = this._settings.accessibility; if (this._bstackOptions.accessibilityOptions) { - this._bstackOptions.accessibilityOptions.authToken = process.env.BS_A11Y_JWT; + this._bstackOptions.accessibilityOptions.authToken = process.env.BSTACK_A11Y_JWT; } else { - this._bstackOptions.accessibilityOptions = {authToken: process.env.BS_A11Y_JWT}; + this._bstackOptions.accessibilityOptions = {authToken: process.env.BSTACK_A11Y_JWT}; } - this._bstackOptions.accessibilityOptions.scannerVersion = JSON.parse( - process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS - ).scannerVersion; + this._bstackOptions.accessibilityOptions.scannerVersion = process.env.BSTACK_A11Y_SCANNER_VERSION; } else if (settings.desiredCapabilities['browserstack.accessibility']) { if (settings.desiredCapabilities['browserstack.accessibilityOptions']) { settings.desiredCapabilities['browserstack.accessibilityOptions'].authToken = - process.env.BS_A11Y_JWT; + process.env.BSTACK_A11Y_JWT; } else { settings.desiredCapabilities['browserstack.accessibilityOptions'] = { - authToken: process.env.BS_A11Y_JWT + authToken: process.env.BSTACK_A11Y_JWT }; } - settings.desiredCapabilities['browserstack.accessibilityOptions'].scannerVersion = JSON.parse( - process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS - ).scannerVersion; + settings.desiredCapabilities['browserstack.accessibilityOptions'].scannerVersion = process.env.BSTACK_A11Y_SCANNER_VERSION; } } } catch (e) { @@ -256,9 +83,9 @@ class AccessibilityAutomation { } const isBrowserstackAccessibilityEnabled = process.env.BROWSERSTACK_ACCESSIBILITY === 'true'; const hasA11yJwtToken = - typeof process.env.BS_A11Y_JWT === 'string' && - process.env.BS_A11Y_JWT.length > 0 && - process.env.BS_A11Y_JWT !== 'null'; + typeof process.env.BSTACK_A11Y_JWT === 'string' && + process.env.BSTACK_A11Y_JWT.length > 0 && + process.env.BSTACK_A11Y_JWT !== 'null'; return isBrowserstackAccessibilityEnabled && hasA11yJwtToken; } catch (error) { @@ -297,23 +124,7 @@ class AccessibilityAutomation { return false; } - fetchPlatformDetails(driver) { - let response = {}; - try { - response = { - os_name: driver.capabilities.platformName, - os_version: helper.getPlatformVersion(driver), - browser_name: driver.capabilities.browserName, - browser_version: driver.capabilities.browserVersion - }; - } catch (error) { - Logger.debug(`Exception in fetching platform details with error : ${error}`); - } - - return response; - } - - setExtension(driver) { + validateA11yCaps(driver) { try { const capabilityConfig = driver.desiredCapabilities || {}; const deviceName = driver.capabilities.deviceName || (capabilityConfig['bstack:options'] ? capabilityConfig['bstack:options'].deviceName : capabilityConfig.device) || ''; @@ -345,7 +156,7 @@ class AccessibilityAutomation { return true; } catch (error) { - Logger.debug(`Exception in setExtension Error: ${error}`); + Logger.debug(`Exception in validateA11yCaps Error: ${error}`); } return false; @@ -358,55 +169,14 @@ class AccessibilityAutomation { testMetaData ); this.currentTest.accessibilityScanStarted = true; - this._isAccessibilitySession = this.setExtension(browser); - - if (this.isAccessibilityAutomationSession() && browser && helper.isAccessibilitySession() && this._isAccessibilitySession) { - try { - const session = await browser.session(); - if (session) { - let pageOpen = true; - const currentURL = await browser.driver.getCurrentUrl(); - - let url = {}; - try { - url = new URL(currentURL); - pageOpen = true; - } catch (e) { - pageOpen = false; - } - pageOpen = url.protocol === 'http:' || url.protocol === 'https:'; - - if (pageOpen) { - if (this.currentTest.shouldScanTestForAccessibility) { - Logger.info( - 'Setup for Accessibility testing has started. Automate test case execution will begin momentarily.' - ); - - await browser.executeAsyncScript(` - const callback = arguments[arguments.length - 1]; - const fn = () => { - window.addEventListener('A11Y_TAP_STARTED', fn2); - const e = new CustomEvent('A11Y_FORCE_START'); - window.dispatchEvent(e); - }; - const fn2 = () => { - window.removeEventListener('A11Y_TAP_STARTED', fn); - callback(); - } - fn(); - `); - } else { - await browser.executeAsyncScript(` - const e = new CustomEvent('A11Y_FORCE_STOP'); - window.dispatchEvent(e); - `); - } - } - this.currentTest.accessibilityScanStarted = + this._isAccessibilitySession = this.validateA11yCaps(browser); + + if (this.isAccessibilityAutomationSession() && browser && this._isAccessibilitySession) { + try { + this.currentTest.accessibilityScanStarted = this.currentTest.shouldScanTestForAccessibility; - if (this.currentTest.shouldScanTestForAccessibility) { - Logger.info('Automate test case execution has started.'); - } + if (this.currentTest.shouldScanTestForAccessibility) { + Logger.info('Automate test case execution has started.'); } } catch (e) { Logger.error('Exception in starting accessibility automation scan for this test case', e); @@ -417,46 +187,23 @@ class AccessibilityAutomation { } } - async afterEachExecution(testMetaData) { + async afterEachExecution(testMetaData, uuid) { try { if (this.currentTest.accessibilityScanStarted && this.isAccessibilityAutomationSession() && this._isAccessibilitySession) { if (this.currentTest.shouldScanTestForAccessibility) { Logger.info( 'Automate test case execution has ended. Processing for accessibility testing is underway. ' ); - } - const dataForExtension = { - saveResults: this.currentTest.shouldScanTestForAccessibility, - testDetails: { - name: testMetaData.testcase, - testRunId: process.env.BS_A11Y_TEST_RUN_ID, - filePath: testMetaData.metadata.modulePath, - scopeList: [testMetaData.metadata.name, testMetaData.testcase] - }, - platform: await this.fetchPlatformDetails(browser) - }; - const final_res = await browser.executeAsyncScript( - ` - const callback = arguments[arguments.length - 1]; - - this.res = null; - if (arguments[0].saveResults) { - window.addEventListener('A11Y_TAP_TRANSPORTER', (event) => { - window.tapTransporterData = event.detail; - this.res = window.tapTransporterData; - callback(this.res); - }); - } - const e = new CustomEvent('A11Y_TEST_END', {detail: arguments[0]}); - window.dispatchEvent(e); - if (arguments[0].saveResults !== true ) { - callback(); - } - `, - dataForExtension - ); - if (this.currentTest.shouldScanTestForAccessibility) { - Logger.info('Accessibility testing for this test case has ended.'); + + const dataForExtension = { + 'thTestRunUuid': uuid, + 'thBuildUuid': process.env.BROWSERSTACK_TESTHUB_UUID, + 'thJwtToken': process.env.BROWSERSTACK_TESTHUB_JWT + }; + AccessibilityAutomation.pendingAllyReq++; + await this.saveAccessibilityResults(browser, dataForExtension); + AccessibilityAutomation.pendingAllyReq--; + Logger.info('Accessibility testing for this test case has ended.'); } } } catch (er) { @@ -474,21 +221,9 @@ class AccessibilityAutomation { return {}; } try { - const results = await browser.executeScript(` - return new Promise(function (resolve, reject) { - try { - const event = new CustomEvent('A11Y_TAP_GET_RESULTS'); - const fn = function (event) { - window.removeEventListener('A11Y_RESULTS_RESPONSE', fn); - resolve(event.detail.data); - }; - window.addEventListener('A11Y_RESULTS_RESPONSE', fn); - window.dispatchEvent(event); - } catch { - reject(); - } - }); - `); + Logger.debug('Performing scan before getting results'); + await this.performScan(browser); + const results = await browser.executeAsyncScript(AccessibilityScripts.getResults); return results; } catch { @@ -507,21 +242,9 @@ class AccessibilityAutomation { return {}; } try { - const summaryResults = await browser.executeScript(` - return new Promise(function (resolve, reject) { - try{ - const event = new CustomEvent('A11Y_TAP_GET_RESULTS_SUMMARY'); - const fn = function (event) { - window.removeEventListener('A11Y_RESULTS_SUMMARY_RESPONSE', fn); - resolve(event.detail.summary); - }; - window.addEventListener('A11Y_RESULTS_SUMMARY_RESPONSE', fn); - window.dispatchEvent(event); - } catch { - reject(); - } - }); - `); + Logger.debug('Performing scan before getting results summary'); + await this.performScan(browser); + const summaryResults = await browser.executeAsyncScript(AccessibilityScripts.getResultsSummary); return summaryResults; } catch { @@ -535,6 +258,95 @@ class AccessibilityAutomation { return Object.fromEntries(Object.entries(accessibilityOptions).filter(([k, v]) => !(k.toLowerCase() === 'excludetagsintestingscope' || k.toLowerCase() === 'includetagsintestingscope'))); } + async performScan(browserInstance = null, commandName = '') { + + if (!this.isAccessibilityAutomationSession() || !this._isAccessibilitySession) { + Logger.warn('Not an Accessibility Automation session, cannot perform Accessibility scan.'); + + return; + } + + if (this.currentTest.shouldScanTestForAccessibility === false) { + Logger.info('Skipping Accessibility scan for this test as it\'s disabled.'); + + return; + } + try { + const browser = browserInstance; + + if (!browser) { + Logger.error('No browser instance available for accessibility scan'); + + return; + } + AccessibilityAutomation.pendingAllyReq++; + const results = await browser.executeAsyncScript(AccessibilityScripts.performScan, { + method: commandName || '' + }); + AccessibilityAutomation.pendingAllyReq--; + Logger.debug(util.inspect(results)); + + return results; + + } catch (err) { + AccessibilityAutomation.pendingAllyReq--; + Logger.error('Accessibility Scan could not be performed: ' + err.message); + Logger.debug('Stack trace:', err.stack); + + return; + } + } + + async saveAccessibilityResults(browser, dataForExtension = {}) { + Logger.debug('Performing scan before saving results'); + await this.performScan(browser); + const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); + + return results; + } + + async commandWrapper() { + const nightwatchMain = require.resolve('nightwatch'); + const nightwatchDir = path.dirname(nightwatchMain); + + const commandJson = AccessibilityScripts.commandsToWrap; + const accessibilityInstance = this; + for (const commandKey in commandJson) { + if (commandJson[commandKey].method === 'protocolAction'){ + commandJson[commandKey].name.forEach(commandName => { + try { + const commandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); + const OriginalClass = require(commandPath); + const originalProtocolAction = OriginalClass.prototype.protocolAction; + + OriginalClass.prototype.protocolAction = async function() { + await accessibilityInstance.performScan(browser, commandName); + + return originalProtocolAction.apply(this); + }; + } catch (error) { + Logger.debug(`Failed to patch protocolAction for command ${commandName}`); + } + }); + } else { + commandJson[commandKey].name.forEach(commandName => { + try { + const webElementCommandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); + const originalCommand = require(webElementCommandPath); + const originalCommandFn = originalCommand.command; + + originalCommand.command = async function(...args) { + await accessibilityInstance.performScan(browser, commandName); + + return originalCommandFn.apply(this, args); + }; + } catch (error) { + Logger.debug(`Failed to patch command ${commandName}`); + } + }); + } + } + } } module.exports = AccessibilityAutomation; diff --git a/src/scripts/accessibilityScripts.js b/src/scripts/accessibilityScripts.js new file mode 100644 index 0000000..a399aa8 --- /dev/null +++ b/src/scripts/accessibilityScripts.js @@ -0,0 +1,107 @@ +const path = require('path'); +const fs = require('fs'); +const os = require('os'); +const Logger = require('../utils/logger.js'); + + +class AccessibilityScripts { + static instance = null; + + performScan = null; + getResults = null; + getResultsSummary = null; + saveTestResults = null; + commandsToWrap = null; + ChromeExtension = {}; + + browserstackFolderPath = ''; + commandsPath = ''; + + // don't allow to create instances from it other than through `checkAndGetInstance` + constructor() { + this.browserstackFolderPath = this.getWritableDir(); + this.commandsPath = path.join(this.browserstackFolderPath, 'commands.json'); + } + + static checkAndGetInstance() { + if (!AccessibilityScripts.instance) { + AccessibilityScripts.instance = new AccessibilityScripts(); + AccessibilityScripts.instance.readFromExistingFile(); + } + + return AccessibilityScripts.instance; + } + + getWritableDir() { + const orderedPaths = [ + path.join(os.homedir(), '.browserstack'), + process.cwd(), + os.tmpdir() + ]; + for (const orderedPath of orderedPaths) { + try { + if (fs.existsSync(orderedPath)) { + fs.accessSync(orderedPath); + + return orderedPath; + } + + fs.mkdirSync(orderedPath, {recursive: true}); + + return orderedPath; + + } catch (e) { + Logger.debug(`Failed to access or create directory ${orderedPath}: ${e.message}`); + } + } + + return ''; + } + + readFromExistingFile() { + try { + if (fs.existsSync(this.commandsPath)) { + const data = fs.readFileSync(this.commandsPath, 'utf8'); + if (data) { + this.update(JSON.parse(data)); + } + } + } catch (error) { + Logger.debug(`Failed to read accessibility commands file: ${error.message}`); + } + } + + update(data) { + if (data.scripts) { + this.performScan = data.scripts.scan; + this.getResults = data.scripts.getResults; + this.getResultsSummary = data.scripts.getResultsSummary; + this.saveTestResults = data.scripts.saveResults; + } + if (data.commands && data.commands.length) { + this.commandsToWrap = data.commands; + } + } + + store() { + try { + if (!fs.existsSync(this.browserstackFolderPath)){ + fs.mkdirSync(this.browserstackFolderPath); + } + + fs.writeFileSync(this.commandsPath, JSON.stringify({ + commands: this.commandsToWrap, + scripts: { + scan: this.performScan, + getResults: this.getResults, + getResultsSummary: this.getResultsSummary, + saveResults: this.saveTestResults + } + })); + } catch (error) { + Logger.debug(`Failed to store accessibility commands file: ${error.message}`); + } + } +} + +module.exports = AccessibilityScripts.checkAndGetInstance(); diff --git a/src/testObservability.js b/src/testObservability.js index eb04010..11f78ee 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -9,7 +9,11 @@ const CrashReporter = require('./utils/crashReporter'); const Logger = require('./utils/logger'); const {API_URL, TAKE_SCREENSHOT_REGEX} = require('./utils/constants'); const OrchestrationUtils = require('./testorchestration/orchestrationUtils'); +const AccessibilityAutomation = require('./accessibilityAutomation'); +const accessibilityScripts = require('./scripts/accessibilityScripts'); +const TestMap = require('./utils/testMap'); const hooksMap = {}; +const accessibilityAutomation = new AccessibilityAutomation(); class TestObservability { configure(settings = {}) { @@ -83,36 +87,49 @@ class TestObservability { async launchTestSession() { // Support both old and new configuration options at different levels - const options = this._settings.test_observability || + const testReportingOptions = this._settings.test_observability || this._settings.test_reporting || this._settings.testReportingOptions || this._settings.testObservabilityOptions || this._parentSettings?.testReportingOptions || this._parentSettings?.testObservabilityOptions || {}; + const accessibility = helper.isAccessibilityEnabled(this._parentSettings); + const accessibilityOptions = accessibility ? this._settings.accessibilityOptions || {} : {}; this._gitMetadata = await helper.getGitMetaData(); const fromProduct = { - test_observability: true, - test_reporting: true + test_observability: this._settings.test_observability?.enabled || this._settings.test_reporting?.enabled || false, + accessibility: accessibility }; const data = { format: 'json', project_name: helper.getProjectName(this._settings, this._bstackOptions, fromProduct), name: helper.getBuildName(this._settings, this._bstackOptions, fromProduct), - build_identifier: options.buildIdentifier, - description: options.buildDescription || '', - start_time: new Date().toISOString(), + build_identifier: testReportingOptions.buildIdentifier, + description: testReportingOptions.buildDescription || '', + started_at: new Date().toISOString(), tags: helper.getObservabilityBuildTags(this._settings, this._bstackOptions), host_info: helper.getHostInfo(), ci_info: helper.getCiInfo(), build_run_identifier: process.env.BROWSERSTACK_BUILD_RUN_IDENTIFIER, failed_tests_rerun: process.env.BROWSERSTACK_RERUN || false, version_control: this._gitMetadata, - observability_version: { + accessibility: { + settings: accessibilityOptions + }, + framework_details: { frameworkName: helper.getFrameworkName(this._testRunner), frameworkVersion: helper.getPackageVersion('nightwatch'), - sdkVersion: helper.getAgentVersion() + sdkVersion: helper.getAgentVersion(), + language: 'javascript', + testFramework: { + name: 'nightwatch', + version: helper.getPackageVersion('nightwatch') + } }, + product_map: this.getProductMapForBuildStartCall(this._parentSettings), + browserstackAutomation: helper.isBrowserstackInfra(this._settings), + config: {}, test_orchestration: this.getTestOrchestrationBuildStartData(this._parentSettings) }; @@ -128,20 +145,18 @@ class TestObservability { }; try { - const response = await makeRequest('POST', 'api/v1/builds', data, config, API_URL); + const response = await makeRequest('POST', 'api/v2/builds', data, config, API_URL); Logger.info('Build creation successful!'); process.env.BS_TESTOPS_BUILD_COMPLETED = true; const responseData = response.data || {}; if (responseData.jwt) { - process.env.BS_TESTOPS_JWT = responseData.jwt; + process.env.BROWSERSTACK_TESTHUB_JWT = responseData.jwt; } if (responseData.build_hashed_id) { - process.env.BS_TESTOPS_BUILD_HASHED_ID = responseData.build_hashed_id; - } - if (responseData.allow_screenshots) { - process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = responseData.allow_screenshots.toString(); + process.env.BROWSERSTACK_TESTHUB_UUID = responseData.build_hashed_id; } + this.processLaunchBuildResponse(responseData, this._settings); } catch (error) { if (error.response) { Logger.error(`EXCEPTION IN BUILD START EVENT : ${error.response.status} ${error.response.statusText} ${JSON.stringify(error.response.data)}`); @@ -159,11 +174,86 @@ class TestObservability { return orchestrationUtils.getBuildStartData(); } + processLaunchBuildResponse(responseData, settings) { + if (helper.isTestObservabilitySession()) { + this.processTestObservabilityResponse(responseData); + } + this.processAccessibilityResponse(responseData, settings); + } + + processTestObservabilityResponse(responseData) { + if (!responseData.observability) { + this.handleErrorForObservability(null); + + return; + } + if (!responseData.observability.success) { + this.handleErrorForObservability(responseData.observability); + + return; + } + process.env.BROWSERSTACK_TEST_OBSERVABILITY = 'true'; + process.env.BROWSERSTACK_TEST_REPORTING = 'true'; + if (responseData.observability.options.allow_screenshots) { + process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = responseData.observability.options.allow_screenshots.toString(); + } + } + + processAccessibilityResponse(responseData, settings) { + if (!responseData.accessibility) { + if (settings.accessibility === true) { + this.handleErrorForAccessibility(null); + } + + return; + } + if (!responseData.accessibility.success) { + this.handleErrorForAccessibility(responseData.accessibility); + + return; + } + + if (responseData.accessibility.options) { + const {accessibilityToken, pollingTimeout, scannerVersion} = helper.jsonifyAccessibilityArray(responseData.accessibility.options.capabilities, 'name', 'value'); + const scriptsJson = { + 'scripts': helper.jsonifyAccessibilityArray(responseData.accessibility.options.scripts, 'name', 'command'), + 'commands': responseData.accessibility.options.commandsToWrap?.commands ?? [] + }; + if (scannerVersion) { + process.env.BSTACK_A11Y_SCANNER_VERSION = scannerVersion; + Logger.debug(`Accessibility scannerVersion ${scannerVersion}`); + } + if (accessibilityToken) { + process.env.BSTACK_A11Y_JWT = accessibilityToken; + process.env.BROWSERSTACK_ACCESSIBILITY = 'true'; + } + if (pollingTimeout) { + process.env.BSTACK_A11Y_POLLING_TIMEOUT = pollingTimeout; + } + if (scriptsJson) { + accessibilityScripts.update(scriptsJson); + accessibilityScripts.store(); + } + } + + } + + handleErrorForObservability(error) { + process.env.BROWSERSTACK_TEST_OBSERVABILITY = 'false'; + process.env.BROWSERSTACK_TEST_REPORTING = 'false'; + helper.logBuildError(error, 'Test Reporting and Analytics'); + } + + handleErrorForAccessibility(error) { + process.env.BROWSERSTACK_ACCESSIBILITY = 'false'; + helper.logBuildError(error, 'Accessibility'); + } + async stopBuildUpstream () { if (!process.env.BS_TESTOPS_BUILD_COMPLETED) { return; } - if (!process.env.BS_TESTOPS_JWT) { + if (!process.env.BROWSERSTACK_TESTHUB_JWT) { Logger.info('[STOP_BUILD] Missing Authentication Token/ Build ID'); return { @@ -172,11 +262,11 @@ class TestObservability { }; } const data = { - 'stop_time': new Date().toISOString() + 'finished_at': new Date().toISOString() }; const config = { headers: { - 'Authorization': `Bearer ${process.env.BS_TESTOPS_JWT}`, + 'Authorization': `Bearer ${process.env.BROWSERSTACK_TESTHUB_JWT}`, 'Content-Type': 'application/json', 'X-BSTACK-TESTOPS': 'true' } @@ -184,7 +274,7 @@ class TestObservability { await helper.uploadPending(); await helper.shutDownRequestHandler(); try { - const response = await makeRequest('PUT', `api/v1/builds/${process.env.BS_TESTOPS_BUILD_HASHED_ID}/stop`, data, config, API_URL, false); + const response = await makeRequest('PUT', `api/v1/builds/${process.env.BROWSERSTACK_TESTHUB_UUID}/stop`, data, config, API_URL, false); if (response.data?.error) { throw {message: response.data.error}; } else { @@ -207,8 +297,8 @@ class TestObservability { } } - async sendEvents(eventData, testFileReport, startEventType, finishedEventType, hookId, hookType, sectionName) { - await this.sendTestRunEvent(eventData, testFileReport, startEventType, hookId, hookType, sectionName); + async sendHookEvents(eventData, testFileReport, startEventType, finishedEventType, hookId, hookType, sectionName) { + await this.sendHookRunEvent(eventData, testFileReport, startEventType, hookId, hookType, sectionName); if (eventData.httpOutput && eventData.httpOutput.length > 0) { for (const [index, output] of eventData.httpOutput.entries()) { if (index % 2 === 0) { @@ -216,7 +306,7 @@ class TestObservability { } } } - await this.sendTestRunEvent(eventData, testFileReport, finishedEventType, hookId, hookType, sectionName); + await this.sendHookRunEvent(eventData, testFileReport, finishedEventType, hookId, hookType, sectionName); } async processTestReportFile(testFileReport) { @@ -232,28 +322,19 @@ class TestObservability { const eventData = completedSections[sectionName]; switch (sectionName) { case '__global_beforeEach_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalBeforeEachHookId, 'GLOBAL_BEFORE_EACH', sectionName); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalBeforeEachHookId, 'GLOBAL_BEFORE_EACH', sectionName); break; } case '__before_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', beforeHookId, 'BEFORE_ALL', sectionName); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', beforeHookId, 'BEFORE_ALL', sectionName); break; } case '__after_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', afterHookId, 'AFTER_ALL', sectionName); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', afterHookId, 'AFTER_ALL', sectionName); break; } case '__global_afterEach_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalAfterEachHookId, 'GLOBAL_AFTER_EACH', sectionName); - break; - } - default: { - if (eventData.retryTestData?.length>0) { - for (const retryTest of eventData.retryTestData) { - await this.processTestRunData(retryTest, sectionName, testFileReport, hookIds); - } - } - await this.processTestRunData(eventData, sectionName, testFileReport, hookIds); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalAfterEachHookId, 'GLOBAL_AFTER_EACH', sectionName); break; } } @@ -266,15 +347,11 @@ class TestObservability { } } - async processTestRunData (eventData, sectionName, testFileReport, hookIds) { - const testUuid = uuidv4(); - const errorData = eventData.commands.find(command => command.result?.stack); - eventData.lastError = errorData ? errorData.result : null; - await this.sendTestRunEvent(eventData, testFileReport, 'TestRunStarted', testUuid, null, sectionName, hookIds); + async processTestRunData (eventData, uuid) { if (eventData.httpOutput && eventData.httpOutput.length > 0) { for (const [index, output] of eventData.httpOutput.entries()) { if (index % 2 === 0) { - await this.createHttpLogEvent(output, eventData.httpOutput[index + 1], testUuid); + await this.createHttpLogEvent(output, eventData.httpOutput[index + 1], uuid); } } } @@ -291,7 +368,7 @@ class TestObservability { try { if (fs.existsSync(screenshotPath)) { const screenshot = fs.readFileSync(screenshotPath, 'base64'); - await this.createScreenshotLogEvent(testUuid, screenshot, command.startTime); + await this.createScreenshotLogEvent(uuid, screenshot, command.startTime); } } catch (err) { Logger.debug(`Failed to upload screenshot from saveScreenshot: ${err.message}`); @@ -299,11 +376,11 @@ class TestObservability { } else if (TAKE_SCREENSHOT_REGEX.test(command.name) && command.result) { try { if (command.result.value) { - await this.createScreenshotLogEvent(testUuid, command.result.value, command.startTime); + await this.createScreenshotLogEvent(uuid, command.result.value, command.startTime); } else if (command.result.valuePath) { if (fs.existsSync(command.result.valuePath)) { const screenshot = fs.readFileSync(command.result.valuePath, 'utf8'); - await this.createScreenshotLogEvent(testUuid, screenshot, command.startTime); + await this.createScreenshotLogEvent(uuid, screenshot, command.startTime); } } } catch (err) { @@ -312,9 +389,8 @@ class TestObservability { } } } - await this.sendTestRunEvent(eventData, testFileReport, 'TestRunFinished', testUuid, null, sectionName, hookIds); } - + async sendSkippedTestEvent(skippedTest, testFileReport) { const testData = { uuid: uuidv4(), @@ -384,7 +460,76 @@ class TestObservability { } } - async sendTestRunEvent(eventData, testFileReport, eventType, uuid, hookType, sectionName, hooks) { + async sendTestRunEvent(eventType, test, uuid) { + Logger.debug(`Sending test run event with eventType: ${eventType}`); + const testMetaData = test.metadata; + const testName = test.testcase; + const settings = test.settings || {}; + const startTimestamp = test.envelope[testName].startTimestamp; + const testResults = {}; + const testBody = this.getTestBody(test.testCaseData); + const provider = helper.getCloudProvider(testMetaData.host); + const testData = { + uuid: uuid, + type: 'test', + name: testName, + body: { + lang: 'javascript', + code: testBody ? testBody.toString() : null + }, + scope: `${testMetaData.name} - ${testName}`, + scopes: [ + testMetaData.name + ], + tags: testMetaData.tags, + identifier: `${testMetaData.name} - ${testName}`, + file_name: path.relative(process.cwd(), testMetaData.modulePath), + location: path.relative(process.cwd(), testMetaData.modulePath), + vc_filepath: (this._gitMetadata && this._gitMetadata.root) ? path.relative(this._gitMetadata.root, testMetaData.modulePath) : null, + started_at: new Date(startTimestamp).toISOString(), + result: 'pending', + framework: 'nightwatch', + integrations: { + [provider]: helper.getIntegrationsObject(testMetaData.sessionCapabilities, testMetaData.sessionId, testMetaData.host, settings?.desiredCapabilities?.['bstack:options']?.osVersion) + }, + product_map: { + observability: helper.isTestObservabilitySession(), + accessibility: helper.isAccessibilitySession() && accessibilityAutomation.shouldScanTestForAccessibility(test) && process.env.VALID_ALLY_PLATFORM + } + }; + + if (eventType === 'TestRunFinished') { + const eventData = test.envelope[testName].testcase; + testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(startTimestamp).toISOString(); + testData.result = 'passed'; + if (eventData && eventData.commands && Array.isArray(eventData.commands)) { + const failedCommand = eventData.commands.find(cmd => cmd.status === 'fail'); + if (failedCommand) { + testData.result = 'failed'; + if (failedCommand.result) { + testData.failure = [ + { + 'backtrace': [stripAnsi(failedCommand.result.message || ''), failedCommand.result.stack || ''] + } + ]; + testData.failure_reason = failedCommand.result.message ? stripAnsi(failedCommand.result.message) : null; + if (failedCommand.result.name) { + testData.failure_type = failedCommand.result.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; + } + } + } + } + await this.processTestRunData (eventData, uuid); + } + + const uploadData = { + event_type: eventType, + test_run: testData + }; + await helper.uploadEventData(uploadData); + } + + async sendHookRunEvent(eventData, testFileReport, eventType, uuid, hookType, sectionName, hooks) { const testData = { uuid: uuid, type: 'hook', @@ -404,12 +549,12 @@ class TestObservability { hook_type: hookType }; - if (eventType === 'HookRunFinished' || eventType === 'TestRunFinished') { + if (eventType === 'HookRunFinished') { testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(eventData.startTimestamp).toISOString(); testData.result = eventData.status === 'pass' ? 'passed' : 'failed'; testData.duration_in_ms = 'timeMs' in eventData ? eventData.timeMs : eventData.time; if (eventData.status === 'fail' && eventData.lastError) { - testData.failure = [ + testData.failure_data = [ { 'backtrace': [stripAnsi(eventData.lastError.message), eventData.lastError.stack] } @@ -420,7 +565,7 @@ class TestObservability { } } else if (eventData.status === 'fail' && (testFileReport?.completed[sectionName]?.lastError || testFileReport?.completed[sectionName]?.stackTrace)) { const testCompletionData = testFileReport.completed[sectionName]; - testData.failure = [ + testData.failure_data = [ {'backtrace': [testCompletionData?.stackTrace]} ]; testData.failure_reason = testCompletionData?.assertions.find(val => val.stackTrace === testCompletionData.stackTrace)?.failure; @@ -434,26 +579,10 @@ class TestObservability { testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); } - if (eventType === 'TestRunStarted') { - testData.type = 'test'; - testData.integrations = {}; - const provider = helper.getCloudProvider(testFileReport.host); - testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); - } - - if (eventType === 'TestRunFinished') { - testData.type = 'test'; - testData.hooks = hooks; - } - const uploadData = { - event_type: eventType + event_type: eventType, + hook_run: testData }; - if (eventType.match(/HookRun/)) { - uploadData['hook_run'] = testData; - } else { - uploadData['test_run'] = testData; - } await helper.uploadEventData(uploadData); } @@ -709,6 +838,25 @@ class TestObservability { Logger.error(`Exception in uploading log data to Test Reporting and Analytics with error : ${error}`); } } -} + + getProductMapForBuildStartCall(settings) { + const product = helper.getObservabilityLinkedProductName(settings.desiredCapabilities, settings?.selenium?.host); + + const buildProductMap = { + automate: product === 'automate', + app_automate: product === 'app-automate', + observability: helper.isTestObservabilitySession(), + accessibility: helper.isAccessibilityEnabled(settings), + turboscale: product === 'turboscale', + percy: false + }; + + return buildProductMap; + } + + getTestBody(testCaseData) { + return testCaseData?.context.__module[testCaseData.testName] || null; + } +} module.exports = TestObservability; diff --git a/src/testorchestration/orchestrationUtils.js b/src/testorchestration/orchestrationUtils.js index c9965f6..63787f3 100644 --- a/src/testorchestration/orchestrationUtils.js +++ b/src/testorchestration/orchestrationUtils.js @@ -145,7 +145,7 @@ class OrchestrationUtils { * Check if the abort build file exists */ static checkAbortBuildFileExists() { - const buildUuid = process.env.BS_TESTOPS_BUILD_HASHED_ID; + const buildUuid = process.env.BROWSERSTACK_TESTHUB_UUID; const filePath = path.join(tmpdir(), `abort_build_${buildUuid}`); return fs.existsSync(filePath); @@ -155,7 +155,7 @@ class OrchestrationUtils { * Write failure to file */ static writeFailureToFile(testName) { - const buildUuid = process.env.BS_TESTOPS_BUILD_HASHED_ID; + const buildUuid = process.env.BROWSERSTACK_TESTHUB_UUID; const failedTestsFile = path.join(tmpdir(), `failed_tests_${buildUuid}.txt`); fs.appendFileSync(failedTestsFile, `${testName}\n`); @@ -433,7 +433,7 @@ class OrchestrationUtils { * Collects build data by making a call to the collect-build-data endpoint */ async collectBuildData(config) { - const buildUuid = process.env.BS_TESTOPS_BUILD_HASHED_ID; + const buildUuid = process.env.BROWSERSTACK_TESTHUB_UUID; this.logger.debug(`[collectBuildData] Collecting build data for build UUID: ${buildUuid}`); try { diff --git a/src/testorchestration/requestUtils.js b/src/testorchestration/requestUtils.js index 254dc9e..ef7e2d0 100644 --- a/src/testorchestration/requestUtils.js +++ b/src/testorchestration/requestUtils.js @@ -36,11 +36,11 @@ class RequestUtils { * Makes an orchestration request with the given method and data */ static async makeOrchestrationRequest(method, reqEndpoint, options) { - const jwtToken = process.env.BS_TESTOPS_JWT || ''; + const jwtToken = process.env.BROWSERSTACK_TESTHUB_JWT || ''; // Validate JWT token if (!jwtToken) { - Logger.error('BS_TESTOPS_JWT environment variable is not set. This is required for test orchestration.'); + Logger.error('BROWSERSTACK_TESTHUB_JWT environment variable is not set. This is required for test orchestration.'); return null; } diff --git a/src/utils/crashReporter.js b/src/utils/crashReporter.js index 8caa23f..1255def 100644 --- a/src/utils/crashReporter.js +++ b/src/utils/crashReporter.js @@ -52,7 +52,7 @@ class CrashReporter { try { const data = { - hashed_id: process.env.BS_TESTOPS_BUILD_HASHED_ID, + hashed_id: process.env.BROWSERSTACK_TESTHUB_UUID, observability_version: { frameworkName: 'nightwatch-default', frameworkVersion: helper.getPackageVersion('nightwatch'), diff --git a/src/utils/helper.js b/src/utils/helper.js index 1054240..ba103d6 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -72,10 +72,42 @@ exports.getObservabilityKey = (config, bstackOptions={}) => { return process.env.BROWSERSTACK_ACCESS_KEY || config?.key || bstackOptions?.accessKey; }; +exports.isAppAutomate = () => { + return process.env.BROWSERSTACK_APP_AUTOMATE === 'true'; +}; + +exports.checkTestEnvironmentForAppAutomate = (testEnvSettings) => { + + const firstEnvKey = Object.keys(testEnvSettings)[0]; + const firstEnv = testEnvSettings[firstEnvKey]; + if (firstEnv?.desiredCapabilities?.['appium:options']?.app) { + return true; + } + + return false; +}; + exports.isAccessibilitySession = () => { return process.env.BROWSERSTACK_ACCESSIBILITY === 'true'; }; +exports.isTestHubBuild = (pluginSettings = {}, isBuildStart = false) => { + if (isBuildStart) { + return pluginSettings?.test_reporting?.enabled === true || pluginSettings?.test_observability?.enabled === true || pluginSettings?.accessibility === true; + } + + return this.isTestObservabilitySession() || this.isAccessibilitySession(); + +}; + +exports.isAccessibilityEnabled = (settings) => { + if (process.argv.includes('--disable-accessibility')) {return false} + + if (process.env.BROWSERSTACK_ACCESSIBILITY === 'false') {return false} + + return settings['@nightwatch/browserstack']?.accessibility === true; +}; + exports.getProjectName = (options, bstackOptions={}, fromProduct={}) => { if ((fromProduct.test_observability || fromProduct.test_reporting) && ((options.test_observability && options.test_observability.projectName) || @@ -376,8 +408,10 @@ exports.getHostInfo = () => { }; }; -exports.isBrowserstackInfra = () => { - return process.env.BROWSERSTACK_INFRA === 'true'; +exports.isBrowserstackInfra = (settings) => { + const isBrowserstackInfra = settings && settings.webdriver && settings.webdriver.host && settings.webdriver.host.indexOf('browserstack') === -1 ? false : true; + + return isBrowserstackInfra; }; const findGitConfig = async (filePath) => { @@ -505,12 +539,12 @@ exports.uploadEventData = async (eventData) => { ['HookRunFinished']: 'Hook_End_Upload' }[eventData.event_type]; - if (process.env.BS_TESTOPS_JWT && process.env.BS_TESTOPS_JWT !== 'null') { + if (process.env.BROWSERSTACK_TESTHUB_JWT && process.env.BROWSERSTACK_TESTHUB_JWT !== 'null') { requestQueueHandler.pending_test_uploads += 1; } if (process.env.BS_TESTOPS_BUILD_COMPLETED === 'true') { - if (process.env.BS_TESTOPS_JWT === 'null') { + if (process.env.BROWSERSTACK_TESTHUB_JWT === 'null') { Logger.info(`EXCEPTION IN ${log_tag} REQUEST TO TEST REPORTING AND ANALYTICS : missing authentication token`); requestQueueHandler.pending_test_uploads = Math.max(0, requestQueueHandler.pending_test_uploads-1); @@ -537,7 +571,7 @@ exports.uploadEventData = async (eventData) => { const config = { headers: { - 'Authorization': `Bearer ${process.env.BS_TESTOPS_JWT}`, + 'Authorization': `Bearer ${process.env.BROWSERSTACK_TESTHUB_JWT}`, 'Content-Type': 'application/json', 'X-BSTACK-TESTOPS': 'true' } @@ -618,7 +652,7 @@ exports.getObservabilityLinkedProductName = (caps, hostname) => { if (hostname) { if (hostname.includes('browserstack.com') && !hostname.includes('hub-ft')) { - if (this.isUndefined(caps.browserName)) { + if (this.isAppAutomate()) { product = 'app-automate'; } else { product = 'automate'; @@ -631,13 +665,14 @@ exports.getObservabilityLinkedProductName = (caps, hostname) => { return product; }; -exports.getIntegrationsObject = (capabilities, sessionId, hostname) => { +exports.getIntegrationsObject = (capabilities, sessionId, hostname, platform_version) => { return { capabilities: capabilities, session_id: sessionId, browser: capabilities.browserName, browser_version: capabilities.browserVersion, platform: capabilities.platformName, + platform_version: capabilities.platformVersion || platform_version, product: this.getObservabilityLinkedProductName(capabilities, hostname) }; }; @@ -1301,4 +1336,67 @@ exports.getGitMetadataForAiSelection = (folders = []) => { })); return formattedResults; -}; \ No newline at end of file +}; + +exports.jsonifyAccessibilityArray = (dataArray, keyName, valueName) => { + const result = {}; + dataArray.forEach((element) => { + result[element[keyName]] = element[valueName]; + }); + + return result; +}; + +exports.logBuildError = (error, product = '') => { + if (!error || !error.errors) { + Logger.error(`${product.toUpperCase()} Build creation failed ${error}`); + + return; + } + + for (const errorJson of error.errors) { + const errorType = errorJson.key; + const errorMessage = errorJson.message; + if (errorMessage) { + switch (errorType) { + case 'ERROR_INVALID_CREDENTIALS': + Logger.error(errorMessage); + break; + case 'ERROR_ACCESS_DENIED': + Logger.info(errorMessage); + break; + case 'ERROR_SDK_DEPRECATED': + Logger.error(errorMessage); + break; + default: + Logger.error(errorMessage); + } + } + } +}; + +exports.patchBrowserTerminateCommand = () =>{ + + const nightwatchDir = path.dirname(require.resolve('nightwatch')); + const CommandPath = path.join(nightwatchDir, 'testsuite/index.js'); + const TestSuite = require(CommandPath); + const originalFn = TestSuite.prototype.terminate; + TestSuite.prototype.terminate = async function(...args) { + const maxWaitTime = 30000; + const pollInterval = 500; + const startTime = Date.now(); + const AccessibilityAutomation = require('../accessibilityAutomation'); + while (Date.now() - startTime < maxWaitTime) { + const pendingAllyReq = AccessibilityAutomation.pendingAllyReq || 0; + + if (pendingAllyReq === 0) { + break; + } + await new Promise(resolve => setTimeout(resolve, pollInterval)); + } + Logger.debug(`Pending Accessibility requests at session end: ${AccessibilityAutomation.pendingAllyReq }`); + + return originalFn.apply(this, args); + }; +}; + diff --git a/src/utils/logPatcher.js b/src/utils/logPatcher.js index 9cd08a6..e0ecd1e 100644 --- a/src/utils/logPatcher.js +++ b/src/utils/logPatcher.js @@ -68,7 +68,7 @@ class LogPatcher extends Transport { process.on('message', (data) => { if (data.uuid !== undefined){ _uuid = data.uuid; - process.env.TEST_OPS_TEST_UUID = _uuid; + process.env.TEST_RUN_UUID = _uuid; } }); process.on('disconnect', async () => { diff --git a/src/utils/requestQueueHandler.js b/src/utils/requestQueueHandler.js index e4363ec..f9ad180 100644 --- a/src/utils/requestQueueHandler.js +++ b/src/utils/requestQueueHandler.js @@ -93,7 +93,7 @@ class RequestQueueHandler { async batchAndPostEvents (eventUrl, kind, data) { const config = { headers: { - 'Authorization': `Bearer ${process.env.BS_TESTOPS_JWT}`, + 'Authorization': `Bearer ${process.env.BROWSERSTACK_TESTHUB_JWT}`, 'Content-Type': 'application/json', 'X-BSTACK-TESTOPS': 'true' } @@ -101,7 +101,7 @@ class RequestQueueHandler { try { const response = await makeRequest('POST', eventUrl, data, config); - if (response.data.error) { + if (response.data && response.data.error) { throw ({message: response.data.error}); } else { this.pending_test_uploads = Math.max(0, this.pending_test_uploads - data.length); diff --git a/src/utils/testMap.js b/src/utils/testMap.js new file mode 100644 index 0000000..39b774c --- /dev/null +++ b/src/utils/testMap.js @@ -0,0 +1,95 @@ +const {v4: uuidv4} = require('uuid'); + +const sharedTestMap = new Map(); +let sharedCurrentTest = null; +const activeTestRuns = new Map(); + +class TestMap { + + static storeTestDetails(test) { + const testIdentifier = this.generateTestIdentifier(test); + const uuid = this.generateUUID(); + + if (!sharedTestMap.has(testIdentifier)) { + sharedTestMap.set(testIdentifier, { + baseUuid: uuid, // Store the first UUID as base + retries: [], + currentUuid: uuid, + test, + createdAt: new Date().toISOString() + }); + } else { + // This is a retry - add new UUID to retries array + const testData = sharedTestMap.get(testIdentifier); + testData.retries.push({ + uuid, + startedAt: new Date().toISOString() + }); + testData.currentUuid = uuid; // Update to latest UUID + sharedTestMap.set(testIdentifier, testData); + } + + // Track this as an active test run + activeTestRuns.set(uuid, { + identifier: testIdentifier, + startedAt: new Date().toISOString(), + hasFinished: false + }); + + sharedCurrentTest = testIdentifier; + + return uuid; + } + + static getUUID(test = null) { + if (test) { + const testIdentifier = typeof test === 'string' ? test : this.generateTestIdentifier(test); + const testData = sharedTestMap.get(testIdentifier); + + return testData ? testData.currentUuid : null; + } + + return null; + } + + static markTestFinished(uuid) { + if (activeTestRuns.has(uuid)) { + const testRun = activeTestRuns.get(uuid); + testRun.hasFinished = true; + testRun.finishedAt = new Date().toISOString(); + activeTestRuns.set(uuid, testRun); + + return true; + } + + return false; + } + + static hasTestFinished(uuid) { + const testRun = activeTestRuns.get(uuid); + + return testRun ? testRun.hasFinished : false; + } + + + static getTestDetails(identifier) { + return sharedTestMap.has(identifier) ? sharedTestMap.get(identifier) : null; + } + + static generateTestIdentifier(test) { + if (!test) { + throw new Error('Test object is required to generate identifier'); + } + const testName = test.testcase; + const moduleName = test.metadata.name; + + return `${moduleName}::${testName}`; + } + + static generateUUID() { + return uuidv4(); + } +} + +module.exports = TestMap; + diff --git a/test/src/test-observability/processTestRunData.js b/test/src/test-observability/processTestRunData.js index 4a4c6aa..c5c46aa 100644 --- a/test/src/test-observability/processTestRunData.js +++ b/test/src/test-observability/processTestRunData.js @@ -8,37 +8,28 @@ describe('TestObservability - processTestRunData', function () { this.sandbox = sinon.createSandbox(); this.testObservability = new TestObservability(); - this.eventData = {commands: []}; - this.sectionName = 'testSection'; - this.testFileReport = {}; - this.hookIds = []; - this.sendTestRunEventStub = this.sandbox.stub(this.testObservability, 'sendTestRunEvent').resolves(); + this.eventData = {commands: [], httpOutput: []}; + this.uuid = 'test-uuid-123'; }); afterEach(() => { this.sandbox.restore(); }); - it('should send test run events', async () => { - await this.testObservability.processTestRunData(this.eventData, this.sectionName, this.testFileReport, this.hookIds); - sinon.assert.calledTwice(this.sendTestRunEventStub); - sinon.assert.calledWith(this.sendTestRunEventStub.firstCall, this.eventData, this.testFileReport, 'TestRunStarted', sinon.match.string, null, this.sectionName, this.hookIds); - sinon.assert.calledWith(this.sendTestRunEventStub.secondCall, this.eventData, this.testFileReport, 'TestRunFinished', sinon.match.string, null, this.sectionName, this.hookIds); - }); - it('should create screenshot log events', async () => { this.eventData = { commands: [ {name: 'saveScreenshot', args: ['path/to/screenshot.png'], startTime: 'start_time'} - ] + ], + httpOutput: [] }; this.sandbox.stub(fs, 'existsSync').callsFake(() => true); this.sandbox.stub(fs, 'readFileSync').callsFake(() => 'screenshot-base-64'); const createScreenshotLogEventStub = this.sandbox.stub(this.testObservability, 'createScreenshotLogEvent'); process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = 'true'; - await this.testObservability.processTestRunData(this.eventData, this.sectionName, this.testFileReport, this.hookIds); + await this.testObservability.processTestRunData(this.eventData, this.uuid); process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = 'false'; - sinon.assert.calledOnceWithExactly(createScreenshotLogEventStub, sinon.match.string, 'screenshot-base-64', 'start_time'); + sinon.assert.calledOnceWithExactly(createScreenshotLogEventStub, this.uuid, 'screenshot-base-64', 'start_time'); }); }); diff --git a/test/src/utils/helper.js b/test/src/utils/helper.js index c82372f..2d54cb7 100644 --- a/test/src/utils/helper.js +++ b/test/src/utils/helper.js @@ -326,21 +326,30 @@ describe('isBrowserstackInfra', () => { isBrowserstackInfra = require('../../../src/utils/helper').isBrowserstackInfra; }); - it('returns false for undefined', async () => { - delete process.env.BROWSERSTACK_INFRA; - expect(isBrowserstackInfra()).to.be.false; + it('returns true for undefined settings', async () => { + expect(isBrowserstackInfra()).to.be.true; }); - it('returns true if env variable is set to true', async () => { - process.env.BROWSERSTACK_INFRA = true; - expect(isBrowserstackInfra()).to.be.true; - delete process.env.BROWSERSTACK_INFRA; + it('returns true for empty settings', async () => { + expect(isBrowserstackInfra({})).to.be.true; }); - it('returns false if env variable is set to false', async () => { - process.env.BROWSERSTACK_INFRA = false; - expect(isBrowserstackInfra()).to.be.false; - delete process.env.BROWSERSTACK_INFRA; + it('returns true if webdriver.host contains browserstack', async () => { + const settings = { + webdriver: { + host: 'hub-cloud.browserstack.com' + } + }; + expect(isBrowserstackInfra(settings)).to.be.true; + }); + + it('returns false if webdriver.host does not contain browserstack', async () => { + const settings = { + webdriver: { + host: 'localhost' + } + }; + expect(isBrowserstackInfra(settings)).to.be.false; }); });