diff --git a/Makefile b/Makefile index d5a2dfe..f9da6b0 100755 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ # Make these targets phony (they don't create files with these names) .PHONY: all setup dev clean clean-all build build-win build-linux \ build-mac build-mac-arm build-mac-universal \ - test css css-watch lint lint-md format validate qa docs-screenshots setup-hooks sonar \ + test perf-test stress-metrics prometheus-verify css css-watch lint lint-md format validate qa docs-screenshots setup-hooks sonar \ security gitleaks sbom renovate renovate-local mend-scan \ icons sample-logo release @@ -50,6 +50,15 @@ build-mac-universal: setup-scripts test: setup-scripts @node scripts/index.js test +perf-test: setup-scripts + @node scripts/index.js perf-test + +stress-metrics: setup-scripts + @node scripts/index.js stress-metrics + +prometheus-verify: setup-scripts + @node scripts/index.js prometheus-verify + css: setup-scripts @node scripts/index.js css diff --git a/package.json b/package.json index 63736d5..9d1590b 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,9 @@ "prebuild:mac-universal": "npm run build:ts && node scripts/prepare-build.js mac", "build:mac-universal": "cross-env NODE_ENV=production electron-builder --mac --universal", "build:linux": "cross-env NODE_ENV=production electron-builder --linux", - "stress:metrics": "node scripts/publish-stress-metrics.js" + "stress:metrics": "node scripts/publish-stress-metrics.js", + "prometheus:verify": "node scripts/verify-prometheus-metrics.js", + "perf:test": "node scripts/run-perf-metrics-job.js" }, "author": "AI Code Fusion ", "license": "GPL-3.0", diff --git a/scripts/README.md b/scripts/README.md index d8bdc2b..8b75e2a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -42,6 +42,10 @@ node scripts/index.js [args...] - `test` - Run all tests - `test:watch` - Watch and run tests on changes +- `test:stress` - Run stress benchmark tests +- `stress:metrics` - Build stress summary and Prometheus payload +- `prometheus:verify` - Verify stress metrics are visible in Prometheus +- `perf-test` - Run stress tests, push metrics to Pushgateway, and verify Prometheus - `lint` - Run linter - `format` - Run code formatter - `validate` - Run all validation (lint + test) diff --git a/scripts/index.js b/scripts/index.js index b1a8cfd..eb22a3b 100755 --- a/scripts/index.js +++ b/scripts/index.js @@ -108,6 +108,31 @@ async function executeCommand() { await utils.runNpmScript('test:watch'); break; + case 'test:stress': + case 'stress-test': + await utils.runNpmScript('test:stress'); + console.log('Stress tests completed successfully'); + break; + + case 'stress:metrics': + case 'stress-metrics': + await utils.runNpmScript('stress:metrics'); + console.log('Stress metrics summary generated successfully'); + break; + + case 'prometheus:verify': + case 'prometheus-verify': + await utils.runNpmScript('prometheus:verify'); + console.log('Prometheus stress metrics verification completed successfully'); + break; + + case 'perf': + case 'perf:test': + case 'perf-test': + await utils.runNpmScript('perf:test'); + console.log('Performance metrics job completed successfully'); + break; + // Code quality commands case 'lint': await utils.runNpmScript('lint'); diff --git a/scripts/lib/utils.js b/scripts/lib/utils.js index e318108..87550d2 100755 --- a/scripts/lib/utils.js +++ b/scripts/lib/utils.js @@ -225,6 +225,10 @@ function printHelp() { console.log('Testing & Quality:'); console.log(' test - Run tests'); console.log(' test:watch - Run tests in watch mode'); + console.log(' test:stress - Run stress benchmark tests'); + console.log(' stress:metrics - Build stress benchmark summary + Prometheus payload'); + console.log(' prometheus:verify - Verify pushed stress metrics in Prometheus'); + console.log(' perf-test - Run stress tests, push metrics, and verify Prometheus'); console.log(' lint - Run linter'); console.log(' lint:md - Validate markdown links, image paths, and no decorative icons'); console.log(' format - Format code'); diff --git a/scripts/publish-stress-metrics.js b/scripts/publish-stress-metrics.js index acaedb2..d390450 100755 --- a/scripts/publish-stress-metrics.js +++ b/scripts/publish-stress-metrics.js @@ -97,6 +97,7 @@ function normalizeBenchmarkRecord(filePath) { scenario, sourceFile: path.basename(filePath), capturedAt: sourceStats.mtime.toISOString(), + capturedAtMs: sourceStats.mtimeMs, p50Ms, p95Ms, p99Ms, @@ -106,7 +107,61 @@ function normalizeBenchmarkRecord(filePath) { }; } -function buildPrometheusPayload(records) { +function resolvePublishTimestampSeconds(providedValue) { + const fromArgument = toFiniteNumber(providedValue); + if (fromArgument !== null) { + return Math.floor(fromArgument); + } + + const fromEnvironment = toFiniteNumber(process.env.STRESS_METRICS_PUBLISH_TS_SECONDS); + if (fromEnvironment !== null) { + return Math.floor(fromEnvironment); + } + + return Math.floor(Date.now() / 1000); +} + +function parseCapturedAtMs(record) { + if (toFiniteNumber(record.capturedAtMs) !== null) { + return Number(record.capturedAtMs); + } + + const parsedDate = Date.parse(record.capturedAt); + return Number.isFinite(parsedDate) ? parsedDate : 0; +} + +function selectLatestRecordPerScenario(records) { + const latestRecordsByScenario = new Map(); + + for (const record of records) { + const existingRecord = latestRecordsByScenario.get(record.scenario); + if (!existingRecord) { + latestRecordsByScenario.set(record.scenario, record); + continue; + } + + const existingCapturedAt = parseCapturedAtMs(existingRecord); + const candidateCapturedAt = parseCapturedAtMs(record); + if (candidateCapturedAt > existingCapturedAt) { + latestRecordsByScenario.set(record.scenario, record); + continue; + } + + if ( + candidateCapturedAt === existingCapturedAt && + String(record.sourceFile).localeCompare(String(existingRecord.sourceFile)) > 0 + ) { + latestRecordsByScenario.set(record.scenario, record); + } + } + + return Array.from(latestRecordsByScenario.values()).sort((leftRecord, rightRecord) => + leftRecord.scenario.localeCompare(rightRecord.scenario) + ); +} + +function buildPrometheusPayload(records, options = {}) { + const publishTimestampSeconds = resolvePublishTimestampSeconds(options.publishTimestampSeconds); const lines = [ `# HELP ${METRIC_PREFIX}_latency_ms Stress benchmark latency in milliseconds.`, `# TYPE ${METRIC_PREFIX}_latency_ms gauge`, @@ -169,6 +224,18 @@ function buildPrometheusPayload(records) { } } + lines.push( + `# HELP ${METRIC_PREFIX}_publish_timestamp_seconds Unix timestamp when stress metrics were published.` + ); + lines.push(`# TYPE ${METRIC_PREFIX}_publish_timestamp_seconds gauge`); + const publishLine = toMetricLine( + `${METRIC_PREFIX}_publish_timestamp_seconds`, + publishTimestampSeconds + ); + if (publishLine) { + lines.push(publishLine); + } + return `${lines.join('\n')}\n`; } @@ -236,22 +303,27 @@ async function main() { const sortedRecords = records.sort((leftRecord, rightRecord) => leftRecord.scenario.localeCompare(rightRecord.scenario) ); + const latestRecords = selectLatestRecordPerScenario(sortedRecords); const summaryPayload = { generatedAt: new Date().toISOString(), benchmarkDirectory: path.relative(ROOT_DIR, BENCHMARK_DIR), benchmarkFiles: benchmarkFiles.map((filePath) => path.basename(filePath)).sort(), scenarios: sortedRecords, + latestScenarios: latestRecords, }; fs.mkdirSync(BENCHMARK_DIR, { recursive: true }); fs.writeFileSync(SUMMARY_FILE, JSON.stringify(summaryPayload, null, 2), 'utf8'); - const prometheusPayload = buildPrometheusPayload(sortedRecords); + const prometheusPayload = buildPrometheusPayload(latestRecords); fs.writeFileSync(PROMETHEUS_FILE, prometheusPayload, 'utf8'); console.log(`Stress summary written: ${path.relative(ROOT_DIR, SUMMARY_FILE)}`); console.log(`Prometheus metrics written: ${path.relative(ROOT_DIR, PROMETHEUS_FILE)}`); + console.log( + `Publishing ${latestRecords.length} unique scenario metric sets from ${sortedRecords.length} benchmark file records.` + ); const pushgatewayUrl = (process.env.PUSHGATEWAY_URL || '').trim(); if (!pushgatewayUrl) { @@ -282,8 +354,27 @@ async function main() { } } -main().catch((error) => { - const safeMessage = error instanceof Error ? error.message : String(error); - console.error(`Failed to publish stress metrics: ${safeMessage}`); - process.exit(1); -}); +module.exports = { + main, + __testUtils: { + buildPrometheusPayload, + buildPushgatewayUrl, + normalizeBenchmarkRecord, + parseCapturedAtMs, + pickFirstNumber, + resolvePublishTimestampSeconds, + sanitizeLabelValue, + selectLatestRecordPerScenario, + toFiniteNumber, + toMetricLine, + trimTrailingSlashes, + }, +}; + +if (require.main === module) { + main().catch((error) => { + const safeMessage = error instanceof Error ? error.message : String(error); + console.error(`Failed to publish stress metrics: ${safeMessage}`); + process.exit(1); + }); +} diff --git a/scripts/run-perf-metrics-job.js b/scripts/run-perf-metrics-job.js new file mode 100644 index 0000000..e8b2f6e --- /dev/null +++ b/scripts/run-perf-metrics-job.js @@ -0,0 +1,200 @@ +#!/usr/bin/env node + +const os = require('os'); +const path = require('path'); +const { execSync } = require('child_process'); +const { waitForStressMetrics } = require('./verify-prometheus-metrics'); + +const ROOT_DIR = path.resolve(__dirname, '..'); +const DEFAULT_PUSHGATEWAY_JOB = 'ai_code_fusion_stress'; + +function toFiniteNumber(value) { + const parsedValue = Number(value); + return Number.isFinite(parsedValue) ? parsedValue : null; +} + +function trimBoundaryDots(value) { + let startIndex = 0; + let endIndex = value.length; + + while (startIndex < endIndex && value[startIndex] === '.') { + startIndex += 1; + } + + while (endIndex > startIndex && value[endIndex - 1] === '.') { + endIndex -= 1; + } + + return value.slice(startIndex, endIndex); +} + +function normalizeEndpointUrl(rawValue, label) { + const input = String(rawValue || '').trim(); + if (!input) { + return ''; + } + + const withProtocol = /^https?:\/\//i.test(input) ? input : `https://${input}`; + let parsed; + try { + parsed = new URL(withProtocol); + } catch (error) { + throw new Error(`Invalid ${label} value: ${input}`); + } + + if (!parsed.hostname) { + throw new Error(`Invalid ${label} value: ${input}`); + } + + return parsed.toString(); +} + +function redactUrlForLogs(rawUrl) { + const input = String(rawUrl || '').trim(); + if (!input) { + return ''; + } + + try { + const parsed = new URL(input); + return `${parsed.protocol}//${parsed.host}`; + } catch (error) { + return ''; + } +} + +function normalizeToolsDomain(rawValue) { + const input = String(rawValue || '').trim(); + if (!input) { + return ''; + } + + const withProtocol = /^https?:\/\//i.test(input) ? input : `https://${input}`; + let parsed; + try { + parsed = new URL(withProtocol); + } catch (error) { + throw new Error(`Invalid TOOLS_DOMAIN value: ${input}`); + } + const normalizedHost = trimBoundaryDots(parsed.hostname).trim().toLowerCase(); + return normalizedHost; +} + +function resolveMonitoringEndpoints(environment) { + const env = environment || process.env; + const toolsDomain = normalizeToolsDomain(env.TOOLS_DOMAIN || ''); + const pushgatewaySource = + (env.PUSHGATEWAY_URL || '').trim() || (toolsDomain ? `https://pushgateway.${toolsDomain}` : ''); + const prometheusSource = + (env.PROMETHEUS_URL || '').trim() || (toolsDomain ? `https://prometheus.${toolsDomain}` : ''); + const pushgatewayUrl = normalizeEndpointUrl(pushgatewaySource, 'PUSHGATEWAY_URL'); + const prometheusUrl = normalizeEndpointUrl(prometheusSource, 'PROMETHEUS_URL'); + + return { + toolsDomain, + pushgatewayUrl, + prometheusUrl, + }; +} + +function buildDefaultInstanceName(nowMs, hostName) { + const safeHost = String(hostName || os.hostname()).replace(/[^a-zA-Z0-9_.-]/g, '-'); + return `local-${safeHost}-${Math.floor(nowMs)}`; +} + +function runCommand(command, options = {}) { + const { env = process.env, execFn = execSync } = options; + console.log(`Running: ${command}`); + execFn(command, { + cwd: ROOT_DIR, + env, + stdio: 'inherit', + }); +} + +async function runPerfMetricsJob(options = {}) { + const env = options.env || process.env; + const nowFn = options.nowFn || Date.now; + const hostName = options.hostName || os.hostname(); + const verifyFn = options.verifyFn || waitForStressMetrics; + const execFn = options.execFn || execSync; + const logger = options.logger || console; + + const { toolsDomain, pushgatewayUrl, prometheusUrl } = resolveMonitoringEndpoints(env); + if (!pushgatewayUrl || !prometheusUrl) { + throw new Error( + 'Unable to resolve monitoring endpoints. Set TOOLS_DOMAIN or provide PUSHGATEWAY_URL and PROMETHEUS_URL.' + ); + } + + const jobName = (env.PUSHGATEWAY_JOB || DEFAULT_PUSHGATEWAY_JOB).trim() || DEFAULT_PUSHGATEWAY_JOB; + const instanceName = + (env.PUSHGATEWAY_INSTANCE || '').trim() || buildDefaultInstanceName(nowFn(), hostName); + const strictMode = (env.PUSHGATEWAY_STRICT || 'true').trim().toLowerCase() === 'false' ? 'false' : 'true'; + const timeoutMs = toFiniteNumber(env.PROMETHEUS_VERIFY_TIMEOUT_MS) || 60_000; + const pollIntervalMs = toFiniteNumber(env.PROMETHEUS_VERIFY_POLL_INTERVAL_MS) || 5_000; + const minPublishTimestampSeconds = Math.floor(nowFn() / 1000); + + const commandEnvironment = { + ...env, + PUSHGATEWAY_JOB: jobName, + PUSHGATEWAY_INSTANCE: instanceName, + PUSHGATEWAY_STRICT: strictMode, + PUSHGATEWAY_URL: pushgatewayUrl, + PROMETHEUS_URL: prometheusUrl, + STRESS_METRICS_PUBLISH_TS_SECONDS: String(minPublishTimestampSeconds), + PROMETHEUS_MIN_PUBLISH_TS_SECONDS: String(minPublishTimestampSeconds), + }; + + if (toolsDomain) { + logger.log(`Resolved monitoring endpoints from TOOLS_DOMAIN=${toolsDomain}`); + } + logger.log(`Pushgateway endpoint: ${redactUrlForLogs(pushgatewayUrl)}`); + logger.log(`Prometheus endpoint: ${redactUrlForLogs(prometheusUrl)}`); + logger.log(`Pushgateway job="${jobName}" instance="${instanceName}"`); + + runCommand('npm run test:stress', { env: commandEnvironment, execFn }); + runCommand('npm run stress:metrics', { env: commandEnvironment, execFn }); + + await verifyFn({ + prometheusUrl, + jobName, + instanceName, + minPublishTimestampSeconds, + timeoutMs, + pollIntervalMs, + }); + + logger.log('Performance metrics published and verified in Prometheus.'); + + return { + toolsDomain, + pushgatewayUrl, + prometheusUrl, + jobName, + instanceName, + minPublishTimestampSeconds, + }; +} + +module.exports = { + runPerfMetricsJob, + __testUtils: { + buildDefaultInstanceName, + normalizeToolsDomain, + resolveMonitoringEndpoints, + normalizeEndpointUrl, + redactUrlForLogs, + runCommand, + trimBoundaryDots, + toFiniteNumber, + }, +}; + +if (require.main === module) { + runPerfMetricsJob().catch((error) => { + const safeMessage = error instanceof Error ? error.message : String(error); + console.error(`Performance metrics job failed: ${safeMessage}`); + process.exit(1); + }); +} diff --git a/scripts/verify-prometheus-metrics.js b/scripts/verify-prometheus-metrics.js new file mode 100644 index 0000000..67f9b2a --- /dev/null +++ b/scripts/verify-prometheus-metrics.js @@ -0,0 +1,261 @@ +#!/usr/bin/env node + +const http = require('http'); +const https = require('https'); + +const DEFAULT_METRIC_NAME = 'ai_code_fusion_stress_publish_timestamp_seconds'; +const DEFAULT_TIMEOUT_MS = 60_000; +const DEFAULT_POLL_INTERVAL_MS = 5_000; +const DEFAULT_REQUEST_TIMEOUT_MS = 10_000; + +function toFiniteNumber(value) { + const parsedValue = Number(value); + return Number.isFinite(parsedValue) ? parsedValue : null; +} + +function trimTrailingSlashes(value) { + let lastIndex = value.length; + while (lastIndex > 0 && value[lastIndex - 1] === '/') { + lastIndex -= 1; + } + + return value.slice(0, lastIndex); +} + +function normalizeBaseUrl(rawUrl) { + const input = String(rawUrl || '').trim(); + if (!input) { + throw new Error('PROMETHEUS_URL is required'); + } + + const normalizedInput = /^https?:\/\//i.test(input) ? input : `https://${input}`; + const parsedUrl = new URL(normalizedInput); + parsedUrl.pathname = ''; + parsedUrl.search = ''; + parsedUrl.hash = ''; + return trimTrailingSlashes(parsedUrl.toString()); +} + +function escapeLabelValue(value) { + return String(value).replace(/\\/g, '\\\\').replace(/"/g, '\\"'); +} + +function buildMetricQueries(metricName, jobName, instanceName) { + const escapedJobName = escapeLabelValue(jobName); + const escapedInstanceName = escapeLabelValue(instanceName); + + return [ + `${metricName}{job="${escapedJobName}",instance="${escapedInstanceName}"}`, + `${metricName}{exported_job="${escapedJobName}",exported_instance="${escapedInstanceName}"}`, + ]; +} + +function requestJson(endpointUrl, options = {}) { + const requestTimeoutMs = toFiniteNumber(options.requestTimeoutMs) || DEFAULT_REQUEST_TIMEOUT_MS; + + return new Promise((resolve, reject) => { + const client = endpointUrl.protocol === 'https:' ? https : http; + let settled = false; + const rejectOnce = (error) => { + if (!settled) { + settled = true; + reject(error); + } + }; + + const resolveOnce = (payload) => { + if (!settled) { + settled = true; + resolve(payload); + } + }; + + const request = client.request( + endpointUrl, + { method: 'GET' }, + (response) => { + const responseChunks = []; + response.on('data', (chunk) => responseChunks.push(chunk)); + response.on('end', () => { + const responseBody = Buffer.concat(responseChunks).toString('utf8'); + if (!response.statusCode || response.statusCode < 200 || response.statusCode >= 300) { + rejectOnce( + new Error(`Prometheus returned HTTP ${response.statusCode || 'unknown status'}`) + ); + return; + } + + try { + resolveOnce(JSON.parse(responseBody)); + } catch (error) { + rejectOnce(new Error('Failed to parse Prometheus response body as JSON')); + } + }); + } + ); + + request.setTimeout(requestTimeoutMs, () => { + request.destroy(new Error(`Prometheus request timed out after ${requestTimeoutMs}ms`)); + }); + + request.on('error', (error) => { + const safeMessage = error instanceof Error ? error.message : String(error); + rejectOnce(new Error(`Prometheus request failed: ${safeMessage}`)); + }); + + request.end(); + }); +} + +function extractNumericValues(resultSet) { + if (!Array.isArray(resultSet)) { + return []; + } + + const values = []; + for (const resultItem of resultSet) { + if (Array.isArray(resultItem?.value) && resultItem.value.length >= 2) { + const parsedValue = toFiniteNumber(resultItem.value[1]); + if (parsedValue !== null) { + values.push(parsedValue); + } + } + } + + return values; +} + +async function queryPrometheus(prometheusUrl, query, options = {}) { + const endpointUrl = new URL('/api/v1/query', normalizeBaseUrl(prometheusUrl)); + endpointUrl.searchParams.set('query', query); + + const payload = await requestJson(endpointUrl, options); + if (!payload || payload.status !== 'success') { + throw new Error(`Prometheus query failed for "${query}"`); + } + + const resultSet = payload?.data?.result; + return extractNumericValues(resultSet); +} + +function sleep(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +async function waitForStressMetrics(options) { + const { + prometheusUrl, + jobName, + instanceName, + metricName = DEFAULT_METRIC_NAME, + minPublishTimestampSeconds = null, + timeoutMs = DEFAULT_TIMEOUT_MS, + pollIntervalMs = DEFAULT_POLL_INTERVAL_MS, + requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, + queryFn = queryPrometheus, + nowFn = Date.now, + sleepFn = sleep, + } = options || {}; + + if (!jobName || String(jobName).trim().length === 0) { + throw new Error('PUSHGATEWAY_JOB is required to verify Prometheus metrics.'); + } + + if (!instanceName || String(instanceName).trim().length === 0) { + throw new Error('PUSHGATEWAY_INSTANCE is required to verify Prometheus metrics.'); + } + + const minimumTimestamp = toFiniteNumber(minPublishTimestampSeconds); + const parsedTimeoutMs = toFiniteNumber(timeoutMs) || DEFAULT_TIMEOUT_MS; + const parsedPollIntervalMs = toFiniteNumber(pollIntervalMs) || DEFAULT_POLL_INTERVAL_MS; + const parsedRequestTimeoutMs = toFiniteNumber(requestTimeoutMs) || DEFAULT_REQUEST_TIMEOUT_MS; + const boundedRequestTimeoutMs = Math.max(1, Math.min(parsedRequestTimeoutMs, parsedTimeoutMs)); + const queries = buildMetricQueries(metricName, String(jobName).trim(), String(instanceName).trim()); + const deadline = nowFn() + parsedTimeoutMs; + let lastError = null; + + while (nowFn() <= deadline) { + for (const query of queries) { + try { + const values = await queryFn(prometheusUrl, query, { + requestTimeoutMs: boundedRequestTimeoutMs, + }); + const hasMatch = values.some((value) => { + if (minimumTimestamp !== null) { + return value >= minimumTimestamp; + } + return value > 0; + }); + + if (hasMatch) { + return { query, values }; + } + } catch (error) { + lastError = error; + } + } + + if (nowFn() > deadline) { + break; + } + + await sleepFn(parsedPollIntervalMs); + } + + const errorSuffix = + lastError instanceof Error ? ` Last error: ${lastError.message}` : ' No Prometheus samples matched.'; + const attemptedQueries = queries.join(' | '); + + throw new Error( + `Timed out after ${parsedTimeoutMs}ms waiting for "${metricName}" in Prometheus. Attempted queries: ${attemptedQueries}.${errorSuffix}` + ); +} + +async function main() { + const prometheusUrl = normalizeBaseUrl(process.env.PROMETHEUS_URL || ''); + const jobName = (process.env.PUSHGATEWAY_JOB || 'ai_code_fusion_stress').trim(); + const instanceName = (process.env.PUSHGATEWAY_INSTANCE || '').trim(); + const timeoutMs = toFiniteNumber(process.env.PROMETHEUS_VERIFY_TIMEOUT_MS) || DEFAULT_TIMEOUT_MS; + const pollIntervalMs = + toFiniteNumber(process.env.PROMETHEUS_VERIFY_POLL_INTERVAL_MS) || DEFAULT_POLL_INTERVAL_MS; + const requestTimeoutMs = + toFiniteNumber(process.env.PROMETHEUS_REQUEST_TIMEOUT_MS) || DEFAULT_REQUEST_TIMEOUT_MS; + const minPublishTimestampSeconds = toFiniteNumber(process.env.PROMETHEUS_MIN_PUBLISH_TS_SECONDS); + + const result = await waitForStressMetrics({ + prometheusUrl, + jobName, + instanceName, + minPublishTimestampSeconds, + timeoutMs, + pollIntervalMs, + requestTimeoutMs, + }); + + console.log( + `Prometheus metrics verified with query "${result.query}" for job "${jobName}" and instance "${instanceName}".` + ); +} + +module.exports = { + main, + waitForStressMetrics, + __testUtils: { + buildMetricQueries, + extractNumericValues, + normalizeBaseUrl, + queryPrometheus, + requestJson, + toFiniteNumber, + trimTrailingSlashes, + DEFAULT_REQUEST_TIMEOUT_MS, + }, +}; + +if (require.main === module) { + main().catch((error) => { + const safeMessage = error instanceof Error ? error.message : String(error); + console.error(`Prometheus verification failed: ${safeMessage}`); + process.exit(1); + }); +} diff --git a/tests/catalog.md b/tests/catalog.md index a6d74e4..b39a899 100644 --- a/tests/catalog.md +++ b/tests/catalog.md @@ -6,6 +6,8 @@ Purpose: quick map of what is covered, why it exists, and which command to run. - Full tests: `npm test -- --runInBand` - Stress metrics summary (+ optional Pushgateway publish): `npm run stress:metrics` +- Stress publish verification in Prometheus: `npm run prometheus:verify` +- End-to-end perf metrics job (`TOOLS_DOMAIN` aware): `npm run perf:test` or `make perf-test` - Lint: `npm run lint` - Markdown docs lint (links/images/icons): `npm run lint:md` - Electron E2E (Playwright): `npm run e2e:playwright` @@ -55,6 +57,8 @@ Stress benchmark outputs: - Prometheus text format: `dist/benchmarks/stress-metrics.prom` - CI artifact: `stress-benchmarks-linux` - Optional publish target: set `PUSHGATEWAY_URL` (and optional `PUSHGATEWAY_JOB`, `PUSHGATEWAY_INSTANCE`, `PUSHGATEWAY_STRICT=true`) +- Optional Prometheus verification tuning: `PROMETHEUS_VERIFY_TIMEOUT_MS`, `PROMETHEUS_VERIFY_POLL_INTERVAL_MS`, `PROMETHEUS_REQUEST_TIMEOUT_MS` +- TOOLS domain mode: set `TOOLS_DOMAIN` (for example `.114.be.tn`) and run `make perf-test` to derive `https://pushgateway.$TOOLS_DOMAIN` and `https://prometheus.$TOOLS_DOMAIN`, publish, then verify scrape visibility ## Electron E2E Tests diff --git a/tests/unit/scripts/perf-metrics-job.test.js b/tests/unit/scripts/perf-metrics-job.test.js new file mode 100644 index 0000000..5171eff --- /dev/null +++ b/tests/unit/scripts/perf-metrics-job.test.js @@ -0,0 +1,125 @@ +const { runPerfMetricsJob, __testUtils } = require('../../../scripts/run-perf-metrics-job'); + +describe('run-perf-metrics-job helpers', () => { + test('normalizes TOOLS_DOMAIN values with prefixes and leading dots', () => { + expect(__testUtils.normalizeToolsDomain('.114.be.tn')).toBe('114.be.tn'); + expect(__testUtils.normalizeToolsDomain('https://example.internal/path')).toBe( + 'example.internal' + ); + expect(__testUtils.normalizeToolsDomain('example.internal:8443')).toBe('example.internal'); + }); + + test('trims only boundary dots from host values', () => { + expect(__testUtils.trimBoundaryDots('.example.internal.')).toBe('example.internal'); + expect(__testUtils.trimBoundaryDots('...a.b.c...')).toBe('a.b.c'); + }); + + test('rejects invalid TOOLS_DOMAIN values', () => { + expect(() => __testUtils.normalizeToolsDomain('http://')).toThrow( + 'Invalid TOOLS_DOMAIN value' + ); + }); + + test('resolves monitoring endpoints from TOOLS_DOMAIN', () => { + const endpoints = __testUtils.resolveMonitoringEndpoints({ + TOOLS_DOMAIN: '.114.be.tn', + PUSHGATEWAY_URL: '', + PROMETHEUS_URL: '', + }); + + expect(endpoints).toEqual({ + toolsDomain: '114.be.tn', + pushgatewayUrl: 'https://pushgateway.114.be.tn/', + prometheusUrl: 'https://prometheus.114.be.tn/', + }); + }); + + test('normalizes explicit endpoint URLs without schemes', () => { + const endpoints = __testUtils.resolveMonitoringEndpoints({ + PUSHGATEWAY_URL: 'pushgateway.internal', + PROMETHEUS_URL: 'prometheus.internal', + }); + + expect(endpoints.pushgatewayUrl).toBe('https://pushgateway.internal/'); + expect(endpoints.prometheusUrl).toBe('https://prometheus.internal/'); + }); + + test('redacts credentials and query parameters in endpoint logs', async () => { + const executedCommands = []; + const log = jest.fn(); + + await runPerfMetricsJob({ + env: { + PUSHGATEWAY_URL: 'https://user:secret@pushgateway.example.com/base?token=abc', + PROMETHEUS_URL: 'https://prometheus.example.com/api?token=xyz', + PUSHGATEWAY_JOB: 'ai_code_fusion_stress', + }, + nowFn: () => 1_700_000_000_000, + hostName: 'dev-host', + execFn: (command, options) => { + executedCommands.push({ command, env: options.env }); + }, + verifyFn: async () => {}, + logger: { log }, + }); + + expect(executedCommands).toHaveLength(2); + const logOutput = log.mock.calls.map((call) => call[0]).join('\n'); + expect(logOutput).toContain('Pushgateway endpoint: https://pushgateway.example.com'); + expect(logOutput).toContain('Prometheus endpoint: https://prometheus.example.com'); + expect(logOutput).not.toContain('secret'); + expect(logOutput).not.toContain('token='); + }); + + test('runs stress publish flow and validates Prometheus', async () => { + const executedCommands = []; + const verifyCalls = []; + const log = jest.fn(); + + await runPerfMetricsJob({ + env: { + TOOLS_DOMAIN: '.114.be.tn', + PUSHGATEWAY_JOB: 'ai_code_fusion_stress', + }, + nowFn: () => 1_700_000_000_000, + hostName: 'dev-host', + execFn: (command, options) => { + executedCommands.push({ command, env: options.env }); + }, + verifyFn: async (options) => { + verifyCalls.push(options); + }, + logger: { log }, + }); + + expect(executedCommands.map((entry) => entry.command)).toEqual([ + 'npm run test:stress', + 'npm run stress:metrics', + ]); + + const stressMetricsEnvironment = executedCommands[1].env; + expect(stressMetricsEnvironment.PUSHGATEWAY_URL).toBe('https://pushgateway.114.be.tn/'); + expect(stressMetricsEnvironment.PROMETHEUS_URL).toBe('https://prometheus.114.be.tn/'); + expect(stressMetricsEnvironment.STRESS_METRICS_PUBLISH_TS_SECONDS).toBe('1700000000'); + + expect(verifyCalls).toHaveLength(1); + expect(verifyCalls[0]).toMatchObject({ + prometheusUrl: 'https://prometheus.114.be.tn/', + jobName: 'ai_code_fusion_stress', + instanceName: 'local-dev-host-1700000000000', + minPublishTimestampSeconds: 1_700_000_000, + }); + }); + + test('fails fast when monitoring endpoints cannot be resolved', async () => { + await expect(runPerfMetricsJob({ env: {} })).rejects.toThrow( + 'Unable to resolve monitoring endpoints' + ); + }); + + test('fails fast when TOOLS_DOMAIN cannot be normalized', async () => { + await expect(runPerfMetricsJob({ env: { TOOLS_DOMAIN: 'http://' } })).rejects.toThrow( + 'Invalid TOOLS_DOMAIN value' + ); + }); +}); diff --git a/tests/unit/scripts/publish-stress-metrics.test.js b/tests/unit/scripts/publish-stress-metrics.test.js new file mode 100644 index 0000000..54aca9e --- /dev/null +++ b/tests/unit/scripts/publish-stress-metrics.test.js @@ -0,0 +1,75 @@ +const { __testUtils } = require('../../../scripts/publish-stress-metrics'); + +describe('publish-stress-metrics helpers', () => { + test('keeps only the latest record per scenario for metric publishing', () => { + const records = [ + { + scenario: 'fs:getDirectoryTree-large-flat', + sourceFile: 'ipc-latency-1000.json', + capturedAt: '2026-02-10T10:00:00.000Z', + capturedAtMs: 1000, + p50Ms: 40, + p95Ms: 60, + p99Ms: 75, + sampleCount: 5, + fileCount: 5000, + iterations: null, + }, + { + scenario: 'fs:getDirectoryTree-large-flat', + sourceFile: 'ipc-latency-2000.json', + capturedAt: '2026-02-10T10:01:00.000Z', + capturedAtMs: 2000, + p50Ms: 42, + p95Ms: 65, + p99Ms: 76, + sampleCount: 5, + fileCount: 5000, + iterations: null, + }, + { + scenario: 'fs:getDirectoryTree-event-loop-lag', + sourceFile: 'event-loop-lag-1500.json', + capturedAt: '2026-02-10T10:00:30.000Z', + capturedAtMs: 1500, + p50Ms: 1, + p95Ms: 4, + p99Ms: 7, + sampleCount: 20, + fileCount: null, + iterations: 20, + }, + ]; + + const latestRecords = __testUtils.selectLatestRecordPerScenario(records); + expect(latestRecords).toHaveLength(2); + + const latestLatencyRecord = latestRecords.find( + (record) => record.scenario === 'fs:getDirectoryTree-large-flat' + ); + expect(latestLatencyRecord).toBeDefined(); + expect(latestLatencyRecord.sourceFile).toBe('ipc-latency-2000.json'); + }); + + test('includes publish timestamp metric in Prometheus payload', () => { + const payload = __testUtils.buildPrometheusPayload( + [ + { + scenario: 'fs:getDirectoryTree-event-loop-lag', + p50Ms: 1, + p95Ms: 4, + p99Ms: 7, + sampleCount: 20, + fileCount: null, + iterations: 20, + }, + ], + { publishTimestampSeconds: 1_700_000_000 } + ); + + expect(payload).toContain( + 'ai_code_fusion_stress_publish_timestamp_seconds 1700000000' + ); + expect(payload).toContain('percentile="p95",scenario="fs:getDirectoryTree-event-loop-lag"'); + }); +}); diff --git a/tests/unit/scripts/verify-prometheus-metrics.test.js b/tests/unit/scripts/verify-prometheus-metrics.test.js new file mode 100644 index 0000000..feaaf95 --- /dev/null +++ b/tests/unit/scripts/verify-prometheus-metrics.test.js @@ -0,0 +1,67 @@ +const { + waitForStressMetrics, + __testUtils, +} = require('../../../scripts/verify-prometheus-metrics'); + +describe('verify-prometheus-metrics helpers', () => { + test('builds strict query set for job and instance labels', () => { + const queries = __testUtils.buildMetricQueries( + 'ai_code_fusion_stress_publish_timestamp_seconds', + 'ai_code_fusion_stress', + 'local-dev-123' + ); + + expect(queries).toEqual([ + 'ai_code_fusion_stress_publish_timestamp_seconds{job="ai_code_fusion_stress",instance="local-dev-123"}', + 'ai_code_fusion_stress_publish_timestamp_seconds{exported_job="ai_code_fusion_stress",exported_instance="local-dev-123"}', + ]); + }); + + test('waits until Prometheus returns publish timestamp at or above the threshold', async () => { + const queryFn = jest.fn(async (prometheusUrl, query, options) => { + expect(options).toMatchObject({ requestTimeoutMs: 250 }); + if (query.includes('exported_job=')) { + return [1_700_000_005]; + } + + return []; + }); + + const result = await waitForStressMetrics({ + prometheusUrl: 'https://prometheus.example.internal', + jobName: 'ai_code_fusion_stress', + instanceName: 'local-dev-123', + minPublishTimestampSeconds: 1_700_000_000, + timeoutMs: 250, + pollIntervalMs: 50, + queryFn, + nowFn: () => 0, + sleepFn: async () => {}, + }); + + expect(result.query).toContain('exported_job='); + expect(result.values).toContain(1_700_000_005); + expect(queryFn).toHaveBeenCalled(); + }); + + test('times out when Prometheus never returns matching values', async () => { + let now = 0; + const queryFn = jest.fn(async () => []); + + await expect( + waitForStressMetrics({ + prometheusUrl: 'https://prometheus.example.internal', + jobName: 'ai_code_fusion_stress', + instanceName: 'local-dev-123', + minPublishTimestampSeconds: 1_700_000_000, + timeoutMs: 150, + pollIntervalMs: 50, + queryFn, + nowFn: () => now, + sleepFn: async (intervalMs) => { + now += intervalMs; + }, + }) + ).rejects.toThrow('Attempted queries:'); + }); +});