Add TOOLS_DOMAIN perf metrics job and Prometheus verification

Makefile

@@ -5,7 +5,7 @@
 # Make these targets phony (they don't create files with these names)
 .PHONY: all setup dev clean clean-all build build-win build-linux \
         build-mac build-mac-arm build-mac-universal \
-        test css css-watch lint lint-md format validate qa docs-screenshots setup-hooks sonar \
+        test perf-test stress-metrics prometheus-verify css css-watch lint lint-md format validate qa docs-screenshots setup-hooks sonar \
         security gitleaks sbom renovate renovate-local mend-scan \
         icons sample-logo release
 
@@ -50,6 +50,15 @@ build-mac-universal: setup-scripts
 test: setup-scripts
 	@node scripts/index.js test
 
+perf-test: setup-scripts
+	@node scripts/index.js perf-test
+
+stress-metrics: setup-scripts
+	@node scripts/index.js stress-metrics
+
+prometheus-verify: setup-scripts
+	@node scripts/index.js prometheus-verify
+
 css: setup-scripts
 	@node scripts/index.js css
 

package.json

@@ -60,7 +60,9 @@
     "prebuild:mac-universal": "npm run build:ts && node scripts/prepare-build.js mac",
     "build:mac-universal": "cross-env NODE_ENV=production electron-builder --mac --universal",
     "build:linux": "cross-env NODE_ENV=production electron-builder --linux",
-    "stress:metrics": "node scripts/publish-stress-metrics.js"
+    "stress:metrics": "node scripts/publish-stress-metrics.js",
+    "prometheus:verify": "node scripts/verify-prometheus-metrics.js",
+    "perf:test": "node scripts/run-perf-metrics-job.js"
   },
   "author": "AI Code Fusion <m@codingworkflow.com>",
   "license": "GPL-3.0",

scripts/README.md

@@ -42,6 +42,10 @@ node scripts/index.js <command> [args...]
 
 - `test` - Run all tests
 - `test:watch` - Watch and run tests on changes
+- `test:stress` - Run stress benchmark tests
+- `stress:metrics` - Build stress summary and Prometheus payload
+- `prometheus:verify` - Verify stress metrics are visible in Prometheus
+- `perf-test` - Run stress tests, push metrics to Pushgateway, and verify Prometheus
 - `lint` - Run linter
 - `format` - Run code formatter
 - `validate` - Run all validation (lint + test)

scripts/index.js

@@ -108,6 +108,31 @@ async function executeCommand() {
         await utils.runNpmScript('test:watch');
         break;
 
+      case 'test:stress':
+      case 'stress-test':
+        await utils.runNpmScript('test:stress');
+        console.log('Stress tests completed successfully');
+        break;
+
+      case 'stress:metrics':
+      case 'stress-metrics':
+        await utils.runNpmScript('stress:metrics');
+        console.log('Stress metrics summary generated successfully');
+        break;
+
+      case 'prometheus:verify':
+      case 'prometheus-verify':
+        await utils.runNpmScript('prometheus:verify');
+        console.log('Prometheus stress metrics verification completed successfully');
+        break;
+
+      case 'perf':
+      case 'perf:test':
+      case 'perf-test':
+        await utils.runNpmScript('perf:test');
+        console.log('Performance metrics job completed successfully');
+        break;
+
       // Code quality commands
       case 'lint':
         await utils.runNpmScript('lint');

scripts/lib/utils.js

@@ -225,6 +225,10 @@ function printHelp() {
   console.log('Testing & Quality:');
   console.log('  test                 - Run tests');
   console.log('  test:watch           - Run tests in watch mode');
+  console.log('  test:stress          - Run stress benchmark tests');
+  console.log('  stress:metrics       - Build stress benchmark summary + Prometheus payload');
+  console.log('  prometheus:verify    - Verify pushed stress metrics in Prometheus');
+  console.log('  perf-test            - Run stress tests, push metrics, and verify Prometheus');
   console.log('  lint                 - Run linter');
   console.log('  lint:md             - Validate markdown links, image paths, and no decorative icons');
   console.log('  format               - Format code');

scripts/publish-stress-metrics.js

@@ -97,6 +97,7 @@ function normalizeBenchmarkRecord(filePath) {
     scenario,
     sourceFile: path.basename(filePath),
     capturedAt: sourceStats.mtime.toISOString(),
+    capturedAtMs: sourceStats.mtimeMs,
     p50Ms,
     p95Ms,
     p99Ms,
@@ -106,7 +107,61 @@ function normalizeBenchmarkRecord(filePath) {
   };
 }
 
-function buildPrometheusPayload(records) {
+function resolvePublishTimestampSeconds(providedValue) {
+  const fromArgument = toFiniteNumber(providedValue);
+  if (fromArgument !== null) {
+    return Math.floor(fromArgument);
+  }
+
+  const fromEnvironment = toFiniteNumber(process.env.STRESS_METRICS_PUBLISH_TS_SECONDS);
+  if (fromEnvironment !== null) {
+    return Math.floor(fromEnvironment);
+  }
+
+  return Math.floor(Date.now() / 1000);
+}
+
+function parseCapturedAtMs(record) {
+  if (toFiniteNumber(record.capturedAtMs) !== null) {
+    return Number(record.capturedAtMs);
+  }
+
+  const parsedDate = Date.parse(record.capturedAt);
+  return Number.isFinite(parsedDate) ? parsedDate : 0;
+}
+
+function selectLatestRecordPerScenario(records) {
+  const latestRecordsByScenario = new Map();
+
+  for (const record of records) {
+    const existingRecord = latestRecordsByScenario.get(record.scenario);
+    if (!existingRecord) {
+      latestRecordsByScenario.set(record.scenario, record);
+      continue;
+    }
+
+    const existingCapturedAt = parseCapturedAtMs(existingRecord);
+    const candidateCapturedAt = parseCapturedAtMs(record);
+    if (candidateCapturedAt > existingCapturedAt) {
+      latestRecordsByScenario.set(record.scenario, record);
+      continue;
+    }
+
+    if (
+      candidateCapturedAt === existingCapturedAt &&
+      String(record.sourceFile).localeCompare(String(existingRecord.sourceFile)) > 0
+    ) {
+      latestRecordsByScenario.set(record.scenario, record);
+    }
+  }
+
+  return Array.from(latestRecordsByScenario.values()).sort((leftRecord, rightRecord) =>
+    leftRecord.scenario.localeCompare(rightRecord.scenario)
+  );
+}
+
+function buildPrometheusPayload(records, options = {}) {
+  const publishTimestampSeconds = resolvePublishTimestampSeconds(options.publishTimestampSeconds);
   const lines = [
     `# HELP ${METRIC_PREFIX}_latency_ms Stress benchmark latency in milliseconds.`,
     `# TYPE ${METRIC_PREFIX}_latency_ms gauge`,
@@ -169,6 +224,18 @@ function buildPrometheusPayload(records) {
     }
   }
 
+  lines.push(
+    `# HELP ${METRIC_PREFIX}_publish_timestamp_seconds Unix timestamp when stress metrics were published.`
+  );
+  lines.push(`# TYPE ${METRIC_PREFIX}_publish_timestamp_seconds gauge`);
+  const publishLine = toMetricLine(
+    `${METRIC_PREFIX}_publish_timestamp_seconds`,
+    publishTimestampSeconds
+  );
+  if (publishLine) {
+    lines.push(publishLine);
+  }
+
   return `${lines.join('\n')}\n`;
 }
 
@@ -236,22 +303,27 @@ async function main() {
   const sortedRecords = records.sort((leftRecord, rightRecord) =>
     leftRecord.scenario.localeCompare(rightRecord.scenario)
   );
+  const latestRecords = selectLatestRecordPerScenario(sortedRecords);
 
   const summaryPayload = {
     generatedAt: new Date().toISOString(),
     benchmarkDirectory: path.relative(ROOT_DIR, BENCHMARK_DIR),
     benchmarkFiles: benchmarkFiles.map((filePath) => path.basename(filePath)).sort(),
     scenarios: sortedRecords,
+    latestScenarios: latestRecords,
   };
 
   fs.mkdirSync(BENCHMARK_DIR, { recursive: true });
   fs.writeFileSync(SUMMARY_FILE, JSON.stringify(summaryPayload, null, 2), 'utf8');
 
-  const prometheusPayload = buildPrometheusPayload(sortedRecords);
+  const prometheusPayload = buildPrometheusPayload(latestRecords);
   fs.writeFileSync(PROMETHEUS_FILE, prometheusPayload, 'utf8');
 
   console.log(`Stress summary written: ${path.relative(ROOT_DIR, SUMMARY_FILE)}`);
   console.log(`Prometheus metrics written: ${path.relative(ROOT_DIR, PROMETHEUS_FILE)}`);
+  console.log(
+    `Publishing ${latestRecords.length} unique scenario metric sets from ${sortedRecords.length} benchmark file records.`
+  );
 
   const pushgatewayUrl = (process.env.PUSHGATEWAY_URL || '').trim();
   if (!pushgatewayUrl) {
@@ -282,8 +354,27 @@ async function main() {
   }
 }
 
-main().catch((error) => {
-  const safeMessage = error instanceof Error ? error.message : String(error);
-  console.error(`Failed to publish stress metrics: ${safeMessage}`);
-  process.exit(1);
-});
+module.exports = {
+  main,
+  __testUtils: {
+    buildPrometheusPayload,
+    buildPushgatewayUrl,
+    normalizeBenchmarkRecord,
+    parseCapturedAtMs,
+    pickFirstNumber,
+    resolvePublishTimestampSeconds,
+    sanitizeLabelValue,
+    selectLatestRecordPerScenario,
+    toFiniteNumber,
+    toMetricLine,
+    trimTrailingSlashes,
+  },
+};
+
+if (require.main === module) {
+  main().catch((error) => {
+    const safeMessage = error instanceof Error ? error.message : String(error);
+    console.error(`Failed to publish stress metrics: ${safeMessage}`);
+    process.exit(1);
+  });
+}