perf(ci): duration-weighted shard sequencer + reshard internal->12 packages->3

nicktrn · nicktrn · commit 273f1be077cb · 2026-06-06T18:51:10.000+01:00
diff --git a/.github/workflows/unit-tests-internal.yml b/.github/workflows/unit-tests-internal.yml
@@ -19,8 +19,8 @@ jobs:
       # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard
       fail-fast: false
       matrix:
-        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
-        shardTotal: [8]
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+        shardTotal: [12]
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       SHARD_INDEX: ${{ matrix.shardIndex }}
diff --git a/.github/workflows/unit-tests-packages.yml b/.github/workflows/unit-tests-packages.yml
@@ -16,9 +16,11 @@ jobs:
     name: "🧪 Unit Tests: Packages"
     runs-on: ubuntu-latest
     strategy:
+      # one flaky shard shouldn't cancel its siblings - lets us re-run only the failed shard
+      fail-fast: false
       matrix:
-        shardIndex: [1]
-        shardTotal: [1]
+        shardIndex: [1, 2, 3]
+        shardTotal: [3]
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       SHARD_INDEX: ${{ matrix.shardIndex }}
diff --git a/.gitignore b/.gitignore
@@ -72,4 +72,6 @@ apps/**/public/build
 .mcp.log
 .mcp.json
 .cursor/debug.log
-ailogger-output.log
+ailogger-output.log
+# per-package vitest timing capture (transient; merged into root test-timings.json)
+.vitest-timing.json
diff --git a/apps/webapp/vitest.config.ts b/apps/webapp/vitest.config.ts
@@ -1,8 +1,10 @@
 import { defineConfig } from "vitest/config";
+import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
 import tsconfigPaths from "vite-tsconfig-paths";
 
 export default defineConfig({
   test: {
+    sequence: { sequencer: DurationShardingSequencer },
     include: ["test/**/*.test.ts"],
     // *.e2e.test.ts: smoke matrix, run via vitest.e2e.config.ts.
     // *.e2e.full.test.ts: full auth suite, runs via vitest.e2e.full.config.ts
diff --git a/internal-packages/clickhouse/vitest.config.ts b/internal-packages/clickhouse/vitest.config.ts
@@ -1,7 +1,9 @@
 import { defineConfig } from "vitest/config";
+import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
 
 export default defineConfig({
   test: {
+    sequence: { sequencer: DurationShardingSequencer },
     include: ["**/*.test.ts"],
     globals: true,
     isolate: true,
diff --git a/internal-packages/llm-model-catalog/vitest.config.ts b/internal-packages/llm-model-catalog/vitest.config.ts
@@ -1,7 +1,9 @@
 import { defineConfig } from "vitest/config";
+import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
 
 export default defineConfig({
   test: {
+    sequence: { sequencer: DurationShardingSequencer },
     include: ["**/*.test.ts"],
     globals: true,
     isolate: true,
diff --git a/internal-packages/replication/vitest.config.ts b/internal-packages/replication/vitest.config.ts
@@ -1,7 +1,9 @@
 import { defineConfig } from "vitest/config";
+import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
 
 export default defineConfig({
   test: {
+    sequence: { sequencer: DurationShardingSequencer },
     include: ["**/*.test.ts"],
     globals: true,
     isolate: true,
diff --git a/internal-packages/run-engine/vitest.config.ts b/internal-packages/run-engine/vitest.config.ts
@@ -1,7 +1,9 @@
 import { defineConfig } from "vitest/config";
+import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
 
 export default defineConfig({
   test: {
+    sequence: { sequencer: DurationShardingSequencer },
     include: ["**/*.test.ts"],
     globals: true,
     // CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention
diff --git a/internal-packages/schedule-engine/vitest.config.ts b/internal-packages/schedule-engine/vitest.config.ts
@@ -1,7 +1,9 @@
 import { defineConfig } from "vitest/config";
+import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
 
 export default defineConfig({
   test: {
+    sequence: { sequencer: DurationShardingSequencer },
     globals: true,
     // CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention
     retry: process.env.CI ? 2 : 0,
diff --git a/internal-packages/testcontainers/package.json b/internal-packages/testcontainers/package.json
@@ -6,7 +6,8 @@
   "types": "./src/index.ts",
   "exports": {
     ".": "./src/index.ts",
-    "./webapp": "./src/webapp.ts"
+    "./webapp": "./src/webapp.ts",
+    "./sequencer": "./src/sequencer.ts"
   },
   "dependencies": {
     "@clickhouse/client": "^1.11.1",
diff --git a/internal-packages/testcontainers/src/sequencer.ts b/internal-packages/testcontainers/src/sequencer.ts
@@ -0,0 +1,79 @@
+import { existsSync, readFileSync } from "node:fs";
+import path from "node:path";
+import { BaseSequencer, type TestSpecification } from "vitest/node";
+
+// Walk up from the package dir (cwd at config-load time) to the monorepo root, identified by
+// pnpm-workspace.yaml. (Can't use __dirname - vitest bundles the config + this import as ESM.)
+function findRepoRoot(start: string): string {
+  let dir = start;
+  for (let i = 0; i < 20; i++) {
+    if (existsSync(path.join(dir, "pnpm-workspace.yaml"))) return dir;
+    const parent = path.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return start;
+}
+
+// test-timings.json lives at the monorepo root: { "<repo-relative path>": <ms> }
+const REPO_ROOT = findRepoRoot(process.cwd());
+const TIMINGS_PATH = path.resolve(REPO_ROOT, "test-timings.json");
+
+let cachedTimings: Record<string, number> | undefined;
+
+function loadTimings(): Record<string, number> {
+  if (!cachedTimings) {
+    cachedTimings = existsSync(TIMINGS_PATH)
+      ? (JSON.parse(readFileSync(TIMINGS_PATH, "utf-8")) as Record<string, number>)
+      : {};
+  }
+  return cachedTimings;
+}
+
+function median(nums: number[]): number {
+  if (nums.length === 0) return 1;
+  const sorted = [...nums].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2;
+}
+
+/**
+ * Duration-weighted interpretation of `--shard=i/N`. Instead of vitest's default file-count split,
+ * this greedily bin-packs test files by recorded duration (test-timings.json at the repo root;
+ * unknown/new files get the median) so each shard does roughly equal work.
+ *
+ * The packing is fully deterministic (sort by duration desc, then moduleId) so every shard computes
+ * the identical bins and just takes its own - no file runs twice or gets dropped. Falls back to the
+ * full set when no shard is configured, and to ~count-based when no timings exist.
+ */
+export class DurationShardingSequencer extends BaseSequencer {
+  async shard(specs: TestSpecification[]): Promise<TestSpecification[]> {
+    const shard = this.ctx.config.shard;
+    if (!shard || specs.length === 0) {
+      return specs;
+    }
+
+    const timings = loadTimings();
+    const fallback = median(Object.values(timings));
+
+    const weighted = specs
+      .map((spec) => ({
+        spec,
+        ms: timings[path.relative(REPO_ROOT, spec.moduleId)] ?? fallback,
+      }))
+      .sort((a, b) => b.ms - a.ms || a.spec.moduleId.localeCompare(b.spec.moduleId));
+
+    const bins = Array.from({ length: shard.count }, () => ({
+      total: 0,
+      specs: [] as TestSpecification[],
+    }));
+
+    for (const { spec, ms } of weighted) {
+      const lightest = bins.reduce((min, bin) => (bin.total < min.total ? bin : min));
+      lightest.total += ms;
+      lightest.specs.push(spec);
+    }
+
+    return bins[shard.index - 1]!.specs;
+  }
+}
diff --git a/packages/redis-worker/vitest.config.ts b/packages/redis-worker/vitest.config.ts
@@ -1,7 +1,9 @@
 import { defineConfig } from "vitest/config";
+import { DurationShardingSequencer } from "@internal/testcontainers/sequencer";
 
 export default defineConfig({
   test: {
+    sequence: { sequencer: DurationShardingSequencer },
     include: ["**/*.test.ts"],
     globals: true,
     // CI-only: absorbs timing races (real-clock waits vs worker poll interval) under shard CPU contention
diff --git a/test-timings.json b/test-timings.json
