From 92d8ea0cf1f192a2786dcf071435829f4badf66d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Jan 2026 00:00:20 +0000 Subject: [PATCH 01/28] Remove Redis dependency, use local JSON file for timing storage Replace external Redis database with a simple local JSON file storage: - Add src/backend/fileStorage.ts with same saveTimings/getTimings interface - Remove src/backend/redis.ts and redis package dependency - Update config.ts to use FAIRSPLICE_TIMINGS_FILE env var (default: .fairsplice-timings.json) - Update commands to use new file-based storage - Update README with new configuration and GitHub Actions cache example This eliminates the need for any external database while maintaining the same functionality (storing last 10 timings per test file). --- README.md | 26 ++++++++++++--- index.ts | 10 ++---- package.json | 3 +- src/backend/fileStorage.ts | 67 ++++++++++++++++++++++++++++++++++++++ src/backend/redis.ts | 61 ---------------------------------- src/commands/save.ts | 2 +- src/commands/split.ts | 2 +- src/config.ts | 4 +-- 8 files changed, 96 insertions(+), 79 deletions(-) create mode 100644 src/backend/fileStorage.ts delete mode 100644 src/backend/redis.ts diff --git a/README.md b/README.md index e71b8f7..553fc48 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ We found Github Actions lacking when compared to CircleCI which has [tests split There are a number of projects like [Split tests](https://github.com/marketplace/actions/split-tests) but they require uploading and downloading Junit XML files and merging them, or committing the Junit files to have them when running the tests. -This tool uses instead a Redis server to store the last 10 timings for each test file and uses the average of these to split tests. It is easy to setup if you have a Redis server running. +This tool stores test timings in a local JSON file, keeping the last 10 timings for each test file and using the average for splitting. No external database required! ## Installation -This project is built using [Bun](https://bun.sh) and [Redis](https://redis.io/). +This project is built using [Bun](https://bun.sh). Ensure you have Bun installed. To launch it, run @@ -23,12 +23,30 @@ bunx fairsplice ## Configuration -Before using Fairsplice, set the environment variable `FAIRSPLICE_REDIS_URL` to your Redis server URL. This is necessary for storing and retrieving test case information. +Fairsplice stores timings in a local JSON file (default: `.fairsplice-timings.json` in the current directory). + +You can customize the file path using the `FAIRSPLICE_TIMINGS_FILE` environment variable: ```bash -export FAIRSPLICE_REDIS_URL='redis://myuser:mypassword@your-redis-url.upstash.io:33683' +export FAIRSPLICE_TIMINGS_FILE='/path/to/my-timings.json' +``` + +### Using with GitHub Actions + +To persist timings across CI runs, you can use GitHub Actions cache: + +```yaml +- name: Cache test timings + uses: actions/cache@v4 + with: + path: .fairsplice-timings.json + key: fairsplice-timings-${{ github.ref }} + restore-keys: | + fairsplice-timings- ``` +Alternatively, you can commit the timings file to your repository for simpler persistence. + ## Usage Fairsplice supports two main commands: `save` and `split`. diff --git a/index.ts b/index.ts index 0f68c0f..cdde780 100755 --- a/index.ts +++ b/index.ts @@ -45,7 +45,8 @@ if (values.help || !command) { console.log(` Usage: fairsplice [save|split] [options] -Make sure the environment variable FAIRSPLICE_REDIS_URL is set. +Timings are stored in a local JSON file (default: .fairsplice-timings.json). +Set FAIRSPLICE_TIMINGS_FILE environment variable to customize the file path. fairsplice save --------------- @@ -69,13 +70,6 @@ Example: fairsplice split --pattern "test_*.py" --pattern "tests*.py" --total 3 process.exit(0); } -if (!process.env.FAIRSPLICE_REDIS_URL) { - console.error( - "Please set the FAIRSPLICE_REDIS_URL environment variable to use fairsplice." - ); - process.exit(1); -} - if (command === "save") { await save({ from: values.from }); process.exit(0); diff --git a/package.json b/package.json index 48a4f43..205613a 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,6 @@ "typescript": "^5.0.0" }, "dependencies": { - "fast-xml-parser": "^4.3.4", - "redis": "^4.6.13" + "fast-xml-parser": "^4.3.4" } } diff --git a/src/backend/fileStorage.ts b/src/backend/fileStorage.ts new file mode 100644 index 0000000..1dd8a3a --- /dev/null +++ b/src/backend/fileStorage.ts @@ -0,0 +1,67 @@ +import { average } from "../lib/average"; +import { NUMBER_OF_TIMINGS_TO_KEEP, TIMINGS_FILE_PATH } from "../config"; + +interface TimingsData { + version: number; + timings: Record; +} + +async function readTimingsFile(): Promise { + const file = Bun.file(TIMINGS_FILE_PATH); + if (!(await file.exists())) { + return { version: 1, timings: {} }; + } + try { + const content = await file.text(); + return JSON.parse(content) as TimingsData; + } catch { + // If file is corrupted or invalid, start fresh + return { version: 1, timings: {} }; + } +} + +async function writeTimingsFile(data: TimingsData): Promise { + await Bun.write(TIMINGS_FILE_PATH, JSON.stringify(data, null, 2)); +} + +export async function saveTimings( + timingByFile: Record +): Promise { + const data = await readTimingsFile(); + + for (const [file, timing] of Object.entries(timingByFile)) { + // Initialize array if doesn't exist + if (!data.timings[file]) { + data.timings[file] = []; + } + + // Add new timing at the beginning (like Redis LPUSH) + data.timings[file].unshift(timing); + + // Keep only the last NUMBER_OF_TIMINGS_TO_KEEP timings (like Redis LTRIM) + if (data.timings[file].length > NUMBER_OF_TIMINGS_TO_KEEP) { + data.timings[file] = data.timings[file].slice( + 0, + NUMBER_OF_TIMINGS_TO_KEEP + ); + } + } + + await writeTimingsFile(data); +} + +export async function getTimings( + files: string[] +): Promise> { + const data = await readTimingsFile(); + + const timingByFile: Record = {}; + for (const file of files) { + const timings = data.timings[file]; + if (timings && timings.length > 0) { + timingByFile[file] = average(timings); + } + } + + return timingByFile; +} diff --git a/src/backend/redis.ts b/src/backend/redis.ts deleted file mode 100644 index 8252176..0000000 --- a/src/backend/redis.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { createClient } from "redis"; -import { average } from "../lib/average"; -import { - NUMBER_OF_TIMINGS_TO_KEEP, - REDIS_KEY_PREFIX, - REDIS_URL, -} from "../config"; - -async function getClient() { - const client = createClient({ url: REDIS_URL, socket: { tls: true } }); - await client.connect(); - return client; -} - -function getKey(file: string) { - return `${REDIS_KEY_PREFIX}:${file}`; -} - -export async function saveTimings(timingByFile: Record) { - const client = await getClient(); - const transaction = client.multi(); - for (const [file, timing] of Object.entries(timingByFile)) { - const key = getKey(file); - // first we push the new timing - transaction.lPush(key, timing.toString()); - // then we trim the list to keep only the last TIMINGS_TO_KEEP timings - transaction.lTrim(key, 0, NUMBER_OF_TIMINGS_TO_KEEP - 2); - // then we set the expiration time for the key (30 days in seconds) - transaction.expire(key, 2592000); - } - await transaction.exec(); -} - -export async function getTimings(files: string[]) { - const client = await getClient(); - // fetch the last NUMBER_OF_TIMINGS_TO_KEEP timings for each file - const transaction = client.multi(); - for (const file of files) { - const key = getKey(file); - transaction.lRange(key, 0, NUMBER_OF_TIMINGS_TO_KEEP - 1); - } - const results = await transaction.exec(); - - // convert results to a map of file -> average timing - const timingByFile: Record = {}; - for (const [i, file] of files.entries()) { - const result = results[i]; - if ( - typeof result === "number" || - typeof result === "string" || - result?.length === 0 || - !result - ) { - continue; - } - const timings = Array.from(result).map(Number); - const timing = average(timings); - timingByFile[file] = timing; - } - return timingByFile; -} diff --git a/src/commands/save.ts b/src/commands/save.ts index 50ba728..47c272c 100644 --- a/src/commands/save.ts +++ b/src/commands/save.ts @@ -1,4 +1,4 @@ -import { saveTimings } from "../backend/redis"; +import { saveTimings } from "../backend/fileStorage"; import { parseJunit } from "../lib/junit"; export async function save({ from }: { from: string | undefined }) { diff --git a/src/commands/split.ts b/src/commands/split.ts index 0a80e51..3eff86a 100644 --- a/src/commands/split.ts +++ b/src/commands/split.ts @@ -1,5 +1,5 @@ import { Glob } from "bun"; -import { getTimings } from "../backend/redis"; +import { getTimings } from "../backend/fileStorage"; import { splitFiles } from "../lib/splitFiles"; import { DEFAULT_TIMING_IF_MISSING } from "../config"; diff --git a/src/config.ts b/src/config.ts index a954339..daefe14 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,4 @@ -export const REDIS_KEY_PREFIX = "fairsplice:timings"; export const NUMBER_OF_TIMINGS_TO_KEEP = 10; -export const REDIS_URL = process.env.FAIRSPLICE_REDIS_URL; export const DEFAULT_TIMING_IF_MISSING = 10000; +export const TIMINGS_FILE_PATH = + process.env.FAIRSPLICE_TIMINGS_FILE || ".fairsplice-timings.json"; From 82932618e103fdafbd58f193f25aca700129fe2b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Jan 2026 07:11:33 +0000 Subject: [PATCH 02/28] Add GitHub Actions tests with dummy test files - Add GitHub Actions workflow (test.yml) with three jobs: - unit-tests: runs existing unit tests in src/ - dummy-tests: runs dummy tests, saves timings, and splits for workers - fairsplice-integration: matrix job testing split across 3 workers - Add dummy test files with varying execution times: - fast.test.ts: ~100ms total (50ms, 30ms, 20ms delays) - medium.test.ts: ~450ms total (150ms, 200ms, 100ms delays) - slow.test.ts: ~900ms total (300ms, 350ms, 250ms delays) - variable.test.ts: ~600ms total (25ms, 175ms, 400ms delays) - Add fileStorage.test.ts to test the JSON file storage backend --- .github/workflows/test.yml | 123 ++++++++++++++++++++++++++++++++ bun.lockb | Bin 7590 -> 3949 bytes src/backend/fileStorage.test.ts | 88 +++++++++++++++++++++++ tests/dummy/fast.test.ts | 19 +++++ tests/dummy/medium.test.ts | 19 +++++ tests/dummy/slow.test.ts | 19 +++++ tests/dummy/variable.test.ts | 19 +++++ 7 files changed, 287 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 src/backend/fileStorage.test.ts create mode 100644 tests/dummy/fast.test.ts create mode 100644 tests/dummy/medium.test.ts create mode 100644 tests/dummy/slow.test.ts create mode 100644 tests/dummy/variable.test.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..67459e2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,123 @@ +name: Tests + +on: + push: + branches: [main, 'claude/**'] + pull_request: + branches: [main] + +jobs: + unit-tests: + name: Unit Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest + + - name: Install dependencies + run: bun install + + - name: Run unit tests + run: bun test src/ + + dummy-tests: + name: Dummy Tests with Variable Timings + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest + + - name: Install dependencies + run: bun install + + - name: Run dummy tests and generate JUnit report + run: bun test tests/dummy/ --reporter=junit --reporter-outfile=junit-results.xml + + - name: Display test results + run: cat junit-results.xml + + - name: Save timings with fairsplice + run: bun run index.ts save --from junit-results.xml + + - name: Display saved timings + run: cat .fairsplice-timings.json + + - name: Split tests for 2 workers + run: | + bun run index.ts split --pattern "tests/dummy/*.test.ts" --total 2 --out split-2.json + echo "Split for 2 workers:" + cat split-2.json + + - name: Split tests for 3 workers + run: | + bun run index.ts split --pattern "tests/dummy/*.test.ts" --total 3 --out split-3.json + echo "Split for 3 workers:" + cat split-3.json + + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + with: + name: test-results + path: | + junit-results.xml + .fairsplice-timings.json + split-2.json + split-3.json + + fairsplice-integration: + name: Fairsplice Integration Test + runs-on: ubuntu-latest + strategy: + matrix: + worker: [0, 1, 2] + steps: + - uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest + + - name: Install dependencies + run: bun install + + - name: Run initial tests to gather timings + run: bun test tests/dummy/ --reporter=junit --reporter-outfile=junit-results.xml + + - name: Save timings + run: bun run index.ts save --from junit-results.xml + + - name: Split tests for 3 workers + run: bun run index.ts split --pattern "tests/dummy/*.test.ts" --total 3 --out split.json + + - name: Display worker ${{ matrix.worker }} tests + run: | + echo "Worker ${{ matrix.worker }} assigned tests:" + cat split.json | bun -e " + const split = JSON.parse(require('fs').readFileSync('/dev/stdin', 'utf8')); + const workerTests = split[${{ matrix.worker }}] || []; + console.log('Files to run:', workerTests); + console.log('Number of files:', workerTests.length); + " + + - name: Run worker ${{ matrix.worker }} tests + run: | + TESTS=$(cat split.json | bun -e " + const split = JSON.parse(require('fs').readFileSync('/dev/stdin', 'utf8')); + const workerTests = split[${{ matrix.worker }}] || []; + console.log(workerTests.join(' ')); + ") + if [ -n "$TESTS" ]; then + echo "Running tests: $TESTS" + bun test $TESTS + else + echo "No tests assigned to this worker" + fi diff --git a/bun.lockb b/bun.lockb index aa480fd814a8c3a400e8854abc411f017669dc62..daa370569f1ae24f2dd0eaf6e6409e600828480d 100755 GIT binary patch delta 1090 zcmbVLO-NKx6ux(6bjCOTI{BQzG_n*;pKq*bG%a*d3dta~2>KD!o+gYkym^We)ksA@ zWGH825wrEN~IwqNdJy^IkZ#>%e>8z32PRx%ZrV?`-q8 zvZx-ungF zN%k)=ma4!ZVF1yKnAFtaks;lu`x`ihqJP4`QA8)s*AZQa*67JgotwY(-`<|8-EmRr zg#>Tp9Omrpu*Mz1CQRFcvb^kyjCM?Bg*jeD3jcYW9YBRm_Y*Vit=E|)^z1?gSIQc+ zS)%@5IF96apIf8;W)wB&c-!)#!K>({3=SiKxeLKuJ@^zAdr(2;lVNGP6`2aZ&B9n4 zD$Ljr?xG%^aoahn3B(*CoN=~pq^z*yBr1bcohygRgK-y8=>ic}aw$7h>j}FB5hmTO@Q{lzeJ?h+tfB$ZLy zs2$X{eEdj~G9-6MQYn#ymZmP6QtB?r2`S`*5%R(bj{=qD=%Kr;-{)8P)lUw}6W<+{ zc#ozeHO2zq-V_W}7`%CpeY`sYv&CC>n-Adh*MT>e3s<*VcvzC9hA3m^Q<$zlI$8O6 zZA!*++6vk_2U#qs^5F|2>oQPiNh(95iYo0N(-d{MCw^E{ z2ZskBE>^*eD2M7$OONhCW8^g-^JG#++)EF97G)?c2@2YHLP_?ig9!~5OFQO&gqFEa SUju)AvHteaL&+GP(##)Jv%%W{ literal 7590 zcmeHM3s_Xu7CykBND2m|h%ZpmRA!h5PenyTP*D*@@DT_MXU4&Kz|8Q{R1^`=GDRp% z(-0_y)O-|INbyPKqZBPf@j)8JGc`=E54>xgIUDAByD)U`_uanxwfHvc>~r?NUVHDg zH;ds8&T$&GbEHh^Oe^E%b9m( z534J+9oSYnWt#PgCl$?oOLBwu_WYpgOuxY}2*_H;D6!b#{rRg~mN!Z-q+2x%%R85ov@^Ow zw%K{%+ao?3@7YV`RbC(Xe(BCvlbyz-(vjf)e?&NoNL*+l@EhEwdO(9%8$<-ZA4X3B zJh8{rAT&i0d<%^CV)2Nz=q7#!D0XA*HxSAUBl_0>-W%{R{kR?sCW1c+_^GV@7z6UY zD?wx&Zl`g2c$@=6h~THdgFA~yTzBzr0^U7xd%$!#O~1 z2oYVA0p!BkkGSrxzj(mA>fyT^zr%nh^N0Ee;x7Y;*lz+3qy3m6(RR!`h7iHK1AZ*v zF|R@XYR|Y`$sqU@fS&|-)NKc?yYV{s8C|JhQ8=%`3>sxbio!k;odt4~`!MDtR*LGXG!Qaf@APcZX1k&CF_5|KclatmjP6H9hGtZ`j1E@9&*&9Fnp% zFJxWT6MoAeJ-ImdggCc)`euBuF4}V9@;CjRe2&c8lIhdu-rF&+I*1)g6!}xmJ--w5 zaaG->OV!&aDS5Q|<~zpwQ*xb-7Slmxso{RyMi4~epuev;X`@ocHc$E~?ics9)((G*PiH-tnVp*O_^563+|$DP${i`q zdUEyi702gnOTU4OL+cK%UpB%yed&VoUgh-TxSG(H?52Iwx2f3HqO!KyJjwA%<_P^tiX5u zQ!c(?2iSWj{n@r_|gHJyW+YHOi@I zRsZQ@U#9J*CKrYrn15Vj&NGR%6Q(!txDSu1J$)BB2q2%g+pC|CIfE1_{oKrzTO{+A z+g12&7_#$Gc1^~&3-Zze7FSu7+a4Zp!J#fw5?pG$$oR4Qq|@^s>~z_-b8B3DX!zaL zyoJkkIZn6VFwYU|uL?vTk>)`mubAQE4=`?eEjWchXXVe7-fq9aZx)Cn!_ zD{6yizkRX;2W|#X8+(yzTw_#M$qln(C{= zBVFk`BQwn$TTe}@wo2T+8FMkQm*jUenzO1$M)i!6VKJ@Sd)Xx1=AVzLJuMAhFsN+W z673`-ev{EEYQWEX#2Ds)ba+%# z=KqSMepx;UB69IOB*ZC*+rIac85ND*{tYuP8kKQEx18ORHg0C3bXLoplPmKVT|Ht` zcrW*<1-<8k&u&(1w(K2O6%zY~(ZQ(o!lVx`ja)6@kAZ(5(n;V*l>QXSunTMlI&alo~D6uXv=I#tTx@^aOh$ z-(DkIMyn)F_Ad4ja+y@AlIY(z+MeqOExe8F`hpV=5X4WbU;{=oT910nv18kAt8iny4RIGO3&O zH41a(@p=4?N+7AcnHkF?v^10$DN-|}*&3a(QJ4!)#B-r2Qk^8#Kx1@;p>X&OHYbz% zp0SaDCqy-r92!I+)fs7fL#;=njkMI}{Boirg}ykYXtA5zyeHWKkX8GlG6mQ+5Cja+oX zI??lD3_2|MBVR}@m()&Se$XFql~i>}6;&4lAr#0+eV5czjg3|c`8r+1gXlO+|t>#Th)Y58YgnX`6EmKM- z*b6$Au1CkxWi)9uVul}oVpE{b+GK+TdVFVjGtaAeaBPWI1D6$$8G6}%^&E+R>oM3& zIB#n`T+bUNTC|z5g$)a~A`dwO_KM*5whE(15+PG-V&pP0txbxdHDa|aM(fQyI>Y}1 zBoP`dH&G$y>Mlu3wAvWWSZ8N7Err^=I*F%@QABHaDzy||K^@?Q(<>=T3t zc4QaU=nDs^ewWK^&o1yU8p}BJ&-R0Bn79mzif&Rsi0*-bx)}z=sRbZ1W*!WLF{=YT z$5q1U6BwqCS#JPjH~=9vBM|l`PAN8CbZ3j{g@q52UU&oZ2PZ@q#^@3kfOQXy=dN8~ zIyN7g5C4*5P67mBhuN8(th4qpESMS3h!DD?3|6{Q6-Bd;37v(ZeGG_8&~SBm(l7^5 z1)z{|b(vS-iO4u*luRt^(h>b(AV$B-Wl}E_M_qQ*WyDUR(vjLgR7VE{iQor}Vp8Fs H@Yg>8B;(>f diff --git a/src/backend/fileStorage.test.ts b/src/backend/fileStorage.test.ts new file mode 100644 index 0000000..91a1f0a --- /dev/null +++ b/src/backend/fileStorage.test.ts @@ -0,0 +1,88 @@ +import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import { saveTimings, getTimings } from "./fileStorage"; +import { unlink } from "node:fs/promises"; +import { TIMINGS_FILE_PATH } from "../config"; + +describe("fileStorage", () => { + let originalFileContent: string | null = null; + + beforeEach(async () => { + // Backup existing file if present + try { + originalFileContent = await Bun.file(TIMINGS_FILE_PATH).text(); + } catch { + originalFileContent = null; + } + // Clean up before each test + try { + await unlink(TIMINGS_FILE_PATH); + } catch { + // File might not exist + } + }); + + afterEach(async () => { + // Clean up after test + try { + await unlink(TIMINGS_FILE_PATH); + } catch { + // File might not exist + } + // Restore original file if it existed + if (originalFileContent !== null) { + await Bun.write(TIMINGS_FILE_PATH, originalFileContent); + } + }); + + it("should save and retrieve timings", async () => { + const timings = { + "test1.ts": 100, + "test2.ts": 200, + "test3.ts": 300, + }; + + await saveTimings(timings); + const retrieved = await getTimings(["test1.ts", "test2.ts", "test3.ts"]); + + expect(retrieved).toEqual(timings); + }); + + it("should return empty object for non-existent files", async () => { + const retrieved = await getTimings(["nonexistent.ts"]); + expect(retrieved).toEqual({}); + }); + + it("should average multiple timing entries", async () => { + // Save first set of timings + await saveTimings({ "test.ts": 100 }); + // Save second set of timings + await saveTimings({ "test.ts": 200 }); + // Save third set of timings + await saveTimings({ "test.ts": 300 }); + + const retrieved = await getTimings(["test.ts"]); + // Average of [300, 200, 100] = 200 + expect(retrieved["test.ts"]).toBe(200); + }); + + it("should handle partial file requests", async () => { + await saveTimings({ + "test1.ts": 100, + "test2.ts": 200, + }); + + const retrieved = await getTimings(["test1.ts", "nonexistent.ts"]); + expect(retrieved).toEqual({ "test1.ts": 100 }); + }); + + it("should persist data to JSON file", async () => { + await saveTimings({ "myfile.ts": 42 }); + + // Verify file exists and contains valid JSON + const content = await Bun.file(TIMINGS_FILE_PATH).text(); + const data = JSON.parse(content); + + expect(data.version).toBe(1); + expect(data.timings["myfile.ts"]).toEqual([42]); + }); +}); diff --git a/tests/dummy/fast.test.ts b/tests/dummy/fast.test.ts new file mode 100644 index 0000000..3041a6d --- /dev/null +++ b/tests/dummy/fast.test.ts @@ -0,0 +1,19 @@ +import { describe, it, expect } from "bun:test"; + +// Fast tests - minimal delay +describe("Fast test suite", () => { + it("should complete quickly - test 1", async () => { + await Bun.sleep(50); + expect(1 + 1).toBe(2); + }); + + it("should complete quickly - test 2", async () => { + await Bun.sleep(30); + expect(true).toBe(true); + }); + + it("should complete quickly - test 3", async () => { + await Bun.sleep(20); + expect("hello").toContain("ell"); + }); +}); diff --git a/tests/dummy/medium.test.ts b/tests/dummy/medium.test.ts new file mode 100644 index 0000000..803a5f2 --- /dev/null +++ b/tests/dummy/medium.test.ts @@ -0,0 +1,19 @@ +import { describe, it, expect } from "bun:test"; + +// Medium duration tests +describe("Medium test suite", () => { + it("should take moderate time - test 1", async () => { + await Bun.sleep(150); + expect([1, 2, 3]).toHaveLength(3); + }); + + it("should take moderate time - test 2", async () => { + await Bun.sleep(200); + expect({ a: 1 }).toHaveProperty("a"); + }); + + it("should take moderate time - test 3", async () => { + await Bun.sleep(100); + expect(Math.max(1, 2, 3)).toBe(3); + }); +}); diff --git a/tests/dummy/slow.test.ts b/tests/dummy/slow.test.ts new file mode 100644 index 0000000..78ba1c8 --- /dev/null +++ b/tests/dummy/slow.test.ts @@ -0,0 +1,19 @@ +import { describe, it, expect } from "bun:test"; + +// Slow tests - longer delays +describe("Slow test suite", () => { + it("should take longer - test 1", async () => { + await Bun.sleep(300); + expect(Array.isArray([])).toBe(true); + }); + + it("should take longer - test 2", async () => { + await Bun.sleep(350); + expect(typeof "string").toBe("string"); + }); + + it("should take longer - test 3", async () => { + await Bun.sleep(250); + expect(null).toBeNull(); + }); +}); diff --git a/tests/dummy/variable.test.ts b/tests/dummy/variable.test.ts new file mode 100644 index 0000000..3942179 --- /dev/null +++ b/tests/dummy/variable.test.ts @@ -0,0 +1,19 @@ +import { describe, it, expect } from "bun:test"; + +// Variable timing tests +describe("Variable timing test suite", () => { + it("should handle variable timing - quick", async () => { + await Bun.sleep(25); + expect(Number.isInteger(42)).toBe(true); + }); + + it("should handle variable timing - medium", async () => { + await Bun.sleep(175); + expect(Object.keys({ a: 1, b: 2 })).toEqual(["a", "b"]); + }); + + it("should handle variable timing - slow", async () => { + await Bun.sleep(400); + expect(new Date()).toBeInstanceOf(Date); + }); +}); From 90acc61dff2276310d50bfc2c56e487653ccef27 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Jan 2026 07:16:17 +0000 Subject: [PATCH 03/28] Add workflow diagram to README Add ASCII diagram explaining the two-phase workflow: 1. Split phase: load timings, distribute tests via bin packing 2. Save phase: extract timings from JUnit XML, update storage Includes key concepts section explaining bin packing and rolling averages. --- README.md | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/README.md b/README.md index 553fc48..dcec440 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,76 @@ There are a number of projects like [Split tests](https://github.com/marketplace This tool stores test timings in a local JSON file, keeping the last 10 timings for each test file and using the average for splitting. No external database required! +## How It Works + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ CI PIPELINE │ +└─────────────────────────────────────────────────────────────────────────────┘ + + ┌─────────────────────┐ + │ 1. SPLIT PHASE │ + └─────────────────────┘ + + .fairsplice-timings.json fairsplice split + ┌──────────────────────┐ ┌─────────────────┐ + │ { │ │ │ + │ "test_a.py": [2.1],│ ──────▶ │ Load timings │ + │ "test_b.py": [5.3],│ │ + glob files │ + │ "test_c.py": [1.8] │ │ │ + │ } │ └────────┬────────┘ + └──────────────────────┘ │ + ▼ + ┌─────────────────────────┐ + │ Distribute tests by │ + │ timing (bin packing) │ + └─────────────────────────┘ + │ + ┌─────────────────────────────┼─────────────────────────────┐ + ▼ ▼ ▼ + ┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐ + │ Worker 0 │ │ Worker 1 │ │ Worker 2 │ + │ ["test_b.py"] │ │ ["test_a.py", │ │ ["test_c.py"] │ + │ ~5.3s │ │ "test_c.py"] │ │ ~1.8s │ + └─────────┬─────────┘ │ ~3.9s │ └─────────┬─────────┘ + │ └─────────┬─────────┘ │ + ▼ ▼ ▼ + ┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐ + │ Run tests │ │ Run tests │ │ Run tests │ + │ Output JUnit │ │ Output JUnit │ │ Output JUnit │ + └─────────┬─────────┘ └─────────┬─────────┘ └─────────┬─────────┘ + │ │ │ + └───────────────────────────┴───────────────────────────┘ + │ + ┌─────────────────────┐ │ + │ 2. SAVE PHASE │ │ + └─────────────────────┘ │ + ▼ + ┌─────────────────────────┐ + │ fairsplice save │ + │ --from junit.xml │ + └─────────────────────────┘ + │ + ▼ + ┌─────────────────────────┐ + │ Extract timings from │ + │ JUnit XML results │ + └─────────────────────────┘ + │ + ▼ + ┌──────────────────────┐ + │ .fairsplice-timings │ + │ Updated with new │ + │ timing data │◀─── Cached/committed + └──────────────────────┘ for next run +``` + +**Key concepts:** +- **Split phase**: Before tests run, fairsplice distributes test files across workers based on historical timing data +- **Save phase**: After tests complete, fairsplice extracts timing from JUnit XML and updates the timings file +- **Bin packing**: Tests are assigned to workers to balance total execution time (heaviest tests first) +- **Rolling average**: Keeps last 10 timings per test file, uses average for predictions + ## Installation This project is built using [Bun](https://bun.sh). From 7cbb4fef0691b08323cb3d97b022807cbd250a41 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Jan 2026 07:20:39 +0000 Subject: [PATCH 04/28] Fix integration test to properly simulate CI workflow The integration test was incorrectly running all tests in each worker before splitting. Now it properly simulates a real CI workflow: 1. integration-prepare: Restores cached timings (or generates once on first run), splits tests, uploads split config as artifact 2. integration-worker (x3): Downloads split config, runs ONLY assigned tests, uploads JUnit results 3. integration-finalize: Collects all worker results, saves updated timings to cache for next run This demonstrates how fairsplice is intended to be used: - Timings persist across CI runs via cache - Tests run only once (not duplicated across workers) - Each worker runs its fair share based on historical timings --- .github/workflows/test.yml | 166 ++++++++++++++++++++++++++++++++++--- 1 file changed, 153 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67459e2..a24cfe2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,12 +72,12 @@ jobs: split-2.json split-3.json - fairsplice-integration: - name: Fairsplice Integration Test + # Integration test: simulates real CI workflow with cached timings + integration-prepare: + name: Prepare Test Split runs-on: ubuntu-latest - strategy: - matrix: - worker: [0, 1, 2] + outputs: + has-timings: ${{ steps.check-cache.outputs.cache-hit }} steps: - uses: actions/checkout@v4 @@ -89,16 +89,72 @@ jobs: - name: Install dependencies run: bun install - - name: Run initial tests to gather timings - run: bun test tests/dummy/ --reporter=junit --reporter-outfile=junit-results.xml + # Try to restore timings from cache (simulates real CI) + - name: Restore timings cache + id: check-cache + uses: actions/cache@v4 + with: + path: .fairsplice-timings.json + key: fairsplice-timings-${{ github.ref }}-${{ github.run_number }} + restore-keys: | + fairsplice-timings-${{ github.ref }}- + fairsplice-timings- - - name: Save timings - run: bun run index.ts save --from junit-results.xml + # If no cached timings, generate initial ones + - name: Generate initial timings (first run only) + if: steps.check-cache.outputs.cache-hit != 'true' + run: | + echo "No cached timings found, generating initial timings..." + bun test tests/dummy/ --reporter=junit --reporter-outfile=junit-results.xml + bun run index.ts save --from junit-results.xml + echo "Initial timings:" + cat .fairsplice-timings.json + - name: Show cached timings + if: steps.check-cache.outputs.cache-hit == 'true' + run: | + echo "Using cached timings:" + cat .fairsplice-timings.json + + # Split tests for workers - name: Split tests for 3 workers - run: bun run index.ts split --pattern "tests/dummy/*.test.ts" --total 3 --out split.json + run: | + bun run index.ts split --pattern "tests/dummy/*.test.ts" --total 3 --out split.json + echo "Test split:" + cat split.json + + # Upload split file for workers to use + - name: Upload split configuration + uses: actions/upload-artifact@v4 + with: + name: test-split + path: split.json + + integration-worker: + name: Worker ${{ matrix.worker }} + needs: integration-prepare + runs-on: ubuntu-latest + strategy: + matrix: + worker: [0, 1, 2] + steps: + - uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest + + - name: Install dependencies + run: bun install + + # Download the split configuration from prepare job + - name: Download split configuration + uses: actions/download-artifact@v4 + with: + name: test-split - - name: Display worker ${{ matrix.worker }} tests + - name: Show assigned tests for worker ${{ matrix.worker }} run: | echo "Worker ${{ matrix.worker }} assigned tests:" cat split.json | bun -e " @@ -108,6 +164,7 @@ jobs: console.log('Number of files:', workerTests.length); " + # Run ONLY this worker's assigned tests - name: Run worker ${{ matrix.worker }} tests run: | TESTS=$(cat split.json | bun -e " @@ -116,8 +173,91 @@ jobs: console.log(workerTests.join(' ')); ") if [ -n "$TESTS" ]; then - echo "Running tests: $TESTS" - bun test $TESTS + echo "Running ONLY these tests: $TESTS" + bun test $TESTS --reporter=junit --reporter-outfile=junit-worker-${{ matrix.worker }}.xml else echo "No tests assigned to this worker" fi + + # Upload this worker's results + - name: Upload worker results + uses: actions/upload-artifact@v4 + with: + name: junit-worker-${{ matrix.worker }} + path: junit-worker-${{ matrix.worker }}.xml + if-no-files-found: ignore + + # Collect results and update timings for next run + integration-finalize: + name: Finalize & Update Timings + needs: integration-worker + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v1 + with: + bun-version: latest + + - name: Install dependencies + run: bun install + + # Restore existing timings + - name: Restore timings cache + uses: actions/cache@v4 + with: + path: .fairsplice-timings.json + key: fairsplice-timings-${{ github.ref }}-${{ github.run_number }} + restore-keys: | + fairsplice-timings-${{ github.ref }}- + fairsplice-timings- + + # Download all worker results + - name: Download worker 0 results + uses: actions/download-artifact@v4 + with: + name: junit-worker-0 + path: results/ + continue-on-error: true + + - name: Download worker 1 results + uses: actions/download-artifact@v4 + with: + name: junit-worker-1 + path: results/ + continue-on-error: true + + - name: Download worker 2 results + uses: actions/download-artifact@v4 + with: + name: junit-worker-2 + path: results/ + continue-on-error: true + + # Merge and save new timings + - name: Save updated timings from all workers + run: | + echo "Worker result files:" + ls -la results/ || echo "No results directory" + + for file in results/junit-worker-*.xml; do + if [ -f "$file" ]; then + echo "Saving timings from $file" + bun run index.ts save --from "$file" + fi + done + + echo "Updated timings:" + cat .fairsplice-timings.json || echo "No timings file" + + - name: Summary + run: | + echo "## Integration Test Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Timings have been updated and cached for the next run." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Current timings:" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + cat .fairsplice-timings.json >> $GITHUB_STEP_SUMMARY || echo "{}" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY From e4625b8364250932447beff61e126b3e0321b46e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Jan 2026 07:28:55 +0000 Subject: [PATCH 05/28] Fix integration test to not run all tests in prepare phase Remove the step that ran all tests to generate initial timings - this defeats the purpose of test splitting in a real scenario where the full suite could take hours. Now the workflow correctly handles cold starts: - First run: No cached timings, fairsplice uses default timing (10s) per test, tests are split evenly across workers - After run: Timings are saved from worker results - Subsequent runs: Cached timings enable optimal distribution This matches real-world usage where you can't afford to run the full test suite just to gather timing data. --- .github/workflows/test.yml | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a24cfe2..365ca36 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -100,21 +100,15 @@ jobs: fairsplice-timings-${{ github.ref }}- fairsplice-timings- - # If no cached timings, generate initial ones - - name: Generate initial timings (first run only) - if: steps.check-cache.outputs.cache-hit != 'true' + - name: Show timings status run: | - echo "No cached timings found, generating initial timings..." - bun test tests/dummy/ --reporter=junit --reporter-outfile=junit-results.xml - bun run index.ts save --from junit-results.xml - echo "Initial timings:" - cat .fairsplice-timings.json - - - name: Show cached timings - if: steps.check-cache.outputs.cache-hit == 'true' - run: | - echo "Using cached timings:" - cat .fairsplice-timings.json + if [ -f .fairsplice-timings.json ]; then + echo "Using cached timings from previous run:" + cat .fairsplice-timings.json + else + echo "No cached timings found - first run will use default timing (10s per test)" + echo "Timings will be collected after this run for future optimization" + fi # Split tests for workers - name: Split tests for 3 workers From a064866cf34c35ede1bbb28ec19043f22effef7b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Jan 2026 07:36:44 +0000 Subject: [PATCH 06/28] Simplify worker jobs - no fairsplice deps needed Workers only need to: 1. Download split.json artifact 2. Parse JSON to get test list (using jq) 3. Run their test framework Changes: - Remove 'bun install' from workers (not needed) - Use jq for JSON parsing (cleaner than inline bun script) - Add comments clarifying workers don't need fairsplice - Use GitHub outputs for cleaner conditional flow --- .github/workflows/test.yml | 41 ++++++++++++-------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 365ca36..69fbe0c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -124,6 +124,7 @@ jobs: name: test-split path: split.json + # Workers don't need fairsplice installed - they just read split.json and run tests integration-worker: name: Worker ${{ matrix.worker }} needs: integration-prepare @@ -139,47 +140,31 @@ jobs: with: bun-version: latest - - name: Install dependencies - run: bun install + # Note: No need to install fairsplice deps - workers just run tests - # Download the split configuration from prepare job - name: Download split configuration uses: actions/download-artifact@v4 with: name: test-split - - name: Show assigned tests for worker ${{ matrix.worker }} + - name: Get assigned tests + id: get-tests run: | - echo "Worker ${{ matrix.worker }} assigned tests:" - cat split.json | bun -e " - const split = JSON.parse(require('fs').readFileSync('/dev/stdin', 'utf8')); - const workerTests = split[${{ matrix.worker }}] || []; - console.log('Files to run:', workerTests); - console.log('Number of files:', workerTests.length); - " - - # Run ONLY this worker's assigned tests - - name: Run worker ${{ matrix.worker }} tests + TESTS=$(jq -r '.[${{ matrix.worker }}] | join(" ")' split.json) + echo "files=$TESTS" >> $GITHUB_OUTPUT + echo "Worker ${{ matrix.worker }} assigned: $TESTS" + + - name: Run tests + if: steps.get-tests.outputs.files != '' run: | - TESTS=$(cat split.json | bun -e " - const split = JSON.parse(require('fs').readFileSync('/dev/stdin', 'utf8')); - const workerTests = split[${{ matrix.worker }}] || []; - console.log(workerTests.join(' ')); - ") - if [ -n "$TESTS" ]; then - echo "Running ONLY these tests: $TESTS" - bun test $TESTS --reporter=junit --reporter-outfile=junit-worker-${{ matrix.worker }}.xml - else - echo "No tests assigned to this worker" - fi + bun test ${{ steps.get-tests.outputs.files }} --reporter=junit --reporter-outfile=junit-worker-${{ matrix.worker }}.xml - # Upload this worker's results - - name: Upload worker results + - name: Upload results + if: steps.get-tests.outputs.files != '' uses: actions/upload-artifact@v4 with: name: junit-worker-${{ matrix.worker }} path: junit-worker-${{ matrix.worker }}.xml - if-no-files-found: ignore # Collect results and update timings for next run integration-finalize: From 15e9f5c3c883ec432097f6a366c7ef9b78c8d035 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Jan 2026 04:55:42 +0000 Subject: [PATCH 07/28] Make --timings-file an explicit required parameter Replace implicit default file path with explicit --timings-file parameter: - Both save and split commands now require --timings-file - Remove TIMINGS_FILE_PATH from config and environment variable support - Update fileStorage functions to accept filePath as first argument - Update README with new usage examples --- README.md | 69 ++++++++++++++++++-------------------- index.ts | 57 ++++++++++++++++++++++--------- src/backend/fileStorage.ts | 21 +++++++----- src/commands/save.ts | 17 +++++----- src/commands/split.ts | 15 ++++----- src/config.ts | 2 -- 6 files changed, 101 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 553fc48..ac8fc1e 100644 --- a/README.md +++ b/README.md @@ -21,50 +21,25 @@ To launch it, run bunx fairsplice ``` -## Configuration - -Fairsplice stores timings in a local JSON file (default: `.fairsplice-timings.json` in the current directory). - -You can customize the file path using the `FAIRSPLICE_TIMINGS_FILE` environment variable: - -```bash -export FAIRSPLICE_TIMINGS_FILE='/path/to/my-timings.json' -``` - -### Using with GitHub Actions - -To persist timings across CI runs, you can use GitHub Actions cache: - -```yaml -- name: Cache test timings - uses: actions/cache@v4 - with: - path: .fairsplice-timings.json - key: fairsplice-timings-${{ github.ref }} - restore-keys: | - fairsplice-timings- -``` - -Alternatively, you can commit the timings file to your repository for simpler persistence. - ## Usage -Fairsplice supports two main commands: `save` and `split`. +Fairsplice supports two main commands: `save` and `split`. Both require a `--timings-file` parameter to specify where timings are stored. ### Saving test results To save test results: ```bash -fairsplice save --from +fairsplice save --timings-file --from ``` -- `--from `: Specify the file path to read test results from. +- `--timings-file `: JSON file to store timings (will be created if it doesn't exist) +- `--from `: JUnit XML file to read test results from Example: ```bash -fairsplice save --from results/junit.xml +fairsplice save --timings-file timings.json --from results/junit.xml ``` ### Splitting test cases @@ -72,21 +47,41 @@ fairsplice save --from results/junit.xml To split test cases for execution: ```bash -fairsplice split --pattern "" [--pattern "" ...] --total --out --replace-from --replace-to [--replace-from --replace-to ] +fairsplice split --timings-file --pattern "" --total --out ``` -- `--pattern ""`: Pattern to match test files. Can be used multiple times to specify multiple patterns. -- `--total `: Total number of workers in the test environment. -- `--out `: File to write split test files to (newline separated) -- `--replace-from `: Substring to replace in the file paths (can be used multiple times) -- `--replace-to `: Replacement for the substring (can be used multiple times but must match the number of --replace-from) +- `--timings-file `: JSON file with stored timings +- `--pattern ""`: Pattern to match test files (can be used multiple times) +- `--total `: Total number of workers +- `--out `: File to write split result to (JSON array of arrays) +- `--replace-from `: (Optional) Substring to replace in file paths +- `--replace-to `: (Optional) Replacement string Example: ```bash -fairsplice split --pattern "test_*.py" --pattern "tests*.py" --total 3 --out split.json +fairsplice split --timings-file timings.json --pattern "test_*.py" --total 3 --out split.json +``` + +## Using with GitHub Actions + +To persist timings across CI runs, use GitHub Actions cache: + +```yaml +- name: Cache test timings + uses: actions/cache@v4 + with: + path: timings.json + key: fairsplice-timings-${{ github.ref }} + restore-keys: | + fairsplice-timings- + +- name: Split tests + run: bunx fairsplice split --timings-file timings.json --pattern "tests/**/*.py" --total 3 --out split.json ``` +Alternatively, you can commit the timings file to your repository for simpler persistence. + ## Help For a detailed list of commands and options, use the help command: diff --git a/index.ts b/index.ts index cdde780..9748c00 100755 --- a/index.ts +++ b/index.ts @@ -11,6 +11,10 @@ const { positionals, values } = parseArgs({ type: "boolean", short: "h", }, + // common options + ["timings-file"]: { + type: "string", + }, // save options from: { type: "string", @@ -45,41 +49,64 @@ if (values.help || !command) { console.log(` Usage: fairsplice [save|split] [options] -Timings are stored in a local JSON file (default: .fairsplice-timings.json). -Set FAIRSPLICE_TIMINGS_FILE environment variable to customize the file path. - fairsplice save --------------- -Available options: - --from File to read test results from +Save test timings from a JUnit XML file. -Example: fairsplice save --from results/junit.xml +Required options: + --timings-file JSON file to store timings + --from JUnit XML file to read test results from + +Example: fairsplice save --timings-file timings.json --from results/junit.xml fairsplice split ------------------ -Available options: - --pattern Pattern to match test files (can be used multiple times) - --total Total number of workers - --out File to write test files to (JSON) - --replace-from Substring to replace in the file paths (can be used multiple times) - --replace-to Replacement for the substring (can be used multiple times but must match the number of --replace-from) +---------------- +Split test files across workers based on historical timings. + +Required options: + --timings-file JSON file with stored timings + --pattern Pattern to match test files (can be used multiple times) + --total Total number of workers + --out File to write split result to (JSON) + +Optional: + --replace-from Substring to replace in file paths (can be used multiple times) + --replace-to Replacement string (must match number of --replace-from) -Example: fairsplice split --pattern "test_*.py" --pattern "tests*.py" --total 3 --out split.json +Example: fairsplice split --timings-file timings.json --pattern "test_*.py" --total 3 --out split.json `); process.exit(0); } if (command === "save") { - await save({ from: values.from }); + if (!values["timings-file"] || !values.from) { + console.error( + "Error: --timings-file and --from are required for the save command." + ); + process.exit(1); + } + await save({ from: values.from, timingsFile: values["timings-file"] }); process.exit(0); } else if (command === "split") { + if ( + !values["timings-file"] || + !values.pattern || + !values.total || + !values.out + ) { + console.error( + "Error: --timings-file, --pattern, --total, and --out are required for the split command." + ); + process.exit(1); + } await split({ patterns: values.pattern, total: values.total, out: values.out, replaceFrom: values["replace-from"], replaceTo: values["replace-to"], + timingsFile: values["timings-file"], }); process.exit(0); } else { diff --git a/src/backend/fileStorage.ts b/src/backend/fileStorage.ts index 1dd8a3a..0f85396 100644 --- a/src/backend/fileStorage.ts +++ b/src/backend/fileStorage.ts @@ -1,13 +1,13 @@ import { average } from "../lib/average"; -import { NUMBER_OF_TIMINGS_TO_KEEP, TIMINGS_FILE_PATH } from "../config"; +import { NUMBER_OF_TIMINGS_TO_KEEP } from "../config"; interface TimingsData { version: number; timings: Record; } -async function readTimingsFile(): Promise { - const file = Bun.file(TIMINGS_FILE_PATH); +async function readTimingsFile(filePath: string): Promise { + const file = Bun.file(filePath); if (!(await file.exists())) { return { version: 1, timings: {} }; } @@ -20,14 +20,18 @@ async function readTimingsFile(): Promise { } } -async function writeTimingsFile(data: TimingsData): Promise { - await Bun.write(TIMINGS_FILE_PATH, JSON.stringify(data, null, 2)); +async function writeTimingsFile( + filePath: string, + data: TimingsData +): Promise { + await Bun.write(filePath, JSON.stringify(data, null, 2)); } export async function saveTimings( + filePath: string, timingByFile: Record ): Promise { - const data = await readTimingsFile(); + const data = await readTimingsFile(filePath); for (const [file, timing] of Object.entries(timingByFile)) { // Initialize array if doesn't exist @@ -47,13 +51,14 @@ export async function saveTimings( } } - await writeTimingsFile(data); + await writeTimingsFile(filePath, data); } export async function getTimings( + filePath: string, files: string[] ): Promise> { - const data = await readTimingsFile(); + const data = await readTimingsFile(filePath); const timingByFile: Record = {}; for (const file of files) { diff --git a/src/commands/save.ts b/src/commands/save.ts index 47c272c..a7ff8e7 100644 --- a/src/commands/save.ts +++ b/src/commands/save.ts @@ -1,14 +1,13 @@ import { saveTimings } from "../backend/fileStorage"; import { parseJunit } from "../lib/junit"; -export async function save({ from }: { from: string | undefined }) { - if (!from) { - console.warn( - "Please provide the --from option to specify the file to read test results from" - ); - process.exit(1); - } - +export async function save({ + from, + timingsFile, +}: { + from: string; + timingsFile: string; +}) { // read junit xml file const junitXmlFile = Bun.file(from); const xmlString = await junitXmlFile.text(); @@ -34,7 +33,7 @@ export async function save({ from }: { from: string | undefined }) { } // save timings - await saveTimings(timingByFile); + await saveTimings(timingsFile, timingByFile); console.log( "Timings saved for files:\n", Object.keys(timingByFile).join("\n - ") diff --git a/src/commands/split.ts b/src/commands/split.ts index 3eff86a..d30e4ed 100644 --- a/src/commands/split.ts +++ b/src/commands/split.ts @@ -9,18 +9,15 @@ export async function split({ replaceFrom, replaceTo, out, + timingsFile, }: { - patterns: string[] | undefined; - total: string | undefined; + patterns: string[]; + total: string; replaceFrom: string[] | undefined; replaceTo: string[] | undefined; - out: string | undefined; + out: string; + timingsFile: string; }) { - if (!patterns || !total || !out) { - console.warn("Please provide the --pattern and --total and --out flags."); - process.exit(1); - } - if (replaceFrom && replaceTo && replaceFrom.length !== replaceTo.length) { console.warn( "The number of --replace-from and --replace-to flags must match." @@ -47,7 +44,7 @@ export async function split({ } // get file times - const filesTimesMap = await getTimings(files); + const filesTimesMap = await getTimings(timingsFile, files); // warn if missing timings for (const file of files) { diff --git a/src/config.ts b/src/config.ts index daefe14..07aa06e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,2 @@ export const NUMBER_OF_TIMINGS_TO_KEEP = 10; export const DEFAULT_TIMING_IF_MISSING = 10000; -export const TIMINGS_FILE_PATH = - process.env.FAIRSPLICE_TIMINGS_FILE || ".fairsplice-timings.json"; From 5ff29e275143aa1ded63a9d714e4d99d93576683 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Jan 2026 09:53:07 +0000 Subject: [PATCH 08/28] Make --timings-file an explicit required parameter Replace implicit default file path with explicit --timings-file parameter: - Both save and split commands now require --timings-file - Remove TIMINGS_FILE_PATH from config and environment variable support - Update fileStorage functions to accept filePath as first argument - Update README with new usage examples and diagram --- README.md | 73 ++++++++++++++++++-------------------- index.ts | 57 +++++++++++++++++++++-------- src/backend/fileStorage.ts | 21 ++++++----- src/commands/save.ts | 17 +++++---- src/commands/split.ts | 15 ++++---- src/config.ts | 2 -- 6 files changed, 103 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index dcec440..54623ae 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ This tool stores test timings in a local JSON file, keeping the last 10 timings │ 1. SPLIT PHASE │ └─────────────────────┘ - .fairsplice-timings.json fairsplice split + timings.json fairsplice split ┌──────────────────────┐ ┌─────────────────┐ │ { │ │ │ │ "test_a.py": [2.1],│ ──────▶ │ Load timings │ @@ -68,7 +68,7 @@ This tool stores test timings in a local JSON file, keeping the last 10 timings │ ▼ ┌──────────────────────┐ - │ .fairsplice-timings │ + │ timings.json │ │ Updated with new │ │ timing data │◀─── Cached/committed └──────────────────────┘ for next run @@ -91,50 +91,25 @@ To launch it, run bunx fairsplice ``` -## Configuration - -Fairsplice stores timings in a local JSON file (default: `.fairsplice-timings.json` in the current directory). - -You can customize the file path using the `FAIRSPLICE_TIMINGS_FILE` environment variable: - -```bash -export FAIRSPLICE_TIMINGS_FILE='/path/to/my-timings.json' -``` - -### Using with GitHub Actions - -To persist timings across CI runs, you can use GitHub Actions cache: - -```yaml -- name: Cache test timings - uses: actions/cache@v4 - with: - path: .fairsplice-timings.json - key: fairsplice-timings-${{ github.ref }} - restore-keys: | - fairsplice-timings- -``` - -Alternatively, you can commit the timings file to your repository for simpler persistence. - ## Usage -Fairsplice supports two main commands: `save` and `split`. +Fairsplice supports two main commands: `save` and `split`. Both require a `--timings-file` parameter to specify where timings are stored. ### Saving test results To save test results: ```bash -fairsplice save --from +fairsplice save --timings-file --from ``` -- `--from `: Specify the file path to read test results from. +- `--timings-file `: JSON file to store timings (will be created if it doesn't exist) +- `--from `: JUnit XML file to read test results from Example: ```bash -fairsplice save --from results/junit.xml +fairsplice save --timings-file timings.json --from results/junit.xml ``` ### Splitting test cases @@ -142,21 +117,41 @@ fairsplice save --from results/junit.xml To split test cases for execution: ```bash -fairsplice split --pattern "" [--pattern "" ...] --total --out --replace-from --replace-to [--replace-from --replace-to ] +fairsplice split --timings-file --pattern "" --total --out ``` -- `--pattern ""`: Pattern to match test files. Can be used multiple times to specify multiple patterns. -- `--total `: Total number of workers in the test environment. -- `--out `: File to write split test files to (newline separated) -- `--replace-from `: Substring to replace in the file paths (can be used multiple times) -- `--replace-to `: Replacement for the substring (can be used multiple times but must match the number of --replace-from) +- `--timings-file `: JSON file with stored timings +- `--pattern ""`: Pattern to match test files (can be used multiple times) +- `--total `: Total number of workers +- `--out `: File to write split result to (JSON array of arrays) +- `--replace-from `: (Optional) Substring to replace in file paths +- `--replace-to `: (Optional) Replacement string Example: ```bash -fairsplice split --pattern "test_*.py" --pattern "tests*.py" --total 3 --out split.json +fairsplice split --timings-file timings.json --pattern "test_*.py" --total 3 --out split.json +``` + +## Using with GitHub Actions + +To persist timings across CI runs, use GitHub Actions cache: + +```yaml +- name: Cache test timings + uses: actions/cache@v4 + with: + path: timings.json + key: fairsplice-timings-${{ github.ref }} + restore-keys: | + fairsplice-timings- + +- name: Split tests + run: bunx fairsplice split --timings-file timings.json --pattern "tests/**/*.py" --total 3 --out split.json ``` +Alternatively, you can commit the timings file to your repository for simpler persistence. + ## Help For a detailed list of commands and options, use the help command: diff --git a/index.ts b/index.ts index cdde780..9748c00 100755 --- a/index.ts +++ b/index.ts @@ -11,6 +11,10 @@ const { positionals, values } = parseArgs({ type: "boolean", short: "h", }, + // common options + ["timings-file"]: { + type: "string", + }, // save options from: { type: "string", @@ -45,41 +49,64 @@ if (values.help || !command) { console.log(` Usage: fairsplice [save|split] [options] -Timings are stored in a local JSON file (default: .fairsplice-timings.json). -Set FAIRSPLICE_TIMINGS_FILE environment variable to customize the file path. - fairsplice save --------------- -Available options: - --from File to read test results from +Save test timings from a JUnit XML file. -Example: fairsplice save --from results/junit.xml +Required options: + --timings-file JSON file to store timings + --from JUnit XML file to read test results from + +Example: fairsplice save --timings-file timings.json --from results/junit.xml fairsplice split ------------------ -Available options: - --pattern Pattern to match test files (can be used multiple times) - --total Total number of workers - --out File to write test files to (JSON) - --replace-from Substring to replace in the file paths (can be used multiple times) - --replace-to Replacement for the substring (can be used multiple times but must match the number of --replace-from) +---------------- +Split test files across workers based on historical timings. + +Required options: + --timings-file JSON file with stored timings + --pattern Pattern to match test files (can be used multiple times) + --total Total number of workers + --out File to write split result to (JSON) + +Optional: + --replace-from Substring to replace in file paths (can be used multiple times) + --replace-to Replacement string (must match number of --replace-from) -Example: fairsplice split --pattern "test_*.py" --pattern "tests*.py" --total 3 --out split.json +Example: fairsplice split --timings-file timings.json --pattern "test_*.py" --total 3 --out split.json `); process.exit(0); } if (command === "save") { - await save({ from: values.from }); + if (!values["timings-file"] || !values.from) { + console.error( + "Error: --timings-file and --from are required for the save command." + ); + process.exit(1); + } + await save({ from: values.from, timingsFile: values["timings-file"] }); process.exit(0); } else if (command === "split") { + if ( + !values["timings-file"] || + !values.pattern || + !values.total || + !values.out + ) { + console.error( + "Error: --timings-file, --pattern, --total, and --out are required for the split command." + ); + process.exit(1); + } await split({ patterns: values.pattern, total: values.total, out: values.out, replaceFrom: values["replace-from"], replaceTo: values["replace-to"], + timingsFile: values["timings-file"], }); process.exit(0); } else { diff --git a/src/backend/fileStorage.ts b/src/backend/fileStorage.ts index 1dd8a3a..0f85396 100644 --- a/src/backend/fileStorage.ts +++ b/src/backend/fileStorage.ts @@ -1,13 +1,13 @@ import { average } from "../lib/average"; -import { NUMBER_OF_TIMINGS_TO_KEEP, TIMINGS_FILE_PATH } from "../config"; +import { NUMBER_OF_TIMINGS_TO_KEEP } from "../config"; interface TimingsData { version: number; timings: Record; } -async function readTimingsFile(): Promise { - const file = Bun.file(TIMINGS_FILE_PATH); +async function readTimingsFile(filePath: string): Promise { + const file = Bun.file(filePath); if (!(await file.exists())) { return { version: 1, timings: {} }; } @@ -20,14 +20,18 @@ async function readTimingsFile(): Promise { } } -async function writeTimingsFile(data: TimingsData): Promise { - await Bun.write(TIMINGS_FILE_PATH, JSON.stringify(data, null, 2)); +async function writeTimingsFile( + filePath: string, + data: TimingsData +): Promise { + await Bun.write(filePath, JSON.stringify(data, null, 2)); } export async function saveTimings( + filePath: string, timingByFile: Record ): Promise { - const data = await readTimingsFile(); + const data = await readTimingsFile(filePath); for (const [file, timing] of Object.entries(timingByFile)) { // Initialize array if doesn't exist @@ -47,13 +51,14 @@ export async function saveTimings( } } - await writeTimingsFile(data); + await writeTimingsFile(filePath, data); } export async function getTimings( + filePath: string, files: string[] ): Promise> { - const data = await readTimingsFile(); + const data = await readTimingsFile(filePath); const timingByFile: Record = {}; for (const file of files) { diff --git a/src/commands/save.ts b/src/commands/save.ts index 47c272c..a7ff8e7 100644 --- a/src/commands/save.ts +++ b/src/commands/save.ts @@ -1,14 +1,13 @@ import { saveTimings } from "../backend/fileStorage"; import { parseJunit } from "../lib/junit"; -export async function save({ from }: { from: string | undefined }) { - if (!from) { - console.warn( - "Please provide the --from option to specify the file to read test results from" - ); - process.exit(1); - } - +export async function save({ + from, + timingsFile, +}: { + from: string; + timingsFile: string; +}) { // read junit xml file const junitXmlFile = Bun.file(from); const xmlString = await junitXmlFile.text(); @@ -34,7 +33,7 @@ export async function save({ from }: { from: string | undefined }) { } // save timings - await saveTimings(timingByFile); + await saveTimings(timingsFile, timingByFile); console.log( "Timings saved for files:\n", Object.keys(timingByFile).join("\n - ") diff --git a/src/commands/split.ts b/src/commands/split.ts index 3eff86a..d30e4ed 100644 --- a/src/commands/split.ts +++ b/src/commands/split.ts @@ -9,18 +9,15 @@ export async function split({ replaceFrom, replaceTo, out, + timingsFile, }: { - patterns: string[] | undefined; - total: string | undefined; + patterns: string[]; + total: string; replaceFrom: string[] | undefined; replaceTo: string[] | undefined; - out: string | undefined; + out: string; + timingsFile: string; }) { - if (!patterns || !total || !out) { - console.warn("Please provide the --pattern and --total and --out flags."); - process.exit(1); - } - if (replaceFrom && replaceTo && replaceFrom.length !== replaceTo.length) { console.warn( "The number of --replace-from and --replace-to flags must match." @@ -47,7 +44,7 @@ export async function split({ } // get file times - const filesTimesMap = await getTimings(files); + const filesTimesMap = await getTimings(timingsFile, files); // warn if missing timings for (const file of files) { diff --git a/src/config.ts b/src/config.ts index daefe14..07aa06e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,2 @@ export const NUMBER_OF_TIMINGS_TO_KEEP = 10; export const DEFAULT_TIMING_IF_MISSING = 10000; -export const TIMINGS_FILE_PATH = - process.env.FAIRSPLICE_TIMINGS_FILE || ".fairsplice-timings.json"; From ecb03aaf860e33ba4f42ffc40d0c4a0089b739db Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 01:29:03 +0000 Subject: [PATCH 09/28] Add merge command to combine JUnit results from parallel workers New command: fairsplice merge --timings-file --prefix - Finds all files matching the prefix pattern (e.g., junit-*.xml) - Parses and aggregates timings from all matched JUnit XML files - Saves combined timings to the specified timings file This is useful in CI pipelines where tests run in parallel and each worker produces its own JUnit XML file. --- README.md | 30 ++++++++++++++++++----- index.ts | 31 +++++++++++++++++++++--- src/commands/merge.ts | 56 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 src/commands/merge.ts diff --git a/README.md b/README.md index 54623ae..5d5bb4f 100644 --- a/README.md +++ b/README.md @@ -52,12 +52,12 @@ This tool stores test timings in a local JSON file, keeping the last 10 timings └───────────────────────────┴───────────────────────────┘ │ ┌─────────────────────┐ │ - │ 2. SAVE PHASE │ │ + │ 2. MERGE PHASE │ │ └─────────────────────┘ │ ▼ ┌─────────────────────────┐ - │ fairsplice save │ - │ --from junit.xml │ + │ fairsplice merge │ + │ --prefix junit- │ └─────────────────────────┘ │ ▼ @@ -93,11 +93,11 @@ bunx fairsplice ## Usage -Fairsplice supports two main commands: `save` and `split`. Both require a `--timings-file` parameter to specify where timings are stored. +Fairsplice supports three commands: `save`, `merge`, and `split`. All require a `--timings-file` parameter to specify where timings are stored. -### Saving test results +### Saving test results (single file) -To save test results: +To save test results from a single JUnit XML file: ```bash fairsplice save --timings-file --from @@ -112,6 +112,24 @@ Example: fairsplice save --timings-file timings.json --from results/junit.xml ``` +### Merging test results (multiple files) + +To merge and save test results from multiple JUnit XML files (e.g., from parallel workers): + +```bash +fairsplice merge --timings-file --prefix +``` + +- `--timings-file `: JSON file to store timings +- `--prefix `: Prefix to match JUnit XML files + +Example: + +```bash +# Merges junit-0.xml, junit-1.xml, junit-2.xml, etc. +fairsplice merge --timings-file timings.json --prefix junit- +``` + ### Splitting test cases To split test cases for execution: diff --git a/index.ts b/index.ts index 9748c00..4d248b2 100755 --- a/index.ts +++ b/index.ts @@ -1,5 +1,6 @@ #!/usr/bin/env bun +import { merge } from "./src/commands/merge"; import { save } from "./src/commands/save"; import { split } from "./src/commands/split"; import { parseArgs } from "util"; @@ -19,6 +20,10 @@ const { positionals, values } = parseArgs({ from: { type: "string", }, + // merge options + prefix: { + type: "string", + }, // split options pattern: { type: "string", @@ -47,11 +52,11 @@ const command = positionals[2]; if (values.help || !command) { console.log(` -Usage: fairsplice [save|split] [options] +Usage: fairsplice [save|merge|split] [options] fairsplice save --------------- -Save test timings from a JUnit XML file. +Save test timings from a single JUnit XML file. Required options: --timings-file JSON file to store timings @@ -60,6 +65,17 @@ Required options: Example: fairsplice save --timings-file timings.json --from results/junit.xml +fairsplice merge +---------------- +Merge and save test timings from multiple JUnit XML files (e.g., from parallel workers). + +Required options: + --timings-file JSON file to store timings + --prefix Prefix to match JUnit XML files (e.g., "junit-" matches junit-0.xml, junit-1.xml) + +Example: fairsplice merge --timings-file timings.json --prefix junit- + + fairsplice split ---------------- Split test files across workers based on historical timings. @@ -88,6 +104,15 @@ if (command === "save") { } await save({ from: values.from, timingsFile: values["timings-file"] }); process.exit(0); +} else if (command === "merge") { + if (!values["timings-file"] || !values.prefix) { + console.error( + "Error: --timings-file and --prefix are required for the merge command." + ); + process.exit(1); + } + await merge({ prefix: values.prefix, timingsFile: values["timings-file"] }); + process.exit(0); } else if (command === "split") { if ( !values["timings-file"] || @@ -111,7 +136,7 @@ if (command === "save") { process.exit(0); } else { console.error( - `Invalid command "${command}". Available commands: save, split.` + `Invalid command "${command}". Available commands: save, merge, split.` ); process.exit(1); } diff --git a/src/commands/merge.ts b/src/commands/merge.ts new file mode 100644 index 0000000..2dbe80c --- /dev/null +++ b/src/commands/merge.ts @@ -0,0 +1,56 @@ +import { Glob } from "bun"; +import { saveTimings } from "../backend/fileStorage"; +import { parseJunit } from "../lib/junit"; + +export async function merge({ + timingsFile, + prefix, +}: { + timingsFile: string; + prefix: string; +}) { + // find all files matching the prefix pattern + const glob = new Glob(`${prefix}*`); + const files = Array.from(glob.scanSync()); + + if (files.length === 0) { + console.warn(`No files found matching prefix: ${prefix}*`); + process.exit(1); + } + + console.log(`Found ${files.length} files to merge:`); + files.forEach((f) => console.log(` - ${f}`)); + + // aggregate timings from all files + const timingByFile: Record = {}; + + for (const file of files) { + const junitXmlFile = Bun.file(file); + const xmlString = await junitXmlFile.text(); + + // parse junit xml + const testCases = parseJunit(xmlString); + + // aggregate timings + for (let testCase of testCases) { + if (testCase.file.includes("..")) { + continue; + } + if (!timingByFile[testCase.file]) { + timingByFile[testCase.file] = 0; + } + timingByFile[testCase.file] += testCase.time; + } + } + + // convert to ms + for (const [file, timing] of Object.entries(timingByFile)) { + timingByFile[file] = Math.round(timing * 1000); + } + + // save timings + await saveTimings(timingsFile, timingByFile); + console.log( + `\nTimings saved for ${Object.keys(timingByFile).length} files` + ); +} From 7a2e5a58498474eec0cdacf2decfe34c2d5848fb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 01:39:20 +0000 Subject: [PATCH 10/28] Remove save command - merge handles all use cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge command with a prefix that matches a single file works identically to save, making save redundant. Simplify to two commands: - merge: Read JUnit XML file(s) → save timings - split: Read timings → output test distribution --- README.md | 27 +++++---------------------- index.ts | 35 +++++------------------------------ src/commands/save.ts | 41 ----------------------------------------- 3 files changed, 10 insertions(+), 93 deletions(-) delete mode 100644 src/commands/save.ts diff --git a/README.md b/README.md index 5d5bb4f..987d899 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ This tool stores test timings in a local JSON file, keeping the last 10 timings **Key concepts:** - **Split phase**: Before tests run, fairsplice distributes test files across workers based on historical timing data -- **Save phase**: After tests complete, fairsplice extracts timing from JUnit XML and updates the timings file +- **Merge phase**: After tests complete, fairsplice extracts timing from JUnit XML and updates the timings file - **Bin packing**: Tests are assigned to workers to balance total execution time (heaviest tests first) - **Rolling average**: Keeps last 10 timings per test file, uses average for predictions @@ -93,28 +93,11 @@ bunx fairsplice ## Usage -Fairsplice supports three commands: `save`, `merge`, and `split`. All require a `--timings-file` parameter to specify where timings are stored. +Fairsplice has two commands: `merge` and `split`. Both require a `--timings-file` parameter. -### Saving test results (single file) +### Merging test results -To save test results from a single JUnit XML file: - -```bash -fairsplice save --timings-file --from -``` - -- `--timings-file `: JSON file to store timings (will be created if it doesn't exist) -- `--from `: JUnit XML file to read test results from - -Example: - -```bash -fairsplice save --timings-file timings.json --from results/junit.xml -``` - -### Merging test results (multiple files) - -To merge and save test results from multiple JUnit XML files (e.g., from parallel workers): +Save test timings from JUnit XML file(s): ```bash fairsplice merge --timings-file --prefix @@ -132,7 +115,7 @@ fairsplice merge --timings-file timings.json --prefix junit- ### Splitting test cases -To split test cases for execution: +Split test files across workers based on historical timings: ```bash fairsplice split --timings-file --pattern "" --total --out diff --git a/index.ts b/index.ts index 4d248b2..664fbb1 100755 --- a/index.ts +++ b/index.ts @@ -1,7 +1,6 @@ #!/usr/bin/env bun import { merge } from "./src/commands/merge"; -import { save } from "./src/commands/save"; import { split } from "./src/commands/split"; import { parseArgs } from "util"; @@ -16,10 +15,6 @@ const { positionals, values } = parseArgs({ ["timings-file"]: { type: "string", }, - // save options - from: { - type: "string", - }, // merge options prefix: { type: "string", @@ -52,26 +47,15 @@ const command = positionals[2]; if (values.help || !command) { console.log(` -Usage: fairsplice [save|merge|split] [options] - -fairsplice save ---------------- -Save test timings from a single JUnit XML file. - -Required options: - --timings-file JSON file to store timings - --from JUnit XML file to read test results from - -Example: fairsplice save --timings-file timings.json --from results/junit.xml - +Usage: fairsplice [merge|split] [options] fairsplice merge ---------------- -Merge and save test timings from multiple JUnit XML files (e.g., from parallel workers). +Save test timings from JUnit XML file(s). Required options: --timings-file JSON file to store timings - --prefix Prefix to match JUnit XML files (e.g., "junit-" matches junit-0.xml, junit-1.xml) + --prefix Prefix to match JUnit XML files (e.g., "junit-" matches junit-*.xml) Example: fairsplice merge --timings-file timings.json --prefix junit- @@ -95,16 +79,7 @@ Example: fairsplice split --timings-file timings.json --pattern "test_*.py" --to process.exit(0); } -if (command === "save") { - if (!values["timings-file"] || !values.from) { - console.error( - "Error: --timings-file and --from are required for the save command." - ); - process.exit(1); - } - await save({ from: values.from, timingsFile: values["timings-file"] }); - process.exit(0); -} else if (command === "merge") { +if (command === "merge") { if (!values["timings-file"] || !values.prefix) { console.error( "Error: --timings-file and --prefix are required for the merge command." @@ -136,7 +111,7 @@ if (command === "save") { process.exit(0); } else { console.error( - `Invalid command "${command}". Available commands: save, merge, split.` + `Invalid command "${command}". Available commands: merge, split.` ); process.exit(1); } diff --git a/src/commands/save.ts b/src/commands/save.ts deleted file mode 100644 index a7ff8e7..0000000 --- a/src/commands/save.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { saveTimings } from "../backend/fileStorage"; -import { parseJunit } from "../lib/junit"; - -export async function save({ - from, - timingsFile, -}: { - from: string; - timingsFile: string; -}) { - // read junit xml file - const junitXmlFile = Bun.file(from); - const xmlString = await junitXmlFile.text(); - - // parse junit xml - const testCases = parseJunit(xmlString); - - // aggregate timings - const timingByFile: Record = {}; - for (let testCase of testCases) { - if (testCase.file.includes("..")) { - continue; - } - if (!timingByFile[testCase.file]) { - timingByFile[testCase.file] = 0; - } - timingByFile[testCase.file] += testCase.time; - } - - // convert to ms - for (const [file, timing] of Object.entries(timingByFile)) { - timingByFile[file] = Math.round(timing * 1000); - } - - // save timings - await saveTimings(timingsFile, timingByFile); - console.log( - "Timings saved for files:\n", - Object.keys(timingByFile).join("\n - ") - ); -} From 2bd6309125c66eda563cc46848d647f35b85f219 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 01:49:44 +0000 Subject: [PATCH 11/28] Add GitHub Action with automatic caching New action.yml provides a simple interface for GitHub Actions: - Split: distributes tests across workers with index output - Merge: saves timings from JUnit XML files - Automatic cache restore/save - no manual setup needed Usage: uses: dashdoc/fairsplice@v1 with: command: split pattern: 'tests/**/*.py' total: 3 index: ${{ matrix.index }} --- README.md | 190 +++++++++++++++++++++++++---------------------------- action.yml | 88 +++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 99 deletions(-) create mode 100644 action.yml diff --git a/README.md b/README.md index 987d899..52a6904 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,52 @@ # Fairsplice -**Warning: this project is still in very early development!** +Fairsplice is a CLI tool and GitHub Action that optimizes test distribution across parallel workers. It provides CircleCI-style test splitting based on timing data for GitHub Actions. -Fairsplice is a CLI tool designed to optimize test distribution across multiple workers. By intelligently splitting and saving test cases, Fairsplice ensures a balanced workload distribution for your CI/CD pipelines, making tests run time more predictable. +## Quick Start (GitHub Action) -We found Github Actions lacking when compared to CircleCI which has [tests splitting](https://circleci.com/docs/parallelism-faster-jobs/#how-test-splitting-works) based on timings. - -There are a number of projects like [Split tests](https://github.com/marketplace/actions/split-tests) but they require uploading and downloading Junit XML files and merging them, or committing the Junit files to have them when running the tests. +```yaml +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + index: [0, 1, 2] + steps: + - uses: actions/checkout@v4 + + - name: Split tests + id: split + uses: dashdoc/fairsplice@v1 + with: + command: split + pattern: 'tests/**/*.py' + total: 3 + index: ${{ matrix.index }} + + - name: Run tests + run: pytest ${{ steps.split.outputs.tests }} --junit-xml=junit-${{ matrix.index }}.xml + + - uses: actions/upload-artifact@v4 + with: + name: junit-${{ matrix.index }} + path: junit-${{ matrix.index }}.xml + + save-timings: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + + - name: Merge timings + uses: dashdoc/fairsplice@v1 + with: + command: merge + prefix: 'junit-*/junit-' +``` -This tool stores test timings in a local JSON file, keeping the last 10 timings for each test file and using the average for splitting. No external database required! +That's it! Caching is handled automatically. ## How It Works @@ -21,7 +59,7 @@ This tool stores test timings in a local JSON file, keeping the last 10 timings │ 1. SPLIT PHASE │ └─────────────────────┘ - timings.json fairsplice split + timings (cached) fairsplice split ┌──────────────────────┐ ┌─────────────────┐ │ { │ │ │ │ "test_a.py": [2.1],│ ──────▶ │ Load timings │ @@ -39,10 +77,9 @@ This tool stores test timings in a local JSON file, keeping the last 10 timings ▼ ▼ ▼ ┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐ │ Worker 0 │ │ Worker 1 │ │ Worker 2 │ - │ ["test_b.py"] │ │ ["test_a.py", │ │ ["test_c.py"] │ - │ ~5.3s │ │ "test_c.py"] │ │ ~1.8s │ - └─────────┬─────────┘ │ ~3.9s │ └─────────┬─────────┘ - │ └─────────┬─────────┘ │ + │ ~5.3s │ │ ~3.9s │ │ ~5.1s │ + └─────────┬─────────┘ └─────────┬─────────┘ └─────────┬─────────┘ + │ │ │ ▼ ▼ ▼ ┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐ │ Run tests │ │ Run tests │ │ Run tests │ @@ -57,130 +94,85 @@ This tool stores test timings in a local JSON file, keeping the last 10 timings ▼ ┌─────────────────────────┐ │ fairsplice merge │ - │ --prefix junit- │ - └─────────────────────────┘ - │ - ▼ - ┌─────────────────────────┐ - │ Extract timings from │ - │ JUnit XML results │ + │ (extracts timings) │ └─────────────────────────┘ │ ▼ ┌──────────────────────┐ - │ timings.json │ - │ Updated with new │ - │ timing data │◀─── Cached/committed + │ timings (cached) │◀─── Auto-cached └──────────────────────┘ for next run ``` **Key concepts:** -- **Split phase**: Before tests run, fairsplice distributes test files across workers based on historical timing data -- **Merge phase**: After tests complete, fairsplice extracts timing from JUnit XML and updates the timings file -- **Bin packing**: Tests are assigned to workers to balance total execution time (heaviest tests first) -- **Rolling average**: Keeps last 10 timings per test file, uses average for predictions +- **Split phase**: Distributes test files across workers based on historical timing data +- **Merge phase**: Extracts timing from JUnit XML and caches for next run +- **Bin packing**: Assigns tests to balance total execution time (heaviest tests first) +- **Rolling average**: Keeps last 10 timings per test file for predictions -## Installation +## GitHub Action Reference -This project is built using [Bun](https://bun.sh). +### Inputs -Ensure you have Bun installed. -To launch it, run +| Input | Required | Description | +|-------|----------|-------------| +| `command` | Yes | `split` or `merge` | +| `timings-file` | No | JSON file for timings (default: `.fairsplice-timings.json`) | +| `pattern` | For split | Glob pattern to match test files | +| `total` | For split | Total number of workers | +| `index` | For split | Current worker index (0-based) | +| `prefix` | For merge | Prefix to match JUnit XML files | -```bash -bunx fairsplice -``` - -## Usage +### Outputs -Fairsplice has two commands: `merge` and `split`. Both require a `--timings-file` parameter. +| Output | Description | +|--------|-------------| +| `tests` | Space-separated list of test files (when `index` provided) | +| `buckets` | JSON array of all test buckets | -### Merging test results +## CLI Usage -Save test timings from JUnit XML file(s): +Install with Bun: ```bash -fairsplice merge --timings-file --prefix -``` - -- `--timings-file `: JSON file to store timings -- `--prefix `: Prefix to match JUnit XML files - -Example: - -```bash -# Merges junit-0.xml, junit-1.xml, junit-2.xml, etc. -fairsplice merge --timings-file timings.json --prefix junit- +bunx fairsplice ``` -### Splitting test cases - -Split test files across workers based on historical timings: +### Commands +**Split tests:** ```bash -fairsplice split --timings-file --pattern "" --total --out +fairsplice split --timings-file timings.json --pattern "tests/**/*.py" --total 3 --out split.json ``` -- `--timings-file `: JSON file with stored timings -- `--pattern ""`: Pattern to match test files (can be used multiple times) -- `--total `: Total number of workers -- `--out `: File to write split result to (JSON array of arrays) -- `--replace-from `: (Optional) Substring to replace in file paths -- `--replace-to `: (Optional) Replacement string - -Example: - +**Merge results:** ```bash -fairsplice split --timings-file timings.json --pattern "test_*.py" --total 3 --out split.json +fairsplice merge --timings-file timings.json --prefix junit- ``` -## Using with GitHub Actions +### CLI Options -To persist timings across CI runs, use GitHub Actions cache: - -```yaml -- name: Cache test timings - uses: actions/cache@v4 - with: - path: timings.json - key: fairsplice-timings-${{ github.ref }} - restore-keys: | - fairsplice-timings- - -- name: Split tests - run: bunx fairsplice split --timings-file timings.json --pattern "tests/**/*.py" --total 3 --out split.json ``` - -Alternatively, you can commit the timings file to your repository for simpler persistence. - -## Help - -For a detailed list of commands and options, use the help command: - -```bash -fairsplice --help +fairsplice split + --timings-file JSON file with stored timings + --pattern Glob pattern for test files (can repeat) + --total Number of workers + --out Output JSON file + +fairsplice merge + --timings-file JSON file to store timings + --prefix Prefix to match JUnit XML files ``` ## Contributing -Contributions are welcome! Please fork the repository and submit a pull request with your improvements. - -### Running locally - -Launch the development version with: - ```bash +# Run locally bun run index.ts -``` - -### Running tests -Launch the following command to run tests: - -```bash -bun test [--watch] +# Run tests +bun test ``` ## License -Fairsplice is open-source software licensed under the MIT license. +MIT diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..a70fcea --- /dev/null +++ b/action.yml @@ -0,0 +1,88 @@ +name: 'Fairsplice' +description: 'Split tests across parallel workers based on timing data' +branding: + icon: 'scissors' + color: 'blue' + +inputs: + command: + description: 'Command to run: split or merge' + required: true + timings-file: + description: 'JSON file to store/read timings (default: .fairsplice-timings.json)' + required: false + default: '.fairsplice-timings.json' + # split inputs + pattern: + description: 'Glob pattern to match test files (for split)' + required: false + total: + description: 'Total number of workers (for split)' + required: false + index: + description: 'Current worker index, 0-based (for split) - outputs only this worker tests' + required: false + # merge inputs + prefix: + description: 'Prefix to match JUnit XML files (for merge)' + required: false + +outputs: + tests: + description: 'Space-separated list of test files for the current worker (when index is provided)' + value: ${{ steps.split.outputs.tests }} + buckets: + description: 'JSON array of test buckets (when index is not provided)' + value: ${{ steps.split.outputs.buckets }} + +runs: + using: 'composite' + steps: + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + + - name: Restore timings cache + uses: actions/cache/restore@v4 + with: + path: ${{ inputs.timings-file }} + key: fairsplice-timings-${{ github.repository }}-${{ github.ref_name }} + restore-keys: | + fairsplice-timings-${{ github.repository }}- + + - name: Run split + id: split + if: inputs.command == 'split' + shell: bash + run: | + # Run fairsplice split + bunx fairsplice@latest split \ + --timings-file "${{ inputs.timings-file }}" \ + --pattern "${{ inputs.pattern }}" \ + --total "${{ inputs.total }}" \ + --out /tmp/fairsplice-buckets.json + + # Output results + if [ -n "${{ inputs.index }}" ]; then + # Extract tests for specific worker index + TESTS=$(jq -r '.[${{ inputs.index }}] | join(" ")' /tmp/fairsplice-buckets.json) + echo "tests=$TESTS" >> $GITHUB_OUTPUT + else + # Output all buckets + BUCKETS=$(cat /tmp/fairsplice-buckets.json) + echo "buckets=$BUCKETS" >> $GITHUB_OUTPUT + fi + + - name: Run merge + if: inputs.command == 'merge' + shell: bash + run: | + bunx fairsplice@latest merge \ + --timings-file "${{ inputs.timings-file }}" \ + --prefix "${{ inputs.prefix }}" + + - name: Save timings cache + if: inputs.command == 'merge' + uses: actions/cache/save@v4 + with: + path: ${{ inputs.timings-file }} + key: fairsplice-timings-${{ github.repository }}-${{ github.ref_name }}-${{ github.run_id }} From bc181805f885c8c7a23dc18d68c8f92b83c11ccc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 01:53:59 +0000 Subject: [PATCH 12/28] Update CI to test GitHub Action with uses: ./ Simplified workflow that tests the action itself: - Unit tests for src/ - Integration tests using the action with 3 parallel workers - Save timings job that merges results --- .github/workflows/test.yml | 229 ++++++------------------------------- 1 file changed, 36 insertions(+), 193 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69fbe0c..59067f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup Bun - uses: oven-sh/setup-bun@v1 - with: - bun-version: latest + uses: oven-sh/setup-bun@v2 - name: Install dependencies run: bun install @@ -24,219 +22,64 @@ jobs: - name: Run unit tests run: bun test src/ - dummy-tests: - name: Dummy Tests with Variable Timings - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup Bun - uses: oven-sh/setup-bun@v1 - with: - bun-version: latest - - - name: Install dependencies - run: bun install - - - name: Run dummy tests and generate JUnit report - run: bun test tests/dummy/ --reporter=junit --reporter-outfile=junit-results.xml - - - name: Display test results - run: cat junit-results.xml - - - name: Save timings with fairsplice - run: bun run index.ts save --from junit-results.xml - - - name: Display saved timings - run: cat .fairsplice-timings.json - - - name: Split tests for 2 workers - run: | - bun run index.ts split --pattern "tests/dummy/*.test.ts" --total 2 --out split-2.json - echo "Split for 2 workers:" - cat split-2.json - - - name: Split tests for 3 workers - run: | - bun run index.ts split --pattern "tests/dummy/*.test.ts" --total 3 --out split-3.json - echo "Split for 3 workers:" - cat split-3.json - - - name: Upload test artifacts - uses: actions/upload-artifact@v4 - with: - name: test-results - path: | - junit-results.xml - .fairsplice-timings.json - split-2.json - split-3.json - - # Integration test: simulates real CI workflow with cached timings - integration-prepare: - name: Prepare Test Split - runs-on: ubuntu-latest - outputs: - has-timings: ${{ steps.check-cache.outputs.cache-hit }} - steps: - - uses: actions/checkout@v4 - - - name: Setup Bun - uses: oven-sh/setup-bun@v1 - with: - bun-version: latest - - - name: Install dependencies - run: bun install - - # Try to restore timings from cache (simulates real CI) - - name: Restore timings cache - id: check-cache - uses: actions/cache@v4 - with: - path: .fairsplice-timings.json - key: fairsplice-timings-${{ github.ref }}-${{ github.run_number }} - restore-keys: | - fairsplice-timings-${{ github.ref }}- - fairsplice-timings- - - - name: Show timings status - run: | - if [ -f .fairsplice-timings.json ]; then - echo "Using cached timings from previous run:" - cat .fairsplice-timings.json - else - echo "No cached timings found - first run will use default timing (10s per test)" - echo "Timings will be collected after this run for future optimization" - fi - - # Split tests for workers - - name: Split tests for 3 workers - run: | - bun run index.ts split --pattern "tests/dummy/*.test.ts" --total 3 --out split.json - echo "Test split:" - cat split.json - - # Upload split file for workers to use - - name: Upload split configuration - uses: actions/upload-artifact@v4 - with: - name: test-split - path: split.json - - # Workers don't need fairsplice installed - they just read split.json and run tests - integration-worker: - name: Worker ${{ matrix.worker }} - needs: integration-prepare + # Integration test using the GitHub Action + test: + name: Test Worker ${{ matrix.index }} runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - worker: [0, 1, 2] + index: [0, 1, 2] steps: - uses: actions/checkout@v4 - name: Setup Bun - uses: oven-sh/setup-bun@v1 - with: - bun-version: latest + uses: oven-sh/setup-bun@v2 - # Note: No need to install fairsplice deps - workers just run tests + - name: Install dependencies + run: bun install - - name: Download split configuration - uses: actions/download-artifact@v4 + - name: Split tests + id: split + uses: ./ with: - name: test-split + command: split + pattern: 'tests/dummy/*.test.ts' + total: 3 + index: ${{ matrix.index }} - - name: Get assigned tests - id: get-tests - run: | - TESTS=$(jq -r '.[${{ matrix.worker }}] | join(" ")' split.json) - echo "files=$TESTS" >> $GITHUB_OUTPUT - echo "Worker ${{ matrix.worker }} assigned: $TESTS" + - name: Show assigned tests + run: echo "Worker ${{ matrix.index }} running:${{ steps.split.outputs.tests }}" - name: Run tests - if: steps.get-tests.outputs.files != '' - run: | - bun test ${{ steps.get-tests.outputs.files }} --reporter=junit --reporter-outfile=junit-worker-${{ matrix.worker }}.xml + if: steps.split.outputs.tests != '' + run: bun test ${{ steps.split.outputs.tests }} --reporter=junit --reporter-outfile=junit-${{ matrix.index }}.xml - name: Upload results - if: steps.get-tests.outputs.files != '' + if: always() uses: actions/upload-artifact@v4 with: - name: junit-worker-${{ matrix.worker }} - path: junit-worker-${{ matrix.worker }}.xml + name: junit-${{ matrix.index }} + path: junit-${{ matrix.index }}.xml - # Collect results and update timings for next run - integration-finalize: - name: Finalize & Update Timings - needs: integration-worker + save-timings: + name: Save Timings + needs: test + if: always() runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Setup Bun - uses: oven-sh/setup-bun@v1 - with: - bun-version: latest + - uses: actions/download-artifact@v4 - - name: Install dependencies - run: bun install + - name: Show downloaded artifacts + run: find . -name "*.xml" -type f - # Restore existing timings - - name: Restore timings cache - uses: actions/cache@v4 + - name: Merge timings + uses: ./ with: - path: .fairsplice-timings.json - key: fairsplice-timings-${{ github.ref }}-${{ github.run_number }} - restore-keys: | - fairsplice-timings-${{ github.ref }}- - fairsplice-timings- - - # Download all worker results - - name: Download worker 0 results - uses: actions/download-artifact@v4 - with: - name: junit-worker-0 - path: results/ - continue-on-error: true - - - name: Download worker 1 results - uses: actions/download-artifact@v4 - with: - name: junit-worker-1 - path: results/ - continue-on-error: true + command: merge + prefix: 'junit-*/junit-' - - name: Download worker 2 results - uses: actions/download-artifact@v4 - with: - name: junit-worker-2 - path: results/ - continue-on-error: true - - # Merge and save new timings - - name: Save updated timings from all workers - run: | - echo "Worker result files:" - ls -la results/ || echo "No results directory" - - for file in results/junit-worker-*.xml; do - if [ -f "$file" ]; then - echo "Saving timings from $file" - bun run index.ts save --from "$file" - fi - done - - echo "Updated timings:" - cat .fairsplice-timings.json || echo "No timings file" - - - name: Summary - run: | - echo "## Integration Test Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Timings have been updated and cached for the next run." >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Current timings:" >> $GITHUB_STEP_SUMMARY - echo '```json' >> $GITHUB_STEP_SUMMARY - cat .fairsplice-timings.json >> $GITHUB_STEP_SUMMARY || echo "{}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY + - name: Show saved timings + run: cat .fairsplice-timings.json From ac37b60d797cce11ebf9a9829b4f4093a750eca8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 01:56:05 +0000 Subject: [PATCH 13/28] Fix action to run local code instead of npm package - Use bun run ${{ github.action_path }}/index.ts instead of bunx fairsplice@latest - Add bun install step to install dependencies - This allows testing the action from the current repo with uses: ./ --- action.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index a70fcea..b68399b 100644 --- a/action.yml +++ b/action.yml @@ -41,6 +41,10 @@ runs: - name: Setup Bun uses: oven-sh/setup-bun@v2 + - name: Install dependencies + shell: bash + run: cd ${{ github.action_path }} && bun install --frozen-lockfile + - name: Restore timings cache uses: actions/cache/restore@v4 with: @@ -55,7 +59,7 @@ runs: shell: bash run: | # Run fairsplice split - bunx fairsplice@latest split \ + bun run ${{ github.action_path }}/index.ts split \ --timings-file "${{ inputs.timings-file }}" \ --pattern "${{ inputs.pattern }}" \ --total "${{ inputs.total }}" \ @@ -76,7 +80,7 @@ runs: if: inputs.command == 'merge' shell: bash run: | - bunx fairsplice@latest merge \ + bun run ${{ github.action_path }}/index.ts merge \ --timings-file "${{ inputs.timings-file }}" \ --prefix "${{ inputs.prefix }}" From a57b88a592584fff8426cd564a2eec88419cb8bc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 01:57:46 +0000 Subject: [PATCH 14/28] Fix fileStorage tests to use explicit file path --- src/backend/fileStorage.test.ts | 48 +++++++++++++++------------------ 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/src/backend/fileStorage.test.ts b/src/backend/fileStorage.test.ts index 91a1f0a..89330f9 100644 --- a/src/backend/fileStorage.test.ts +++ b/src/backend/fileStorage.test.ts @@ -1,21 +1,14 @@ import { describe, it, expect, beforeEach, afterEach } from "bun:test"; import { saveTimings, getTimings } from "./fileStorage"; import { unlink } from "node:fs/promises"; -import { TIMINGS_FILE_PATH } from "../config"; -describe("fileStorage", () => { - let originalFileContent: string | null = null; +const TEST_TIMINGS_FILE = "/tmp/fairsplice-test-timings.json"; +describe("fileStorage", () => { beforeEach(async () => { - // Backup existing file if present - try { - originalFileContent = await Bun.file(TIMINGS_FILE_PATH).text(); - } catch { - originalFileContent = null; - } // Clean up before each test try { - await unlink(TIMINGS_FILE_PATH); + await unlink(TEST_TIMINGS_FILE); } catch { // File might not exist } @@ -24,14 +17,10 @@ describe("fileStorage", () => { afterEach(async () => { // Clean up after test try { - await unlink(TIMINGS_FILE_PATH); + await unlink(TEST_TIMINGS_FILE); } catch { // File might not exist } - // Restore original file if it existed - if (originalFileContent !== null) { - await Bun.write(TIMINGS_FILE_PATH, originalFileContent); - } }); it("should save and retrieve timings", async () => { @@ -41,45 +30,52 @@ describe("fileStorage", () => { "test3.ts": 300, }; - await saveTimings(timings); - const retrieved = await getTimings(["test1.ts", "test2.ts", "test3.ts"]); + await saveTimings(TEST_TIMINGS_FILE, timings); + const retrieved = await getTimings(TEST_TIMINGS_FILE, [ + "test1.ts", + "test2.ts", + "test3.ts", + ]); expect(retrieved).toEqual(timings); }); it("should return empty object for non-existent files", async () => { - const retrieved = await getTimings(["nonexistent.ts"]); + const retrieved = await getTimings(TEST_TIMINGS_FILE, ["nonexistent.ts"]); expect(retrieved).toEqual({}); }); it("should average multiple timing entries", async () => { // Save first set of timings - await saveTimings({ "test.ts": 100 }); + await saveTimings(TEST_TIMINGS_FILE, { "test.ts": 100 }); // Save second set of timings - await saveTimings({ "test.ts": 200 }); + await saveTimings(TEST_TIMINGS_FILE, { "test.ts": 200 }); // Save third set of timings - await saveTimings({ "test.ts": 300 }); + await saveTimings(TEST_TIMINGS_FILE, { "test.ts": 300 }); - const retrieved = await getTimings(["test.ts"]); + const retrieved = await getTimings(TEST_TIMINGS_FILE, ["test.ts"]); // Average of [300, 200, 100] = 200 expect(retrieved["test.ts"]).toBe(200); }); it("should handle partial file requests", async () => { - await saveTimings({ + await saveTimings(TEST_TIMINGS_FILE, { "test1.ts": 100, "test2.ts": 200, }); - const retrieved = await getTimings(["test1.ts", "nonexistent.ts"]); + const retrieved = await getTimings(TEST_TIMINGS_FILE, [ + "test1.ts", + "nonexistent.ts", + ]); expect(retrieved).toEqual({ "test1.ts": 100 }); }); it("should persist data to JSON file", async () => { - await saveTimings({ "myfile.ts": 42 }); + await saveTimings(TEST_TIMINGS_FILE, { "myfile.ts": 42 }); // Verify file exists and contains valid JSON - const content = await Bun.file(TIMINGS_FILE_PATH).text(); + const content = await Bun.file(TEST_TIMINGS_FILE).text(); const data = JSON.parse(content); expect(data.version).toBe(1); From a37b50bba1c47df1a28a3f7346f7a5c8850ed533 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 02:15:57 +0000 Subject: [PATCH 15/28] Add convert command and refactor architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Separate concerns by introducing a new convert command: - convert: JUnit XML → timing JSON (single file) - merge: timing JSONs → combined timing history - split: timing history + pattern → test distribution This allows each worker to convert their JUnit results to timing JSON, then merge job combines all timing JSONs together. --- .github/workflows/test.yml | 18 +++++++--- action.yml | 20 +++++++++-- index.ts | 74 +++++++++++++++++++++++++------------- src/commands/convert.ts | 37 +++++++++++++++++++ src/commands/merge.ts | 37 ++++++------------- 5 files changed, 128 insertions(+), 58 deletions(-) create mode 100644 src/commands/convert.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59067f0..c1a1ba7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,12 +55,20 @@ jobs: if: steps.split.outputs.tests != '' run: bun test ${{ steps.split.outputs.tests }} --reporter=junit --reporter-outfile=junit-${{ matrix.index }}.xml - - name: Upload results + - name: Convert JUnit to timing JSON + if: steps.split.outputs.tests != '' + uses: ./ + with: + command: convert + from: junit-${{ matrix.index }}.xml + out: timing-${{ matrix.index }}.json + + - name: Upload timing artifact if: always() uses: actions/upload-artifact@v4 with: - name: junit-${{ matrix.index }} - path: junit-${{ matrix.index }}.xml + name: timing-${{ matrix.index }} + path: timing-${{ matrix.index }}.json save-timings: name: Save Timings @@ -73,13 +81,13 @@ jobs: - uses: actions/download-artifact@v4 - name: Show downloaded artifacts - run: find . -name "*.xml" -type f + run: find . -name "*.json" -type f - name: Merge timings uses: ./ with: command: merge - prefix: 'junit-*/junit-' + prefix: 'timing-*/timing-' - name: Show saved timings run: cat .fairsplice-timings.json diff --git a/action.yml b/action.yml index b68399b..48629bd 100644 --- a/action.yml +++ b/action.yml @@ -6,7 +6,7 @@ branding: inputs: command: - description: 'Command to run: split or merge' + description: 'Command to run: split, convert, or merge' required: true timings-file: description: 'JSON file to store/read timings (default: .fairsplice-timings.json)' @@ -22,9 +22,16 @@ inputs: index: description: 'Current worker index, 0-based (for split) - outputs only this worker tests' required: false + # convert inputs + from: + description: 'JUnit XML file to read (for convert)' + required: false + out: + description: 'Output file (for convert: timing JSON)' + required: false # merge inputs prefix: - description: 'Prefix to match JUnit XML files (for merge)' + description: 'Prefix to match timing JSON files (for merge)' required: false outputs: @@ -46,6 +53,7 @@ runs: run: cd ${{ github.action_path }} && bun install --frozen-lockfile - name: Restore timings cache + if: inputs.command == 'split' || inputs.command == 'merge' uses: actions/cache/restore@v4 with: path: ${{ inputs.timings-file }} @@ -76,6 +84,14 @@ runs: echo "buckets=$BUCKETS" >> $GITHUB_OUTPUT fi + - name: Run convert + if: inputs.command == 'convert' + shell: bash + run: | + bun run ${{ github.action_path }}/index.ts convert \ + --from "${{ inputs.from }}" \ + --out "${{ inputs.out }}" + - name: Run merge if: inputs.command == 'merge' shell: bash diff --git a/index.ts b/index.ts index 664fbb1..9dca6ca 100755 --- a/index.ts +++ b/index.ts @@ -1,5 +1,6 @@ #!/usr/bin/env bun +import { convert } from "./src/commands/convert"; import { merge } from "./src/commands/merge"; import { split } from "./src/commands/split"; import { parseArgs } from "util"; @@ -11,11 +12,14 @@ const { positionals, values } = parseArgs({ type: "boolean", short: "h", }, - // common options - ["timings-file"]: { + // convert options + from: { type: "string", }, // merge options + ["timings-file"]: { + type: "string", + }, prefix: { type: "string", }, @@ -47,18 +51,7 @@ const command = positionals[2]; if (values.help || !command) { console.log(` -Usage: fairsplice [merge|split] [options] - -fairsplice merge ----------------- -Save test timings from JUnit XML file(s). - -Required options: - --timings-file JSON file to store timings - --prefix Prefix to match JUnit XML files (e.g., "junit-" matches junit-*.xml) - -Example: fairsplice merge --timings-file timings.json --prefix junit- - +Usage: fairsplice [split|convert|merge] [options] fairsplice split ---------------- @@ -75,20 +68,33 @@ Optional: --replace-to Replacement string (must match number of --replace-from) Example: fairsplice split --timings-file timings.json --pattern "test_*.py" --total 3 --out split.json + + +fairsplice convert +------------------ +Convert JUnit XML to timing JSON (for a single worker). + +Required options: + --from JUnit XML file to read + --out Timing JSON file to write + +Example: fairsplice convert --from junit.xml --out timing.json + + +fairsplice merge +---------------- +Merge timing JSON files and save to timings history. + +Required options: + --timings-file JSON file to store timing history + --prefix Prefix to match timing JSON files + +Example: fairsplice merge --timings-file timings.json --prefix timing- `); process.exit(0); } -if (command === "merge") { - if (!values["timings-file"] || !values.prefix) { - console.error( - "Error: --timings-file and --prefix are required for the merge command." - ); - process.exit(1); - } - await merge({ prefix: values.prefix, timingsFile: values["timings-file"] }); - process.exit(0); -} else if (command === "split") { +if (command === "split") { if ( !values["timings-file"] || !values.pattern || @@ -109,9 +115,27 @@ if (command === "merge") { timingsFile: values["timings-file"], }); process.exit(0); +} else if (command === "convert") { + if (!values.from || !values.out) { + console.error( + "Error: --from and --out are required for the convert command." + ); + process.exit(1); + } + await convert({ from: values.from, out: values.out }); + process.exit(0); +} else if (command === "merge") { + if (!values["timings-file"] || !values.prefix) { + console.error( + "Error: --timings-file and --prefix are required for the merge command." + ); + process.exit(1); + } + await merge({ prefix: values.prefix, timingsFile: values["timings-file"] }); + process.exit(0); } else { console.error( - `Invalid command "${command}". Available commands: merge, split.` + `Invalid command "${command}". Available commands: split, convert, merge.` ); process.exit(1); } diff --git a/src/commands/convert.ts b/src/commands/convert.ts new file mode 100644 index 0000000..b7967b2 --- /dev/null +++ b/src/commands/convert.ts @@ -0,0 +1,37 @@ +import { parseJunit } from "../lib/junit"; + +export async function convert({ + from, + out, +}: { + from: string; + out: string; +}) { + // read junit xml file + const junitXmlFile = Bun.file(from); + const xmlString = await junitXmlFile.text(); + + // parse junit xml + const testCases = parseJunit(xmlString); + + // aggregate timings by file + const timingByFile: Record = {}; + for (const testCase of testCases) { + if (testCase.file.includes("..")) { + continue; + } + if (!timingByFile[testCase.file]) { + timingByFile[testCase.file] = 0; + } + timingByFile[testCase.file] += testCase.time; + } + + // convert to ms + for (const [file, timing] of Object.entries(timingByFile)) { + timingByFile[file] = Math.round(timing * 1000); + } + + // write timings JSON + await Bun.write(out, JSON.stringify(timingByFile, null, 2)); + console.log(`Converted ${Object.keys(timingByFile).length} test timings to ${out}`); +} diff --git a/src/commands/merge.ts b/src/commands/merge.ts index 2dbe80c..1e86529 100644 --- a/src/commands/merge.ts +++ b/src/commands/merge.ts @@ -1,6 +1,5 @@ import { Glob } from "bun"; import { saveTimings } from "../backend/fileStorage"; -import { parseJunit } from "../lib/junit"; export async function merge({ timingsFile, @@ -9,7 +8,7 @@ export async function merge({ timingsFile: string; prefix: string; }) { - // find all files matching the prefix pattern + // find all timing JSON files matching the prefix pattern const glob = new Glob(`${prefix}*`); const files = Array.from(glob.scanSync()); @@ -18,39 +17,25 @@ export async function merge({ process.exit(1); } - console.log(`Found ${files.length} files to merge:`); + console.log(`Found ${files.length} timing files to merge:`); files.forEach((f) => console.log(` - ${f}`)); - // aggregate timings from all files + // aggregate timings from all JSON files const timingByFile: Record = {}; for (const file of files) { - const junitXmlFile = Bun.file(file); - const xmlString = await junitXmlFile.text(); + const content = await Bun.file(file).text(); + const timings = JSON.parse(content) as Record; - // parse junit xml - const testCases = parseJunit(xmlString); - - // aggregate timings - for (let testCase of testCases) { - if (testCase.file.includes("..")) { - continue; - } - if (!timingByFile[testCase.file]) { - timingByFile[testCase.file] = 0; + for (const [testFile, timing] of Object.entries(timings)) { + if (!timingByFile[testFile]) { + timingByFile[testFile] = 0; } - timingByFile[testCase.file] += testCase.time; + timingByFile[testFile] += timing; } } - // convert to ms - for (const [file, timing] of Object.entries(timingByFile)) { - timingByFile[file] = Math.round(timing * 1000); - } - - // save timings + // save merged timings await saveTimings(timingsFile, timingByFile); - console.log( - `\nTimings saved for ${Object.keys(timingByFile).length} files` - ); + console.log(`\nMerged timings for ${Object.keys(timingByFile).length} files`); } From b714ca93c6f7c2a83ccab3315311358140232ecb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 02:17:55 +0000 Subject: [PATCH 16/28] Rename convert --out to --to for consistency with --from --- .github/workflows/test.yml | 2 +- action.yml | 6 +++--- index.ts | 13 ++++++++----- src/commands/convert.ts | 8 ++++---- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c1a1ba7..c66a6d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,7 @@ jobs: with: command: convert from: junit-${{ matrix.index }}.xml - out: timing-${{ matrix.index }}.json + to: timing-${{ matrix.index }}.json - name: Upload timing artifact if: always() diff --git a/action.yml b/action.yml index 48629bd..780f3f9 100644 --- a/action.yml +++ b/action.yml @@ -26,8 +26,8 @@ inputs: from: description: 'JUnit XML file to read (for convert)' required: false - out: - description: 'Output file (for convert: timing JSON)' + to: + description: 'Timing JSON file to write (for convert)' required: false # merge inputs prefix: @@ -90,7 +90,7 @@ runs: run: | bun run ${{ github.action_path }}/index.ts convert \ --from "${{ inputs.from }}" \ - --out "${{ inputs.out }}" + --to "${{ inputs.to }}" - name: Run merge if: inputs.command == 'merge' diff --git a/index.ts b/index.ts index 9dca6ca..4d801e1 100755 --- a/index.ts +++ b/index.ts @@ -16,6 +16,9 @@ const { positionals, values } = parseArgs({ from: { type: "string", }, + to: { + type: "string", + }, // merge options ["timings-file"]: { type: "string", @@ -76,9 +79,9 @@ Convert JUnit XML to timing JSON (for a single worker). Required options: --from JUnit XML file to read - --out Timing JSON file to write + --to Timing JSON file to write -Example: fairsplice convert --from junit.xml --out timing.json +Example: fairsplice convert --from junit.xml --to timing.json fairsplice merge @@ -116,13 +119,13 @@ if (command === "split") { }); process.exit(0); } else if (command === "convert") { - if (!values.from || !values.out) { + if (!values.from || !values.to) { console.error( - "Error: --from and --out are required for the convert command." + "Error: --from and --to are required for the convert command." ); process.exit(1); } - await convert({ from: values.from, out: values.out }); + await convert({ from: values.from, to: values.to }); process.exit(0); } else if (command === "merge") { if (!values["timings-file"] || !values.prefix) { diff --git a/src/commands/convert.ts b/src/commands/convert.ts index b7967b2..cb393e7 100644 --- a/src/commands/convert.ts +++ b/src/commands/convert.ts @@ -2,10 +2,10 @@ import { parseJunit } from "../lib/junit"; export async function convert({ from, - out, + to, }: { from: string; - out: string; + to: string; }) { // read junit xml file const junitXmlFile = Bun.file(from); @@ -32,6 +32,6 @@ export async function convert({ } // write timings JSON - await Bun.write(out, JSON.stringify(timingByFile, null, 2)); - console.log(`Converted ${Object.keys(timingByFile).length} test timings to ${out}`); + await Bun.write(to, JSON.stringify(timingByFile, null, 2)); + console.log(`Converted ${Object.keys(timingByFile).length} test timings to ${to}`); } From 3019ec7752d70bb910ecf46d643b4f7f02e66a79 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Jan 2026 22:29:49 +0000 Subject: [PATCH 17/28] Add cache-key input for multi-workflow support - Add cache-key input to differentiate frontend/backend workflows - Remove branch reference from cache key to share timings across branches - Cache is now shared across all branches by default --- action.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index 780f3f9..0f622d4 100644 --- a/action.yml +++ b/action.yml @@ -12,6 +12,10 @@ inputs: description: 'JSON file to store/read timings (default: .fairsplice-timings.json)' required: false default: '.fairsplice-timings.json' + cache-key: + description: 'Cache key for storing timings (use different keys for frontend/backend workflows)' + required: false + default: 'default' # split inputs pattern: description: 'Glob pattern to match test files (for split)' @@ -57,9 +61,9 @@ runs: uses: actions/cache/restore@v4 with: path: ${{ inputs.timings-file }} - key: fairsplice-timings-${{ github.repository }}-${{ github.ref_name }} + key: fairsplice-${{ inputs.cache-key }}-${{ github.repository }} restore-keys: | - fairsplice-timings-${{ github.repository }}- + fairsplice-${{ inputs.cache-key }}-${{ github.repository }} - name: Run split id: split @@ -105,4 +109,4 @@ runs: uses: actions/cache/save@v4 with: path: ${{ inputs.timings-file }} - key: fairsplice-timings-${{ github.repository }}-${{ github.ref_name }}-${{ github.run_id }} + key: fairsplice-${{ inputs.cache-key }}-${{ github.repository }}-${{ github.run_id }} From f268bdfe037b0073fe4e62e105cf88f555f0dd27 Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Sun, 11 Jan 2026 16:17:55 +0100 Subject: [PATCH 18/28] Make cache-key required and fix cache restore - Make cache-key input required to prevent configuration mistakes - Fix cache key format: use run_id suffix for saves, prefix matching for restores - Add cache-key to test workflow - Document cache behavior in README (branch scoping, cross-branch sharing) - Add CLAUDE.md for Claude Code guidance Co-Authored-By: Claude Opus 4.5 --- .github/workflows/test.yml | 2 ++ CLAUDE.md | 57 ++++++++++++++++++++++++++++++++++++++ README.md | 14 ++++++++++ action.yml | 8 ++---- 4 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 CLAUDE.md diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c66a6d3..2160cde 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,7 @@ jobs: pattern: 'tests/dummy/*.test.ts' total: 3 index: ${{ matrix.index }} + cache-key: integration-tests - name: Show assigned tests run: echo "Worker ${{ matrix.index }} running:${{ steps.split.outputs.tests }}" @@ -88,6 +89,7 @@ jobs: with: command: merge prefix: 'timing-*/timing-' + cache-key: integration-tests - name: Show saved timings run: cat .fairsplice-timings.json diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..de045f4 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,57 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Fairsplice is a TypeScript/Bun CLI tool and GitHub Action that optimizes test distribution across parallel workers. It provides CircleCI-style test splitting based on historical timing data for GitHub Actions. + +## Commands + +```bash +# Run locally +bun run index.ts + +# Run all tests +bun test + +# Run tests in src directory +bun test src/ + +# Run a specific test file +bun test src/lib/splitFiles.test.ts + +# Compile to standalone binary +bun build ./index.ts --compile --outfile fairsplice +``` + +## Architecture + +**Entry Point**: `index.ts` - CLI with three commands: `split`, `convert`, `merge` + +**Source Structure**: +- `src/commands/` - CLI command implementations + - `split.ts` - Distributes test files across workers using bin packing + - `merge.ts` - Aggregates timing JSON files and updates history + - `convert.ts` - Converts JUnit XML to timing JSON +- `src/lib/` - Core algorithms + - `splitFiles.ts` - Greedy bin packing algorithm (assigns heaviest tests first to balance workload) + - `junit.ts` - JUnit XML parser using `fast-xml-parser` + - `average.ts` - Timing averaging utility +- `src/backend/` - Storage layer + - `fileStorage.ts` - JSON-based timing persistence with rolling window of last 10 timings per file +- `src/config.ts` - Constants (`NUMBER_OF_TIMINGS_TO_KEEP=10`, `DEFAULT_TIMING_IF_MISSING=10000ms`) + +**GitHub Action**: `action.yml` - Composite action wrapping the CLI with automatic cache handling + +**Data Flow**: +1. `split` loads cached timings, globs test files, applies bin packing, outputs bucket assignments +2. Tests run in parallel workers, each outputting JUnit XML +3. `convert` transforms JUnit XML to timing JSON (one per worker) +4. `merge` aggregates timing JSONs into cached timings history + +## Testing + +Tests are co-located with source files (`*.test.ts`). Test fixtures for JUnit parsing are in `src/lib/fixtures/`. + +The CI workflow (`.github/workflows/test.yml`) runs unit tests plus a 3-worker integration test that exercises the full split→run→convert→merge pipeline. diff --git a/README.md b/README.md index 52a6904..0fa14d1 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ jobs: pattern: 'tests/**/*.py' total: 3 index: ${{ matrix.index }} + cache-key: python-tests - name: Run tests run: pytest ${{ steps.split.outputs.tests }} --junit-xml=junit-${{ matrix.index }}.xml @@ -44,6 +45,7 @@ jobs: with: command: merge prefix: 'junit-*/junit-' + cache-key: python-tests ``` That's it! Caching is handled automatically. @@ -116,12 +118,24 @@ That's it! Caching is handled automatically. | Input | Required | Description | |-------|----------|-------------| | `command` | Yes | `split` or `merge` | +| `cache-key` | Yes | Cache key for storing timings (use different keys for frontend/backend workflows) | | `timings-file` | No | JSON file for timings (default: `.fairsplice-timings.json`) | | `pattern` | For split | Glob pattern to match test files | | `total` | For split | Total number of workers | | `index` | For split | Current worker index (0-based) | | `prefix` | For merge | Prefix to match JUnit XML files | +### Cache Behavior + +Fairsplice uses GitHub Actions cache for storing timing history. Important characteristics: + +- **Repository-scoped, branch-gated**: Caches are repository-scoped but restore access is gated by branch context +- **Default branch is global**: Caches saved from the default branch (usually `main`) are restorable by all branches +- **Immutable, single-writer**: Each cache key can only be written once; updates require a new key (handled automatically via run ID suffix) +- **Asymmetric cross-branch sharing**: Restore is permissive (branches can read from main), save is restricted (branches can only write to their own scope) + +To seed shared timings for all branches, run the workflow on `main` first. Subsequent PRs and feature branches will restore timings from main's cache. + ### Outputs | Output | Description | diff --git a/action.yml b/action.yml index 0f622d4..82a361b 100644 --- a/action.yml +++ b/action.yml @@ -14,8 +14,7 @@ inputs: default: '.fairsplice-timings.json' cache-key: description: 'Cache key for storing timings (use different keys for frontend/backend workflows)' - required: false - default: 'default' + required: true # split inputs pattern: description: 'Glob pattern to match test files (for split)' @@ -61,9 +60,8 @@ runs: uses: actions/cache/restore@v4 with: path: ${{ inputs.timings-file }} - key: fairsplice-${{ inputs.cache-key }}-${{ github.repository }} - restore-keys: | - fairsplice-${{ inputs.cache-key }}-${{ github.repository }} + key: fairsplice-${{ inputs.cache-key }}-${{ github.repository }}-${{ github.run_id }} + restore-keys: fairsplice-${{ inputs.cache-key }}-${{ github.repository }}- - name: Run split id: split From 3e8d5fa5fb64a7e4445f54ffb94d6994167154e7 Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Sun, 11 Jan 2026 16:20:50 +0100 Subject: [PATCH 19/28] Trigger CI to test cache restore From bb0ebb16d0545701266dca925e18d650718c47d3 Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Sun, 11 Jan 2026 16:26:37 +0100 Subject: [PATCH 20/28] docs: update README with convert command and fix action schema - Make cache-key not required at schema level (only needed for split/merge) - Add validation step to ensure cache-key is provided for split/merge - Update README example to show correct flow with convert step - Document convert command inputs in reference table - Fix merge description (expects timing JSON, not JUnit XML) Co-Authored-By: Claude Opus 4.5 --- README.md | 41 ++++++++++++++++++++++++++++++----------- action.yml | 11 +++++++++-- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 0fa14d1..dad816a 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,19 @@ jobs: cache-key: python-tests - name: Run tests - run: pytest ${{ steps.split.outputs.tests }} --junit-xml=junit-${{ matrix.index }}.xml + run: pytest ${{ steps.split.outputs.tests }} --junit-xml=junit.xml + + - name: Convert JUnit to timing JSON + uses: dashdoc/fairsplice@v1 + with: + command: convert + from: junit.xml + to: timing.json - uses: actions/upload-artifact@v4 with: - name: junit-${{ matrix.index }} - path: junit-${{ matrix.index }}.xml + name: timing-${{ matrix.index }} + path: timing.json save-timings: needs: test @@ -44,7 +51,7 @@ jobs: uses: dashdoc/fairsplice@v1 with: command: merge - prefix: 'junit-*/junit-' + prefix: 'timing-*/timing' cache-key: python-tests ``` @@ -107,7 +114,8 @@ That's it! Caching is handled automatically. **Key concepts:** - **Split phase**: Distributes test files across workers based on historical timing data -- **Merge phase**: Extracts timing from JUnit XML and caches for next run +- **Convert phase**: Extracts timing from JUnit XML into timing JSON (one per worker) +- **Merge phase**: Combines timing JSON files from all workers and caches for next run - **Bin packing**: Assigns tests to balance total execution time (heaviest tests first) - **Rolling average**: Keeps last 10 timings per test file for predictions @@ -117,13 +125,15 @@ That's it! Caching is handled automatically. | Input | Required | Description | |-------|----------|-------------| -| `command` | Yes | `split` or `merge` | -| `cache-key` | Yes | Cache key for storing timings (use different keys for frontend/backend workflows) | +| `command` | Yes | `split`, `convert`, or `merge` | +| `cache-key` | For split/merge | Cache key for storing timings (use different keys for frontend/backend workflows) | | `timings-file` | No | JSON file for timings (default: `.fairsplice-timings.json`) | | `pattern` | For split | Glob pattern to match test files | | `total` | For split | Total number of workers | | `index` | For split | Current worker index (0-based) | -| `prefix` | For merge | Prefix to match JUnit XML files | +| `from` | For convert | JUnit XML file to read | +| `to` | For convert | Timing JSON file to write | +| `prefix` | For merge | Prefix to match timing JSON files | ### Cache Behavior @@ -158,9 +168,14 @@ bunx fairsplice fairsplice split --timings-file timings.json --pattern "tests/**/*.py" --total 3 --out split.json ``` -**Merge results:** +**Convert JUnit XML to timing JSON:** ```bash -fairsplice merge --timings-file timings.json --prefix junit- +fairsplice convert --from junit.xml --to timing.json +``` + +**Merge timing results:** +```bash +fairsplice merge --timings-file timings.json --prefix timing- ``` ### CLI Options @@ -172,9 +187,13 @@ fairsplice split --total Number of workers --out Output JSON file +fairsplice convert + --from JUnit XML file to read + --to Timing JSON file to write + fairsplice merge --timings-file JSON file to store timings - --prefix Prefix to match JUnit XML files + --prefix Prefix to match timing JSON files ``` ## Contributing diff --git a/action.yml b/action.yml index 82a361b..8d1c01e 100644 --- a/action.yml +++ b/action.yml @@ -13,8 +13,8 @@ inputs: required: false default: '.fairsplice-timings.json' cache-key: - description: 'Cache key for storing timings (use different keys for frontend/backend workflows)' - required: true + description: 'Cache key for storing timings (required for split and merge commands)' + required: false # split inputs pattern: description: 'Glob pattern to match test files (for split)' @@ -48,6 +48,13 @@ outputs: runs: using: 'composite' steps: + - name: Validate cache-key for split/merge + if: (inputs.command == 'split' || inputs.command == 'merge') && inputs.cache-key == '' + shell: bash + run: | + echo "::error::cache-key is required for split and merge commands" + exit 1 + - name: Setup Bun uses: oven-sh/setup-bun@v2 From 711cd12892489a7bd26f8c619acbb030327accd6 Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Sun, 11 Jan 2026 17:01:18 +0100 Subject: [PATCH 21/28] fix: don't fail merge when no timing files found When all tests fail or are skipped, there won't be any timing artifacts. The merge command should handle this gracefully instead of failing. Co-Authored-By: Claude Opus 4.5 --- src/commands/merge.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commands/merge.ts b/src/commands/merge.ts index 1e86529..f989de3 100644 --- a/src/commands/merge.ts +++ b/src/commands/merge.ts @@ -14,7 +14,8 @@ export async function merge({ if (files.length === 0) { console.warn(`No files found matching prefix: ${prefix}*`); - process.exit(1); + console.warn(`Skipping merge (this is normal if all tests failed or were skipped)`); + return; } console.log(`Found ${files.length} timing files to merge:`); From b663e47397e8e3948c49d223ccf1234cad54e58c Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Sun, 11 Jan 2026 17:04:33 +0100 Subject: [PATCH 22/28] fix: don't fail convert when input file doesn't exist When tests are skipped or fail early, the JUnit XML file won't exist. The convert command should handle this gracefully instead of failing. Co-Authored-By: Claude Opus 4.5 --- src/commands/convert.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/commands/convert.ts b/src/commands/convert.ts index cb393e7..f82e9e7 100644 --- a/src/commands/convert.ts +++ b/src/commands/convert.ts @@ -7,8 +7,15 @@ export async function convert({ from: string; to: string; }) { - // read junit xml file + // check if input file exists const junitXmlFile = Bun.file(from); + if (!(await junitXmlFile.exists())) { + console.warn(`Input file not found: ${from}`); + console.warn(`Skipping convert (this is normal if tests were skipped or failed early)`); + return; + } + + // read junit xml file const xmlString = await junitXmlFile.text(); // parse junit xml From f9b3d60a0f4bfef4fc752c0264d7183502bb203d Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Sun, 11 Jan 2026 17:54:07 +0100 Subject: [PATCH 23/28] fix: pin bun version to 1.3.5 --- action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/action.yml b/action.yml index 8d1c01e..8e33340 100644 --- a/action.yml +++ b/action.yml @@ -57,6 +57,8 @@ runs: - name: Setup Bun uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.5" - name: Install dependencies shell: bash From 5d0ce25950ffe812bbbb6b518e7ca7b6b752f6ac Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Sun, 11 Jan 2026 17:55:22 +0100 Subject: [PATCH 24/28] fix: pin bun to 1.2.23 for dashdoc compatibility --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 8e33340..d5dfe1a 100644 --- a/action.yml +++ b/action.yml @@ -58,7 +58,7 @@ runs: - name: Setup Bun uses: oven-sh/setup-bun@v2 with: - bun-version: "1.3.5" + bun-version: "1.2.23" - name: Install dependencies shell: bash From 230d1a5a9e0f62b9b3b9951e44f1b24986594eec Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Sun, 11 Jan 2026 18:41:32 +0100 Subject: [PATCH 25/28] feat: add path-prefix option to convert command Playwright and other test runners may output paths relative to their config directory, but split patterns use repo-root paths. The path-prefix option allows prepending a prefix to all file paths during conversion so they match the split pattern. Example: --path-prefix "frontends/apps/e2e/" converts "tests/foo.spec.ts" to "frontends/apps/e2e/tests/foo.spec.ts" Co-Authored-By: Claude Opus 4.5 --- action.yml | 6 +++++- bun.lockb | Bin 3949 -> 3477 bytes index.ts | 14 ++++++++++---- src/commands/convert.ts | 10 +++++++--- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/action.yml b/action.yml index d5dfe1a..ff5ce72 100644 --- a/action.yml +++ b/action.yml @@ -32,6 +32,9 @@ inputs: to: description: 'Timing JSON file to write (for convert)' required: false + path-prefix: + description: 'Prefix to prepend to file paths (for convert) - use to match paths with split pattern' + required: false # merge inputs prefix: description: 'Prefix to match timing JSON files (for merge)' @@ -101,7 +104,8 @@ runs: run: | bun run ${{ github.action_path }}/index.ts convert \ --from "${{ inputs.from }}" \ - --to "${{ inputs.to }}" + --to "${{ inputs.to }}" \ + ${{ inputs.path-prefix != '' && format('--path-prefix "{0}"', inputs.path-prefix) || '' }} - name: Run merge if: inputs.command == 'merge' diff --git a/bun.lockb b/bun.lockb index daa370569f1ae24f2dd0eaf6e6409e600828480d..40412cb9bc63a15ca986f5cd2dc27c4a01b353a3 100755 GIT binary patch delta 1060 zcmaDWH&uFqmM#+m1B0nmhHdMeYkH|_mm+V@_-ox4Ir-mt#;Zv`7a!cyu$o7Rl^3W8 z2skE&%QIzgOsufrEM;V1;AUWG*fH^-`6R}G$qN`cCMPiRuyO*GF;70os6TlHkjDz; zS+fF_GcbY0Etq&%nSngk$$?D9yjDOnKynZfpgutm*8(Ui2^0kZHjpmfBS29{peQ>~ zL}0QY(EK^f8oVH#Y(Na6*(XnAHbxUsWGSidi_K3rKHOFG`s-8OJM-VZjcaYVXz-Qe zJewR+o7jI^+3@!?N9Jf!Jt6oy#ebR+# zT(dNdCh`0aD9K(MCusG|W$~9UU6XZLCF=KWu4CVDP^sl|n&o-ZHwNu;b6EI~|7h90 z_Uo=sb=QyR7QK6=B*{>ivF6pzgHsw;U%x)xvq^FDmQ6MLrCZK(HI^W`js@m4AkE@d zvw&-Hc6qh7!bdUId)}f?C8T$yay5QS<`2u9xXCO%Sn$QZ#;t4q@N3S%o8lk!uA9 z#Kn6!CU57olKS@_0ziVGboc@)&B8gEkIRnB0F?HDzz4)tn%ux8A_Vd@NM!?5%8YCB zI<8)3Lp{UEcHCm@#(Jhe$$CgfUe;Ism!-I_9}4JZYIV$b3H>dJp+agpyNQ` z1lQy<+%BMuGMS&pk?RLk_%F}oD4r>dmXqJ}1T&lJ8Bez2lb@WzD+IK~2*_5OJfD|` z8EDhwb-WHjKw7UPy~?4avLLlsKQBKeb+R3wKC7vofu7mq5h@BuclN;G*004R; BTI2u# delta 1448 zcmZ`(4NQ|q7{0G9&_4>6LMSkhpTGvvU&_C;7$P$$7C}%Dm`l1cbOS{<^_}$daY*uC#_N+ml@0%X82BzR!2}-Cc3k zyME@&BLe_}@7{5>9{b2GraFIUc;NnzH*LvVGC#bWE%&S~PhPn1HNiq61RBuF9JD!9 zQY%ec7`*9I0#iU!fFc9MHEm}Q^~MiXsxPxfj%k-<(3YS=DZ#PExwsDo84`pK)clL3 z7}*{ZyRHPl0<$_2$HGWJEh&f_hI)meUMTzn&c*^#g32vJO94nhHLb5F13-VL=t7E6 zq~Mqo$h#4C?-Udg01-%G&UG>=upvK4A-`{r1i%Un>;regkp}-HiT`IsMOH`~^J`mg z?;2T@Mlb9hBDyIl@M%+V*zQ7k%*x*0qK4$>ot);S`oCBiO<&xb9MRe^^G6d;9wge+ z%S5WWVV`8^PjOT~7+tEF&^;}mA{QQVI~?Aem^LQMs%b6`)>J>Ti*?Xv3H65S%?57& zE2oB;6tSgUiLP{%FmMmrJJLz^EY1u`uXvX4@)v$ z`RJ%#Yu`nTvkPx&IUCHGx87(@ZCN_L>w!n^S}l#EoDvM?&h*I>4!Csh%v3}W0zIJy z+}S2SL#ZRZXJ&*x9bWmp+s|(OTK!U`}L^_(c9<*ohQ}C(776WBV=EFVxxOW+vDUqD*OCJ_13O46vwHyl@0Fj z(VIc}-OeXIitC^!m+kP}KNs855f>i0Q1dXgDRMO_(z8|M#kLSVkte9zw~d}3=<}Y8 z3*4EPz4SUwIbZZ>X~H+DaYJGAUyF-Vyk+HKzt$_Z)0<;Y!s~7c8csr{Sl=HuxNZ7% zfubeoR-5~b>`KftFaEa|st%|u4)RMrY1g^GneL9ZKRSQ90oC0A$RcPk-trrCEjb;w zR{IH9M;6kT&<)0*(BUa+=zC~pc)JeFVmkslo&g}ELl(^g&Qe`inAVu+1|yCCCCI1Q zJK>$NL<2`Gdv22TGGGF&LM`A)01zQ#_~AISD1TPMUhp6hLY{yp3If3VlxX=$72fx@ zhLDINk$@)@^2FxH#{9mZm0+HPClvfgjxXwqv4;9EQ6Jig@PYZ*>W3tPT_oX&0(k+z z7V#J|CX~`6VF_JI*O;h&7962h5@IN4#F53k0JzL>gX0V~8F@!@5sHD(n<;g{mU#;K zX{kb$n~}%QIgy@Wnkw_4kl_q%nN}=i9^ap@%=2G2i}Jl;HPhP4_))?Tc$*pJWV9r~ VmDW80a3tnvRKec77`Vx*>ThBEs%8KH diff --git a/index.ts b/index.ts index 4d801e1..e9c62ee 100755 --- a/index.ts +++ b/index.ts @@ -19,6 +19,9 @@ const { positionals, values } = parseArgs({ to: { type: "string", }, + ["path-prefix"]: { + type: "string", + }, // merge options ["timings-file"]: { type: "string", @@ -78,10 +81,13 @@ fairsplice convert Convert JUnit XML to timing JSON (for a single worker). Required options: - --from JUnit XML file to read - --to Timing JSON file to write + --from JUnit XML file to read + --to Timing JSON file to write + +Optional: + --path-prefix Prefix to prepend to all file paths (e.g., "src/tests/") -Example: fairsplice convert --from junit.xml --to timing.json +Example: fairsplice convert --from junit.xml --to timing.json --path-prefix "frontends/apps/e2e/" fairsplice merge @@ -125,7 +131,7 @@ if (command === "split") { ); process.exit(1); } - await convert({ from: values.from, to: values.to }); + await convert({ from: values.from, to: values.to, pathPrefix: values["path-prefix"] }); process.exit(0); } else if (command === "merge") { if (!values["timings-file"] || !values.prefix) { diff --git a/src/commands/convert.ts b/src/commands/convert.ts index f82e9e7..cb031e6 100644 --- a/src/commands/convert.ts +++ b/src/commands/convert.ts @@ -3,9 +3,11 @@ import { parseJunit } from "../lib/junit"; export async function convert({ from, to, + pathPrefix, }: { from: string; to: string; + pathPrefix?: string; }) { // check if input file exists const junitXmlFile = Bun.file(from); @@ -27,10 +29,12 @@ export async function convert({ if (testCase.file.includes("..")) { continue; } - if (!timingByFile[testCase.file]) { - timingByFile[testCase.file] = 0; + // Apply path prefix if provided + const filePath = pathPrefix ? `${pathPrefix}${testCase.file}` : testCase.file; + if (!timingByFile[filePath]) { + timingByFile[filePath] = 0; } - timingByFile[testCase.file] += testCase.time; + timingByFile[filePath] += testCase.time; } // convert to ms From 943436153a9be9d9375c8cc868336577584c357b Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Sun, 11 Jan 2026 21:38:08 +0100 Subject: [PATCH 26/28] fix: add run_attempt to cache key for re-run support GitHub cache keys are immutable, so re-running a workflow would fail to save because the key already exists. Adding run_attempt ensures each attempt gets a unique key. Co-Authored-By: Claude Opus 4.5 --- action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index ff5ce72..8a84ea7 100644 --- a/action.yml +++ b/action.yml @@ -72,7 +72,7 @@ runs: uses: actions/cache/restore@v4 with: path: ${{ inputs.timings-file }} - key: fairsplice-${{ inputs.cache-key }}-${{ github.repository }}-${{ github.run_id }} + key: fairsplice-${{ inputs.cache-key }}-${{ github.repository }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: fairsplice-${{ inputs.cache-key }}-${{ github.repository }}- - name: Run split @@ -120,4 +120,4 @@ runs: uses: actions/cache/save@v4 with: path: ${{ inputs.timings-file }} - key: fairsplice-${{ inputs.cache-key }}-${{ github.repository }}-${{ github.run_id }} + key: fairsplice-${{ inputs.cache-key }}-${{ github.repository }}-${{ github.run_id }}-${{ github.run_attempt }} From c94e522fbca7755d00fec6337a01547f3a2978b8 Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Mon, 12 Jan 2026 11:04:01 +0100 Subject: [PATCH 27/28] docs: add path-prefix option to README Co-Authored-By: Claude Opus 4.5 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index dad816a..bd5b72e 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,7 @@ That's it! Caching is handled automatically. | `index` | For split | Current worker index (0-based) | | `from` | For convert | JUnit XML file to read | | `to` | For convert | Timing JSON file to write | +| `path-prefix` | For convert | Prefix to prepend to file paths (to match split pattern) | | `prefix` | For merge | Prefix to match timing JSON files | ### Cache Behavior @@ -190,6 +191,7 @@ fairsplice split fairsplice convert --from JUnit XML file to read --to Timing JSON file to write + --path-prefix Prefix to prepend to file paths fairsplice merge --timings-file JSON file to store timings From 0842c1970feff83c2427ea2a74aabcc3982c883b Mon Sep 17 00:00:00 2001 From: Corentin Smith Date: Wed, 14 Jan 2026 11:15:05 +0100 Subject: [PATCH 28/28] docs: recommend computing splits once for consistent re-runs Update Quick Start to show the recommended pattern of computing splits in a dedicated job and passing to test jobs via outputs. This ensures that re-running a failed job runs the same tests, since GitHub Actions preserves workflow outputs on re-runs. Co-Authored-By: Claude Opus 4.5 --- README.md | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bd5b72e..fc6dcb1 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,15 @@ Fairsplice is a CLI tool and GitHub Action that optimizes test distribution acro ## Quick Start (GitHub Action) +**Recommended:** Compute splits once and pass to test jobs. This ensures re-running a failed job runs the same tests. + ```yaml jobs: - test: + # Compute splits once - ensures consistent re-runs + compute-splits: runs-on: ubuntu-latest - strategy: - matrix: - index: [0, 1, 2] + outputs: + test-buckets: ${{ steps.split.outputs.buckets }} steps: - uses: actions/checkout@v4 @@ -21,8 +23,22 @@ jobs: command: split pattern: 'tests/**/*.py' total: 3 - index: ${{ matrix.index }} cache-key: python-tests + # No index = outputs all buckets as JSON array + + test: + needs: compute-splits + runs-on: ubuntu-latest + strategy: + matrix: + index: [0, 1, 2] + steps: + - uses: actions/checkout@v4 + + - name: Get test files + id: split + run: | + echo "tests=$(echo '${{ needs.compute-splits.outputs.test-buckets }}' | jq -r '.[${{ matrix.index }}] | join(" ")')" >> "$GITHUB_OUTPUT" - name: Run tests run: pytest ${{ steps.split.outputs.tests }} --junit-xml=junit.xml @@ -57,6 +73,15 @@ jobs: That's it! Caching is handled automatically. +### Why compute splits once? + +When you compute splits inside each matrix job (using `index`), re-running a failed job can run different tests: +1. Other jobs may have updated the timing cache +2. The re-run computes a new split with updated timings +3. The failed test might now be assigned to a different worker + +By computing splits once in a dedicated job and passing via workflow outputs, GitHub Actions preserves the same split on re-runs. + ## How It Works ```