From 41ebc01c10ee410024305e2739c1de26f2bea376 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 23 Jul 2026 09:26:21 -0400 Subject: [PATCH 1/3] chore: bump mongodb and bson dependencies --- package-lock.json | 120 +++++++++++++++--- package.json | 2 +- packages/devtools-connect/package.json | 10 +- packages/mongodb-constants/package.json | 2 +- packages/mongodb-runner/package.json | 2 +- packages/mongodb-ts-autocomplete/package.json | 10 +- packages/mql-typescript/package.json | 6 +- 7 files changed, 116 insertions(+), 36 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7af72e86..b9c05bec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11771,6 +11771,7 @@ "version": "6.10.4", "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz", "integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==", + "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=16.20.1" @@ -21178,14 +21179,14 @@ } }, "node_modules/mongodb": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.2.0.tgz", - "integrity": "sha512-F/2+BMZtLVhY30ioZp0dAmZ+IRZMBqI+nrv6t5+9/1AIwCa8sMRC3jBf81lpxMhnZgqq8CoUD503Z1oZWq1/sw==", + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.5.0.tgz", + "integrity": "sha512-5FnrEDLnvp6ycUOGLNLLU33BfCx2qmp2mJjGPDwKLruYsVzXVSK5fsGpoDXvsXJwBfBsD7ebMRdawbDxC2814g==", "license": "Apache-2.0", "dependencies": { - "@mongodb-js/saslprep": "^1.3.0", + "@mongodb-js/saslprep": "^1.4.11", "bson": "^7.2.0", - "mongodb-connection-string-url": "^7.0.0" + "mongodb-connection-string-url": "^7.0.1" }, "engines": { "node": ">=20.19.0" @@ -21195,7 +21196,7 @@ "@mongodb-js/zstd": "^7.0.0", "gcp-metadata": "^7.0.1", "kerberos": "^7.0.0", - "mongodb-client-encryption": ">=7.0.0 <7.1.0", + "mongodb-client-encryption": "^7.2.0", "snappy": "^7.3.2", "socks": "^2.8.6" }, @@ -28246,7 +28247,7 @@ "eslint-plugin-promise": "^6.1.1", "gen-esm-wrapper": "^1.1.3", "mocha": "^8.4.0", - "mongodb": "^7.2.0", + "mongodb": "^7.5.0", "mongodb-log-writer": "^2.5.13", "mongodb-runner": "^6.8.3", "mongodb6": "npm:mongodb@^6.9.0", @@ -28793,6 +28794,16 @@ "@types/sinon": "*" } }, + "packages/mongodb-constants/node_modules/bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0" + } + }, "packages/mongodb-constants/node_modules/semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", @@ -29029,7 +29040,7 @@ "@mongodb-js/saslprep": "^1.4.12", "@peculiar/x509": "^2.0.0", "debug": "^4.4.0", - "mongodb": "^7.2.0", + "mongodb": "^7.5.0", "mongodb-connection-string-url": "^3.0.0 || ^7.0.0", "reflect-metadata": "^0.2.2", "yargs": "^18.0.0" @@ -29288,7 +29299,7 @@ "eslint": "^7.25.0 || ^8.0.0", "gen-esm-wrapper": "^1.1.3", "mocha": "^8.4.0", - "mongodb": "^7.2.0", + "mongodb": "^7.5.0", "nyc": "^15.1.0", "prettier": "^3.8.1", "sinon": "^9.2.3", @@ -29316,6 +29327,16 @@ "@types/sinon": "*" } }, + "packages/mongodb-ts-autocomplete/node_modules/bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0" + } + }, "packages/mongodb-ts-autocomplete/node_modules/undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", @@ -29641,7 +29662,7 @@ "license": "Apache-2.0", "dependencies": { "bson": "^4.6.3 || ^5 || ^6.10.3 || ^7.0.0", - "mongodb": "^7.2.0" + "mongodb": "^7.5.0" }, "bin": { "mql-typescript": "bin/runner.js" @@ -29722,6 +29743,15 @@ "dev": true, "license": "Python-2.0" }, + "packages/mql-typescript/node_modules/bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0" + } + }, "packages/mql-typescript/node_modules/cliui": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", @@ -30251,6 +30281,16 @@ "@types/sinon": "*" } }, + "packages/query-parser/node_modules/bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0" + } + }, "packages/query-parser/node_modules/undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", @@ -30660,6 +30700,16 @@ "@types/sinon": "*" } }, + "packages/shell-bson-parser/node_modules/bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0" + } + }, "packages/signing-utils": { "name": "@mongodb-js/signing-utils", "version": "0.5.12", @@ -36058,7 +36108,7 @@ "kerberos": "^2.1.0 || ^7.0.0", "lodash.merge": "^4.6.2", "mocha": "^8.4.0", - "mongodb": "^7.2.0", + "mongodb": "^7.5.0", "mongodb-client-encryption": "^6.5.0 || ^7.0.0", "mongodb-connection-string-url": "^3.0.1 || ^7.0.0", "mongodb-log-writer": "^2.5.13", @@ -36423,6 +36473,12 @@ "@types/sinon": "*" } }, + "bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "dev": true + }, "semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", @@ -36589,7 +36645,7 @@ "gen-esm-wrapper": "^1.1.3", "lodash": "^4.17.21", "mocha": "^8.4.0", - "mongodb": "^7.2.0", + "mongodb": "^7.5.0", "mongodb-schema": "^12.6.2", "node-cache": "^5.1.2", "nyc": "^15.1.0", @@ -36618,6 +36674,12 @@ "@types/sinon": "*" } }, + "bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "dev": true + }, "undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", @@ -36836,7 +36898,7 @@ "js-yaml": "^4.1.1", "jsdom": "^24.1.3", "json-schema-to-zod": "^2.6.0", - "mongodb": "^7.2.0", + "mongodb": "^7.5.0", "mongodb-schema": "^12.6.2", "prettier": "^3.8.1", "ts-node": "^10.9.2", @@ -36880,6 +36942,11 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, + "bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==" + }, "cliui": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", @@ -37596,6 +37663,12 @@ "@types/chai": "*", "@types/sinon": "*" } + }, + "bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "dev": true } } }, @@ -41312,7 +41385,8 @@ "bson": { "version": "6.10.4", "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz", - "integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==" + "integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==", + "devOptional": true }, "buffer": { "version": "5.7.1", @@ -48035,13 +48109,13 @@ "dev": true }, "mongodb": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.2.0.tgz", - "integrity": "sha512-F/2+BMZtLVhY30ioZp0dAmZ+IRZMBqI+nrv6t5+9/1AIwCa8sMRC3jBf81lpxMhnZgqq8CoUD503Z1oZWq1/sw==", + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.5.0.tgz", + "integrity": "sha512-5FnrEDLnvp6ycUOGLNLLU33BfCx2qmp2mJjGPDwKLruYsVzXVSK5fsGpoDXvsXJwBfBsD7ebMRdawbDxC2814g==", "requires": { - "@mongodb-js/saslprep": "^1.3.0", + "@mongodb-js/saslprep": "^1.4.11", "bson": "^7.2.0", - "mongodb-connection-string-url": "^7.0.0" + "mongodb-connection-string-url": "^7.0.1" }, "dependencies": { "bson": { @@ -48364,6 +48438,12 @@ "@types/sinon": "*" } }, + "bson": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-7.3.1.tgz", + "integrity": "sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==", + "dev": true + }, "undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", @@ -48431,7 +48511,7 @@ "eslint": "^7.25.0 || ^8.0.0", "gen-esm-wrapper": "^1.1.3", "mocha": "^8.4.0", - "mongodb": "^7.2.0", + "mongodb": "^7.5.0", "mongodb-connection-string-url": "^3.0.0 || ^7.0.0", "nyc": "^15.1.0", "prettier": "^3.8.1", diff --git a/package.json b/package.json index 2edbe4b8..8dc49b84 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "test-changed": "lerna run test --stream --concurrency 1 --since origin/HEAD", "test-ci": "lerna run test-ci --concurrency 1", "test": "lerna run test --concurrency 1 --stream", - "where": "node ./scripts/src/where.js", + "where": "node packages/monorepo-tools/dist/where.js", "request-npm-token": "request-npm-token" }, "dependencies": { diff --git a/packages/devtools-connect/package.json b/packages/devtools-connect/package.json index 929578eb..4ec8a9e0 100644 --- a/packages/devtools-connect/package.json +++ b/packages/devtools-connect/package.json @@ -47,8 +47,8 @@ }, "license": "Apache-2.0", "dependencies": { - "@mongodb-js/oidc-http-server-pages": "1.2.12", "@mongodb-js/devtools-proxy-support": "^0.7.17", + "@mongodb-js/oidc-http-server-pages": "1.2.12", "lodash.merge": "^4.6.2", "mongodb-connection-string-url": "^3.0.1 || ^7.0.0", "socks": "^2.7.3" @@ -75,18 +75,18 @@ "eslint-plugin-promise": "^6.1.1", "gen-esm-wrapper": "^1.1.3", "mocha": "^8.4.0", - "mongodb": "^7.2.0", - "mongodb6": "npm:mongodb@^6.9.0", + "mongodb": "^7.5.0", "mongodb-log-writer": "^2.5.13", "mongodb-runner": "^6.8.3", + "mongodb6": "npm:mongodb@^6.9.0", "nyc": "^15.1.0", "os-dns-native": "^2.0.1", + "prettier": "^3.8.1", "resolve-mongodb-srv": "^1.1.1", "sinon-chai": "^4.0.1", "ts-node": "^10.9.2", "ts-sinon": "^2.0.1", - "typescript": "^5.9.3", - "prettier": "^3.8.1" + "typescript": "^5.9.3" }, "optionalDependencies": { "kerberos": "^2.1.0 || ^7.0.0", diff --git a/packages/mongodb-constants/package.json b/packages/mongodb-constants/package.json index cfe273ef..16cdd779 100644 --- a/packages/mongodb-constants/package.json +++ b/packages/mongodb-constants/package.json @@ -63,8 +63,8 @@ "@types/mocha": "^9.1.1", "@types/semver": "^7.7.1", "@types/sinon-chai": "^4.0.0", - "bson": "^4.6.3 || ^5 || ^6.10.3 || ^7.0.0", "acorn": "^8.16.0", + "bson": "^4.6.3 || ^5 || ^6.10.3 || ^7.0.0", "chai": "^4.5.0", "depcheck": "^1.4.7", "eslint": "^7.25.0 || ^8.0.0", diff --git a/packages/mongodb-runner/package.json b/packages/mongodb-runner/package.json index 4a56cca3..45d8ece6 100644 --- a/packages/mongodb-runner/package.json +++ b/packages/mongodb-runner/package.json @@ -61,7 +61,7 @@ "@mongodb-js/saslprep": "^1.4.12", "@peculiar/x509": "^2.0.0", "debug": "^4.4.0", - "mongodb": "^7.2.0", + "mongodb": "^7.5.0", "mongodb-connection-string-url": "^3.0.0 || ^7.0.0", "reflect-metadata": "^0.2.2", "yargs": "^18.0.0" diff --git a/packages/mongodb-ts-autocomplete/package.json b/packages/mongodb-ts-autocomplete/package.json index ec2223d3..318e4efc 100644 --- a/packages/mongodb-ts-autocomplete/package.json +++ b/packages/mongodb-ts-autocomplete/package.json @@ -55,29 +55,29 @@ "dependencies": { "@mongodb-js/ts-autocomplete": "^0.5.12", "@mongosh/shell-api": "^5.1.6", + "debug": "^4.4.0", + "lodash": "^4.17.21", "mongodb-schema": "^12.6.2", "node-cache": "^5.1.2", - "typescript": "^5.9.3", - "debug": "^4.4.0", - "lodash": "^4.17.21" + "typescript": "^5.9.3" }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "^0.11.7", "@mongodb-js/mocha-config-devtools": "^1.1.2", + "@mongodb-js/mql-typescript": "^0.6.0", "@mongodb-js/prettier-config-devtools": "^1.0.3", "@mongodb-js/tsconfig-devtools": "^1.1.2", - "@mongodb-js/mql-typescript": "^0.6.0", "@types/chai": "^4.2.21", "@types/mocha": "^9.1.1", "@types/node": "^25.5.2", "@types/sinon-chai": "^4.0.0", "bson": "^4.6.3 || ^5 || ^6.10.3 || ^7.0.0", - "mongodb": "^7.2.0", "chai": "^4.5.0", "depcheck": "^1.4.7", "eslint": "^7.25.0 || ^8.0.0", "gen-esm-wrapper": "^1.1.3", "mocha": "^8.4.0", + "mongodb": "^7.5.0", "nyc": "^15.1.0", "prettier": "^3.8.1", "sinon": "^9.2.3", diff --git a/packages/mql-typescript/package.json b/packages/mql-typescript/package.json index a3098601..ed7e0f8c 100644 --- a/packages/mql-typescript/package.json +++ b/packages/mql-typescript/package.json @@ -74,25 +74,25 @@ "@mongodb-js/prettier-config-devtools": "^1.0.3", "@mongodb-js/shell-bson-parser": "^1.5.13", "@mongodb-js/tsconfig-devtools": "^1.1.2", + "@types/chai": "^4.2.21", "@types/js-yaml": "^4.0.9", "@types/jsdom": "^21.1.7", "@types/node": "^25.5.2", - "@types/chai": "^4.2.21", "depcheck": "^1.4.7", "eslint": "^7.25.0 || ^8.0.0", "gen-esm-wrapper": "^1.1.3", "js-yaml": "^4.1.1", "jsdom": "^24.1.3", "json-schema-to-zod": "^2.6.0", + "mongodb-schema": "^12.6.2", "prettier": "^3.8.1", "ts-node": "^10.9.2", "typescript": "^5.9.3", - "mongodb-schema": "^12.6.2", "yargs": "^18.0.0", "zod": "^3.24.2" }, "dependencies": { "bson": "^4.6.3 || ^5 || ^6.10.3 || ^7.0.0", - "mongodb": "^7.2.0" + "mongodb": "^7.5.0" } } From abfe41bf9fe609d17da5a34ef82502edbb466c26 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 23 Jul 2026 11:57:20 -0400 Subject: [PATCH 2/3] let's try sharding --- .github/workflows/check-test.yaml | 30 ++++++++++- .github/workflows/compute-test-shard.mjs | 66 ++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/compute-test-shard.mjs diff --git a/.github/workflows/check-test.yaml b/.github/workflows/check-test.yaml index a6dadfa3..860e5b1e 100644 --- a/.github/workflows/check-test.yaml +++ b/.github/workflows/check-test.yaml @@ -25,10 +25,16 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] + # Tests are sharded across hosts by package (see compute-test-shard.mjs). + # When changing the shard count, update every list entry below. + shard: [1, 2, 3, 4] fail-fast: false runs-on: ${{ matrix.os }} + env: + TOTAL_SHARDS: 4 + # Steps represent a sequence of tasks that will be executed as part of the job steps: - name: Sets env vars @@ -90,14 +96,21 @@ jobs: npx lerna@${LERNA_VERSION} ls --all --since ${SINCE_REF} shell: bash + # Checks are not sharded; run them once per OS on the first shard. - name: Run Checks + if: ${{ matrix.shard == 1 }} run: npm run check-ci -- --stream shell: bash - name: Run Tests - run: npm run test-ci -- --stream + run: | + scopes=$(node .github/workflows/compute-test-shard.mjs "${{ matrix.shard }}" "${TOTAL_SHARDS}") + echo "Testing shard ${{ matrix.shard }}/${TOTAL_SHARDS} with: ${scopes}" + npm run test-ci -- --stream ${scopes} shell: bash + # Each shard uploads coverage for the packages it tested; the parallel + # build is finalized once by the separate coverage-finalize job below. - name: Report Coverage if: ${{ runner.os == 'Linux' }} run: | @@ -108,7 +121,22 @@ jobs: flag_name=$(sed -E 's/packages\/([^\/]*)\/coverage\/lcov.info/\1/g' <<<$report) coveralls report --base-path . --job-flag=$flag_name $report --parallel --no-logo done + env: + COVERALLS_GIT_BRANCH: ${{ github.head_ref || github.ref_name }} + COVERALLS_REPO_TOKEN: ${{ github.token }} + COVERALLS_GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + # Signals to Coveralls that all parallel coverage uploads from the sharded + # check-and-test jobs are complete. + coverage-finalize: + name: Finalize Coverage + needs: check-and-test + if: ${{ always() && needs.check-and-test.result != 'skipped' }} + runs-on: ubuntu-latest + steps: + - name: Finalize Coveralls parallel build + run: | + curl -L https://coveralls.io/coveralls-linux.tar.gz | tar -xz -C /usr/local/bin coveralls done env: COVERALLS_GIT_BRANCH: ${{ github.head_ref || github.ref_name }} diff --git a/.github/workflows/compute-test-shard.mjs b/.github/workflows/compute-test-shard.mjs new file mode 100644 index 00000000..ebcf4b70 --- /dev/null +++ b/.github/workflows/compute-test-shard.mjs @@ -0,0 +1,66 @@ +#! /usr/bin/env node +// Prints the `lerna run` filter flags (`--scope ...`) for the packages +// that belong to a given test shard. Packages are distributed round-robin over +// the shards after sorting by name, so the assignment is stable across runs and +// every package with a `test-ci` script lands in exactly one shard. +// +// Usage: node compute-test-shard.mjs +// shardIndex is 1-based. + +import { readFileSync, readdirSync, existsSync } from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const shardIndex = Number.parseInt(process.argv[2], 10); +const totalShards = Number.parseInt(process.argv[3], 10); + +if ( + !Number.isInteger(shardIndex) || + !Number.isInteger(totalShards) || + shardIndex < 1 || + totalShards < 1 || + shardIndex > totalShards +) { + throw new Error( + `Invalid shard arguments: shardIndex=${process.argv[2]} totalShards=${process.argv[3]} ` + + `(expected 1 <= shardIndex <= totalShards)`, + ); +} + +// Resolve the repo root relative to this script (.github/workflows/). +const repoRoot = path.resolve(fileURLToPath(import.meta.url), '../../..'); + +// Keep this in sync with the `packages` globs in lerna.json. +const workspaceGlobs = ['packages', 'configs']; +const explicitDirs = ['scripts']; + +const packageDirs = [ + ...workspaceGlobs.flatMap((glob) => { + const base = path.join(repoRoot, glob); + if (!existsSync(base)) return []; + return readdirSync(base, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => path.join(base, entry.name)); + }), + ...explicitDirs.map((dir) => path.join(repoRoot, dir)), +]; + +const testablePackages = packageDirs + .map((dir) => path.join(dir, 'package.json')) + .filter((manifestPath) => existsSync(manifestPath)) + .map((manifestPath) => JSON.parse(readFileSync(manifestPath, 'utf8'))) + .filter((manifest) => manifest.scripts && manifest.scripts['test-ci']) + .map((manifest) => manifest.name) + .sort(); + +const shardPackages = testablePackages.filter( + (_name, index) => index % totalShards === shardIndex - 1, +); + +if (shardPackages.length === 0) { + // Emit a filter that matches nothing so `lerna run` becomes a no-op rather + // than falling back to running every package. + process.stdout.write('--scope __no_packages_in_this_shard__'); +} else { + process.stdout.write(shardPackages.map((name) => `--scope ${name}`).join(' ')); +} From fe99fe025d3700ac45bcd37c40fddcfba6f98025 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 23 Jul 2026 13:14:01 -0400 Subject: [PATCH 3/3] shard -> group --- .github/workflows/check-test.yaml | 21 +++++----- ...-test-shard.mjs => compute-test-group.mjs} | 38 +++++++++---------- 2 files changed, 30 insertions(+), 29 deletions(-) rename .github/workflows/{compute-test-shard.mjs => compute-test-group.mjs} (60%) diff --git a/.github/workflows/check-test.yaml b/.github/workflows/check-test.yaml index 860e5b1e..c1db3b68 100644 --- a/.github/workflows/check-test.yaml +++ b/.github/workflows/check-test.yaml @@ -25,15 +25,16 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - # Tests are sharded across hosts by package (see compute-test-shard.mjs). - # When changing the shard count, update every list entry below. - shard: [1, 2, 3, 4] + # Tests are split across hosts into groups by package (see + # compute-test-group.mjs). When changing the group count, keep + # TOTAL_GROUPS below in sync with the number of list entries here. + group: [1, 2, 3, 4] fail-fast: false runs-on: ${{ matrix.os }} env: - TOTAL_SHARDS: 4 + TOTAL_GROUPS: 4 # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -96,20 +97,20 @@ jobs: npx lerna@${LERNA_VERSION} ls --all --since ${SINCE_REF} shell: bash - # Checks are not sharded; run them once per OS on the first shard. + # Checks are not split into groups; run them once per OS on the first group. - name: Run Checks - if: ${{ matrix.shard == 1 }} + if: ${{ matrix.group == 1 }} run: npm run check-ci -- --stream shell: bash - name: Run Tests run: | - scopes=$(node .github/workflows/compute-test-shard.mjs "${{ matrix.shard }}" "${TOTAL_SHARDS}") - echo "Testing shard ${{ matrix.shard }}/${TOTAL_SHARDS} with: ${scopes}" + scopes=$(node .github/workflows/compute-test-group.mjs "${{ matrix.group }}" "${TOTAL_GROUPS}") + echo "Testing group ${{ matrix.group }}/${TOTAL_GROUPS} with: ${scopes}" npm run test-ci -- --stream ${scopes} shell: bash - # Each shard uploads coverage for the packages it tested; the parallel + # Each group uploads coverage for the packages it tested; the parallel # build is finalized once by the separate coverage-finalize job below. - name: Report Coverage if: ${{ runner.os == 'Linux' }} @@ -126,7 +127,7 @@ jobs: COVERALLS_REPO_TOKEN: ${{ github.token }} COVERALLS_GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} - # Signals to Coveralls that all parallel coverage uploads from the sharded + # Signals to Coveralls that all parallel coverage uploads from the grouped # check-and-test jobs are complete. coverage-finalize: name: Finalize Coverage diff --git a/.github/workflows/compute-test-shard.mjs b/.github/workflows/compute-test-group.mjs similarity index 60% rename from .github/workflows/compute-test-shard.mjs rename to .github/workflows/compute-test-group.mjs index ebcf4b70..39ed4a41 100644 --- a/.github/workflows/compute-test-shard.mjs +++ b/.github/workflows/compute-test-group.mjs @@ -1,29 +1,29 @@ #! /usr/bin/env node // Prints the `lerna run` filter flags (`--scope ...`) for the packages -// that belong to a given test shard. Packages are distributed round-robin over -// the shards after sorting by name, so the assignment is stable across runs and -// every package with a `test-ci` script lands in exactly one shard. +// that belong to a given test group. Packages are distributed round-robin over +// the groups after sorting by name, so the assignment is stable across runs and +// every package with a `test-ci` script lands in exactly one group. // -// Usage: node compute-test-shard.mjs -// shardIndex is 1-based. +// Usage: node compute-test-group.mjs +// groupIndex is 1-based. import { readFileSync, readdirSync, existsSync } from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; -const shardIndex = Number.parseInt(process.argv[2], 10); -const totalShards = Number.parseInt(process.argv[3], 10); +const groupIndex = Number.parseInt(process.argv[2], 10); +const totalGroups = Number.parseInt(process.argv[3], 10); if ( - !Number.isInteger(shardIndex) || - !Number.isInteger(totalShards) || - shardIndex < 1 || - totalShards < 1 || - shardIndex > totalShards + !Number.isInteger(groupIndex) || + !Number.isInteger(totalGroups) || + groupIndex < 1 || + totalGroups < 1 || + groupIndex > totalGroups ) { throw new Error( - `Invalid shard arguments: shardIndex=${process.argv[2]} totalShards=${process.argv[3]} ` + - `(expected 1 <= shardIndex <= totalShards)`, + `Invalid group arguments: groupIndex=${process.argv[2]} totalGroups=${process.argv[3]} ` + + `(expected 1 <= groupIndex <= totalGroups)`, ); } @@ -53,14 +53,14 @@ const testablePackages = packageDirs .map((manifest) => manifest.name) .sort(); -const shardPackages = testablePackages.filter( - (_name, index) => index % totalShards === shardIndex - 1, +const groupPackages = testablePackages.filter( + (_name, index) => index % totalGroups === groupIndex - 1, ); -if (shardPackages.length === 0) { +if (groupPackages.length === 0) { // Emit a filter that matches nothing so `lerna run` becomes a no-op rather // than falling back to running every package. - process.stdout.write('--scope __no_packages_in_this_shard__'); + process.stdout.write('--scope __no_packages_in_this_group__'); } else { - process.stdout.write(shardPackages.map((name) => `--scope ${name}`).join(' ')); + process.stdout.write(groupPackages.map((name) => `--scope ${name}`).join(' ')); }