From 75c79f5eedeeae9420da31488141a11fa4203a61 Mon Sep 17 00:00:00 2001 From: NotNite Date: Sat, 12 Apr 2025 19:18:29 -0400 Subject: [PATCH 1/6] Initial rewrite work --- .dockerignore | 15 +- .gitignore | 10 +- Dockerfile | 28 +++- builder/index.mjs | 47 ------ package.json | 36 ++++- pnpm-lock.yaml | 164 +++++++++++++++++++- runner/index.mjs | 188 ----------------------- src/group.ts | 75 ++++++++++ src/index.ts | 11 ++ src/run.ts | 349 +++++++++++++++++++++++++++++++++++++++++++ src/util/env.ts | 18 +++ src/util/exec.ts | 20 +++ src/util/fs.ts | 36 +++++ src/util/git.ts | 63 ++++++++ src/util/manifest.ts | 51 +++++++ tsconfig.json | 24 +++ 16 files changed, 885 insertions(+), 250 deletions(-) delete mode 100644 builder/index.mjs delete mode 100644 runner/index.mjs create mode 100644 src/group.ts create mode 100644 src/index.ts create mode 100644 src/run.ts create mode 100644 src/util/env.ts create mode 100644 src/util/exec.ts create mode 100644 src/util/fs.ts create mode 100644 src/util/git.ts create mode 100644 src/util/manifest.ts create mode 100644 tsconfig.json diff --git a/.dockerignore b/.dockerignore index 05d367c..e8c42ed 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,11 @@ -.git -.gitignore -/node_modules -/runner +**/.dockerignore +**/.git +**/.gitignore +**/.vscode +**/.idea +**/Dockerfile* +**/node_modules +**/dist +**/local +LICENSE +README.md diff --git a/.gitignore b/.gitignore index f4f5291..99bc627 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ -/node_modules -local/ +node_modules/ +dist/ +/local +.DS_Store + +# IDEs +.vscode/ +.idea/ diff --git a/Dockerfile b/Dockerfile index b7195de..4cd7964 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,28 @@ -# TODO: pin this better -FROM node:22 AS base + +ARG NODE_VERSION=22 +ARG PNPM_VERSION=10 + +FROM node:${NODE_VERSION}-alpine AS base ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" -RUN npm install -g pnpm@10 -FROM base AS builder +RUN apk add --no-cache git git-lfs docker +RUN git lfs install +RUN --mount=type=cache,target=/root/.npm npm install -g pnpm@${PNPM_VERSION} + COPY . /app WORKDIR /app -CMD [ "node", "builder/index.mjs" ] + +FROM base AS prod-deps +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile + +FROM base AS build +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile +RUN pnpm run build + +FROM base AS final +COPY --from=prod-deps /app/node_modules /app/node_modules +COPY --from=build /app/dist /app/dist + +WORKDIR /app +CMD ["node", "dist"] diff --git a/builder/index.mjs b/builder/index.mjs deleted file mode 100644 index b3ebcd2..0000000 --- a/builder/index.mjs +++ /dev/null @@ -1,47 +0,0 @@ -import fs from "fs"; -import path from "path"; -import { spawn } from "child_process"; - -async function exec(cmd, args, opts) { - return await new Promise((resolve, reject) => { - const proc = spawn(cmd, args, { - ...opts, - stdio: "inherit" - }); - proc.on("error", reject); - proc.on("close", (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(`Process exited with code ${code}`)); - } - }); - }); -} - -const workPath = "/work"; -if (!fs.existsSync(workPath)) fs.mkdirSync(workPath); - -const gitPath = path.join(workPath, "git"); -const artifactPath = path.join(workPath, "artifact"); - -const manifestPath = path.join(workPath, "manifest.json"); -const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); - -// Apply some common defaults -manifest.scripts ??= ["build", "repo"]; -manifest.artifact ??= `repo/${process.env.EXT_ID}.asar`; - -// TODO: reuse clone for multiple extensions being updated at once -await exec("git", ["clone", manifest.repository, gitPath]); -await exec("git", ["checkout", manifest.commit], { cwd: gitPath }); - -// TODO: support for other package managers -await exec("pnpm", ["install", "--recursive"], { cwd: gitPath }); -for (const script of manifest.scripts) { - await exec("pnpm", ["run", script], { cwd: gitPath }); -} - -const artifactFile = path.join(gitPath, manifest.artifact); -const artifactOutFile = path.join(artifactPath, process.env.EXT_ID + ".asar"); -fs.copyFileSync(artifactFile, artifactOutFile); diff --git a/package.json b/package.json index d99d769..9303eb9 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,39 @@ { "name": "@moonlight-mod/extensions-runner", - "version": "1.0.0", "private": true, + "packageManager": "pnpm@10.7.1", + "engineStrict": true, + "type": "module", + "engines": { + "node": ">=22", + "pnpm": ">=10", + "npm": "pnpm", + "yarn": "pnpm" + }, "scripts": { - "runner": "node runner/index.mjs", - "builder": "node builder/index.mjs" + "build": "tsc --build", + "clean": "pnpm build --clean" + }, + "files": [ + "dist" + ], + "main": "./dist/index.js", + "homepage": "https://moonlight-mod.github.io/", + "repository": { + "type": "git", + "url": "git+https://github.com/moonlight-mod/extensions-runner.git" + }, + "bugs": { + "url": "https://github.com/moonlight-mod/extensions-runner/issues" + }, + "license": "MIT", + "dependencies": { + "@electron/asar": "^3.4.1", + "zod": "^3.24.2" + }, + "devDependencies": { + "@types/node": "^22.14.1", + "prettier": "^3.5.3", + "typescript": "^5.8.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9b60ae1..ce3c759 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,4 +6,166 @@ settings: importers: - .: {} + .: + dependencies: + '@electron/asar': + specifier: ^3.4.1 + version: 3.4.1 + zod: + specifier: ^3.24.2 + version: 3.24.2 + devDependencies: + '@types/node': + specifier: ^22.14.1 + version: 22.14.1 + prettier: + specifier: ^3.5.3 + version: 3.5.3 + typescript: + specifier: ^5.8.3 + version: 5.8.3 + + packages/extensions-builder: + dependencies: + '@moonlight-mod/extensions-shared': + specifier: workspace:* + version: link:../extensions-shared + + packages/extensions-runner: + dependencies: + '@moonlight-mod/extensions-shared': + specifier: workspace:* + version: link:../extensions-shared + + packages/extensions-shared: + dependencies: + zod: + specifier: catalog:prod + version: 3.24.2 + +packages: + + '@electron/asar@3.4.1': + resolution: {integrity: sha512-i4/rNPRS84t0vSRa2HorerGRXWyF4vThfHesw0dmcWHp+cspK743UanA0suA5Q5y8kzY2y6YKrvbIUn69BCAiA==} + engines: {node: '>=10.12.0'} + hasBin: true + + '@types/node@22.14.1': + resolution: {integrity: sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==} + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + + brace-expansion@1.1.11: + resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} + + commander@5.1.0: + resolution: {integrity: sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==} + engines: {node: '>= 6'} + + concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + + fs.realpath@1.0.0: + resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + + glob@7.2.3: + resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + deprecated: Glob versions prior to v9 are no longer supported + + inflight@1.0.6: + resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + minimatch@3.1.2: + resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + path-is-absolute@1.0.1: + resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} + engines: {node: '>=0.10.0'} + + prettier@3.5.3: + resolution: {integrity: sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==} + engines: {node: '>=14'} + hasBin: true + + typescript@5.8.3: + resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + zod@3.24.2: + resolution: {integrity: sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==} + +snapshots: + + '@electron/asar@3.4.1': + dependencies: + commander: 5.1.0 + glob: 7.2.3 + minimatch: 3.1.2 + + '@types/node@22.14.1': + dependencies: + undici-types: 6.21.0 + + balanced-match@1.0.2: {} + + brace-expansion@1.1.11: + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + + commander@5.1.0: {} + + concat-map@0.0.1: {} + + fs.realpath@1.0.0: {} + + glob@7.2.3: + dependencies: + fs.realpath: 1.0.0 + inflight: 1.0.6 + inherits: 2.0.4 + minimatch: 3.1.2 + once: 1.4.0 + path-is-absolute: 1.0.1 + + inflight@1.0.6: + dependencies: + once: 1.4.0 + wrappy: 1.0.2 + + inherits@2.0.4: {} + + minimatch@3.1.2: + dependencies: + brace-expansion: 1.1.11 + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + path-is-absolute@1.0.1: {} + + prettier@3.5.3: {} + + typescript@5.8.3: {} + + undici-types@6.21.0: {} + + wrappy@1.0.2: {} + + zod@3.24.2: {} diff --git a/runner/index.mjs b/runner/index.mjs deleted file mode 100644 index e0a654c..0000000 --- a/runner/index.mjs +++ /dev/null @@ -1,188 +0,0 @@ -import fs from "fs"; -import path from "path"; -import { spawn } from "child_process"; - -function checkEnv(name) { - if (process.env[name] == null) { - console.error(`Missing environment variable ${name}`); - process.exit(1); - } - - return process.env[name]; -} - -async function exec(cmd, args, opts) { - return await new Promise((resolve, reject) => { - const proc = spawn(cmd, args, { - ...opts, - stdio: "inherit" - }); - proc.on("error", reject); - proc.on("close", (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(`Process exited with code ${code}`)); - } - }); - }); -} - -const manifestsPath = checkEnv("EXT_MANIFESTS_PATH"); -const distPath = checkEnv("EXT_DIST_PATH"); -const workPath = checkEnv("EXT_WORK_PATH"); - -if (fs.existsSync(workPath)) fs.rmSync(workPath, { recursive: true }); -fs.mkdirSync(workPath); - -const changedPath = path.join(workPath, "changed"); -if (fs.existsSync(changedPath)) fs.rmSync(changedPath, { recursive: true }); -fs.mkdirSync(changedPath); - -const manifests = {}; -for (const file of fs.readdirSync(manifestsPath)) { - if (!file.endsWith(".json")) continue; - const name = file.replace(/\.json$/, ""); - const manifest = JSON.parse(fs.readFileSync(path.join(manifestsPath, file), "utf8")); - manifests[name] = manifest; -} - -const stateStrFile = path.join(workPath, "state.md"); -const stateFile = path.join(distPath, "state.json"); -const state = fs.existsSync(stateFile) ? JSON.parse(fs.readFileSync(stateFile, "utf8")) : {}; - -const changed = {}; -const deleted = []; - -for (const name in manifests) { - const manifest = manifests[name]; - const old = state[name]; - - // TODO: this is probably not that good of a check - if (!manifest.repository.startsWith("https:")) { - throw new Error("Only HTTPS Git urls are supported for repositories"); - } - - if (old == null || old.commit !== manifest.commit) { - changed[name] = manifest; - } -} - -for (const name in state) { - if (manifests[name] == null) { - deleted.push(name); - } -} - -console.log("Changed:", changed); -console.log("Deleted:", deleted); - -if (Object.keys(changed).length === 0 && deleted.length === 0) { - console.log("No changes"); - process.exit(0); -} - -async function run(ext, manifest) { - const extWorkPath = path.join(workPath, ext); - const extArtifactPath = path.join(extWorkPath, "artifact"); - fs.mkdirSync(extArtifactPath, { recursive: true }); - - const extManifestPath = path.join(extWorkPath, "manifest.json"); - fs.writeFileSync(extManifestPath, JSON.stringify(manifest)); - - // TODO: run this without network access? - await exec("docker", [ - "run", - "--rm", - "-v", - `${extArtifactPath}:/work/artifact`, - "-v", - `${extManifestPath}:/work/manifest.json`, - "-e", - `EXT_ID=${ext}`, - "moonlight-mod/extensions-runner:latest" - ]); - - const artifactOutFile = path.join(extArtifactPath, ext + ".asar"); - if (!fs.existsSync(artifactOutFile)) { - console.error(`Artifact file ${artifactOutFile} not found`); - process.exit(1); - } - - fs.copyFileSync(artifactOutFile, path.join(distPath, "exts", ext + ".asar")); - fs.copyFileSync(artifactOutFile, path.join(changedPath, ext + ".asar")); - state[ext] = manifest; -} - -const stateBak = { ...state }; - -for (const name in changed) { - console.log(`Running ${name}`); - // TODO: post about failed builds here? - await run(name, changed[name]); -} - -for (const name of deleted) { - console.log(`Deleting ${name}`); - const asarPath = path.join(distPath, name + ".asar"); - if (fs.existsSync(asarPath)) fs.rmSync(asarPath); - delete state[name]; -} - -let stateStr = "# Extensions state\n\n"; - -for (const name in changed) { - const oldCommit = stateBak[name]?.commit ?? "none"; - const newCommit = changed[name].commit; - - let oldCommitStr = oldCommit; - let newCommitStr = newCommit; - let diffStr = null; - - // Flawed, but w/e - if (changed[name].repository.startsWith("https://github.com/")) { - const repoUrl = changed[name].repository.replace(".git", ""); - - if (oldCommit !== "none") { - const oldCommitUrl = `${repoUrl}/commit/${oldCommit}`; - oldCommitStr = `[${oldCommit}](${oldCommitUrl})`; - } - - const newCommitUrl = `${repoUrl}/commit/${newCommit}`; - newCommitStr = `[${newCommit}](${newCommitUrl})`; - - const diffUrl = - oldCommit === "none" ? `${repoUrl}/tree/${newCommit}` : `${repoUrl}/compare/${oldCommit}...${newCommit}`; - diffStr = `[Diff](${diffUrl})`; - } - - let msg = `## ${name}\n\n- Repository: <${changed[name].repository}>\n- Old commit: ${oldCommitStr}\n- New commit: ${newCommitStr}`; - if (diffStr != null) { - msg += `\n- ${diffStr}`; - } - - msg += "\n\n"; - console.log(msg); - - stateStr += msg; -} - -for (const name of deleted) { - const oldCommit = stateBak[name]?.commit ?? "none"; - let oldCommitStr = oldCommit; - - const repo = stateBak[name]?.repository ?? "unknown"; - - if (repo.startsWith("https://github.com/")) { - const repoUrl = repo.replace(".git", ""); - const oldCommitUrl = `${repoUrl}/commit/${oldCommit}`; - oldCommitStr = `[${oldCommit}](${oldCommitUrl})`; - } - - const msg = `## ${name}\n\n- Repository: <${repo}>\n- Old commit: ${oldCommitStr}\n- Deleted\n\n`; - console.log(msg); - stateStr += msg; -} - -fs.writeFileSync(stateFile, JSON.stringify(state, null, 2)); -fs.writeFileSync(stateStrFile, stateStr.trim() + "\n"); diff --git a/src/group.ts b/src/group.ts new file mode 100644 index 0000000..e8c2c15 --- /dev/null +++ b/src/group.ts @@ -0,0 +1,75 @@ +import asar from "@electron/asar"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { + currentApiLevel, + ExtensionManifestSchema, + type BuildGroupResult, + type BuildGroupState +} from "./util/manifest.js"; +import { exec } from "./util/exec.js"; +import { envVariables } from "./util/env.js"; +import { pathExists } from "./util/fs.js"; + +export default async function buildGroup() { + const groupPath = process.env[envVariables.groupPath] ?? "/moonlight/group"; + + const groupStatePath = path.join(groupPath, "state.json"); + const groupState: BuildGroupState = JSON.parse(await fs.readFile(groupStatePath, "utf8")); + + const sourceDir = path.join(groupPath, "source"); + const storeDir = path.join(groupPath, "store"); + const outputDir = path.join(groupPath, "output"); + + await exec( + "pnpm", + [ + "install", + "--frozen-lockfile", + "--offline", + // https://github.com/pnpm/pnpm/issues/6778 + "--config.confirmModulesPurge=false" + ], + { + cwd: sourceDir, + env: { + [envVariables.npmStoreDir]: storeDir + } + } + ); + + for (const script of groupState.scripts) { + await exec("pnpm", ["run", script], { + cwd: sourceDir + }); + } + + const result: BuildGroupResult = { + versions: {} + }; + + for (const [id, output] of Object.entries(groupState.output)) { + const normalized = path.normalize(output!); + if (normalized.startsWith(".")) throw new Error(`Detected possible path traversal: ${normalized}`); + + const folder = path.join(sourceDir, output!); + if (!(await pathExists(folder))) throw new Error(`Missing output directory for ${id}: ${folder}`); + + const manifestPath = path.join(folder, "manifest.json"); + if (!(await pathExists(manifestPath))) throw new Error(`Missing manifest for ${id}: ${manifestPath}`); + const manifestStr = await fs.readFile(manifestPath, "utf8"); + + const manifest = ExtensionManifestSchema.parse(JSON.parse(manifestStr)); + if (manifest.version == null) throw new Error(`Missing version for ${id}`); + if (manifest.apiLevel !== currentApiLevel) { + throw new Error(`Mismatched API level (expected ${currentApiLevel}, got ${manifest.apiLevel ?? "none"})`); + } + result.versions[id] = manifest.version; + + const file = path.join(outputDir, `${id}.asar`); + await asar.createPackage(folder, file); + } + + const groupResultPath = path.join(groupPath, "result.json"); + await fs.writeFile(groupResultPath, JSON.stringify(result)); +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..a92c7a7 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,11 @@ +import { mode } from "./util/env.js"; +import buildGroup from "./group.js"; +import run from "./run.js"; + +console.log("Current mode:", mode ?? "none"); + +if (mode === "group") { + await buildGroup(); +} else { + await run(); +} diff --git a/src/run.ts b/src/run.ts new file mode 100644 index 0000000..59e5791 --- /dev/null +++ b/src/run.ts @@ -0,0 +1,349 @@ +import { + BuildGroupResultSchema, + BuildManifestSchema, + compare, + defaultScripts, + type BuildGroupState, + type BuildManifest, + type BuildStates +} from "./util/manifest.js"; +import { ensureEnv, envVariables, mode } from "./util/env.js"; +import { ensureDir, pathExists } from "./util/fs.js"; +import { getCommitLink, getCommitTree, getCommitDiff, maybeWrapLink } from "./util/git.js"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { exec } from "./util/exec.js"; + +type Manifests = Record; +type ExtensionChange = + | { + type: "add"; + newManifest: BuildManifest; + } + | { + type: "update"; + oldManifest: BuildManifest; + newManifest: BuildManifest; + } + | { + type: "remove"; + oldManifest: BuildManifest; + }; + +async function getManifests(dir: string) { + const manifests: Manifests = {}; + + for (const filename of await fs.readdir(dir)) { + if (!filename.endsWith(".json")) continue; + + const id = filename.replace(/\.json$/, ""); + const filePath = path.join(dir, filename); + + const manifestStr = await fs.readFile(filePath, "utf8"); + const manifest = BuildManifestSchema.parse(JSON.parse(manifestStr)); + manifests[id] = manifest; + } + + return manifests; +} + +async function diffManifests(manifests: Manifests, state: BuildStates) { + const changed: Record = {}; + const buildAll = mode === "all"; + + for (const [id, newManifest] of Object.entries(manifests)) { + const oldManifest = state[id]?.manifest; + + const url = new URL(newManifest.repository); + if (url.protocol !== "https:") throw new Error("Only HTTPS Git urls are supported"); + + if (oldManifest == null) { + changed[id] = { + type: "add", + newManifest + }; + } else { + const diff = + buildAll || + compare(oldManifest.repository, newManifest.repository) || + compare(oldManifest.commit, newManifest.commit) || + compare(oldManifest.scripts, newManifest.scripts) || + compare(oldManifest.output, newManifest.output); + + if (diff) { + changed[id] = { + type: "update", + oldManifest, + newManifest + }; + } + } + } + + for (const [id, extState] of Object.entries(state)) { + if (manifests[id] == null) { + changed[id] = { + type: "remove", + oldManifest: extState!.manifest + }; + } + } + + return changed; +} + +type BuildGroup = { + repository: string; + commit: string; + scripts: string[]; + output: Partial>; + + directory: string; + hostDirectory: string; +}; + +async function build(group: BuildGroup) { + // Write the instructions for the builder + const groupStatePath = path.join(group.directory, "state.json"); + const groupState: BuildGroupState = { + scripts: group.scripts, + output: group.output + }; + await fs.writeFile(groupStatePath, JSON.stringify(groupState)); + + // Run the builder (without network access!) + await exec("docker", [ + "run", + "--rm", + + "--net", + "none", + + // Remember that we're using the socket from the host! + "-v", + `${group.hostDirectory}:/moonlight/group`, + + // Tell the container to build this group + "-e", + "MOONLIGHT_BUILD_MODE=group", + + "moonlight-mod/extensions-runner:latest" + ]); + + // Pass through a schema in case it gets tampered with + const groupResultPath = path.join(group.directory, "result.json"); + const groupResultStr = await fs.readFile(groupResultPath, "utf8"); + const groupResult = BuildGroupResultSchema.parse(JSON.parse(groupResultStr)); + + return groupResult; +} + +async function processGroups(groupDir: string, groupHostDir: string, changes: Record) { + const groups: Partial> = {}; + const mapping: Partial> = {}; + let currentIdx = 0; + + for (const [id, change] of Object.entries(changes)) { + if (change.type === "remove") continue; + + const manifest = change.newManifest; + const scripts = manifest.scripts ?? [...defaultScripts]; + const key = `${manifest.repository}-${manifest.commit}-${JSON.stringify(scripts)}`; + + if (groups[key] == null) { + console.log("Creating group for", id, manifest); + + // Unique ID for this prefetch directory + const thisIdx = currentIdx++; + const thisDir = path.join(groupDir, thisIdx.toString()); + const thisDirHost = path.join(groupHostDir, thisIdx.toString()); + await ensureDir(thisDir); + + const sourceDir = path.join(thisDir, "source"); + const storeDir = path.join(thisDir, "store"); + const outputDir = path.join(thisDir, "output"); + await ensureDir(sourceDir); + await ensureDir(storeDir); + await ensureDir(outputDir); + + // https://stackoverflow.com/a/3489576 + await exec("git", ["init"], { + cwd: sourceDir + }); + await exec("git", ["remote", "add", "origin", manifest.repository], { + cwd: sourceDir + }); + await exec("git", ["fetch", "origin", manifest.commit], { + cwd: sourceDir + }); + await exec("git", ["reset", "--hard", "FETCH_HEAD"], { + cwd: sourceDir + }); + + // Prefetch packages into the store so we can build offline + await exec("pnpm", ["fetch"], { + cwd: sourceDir, + env: { + [envVariables.npmStoreDir]: storeDir + } + }); + + groups[key] = { + repository: manifest.repository, + commit: manifest.commit, + scripts, + output: {}, + + hostDirectory: thisDirHost, + directory: thisDir + }; + } + + // Add our output dir to the group config so we pack it into an .asar + const output = manifest.output ?? `dist/${id}`; + groups[key].output[id] = output; + + // Add our extension to the mapping so we know what group we're in + mapping[id] = key; + } + + return { groups, mapping }; +} + +export default async function run() { + const manifestsEnv = process.env[envVariables.manifestsPath] ?? "/moonlight/manifests"; + const manifestsPath = path.join(manifestsEnv, "exts"); + + const distEnv = process.env[envVariables.distPath] ?? "/moonlight/dist"; + const distPath = path.join(distEnv, "exts"); + const statePath = path.join(distEnv, "state.json"); + + const workEnv = process.env[envVariables.workPath] ?? "/moonlight/work"; + const summaryPath = path.join(workEnv, "summary.md"); + await ensureDir(workEnv); + + const outputPath = path.join(workEnv, "output"); + await ensureDir(outputPath); + + const groupPath = path.join(workEnv, "group"); + await ensureDir(groupPath); + + const workHostEnv = ensureEnv(envVariables.workHostPath); + const groupHostPath = path.join(workHostEnv, "group"); + + const state: BuildStates = (await pathExists(statePath)) ? JSON.parse(await fs.readFile(statePath, "utf8")) : {}; + + const manifests = await getManifests(manifestsPath); + console.log(`Loaded ${Object.keys(manifests).length} manifests`); + + const diff = await diffManifests(manifests, state); + console.log("Diff results:", diff); + + let summary = "# Extensions state\n\n"; + + if (Object.keys(diff).length === 0) { + console.log("No changes"); + summary += "No changes."; + } + + // Build all extensions and get their new versions + const { groups, mapping } = await processGroups(groupPath, groupHostPath, diff); + console.log("Processed groups", mapping); + + const versions: Partial> = {}; + for (const [key, group] of Object.entries(groups)) { + console.log("Building group", key); + + const result = await build(group!); + for (const [ext, version] of Object.entries(result.versions)) { + versions[ext] = version; + } + + // Copy our .asars into the output directories + const groupOutputPath = path.join(group!.directory, "output"); + for (const filename of await fs.readdir(groupOutputPath)) { + const filePath = path.join(groupOutputPath, filename); + + const outputDestPath = path.join(outputPath, filename); + await fs.copyFile(filePath, outputDestPath); + + const distDestPath = path.join(distPath, filename); + await fs.copyFile(filePath, distDestPath); + } + } + + for (const [id, change] of Object.entries(diff)) { + let summaryMsg = `## ${id}\n\n`; + const oldState = state[id]; + + if (change.type === "remove") { + console.log("Removing", id); + + const asarPath = path.join(distPath, `${id}.asar`); + await fs.rm(asarPath, { force: true }); + + delete state[id]; + } else { + const version = versions[id]; + if (version == null) throw new Error(`Couldn't get version for ${id}`); + state[id] = { + version, + manifest: change.newManifest + }; + } + + const newState = state[id]; + summaryMsg += `- Type: ${change.type}\n`; + + const repository = change.type === "remove" ? change.oldManifest.repository : change.newManifest.repository; + summaryMsg += `- Repository: <${repository}>`; + if (change.type === "update" && change.newManifest.repository !== change.oldManifest.repository) { + summaryMsg += ` **(changed from <${change.oldManifest.repository}>)**`; + } + summaryMsg += "\n"; + + if (change.type === "update") { + const { repository, commit } = change.oldManifest; + const link = getCommitLink(repository, commit); + summaryMsg += `- Old commit: ${maybeWrapLink(commit, link)}`; + + const tree = getCommitTree(repository, commit); + if (tree != null) summaryMsg += ` ([Tree](${tree}))`; + + summaryMsg += "\n"; + } + + if (change.type !== "remove") { + const { repository, commit } = change.newManifest; + const link = getCommitLink(repository, commit); + summaryMsg += `- New commit: ${maybeWrapLink(commit, link)}`; + + const tree = getCommitTree(repository, commit); + if (tree != null) summaryMsg += ` ([Tree](${tree}))`; + + if (change.type === "update" && change.oldManifest.repository === repository) { + const diff = getCommitDiff(repository, change.oldManifest.commit, commit); + if (diff != null) { + summaryMsg += ` ([Diff](${diff}))`; + } + } + + summaryMsg += "\n"; + } + + const newVersion = newState?.version; + if (newVersion != null) { + summaryMsg += `- Version: ${newVersion}`; + const oldVersion = oldState?.version; + if (oldVersion === newVersion) summaryMsg += ` **(same version)**`; + summaryMsg += "\n"; + } + + summaryMsg += "\n\n"; + summary += summaryMsg; + console.log(summaryMsg.trim()); + } + + await fs.writeFile(statePath, JSON.stringify(state, null, 2)); + await fs.writeFile(summaryPath, summary.trim()); +} diff --git a/src/util/env.ts b/src/util/env.ts new file mode 100644 index 0000000..0d2f212 --- /dev/null +++ b/src/util/env.ts @@ -0,0 +1,18 @@ +export const envVariables = { + manifestsPath: "MOONLIGHT_MANIFESTS_PATH", + distPath: "MOONLIGHT_DIST_PATH", + workPath: "MOONLIGHT_WORK_PATH", + workHostPath: "MOONLIGHT_WORK_HOST_PATH", + groupPath: "MOONLIGHT_GROUP_PATH", + buildMode: "MOONLIGHT_BUILD_MODE", + + npmStoreDir: "NPM_CONFIG_STORE_DIR" +}; + +export type BuildMode = "all" | "group" | null; +export const mode = (process.env[envVariables.buildMode] ?? null) as BuildMode; + +export function ensureEnv(name: string) { + if (process.env[name] == null) throw new Error(`Missing environment variable: ${name}`); + return process.env[name]; +} diff --git a/src/util/exec.ts b/src/util/exec.ts new file mode 100644 index 0000000..33fc9a1 --- /dev/null +++ b/src/util/exec.ts @@ -0,0 +1,20 @@ +import { spawn, type SpawnOptions } from "node:child_process"; + +export function exec(cmd: string, args: string[], opts?: SpawnOptions): Promise { + return new Promise((resolve, reject) => { + const proc = spawn(cmd, args, { + ...opts, + stdio: "inherit" + }); + + proc.on("error", reject); + + proc.on("close", (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`Process exited with code ${code}`)); + } + }); + }); +} diff --git a/src/util/fs.ts b/src/util/fs.ts new file mode 100644 index 0000000..d15682f --- /dev/null +++ b/src/util/fs.ts @@ -0,0 +1,36 @@ +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +export const pathExists = (path: string) => + fs + .stat(path) + .then(() => true) + .catch(() => false); + +export async function ensureDir(path: string) { + const isDirectory = await fs + .stat(path) + .then((s) => s.isDirectory()) + .catch(() => null); + + // Exists, but is a file + if (isDirectory === false) throw new Error(`Tried to use file as directory: ${path}`); + + // Create if it doesn't exist + if (isDirectory === null) await fs.mkdir(path, { recursive: true }); +} + +export async function recursiveCopy(src: string, dst: string) { + for (const filename of await fs.readdir(src)) { + const srcPath = path.join(src, filename); + const dstPath = path.join(dst, filename); + + const isDirectory = (await fs.stat(srcPath)).isDirectory(); + if (isDirectory) { + await ensureDir(dstPath); + await recursiveCopy(srcPath, dstPath); + } else { + await fs.copyFile(srcPath, dstPath); + } + } +} diff --git a/src/util/git.ts b/src/util/git.ts new file mode 100644 index 0000000..ecfb170 --- /dev/null +++ b/src/util/git.ts @@ -0,0 +1,63 @@ +export type GitRepository = { + type: "github"; + owner: string; + repo: string; +}; + +export function parseRepository(repository: string): GitRepository | null { + const url = new URL(repository); + + if (url.hostname === "github.com") { + const path = url.pathname.replace(/\.git$/, "").replace(/^\//, ""); + const [owner, repo] = path.split("/"); + return { type: "github", owner, repo }; + } + + // TODO: other git forges? + return null; +} + +export function getCommitLink(repository: string, commit: string) { + const parsed = parseRepository(repository); + + switch (parsed?.type) { + case "github": { + return `https://github.com/${parsed.owner}/${parsed.repo}/commit/${commit}`; + } + } + + return undefined; +} + +export function getCommitTree(repository: string, commit: string) { + const parsed = parseRepository(repository); + + switch (parsed?.type) { + case "github": { + return `https://github.com/${parsed.owner}/${parsed.repo}/tree/${commit}`; + } + } + + return undefined; +} + +export function getCommitDiff(repository: string, oldCommit: string, newCommit: string) { + const parsed = parseRepository(repository); + + switch (parsed?.type) { + case "github": { + return `https://github.com/${parsed.owner}/${parsed.repo}/compare/${oldCommit}...${newCommit}`; + } + } + + return undefined; +} + +// kinda meh about this being in this file but w/e +export function maybeWrapLink(text: string, link?: string) { + if (link != null) { + return `[${text}](${link})`; + } else { + return text; + } +} diff --git a/src/util/manifest.ts b/src/util/manifest.ts new file mode 100644 index 0000000..17f68b2 --- /dev/null +++ b/src/util/manifest.ts @@ -0,0 +1,51 @@ +import { z } from "zod"; + +// https://stackoverflow.com/a/78709590 +const GitHashSchema = z.custom((val) => { + return typeof val === "string" ? /^[a-f0-9]+$/i.test(val) : false; +}); + +export type BuildManifest = z.infer; +export const BuildManifestSchema = z.object({ + repository: z.string().url(), + commit: GitHashSchema, + scripts: z.string().array().optional(), + output: z.string().optional() +}); + +export const ExtensionManifestSchema = z.object({ + id: z.string(), + apiLevel: z.number().optional(), + version: z.string().optional(), + meta: z.object({ + name: z.string(), + source: z.string() + }) +}); + +export type BuildState = { + version: string; + manifest: BuildManifest; +}; +export type BuildStates = Partial>; + +export type BuildGroupState = { + scripts: string[]; + output: Partial>; +}; + +export type BuildGroupResult = z.infer; +export const BuildGroupResultSchema = z.object({ + versions: z.record(z.string(), z.string()) +}); + +export const defaultScripts = ["build"]; +export const currentApiLevel = 2; + +// Simple compare function for the manifest diffing +export function compare(old?: T, current?: T) { + if (old == null || current == null) return old != current; + if (typeof old === "string" || typeof current === "string") return old !== current; + // This would be used on arrays, so positioning is fine + return JSON.stringify(old) !== JSON.stringify(current); +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..29537e1 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig.json", + "compilerOptions": { + "target": "ESNext", + "lib": ["ESNext", "DOM"], + "module": "NodeNext", + "moduleResolution": "nodenext", + "outDir": "dist", + "resolveJsonModule": true, + "allowArbitraryExtensions": false, + "allowJs": true, + "strict": true, + "strictNullChecks": true, + "noFallthroughCasesInSwitch": true, + "noImplicitReturns": true, + "noErrorTruncation": true, + "declaration": true, + "stripInternal": true, + "esModuleInterop": true, + "verbatimModuleSyntax": true + }, + "include": ["./src"], + "exclude": ["**/node_modules/**", "**/dist/**"] +} From 3e4d7bc635a987f2624c993d100da47107e12ecf Mon Sep 17 00:00:00 2001 From: NotNite Date: Sat, 12 Apr 2025 19:35:55 -0400 Subject: [PATCH 2/6] Log previous version --- src/run.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/run.ts b/src/run.ts index 59e5791..ecfc07e 100644 --- a/src/run.ts +++ b/src/run.ts @@ -334,8 +334,16 @@ export default async function run() { const newVersion = newState?.version; if (newVersion != null) { summaryMsg += `- Version: ${newVersion}`; + const oldVersion = oldState?.version; - if (oldVersion === newVersion) summaryMsg += ` **(same version)**`; + if (oldVersion != null) { + if (oldVersion === newVersion) { + summaryMsg += ` **(same version)**`; + } else { + summaryMsg += ` (prev ${oldVersion})`; + } + } + summaryMsg += "\n"; } From 2e672fb833e8a38d87bb0640e9ee1451a2ef9b4c Mon Sep 17 00:00:00 2001 From: NotNite Date: Sat, 12 Apr 2025 19:39:05 -0400 Subject: [PATCH 3/6] Set initial branch when initializing repo --- src/run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/run.ts b/src/run.ts index ecfc07e..03731c6 100644 --- a/src/run.ts +++ b/src/run.ts @@ -167,7 +167,7 @@ async function processGroups(groupDir: string, groupHostDir: string, changes: Re await ensureDir(outputDir); // https://stackoverflow.com/a/3489576 - await exec("git", ["init"], { + await exec("git", ["init", "--initial-branch=main"], { cwd: sourceDir }); await exec("git", ["remote", "add", "origin", manifest.repository], { From e0ef81399426e8274b1106e6db66ec6a7bfcd9f1 Mon Sep 17 00:00:00 2001 From: NotNite Date: Sat, 12 Apr 2025 19:43:49 -0400 Subject: [PATCH 4/6] Show version on removals --- src/run.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/run.ts b/src/run.ts index 03731c6..937466d 100644 --- a/src/run.ts +++ b/src/run.ts @@ -332,7 +332,10 @@ export default async function run() { } const newVersion = newState?.version; - if (newVersion != null) { + const oldVersion = oldState?.version; + if (change.type === "remove") { + if (oldVersion != null) summaryMsg += `- Version: ${oldVersion}`; + } else if (newVersion != null) { summaryMsg += `- Version: ${newVersion}`; const oldVersion = oldState?.version; From 64f276257a47154bd4167b9c831755db8eb14f47 Mon Sep 17 00:00:00 2001 From: NotNite Date: Sun, 13 Apr 2025 11:44:44 -0400 Subject: [PATCH 5/6] Use Docker Engine API --- Dockerfile | 9 ++-- package.json | 1 + pnpm-lock.yaml | 9 ++++ src/group.ts | 11 +++-- src/run.ts | 24 +++------- src/util/docker.ts | 106 +++++++++++++++++++++++++++++++++++++++++++++ src/util/exec.ts | 1 - 7 files changed, 135 insertions(+), 26 deletions(-) create mode 100644 src/util/docker.ts diff --git a/Dockerfile b/Dockerfile index 4cd7964..6843780 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,21 +6,22 @@ FROM node:${NODE_VERSION}-alpine AS base ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" -RUN apk add --no-cache git git-lfs docker +RUN apk add --no-cache git git-lfs RUN git lfs install RUN --mount=type=cache,target=/root/.npm npm install -g pnpm@${PNPM_VERSION} +FROM base AS app COPY . /app WORKDIR /app -FROM base AS prod-deps +FROM app AS prod-deps RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile -FROM base AS build +FROM app AS build RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile RUN pnpm run build -FROM base AS final +FROM app AS final COPY --from=prod-deps /app/node_modules /app/node_modules COPY --from=build /app/dist /app/dist diff --git a/package.json b/package.json index 9303eb9..79b2cfb 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "license": "MIT", "dependencies": { "@electron/asar": "^3.4.1", + "undici": "^7.8.0", "zod": "^3.24.2" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce3c759..8ed8c77 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@electron/asar': specifier: ^3.4.1 version: 3.4.1 + undici: + specifier: ^7.8.0 + version: 7.8.0 zod: specifier: ^3.24.2 version: 3.24.2 @@ -103,6 +106,10 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici@7.8.0: + resolution: {integrity: sha512-vFv1GA99b7eKO1HG/4RPu2Is3FBTWBrmzqzO0mz+rLxN3yXkE4mqRcb8g8fHxzX4blEysrNZLqg5RbJLqX5buA==} + engines: {node: '>=20.18.1'} + wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} @@ -166,6 +173,8 @@ snapshots: undici-types@6.21.0: {} + undici@7.8.0: {} + wrappy@1.0.2: {} zod@3.24.2: {} diff --git a/src/group.ts b/src/group.ts index e8c2c15..6b885fc 100644 --- a/src/group.ts +++ b/src/group.ts @@ -27,12 +27,13 @@ export default async function buildGroup() { "install", "--frozen-lockfile", "--offline", - // https://github.com/pnpm/pnpm/issues/6778 - "--config.confirmModulesPurge=false" + "--config.confirmModulesPurge=false", // auto-confirm yes to remaking node_modules + "--config.managePackageManagerVersions=false" // skip trying to pin pnpm without the network ], { cwd: sourceDir, env: { + PATH: process.env["PATH"], [envVariables.npmStoreDir]: storeDir } } @@ -40,7 +41,11 @@ export default async function buildGroup() { for (const script of groupState.scripts) { await exec("pnpm", ["run", script], { - cwd: sourceDir + cwd: sourceDir, + env: { + PATH: process.env["PATH"], + [envVariables.npmStoreDir]: storeDir + } }); } diff --git a/src/run.ts b/src/run.ts index 937466d..3f373f7 100644 --- a/src/run.ts +++ b/src/run.ts @@ -13,6 +13,7 @@ import { getCommitLink, getCommitTree, getCommitDiff, maybeWrapLink } from "./ut import * as fs from "node:fs/promises"; import * as path from "node:path"; import { exec } from "./util/exec.js"; +import runContainer from "./util/docker.js"; type Manifests = Record; type ExtensionChange = @@ -111,24 +112,7 @@ async function build(group: BuildGroup) { }; await fs.writeFile(groupStatePath, JSON.stringify(groupState)); - // Run the builder (without network access!) - await exec("docker", [ - "run", - "--rm", - - "--net", - "none", - - // Remember that we're using the socket from the host! - "-v", - `${group.hostDirectory}:/moonlight/group`, - - // Tell the container to build this group - "-e", - "MOONLIGHT_BUILD_MODE=group", - - "moonlight-mod/extensions-runner:latest" - ]); + await runContainer(group.hostDirectory); // Pass through a schema in case it gets tampered with const groupResultPath = path.join(group.directory, "result.json"); @@ -179,11 +163,15 @@ async function processGroups(groupDir: string, groupHostDir: string, changes: Re await exec("git", ["reset", "--hard", "FETCH_HEAD"], { cwd: sourceDir }); + await exec("git", ["submodule", "update", "--init", "--recursive"], { + cwd: sourceDir + }); // Prefetch packages into the store so we can build offline await exec("pnpm", ["fetch"], { cwd: sourceDir, env: { + PATH: process.env["PATH"], [envVariables.npmStoreDir]: storeDir } }); diff --git a/src/util/docker.ts b/src/util/docker.ts new file mode 100644 index 0000000..abb4f27 --- /dev/null +++ b/src/util/docker.ts @@ -0,0 +1,106 @@ +import { Readable } from "node:stream"; +import { Agent, fetch } from "undici"; + +type CreateRequest = { + Image: string; + Env: string[]; + Tty: true; + NetworkDisabled: boolean; + HostConfig: { + Binds: string[]; + }; +}; + +const agent = new Agent({ + connect: { + socketPath: "/var/run/docker.sock" + }, + bodyTimeout: 0 +}); +const version = "v1.40"; // picked randomly since idk what's recent-ish + +async function create(hostDirectory: string) { + const resp = await fetch(`http://localhost/${version}/containers/create`, { + dispatcher: agent, + method: "POST", + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + Image: "moonlight-mod/extensions-runner:latest", + Env: ["MOONLIGHT_BUILD_MODE=group"], + Tty: true, + NetworkDisabled: true, + HostConfig: { + Binds: [`${hostDirectory}:/moonlight/group`] + } + } satisfies CreateRequest) + }); + + if (!resp.ok) throw new Error(resp.statusText); + + const data = (await resp.json()) as { Id: string }; + return data.Id; +} + +async function start(id: string) { + const resp = await fetch(`http://localhost/${version}/containers/${id}/start`, { + dispatcher: agent, + method: "POST" + }); + + if (!resp.ok) throw new Error(resp.statusText); +} + +async function attach(id: string) { + const url = new URL(`http://localhost/${version}/containers/${id}/attach`); + url.searchParams.append("stream", "true"); + url.searchParams.append("stdout", "true"); + url.searchParams.append("stderr", "true"); + url.searchParams.append("logs", "true"); + + const resp = await fetch(url, { + dispatcher: agent, + method: "POST" + }); + if (!resp.ok) throw new Error(resp.statusText); + + return Readable.fromWeb(resp.body!).pipe(process.stdout); +} + +async function wait(id: string) { + const resp = await fetch(`http://localhost/${version}/containers/${id}/wait`, { + dispatcher: agent, + method: "POST" + }); + + if (!resp.ok) throw new Error(resp.statusText); + + const data = (await resp.json()) as { StatusCode: number }; + if (data.StatusCode !== 0) throw new Error(`Container exited with code ${data.StatusCode}`); +} + +async function kill(id: string) { + const url = new URL(`http://localhost/${version}/containers/${id}`); + url.searchParams.append("force", "true"); + + const resp = await fetch(url, { + dispatcher: agent, + method: "DELETE" + }); + + if (!resp.ok) throw new Error(resp.statusText); +} + +export default async function runContainer(hostDirectory: string) { + const id = await create(hostDirectory); + await start(id); + const stream = await attach(id); + + try { + await wait(id); + } finally { + await kill(id); + stream.destroy(); + } +} diff --git a/src/util/exec.ts b/src/util/exec.ts index 33fc9a1..b8d6301 100644 --- a/src/util/exec.ts +++ b/src/util/exec.ts @@ -8,7 +8,6 @@ export function exec(cmd: string, args: string[], opts?: SpawnOptions): Promise< }); proc.on("error", reject); - proc.on("close", (code) => { if (code === 0) { resolve(); From 45ba3b57da24d80062604c8ecac9454f8d021e8a Mon Sep 17 00:00:00 2001 From: NotNite Date: Sun, 13 Apr 2025 23:20:47 -0400 Subject: [PATCH 6/6] Rewrite the rewrite --- Dockerfile | 3 +- package.json | 5 +- pnpm-lock.yaml | 18 +- src/group.ts | 80 -------- src/index.ts | 29 ++- src/modes/group/build.ts | 94 ++++++++++ src/modes/group/fetch.ts | 61 ++++++ src/modes/group/index.ts | 26 +++ src/modes/run/build.ts | 390 +++++++++++++++++++++++++++++++++++++++ src/modes/run/index.ts | 48 +++++ src/modes/run/state.ts | 331 +++++++++++++++++++++++++++++++++ src/modes/run/summary.ts | 287 ++++++++++++++++++++++++++++ src/run.ts | 348 ---------------------------------- src/util/docker.ts | 51 ++--- src/util/env.ts | 34 +++- src/util/exec.ts | 14 +- src/util/fs.ts | 41 ++-- src/util/git.ts | 9 - src/util/manifest.ts | 67 ++++--- src/util/version.ts | 23 +++ 20 files changed, 1421 insertions(+), 538 deletions(-) delete mode 100644 src/group.ts create mode 100644 src/modes/group/build.ts create mode 100644 src/modes/group/fetch.ts create mode 100644 src/modes/group/index.ts create mode 100644 src/modes/run/build.ts create mode 100644 src/modes/run/index.ts create mode 100644 src/modes/run/state.ts create mode 100644 src/modes/run/summary.ts delete mode 100644 src/run.ts create mode 100644 src/util/version.ts diff --git a/Dockerfile b/Dockerfile index 6843780..5b87d9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ +# Make sure these directly line up with the README ARG NODE_VERSION=22 -ARG PNPM_VERSION=10 +ARG PNPM_VERSION="10.7.1" FROM node:${NODE_VERSION}-alpine AS base ENV PNPM_HOME="/pnpm" diff --git a/package.json b/package.json index 79b2cfb..01bf435 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "yarn": "pnpm" }, "scripts": { + "start": "node dist", "build": "tsc --build", "clean": "pnpm build --clean" }, @@ -29,8 +30,8 @@ "license": "MIT", "dependencies": { "@electron/asar": "^3.4.1", - "undici": "^7.8.0", - "zod": "^3.24.2" + "@zod/mini": "4.0.0-beta.20250412T085909", + "undici": "^7.8.0" }, "devDependencies": { "@types/node": "^22.14.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8ed8c77..130e671 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,12 +11,12 @@ importers: '@electron/asar': specifier: ^3.4.1 version: 3.4.1 + '@zod/mini': + specifier: 4.0.0-beta.20250412T085909 + version: 4.0.0-beta.20250412T085909 undici: specifier: ^7.8.0 version: 7.8.0 - zod: - specifier: ^3.24.2 - version: 3.24.2 devDependencies: '@types/node': specifier: ^22.14.1 @@ -56,6 +56,12 @@ packages: '@types/node@22.14.1': resolution: {integrity: sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==} + '@zod/core@0.4.6': + resolution: {integrity: sha512-0IeEldTobOdkoJPyINVQgOHlgCWbpozSsDjdcd2+VcxKvf8T+SYLDVk7NKt0XJx3C0ImNXAXh3V3yIw46NJyQA==} + + '@zod/mini@4.0.0-beta.20250412T085909': + resolution: {integrity: sha512-L4JpGahto3ZZot2QlCTlKnNwYjqqDUAd+cRiOocbDiBwosi+TyLlHFE63hIKS0FUDQPIqlEXMsL0VFThow7hkg==} + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -128,6 +134,12 @@ snapshots: dependencies: undici-types: 6.21.0 + '@zod/core@0.4.6': {} + + '@zod/mini@4.0.0-beta.20250412T085909': + dependencies: + '@zod/core': 0.4.6 + balanced-match@1.0.2: {} brace-expansion@1.1.11: diff --git a/src/group.ts b/src/group.ts deleted file mode 100644 index 6b885fc..0000000 --- a/src/group.ts +++ /dev/null @@ -1,80 +0,0 @@ -import asar from "@electron/asar"; -import * as fs from "node:fs/promises"; -import * as path from "node:path"; -import { - currentApiLevel, - ExtensionManifestSchema, - type BuildGroupResult, - type BuildGroupState -} from "./util/manifest.js"; -import { exec } from "./util/exec.js"; -import { envVariables } from "./util/env.js"; -import { pathExists } from "./util/fs.js"; - -export default async function buildGroup() { - const groupPath = process.env[envVariables.groupPath] ?? "/moonlight/group"; - - const groupStatePath = path.join(groupPath, "state.json"); - const groupState: BuildGroupState = JSON.parse(await fs.readFile(groupStatePath, "utf8")); - - const sourceDir = path.join(groupPath, "source"); - const storeDir = path.join(groupPath, "store"); - const outputDir = path.join(groupPath, "output"); - - await exec( - "pnpm", - [ - "install", - "--frozen-lockfile", - "--offline", - "--config.confirmModulesPurge=false", // auto-confirm yes to remaking node_modules - "--config.managePackageManagerVersions=false" // skip trying to pin pnpm without the network - ], - { - cwd: sourceDir, - env: { - PATH: process.env["PATH"], - [envVariables.npmStoreDir]: storeDir - } - } - ); - - for (const script of groupState.scripts) { - await exec("pnpm", ["run", script], { - cwd: sourceDir, - env: { - PATH: process.env["PATH"], - [envVariables.npmStoreDir]: storeDir - } - }); - } - - const result: BuildGroupResult = { - versions: {} - }; - - for (const [id, output] of Object.entries(groupState.output)) { - const normalized = path.normalize(output!); - if (normalized.startsWith(".")) throw new Error(`Detected possible path traversal: ${normalized}`); - - const folder = path.join(sourceDir, output!); - if (!(await pathExists(folder))) throw new Error(`Missing output directory for ${id}: ${folder}`); - - const manifestPath = path.join(folder, "manifest.json"); - if (!(await pathExists(manifestPath))) throw new Error(`Missing manifest for ${id}: ${manifestPath}`); - const manifestStr = await fs.readFile(manifestPath, "utf8"); - - const manifest = ExtensionManifestSchema.parse(JSON.parse(manifestStr)); - if (manifest.version == null) throw new Error(`Missing version for ${id}`); - if (manifest.apiLevel !== currentApiLevel) { - throw new Error(`Mismatched API level (expected ${currentApiLevel}, got ${manifest.apiLevel ?? "none"})`); - } - result.versions[id] = manifest.version; - - const file = path.join(outputDir, `${id}.asar`); - await asar.createPackage(folder, file); - } - - const groupResultPath = path.join(groupPath, "result.json"); - await fs.writeFile(groupResultPath, JSON.stringify(result)); -} diff --git a/src/index.ts b/src/index.ts index a92c7a7..9361f70 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,24 @@ -import { mode } from "./util/env.js"; -import buildGroup from "./group.js"; -import run from "./run.js"; +import { buildMode } from "./util/env.js"; -console.log("Current mode:", mode ?? "none"); +import run from "./modes/run/index.js"; +import runFetch from "./modes/group/fetch.js"; +import runBuild from "./modes/group/build.js"; -if (mode === "group") { - await buildGroup(); -} else { - await run(); +console.log("Current mode:", buildMode ?? "none"); + +switch (buildMode) { + case "fetch": { + await runFetch(); + break; + } + + case "build": { + await runBuild(); + break; + } + + default: { + await run(); + break; + } } diff --git a/src/modes/group/build.ts b/src/modes/group/build.ts new file mode 100644 index 0000000..978fb38 --- /dev/null +++ b/src/modes/group/build.ts @@ -0,0 +1,94 @@ +import { getGroupDir, readGroupResult, readGroupState, writeGroupResult } from "./index.js"; +import { envVariables, storeDir } from "../../util/env.js"; +import { exec } from "../../util/exec.js"; +import { pathExists } from "../../util/fs.js"; +import { ExtensionManifestSchema } from "../../util/manifest.js"; +import * as fs from "node:fs/promises"; +import path from "node:path"; +import asar from "@electron/asar"; + +export default async function runBuild() { + const groupDir = getGroupDir(); + const groupState = await readGroupState(); + const groupResult = await readGroupResult(); + + const sourceDir = path.join(groupDir, "source"); + const outputDir = path.join(groupDir, "output"); + + let failed = false; + + try { + await exec( + "pnpm", + [ + "install", + "--frozen-lockfile", + "--offline", + + // FIXME: https://github.com/orgs/pnpm/discussions/9418 + "--config.confirmModulesPurge=false", // auto-confirm yes to remaking node_modules + "--config.managePackageManagerVersions=false" // skip trying to pin pnpm without the network + ], + { + cwd: sourceDir, + env: { + // Not entirely sure why this is needed to forward the path + PATH: process.env["PATH"], + [envVariables.npmStoreDir]: storeDir + } + } + ); + } catch (e) { + console.error("Failed to install", e); + groupResult.errors.push({ type: "installFailed", err: `${e}` }); + failed = true; + } + + if (!failed) { + for (const script of groupState.scripts) { + try { + await exec("pnpm", ["run", script], { + cwd: sourceDir, + env: { + PATH: process.env["PATH"], + [envVariables.npmStoreDir]: storeDir + } + }); + } catch (e) { + console.error("Failed to run script", script, e); + groupResult.errors.push({ type: "scriptFailed", script, err: `${e}` }); + failed = true; + break; + } + } + } + + if (!failed) { + for (const [ext, outputPath] of Object.entries(groupState.outputs)) { + try { + const normalized = path.normalize(outputPath); + if (normalized.startsWith(".")) throw new Error(`Detected possible path traversal: ${normalized}`); + + const extOutputDir = path.join(sourceDir, normalized); + if (!(await pathExists(extOutputDir))) throw new Error(`Missing output directory: ${extOutputDir}`); + + const manifestPath = path.join(extOutputDir, "manifest.json"); + if (!(await pathExists(manifestPath))) throw new Error(`Missing manifest: ${manifestPath}`); + + const manifestStr = await fs.readFile(manifestPath, "utf8"); + const manifest = ExtensionManifestSchema.parse(JSON.parse(manifestStr)); + + const asarOutputPath = path.join(outputDir, `${ext}.asar`); + await asar.createPackage(extOutputDir, asarOutputPath); + + groupResult.manifests[ext] = manifest; + } catch (e) { + console.error("Failed to package", e); + groupResult.errors.push({ type: "packageFailed", ext, err: `${e}` }); + failed = true; + } + } + } + + await writeGroupResult(groupResult); +} diff --git a/src/modes/group/fetch.ts b/src/modes/group/fetch.ts new file mode 100644 index 0000000..1cc326c --- /dev/null +++ b/src/modes/group/fetch.ts @@ -0,0 +1,61 @@ +import { getGroupDir, readGroupState, writeGroupResult } from "./index.js"; +import type { MiniGroupResult } from "../../util/manifest.js"; +import { envVariables, storeDir } from "../../util/env.js"; +import { exec } from "../../util/exec.js"; +import { ensureDir } from "../../util/fs.js"; +import path from "node:path"; + +export default async function runFetch() { + const groupDir = getGroupDir(); + const groupState = await readGroupState(); + const groupResult: MiniGroupResult = { + errors: [], + manifests: {} + }; + + const sourceDir = path.join(groupDir, "source"); + await ensureDir(sourceDir, true); + + let failed = false; + + try { + await exec("git", ["init", "--initial-branch=main"], { + cwd: sourceDir + }); + await exec("git", ["remote", "add", "origin", groupState.repository], { + cwd: sourceDir + }); + await exec("git", ["fetch", "origin", groupState.commit], { + cwd: sourceDir + }); + await exec("git", ["reset", "--hard", "FETCH_HEAD"], { + cwd: sourceDir + }); + await exec("git", ["submodule", "update", "--init", "--recursive"], { + cwd: sourceDir + }); + } catch (e) { + console.error("Failed to clone", e); + groupResult.errors.push({ type: "cloneFailed", err: `${e}` }); + failed = true; + } + + if (!failed) { + try { + // FIXME: don't run scripts here + await exec("pnpm", ["fetch"], { + cwd: sourceDir, + env: { + PATH: process.env["PATH"], + [envVariables.npmStoreDir]: storeDir + } + }); + } catch (e) { + console.error("Failed to fetch", e); + groupResult.errors.push({ type: "fetchFailed", err: `${e}` }); + failed = true; + } + } + + await writeGroupResult(groupResult); +} diff --git a/src/modes/group/index.ts b/src/modes/group/index.ts new file mode 100644 index 0000000..77e185c --- /dev/null +++ b/src/modes/group/index.ts @@ -0,0 +1,26 @@ +import { groupDir } from "../../util/env.js"; +import { MiniGroupResultSchema, type MiniGroupResult, type MiniGroupState } from "../../util/manifest.js"; +import * as fs from "node:fs/promises"; +import path from "node:path"; + +export function getGroupDir() { + if (groupDir == null) throw new Error("Group directory not set"); + return groupDir; +} + +export async function readGroupState(): Promise { + const groupStatePath = path.join(getGroupDir(), "state.json"); + const groupStateStr = await fs.readFile(groupStatePath, "utf8"); + return JSON.parse(groupStateStr); +} + +export async function readGroupResult() { + const groupResultPath = path.join(getGroupDir(), "result.json"); + const groupResultStr = await fs.readFile(groupResultPath, "utf8"); + return MiniGroupResultSchema.parse(JSON.parse(groupResultStr)); +} + +export async function writeGroupResult(result: MiniGroupResult) { + const groupResultPath = path.join(getGroupDir(), "result.json"); + await fs.writeFile(groupResultPath, JSON.stringify(result)); +} diff --git a/src/modes/run/build.ts b/src/modes/run/build.ts new file mode 100644 index 0000000..deaca2b --- /dev/null +++ b/src/modes/run/build.ts @@ -0,0 +1,390 @@ +import type { RunnerState } from "./state.js"; +import { defaultGroup, defaultStore, distRepo, workDir, workHostDir } from "../../util/env.js"; +import { ensureDir, pathExists } from "../../util/fs.js"; +import { + currentApiLevel, + MiniGroupResultSchema, + type MiniGroupResult, + type MiniGroupState +} from "../../util/manifest.js"; +import runContainer from "../../util/docker.js"; +import { parseVersion, versionGreaterThan } from "../../util/version.js"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +type Group = { + key: string; + directory: string; + hostDirectory: string; // Since this is running in Docker, we need to mount the volume from the perspective of the host + + repository: string; + commit: string; + scripts: string[]; + + extensions: string[]; + outputs: Record; + + result?: MiniGroupResult; +}; + +type GroupState = { + groups: Group[]; +}; + +function getWorkHostDir() { + if (workHostDir == null) throw new Error("Work host directory not set"); + return workHostDir; +} + +async function createGroups(runnerState: RunnerState) { + const groupState: GroupState = { + groups: [] + }; + + const groupDir = path.join(workDir, "group"); + const groupHostDir = path.join(getWorkHostDir(), "group"); + await ensureDir(groupDir); + + for (const [ext, change] of Object.entries(runnerState.changes)) { + if (change.type === "remove" || change.type === "updateNoBuild") continue; + const manifest = change.newManifest; + + // These are the defaults for create-extension + const scripts = manifest.scripts ?? ["build"]; + const output = manifest.output ?? `dist/${ext}`; + + // Unique key to represent this combination of repository-commit-script + const key = `${manifest.repository}-${manifest.commit}-${JSON.stringify(scripts)}`; + + let groupIdx = groupState.groups.findIndex((group) => group.key === key); + if (groupIdx === -1) { + groupIdx = groupState.groups.length; + console.log("Creating group for key", key, groupIdx); + + const thisGroupDir = path.join(groupDir, groupIdx.toString()); + const thisGroupHostDir = path.join(groupHostDir, groupIdx.toString()); + await ensureDir(thisGroupDir, true); + + groupState.groups.push({ + key, + directory: thisGroupDir, + hostDirectory: thisGroupHostDir, + + repository: manifest.repository, + commit: manifest.commit, + scripts: scripts, + + extensions: [], + outputs: {} + }); + } + + console.log("Adding extension", ext, "to", groupIdx); + const group = groupState.groups[groupIdx]; + if (group == null) { + console.warn("Group is null"); + runnerState.warnings.push({ type: "unknown" }); + } + + group.extensions.push(ext); + group.outputs[ext] = output; + } + + return groupState; +} + +async function writeGroupState(group: Group, state: MiniGroupState) { + const groupStatePath = path.join(group.directory, "state.json"); + await fs.writeFile(groupStatePath, JSON.stringify(state)); +} + +async function readGroupResult(group: Group) { + const groupResultPath = path.join(group.directory, "result.json"); + const groupResultStr = await fs.readFile(groupResultPath, "utf8"); + return MiniGroupResultSchema.parse(JSON.parse(groupResultStr)); +} + +async function writeGroupResult(group: Group, result: MiniGroupResult) { + const groupResultPath = path.join(group.directory, "result.json"); + await fs.writeFile(groupResultPath, JSON.stringify(result)); +} + +async function propagateErrors(runnerState: RunnerState, group: Group, result: MiniGroupResult) { + const sharedExts = []; + for (const ext of group.extensions) { + const change = runnerState.changes[ext]; + if (change == null) { + console.warn("Null change when propagating errors", ext, result); + runnerState.warnings.push({ type: "unknown" }); + continue; + } + sharedExts.push(change); + } + + for (const error of result.errors) { + switch (error.type) { + case "cloneFailed": + case "fetchFailed": + case "installFailed": { + for (const change of sharedExts) { + change.errors.push({ type: error.type, err: error.err }); + } + break; + } + + case "scriptFailed": { + for (const change of sharedExts) { + change.errors.push({ type: "scriptFailed", script: error.script, err: error.err }); + } + break; + } + + case "packageFailed": { + const change = runnerState.changes[error.ext]; + if (change == null) { + console.warn("Null change when propagating package failed error", error); + runnerState.warnings.push({ type: "unknown" }); + continue; + } + change.errors.push({ type: "packageFailed", err: error.err }); + break; + } + } + } +} + +async function buildGroup(runnerState: RunnerState, group: Group) { + const storeDir = path.join(workDir, "store"); + const storeHostDir = path.join(getWorkHostDir(), "store"); + await ensureDir(storeDir); + + const groupOutputDir = path.join(group.directory, "output"); + const groupOutputHostDir = path.join(group.hostDirectory, "output"); + await ensureDir(groupOutputDir, true); + + const distOutputDir = path.join(distRepo, "exts"); + await ensureDir(distOutputDir); + + const artifactOutputDir = path.join(workDir, "output"); + await ensureDir(artifactOutputDir); + + // Only pass in what the build needs to know + await writeGroupState(group, { + repository: group.repository, + commit: group.commit, + scripts: group.scripts, + outputs: group.outputs + }); + + // Write this so we can mount it + await writeGroupResult(group, { + errors: [], + manifests: {} + }); + + // Fetch dependencies (with network access) + // FIXME: pnpm fetch is buggy for this use case right now, re-evaluate if we should use this in prod + await runContainer({ + Image: "moonlight-mod/extensions-runner:latest", + Env: ["MOONLIGHT_BUILD_MODE=fetch"], + Tty: true, + HostConfig: { + AutoRemove: true, + Mounts: [ + { + Target: path.join(defaultGroup, "state.json"), + Source: path.join(group.hostDirectory, "state.json"), + Type: "bind", + ReadOnly: true + }, + { + Target: path.join(defaultGroup, "result.json"), + Source: path.join(group.hostDirectory, "result.json"), + Type: "bind" + }, + { + Target: path.join(defaultGroup, "source"), + Source: path.join(group.hostDirectory, "source"), + Type: "bind" + // FIXME: make this readonly when pnpm stops creating `node_modules` after running fetch + }, + { + // SECURITY ASSUMPTION: poisoning the pnpm store is not possible + // if this is possible (either a pnpm bug or someone gets code exec) then whelp I fucked up + Target: defaultStore, + Source: storeHostDir, + Type: "bind" + } + ] + } + }); + + let result = await readGroupResult(group); + if (result.errors.length !== 0) { + console.error("Fetch failed", group, result); + await propagateErrors(runnerState, group, result); + return; + } + + // Build (without network access) + await runContainer({ + Image: "moonlight-mod/extensions-runner:latest", + Env: ["MOONLIGHT_BUILD_MODE=build"], + Tty: true, + NetworkDisabled: true, + HostConfig: { + AutoRemove: true, + Mounts: [ + { + Target: path.join(defaultGroup, "state.json"), + Source: path.join(group.hostDirectory, "state.json"), + Type: "bind", + ReadOnly: true + }, + { + Target: path.join(defaultGroup, "result.json"), + Source: path.join(group.hostDirectory, "result.json"), + Type: "bind" + }, + { + Target: path.join(defaultGroup, "source"), + Source: path.join(group.hostDirectory, "source"), + Type: "bind" + }, + { + Target: defaultStore, + Source: storeHostDir, + Type: "bind", + ReadOnly: true // store doesn't need to be writable anymore + }, + { + Target: path.join(defaultGroup, "output"), + Source: groupOutputHostDir, + Type: "bind" + } + ] + } + }); + + result = await readGroupResult(group); + if (result.errors.length !== 0) { + console.error("Build failed", group, result); + await propagateErrors(runnerState, group, result); + return; + } + + for (const [ext, manifest] of Object.entries(result.manifests)) { + const change = runnerState.changes[ext]; + if (change == null) { + console.warn("Null change when applying build results", ext, result); + runnerState.warnings.push({ type: "unknown" }); + continue; + } + + // This should never happen, but just in case + if (change.type !== "add" && change.type !== "update") { + console.warn("Mismatched change type when applying build result", ext, result, change); + runnerState.warnings.push({ type: "unknown" }); + continue; + } + + runnerState.buildState[ext] = { + version: manifest.version, + manifest: change.newManifest + }; + + if (manifest.apiLevel !== currentApiLevel) { + // Manifest does not specify API level or it is mismatched + change.warnings.push({ type: "invalidApiLevel", value: manifest.apiLevel }); + } + + if (manifest.id !== ext) { + // Extension ID is mismatched between CI and the manifest + change.warnings.push({ type: "invalidId", value: manifest.id }); + } + + const ver = manifest.version != null ? parseVersion(manifest.version) : null; + const oldState = runnerState.oldBuildState[ext]; + + if (manifest.version != null && oldState?.version != null && manifest.version === oldState.version) { + // Version string is the same + change.warnings.push({ type: "sameOrLowerVersion", oldVersion: oldState.version, newVersion: manifest.version }); + } + + // Version parsing isn't enforced by moonlight, but we'll check in case we want to in the future + if (ver == null) { + // Couldn't parse version + change.warnings.push({ type: "irregularVersion" }); + } else if (change.type === "update" && oldState?.version != null) { + const oldVer = parseVersion(oldState.version, true); + + if (oldVer != null && !versionGreaterThan(ver, oldVer)) { + // Version string was parsed and it's a downgrade + change.warnings.push({ + type: "sameOrLowerVersion", + oldVersion: oldState.version, + newVersion: manifest.version! + }); + } + } + + const asarFilename = `${ext}.asar`; + const asarOutputPath = path.join(groupOutputDir, asarFilename); + const asarDistPath = path.join(distOutputDir, asarFilename); + const asarArtifactPath = path.join(artifactOutputDir, asarFilename); + + if (!(await pathExists(asarOutputPath))) { + console.warn("Output .asar does not exist", ext, change, manifest); + change.errors.push({ type: "packageFailed", err: "Output .asar does not exist" }); + continue; + } + + // Copy into extensions-dist + await fs.copyFile(asarOutputPath, asarDistPath); + + // Copy into the build artifact folder (remember, this is the group-specific output folder so far) + await fs.copyFile(asarOutputPath, asarArtifactPath); + } +} + +export default async function build(runnerState: RunnerState) { + const groupState = await createGroups(runnerState); + + for (const group of groupState.groups) { + try { + await buildGroup(runnerState, group); + } catch (e) { + console.error("Failed to build group", e); + + for (const ext of group.extensions) { + const change = runnerState.changes[ext]; + if (change == null) { + console.warn("Null change when propagating uncaught error", ext); + runnerState.warnings.push({ type: "unknown" }); + continue; + } + + change.errors.push({ type: "unknown", err: `${e}` }); + } + } + } + + // Handle deletes separately to builds + try { + const distOutputDir = path.join(distRepo, "exts"); + await ensureDir(distOutputDir); + + for (const [ext, change] of Object.entries(runnerState.changes)) { + if (change.type !== "remove") continue; + + const asarFilename = `${ext}.asar`; + const asarDistPath = path.join(distOutputDir, asarFilename); + if (await pathExists(asarDistPath)) await fs.rm(asarDistPath, { force: true }); + + delete runnerState.buildState[ext]; + } + } catch (e) { + console.error(e); + runnerState.errors.push({ type: "deleteChangeFailed", err: `${e}` }); + } +} diff --git a/src/modes/run/index.ts b/src/modes/run/index.ts new file mode 100644 index 0000000..a0b42e4 --- /dev/null +++ b/src/modes/run/index.ts @@ -0,0 +1,48 @@ +import { distRepo, workDir } from "../../util/env.js"; +import { ensureDir } from "../../util/fs.js"; +import computeState from "./state.js"; +import build from "./build.js"; +import writeSummary from "./summary.js"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +export default async function run() { + await ensureDir(workDir, true); + + const outputDir = path.join(workDir, "output"); + await ensureDir(outputDir); + + const groupDir = path.join(workDir, "group"); + await ensureDir(groupDir); + + const runnerState = await computeState(); + await build(runnerState); + + // This is included for debugging locally, this isn't uploaded anywhere + const runnerStatePath = path.join(workDir, "runnerState.json"); + await fs.writeFile(runnerStatePath, JSON.stringify(runnerState)); + + // This is used by extensions-dist + const buildStatePath = path.join(distRepo, "state.json"); + await fs.writeFile(buildStatePath, JSON.stringify(runnerState.buildState, null, 2)); + + // This is included for CI previews + await writeSummary(runnerState); + + const shouldFail = + runnerState.errors.length !== 0 || Object.values(runnerState.changes).some((change) => change.errors.length !== 0); + if (shouldFail) { + console.log("Exiting with errors :("); + process.exit(1); + } else { + console.log("aight cya"); + + // Clean up after ourselves since we won't need to debug anything + // (leave output/summary/state for CI though) + await fs.rm(groupDir, { recursive: true, force: true }); + + // `storeDir` global is for the fetch/build modes + const storeDir = path.join(workDir, "store"); + await fs.rm(storeDir, { recursive: true, force: true }); + } +} diff --git a/src/modes/run/state.ts b/src/modes/run/state.ts new file mode 100644 index 0000000..3659047 --- /dev/null +++ b/src/modes/run/state.ts @@ -0,0 +1,331 @@ +import { authorId, authorPr, authorUsername, buildMode, distRepo, manifestsRepo } from "../../util/env.js"; +import { pathExists } from "../../util/fs.js"; +import { + BuildManifestSchema, + hasChanged, + moonlightReviewers, + type BuildManifest, + type BuildState +} from "../../util/manifest.js"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +// Based off of BuildGroupResult +export type ExtensionError = + // Unhandled exceptions + | { + type: "unknown"; + err: string; + } + // Logged "properly" by BuildGroupResult + | { + type: "cloneFailed"; + err: string; + } + | { + type: "fetchFailed"; + err: string; + } + | { + type: "installFailed"; + err: string; + } + | { + type: "scriptFailed"; + script: string; + err: string; + } + | { + type: "packageFailed"; + err: string; + }; + +export type ExtensionWarning = + // Extension manifest warnings + | { + type: "invalidApiLevel"; + value?: number; + } + | { + type: "invalidId"; + value: string; + } + | { + type: "irregularVersion"; + value?: string; + } + | { + type: "sameOrLowerVersion"; + oldVersion: string; + newVersion: string; + } + // Build manifest warnings + | { + type: "noOwnersSpecified"; + } + | { + type: "repositoryChanged"; + } + | { + type: "buildConfigChanged"; + } + | { + type: "authorNotInOwners"; + } + | { + type: "ownersChanged"; + } + | { + type: "authorAddedToOwners"; + }; + +export type ExtensionChange = { + errors: ExtensionError[]; + warnings: ExtensionWarning[]; +} & ( + | { + type: "add"; + newManifest: BuildManifest; + } + | { + type: "updateNoBuild"; + oldManifest: BuildManifest; + newManifest: BuildManifest; + } + | { + type: "update"; + oldManifest: BuildManifest; + newManifest: BuildManifest; + } + | { + type: "remove"; + oldManifest: BuildManifest; + } +); + +// FIXME: warning for out-of-sync or conflicting PR +export type RunnerWarning = + | { + type: "unknown"; + } + | { + type: "missingAuthor"; + }; + +export type RunnerError = + | { + type: "parseManifestFailed"; + ext: string; + err: string; + } + | { + type: "deleteChangeFailed"; + err: string; + }; + +export type RunnerAuthor = { + username: string; + id: string; + pr?: string; +}; + +export type RunnerState = { + author?: RunnerAuthor; + warnings: RunnerWarning[]; + errors: RunnerError[]; + oldBuildState: BuildState; + buildState: BuildState; + changes: Record; +}; + +function authorCanEdit(manifest: BuildManifest, author: RunnerAuthor) { + return ( + manifest.owners == null || + moonlightReviewers.includes(author.id) || + manifest.owners.some((owner) => owner === author.username || owner === `id:${author.id}`) + ); +} + +async function getManifests(runnerState: RunnerState, dir: string) { + const manifests: Record = {}; + + for (const filename of await fs.readdir(dir)) { + if (!filename.endsWith(".json")) continue; + + const ext = filename.replace(/\.json$/, ""); + const filePath = path.join(dir, filename); + + try { + const manifestStr = await fs.readFile(filePath, "utf8"); + const manifest = BuildManifestSchema.parse(JSON.parse(manifestStr)); + manifests[ext] = manifest; + } catch (e) { + runnerState.errors.push({ type: "parseManifestFailed", ext: ext, err: `${e}` }); + } + } + + return manifests; +} + +async function diffManifests(runnerState: RunnerState, manifests: Record) { + const changed: Record = {}; + + for (const [ext, newManifest] of Object.entries(manifests)) { + const oldManifest = runnerState.oldBuildState[ext]?.manifest; + + const url = new URL(newManifest.repository); + if (url.protocol !== "https:") throw new Error("Only HTTPS Git URLs are supported"); + if (url.username !== "" || url.password !== "") throw new Error("Cannot provide credentials to Git repository"); + + if (oldManifest == null) { + // Build a new extension + const change: ExtensionChange = { + warnings: [], + errors: [], + + type: "add", + newManifest + }; + + if (newManifest.owners != null) { + if (runnerState.author != null && !authorCanEdit(newManifest, runnerState.author)) { + // Author forgot to add themselves to the owner list + change.warnings.push({ type: "authorNotInOwners" }); + } + } else { + // Author forgot to add an owner list + change.warnings.push({ type: "noOwnersSpecified" }); + } + + runnerState.changes[ext] = change; + } else { + const repositoryChanged = hasChanged(oldManifest.repository, newManifest.repository); + // Count a repo change as a commit change too just to be safe + const commitChanged = repositoryChanged || hasChanged(oldManifest.commit, newManifest.commit); + + // Changes to the way the build system runs + const buildConfigChanged = + hasChanged(oldManifest.scripts, newManifest.scripts) || hasChanged(oldManifest.output, newManifest.output); + const ownersChanged = hasChanged(oldManifest.owners, newManifest.owners); + + const shouldBuild = buildMode === "all" || commitChanged || buildConfigChanged; + const shouldUpdate = shouldBuild || ownersChanged; + + let change: ExtensionChange; + if (shouldBuild) { + // Build the new version + change = { + warnings: [], + errors: [], + + type: "update", + oldManifest, + newManifest + }; + } else if (shouldUpdate) { + // Emit a change without a rebuild, just to stick a warning in + change = { + warnings: [], + errors: [], + + type: "updateNoBuild", + oldManifest, + newManifest + }; + } else { + // No change here, just move on + continue; + } + + if (newManifest.owners == null) { + // Extension does not have any owners yet + change.warnings.push({ type: "noOwnersSpecified" }); + } + + if (repositoryChanged) { + // Repository URL changed (*not* commit) + change.warnings.push({ type: "repositoryChanged" }); + } + + if (buildConfigChanged) { + // Build config changed (even if repo/commit is the same) + change.warnings.push({ type: "buildConfigChanged" }); + } + + if (ownersChanged) { + // Author added/removed owners + change.warnings.push({ type: "ownersChanged" }); + } + + if (runnerState.author != null) { + const ownerForOld = authorCanEdit(oldManifest, runnerState.author); + const ownerForNew = authorCanEdit(newManifest, runnerState.author); + + if (!ownerForNew) { + // Author isn't in the owners list (!!!) + change.warnings.push({ type: "authorNotInOwners" }); + } + + if (!ownerForOld && ownerForNew) { + // Author added themselves to the owners list (!!!!!) + change.warnings.push({ type: "authorAddedToOwners" }); + } + } + + runnerState.changes[ext] = change; + } + } + + for (const [id, extState] of Object.entries(runnerState.oldBuildState)) { + if (manifests[id] == null) { + // Removed a previous extension + runnerState.changes[id] = { + warnings: [], + errors: [], + + type: "remove", + oldManifest: extState.manifest + }; + } + } + + return changed; +} + +export default async function computeState() { + const buildStatePath = path.join(distRepo, "state.json"); + const oldBuildState: BuildState = (await pathExists(buildStatePath)) + ? JSON.parse(await fs.readFile(buildStatePath, "utf8")) + : {}; + console.log(`Loaded ${Object.keys(oldBuildState).length} state entries`); + + const runnerState: RunnerState = { + author: + authorId != null && authorUsername != null + ? { + id: authorId, + username: authorUsername, + pr: authorPr + } + : undefined, + warnings: [], + errors: [], + oldBuildState, + buildState: JSON.parse(JSON.stringify(oldBuildState)), + changes: {} + }; + + if (buildMode === "pr" && runnerState.author == null) { + // CI isn't providing the author right (maybe the workflow is broken?) + runnerState.warnings.push({ type: "missingAuthor" }); + } + + const extManifestsDir = path.join(manifestsRepo, "exts"); + const manifests = await getManifests(runnerState, extManifestsDir); + console.log(`Loaded ${Object.keys(manifests).length} manifests`); + + await diffManifests(runnerState, manifests); + console.log(`Processing ${Object.keys(runnerState.changes).length} changes`); + + return runnerState; +} diff --git a/src/modes/run/summary.ts b/src/modes/run/summary.ts new file mode 100644 index 0000000..362c091 --- /dev/null +++ b/src/modes/run/summary.ts @@ -0,0 +1,287 @@ +import { buildMode, workDir, type BuildMode } from "../../util/env.js"; +import { getCommitDiff, getCommitLink, getCommitTree } from "../../util/git.js"; +import { currentApiLevel } from "../../util/manifest.js"; +import type { ExtensionChange, RunnerState } from "./state.js"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +const buildModeEmojis: Partial> = { + push: [":shipit:", "push"], + pr: [":hammer:", "PR"] +}; + +const changeEmojis: Record = { + add: [":new:", "New extension."], + update: [":repeat:", "Updating extension."], + updateNoBuild: [":repeat_one:", "Updating build manifest."], + remove: [":put_litter_in_its_place:", "Deleting extension."] // I swear I'm not being mean that emoji shortcode is just rude +}; + +function filterByType(values: T) { + return [...values] + .reduce( + (prev, current) => { + return prev.some((i) => i.type === current.type) ? prev : ([...prev, current] as T); + }, + [] as unknown as T + ) + .sort(); +} + +function formatCommit(repository: string, commit: string, oldCommit?: string) { + let result = ""; + + const link = getCommitLink(repository, commit); + if (link != null) { + result += `[${commit}](${link})`; + } else { + result += commit; + } + + const tree = getCommitTree(repository, commit); + if (tree != null) result += ` ([Tree](${tree}))`; + + if (oldCommit != null) { + const diff = getCommitDiff(repository, oldCommit, commit); + if (diff != null) result += ` ([Diff](${diff}))`; + } + + return result; +} + +export default async function writeSummary(state: RunnerState) { + let summary = "# Extensions state\n\n"; + + const modeEmoji = buildMode != null ? buildModeEmojis[buildMode] : null; + if (modeEmoji != null) { + summary += `- ${modeEmoji[0]} Running in ${modeEmoji[1]} mode.\n`; + if (state.author != null) { + summary += + state.author.pr != null + ? ` - Running on behalf of \`${state.author.username}\` for PR ${state.author.pr}.\n` + : ` - Running on behalf of \`${state.author.username}\`.\n`; + } + } + + const allChanges = Object.values(state.changes); + const successCount = allChanges.filter((change) => change.errors.length === 0 && change.warnings.length === 0).length; + const warnCount = allChanges.filter((change) => change.errors.length === 0 && change.warnings.length !== 0).length; + const failCount = allChanges.filter((change) => change.errors.length !== 0).length; + + const warningMergeMessage = "Review all warnings before merging."; + const errorMergeMessage = "Do not merge."; + + if (allChanges.length !== 0) { + summary += `- Processed ${allChanges.length} extension change(s).\n`; + if (successCount !== 0) summary += ` - :white_check_mark: ${successCount} extension(s) built successfully.\n`; + if (warnCount !== 0) { + summary += ` - :warning: ${warnCount} extension(s) **built with warnings**.`; + if (buildMode === "pr") summary += ` ${warningMergeMessage}`; + summary += "\n"; + } + if (failCount !== 0) { + summary += ` - :x: ${failCount} extension(s) **failed to build**.`; + if (buildMode === "pr") summary += ` ${errorMergeMessage}`; + summary += "\n"; + } + } else { + summary += `- No extension changes.\n`; + } + + if (state.warnings.length !== 0) { + summary += `- :warning: Runner completed with **${state.warnings.length} warning(s).**\n`; + + for (const warning of filterByType(state.warnings)) { + switch (warning.type) { + case "unknown": { + summary += ` - **Unknown warning.** Check the build log for more info.\n`; + break; + } + + case "missingAuthor": { + summary += ` - **Author context is missing.** Check that the build workflows are correct.\n`; + break; + } + } + } + } + + if (state.errors.length !== 0) { + summary += `- :x: Runner completed with **${state.errors.length} error(s).**\n`; + + for (const error of filterByType(state.errors)) { + switch (error.type) { + case "parseManifestFailed": { + summary += ` - **Build manifests failed to parse.** Check that all manifests are valid.\n`; + break; + } + + case "deleteChangeFailed": { + summary += ` - **Failed to delete extensions.** Check the build log for more info.\n`; + break; + } + } + } + } + + summary += "\n"; + + for (const [ext, change] of Object.entries(state.changes)) { + summary += `## ${ext}\n\n`; + + const [emoji, typeName] = changeEmojis[change.type]; + summary += `- ${emoji} ${typeName}\n`; + + if (change.type === "remove" || change.type === "add") { + const manifest = change.type === "remove" ? change.oldManifest : change.newManifest; + summary += `- Repository: <${manifest.repository}>\n`; + summary += `- Commit: ${formatCommit(manifest.repository, manifest.commit)}\n`; + } else { + if (change.oldManifest.repository !== change.newManifest.repository) { + summary += `- Old repository: <${change.oldManifest.repository}>\n`; + summary += `- New repository: <${change.newManifest.repository}>\n`; + } else { + summary += `- Repository: <${change.newManifest.repository}>\n`; + } + + if (change.oldManifest.commit !== change.newManifest.commit) { + summary += `- Old commit: ${formatCommit(change.oldManifest.repository, change.oldManifest.commit)}\n`; + + // Don't show diff URL if repository changed + const newCommit = + change.oldManifest.repository === change.newManifest.repository + ? formatCommit(change.newManifest.repository, change.newManifest.commit, change.oldManifest.commit) + : formatCommit(change.newManifest.repository, change.newManifest.commit); + summary += `- New commit: ${newCommit}\n`; + } else { + summary += `- Commit: ${formatCommit(change.newManifest.repository, change.newManifest.commit)}\n`; + } + + // Theoretically this will only be null if there are already warnings/errors for it, so this should be safe + const oldBuildState = state.oldBuildState[ext]; + const buildState = state.buildState[ext]; + if (buildState != null && buildState.version != null) { + if (oldBuildState != null && oldBuildState.version != null) { + summary += `- Old version: ${oldBuildState.version}\n`; + summary += `- New version: ${buildState.version}\n`; + } else { + summary += `- Version: ${buildState.version}\n`; + } + } + } + + if (change.errors.length === 0 && change.warnings.length === 0) { + summary += `- :white_check_mark: Built successfully.\n`; + } else if (change.errors.length === 0 && change.warnings.length !== 0) { + summary += `- :warning: **Built with warnings.**`; + if (buildMode === "pr") summary += ` ${warningMergeMessage}`; + summary += "\n"; + } else { + summary += `- :x: **Failed to build.**`; + if (buildMode === "pr") summary += ` ${errorMergeMessage}`; + summary += "\n"; + } + + for (const warning of filterByType(change.warnings)) { + switch (warning.type) { + case "invalidApiLevel": { + summary += ` - **Invalid API level** (expected ${currentApiLevel}, got ${warning.value ?? "none"}). This extension will not load in moonlight.\n`; + break; + } + + case "invalidId": { + summary += ` - **Mismatched IDs** (expected ${ext}, got ${warning.value}). Ensure the same ID is used across all manifests.\n`; + break; + } + + case "irregularVersion": { + if (warning.value == null) { + summary += ` - **Missing version.** Updates may fail in Moonbase.\n`; + } else { + summary += ` - **Irregular version** (got ${warning.value}). This does not currently cause issues, but using a standard version format may be required in the future.\n`; + } + break; + } + + case "sameOrLowerVersion": { + if (warning.newVersion === warning.oldVersion) { + summary += ` - **Same version.** Updates will fail in Moonbase.\n`; + } else { + summary += ` - **Downgraded version.** This does not currently cause issues, but always incrementing versions may be required in the future.\n`; + } + break; + } + + case "noOwnersSpecified": { + summary += ` - **No owners specified.** This should be set to prevent extension hijacking.\n`; + break; + } + + case "repositoryChanged": { + summary += ` - **Repository changed.** Check that the new repository is not malicious.\n`; + break; + } + + case "buildConfigChanged": { + summary += ` - **Build config changed.** Recheck the build script and output artifact.\n`; + break; + } + + case "authorNotInOwners": { + summary += ` - **Author not in owners.** Check that the author has permission to update this extension.\n`; + break; + } + + case "ownersChanged": { + summary += ` - **Owners changed.** Check that all owners should be able to update this extension.\n`; + break; + } + + case "authorAddedToOwners": { + summary += ` - **Author added themselves to owners.** Check that the author has permission to adopt this extension.\n`; + break; + } + } + } + + for (const error of filterByType(change.errors)) { + switch (error.type) { + case "unknown": { + summary += ` - **Unknown error.** Check the build log for more info.\n`; + break; + } + + case "cloneFailed": { + summary += ` - **Clone failed.** Check that the target Git forge is online.\n`; + break; + } + + case "fetchFailed": { + summary += ` - **Fetch failed.** Check that the lockfile is up to date. The extension repository might be broken, or this might be a bug in the runner.\n`; + break; + } + + case "installFailed": { + summary += ` - **Install failed.** Check that the lockfile is up to date. The extension repository might be broken, or this might be a bug in the runner.\n`; + break; + } + + case "scriptFailed": { + summary += ` - **Running script ${error.script} failed.** The extension repository might be broken, or this might be a bug in the runner.\n`; + break; + } + + case "packageFailed": { + summary += ` - **Package failed.** Check that the output path is correct. This might be a bug in the runner.\n`; + break; + } + } + } + + summary += "\n"; + } + + // newline to make my markdown linting chimpanzee brain happy + const summaryPath = path.join(workDir, "summary.md"); + await fs.writeFile(summaryPath, summary.trim() + "\n"); +} diff --git a/src/run.ts b/src/run.ts deleted file mode 100644 index 3f373f7..0000000 --- a/src/run.ts +++ /dev/null @@ -1,348 +0,0 @@ -import { - BuildGroupResultSchema, - BuildManifestSchema, - compare, - defaultScripts, - type BuildGroupState, - type BuildManifest, - type BuildStates -} from "./util/manifest.js"; -import { ensureEnv, envVariables, mode } from "./util/env.js"; -import { ensureDir, pathExists } from "./util/fs.js"; -import { getCommitLink, getCommitTree, getCommitDiff, maybeWrapLink } from "./util/git.js"; -import * as fs from "node:fs/promises"; -import * as path from "node:path"; -import { exec } from "./util/exec.js"; -import runContainer from "./util/docker.js"; - -type Manifests = Record; -type ExtensionChange = - | { - type: "add"; - newManifest: BuildManifest; - } - | { - type: "update"; - oldManifest: BuildManifest; - newManifest: BuildManifest; - } - | { - type: "remove"; - oldManifest: BuildManifest; - }; - -async function getManifests(dir: string) { - const manifests: Manifests = {}; - - for (const filename of await fs.readdir(dir)) { - if (!filename.endsWith(".json")) continue; - - const id = filename.replace(/\.json$/, ""); - const filePath = path.join(dir, filename); - - const manifestStr = await fs.readFile(filePath, "utf8"); - const manifest = BuildManifestSchema.parse(JSON.parse(manifestStr)); - manifests[id] = manifest; - } - - return manifests; -} - -async function diffManifests(manifests: Manifests, state: BuildStates) { - const changed: Record = {}; - const buildAll = mode === "all"; - - for (const [id, newManifest] of Object.entries(manifests)) { - const oldManifest = state[id]?.manifest; - - const url = new URL(newManifest.repository); - if (url.protocol !== "https:") throw new Error("Only HTTPS Git urls are supported"); - - if (oldManifest == null) { - changed[id] = { - type: "add", - newManifest - }; - } else { - const diff = - buildAll || - compare(oldManifest.repository, newManifest.repository) || - compare(oldManifest.commit, newManifest.commit) || - compare(oldManifest.scripts, newManifest.scripts) || - compare(oldManifest.output, newManifest.output); - - if (diff) { - changed[id] = { - type: "update", - oldManifest, - newManifest - }; - } - } - } - - for (const [id, extState] of Object.entries(state)) { - if (manifests[id] == null) { - changed[id] = { - type: "remove", - oldManifest: extState!.manifest - }; - } - } - - return changed; -} - -type BuildGroup = { - repository: string; - commit: string; - scripts: string[]; - output: Partial>; - - directory: string; - hostDirectory: string; -}; - -async function build(group: BuildGroup) { - // Write the instructions for the builder - const groupStatePath = path.join(group.directory, "state.json"); - const groupState: BuildGroupState = { - scripts: group.scripts, - output: group.output - }; - await fs.writeFile(groupStatePath, JSON.stringify(groupState)); - - await runContainer(group.hostDirectory); - - // Pass through a schema in case it gets tampered with - const groupResultPath = path.join(group.directory, "result.json"); - const groupResultStr = await fs.readFile(groupResultPath, "utf8"); - const groupResult = BuildGroupResultSchema.parse(JSON.parse(groupResultStr)); - - return groupResult; -} - -async function processGroups(groupDir: string, groupHostDir: string, changes: Record) { - const groups: Partial> = {}; - const mapping: Partial> = {}; - let currentIdx = 0; - - for (const [id, change] of Object.entries(changes)) { - if (change.type === "remove") continue; - - const manifest = change.newManifest; - const scripts = manifest.scripts ?? [...defaultScripts]; - const key = `${manifest.repository}-${manifest.commit}-${JSON.stringify(scripts)}`; - - if (groups[key] == null) { - console.log("Creating group for", id, manifest); - - // Unique ID for this prefetch directory - const thisIdx = currentIdx++; - const thisDir = path.join(groupDir, thisIdx.toString()); - const thisDirHost = path.join(groupHostDir, thisIdx.toString()); - await ensureDir(thisDir); - - const sourceDir = path.join(thisDir, "source"); - const storeDir = path.join(thisDir, "store"); - const outputDir = path.join(thisDir, "output"); - await ensureDir(sourceDir); - await ensureDir(storeDir); - await ensureDir(outputDir); - - // https://stackoverflow.com/a/3489576 - await exec("git", ["init", "--initial-branch=main"], { - cwd: sourceDir - }); - await exec("git", ["remote", "add", "origin", manifest.repository], { - cwd: sourceDir - }); - await exec("git", ["fetch", "origin", manifest.commit], { - cwd: sourceDir - }); - await exec("git", ["reset", "--hard", "FETCH_HEAD"], { - cwd: sourceDir - }); - await exec("git", ["submodule", "update", "--init", "--recursive"], { - cwd: sourceDir - }); - - // Prefetch packages into the store so we can build offline - await exec("pnpm", ["fetch"], { - cwd: sourceDir, - env: { - PATH: process.env["PATH"], - [envVariables.npmStoreDir]: storeDir - } - }); - - groups[key] = { - repository: manifest.repository, - commit: manifest.commit, - scripts, - output: {}, - - hostDirectory: thisDirHost, - directory: thisDir - }; - } - - // Add our output dir to the group config so we pack it into an .asar - const output = manifest.output ?? `dist/${id}`; - groups[key].output[id] = output; - - // Add our extension to the mapping so we know what group we're in - mapping[id] = key; - } - - return { groups, mapping }; -} - -export default async function run() { - const manifestsEnv = process.env[envVariables.manifestsPath] ?? "/moonlight/manifests"; - const manifestsPath = path.join(manifestsEnv, "exts"); - - const distEnv = process.env[envVariables.distPath] ?? "/moonlight/dist"; - const distPath = path.join(distEnv, "exts"); - const statePath = path.join(distEnv, "state.json"); - - const workEnv = process.env[envVariables.workPath] ?? "/moonlight/work"; - const summaryPath = path.join(workEnv, "summary.md"); - await ensureDir(workEnv); - - const outputPath = path.join(workEnv, "output"); - await ensureDir(outputPath); - - const groupPath = path.join(workEnv, "group"); - await ensureDir(groupPath); - - const workHostEnv = ensureEnv(envVariables.workHostPath); - const groupHostPath = path.join(workHostEnv, "group"); - - const state: BuildStates = (await pathExists(statePath)) ? JSON.parse(await fs.readFile(statePath, "utf8")) : {}; - - const manifests = await getManifests(manifestsPath); - console.log(`Loaded ${Object.keys(manifests).length} manifests`); - - const diff = await diffManifests(manifests, state); - console.log("Diff results:", diff); - - let summary = "# Extensions state\n\n"; - - if (Object.keys(diff).length === 0) { - console.log("No changes"); - summary += "No changes."; - } - - // Build all extensions and get their new versions - const { groups, mapping } = await processGroups(groupPath, groupHostPath, diff); - console.log("Processed groups", mapping); - - const versions: Partial> = {}; - for (const [key, group] of Object.entries(groups)) { - console.log("Building group", key); - - const result = await build(group!); - for (const [ext, version] of Object.entries(result.versions)) { - versions[ext] = version; - } - - // Copy our .asars into the output directories - const groupOutputPath = path.join(group!.directory, "output"); - for (const filename of await fs.readdir(groupOutputPath)) { - const filePath = path.join(groupOutputPath, filename); - - const outputDestPath = path.join(outputPath, filename); - await fs.copyFile(filePath, outputDestPath); - - const distDestPath = path.join(distPath, filename); - await fs.copyFile(filePath, distDestPath); - } - } - - for (const [id, change] of Object.entries(diff)) { - let summaryMsg = `## ${id}\n\n`; - const oldState = state[id]; - - if (change.type === "remove") { - console.log("Removing", id); - - const asarPath = path.join(distPath, `${id}.asar`); - await fs.rm(asarPath, { force: true }); - - delete state[id]; - } else { - const version = versions[id]; - if (version == null) throw new Error(`Couldn't get version for ${id}`); - state[id] = { - version, - manifest: change.newManifest - }; - } - - const newState = state[id]; - summaryMsg += `- Type: ${change.type}\n`; - - const repository = change.type === "remove" ? change.oldManifest.repository : change.newManifest.repository; - summaryMsg += `- Repository: <${repository}>`; - if (change.type === "update" && change.newManifest.repository !== change.oldManifest.repository) { - summaryMsg += ` **(changed from <${change.oldManifest.repository}>)**`; - } - summaryMsg += "\n"; - - if (change.type === "update") { - const { repository, commit } = change.oldManifest; - const link = getCommitLink(repository, commit); - summaryMsg += `- Old commit: ${maybeWrapLink(commit, link)}`; - - const tree = getCommitTree(repository, commit); - if (tree != null) summaryMsg += ` ([Tree](${tree}))`; - - summaryMsg += "\n"; - } - - if (change.type !== "remove") { - const { repository, commit } = change.newManifest; - const link = getCommitLink(repository, commit); - summaryMsg += `- New commit: ${maybeWrapLink(commit, link)}`; - - const tree = getCommitTree(repository, commit); - if (tree != null) summaryMsg += ` ([Tree](${tree}))`; - - if (change.type === "update" && change.oldManifest.repository === repository) { - const diff = getCommitDiff(repository, change.oldManifest.commit, commit); - if (diff != null) { - summaryMsg += ` ([Diff](${diff}))`; - } - } - - summaryMsg += "\n"; - } - - const newVersion = newState?.version; - const oldVersion = oldState?.version; - if (change.type === "remove") { - if (oldVersion != null) summaryMsg += `- Version: ${oldVersion}`; - } else if (newVersion != null) { - summaryMsg += `- Version: ${newVersion}`; - - const oldVersion = oldState?.version; - if (oldVersion != null) { - if (oldVersion === newVersion) { - summaryMsg += ` **(same version)**`; - } else { - summaryMsg += ` (prev ${oldVersion})`; - } - } - - summaryMsg += "\n"; - } - - summaryMsg += "\n\n"; - summary += summaryMsg; - console.log(summaryMsg.trim()); - } - - await fs.writeFile(statePath, JSON.stringify(state, null, 2)); - await fs.writeFile(summaryPath, summary.trim()); -} diff --git a/src/util/docker.ts b/src/util/docker.ts index abb4f27..2d71bc3 100644 --- a/src/util/docker.ts +++ b/src/util/docker.ts @@ -1,13 +1,20 @@ import { Readable } from "node:stream"; import { Agent, fetch } from "undici"; -type CreateRequest = { +export type CreateRequest = { Image: string; - Env: string[]; - Tty: true; - NetworkDisabled: boolean; - HostConfig: { - Binds: string[]; + Env?: string[]; + Tty?: boolean; + NetworkDisabled?: boolean; + HostConfig?: { + Mounts?: { + Target: string; // container path + Source: string; // host path + Type: "bind"; + ReadOnly?: boolean; + }[]; + AutoRemove?: boolean; + // FIXME: CPU/RAM/disk/time limits }; }; @@ -15,26 +22,18 @@ const agent = new Agent({ connect: { socketPath: "/var/run/docker.sock" }, - bodyTimeout: 0 + bodyTimeout: 0 // for log streaming }); const version = "v1.40"; // picked randomly since idk what's recent-ish -async function create(hostDirectory: string) { +async function create(config: CreateRequest) { const resp = await fetch(`http://localhost/${version}/containers/create`, { dispatcher: agent, method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - Image: "moonlight-mod/extensions-runner:latest", - Env: ["MOONLIGHT_BUILD_MODE=group"], - Tty: true, - NetworkDisabled: true, - HostConfig: { - Binds: [`${hostDirectory}:/moonlight/group`] - } - } satisfies CreateRequest) + body: JSON.stringify(config) }); if (!resp.ok) throw new Error(resp.statusText); @@ -80,27 +79,17 @@ async function wait(id: string) { if (data.StatusCode !== 0) throw new Error(`Container exited with code ${data.StatusCode}`); } -async function kill(id: string) { - const url = new URL(`http://localhost/${version}/containers/${id}`); - url.searchParams.append("force", "true"); +export default async function runContainer(config: CreateRequest) { + console.log("Starting container:", config); + const id = await create(config); + console.log("Container created:", id); - const resp = await fetch(url, { - dispatcher: agent, - method: "DELETE" - }); - - if (!resp.ok) throw new Error(resp.statusText); -} - -export default async function runContainer(hostDirectory: string) { - const id = await create(hostDirectory); await start(id); const stream = await attach(id); try { await wait(id); } finally { - await kill(id); stream.destroy(); } } diff --git a/src/util/env.ts b/src/util/env.ts index 0d2f212..216f750 100644 --- a/src/util/env.ts +++ b/src/util/env.ts @@ -1,18 +1,38 @@ export const envVariables = { + buildMode: "MOONLIGHT_BUILD_MODE", + authorId: "MOONLIGHT_AUTHOR_ID", + authorUsername: "MOONLIGHT_AUTHOR_USERNAME", + authorPr: "MOONLIGHT_AUTHOR_PR", + manifestsPath: "MOONLIGHT_MANIFESTS_PATH", distPath: "MOONLIGHT_DIST_PATH", workPath: "MOONLIGHT_WORK_PATH", workHostPath: "MOONLIGHT_WORK_HOST_PATH", + groupPath: "MOONLIGHT_GROUP_PATH", - buildMode: "MOONLIGHT_BUILD_MODE", + storePath: "MOONLIGHT_STORE_PATH", npmStoreDir: "NPM_CONFIG_STORE_DIR" }; -export type BuildMode = "all" | "group" | null; -export const mode = (process.env[envVariables.buildMode] ?? null) as BuildMode; +export type BuildMode = "push" | "pr" | "all" | "fetch" | "build"; +export const buildMode = (process.env[envVariables.buildMode] ?? null) as BuildMode | null; + +export const defaultManifests = "/moonlight/manifests"; +export const defaultDist = "/moonlight/dist"; +export const defaultWork = "/moonlight/work"; + +export const manifestsRepo = process.env[envVariables.manifestsPath] ?? defaultManifests; +export const distRepo = process.env[envVariables.distPath] ?? defaultDist; +export const workDir = process.env[envVariables.workPath] ?? defaultWork; +export const workHostDir = process.env[envVariables.workHostPath]; + +export const defaultGroup = "/moonlight/group"; +export const defaultStore = "/moonlight/store"; + +export const groupDir = process.env[envVariables.groupPath] ?? defaultGroup; +export const storeDir = process.env[envVariables.storePath] ?? defaultStore; -export function ensureEnv(name: string) { - if (process.env[name] == null) throw new Error(`Missing environment variable: ${name}`); - return process.env[name]; -} +export const authorId = process.env[envVariables.authorId]; +export const authorUsername = process.env[envVariables.authorUsername]; +export const authorPr = process.env[envVariables.authorPr]; diff --git a/src/util/exec.ts b/src/util/exec.ts index b8d6301..72a01d5 100644 --- a/src/util/exec.ts +++ b/src/util/exec.ts @@ -1,19 +1,15 @@ import { spawn, type SpawnOptions } from "node:child_process"; -export function exec(cmd: string, args: string[], opts?: SpawnOptions): Promise { - return new Promise((resolve, reject) => { +export async function exec(cmd: string, args: string[] = [], opts?: SpawnOptions) { + const code = await new Promise((resolve, reject) => { const proc = spawn(cmd, args, { ...opts, stdio: "inherit" }); proc.on("error", reject); - proc.on("close", (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(`Process exited with code ${code}`)); - } - }); + proc.on("close", (code) => resolve(code)); }); + + if (code !== 0) throw new Error(`Process exited with code ${code}`); } diff --git a/src/util/fs.ts b/src/util/fs.ts index d15682f..454c423 100644 --- a/src/util/fs.ts +++ b/src/util/fs.ts @@ -7,30 +7,35 @@ export const pathExists = (path: string) => .then(() => true) .catch(() => false); -export async function ensureDir(path: string) { +export async function cleanDir(dir: string) { const isDirectory = await fs - .stat(path) + .stat(dir) .then((s) => s.isDirectory()) .catch(() => null); - // Exists, but is a file - if (isDirectory === false) throw new Error(`Tried to use file as directory: ${path}`); + if (isDirectory !== true) throw new Error(`Tried to clean a directory that doesn't exist: ${dir}`); - // Create if it doesn't exist - if (isDirectory === null) await fs.mkdir(path, { recursive: true }); + const entries = await fs.readdir(dir); + for (const entry of entries) { + const fullEntry = path.join(dir, entry); + await fs.rm(fullEntry, { recursive: true, force: true }); + } } -export async function recursiveCopy(src: string, dst: string) { - for (const filename of await fs.readdir(src)) { - const srcPath = path.join(src, filename); - const dstPath = path.join(dst, filename); - - const isDirectory = (await fs.stat(srcPath)).isDirectory(); - if (isDirectory) { - await ensureDir(dstPath); - await recursiveCopy(srcPath, dstPath); - } else { - await fs.copyFile(srcPath, dstPath); - } +export async function ensureDir(dir: string, clean?: boolean) { + const isDirectory = await fs + .stat(dir) + .then((s) => s.isDirectory()) + .catch(() => null); + + // Exists, but is a file + if (isDirectory === false) throw new Error(`Tried to use file as directory: ${dir}`); + + // Clean if needed (removing the files inside instead of the folder since it may be mounted) + if (clean && isDirectory === true) { + await cleanDir(dir); } + + // Create if it doesn't exist + if (isDirectory === null) await fs.mkdir(dir, { recursive: true }); } diff --git a/src/util/git.ts b/src/util/git.ts index ecfb170..004b453 100644 --- a/src/util/git.ts +++ b/src/util/git.ts @@ -52,12 +52,3 @@ export function getCommitDiff(repository: string, oldCommit: string, newCommit: return undefined; } - -// kinda meh about this being in this file but w/e -export function maybeWrapLink(text: string, link?: string) { - if (link != null) { - return `[${text}](${link})`; - } else { - return text; - } -} diff --git a/src/util/manifest.ts b/src/util/manifest.ts index 17f68b2..87926d8 100644 --- a/src/util/manifest.ts +++ b/src/util/manifest.ts @@ -1,4 +1,4 @@ -import { z } from "zod"; +import { z } from "@zod/mini"; // https://stackoverflow.com/a/78709590 const GitHashSchema = z.custom((val) => { @@ -6,46 +6,69 @@ const GitHashSchema = z.custom((val) => { }); export type BuildManifest = z.infer; -export const BuildManifestSchema = z.object({ - repository: z.string().url(), +export const BuildManifestSchema = z.strictObject({ + repository: z.url(), commit: GitHashSchema, - scripts: z.string().array().optional(), - output: z.string().optional() + owners: z.optional(z.array(z.string())), + scripts: z.optional(z.array(z.string())), + output: z.optional(z.string()) }); export const ExtensionManifestSchema = z.object({ id: z.string(), - apiLevel: z.number().optional(), - version: z.string().optional(), - meta: z.object({ - name: z.string(), - source: z.string() - }) + version: z.optional(z.string()), + apiLevel: z.optional(z.number()), + meta: z.optional( + z.object({ + name: z.optional(z.string()), + source: z.optional(z.string()) + }) + ) }); -export type BuildState = { - version: string; +// This type shouldn't be changed as it's saved in extensions-dist +export type ExtensionState = { + version?: string; manifest: BuildManifest; }; -export type BuildStates = Partial>; +export type BuildState = Record; -export type BuildGroupState = { +// This is passed into groups when fetching/building +export type MiniGroupState = { + repository: string; + commit: string; scripts: string[]; - output: Partial>; + outputs: Record; }; -export type BuildGroupResult = z.infer; -export const BuildGroupResultSchema = z.object({ - versions: z.record(z.string(), z.string()) +// This is a schema in case a build script tries to tamper with it +export type MiniGroupResult = z.infer; +export const MiniGroupResultSchema = z.strictObject({ + errors: z.array( + z.discriminatedUnion([ + z.interface({ type: z.literal("cloneFailed"), err: z.string() }), + z.interface({ type: z.literal("fetchFailed"), err: z.string() }), + z.interface({ type: z.literal("installFailed"), err: z.string() }), + z.interface({ type: z.literal("scriptFailed"), script: z.string(), err: z.string() }), + z.interface({ type: z.literal("packageFailed"), ext: z.string(), err: z.string() }) + ]) + ), + + manifests: z.record(z.string(), ExtensionManifestSchema) }); -export const defaultScripts = ["build"]; export const currentApiLevel = 2; +export const moonlightReviewers = [ + "44414597", // NotNite + "1606710", // Cynosphere + "42352565", // redstonekasi + "48024900" // adryd325 +]; // Simple compare function for the manifest diffing -export function compare(old?: T, current?: T) { +export function hasChanged(old?: T, current?: T) { if (old == null || current == null) return old != current; if (typeof old === "string" || typeof current === "string") return old !== current; - // This would be used on arrays, so positioning is fine + // This would be used on arrays only, so positioning is fine return JSON.stringify(old) !== JSON.stringify(current); } diff --git a/src/util/version.ts b/src/util/version.ts new file mode 100644 index 0000000..64e15c0 --- /dev/null +++ b/src/util/version.ts @@ -0,0 +1,23 @@ +// we have semver at home +export type ParsedVersion = [number, number, number]; + +const regex = /^(\d+)\.(\d+)\.(\d+)$/; + +export function parseVersion(version: string, silent: boolean = false) { + try { + const matches = regex.exec(version); + if (matches == null) return null; + return [matches[1], matches[2], matches[3]].map((value) => parseInt(value)) as ParsedVersion; + } catch (e) { + if (!silent) console.warn("Failed to parse version", version, e); + return null; + } +} + +export function versionGreaterThan(newVersion: ParsedVersion, oldVersion: ParsedVersion) { + return ( + newVersion[0] > oldVersion[0] || + (newVersion[0] === oldVersion[0] && newVersion[1] > oldVersion[1]) || + (newVersion[0] === oldVersion[0] && newVersion[1] === oldVersion[1] && newVersion[2] > oldVersion[2]) + ); +}