From 677c149345f5ed0a3fb823b52540877c3c7ec228 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Fri, 12 Jun 2026 11:35:46 -0700 Subject: [PATCH 1/3] =?UTF-8?q?[SLOP(claude-opus-4-8-high)]=20chore(agent-?= =?UTF-8?q?os):=20match=20rivet-dev/rivet=20for=20native=20binary=20?= =?UTF-8?q?=E2=80=94=20npm-publish=20exec=20bit=20+=20typed-config=20spawn?= =?UTF-8?q?=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yml | 23 ++++++++++++++++++----- CLAUDE.md | 6 ++++++ Cargo.lock | 14 +++++++------- crates/client/src/agent_os.rs | 6 ++++-- crates/client/src/config.rs | 10 ++++++++++ crates/client/src/sidecar.rs | 12 ++++++++++-- crates/client/src/transport.rs | 10 ++++++++-- packages/sidecar-binary/index.js | 25 ++++++------------------- 8 files changed, 69 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a220ab81..5c357df39 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -148,8 +148,14 @@ jobs: set -u FAILURES="" + # publisher: "npm" preserves the binary executable bit (pnpm publish + # normalizes file modes to 0644, stripping +x). Platform binary + # packages must use npm publish; workspace packages use pnpm publish so + # the `workspace:*` protocol is rewritten to concrete versions. This + # mirrors how rivet-dev/rivet ships rivet-engine via @rivetkit/engine-cli. publish_dir() { local dir="$1" + local publisher="${2:-pnpm}" local name version name=$(jq -r .name "$dir/package.json") version=$(jq -r .version "$dir/package.json") @@ -157,15 +163,22 @@ jobs: echo "⏭ ${name}@${version} already published, skipping." return 0 fi - echo "Publishing ${name}@${version}..." - if ! (cd "$dir" && pnpm publish --access public --tag "${NPM_TAG}" --no-git-checks); then - FAILURES="${FAILURES} ${name}" + echo "Publishing ${name}@${version} via ${publisher}..." + if [ "$publisher" = "npm" ]; then + if ! (cd "$dir" && npm publish --access public --tag "${NPM_TAG}"); then + FAILURES="${FAILURES} ${name}" + fi + else + if ! (cd "$dir" && pnpm publish --access public --tag "${NPM_TAG}" --no-git-checks); then + FAILURES="${FAILURES} ${name}" + fi fi } - # 1. Platform binary packages (not pnpm workspace members). + # 1. Platform binary packages (not pnpm workspace members). Published + # with npm to keep the binary's executable bit (see publish_dir). for p in ${PRESENT}; do - publish_dir "packages/sidecar-binary/npm/${p}" + publish_dir "packages/sidecar-binary/npm/${p}" npm done # 2. Workspace packages (skip root, skip registry/software/). This diff --git a/CLAUDE.md b/CLAUDE.md index 250b20bec..915f5c874 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,6 +40,12 @@ Agent OS is a **fully virtualized operating system**. The kernel, written as a R - **The default VM filesystem model should be Docker-like.** Layered overlay view with one writable upper layer on top of one or more immutable lower snapshot layers. - **Everything runs inside the VM.** Agent processes, servers, network requests -- all spawned inside the Agent OS kernel, never on the host. This is a hard rule with no exceptions. +## Native Binary Distribution + +- **Ship the `agent-os-sidecar` binary to npm the same way `rivet-dev/rivet` ships `rivet-engine`** via `@rivetkit/engine-cli` (reference: the rivet repo's `rivetkit-typescript/packages/engine-cli/index.js` resolver and `.github/workflows/publish.yaml`). The pattern: a meta resolver package (`@rivet-dev/agent-os-sidecar`) with platform packages (`@rivet-dev/agent-os-sidecar-`) declared as `optionalDependencies`, the binary bundled in each platform tarball, selected at install by npm `os`/`cpu`/`libc`. +- **Platform binary packages MUST be published with `npm publish`, never `pnpm publish`.** `pnpm publish` normalizes file modes to `0644` and strips the binary's executable bit; `npm publish` preserves `0755`. The release workflow publishes platform packages via npm and workspace packages via pnpm (for `workspace:*` rewriting). Because `npm publish` preserves `0755`, the resolver does NOT `chmod` at runtime, matching `@rivetkit/engine-cli`'s `getEnginePath()`. +- **Spawn the native binary the same way `rivet-dev/rivet` spawns `rivet-engine`: thread the resolved absolute path as a typed value down to `Command::new(path)`, not through a process-global env var.** Reference `rivetkit-core/src/engine_process.rs`, where `engineBinaryPath` flows TS (`getEnginePath()`) → typed serve config → `Command::new(binary_path)`. The agent-os client must mirror this: the npm-resolved `getSidecarPath()` value is passed through `AgentOsConfig` to `SidecarTransport::spawn(...)` and into `Command::new(...)`. The `AGENT_OS_SIDECAR_BIN` env var stays only as a debug/override fallback; callers must not rely on `std::env::set_var` to hand the path to the spawn. + ## Secure-Exec Reference Implementation The Rust sidecar kernel was migrated from a working JavaScript kernel (`@secure-exec/core` + `@secure-exec/nodejs` + `@secure-exec/v8`). The original source is at `/home/nathan/secure-exec-1/` (tagged `v0.2.1`), and recovered polyfill/bridge code lives at `~/.agents/recovery/secure-exec/`. **When something doesn't work in the Rust V8 isolate runtime, check how secure-exec handled it first** — the answer is almost always already there. Key reference files: diff --git a/Cargo.lock b/Cargo.lock index 93aa4a9dc..ab9ed6146 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,7 +16,7 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "agent-os-bridge" -version = "0.1.0" +version = "0.2.0-rc.3" dependencies = [ "serde", "serde_json", @@ -24,7 +24,7 @@ dependencies = [ [[package]] name = "agent-os-client" -version = "0.1.0" +version = "0.2.0-rc.3" dependencies = [ "agent-os-bridge", "agent-os-sidecar", @@ -50,7 +50,7 @@ dependencies = [ [[package]] name = "agent-os-execution" -version = "0.1.0" +version = "0.2.0-rc.3" dependencies = [ "agent-os-bridge", "agent-os-v8-runtime", @@ -67,7 +67,7 @@ dependencies = [ [[package]] name = "agent-os-kernel" -version = "0.1.0" +version = "0.2.0-rc.3" dependencies = [ "agent-os-bridge", "base64 0.22.1", @@ -80,7 +80,7 @@ dependencies = [ [[package]] name = "agent-os-sidecar" -version = "0.1.0" +version = "0.2.0-rc.3" dependencies = [ "agent-os-bridge", "agent-os-execution", @@ -122,7 +122,7 @@ dependencies = [ [[package]] name = "agent-os-sidecar-browser" -version = "0.1.0" +version = "0.2.0-rc.3" dependencies = [ "agent-os-bridge", "agent-os-kernel", @@ -130,7 +130,7 @@ dependencies = [ [[package]] name = "agent-os-v8-runtime" -version = "0.1.0" +version = "0.2.0-rc.3" dependencies = [ "agent-os-bridge", "ciborium", diff --git a/crates/client/src/agent_os.rs b/crates/client/src/agent_os.rs index c0301345a..f4fe0d402 100644 --- a/crates/client/src/agent_os.rs +++ b/crates/client/src/agent_os.rs @@ -178,9 +178,11 @@ impl AgentOs { let sidecar = match &config.sidecar { Some(crate::config::AgentOsSidecarConfig::Explicit { handle }) => handle.clone(), Some(crate::config::AgentOsSidecarConfig::Shared { pool }) => { - AgentOs::get_shared_sidecar(pool.clone()).await? + AgentOs::get_shared_sidecar(pool.clone(), config.sidecar_binary_path.clone()).await? + } + None => { + AgentOs::get_shared_sidecar(None, config.sidecar_binary_path.clone()).await? } - None => AgentOs::get_shared_sidecar(None).await?, }; let (transport, connection_id, max_frame_bytes) = sidecar.ensure_connection().await?; diff --git a/crates/client/src/config.rs b/crates/client/src/config.rs index df86117f4..5fb6fe4e6 100644 --- a/crates/client/src/config.rs +++ b/crates/client/src/config.rs @@ -40,6 +40,11 @@ pub struct AgentOsConfig { pub permissions: Option, /// Sidecar placement/config. Default: shared `default` pool. pub sidecar: Option, + /// Absolute path to the `agent-os-sidecar` binary, resolved from the npm + /// package on the TypeScript side. Threaded to `SidecarTransport::spawn` + /// (mirroring rivetkit's `engine_binary_path`) instead of relying on the + /// `AGENT_OS_SIDECAR_BIN` env var. `None` falls back to env, then `PATH`. + pub sidecar_binary_path: Option, } /// Builder for [`AgentOsConfig`]. @@ -108,6 +113,11 @@ impl AgentOsConfigBuilder { self } + pub fn sidecar_binary_path(mut self, path: impl Into) -> Self { + self.config.sidecar_binary_path = Some(path.into()); + self + } + pub fn build(self) -> AgentOsConfig { self.config } diff --git a/crates/client/src/sidecar.rs b/crates/client/src/sidecar.rs index aa93c7f8a..c57cb97e8 100644 --- a/crates/client/src/sidecar.rs +++ b/crates/client/src/sidecar.rs @@ -110,6 +110,10 @@ pub struct AgentOsSidecar { pub(crate) shared_pool: Option, pub(crate) state: AtomicU8, pub(crate) active_vm_count: AtomicU32, + /// Absolute path to the `agent-os-sidecar` binary, threaded from + /// `AgentOsConfig` (resolved from the npm package on the TS side) and passed + /// to `SidecarTransport::spawn` instead of relying on a process-global env. + pub(crate) sidecar_binary_path: Option, /// The shared sidecar process + authenticated connection, established on the first VM `create` /// against this sidecar and reused by every subsequent VM in the same (shared) sidecar. pub(crate) connection: tokio::sync::Mutex>, @@ -121,6 +125,7 @@ impl AgentOsSidecar { sidecar_id: impl Into, placement: AgentOsSidecarPlacement, shared_pool: Option, + sidecar_binary_path: Option, ) -> Self { Self { sidecar_id: sidecar_id.into(), @@ -128,6 +133,7 @@ impl AgentOsSidecar { shared_pool, state: AtomicU8::new(SidecarState::Ready.as_u8()), active_vm_count: AtomicU32::new(0), + sidecar_binary_path, connection: tokio::sync::Mutex::new(None), } } @@ -149,7 +155,7 @@ impl AgentOsSidecar { )); } - let transport = SidecarTransport::spawn().await?; + let transport = SidecarTransport::spawn(self.sidecar_binary_path.clone()).await?; let authed = match transport .request( OwnershipScope::connection("client-hint"), @@ -323,7 +329,7 @@ impl AgentOs { let placement = AgentOsSidecarPlacement::Explicit { sidecar_id: sidecar_id.clone(), }; - Ok(Arc::new(AgentOsSidecar::new(sidecar_id, placement, None))) + Ok(Arc::new(AgentOsSidecar::new(sidecar_id, placement, None, None))) } /// Get (or create) a pooled shared sidecar. Pool defaults to `"default"`. Uses the process-global @@ -336,6 +342,7 @@ impl AgentOs { /// atomically with `entry`/`insert` so two racing callers converge on a single live handle. pub async fn get_shared_sidecar( pool: Option, + sidecar_binary_path: Option, ) -> Result, ClientError> { let pool = pool.unwrap_or_else(|| "default".to_string()); let cache = shared_sidecars(); @@ -361,6 +368,7 @@ impl AgentOs { pool: placement_pool, }, Some(pool.clone()), + sidecar_binary_path, )); // Insert atomically, replacing a stale (disposed) entry but yielding to a live one that a diff --git a/crates/client/src/transport.rs b/crates/client/src/transport.rs index 2a16e57ed..42d16e3b4 100644 --- a/crates/client/src/transport.rs +++ b/crates/client/src/transport.rs @@ -69,8 +69,14 @@ impl SidecarTransport { /// /// Does NOT run the handshake; `AgentOs::create` drives Authenticate -> OpenSession -> CreateVm -> /// ConfigureVm using [`request`](Self::request) once the transport is live. - pub(crate) async fn spawn() -> Result, ClientError> { - let bin = std::env::var(SIDECAR_BIN_ENV).unwrap_or_else(|_| "agent-os-sidecar".to_string()); + pub(crate) async fn spawn(binary_path: Option) -> Result, ClientError> { + // Prefer the typed path threaded from `AgentOsConfig` (resolved from the + // npm package on the TypeScript side), mirroring how rivetkit threads + // `engine_binary_path` into `Command::new`. The `AGENT_OS_SIDECAR_BIN` + // env var stays only as a debug/override fallback. + let bin = binary_path + .or_else(|| std::env::var(SIDECAR_BIN_ENV).ok()) + .unwrap_or_else(|| "agent-os-sidecar".to_string()); let mut child = Command::new(&bin) .stdin(Stdio::piped()) .stdout(Stdio::piped()) diff --git a/packages/sidecar-binary/index.js b/packages/sidecar-binary/index.js index c1e2540c3..128861a8c 100644 --- a/packages/sidecar-binary/index.js +++ b/packages/sidecar-binary/index.js @@ -10,25 +10,15 @@ // 2. A `agent-os-sidecar` binary placed next to this package (dev builds). // 3. The platform-specific `@rivet-dev/agent-os-sidecar-` package. -const { existsSync, chmodSync, statSync } = require("node:fs"); +const { existsSync } = require("node:fs"); const { join, dirname } = require("node:path"); const BINARY_NAME = "agent-os-sidecar"; -// npm normalizes published non-`bin` file modes to 0644, stripping the -// executable bit from the platform binary. Restore it (best effort) so the -// resolved path is directly spawnable. -function ensureExecutable(binaryPath) { - try { - const mode = statSync(binaryPath).mode; - if ((mode & 0o111) === 0) { - chmodSync(binaryPath, mode | 0o755); - } - } catch { - // Best effort: a read-only install or unsupported platform should not - // break resolution. The spawn will surface a clear error if needed. - } -} +// No runtime chmod: the platform packages are published with `npm publish`, +// which preserves the binary's 0755 executable bit (pnpm publish would strip +// it to 0644). This mirrors how @rivetkit/engine-cli ships rivet-engine. See +// the "Native Binary Distribution" section in CLAUDE.md. function getPlatformPackageName() { const { platform, arch } = process; @@ -56,7 +46,6 @@ function getSidecarPath() { const localBinary = join(__dirname, BINARY_NAME); if (existsSync(localBinary)) { - ensureExecutable(localBinary); return localBinary; } @@ -81,9 +70,7 @@ function getSidecarPath() { ); } - const binaryPath = join(dirname(pkgJsonPath), BINARY_NAME); - ensureExecutable(binaryPath); - return binaryPath; + return join(dirname(pkgJsonPath), BINARY_NAME); } module.exports = { getSidecarPath }; From 5c2cbb2d8acee68a9f6f45fd036924ed623cd426 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Fri, 12 Jun 2026 13:56:51 -0700 Subject: [PATCH 2/3] [SLOP(claude-opus-4-8)] feat(base-filesystem): bake /etc/agentos/instructions.md into the base layer so createSession has OS instructions (single prompt source) --- crates/kernel/assets/base-filesystem.json | 17 ++++++++++- crates/sidecar/assets/base-filesystem.json | 17 ++++++++++- packages/core/fixtures/base-filesystem.json | 17 ++++++++++- .../core/scripts/build-base-filesystem.mjs | 28 ++++++++++++++++++- 4 files changed, 75 insertions(+), 4 deletions(-) diff --git a/crates/kernel/assets/base-filesystem.json b/crates/kernel/assets/base-filesystem.json index 3e24bdbb8..88b854248 100644 --- a/crates/kernel/assets/base-filesystem.json +++ b/crates/kernel/assets/base-filesystem.json @@ -3,7 +3,7 @@ "snapshotPath": "alpine-defaults.json", "image": "alpine:3.22", "snapshotCreatedAt": "2026-04-01T18:44:12.482Z", - "builtAt": "2026-04-01T19:03:51.260Z", + "builtAt": "2026-06-12T21:26:11.555Z", "transforms": [ "Normalize HOSTNAME to agent-os", "Preserve the captured user-level environment and filesystem layout as the AgentOs base layer" @@ -522,6 +522,21 @@ "mode": "1777", "uid": 0, "gid": 0 + }, + { + "path": "/etc/agentos", + "type": "directory", + "mode": "755", + "uid": 0, + "gid": 0 + }, + { + "path": "/etc/agentos/instructions.md", + "type": "file", + "mode": "644", + "uid": 0, + "gid": 0, + "content": "# agentOS\n\nYou are running inside agentOS, a Linux-like operating system for coding agents. \n\nKnown limitations:\n\n- Binaries inside agentOS are special. You cannot download and install arbitrary binaries (e.g. via curl or apt).\n- No file watching (inotify, fs.watch).\n\n## Bug Reports\n\nIf you find a bug in something that does not behave like you would expect in agentOS, generate a bug report and indicate the user should file it under: https://github.com/rivet-dev/rivet/issues (This repository holds agentOS inside of it.)\n\n## Tools CLI\n\nTools are available as CLI commands:\n\n- `agentos list-tools` — list all available toolkits and tools\n- `agentos-{toolkit} {tool} --help` — show usage for a specific tool\n- `agentos-{toolkit} {tool} --flag value` — invoke a tool\n\n" } ] } diff --git a/crates/sidecar/assets/base-filesystem.json b/crates/sidecar/assets/base-filesystem.json index 3e24bdbb8..88b854248 100644 --- a/crates/sidecar/assets/base-filesystem.json +++ b/crates/sidecar/assets/base-filesystem.json @@ -3,7 +3,7 @@ "snapshotPath": "alpine-defaults.json", "image": "alpine:3.22", "snapshotCreatedAt": "2026-04-01T18:44:12.482Z", - "builtAt": "2026-04-01T19:03:51.260Z", + "builtAt": "2026-06-12T21:26:11.555Z", "transforms": [ "Normalize HOSTNAME to agent-os", "Preserve the captured user-level environment and filesystem layout as the AgentOs base layer" @@ -522,6 +522,21 @@ "mode": "1777", "uid": 0, "gid": 0 + }, + { + "path": "/etc/agentos", + "type": "directory", + "mode": "755", + "uid": 0, + "gid": 0 + }, + { + "path": "/etc/agentos/instructions.md", + "type": "file", + "mode": "644", + "uid": 0, + "gid": 0, + "content": "# agentOS\n\nYou are running inside agentOS, a Linux-like operating system for coding agents. \n\nKnown limitations:\n\n- Binaries inside agentOS are special. You cannot download and install arbitrary binaries (e.g. via curl or apt).\n- No file watching (inotify, fs.watch).\n\n## Bug Reports\n\nIf you find a bug in something that does not behave like you would expect in agentOS, generate a bug report and indicate the user should file it under: https://github.com/rivet-dev/rivet/issues (This repository holds agentOS inside of it.)\n\n## Tools CLI\n\nTools are available as CLI commands:\n\n- `agentos list-tools` — list all available toolkits and tools\n- `agentos-{toolkit} {tool} --help` — show usage for a specific tool\n- `agentos-{toolkit} {tool} --flag value` — invoke a tool\n\n" } ] } diff --git a/packages/core/fixtures/base-filesystem.json b/packages/core/fixtures/base-filesystem.json index 3e24bdbb8..88b854248 100644 --- a/packages/core/fixtures/base-filesystem.json +++ b/packages/core/fixtures/base-filesystem.json @@ -3,7 +3,7 @@ "snapshotPath": "alpine-defaults.json", "image": "alpine:3.22", "snapshotCreatedAt": "2026-04-01T18:44:12.482Z", - "builtAt": "2026-04-01T19:03:51.260Z", + "builtAt": "2026-06-12T21:26:11.555Z", "transforms": [ "Normalize HOSTNAME to agent-os", "Preserve the captured user-level environment and filesystem layout as the AgentOs base layer" @@ -522,6 +522,21 @@ "mode": "1777", "uid": 0, "gid": 0 + }, + { + "path": "/etc/agentos", + "type": "directory", + "mode": "755", + "uid": 0, + "gid": 0 + }, + { + "path": "/etc/agentos/instructions.md", + "type": "file", + "mode": "644", + "uid": 0, + "gid": 0, + "content": "# agentOS\n\nYou are running inside agentOS, a Linux-like operating system for coding agents. \n\nKnown limitations:\n\n- Binaries inside agentOS are special. You cannot download and install arbitrary binaries (e.g. via curl or apt).\n- No file watching (inotify, fs.watch).\n\n## Bug Reports\n\nIf you find a bug in something that does not behave like you would expect in agentOS, generate a bug report and indicate the user should file it under: https://github.com/rivet-dev/rivet/issues (This repository holds agentOS inside of it.)\n\n## Tools CLI\n\nTools are available as CLI commands:\n\n- `agentos list-tools` — list all available toolkits and tools\n- `agentos-{toolkit} {tool} --help` — show usage for a specific tool\n- `agentos-{toolkit} {tool} --flag value` — invoke a tool\n\n" } ] } diff --git a/packages/core/scripts/build-base-filesystem.mjs b/packages/core/scripts/build-base-filesystem.mjs index af02e1b88..b0e7ad17d 100644 --- a/packages/core/scripts/build-base-filesystem.mjs +++ b/packages/core/scripts/build-base-filesystem.mjs @@ -10,6 +10,9 @@ const DEFAULT_INPUT = fileURLToPath( const DEFAULT_OUTPUT = fileURLToPath( new URL("../fixtures/base-filesystem.json", import.meta.url), ); +const OS_INSTRUCTIONS_FIXTURE = fileURLToPath( + new URL("../fixtures/AGENTOS_SYSTEM_PROMPT.md", import.meta.url), +); const BASE_HOSTNAME = "agent-os"; const BASE_USER = "user"; @@ -53,11 +56,34 @@ function buildBaseFilesystem(snapshot, inputPath) { prompt: snapshot.environment.prompt, }, filesystem: { - entries: snapshot.filesystem.entries.map(normalizeEntry), + entries: [ + ...snapshot.filesystem.entries.map(normalizeEntry), + ...osInstructionsEntries(), + ], }, }; } +// Bake the base agentOS system prompt into the snapshot so every VM has +// `/etc/agentos/instructions.md` by default. This is the single source for the +// prompt: the TS core, Rust client, and sidecar all consume the file from the +// VM rather than embedding their own copy. Session-level additional +// instructions are appended at session creation, not here. +function osInstructionsEntries() { + const content = readFileSync(OS_INSTRUCTIONS_FIXTURE, "utf-8"); + return [ + { path: "/etc/agentos", type: "directory", mode: "755", uid: 0, gid: 0 }, + { + path: "/etc/agentos/instructions.md", + type: "file", + mode: "644", + uid: 0, + gid: 0, + content, + }, + ]; +} + function main() { const [inputPath = DEFAULT_INPUT, outputPath = DEFAULT_OUTPUT] = process.argv.slice(2); const snapshot = readJson(inputPath); From 1f6524e87a8ae22c9e5b62a3e6712df07a14a375 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Fri, 12 Jun 2026 15:42:11 -0700 Subject: [PATCH 3/3] [SLOP(claude-opus-4-8)] feat(release): mirror rivetkit publish/preview-publish flow (scripts/publish package + unified publish.yaml workflow) --- .claude/skills/release/SKILL.md | 81 +++--- .github/workflows/publish.yaml | 344 +++++++++++++++++++++++ .github/workflows/release.yml | 344 ----------------------- justfile | 5 +- pnpm-lock.yaml | 126 +++++++++ pnpm-workspace.yaml | 1 + scripts/publish/package.json | 23 ++ scripts/publish/src/ci/bin.ts | 311 ++++++++++++++++++++ scripts/publish/src/lib/context.ts | 247 ++++++++++++++++ scripts/publish/src/lib/git.ts | 66 +++++ scripts/publish/src/lib/logger.ts | 28 ++ scripts/publish/src/lib/npm.ts | 264 +++++++++++++++++ scripts/publish/src/lib/packages.ts | 203 +++++++++++++ scripts/publish/src/lib/r2.ts | 155 ++++++++++ scripts/publish/src/lib/version.ts | 302 ++++++++++++++++++++ scripts/publish/src/local/cut-release.ts | 197 +++++++++++++ scripts/publish/tsconfig.json | 17 ++ scripts/release.ts | 243 ---------------- 18 files changed, 2324 insertions(+), 633 deletions(-) create mode 100644 .github/workflows/publish.yaml delete mode 100644 .github/workflows/release.yml create mode 100644 scripts/publish/package.json create mode 100644 scripts/publish/src/ci/bin.ts create mode 100644 scripts/publish/src/lib/context.ts create mode 100644 scripts/publish/src/lib/git.ts create mode 100644 scripts/publish/src/lib/logger.ts create mode 100644 scripts/publish/src/lib/npm.ts create mode 100644 scripts/publish/src/lib/packages.ts create mode 100644 scripts/publish/src/lib/r2.ts create mode 100644 scripts/publish/src/lib/version.ts create mode 100644 scripts/publish/src/local/cut-release.ts create mode 100644 scripts/publish/tsconfig.json delete mode 100644 scripts/release.ts diff --git a/.claude/skills/release/SKILL.md b/.claude/skills/release/SKILL.md index f3f2fba84..ac8eef1a5 100644 --- a/.claude/skills/release/SKILL.md +++ b/.claude/skills/release/SKILL.md @@ -1,68 +1,59 @@ --- name: release -description: Bump version, build, test, commit, tag, push, and publish to npm. Use when the user asks to release, publish, cut a release, or bump the version. +description: Cut a release or trigger a preview publish via the scripts/publish flow. Use when the user asks to release, publish, cut a release, bump the version, or preview-publish a branch. --- # Release -## Usage -- `/release patch` / `minor` / `major` — semver bump from current version -- `/release 0.2.0` or `/release 0.2.0-rc.1` — exact version +The publish flow lives in `scripts/publish` and is driven by the unified +`.github/workflows/publish.yaml` workflow. There are two modes: -## Workflow +- **Release** — a versioned cut that publishes to npm + crates.io, creates a + GitHub release with binaries, and tags `v`. +- **Preview publish** — a branch snapshot published to npm only, under a + branch-named dist-tag, using fast debug builds. No git tag, no crates.io + release, no GitHub release. + +## Release -### 1. Preflight ```bash -git branch --show-current # must be main -git status --porcelain # must be clean -git fetch origin main && git rev-parse HEAD && git rev-parse origin/main # must match +just release --version 0.2.0 # exact version +just release --version 0.2.0-rc.1 # rc (npm tag `rc`, prerelease) +just release --patch # semver bump from latest git tag ``` -Stop and report if any check fails. -### 2. Determine version -- `patch` / `minor` / `major`: strip any prerelease suffix from current `package.json` version, then bump (e.g. `0.1.0-rc.1` + patch = `0.1.1`) -- Exact version (e.g. `0.2.0`, `0.3.0-rc.1`): use as-is +`just release` runs `scripts/publish/src/local/cut-release.ts`, which: +1. Resolves the version and the `latest` flag (auto-detected from git tags). +2. Validates the working tree is clean and prints a plan to confirm. +3. Rewrites `Cargo.toml` + every publishable `package.json` version. +4. Runs a local core build + type-check fail-fast (`--skip-checks` to skip). +5. Commits + pushes the version bump. +6. Triggers `publish.yaml` with the version, which builds release binaries, + publishes npm + crates.io, uploads release assets, and tags `v`. -### 3. Determine npm tag -- Check the latest stable (non-prerelease) version published on npm: `npm view antiox versions --json` -- If the target version has a prerelease suffix (contains `-`): use `--tag rc` -- If the target version is stable and greater than the current latest stable on npm: use `--tag latest` -- If the target version is stable but older than the current latest on npm: stop and warn the user (this would move the `latest` tag backwards) +Flags: `--latest` / `--no-latest`, `--dry-run` (mutate files only), `-y`. -Show the user: current version, target version, npm tag. Ask to confirm. +## Preview publish -### 4. Build and test ```bash -pnpm build -pnpm check-types -pnpm test +just preview-publish ``` -Stop if any step fails. - -### 5. Bump version -Update `version` in `package.json` to the target version. -### 6. Commit, tag, push -```bash -git add package.json -git commit -m "release: v" -git tag v -git push origin main -git push origin v -``` +Dispatches `publish.yaml` on the branch with no version. The context resolver +computes `version = 0.0.0-.` and `npm_tag = `, +builds a debug sidecar, and publishes every package to npm under that tag. +Install a preview with: -### 7. Trigger GitHub release workflow ```bash -gh workflow run release.yml -f version= -f npm-tag= +npm install @rivet-dev/agent-os-core@ ``` -This dispatches the `release.yml` workflow, which checks out the tag, builds, publishes to npm, and creates a GitHub release. -### 8. Report -Show the user: final commit hash, tag, and the link to the triggered workflow run. +## Notes -## Rules -- Never skip git checks unless the user explicitly asks. -- Never force-push. -- Always build + test before publishing. -- Never publish to npm locally; always use the GitHub release workflow. +- Never publish to npm or crates.io locally; always go through `publish.yaml`. +- Platform binary packages publish with `npm publish` (preserves the `0755` + executable bit). `workspace:*` deps are rewritten to literal versions by the + full `bump-versions` pass before publish, so `npm publish` resolves them. +- `SIDECAR_PLATFORMS` (workflow env + `scripts/publish` discovery) is the single + source of truth for which platform binary packages are built and published. - If anything fails, stop and report — do not retry automatically. diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 000000000..7c9a2af8c --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,344 @@ +# ============================================================================= +# Publish +# ============================================================================= +# +# Single workflow for preview publish AND release cuts. Both triggers are +# manual (workflow_dispatch). The build + npm publish steps run identically for +# both modes. Only the build profile (debug vs release), npm dist-tag, and the +# release-only tail (crates.io, GitHub release assets, R2, git tag) differ. +# +# Triggers and mapping: +# workflow_dispatch (no version) → trigger=branch npm_tag= build=debug +# workflow_dispatch (with version) → trigger=release npm_tag=latest build=release +# (npm_tag becomes `rc` if version contains `-rc.`, or +# `next` if `latest=false`) +# +# Preview publishes are fast: debug sidecar build, npm only (the binary ships +# inside the npm platform package), crates dry-run. Releases add the crates.io +# publish, GitHub release assets, R2 mirror, and git tag. +# ============================================================================= + +name: publish + +on: + workflow_dispatch: + inputs: + version: + description: "Version to release (e.g. 0.2.0 or 0.2.0-rc.1). Leave empty for preview publish on the dispatched branch." + required: false + type: string + latest: + description: "Tag as @latest (release only)" + required: true + type: boolean + default: true + +concurrency: + group: publish-${{ github.ref }} + cancel-in-progress: false + +env: + R2_BUCKET: rivet-releases + R2_ENDPOINT: https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com + # Platform list kept in sync with packages/sidecar-binary/npm/* and the build + # matrix below. Also consumed by scripts/publish discovery via SIDECAR_PLATFORMS. + # arm64 is deferred until its portability fix is verified end-to-end. + SIDECAR_PLATFORMS: "linux-x64-gnu" + +jobs: + # --------------------------------------------------------------------------- + # context — resolve PublishContext once, pin as job outputs + # --------------------------------------------------------------------------- + context: + name: "Context" + runs-on: ubuntu-latest + outputs: + trigger: ${{ steps.ctx.outputs.trigger }} + version: ${{ steps.ctx.outputs.version }} + npm_tag: ${{ steps.ctx.outputs.npm_tag }} + sha: ${{ steps.ctx.outputs.sha }} + latest: ${{ steps.ctx.outputs.latest }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - name: Install publish scripts + run: pnpm install --frozen-lockfile --filter=publish + - id: ctx + name: Resolve publish context + run: pnpm --filter=publish exec tsx src/ci/bin.ts context-output + + # --------------------------------------------------------------------------- + # build-sidecar — debug (preview) or release (release) sidecar binary + # --------------------------------------------------------------------------- + build-sidecar: + needs: [context] + name: "Build sidecar (${{ matrix.platform }})" + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x64-gnu + runner: ubuntu-22.04 + target: x86_64-unknown-linux-gnu + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - uses: Swatinem/rust-cache@v2 + with: + workspaces: . -> target + key: ${{ matrix.target }}-${{ needs.context.outputs.trigger }} + - uses: actions/cache@v4 + with: + path: ~/.cargo/.rusty_v8 + key: ${{ runner.os }}-${{ matrix.target }}-rusty-v8-${{ hashFiles('Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.target }}-rusty-v8- + # The V8 bridge build script (invoked by cargo build.rs) needs the pnpm + # workspace installed so esbuild and v8-bridge.source.js are available. + - run: pnpm install --frozen-lockfile + - name: Build sidecar binary + id: build + run: | + if [ "${{ needs.context.outputs.trigger }}" = "release" ]; then + cargo build --release -p agent-os-sidecar --target ${{ matrix.target }} + echo "bin=target/${{ matrix.target }}/release/agent-os-sidecar" >> "$GITHUB_OUTPUT" + else + cargo build -p agent-os-sidecar --target ${{ matrix.target }} + echo "bin=target/${{ matrix.target }}/debug/agent-os-sidecar" >> "$GITHUB_OUTPUT" + fi + - uses: actions/upload-artifact@v4 + with: + name: sidecar-${{ matrix.platform }} + path: ${{ steps.build.outputs.bin }} + if-no-files-found: error + + # --------------------------------------------------------------------------- + # publish-npm — place binaries, build TS, publish all packages (all triggers) + # --------------------------------------------------------------------------- + publish-npm: + needs: [context, build-sidecar] + name: "Publish npm" + if: ${{ !cancelled() && needs.build-sidecar.result == 'success' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + registry-url: https://registry.npmjs.org + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@v4 + with: + pattern: sidecar-* + path: artifacts + - name: Place sidecar binaries into platform packages + run: | + set -euo pipefail + for p in $SIDECAR_PLATFORMS; do + bin="artifacts/sidecar-${p}/agent-os-sidecar" + dest="packages/sidecar-binary/npm/${p}" + if [ ! -f "$bin" ]; then + echo "::error::missing sidecar binary artifact for ${p}" + exit 1 + fi + if [ ! -d "$dest" ]; then + echo "::error::missing platform package dir $dest" + exit 1 + fi + cp "$bin" "${dest}/agent-os-sidecar" + chmod +x "${dest}/agent-os-sidecar" + echo "Placed binary for ${p}" + done + - name: Bump package versions for build (version-only) + run: | + pnpm --filter=publish exec tsx src/ci/bin.ts bump-versions \ + --version ${{ needs.context.outputs.version }} \ + --version-only + - name: Build TypeScript packages + run: | + npx turbo build \ + --filter='!./registry/software/*' \ + --filter='!@rivet-dev/agent-os-playground' \ + --filter='!./examples/*' + - name: Finalize package versions for publish (inject optionalDeps) + run: | + pnpm --filter=publish exec tsx src/ci/bin.ts bump-versions \ + --version ${{ needs.context.outputs.version }} + - name: Publish npm packages + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + pnpm --filter=publish exec tsx src/ci/bin.ts publish-npm \ + --tag ${{ needs.context.outputs.npm_tag }} \ + --parallel 16 \ + --retries 3 \ + ${{ needs.context.outputs.trigger == 'release' && '--release-mode' || '' }} + + # --------------------------------------------------------------------------- + # release-assets — GitHub release + sidecar/pyodide assets + R2 (release only) + # --------------------------------------------------------------------------- + release-assets: + needs: [context, build-sidecar] + name: "Release assets" + if: ${{ !cancelled() && needs.build-sidecar.result == 'success' && needs.context.outputs.trigger == 'release' }} + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - run: pnpm install --frozen-lockfile --filter=publish + - uses: actions/download-artifact@v4 + with: + pattern: sidecar-* + path: artifacts + - name: Stage release assets + env: + VERSION: ${{ needs.context.outputs.version }} + run: | + set -euo pipefail + declare -A PLATFORM_TARGET=( + [linux-x64-gnu]=x86_64-unknown-linux-gnu + [linux-arm64-gnu]=aarch64-unknown-linux-gnu + ) + # Stage release assets: platform binaries + externalized pyodide assets + # (downloaded by the published execution crate's build.rs). + mkdir -p release-assets + for p in $SIDECAR_PLATFORMS; do + bin="artifacts/sidecar-${p}/agent-os-sidecar" + if [ -f "$bin" ]; then + target="${PLATFORM_TARGET[$p]}" + cp "$bin" "release-assets/agent-os-sidecar-${target}" + else + echo "::warning::no binary for ${p}" + fi + done + for f in \ + pyodide.asm.wasm \ + pyodide.asm.js \ + python_stdlib.zip \ + numpy-2.2.5-cp313-cp313-pyodide_2025_0_wasm32.whl \ + pandas-2.3.3-cp313-cp313-pyodide_2025_0_wasm32.whl; do + cp "crates/execution/assets/pyodide/${f}" "release-assets/${f}" + done + - name: Create GitHub release and upload assets + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.context.outputs.version }} + NPM_TAG: ${{ needs.context.outputs.npm_tag }} + run: | + set -euo pipefail + if ! gh release view "v${VERSION}" >/dev/null 2>&1; then + PRERELEASE="" + if [ "${NPM_TAG}" = "rc" ]; then PRERELEASE="--prerelease"; fi + gh release create "v${VERSION}" --title "v${VERSION}" --generate-notes $PRERELEASE + fi + # --clobber so re-runs overwrite assets idempotently. + gh release upload "v${VERSION}" release-assets/* --clobber + - name: Upload sidecar binaries to R2 (best-effort) + env: + R2_RELEASES_ACCESS_KEY_ID: ${{ secrets.R2_RELEASES_ACCESS_KEY_ID }} + R2_RELEASES_SECRET_ACCESS_KEY: ${{ secrets.R2_RELEASES_SECRET_ACCESS_KEY }} + VERSION: ${{ needs.context.outputs.version }} + SHA: ${{ needs.context.outputs.sha }} + LATEST: ${{ needs.context.outputs.latest }} + run: | + set -uo pipefail + if [ -z "${R2_RELEASES_ACCESS_KEY_ID:-}" ] || [ -z "${R2_RELEASES_SECRET_ACCESS_KEY:-}" ]; then + echo "R2 credentials not configured; skipping R2 upload (GitHub release assets are authoritative)." + exit 0 + fi + set -e + pnpm --filter=publish exec tsx src/ci/bin.ts upload-r2 \ + --source "$GITHUB_WORKSPACE/release-assets" --sha "$SHA" + pnpm --filter=publish exec tsx src/ci/bin.ts copy-r2 \ + --sha "$SHA" --version "$VERSION" --latest "$LATEST" + + # --------------------------------------------------------------------------- + # publish-crates — crates.io (dry-run on preview, full publish on release) + # --------------------------------------------------------------------------- + publish-crates: + needs: [context, build-sidecar, release-assets] + name: "Publish crates.io" + if: ${{ !cancelled() && needs.build-sidecar.result == 'success' && (needs.release-assets.result == 'success' || needs.release-assets.result == 'skipped') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: . -> target + - uses: actions/cache@v4 + with: + path: ~/.cargo/.rusty_v8 + key: ${{ runner.os }}-x86_64-unknown-linux-gnu-rusty-v8-${{ hashFiles('Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-x86_64-unknown-linux-gnu-rusty-v8- + - run: pnpm install --frozen-lockfile + - name: Bump Cargo versions + run: | + pnpm --filter=publish exec tsx src/ci/bin.ts bump-versions \ + --version ${{ needs.context.outputs.version }} \ + --version-only + - name: Stage vendored V8 bridge bundles + base-filesystem for publishing + run: | + set -euo pipefail + for crate in execution v8-runtime; do + out="crates/${crate}/assets/generated" + mkdir -p "$out" + node packages/core/scripts/build-v8-bridge.mjs --out-dir "$out" + done + # The generated bundles are gitignored, and `cargo package` honors + # gitignore. Force them into the git index so they get packaged into + # the published crate (consumed by build_support.rs at build time). + git add -f crates/execution/assets/generated crates/v8-runtime/assets/generated + # Refresh the vendored base-filesystem fixtures used by published crates. + cp packages/core/fixtures/base-filesystem.json crates/kernel/assets/base-filesystem.json + cp packages/core/fixtures/base-filesystem.json crates/sidecar/assets/base-filesystem.json + - name: Dry-run crate publish (preview) + if: ${{ needs.context.outputs.trigger != 'release' }} + run: | + pnpm --filter=publish exec tsx src/ci/bin.ts publish-crates \ + --version ${{ needs.context.outputs.version }} \ + --dry-run \ + --allow-dirty + - name: Publish crates (release) + if: ${{ needs.context.outputs.trigger == 'release' }} + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: | + pnpm --filter=publish exec tsx src/ci/bin.ts publish-crates \ + --version ${{ needs.context.outputs.version }} \ + --allow-dirty diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 5c357df39..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,344 +0,0 @@ -name: Release - -on: - workflow_dispatch: - inputs: - version: - description: "Version to publish (e.g. 0.2.0, 0.2.0-rc.2)" - required: true - type: string - npm-tag: - description: "npm dist-tag" - required: true - type: choice - options: - - latest - - rc - -concurrency: - group: release - cancel-in-progress: false - -env: - R2_BUCKET: rivet-releases - R2_ENDPOINT: https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com - # Platform list kept in sync with packages/sidecar-binary/npm/* and the build matrix. - # arm64 is deferred: it needs a portability fix verified end-to-end before the - # meta package advertises it as an optionalDependency. - SIDECAR_PLATFORMS: "linux-x64-gnu" - -jobs: - build-sidecar: - name: "Build sidecar (${{ matrix.platform }})" - strategy: - fail-fast: false - matrix: - include: - - platform: linux-x64-gnu - runner: ubuntu-22.04 - target: x86_64-unknown-linux-gnu - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v4 - with: - ref: v${{ inputs.version }} - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - cache-dependency-path: pnpm-lock.yaml - - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} - - uses: Swatinem/rust-cache@v2 - with: - workspaces: . -> target - key: ${{ matrix.target }} - - uses: actions/cache@v4 - with: - path: ~/.cargo/.rusty_v8 - key: ${{ runner.os }}-${{ matrix.target }}-rusty-v8-${{ hashFiles('Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.target }}-rusty-v8- - # The V8 bridge build script (invoked by cargo build.rs) needs the pnpm - # workspace installed so esbuild and v8-bridge.source.js are available. - - run: pnpm install --frozen-lockfile - - name: Build sidecar binary - run: cargo build --release -p agent-os-sidecar --target ${{ matrix.target }} - - uses: actions/upload-artifact@v4 - with: - name: sidecar-${{ matrix.platform }} - path: target/${{ matrix.target }}/release/agent-os-sidecar - if-no-files-found: error - - publish-npm: - name: "Publish npm" - needs: build-sidecar - if: ${{ !cancelled() }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: v${{ inputs.version }} - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - cache-dependency-path: pnpm-lock.yaml - registry-url: https://registry.npmjs.org - - run: pnpm install --frozen-lockfile - - name: Build TypeScript packages - run: npx turbo build --filter='!./registry/software/*' --filter='!@rivet-dev/agent-os-playground' --filter='!./examples/*' - - uses: actions/download-artifact@v4 - with: - pattern: sidecar-* - path: artifacts - - name: Place sidecar binaries into platform packages - id: place - run: | - PRESENT="" - for p in $SIDECAR_PLATFORMS; do - bin="artifacts/sidecar-${p}/agent-os-sidecar" - dest="packages/sidecar-binary/npm/${p}" - if [ -f "$bin" ]; then - if [ ! -d "$dest" ]; then - echo "::error::missing platform package dir $dest" - exit 1 - fi - cp "$bin" "${dest}/agent-os-sidecar" - chmod +x "${dest}/agent-os-sidecar" - echo "Placed binary for ${p}" - PRESENT="${PRESENT} ${p}" - else - echo "::warning::no binary artifact for ${p}; skipping" - fi - done - case " ${PRESENT} " in - *" linux-x64-gnu "*) ;; - *) echo "::error::linux-x64-gnu binary is required but missing"; exit 1 ;; - esac - echo "present=${PRESENT}" >> "$GITHUB_OUTPUT" - - name: Inject optionalDependencies into meta package - env: - VERSION: ${{ inputs.version }} - PRESENT: ${{ steps.place.outputs.present }} - run: | - node -e ' - const fs = require("fs"); - const version = process.env.VERSION; - const present = process.env.PRESENT.trim().split(/\s+/).filter(Boolean); - const path = "packages/sidecar-binary/package.json"; - const pkg = JSON.parse(fs.readFileSync(path, "utf8")); - pkg.optionalDependencies = pkg.optionalDependencies || {}; - for (const plat of present) { - pkg.optionalDependencies[`@rivet-dev/agent-os-sidecar-${plat}`] = version; - } - fs.writeFileSync(path, JSON.stringify(pkg, null, "\t") + "\n"); - console.log("optionalDependencies:", pkg.optionalDependencies); - ' - - name: Publish to npm - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - VERSION: ${{ inputs.version }} - NPM_TAG: ${{ inputs.npm-tag }} - PRESENT: ${{ steps.place.outputs.present }} - run: | - set -u - FAILURES="" - - # publisher: "npm" preserves the binary executable bit (pnpm publish - # normalizes file modes to 0644, stripping +x). Platform binary - # packages must use npm publish; workspace packages use pnpm publish so - # the `workspace:*` protocol is rewritten to concrete versions. This - # mirrors how rivet-dev/rivet ships rivet-engine via @rivetkit/engine-cli. - publish_dir() { - local dir="$1" - local publisher="${2:-pnpm}" - local name version - name=$(jq -r .name "$dir/package.json") - version=$(jq -r .version "$dir/package.json") - if npm view "${name}@${version}" version >/dev/null 2>&1; then - echo "⏭ ${name}@${version} already published, skipping." - return 0 - fi - echo "Publishing ${name}@${version} via ${publisher}..." - if [ "$publisher" = "npm" ]; then - if ! (cd "$dir" && npm publish --access public --tag "${NPM_TAG}"); then - FAILURES="${FAILURES} ${name}" - fi - else - if ! (cd "$dir" && pnpm publish --access public --tag "${NPM_TAG}" --no-git-checks); then - FAILURES="${FAILURES} ${name}" - fi - fi - } - - # 1. Platform binary packages (not pnpm workspace members). Published - # with npm to keep the binary's executable bit (see publish_dir). - for p in ${PRESENT}; do - publish_dir "packages/sidecar-binary/npm/${p}" npm - done - - # 2. Workspace packages (skip root, skip registry/software/). This - # includes the @rivet-dev/agent-os-sidecar meta package and core. - for dir in $(pnpm -r ls --json --depth -1 | jq -r '.[] | select(.private != true) | .path'); do - if [ "$dir" = "$(pwd)" ]; then - continue - fi - if [[ "$dir" == *"/registry/software/"* ]]; then - echo "⏭ Skipping software package: $(jq -r .name "$dir/package.json")" - continue - fi - publish_dir "$dir" - done - - if [ -n "$FAILURES" ]; then - echo "::error::Failed to publish:${FAILURES}" - exit 1 - fi - - release-assets: - name: "Release assets" - needs: build-sidecar - if: ${{ !cancelled() }} - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - ref: v${{ inputs.version }} - - uses: actions/download-artifact@v4 - with: - pattern: sidecar-* - path: artifacts - - name: Create GitHub release and upload assets - env: - GH_TOKEN: ${{ github.token }} - VERSION: ${{ inputs.version }} - NPM_TAG: ${{ inputs.npm-tag }} - run: | - set -euo pipefail - declare -A PLATFORM_TARGET=( - [linux-x64-gnu]=x86_64-unknown-linux-gnu - [linux-arm64-gnu]=aarch64-unknown-linux-gnu - ) - - # Stage release assets: platform binaries + externalized pyodide - # assets (downloaded by the published execution crate's build.rs). - mkdir -p release-assets - for p in $SIDECAR_PLATFORMS; do - bin="artifacts/sidecar-${p}/agent-os-sidecar" - if [ -f "$bin" ]; then - target="${PLATFORM_TARGET[$p]}" - cp "$bin" "release-assets/agent-os-sidecar-${target}" - else - echo "::warning::no binary for ${p}" - fi - done - for f in \ - pyodide.asm.wasm \ - pyodide.asm.js \ - python_stdlib.zip \ - numpy-2.2.5-cp313-cp313-pyodide_2025_0_wasm32.whl \ - pandas-2.3.3-cp313-cp313-pyodide_2025_0_wasm32.whl; do - cp "crates/execution/assets/pyodide/${f}" "release-assets/${f}" - done - - if ! gh release view "v${VERSION}" >/dev/null 2>&1; then - PRERELEASE="" - if [ "${NPM_TAG}" = "rc" ]; then PRERELEASE="--prerelease"; fi - gh release create "v${VERSION}" --title "v${VERSION}" --generate-notes $PRERELEASE - fi - # --clobber so re-runs overwrite assets idempotently. - gh release upload "v${VERSION}" release-assets/* --clobber - - - name: Best-effort upload to R2 (skipped without credentials) - env: - AWS_ACCESS_KEY_ID: ${{ secrets.R2_RELEASES_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_RELEASES_SECRET_ACCESS_KEY }} - VERSION: ${{ inputs.version }} - run: | - set -uo pipefail - if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then - echo "R2 credentials not configured; skipping R2 upload (GitHub release assets are authoritative)." - exit 0 - fi - set -e - for f in release-assets/agent-os-sidecar-*; do - base=$(basename "$f") - aws s3 cp "$f" "s3://${R2_BUCKET}/agent-os/${VERSION}/sidecar/${base}" \ - --endpoint-url "$R2_ENDPOINT" --checksum-algorithm CRC32 - done - - publish-crates: - name: "Publish crates.io" - needs: release-assets - if: ${{ !cancelled() && needs.release-assets.result == 'success' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: v${{ inputs.version }} - - uses: pnpm/action-setup@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - cache-dependency-path: pnpm-lock.yaml - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - with: - workspaces: . -> target - - uses: actions/cache@v4 - with: - path: ~/.cargo/.rusty_v8 - key: ${{ runner.os }}-x86_64-unknown-linux-gnu-rusty-v8-${{ hashFiles('Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-x86_64-unknown-linux-gnu-rusty-v8- - - run: pnpm install --frozen-lockfile - - name: Stage vendored V8 bridge bundles for publishing - run: | - set -euo pipefail - for crate in execution v8-runtime; do - out="crates/${crate}/assets/generated" - mkdir -p "$out" - node packages/core/scripts/build-v8-bridge.mjs --out-dir "$out" - done - # The generated bundles are gitignored, and `cargo package` honors - # gitignore. Force them into the git index so they get packaged into - # the published crate (consumed by build_support.rs at build time). - git add -f crates/execution/assets/generated crates/v8-runtime/assets/generated - # Refresh the vendored base-filesystem fixtures used by published crates. - cp packages/core/fixtures/base-filesystem.json crates/kernel/assets/base-filesystem.json - cp packages/core/fixtures/base-filesystem.json crates/sidecar/assets/base-filesystem.json - - name: Publish crates in dependency order - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - VERSION: ${{ inputs.version }} - run: | - set -uo pipefail - publish_crate() { - local crate="$1" - if curl -sf "https://index.crates.io/$(echo "$crate" | cut -c1-2)/$(echo "$crate" | cut -c3-4)/$crate" \ - | grep -q "\"vers\":\"${VERSION}\""; then - echo "⏭ ${crate}@${VERSION} already published, skipping." - return 0 - fi - echo "Publishing ${crate}@${VERSION}..." - # --allow-dirty: vendored bundles/fixtures are staged but uncommitted. - cargo publish -p "$crate" --allow-dirty - } - - # Dependency order: bridge -> {kernel, v8-runtime} -> execution -> sidecar -> client. - for crate in \ - agent-os-bridge \ - agent-os-kernel \ - agent-os-v8-runtime \ - agent-os-execution \ - agent-os-sidecar \ - agent-os-client; do - publish_crate "$crate" - done diff --git a/justfile b/justfile index eeb2bd91f..4f08cb16b 100644 --- a/justfile +++ b/justfile @@ -1,7 +1,10 @@ set positional-arguments := true release *args: - npx tsx scripts/release.ts "$@" + pnpm --filter=publish release "$@" + +preview-publish REF: + gh workflow run .github/workflows/publish.yaml --ref "{{ REF }}" dev-shell *args: pnpm --filter @rivet-dev/agent-os-dev-shell dev-shell -- "$@" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5702f78be..b36e25765 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1062,6 +1062,34 @@ importers: specifier: ^2.1.8 version: 2.1.9(@types/node@22.19.15) + scripts/publish: + dependencies: + commander: + specifier: ^12.1.0 + version: 12.1.0 + execa: + specifier: ^8.0.1 + version: 8.0.1 + glob: + specifier: ^10.3.10 + version: 10.5.0 + semver: + specifier: ^7.6.0 + version: 7.7.4 + devDependencies: + '@types/node': + specifier: ^24.3.0 + version: 24.13.2 + '@types/semver': + specifier: ^7.5.8 + version: 7.7.1 + tsx: + specifier: ^4.0.0 + version: 4.21.0 + typescript: + specifier: ^5.5.2 + version: 5.9.3 + packages: '@agentclientprotocol/sdk@0.12.0': @@ -1473,11 +1501,13 @@ packages: '@browserbasehq/browse-cli@0.5.0': resolution: {integrity: sha512-FRkT6rLAy1RqTo90gZ19pcQQMQ37M0OjzKmqQt6KeUsifLoGwZGiUBYMtuspVSA1FjNbzr46rwIEwT34TMS4ug==} engines: {node: ^20.19.0 || >=22.12.0} + deprecated: 'The Browserbase CLI is now published as ''browse''. Please migrate by running: npm uninstall -g @browserbasehq/browse-cli && npm install -g browse' hasBin: true '@browserbasehq/cli@0.5.4': resolution: {integrity: sha512-NXv1+Ad3KpGQjlpzb8SgwD6E0m8Yol2UqwsCBrMZuP/hCQeWicy0KOLxDVqlnx1VqtclTNUI8kyv7/MHMaKVkg==} engines: {node: '>=18'} + deprecated: 'The Browserbase CLI is now published as ''browse''. Please migrate by running: npm uninstall -g @browserbasehq/cli && npm install -g browse' hasBin: true '@browserbasehq/sdk@2.10.0': @@ -2041,20 +2071,24 @@ packages: '@mariozechner/pi-agent-core@0.60.0': resolution: {integrity: sha512-1zQcfFp8r0iwZCxCBQ9/ccFJoagns68cndLPTJJXl1ZqkYirzSld1zBOPxLAgeAKWIz3OX8dB2WQwTJFhmEojQ==} engines: {node: '>=20.0.0'} + deprecated: please use @earendil-works/pi-agent-core instead going forward '@mariozechner/pi-ai@0.60.0': resolution: {integrity: sha512-OiMuXQturnEDPmA+ho7eLe4G8plO2z21yjNMs9niQREauoblWOz7Glv58I66KPzczLED4aZTlQLTRdU6t1rz8A==} engines: {node: '>=20.0.0'} + deprecated: please use @earendil-works/pi-ai instead going forward hasBin: true '@mariozechner/pi-coding-agent@0.60.0': resolution: {integrity: sha512-IOv7cTU4nbznFNUE5ofi13k2dmSG39coBoGWIBQTVw3iVyl0HxuHbg0NiTx3ktrPIDNtkii+y7tWXzWqwoo4lw==} engines: {node: '>=20.6.0'} + deprecated: please use @earendil-works/pi-coding-agent instead going forward hasBin: true '@mariozechner/pi-tui@0.60.0': resolution: {integrity: sha512-ZAK5gxYhGmfJqMjfWcRBjB8glITltDbTrYJXvcDtfengbKTZN0p39p5uO5pvUB8/PiAWKTRS06yaNMhf/LG26g==} engines: {node: '>=20.0.0'} + deprecated: please use @earendil-works/pi-tui instead going forward '@mistralai/mistralai@1.14.1': resolution: {integrity: sha512-IiLmmZFCCTReQgPAT33r7KQ1nYo5JPdvGkrkZqA8qQ2qB1GHgs5LoP5K2ICyrjnpw2n8oSxMM/VP+liiKcGNlQ==} @@ -2702,9 +2736,15 @@ packages: '@types/node@22.19.15': resolution: {integrity: sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg==} + '@types/node@24.13.2': + resolution: {integrity: sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==} + '@types/retry@0.12.0': resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} + '@types/semver@7.7.1': + resolution: {integrity: sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==} + '@types/uuid@10.0.0': resolution: {integrity: sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==} @@ -3436,6 +3476,10 @@ packages: evp_bytestokey@1.0.3: resolution: {integrity: sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==} + execa@8.0.1: + resolution: {integrity: sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==} + engines: {node: '>=16.17'} + expand-template@2.0.3: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} @@ -3609,6 +3653,10 @@ packages: resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} engines: {node: '>=8'} + get-stream@8.0.1: + resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} + engines: {node: '>=16'} + get-tsconfig@4.13.7: resolution: {integrity: sha512-7tN6rFgBlMgpBML5j8typ92BKFi2sFQvIdpAqLA2beia5avZDrMs0FLZiM5etShWq5irVyGcGMEA1jcDaK7A/Q==} @@ -3729,6 +3777,10 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + human-signals@5.0.0: + resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} + engines: {node: '>=16.17.0'} + humanize-ms@1.2.1: resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} @@ -3800,6 +3852,10 @@ packages: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} + is-stream@3.0.0: + resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + is-typed-array@1.1.15: resolution: {integrity: sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==} engines: {node: '>= 0.4'} @@ -3952,6 +4008,9 @@ packages: resolution: {integrity: sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==} engines: {node: '>=18'} + merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + miller-rabin@4.0.1: resolution: {integrity: sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==} hasBin: true @@ -3972,6 +4031,10 @@ packages: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} + mimic-fn@4.0.0: + resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} + engines: {node: '>=12'} + mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} @@ -4084,6 +4147,10 @@ packages: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} + npm-run-path@5.3.0: + resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} @@ -4124,6 +4191,10 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + onetime@6.0.0: + resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==} + engines: {node: '>=12'} + openai@4.104.0: resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} hasBin: true @@ -4229,6 +4300,10 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + path-key@4.0.0: + resolution: {integrity: sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==} + engines: {node: '>=12'} + path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} @@ -4663,6 +4738,10 @@ packages: resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} engines: {node: '>=12'} + strip-final-newline@3.0.0: + resolution: {integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==} + engines: {node: '>=12'} + strip-json-comments@2.0.1: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} @@ -4835,6 +4914,9 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + undici@7.24.6: resolution: {integrity: sha512-Xi4agocCbRzt0yYMZGMA6ApD7gvtUFaxm4ZmeacWI4cZxaF6C+8I8QfofC20NAePiB/IcvZmzkJ7XPa471AEtA==} engines: {node: '>=20.18.1'} @@ -4861,6 +4943,7 @@ packages: uuid@10.0.0: resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true uuid@11.1.0: @@ -4869,6 +4952,7 @@ packages: uuid@9.0.1: resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true vary@1.1.2: @@ -7161,8 +7245,14 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/node@24.13.2': + dependencies: + undici-types: 7.18.2 + '@types/retry@0.12.0': {} + '@types/semver@7.7.1': {} + '@types/uuid@10.0.0': {} '@types/yauzl@2.10.3': @@ -8012,6 +8102,18 @@ snapshots: md5.js: 1.3.5 safe-buffer: 5.2.1 + execa@8.0.1: + dependencies: + cross-spawn: 7.0.6 + get-stream: 8.0.1 + human-signals: 5.0.0 + is-stream: 3.0.0 + merge-stream: 2.0.0 + npm-run-path: 5.3.0 + onetime: 6.0.0 + signal-exit: 4.1.0 + strip-final-newline: 3.0.0 + expand-template@2.0.3: {} expect-type@1.3.0: {} @@ -8239,6 +8341,8 @@ snapshots: dependencies: pump: 3.0.4 + get-stream@8.0.1: {} + get-tsconfig@4.13.7: dependencies: resolve-pkg-maps: 1.0.0 @@ -8402,6 +8506,8 @@ snapshots: transitivePeerDependencies: - supports-color + human-signals@5.0.0: {} + humanize-ms@1.2.1: dependencies: ms: 2.1.3 @@ -8464,6 +8570,8 @@ snapshots: is-stream@2.0.1: {} + is-stream@3.0.0: {} + is-typed-array@1.1.15: dependencies: which-typed-array: 1.1.20 @@ -8606,6 +8714,8 @@ snapshots: merge-descriptors@2.0.0: {} + merge-stream@2.0.0: {} + miller-rabin@4.0.1: dependencies: bn.js: 4.12.3 @@ -8623,6 +8733,8 @@ snapshots: dependencies: mime-db: 1.54.0 + mimic-fn@4.0.0: {} + mimic-response@3.1.0: {} minimalistic-assert@1.0.1: {} @@ -8737,6 +8849,10 @@ snapshots: normalize-path@3.0.0: {} + npm-run-path@5.3.0: + dependencies: + path-key: 4.0.0 + nth-check@2.1.1: dependencies: boolbase: 1.0.0 @@ -8778,6 +8894,10 @@ snapshots: dependencies: wrappy: 1.0.2 + onetime@6.0.0: + dependencies: + mimic-fn: 4.0.0 + openai@4.104.0(ws@8.20.0(bufferutil@4.1.0))(zod@3.25.76): dependencies: '@types/node': 18.19.130 @@ -8892,6 +9012,8 @@ snapshots: path-key@3.1.1: {} + path-key@4.0.0: {} + path-parse@1.0.7: {} path-scurry@1.11.1: @@ -9460,6 +9582,8 @@ snapshots: dependencies: ansi-regex: 6.2.2 + strip-final-newline@3.0.0: {} + strip-json-comments@2.0.1: {} strip-json-comments@5.0.3: {} @@ -9660,6 +9784,8 @@ snapshots: undici-types@6.21.0: {} + undici-types@7.18.2: {} + undici@7.24.6: {} unpipe@1.0.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c45b1e461..0fb4d4179 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,6 +5,7 @@ packages: - registry/agent/* - registry/file-system/* - registry/tool/* + - scripts/publish ignoredBuiltDependencies: - '@clerk/shared' diff --git a/scripts/publish/package.json b/scripts/publish/package.json new file mode 100644 index 000000000..c717d22f7 --- /dev/null +++ b/scripts/publish/package.json @@ -0,0 +1,23 @@ +{ + "name": "publish", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "ci": "tsx src/ci/bin.ts", + "release": "tsx src/local/cut-release.ts", + "check-types": "tsc --noEmit" + }, + "dependencies": { + "commander": "^12.1.0", + "execa": "^8.0.1", + "glob": "^10.3.10", + "semver": "^7.6.0" + }, + "devDependencies": { + "@types/node": "^24.3.0", + "@types/semver": "^7.5.8", + "tsx": "^4.0.0", + "typescript": "^5.5.2" + } +} diff --git a/scripts/publish/src/ci/bin.ts b/scripts/publish/src/ci/bin.ts new file mode 100644 index 000000000..1aecc37f5 --- /dev/null +++ b/scripts/publish/src/ci/bin.ts @@ -0,0 +1,311 @@ +#!/usr/bin/env tsx +/** + * CI entrypoint. Every workflow step calls exactly one subcommand. + * + * Each subcommand is a pure function of its flags — nothing orchestrates + * other subcommands. The GitHub Actions workflow is the orchestrator. + * + * Subcommands accept inputs via flags AND will fall back to re-resolving + * the `PublishContext` from env vars when flags aren't passed. The workflow + * uses the `context-output` subcommand to resolve once and pin values as + * job outputs, then passes those outputs to each subsequent step as flags. + */ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { Command } from "commander"; +import { $ } from "execa"; +import { + resolveContext, + writeContextToGithubOutput, + type Trigger, +} from "../lib/context.js"; +import { createGhRelease, tagAndPush } from "../lib/git.js"; +import { scoped } from "../lib/logger.js"; +import { publishAll } from "../lib/npm.js"; +import { copyPrefix, uploadDir } from "../lib/r2.js"; +import { bumpCargoVersions, bumpPackageJsons } from "../lib/version.js"; + +const log = scoped("ci"); + +// Internal crates published to crates.io in dependency order: +// bridge -> {kernel, v8-runtime} -> execution -> sidecar -> client. +const RUST_CRATES = [ + "agent-os-bridge", + "agent-os-kernel", + "agent-os-v8-runtime", + "agent-os-execution", + "agent-os-sidecar", + "agent-os-client", +] as const; + +function findRepoRoot(): string { + if (process.env.GITHUB_WORKSPACE) return process.env.GITHUB_WORKSPACE; + let dir = dirname(fileURLToPath(import.meta.url)); + for (let i = 0; i < 10; i++) { + try { + readFileSync(join(dir, "pnpm-workspace.yaml"), "utf-8"); + return dir; + } catch { + dir = dirname(dir); + } + } + throw new Error("could not locate repo root"); +} + +async function crateVersionExists(name: string, version: string): Promise { + const response = await fetch( + `https://crates.io/api/v1/crates/${encodeURIComponent(name)}/${encodeURIComponent(version)}`, + { + headers: { + "User-Agent": "agent-os-release-publisher (https://github.com/rivet-dev/agent-os)", + }, + }, + ); + if (response.status === 200) return true; + if (response.status === 404) return false; + throw new Error( + `crates.io lookup failed for ${name}@${version}: ${response.status} ${response.statusText}`, + ); +} + +async function waitForCrateVersion( + name: string, + version: string, + timeoutSeconds: number, +): Promise { + const deadline = Date.now() + timeoutSeconds * 1000; + while (Date.now() < deadline) { + if (await crateVersionExists(name, version)) return; + await sleep(10_000); + } + throw new Error(`timed out waiting for crates.io to index ${name}@${version}`); +} + +async function cargoPublishWithRateLimitRetry( + repoRoot: string, + args: string[], +): Promise { + for (;;) { + const result = await $({ + all: true, + cwd: repoRoot, + reject: false, + })`cargo ${args}`; + + if (result.all) process.stdout.write(result.all); + if (result.exitCode === 0) return; + + const retryAt = parseCratesIoRateLimitRetry(result.all ?? ""); + if (retryAt === undefined) { + throw new Error(`cargo ${args.join(" ")} failed with exit code ${result.exitCode}`); + } + + const waitMs = Math.max(retryAt.getTime() - Date.now() + 5_000, 10_000); + log.info(`crates.io rate limited publish; retrying at ${retryAt.toISOString()}`); + await sleep(waitMs); + } +} + +function parseCratesIoRateLimitRetry(output: string): Date | undefined { + const match = output.match(/try again after (.+? GMT)/); + if (!match) return undefined; + const retryAt = new Date(match[1]); + if (Number.isNaN(retryAt.getTime())) return undefined; + return retryAt; +} + +const program = new Command(); +program.name("ci").description("CI subcommands for the publish flow"); + +// --------------------------------------------------------------------------- +// context-output — resolve once, write to $GITHUB_OUTPUT for downstream steps +// --------------------------------------------------------------------------- +program + .command("context-output") + .description("Resolve publish context and write to $GITHUB_OUTPUT") + .option("--trigger ", "Override trigger (branch|release)") + .option("--version ", "Override version") + .option("--latest ", "Override latest") + .option("--branch ", "Override branch name") + .action(async (opts) => { + const overrides: Parameters[0] = {}; + if (opts.trigger) overrides.trigger = opts.trigger as Trigger; + if (opts.version) overrides.version = opts.version; + if (opts.latest !== undefined) overrides.latest = opts.latest === "true"; + if (opts.branch) overrides.branch = opts.branch; + const ctx = await resolveContext(overrides); + log.info( + `resolved: trigger=${ctx.trigger} version=${ctx.version} npm_tag=${ctx.npmTag} sha=${ctx.sha} latest=${ctx.latest}${ctx.branch !== undefined ? ` branch=${ctx.branch}` : ""}`, + ); + writeContextToGithubOutput(ctx); + }); + +// --------------------------------------------------------------------------- +// bump-versions — rewrite package.jsons + Cargo.toml to a version +// --------------------------------------------------------------------------- +program + .command("bump-versions") + .description("Rewrite every publishable package.json and Cargo.toml to the given version") + .option("--version ", "Version to write (defaults to resolved context)") + .option( + "--version-only", + "Only rewrite version fields without publish-time dependency injection", + ) + .option("--dry-run", "Do not write, only report") + .action(async (opts) => { + const repoRoot = findRepoRoot(); + const ctx = await resolveContext(); + const version = opts.version ?? ctx.version; + await bumpPackageJsons(repoRoot, version, { + dryRun: !!opts.dryRun, + versionOnly: !!opts.versionOnly, + }); + await bumpCargoVersions(repoRoot, version, { dryRun: !!opts.dryRun }); + }); + +// --------------------------------------------------------------------------- +// publish-npm — parallel npm publish with retries +// --------------------------------------------------------------------------- +program + .command("publish-npm") + .description("Publish all discovered packages to npm") + .option("--tag ", "npm dist-tag (defaults to resolved context)") + .option("--parallel ", "Max simultaneous publishes", "16") + .option("--retries ", "Retries per package", "3") + .option("--release-mode", "Fail if every package is already published") + .option("--dry-run", "Pass --dry-run to npm publish (publishes nothing)") + .action(async (opts) => { + const repoRoot = findRepoRoot(); + let tag: string = opts.tag; + let releaseMode: boolean | undefined = opts.releaseMode; + if (!tag || releaseMode === undefined) { + const ctx = await resolveContext(); + tag = tag ?? ctx.npmTag; + if (opts.releaseMode === undefined) { + releaseMode = ctx.trigger === "release"; + } + } + await publishAll(repoRoot, { + tag, + parallel: Number(opts.parallel), + retries: Number(opts.retries), + releaseMode, + dryRun: !!opts.dryRun, + }); + }); + +// --------------------------------------------------------------------------- +// publish-crates — ordered, idempotent crates.io publish +// --------------------------------------------------------------------------- +program + .command("publish-crates") + .description("Publish Rust crates to crates.io in dependency order") + .option("--version ", "Version to publish (defaults to resolved context)") + .option("--wait-seconds ", "Max wait for crates.io indexing per crate", "600") + .option("--dry-run", "Run cargo publish --dry-run for the first crate only") + .option("--allow-dirty", "Pass --allow-dirty to cargo publish") + .action(async (opts) => { + const repoRoot = findRepoRoot(); + const version = opts.version ?? (await resolveContext()).version; + const waitSeconds = Number(opts.waitSeconds); + if (!Number.isFinite(waitSeconds) || waitSeconds <= 0) { + throw new Error("--wait-seconds must be a positive number"); + } + + if (opts.dryRun) { + const crate = RUST_CRATES[0]; + log.info( + `dry-running ${crate}; later crates require earlier versions to exist in the crates.io index`, + ); + const args = ["publish", "-p", crate, "--dry-run"]; + if (opts.allowDirty) args.push("--allow-dirty"); + await $({ stdio: "inherit", cwd: repoRoot })`cargo ${args}`; + return; + } + + if (!process.env.CARGO_REGISTRY_TOKEN) { + throw new Error("CARGO_REGISTRY_TOKEN must be set to publish crates"); + } + + for (const crate of RUST_CRATES) { + if (await crateVersionExists(crate, version)) { + log.info(`skipping ${crate}@${version}; already published`); + continue; + } + + log.info(`publishing ${crate}@${version}`); + const args = ["publish", "-p", crate]; + if (opts.allowDirty) args.push("--allow-dirty"); + await cargoPublishWithRateLimitRetry(repoRoot, args); + log.info(`waiting for crates.io to index ${crate}@${version}`); + await waitForCrateVersion(crate, version, waitSeconds); + } + }); + +// --------------------------------------------------------------------------- +// upload-r2 — upload the sidecar artifact dir to agent-os/{sha}/sidecar/ +// --------------------------------------------------------------------------- +program + .command("upload-r2") + .description("Upload an artifact directory to R2") + .requiredOption("--source ", "Local directory to upload") + .option("--sha ", "Short sha (defaults to resolved context)") + .option("--name ", "R2 sub-path name", "sidecar") + .action(async (opts) => { + const sha = opts.sha ?? (await resolveContext()).sha; + const prefix = `agent-os/${sha}/${opts.name}/`; + await uploadDir(opts.source, prefix); + }); + +// --------------------------------------------------------------------------- +// copy-r2 — copy agent-os/{sha}/sidecar/ to agent-os/{version}/sidecar/ (+latest) +// --------------------------------------------------------------------------- +program + .command("copy-r2") + .description("Copy R2 artifacts from {sha} to {version} (+latest)") + .option("--sha ", "Source sha (defaults to resolved context)") + .option("--version ", "Target version (defaults to resolved context)") + .option("--latest ", "Also copy to /latest/ (defaults to resolved context)") + .option("--name ", "R2 sub-path name", "sidecar") + .action(async (opts) => { + const ctx = await resolveContext(); + const sha: string = opts.sha ?? ctx.sha; + const version: string = opts.version ?? ctx.version; + const latest = opts.latest !== undefined ? opts.latest === "true" : ctx.latest; + const source = `agent-os/${sha}/${opts.name}/`; + await copyPrefix(source, `agent-os/${version}/${opts.name}/`); + if (latest) { + await copyPrefix(source, `agent-os/latest/${opts.name}/`); + } + }); + +// --------------------------------------------------------------------------- +// git-tag — force-create and push v{version} +// --------------------------------------------------------------------------- +program + .command("git-tag") + .description("Create and force-push v{version} tag") + .option("--version ", "Version (defaults to resolved context)") + .action(async (opts) => { + const version = opts.version ?? (await resolveContext()).version; + await tagAndPush(version); + }); + +// --------------------------------------------------------------------------- +// gh-release — create or update GitHub release for the version +// --------------------------------------------------------------------------- +program + .command("gh-release") + .description("Create or update GitHub release") + .option("--version ", "Version (defaults to resolved context)") + .action(async (opts) => { + const version = opts.version ?? (await resolveContext()).version; + await createGhRelease(version); + }); + +program.parseAsync(process.argv).catch((err) => { + log.error(String(err?.stack ?? err)); + process.exit(1); +}); diff --git a/scripts/publish/src/lib/context.ts b/scripts/publish/src/lib/context.ts new file mode 100644 index 000000000..28792057b --- /dev/null +++ b/scripts/publish/src/lib/context.ts @@ -0,0 +1,247 @@ +import { appendFileSync, existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { $ } from "execa"; + +/** + * Publish context. Resolved once per workflow run by the `context-output` CI + * subcommand and passed through every subsequent step via GitHub Actions job + * outputs + per-step flags. + */ +export type Trigger = "branch" | "release"; + +export interface PublishContext { + trigger: Trigger; + /** Resolved version string, never null. */ + version: string; + /** npm dist-tag. */ + npmTag: string; + /** Short commit sha (7 chars). */ + sha: string; + /** Only meaningful when trigger === "release". */ + latest: boolean; + /** Branch name. Only set when trigger === "branch". */ + branch?: string; + repoRoot: string; +} + +/** Override set accepted by the local release cutter. */ +export interface ResolveOverrides { + trigger?: Trigger; + version?: string; + latest?: boolean; + branch?: string; + sha?: string; +} + +function findRepoRoot(): string { + if (process.env.GITHUB_WORKSPACE && existsSync(process.env.GITHUB_WORKSPACE)) { + return process.env.GITHUB_WORKSPACE; + } + let dir = dirname(fileURLToPath(import.meta.url)); + for (let i = 0; i < 10; i++) { + if (existsSync(join(dir, "pnpm-workspace.yaml"))) return dir; + dir = dirname(dir); + } + throw new Error("Could not locate repo root (no pnpm-workspace.yaml)"); +} + +/** + * Base for all preview pre-release strings. Hardcoded to `0.0.0` so preview + * versions like `0.0.0-my-branch.abc1234` never look like real releases and + * always sort below any published `X.Y.Z`. Using a committed `package.json` + * version as the base would just embed whatever stale number happened to be + * committed there. It has no semantic relationship to the branch being + * previewed. + */ +const PREVIEW_BASE_VERSION = "0.0.0"; + +async function readShortSha(repoRoot: string): Promise { + const envSha = process.env.GITHUB_SHA; + if (envSha) return envSha.slice(0, 7); + const { stdout } = await $({ cwd: repoRoot })`git rev-parse HEAD`; + return stdout.trim().slice(0, 7); +} + +async function readBranchName(repoRoot: string): Promise { + const envRef = process.env.GITHUB_REF_NAME; + if (envRef) return envRef; + const { stdout } = await $({ + cwd: repoRoot, + })`git rev-parse --abbrev-ref HEAD`; + return stdout.trim(); +} + +/** + * Sanitize a branch name into something safe to use as both an npm dist-tag + * and a semver prerelease identifier. Lowercases, replaces any + * non-alphanumeric character (other than hyphens) with a hyphen, collapses + * runs of hyphens, and trims leading/trailing hyphens. + */ +function sanitizeBranch(branch: string): string { + const cleaned = branch + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^-+|-+$/g, ""); + if (cleaned.length === 0) { + throw new Error(`branch name "${branch}" sanitized to empty string`); + } + return cleaned; +} + +function readInputFromEvent(name: string): T | undefined { + const path = process.env.GITHUB_EVENT_PATH; + if (!path || !existsSync(path)) return undefined; + try { + const event = JSON.parse(readFileSync(path, "utf-8")) as { + inputs?: Record; + }; + const v = event.inputs?.[name]; + return v as T | undefined; + } catch { + return undefined; + } +} + +function parseBoolInput(v: unknown, fallback: boolean): boolean { + if (typeof v === "boolean") return v; + if (typeof v === "string") { + if (v === "true") return true; + if (v === "false") return false; + } + return fallback; +} + +function deriveTrigger(overrides: ResolveOverrides | undefined): Trigger { + if (overrides?.trigger) return overrides.trigger; + const eventName = process.env.GITHUB_EVENT_NAME; + if (eventName === "workflow_dispatch") { + const version = readInputFromEvent("version"); + if (typeof version === "string" && version.length > 0) return "release"; + return "branch"; + } + // Default for local invocation without overrides (unusual): assume release + // so missing fields are caught loudly. + return "release"; +} + +function computeNpmTag( + trigger: Trigger, + version: string, + latest: boolean, + branch?: string, +): string { + if (trigger === "branch") { + if (!branch) { + throw new Error("branch trigger requires branch to compute npm tag"); + } + return sanitizeBranch(branch); + } + // release + if (version.includes("-rc.")) return "rc"; + return latest ? "latest" : "next"; +} + +function computeVersion( + trigger: Trigger, + base: string, + sha: string, + branch: string | undefined, + overrideVersion: string | undefined, +): string { + if (overrideVersion) return overrideVersion; + if (trigger === "branch") { + if (!branch) { + throw new Error("branch trigger requires branch to compute version"); + } + return `${base}-${sanitizeBranch(branch)}.${sha}`; + } + throw new Error("release trigger requires an explicit version override"); +} + +/** + * Resolve the publish context. Pure function of environment + overrides. + * Not memoized: each subcommand process re-reads env, and the `context-output` + * subcommand exists specifically so downstream steps receive stable values via + * `$GITHUB_OUTPUT` / flags instead of re-resolving. + */ +export async function resolveContext( + overrides: ResolveOverrides = {}, +): Promise { + const repoRoot = findRepoRoot(); + const trigger = deriveTrigger(overrides); + + const sha = overrides.sha ?? (await readShortSha(repoRoot)); + + let branch = overrides.branch; + if (trigger === "branch" && !branch) { + branch = await readBranchName(repoRoot); + } + + // Release version: override > workflow_dispatch input > error. + let version = overrides.version; + if (!version && trigger === "release") { + const input = readInputFromEvent("version"); + if (typeof input === "string" && input.length > 0) version = input; + } + + if (trigger !== "release") { + version = computeVersion( + trigger, + PREVIEW_BASE_VERSION, + sha, + branch, + version, + ); + } else if (!version) { + throw new Error( + "release trigger requires version (pass --version or workflow_dispatch input)", + ); + } + + // Latest: override > workflow_dispatch input > false. + let latest = overrides.latest; + if (latest === undefined) { + const input = readInputFromEvent("latest"); + latest = parseBoolInput(input, false); + } + if (trigger !== "release") latest = false; + + const npmTag = computeNpmTag(trigger, version, latest, branch); + + return { + trigger, + version, + npmTag, + sha, + latest, + branch, + repoRoot, + }; +} + +/** Write every context field to `$GITHUB_OUTPUT` so downstream steps read via needs.*. */ +export function writeContextToGithubOutput(ctx: PublishContext): void { + const path = process.env.GITHUB_OUTPUT; + if (!path) { + // When invoked locally for debugging, print to stdout in the same format. + console.log(`trigger=${ctx.trigger}`); + console.log(`version=${ctx.version}`); + console.log(`npm_tag=${ctx.npmTag}`); + console.log(`sha=${ctx.sha}`); + console.log(`latest=${ctx.latest}`); + if (ctx.branch !== undefined) console.log(`branch=${ctx.branch}`); + return; + } + const lines = [ + `trigger=${ctx.trigger}`, + `version=${ctx.version}`, + `npm_tag=${ctx.npmTag}`, + `sha=${ctx.sha}`, + `latest=${ctx.latest}`, + ]; + if (ctx.branch !== undefined) lines.push(`branch=${ctx.branch}`); + // Append (do not overwrite) in case other steps also wrote to GITHUB_OUTPUT. + appendFileSync(path, `${lines.join("\n")}\n`); +} diff --git a/scripts/publish/src/lib/git.ts b/scripts/publish/src/lib/git.ts new file mode 100644 index 000000000..b1823c5fc --- /dev/null +++ b/scripts/publish/src/lib/git.ts @@ -0,0 +1,66 @@ +/** + * Git helpers for the release tail: clean-tree validation, tag + push, and + * GitHub release creation. + */ +import { $ } from "execa"; +import { scoped } from "./logger.js"; + +const log = scoped("git"); + +/** Refuse to proceed if there are uncommitted changes. */ +export async function validateClean(): Promise { + const { stdout } = await $`git status --porcelain`; + if (stdout.trim().length > 0) { + throw new Error( + "there are uncommitted changes — commit or stash them before proceeding", + ); + } +} + +/** Force-create and force-push `v{version}`. */ +export async function tagAndPush(version: string): Promise { + log.info(`creating tag v${version}`); + await $({ stdio: "inherit" })`git tag -f v${version}`; + await $({ stdio: "inherit" })`git push origin v${version} -f`; + log.info(`pushed v${version}`); +} + +/** + * Create (or update) a GitHub release for the version. If a release with the + * same name already exists, its tag is updated to match. New releases with a + * prerelease identifier (anything containing `-`) are marked as prerelease. + */ +export async function createGhRelease(version: string): Promise { + log.info(`creating GitHub release for ${version}`); + + const { stdout: currentTag } = await $`git describe --tags --exact-match`; + const tagName = currentTag.trim(); + + const { stdout: releaseJson } = + await $`gh release list --json name,tagName --limit 200`; + const releases = JSON.parse(releaseJson) as Array<{ + name: string; + tagName: string; + }>; + const existing = releases.find((r) => r.name === version); + + if (existing) { + log.info(`updating existing release ${version} -> tag ${tagName}`); + await $({ + stdio: "inherit", + })`gh release edit ${existing.tagName} --tag ${tagName}`; + return; + } + + log.info(`creating new release ${version} -> tag ${tagName}`); + await $({ + stdio: "inherit", + })`gh release create ${tagName} --title ${version} --generate-notes`; + + // Prerelease detection: anything with a `-` is a prerelease per semver. + if (version.includes("-")) { + await $({ + stdio: "inherit", + })`gh release edit ${tagName} --prerelease`; + } +} diff --git a/scripts/publish/src/lib/logger.ts b/scripts/publish/src/lib/logger.ts new file mode 100644 index 000000000..aa5056ba1 --- /dev/null +++ b/scripts/publish/src/lib/logger.ts @@ -0,0 +1,28 @@ +/** + * Prefixed console logger for consistent CI log scanning. + * + * Usage: + * const log = scoped("npm"); + * log.info("publishing ..."); -> [npm] publishing ... + */ + +export interface Logger { + info(msg: string): void; + warn(msg: string): void; + error(msg: string): void; +} + +export function scoped(prefix: string): Logger { + const tag = `[${prefix}]`; + return { + info(msg) { + console.log(`${tag} ${msg}`); + }, + warn(msg) { + console.warn(`${tag} ${msg}`); + }, + error(msg) { + console.error(`${tag} ${msg}`); + }, + }; +} diff --git a/scripts/publish/src/lib/npm.ts b/scripts/publish/src/lib/npm.ts new file mode 100644 index 000000000..4d05a2960 --- /dev/null +++ b/scripts/publish/src/lib/npm.ts @@ -0,0 +1,264 @@ +/** + * Parallel npm publish with bounded concurrency, exponential backoff retries, + * and idempotent "already published" handling. + * + * Works for both preview and release flows — the only per-flow input is the + * dist-tag and the `releaseMode` flag (which toggles strict preflight). + * + * All packages publish with `npm publish` (never `pnpm publish`). `npm publish` + * preserves the `0755` executable bit on the bundled sidecar binary, which + * `pnpm publish` normalizes to `0644`. `workspace:*` dependency specs are + * rewritten to literal versions by `bumpPackageJsons` (full mode) before this + * runs, so plain `npm publish` resolves them correctly. + */ +import { spawn } from "node:child_process"; +import { scoped } from "./logger.js"; +import { + assertDiscoverySanity, + discoverPackages, + type Package, +} from "./packages.js"; + +const log = scoped("npm"); + +export interface PublishAllOptions { + /** npm dist-tag (e.g. my-branch, rc, next, latest). */ + tag: string; + /** Max simultaneous publishes. */ + parallel?: number; + /** Max retries per package. */ + retries?: number; + /** Initial backoff in ms (doubled per retry). */ + initialBackoffMs?: number; + /** + * When true, fail hard if every package is already published. Preview + * mode treats this as an idempotent no-op; release mode treats it as a + * "you forgot to bump the version" error. + */ + releaseMode?: boolean; + /** Pass `--dry-run` to npm publish (local verification, publishes nothing). */ + dryRun?: boolean; +} + +export type PublishStatus = + | "success" + | "retried-success" + | "already-exists" + | "failed"; + +export interface PublishResult { + pkg: Package; + status: PublishStatus; + attempts: number; + lastError?: string; +} + +export interface PublishSummary { + results: PublishResult[]; + counts: { + success: number; + retried: number; + alreadyExists: number; + failed: number; + }; + elapsedSeconds: number; +} + +const ALREADY_PUBLISHED_PATTERNS = [ + "cannot publish over the previously published versions", + "cannot publish over previously published version", + "You cannot publish over", +]; + +function isAlreadyPublished(output: string): boolean { + return ALREADY_PUBLISHED_PATTERNS.some((p) => output.includes(p)); +} + +function isRetryable(output: string): boolean { + if (isAlreadyPublished(output)) return false; + return ( + output.includes("ECONNRESET") || + output.includes("ETIMEDOUT") || + output.includes("ENOTFOUND") || + output.includes("EAI_AGAIN") || + output.includes("socket hang up") || + output.includes("npm error 503") || + output.includes("npm error 502") || + output.includes("npm error 504") || + output.includes("npm error 429") || + output.includes("ERR_STREAM_PREMATURE_CLOSE") || + // Some npm errors don't tag the status clearly; if we don't see a + // definitive "already published" we can retry once. + !/npm error (code|E[A-Z]+)/.test(output) + ); +} + +function extractError(output: string, maxLines = 3): string { + const lines = output + .split("\n") + .filter((l) => /npm error/i.test(l) && !l.includes("A complete log")) + .slice(0, maxLines); + if (lines.length === 0) { + return output.trim().split("\n").slice(-maxLines).join(" | "); + } + return lines.join(" | "); +} + +function runNpmPublish( + pkg: Package, + tag: string, + dryRun: boolean, +): Promise<{ code: number; output: string }> { + return new Promise((resolvePromise) => { + const args = ["publish", "--access", "public", "--tag", tag]; + if (dryRun) args.push("--dry-run"); + const child = spawn("npm", args, { + cwd: pkg.dir, + stdio: ["ignore", "pipe", "pipe"], + env: process.env, + }); + const chunks: Buffer[] = []; + child.stdout.on("data", (c) => chunks.push(c)); + child.stderr.on("data", (c) => chunks.push(c)); + child.on("close", (code) => { + resolvePromise({ + code: code ?? 1, + output: Buffer.concat(chunks).toString("utf8"), + }); + }); + }); +} + +async function publishOne( + pkg: Package, + opts: Required< + Pick + >, +): Promise { + for (let attempt = 1; attempt <= opts.retries + 1; attempt++) { + const { code, output } = await runNpmPublish(pkg, opts.tag, opts.dryRun); + if (code === 0) { + return { + pkg, + status: attempt === 1 ? "success" : "retried-success", + attempts: attempt, + }; + } + if (isAlreadyPublished(output)) { + return { pkg, status: "already-exists", attempts: attempt }; + } + if (!isRetryable(output) || attempt > opts.retries) { + return { + pkg, + status: "failed", + attempts: attempt, + lastError: extractError(output), + }; + } + const delay = opts.initialBackoffMs * 2 ** (attempt - 1); + log.info(` [retry ${attempt}/${opts.retries}] ${pkg.name} — waiting ${delay}ms`); + await new Promise((r) => setTimeout(r, delay)); + } + return { pkg, status: "failed", attempts: opts.retries + 1 }; +} + +function printResult(r: PublishResult): void { + const name = r.pkg.name.padEnd(48); + const symbol = + r.status === "success" || r.status === "retried-success" + ? "✓" + : r.status === "already-exists" + ? "=" + : "✗"; + const suffix = + r.status === "retried-success" + ? ` (after ${r.attempts} attempts)` + : r.status === "failed" + ? ` — ${r.lastError ?? "unknown error"}` + : ""; + log.info(` ${symbol} ${name}${suffix}`); +} + +export async function publishAll( + repoRoot: string, + opts: PublishAllOptions, +): Promise { + const parallel = opts.parallel ?? 16; + const retries = opts.retries ?? 3; + const initialBackoffMs = opts.initialBackoffMs ?? 2000; + const tag = opts.tag; + const dryRun = opts.dryRun ?? false; + + const packages = discoverPackages(repoRoot); + assertDiscoverySanity(packages); + + log.info( + `publishing ${packages.length} packages | tag=${tag} | parallel=${parallel} | retries=${retries}${dryRun ? " | DRY RUN" : ""}`, + ); + + const queue = [...packages]; + const results: PublishResult[] = []; + const startedAt = Date.now(); + + async function worker(): Promise { + while (true) { + const pkg = queue.shift(); + if (!pkg) return; + const result = await publishOne(pkg, { + tag, + retries, + initialBackoffMs, + dryRun, + }); + printResult(result); + results.push(result); + } + } + + const workers: Promise[] = []; + for (let i = 0; i < Math.min(parallel, packages.length); i++) { + workers.push(worker()); + } + await Promise.all(workers); + + const elapsed = (Date.now() - startedAt) / 1000; + const counts = { + success: results.filter((r) => r.status === "success").length, + retried: results.filter((r) => r.status === "retried-success").length, + alreadyExists: results.filter((r) => r.status === "already-exists").length, + failed: results.filter((r) => r.status === "failed").length, + }; + + log.info(""); + log.info(`summary (${elapsed.toFixed(1)}s)`); + log.info(` ${counts.success} succeeded`); + if (counts.retried > 0) log.info(` ${counts.retried} succeeded after retry`); + if (counts.alreadyExists > 0) + log.info(` ${counts.alreadyExists} already published (no-op)`); + if (counts.failed > 0) { + log.error(` ${counts.failed} FAILED`); + for (const r of results.filter((x) => x.status === "failed")) { + log.error(` - ${r.pkg.name}: ${r.lastError}`); + } + throw new Error(`${counts.failed} package(s) failed to publish`); + } + + // In release mode, if *every* package was already published, treat it as + // an error — almost certainly a missed version bump. Reruns of successful + // releases are OK because partial rerun (a few packages re-publish) still + // has at least one success. + if ( + opts.releaseMode && + !dryRun && + counts.success === 0 && + counts.retried === 0 && + counts.failed === 0 && + counts.alreadyExists === packages.length + ) { + throw new Error( + `release mode: all ${packages.length} packages already published at this version. Did you forget to bump the version?`, + ); + } + + return { results, counts, elapsedSeconds: elapsed }; +} diff --git a/scripts/publish/src/lib/packages.ts b/scripts/publish/src/lib/packages.ts new file mode 100644 index 000000000..6d63c2eef --- /dev/null +++ b/scripts/publish/src/lib/packages.ts @@ -0,0 +1,203 @@ +/** + * Single source of truth for the set of packages we publish. + * + * Discovery order matters: platform-specific packages are returned first so + * they land on npm before the meta package that lists them as + * `optionalDependencies`. `@rivet-dev/agent-os-sidecar` (the meta package users + * install) resolves the platform-specific binary package for the current host + * at install time via npm `os`/`cpu`/`libc`, so those platform packages must + * exist on the registry before anyone installs the meta. + */ +import { execSync } from "node:child_process"; +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join, relative, resolve } from "node:path"; + +export interface Package { + name: string; + /** Directory containing the package.json (absolute). */ + dir: string; + /** Directory relative to repo root. */ + relDir: string; +} + +export interface DiscoverPackagesOptions { + /** Reserved for parity with the rivetkit discovery API. */ + includeReleaseOnly?: boolean; +} + +/** + * Packages excluded from discovery (private, built separately, or otherwise + * not publishable). The `private: true` flag in package.json already excludes + * most of these; the explicit list is a belt-and-suspenders guard for names + * that must never be published even if their `private` flag is dropped. + */ +export const EXCLUDED = new Set([ + "@rivet-dev/agent-os-workspace", + "@rivet-dev/agent-os-dev-shell", + "@rivet-dev/agent-os-playground", + "@rivet-dev/agent-os-shell", + "@rivet-dev/agent-os-quickstart", + "@rivet-dev/agent-os-registry-types", + "secure-exec", + "@secure-exec/typescript", + "publish", +]); + +/** + * Meta packages that need `optionalDependencies` injected at publish time. + * The meta package's runtime resolver requires the platform-specific package + * for the current host. The committed `package.json` deliberately does NOT + * include these — they would pollute non-CI installs with version pins that do + * not exist yet — so `bumpPackageJsons` injects them in full (publish-time) + * mode. + */ +export interface MetaPackageSpec { + /** Name of the meta package. */ + meta: string; + /** Prefix of the platform-specific packages to inject. */ + platformPrefix: string; +} + +export const META_PACKAGES: readonly MetaPackageSpec[] = [ + { + meta: "@rivet-dev/agent-os-sidecar", + platformPrefix: "@rivet-dev/agent-os-sidecar-", + }, +]; + +/** + * Platforms whose sidecar binary package is built and published. Kept in sync + * with the build matrix in `.github/workflows/publish.yaml`. Override via the + * `SIDECAR_PLATFORMS` env var (space-separated) to publish a different set. + * arm64 is deferred until its portability fix is verified end-to-end. + */ +export const DEFAULT_SIDECAR_PLATFORMS = ["linux-x64-gnu"] as const; + +export function sidecarPlatforms(): string[] { + const env = process.env.SIDECAR_PLATFORMS?.trim(); + if (env) return env.split(/\s+/).filter(Boolean); + return [...DEFAULT_SIDECAR_PLATFORMS]; +} + +function isPublishable(pkg: { name?: string; private?: boolean }): boolean { + if (!pkg.name) return false; + if (pkg.private) return false; + if (EXCLUDED.has(pkg.name)) return false; + return true; +} + +function readPackageJson( + dir: string, +): { name?: string; private?: boolean } | null { + const pkgPath = join(dir, "package.json"); + if (!existsSync(pkgPath)) return null; + try { + return JSON.parse(readFileSync(pkgPath, "utf8")); + } catch { + return null; + } +} + +export function discoverPackages( + repoRoot: string, + _opts: DiscoverPackagesOptions = {}, +): Package[] { + const packages: Package[] = []; + const seen = new Set(); + + const add = (dir: string) => { + const absDir = resolve(dir); + const pkg = readPackageJson(absDir); + if (!pkg) return; + if (!pkg.name) return; + if (!isPublishable(pkg)) return; + if (seen.has(pkg.name)) return; + seen.add(pkg.name); + packages.push({ + name: pkg.name, + dir: absDir, + relDir: relative(repoRoot, absDir), + }); + }; + + // 1. Platform-specific sidecar binary packages first. These are + // `optionalDependencies` of the meta package and must exist on npm before + // the meta package resolves at install time. Only the allowlisted + // platforms are included so unbuilt platform dirs are never published. + const platformAllowlist = new Set(sidecarPlatforms()); + const npmDir = join(repoRoot, "packages/sidecar-binary/npm"); + if (existsSync(npmDir)) { + for (const entry of readdirSync(npmDir).sort()) { + if (!platformAllowlist.has(entry)) continue; + const platDir = join(npmDir, entry); + if (!statSync(platDir).isDirectory()) continue; + add(platDir); + } + } + + // 2. pnpm workspace packages (@rivet-dev/agent-os-*). Skip the + // registry/software/* WASM command packages — they are built and shipped + // separately, never published to npm from this flow. + const pnpmList = execSync("pnpm -r list --json --depth -1", { + cwd: repoRoot, + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }); + const workspacePkgs: Array<{ + name: string; + path: string; + private?: boolean; + }> = JSON.parse(pnpmList); + for (const p of workspacePkgs) { + if (!p.name) continue; + if (!p.name.startsWith("@rivet-dev/agent-os-")) continue; + if (p.path.includes("/registry/software/")) continue; + add(p.path); + } + + return packages; +} + +/** + * Returns a map of meta package name → list of platform package names that + * should be injected as its `optionalDependencies`. + */ +export function buildMetaPlatformMap( + packages: Package[], +): Map { + return new Map( + META_PACKAGES.map(({ meta, platformPrefix }) => [ + meta, + packages + .filter((p) => p.name.startsWith(platformPrefix)) + .map((p) => p.name) + .sort(), + ]), + ); +} + +/** + * Sanity check — asserts the expected root packages are present. Fail loud in + * CI if discovery silently regressed. Called at the top of subcommands that + * touch the full set. + */ +export function assertDiscoverySanity(packages: Package[]): void { + const byName = new Set(packages.map((p) => p.name)); + const required = ["@rivet-dev/agent-os-core", "@rivet-dev/agent-os-sidecar"]; + const missing = required.filter((r) => !byName.has(r)); + if (missing.length > 0) { + throw new Error( + `package discovery missing required packages: ${missing.join(", ")}`, + ); + } + // Each meta must have at least one platform package. + const metaMap = buildMetaPlatformMap(packages); + for (const { meta } of META_PACKAGES) { + const plats = metaMap.get(meta) ?? []; + if (plats.length === 0) { + throw new Error( + `meta package ${meta} has zero platform packages discovered`, + ); + } + } +} diff --git a/scripts/publish/src/lib/r2.ts b/scripts/publish/src/lib/r2.ts new file mode 100644 index 000000000..70b3d3194 --- /dev/null +++ b/scripts/publish/src/lib/r2.ts @@ -0,0 +1,155 @@ +/** + * R2 client + upload + copy helpers for the releases bucket. + * + * Credentials come from env vars (`R2_RELEASES_ACCESS_KEY_ID` + + * `R2_RELEASES_SECRET_ACCESS_KEY`). R2 hosting of the sidecar binary is a + * convenience mirror for crates.io consumers — the npm platform packages bundle + * the binary directly, so R2 upload is release-only and best-effort. + * + * Implementation note: we shell out to `aws s3 cp` / `aws s3api copy-object` + * because Cloudflare R2 does not support the `x-amz-tagging-directive` header + * the AWS SDK sends even with `--copy-props none`. The `s3api copy-object` path + * avoids it. See: + * https://community.cloudflare.com/t/r2-s3-compat-doesnt-support-net-sdk-for-copy-operations-due-to-tagging-header/616867 + */ +import { $ } from "execa"; +import { scoped } from "./logger.js"; + +const log = scoped("r2"); + +const BUCKET = "rivet-releases"; +const ENDPOINT_URL = + "https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com"; + +type R2Env = Record; + +let cached: R2Env | null = null; + +function getR2Env(): R2Env { + if (cached) return cached; + const ak = process.env.R2_RELEASES_ACCESS_KEY_ID; + const sk = process.env.R2_RELEASES_SECRET_ACCESS_KEY; + if (!ak || !sk) { + throw new Error( + "R2_RELEASES_ACCESS_KEY_ID and R2_RELEASES_SECRET_ACCESS_KEY must be set", + ); + } + cached = { + AWS_ACCESS_KEY_ID: ak, + AWS_SECRET_ACCESS_KEY: sk, + AWS_DEFAULT_REGION: "auto", + }; + return cached; +} + +export interface ListEntry { + Key: string; + Size?: number; +} +export interface ListResult { + Contents: ListEntry[]; +} + +export async function listObjects(prefix: string): Promise { + const env = getR2Env(); + const contents: ListEntry[] = []; + let continuationToken: string | undefined; + + while (true) { + const { stdout } = continuationToken + ? await $({ + env, + })`aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${prefix} --continuation-token ${continuationToken} --endpoint-url ${ENDPOINT_URL}` + : await $({ + env, + })`aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${prefix} --endpoint-url ${ENDPOINT_URL}`; + if (!stdout.trim()) break; + + const page = JSON.parse(stdout) as { + Contents?: ListEntry[]; + IsTruncated?: boolean; + NextContinuationToken?: string; + }; + if (Array.isArray(page.Contents)) { + contents.push(...page.Contents); + } + if (!page.IsTruncated || !page.NextContinuationToken) { + break; + } + continuationToken = page.NextContinuationToken; + } + + return { Contents: contents }; +} + +/** Upload a single file to R2. */ +export async function uploadFile( + localPath: string, + r2Key: string, +): Promise { + const env = getR2Env(); + log.info(`uploading ${localPath} -> ${r2Key}`); + await $({ + env, + stdio: "inherit", + })`aws s3 cp ${localPath} s3://${BUCKET}/${r2Key} --endpoint-url ${ENDPOINT_URL} --checksum-algorithm CRC32`; +} + +/** Recursively upload a directory to an R2 prefix. */ +export async function uploadDir( + localDir: string, + r2Prefix: string, +): Promise { + const env = getR2Env(); + log.info(`uploading directory ${localDir} -> ${r2Prefix}`); + await $({ + env, + stdio: "inherit", + })`aws s3 cp ${localDir} s3://${BUCKET}/${r2Prefix} --recursive --endpoint-url ${ENDPOINT_URL} --checksum-algorithm CRC32`; +} + +/** Delete every object under an R2 prefix. */ +export async function deletePrefix(r2Prefix: string): Promise { + const env = getR2Env(); + log.info(`deleting ${r2Prefix}`); + await $({ + env, + stdio: "inherit", + })`aws s3 rm s3://${BUCKET}/${r2Prefix} --recursive --endpoint-url ${ENDPOINT_URL}`; +} + +/** + * Copy every object under `sourcePrefix` to `targetPrefix`. Uses `s3api + * copy-object` per-object to avoid the R2 tagging-directive bug. + */ +export async function copyPrefix( + sourcePrefix: string, + targetPrefix: string, +): Promise { + const env = getR2Env(); + log.info(`copying ${sourcePrefix} -> ${targetPrefix}`); + + const list = await listObjects(sourcePrefix); + if (list.Contents.length === 0) { + log.warn( + `source prefix ${sourcePrefix} is empty. Skipping copy to ${targetPrefix}.`, + ); + return; + } + + // Delete the target first so stale files from a prior publish are cleaned. + try { + await deletePrefix(targetPrefix); + } catch { + // Target may not exist yet — that's fine. + } + + for (const obj of list.Contents) { + const sourceKey = obj.Key; + const targetKey = sourceKey.replace(sourcePrefix, targetPrefix); + log.info(` ${sourceKey} -> ${targetKey}`); + await $({ + env, + })`aws s3api copy-object --bucket ${BUCKET} --key ${targetKey} --copy-source ${BUCKET}/${sourceKey} --endpoint-url ${ENDPOINT_URL}`; + } +} diff --git a/scripts/publish/src/lib/version.ts b/scripts/publish/src/lib/version.ts new file mode 100644 index 000000000..e76db9133 --- /dev/null +++ b/scripts/publish/src/lib/version.ts @@ -0,0 +1,302 @@ +/** + * Version management split across two surfaces: + * + * - `bumpPackageJsons` — rewrites every discovered publishable package.json + * `version` field and injects `optionalDependencies` on meta packages. + * Safe to call in CI on every run. Uses discovery as the source of truth. + * Does NOT touch Cargo.toml or non-discovered files. + * + * - `bumpCargoVersions` — rewrites the Rust `[workspace.package]` version and + * the lock-step internal crate `version` fields under `[workspace.dependencies]` + * so the crates.io publish chain stays consistent. + * + * - `updateSourceFiles` — rewrites non-package.json, non-Cargo source files + * (example dependency specs). Called only by the local `cut-release.ts`. + * + * - `resolveVersion` / `shouldTagAsLatest` — semver helpers for the local cut. + */ +import * as fs from "node:fs/promises"; +import { join, resolve as resolvePath } from "node:path"; +import { $ } from "execa"; +import { glob } from "glob"; +import * as semver from "semver"; +import { scoped } from "./logger.js"; +import { buildMetaPlatformMap, discoverPackages } from "./packages.js"; + +const log = scoped("version"); + +interface PackageJson { + name?: string; + version?: string; + dependencies?: Record; + devDependencies?: Record; + peerDependencies?: Record; + optionalDependencies?: Record; +} + +const DEP_FIELDS = [ + "dependencies", + "devDependencies", + "peerDependencies", + "optionalDependencies", +] as const; + +export interface BumpOptions { + /** If true, report actions but do not write. */ + dryRun?: boolean; + /** + * When true, only rewrite the `version` field. Does not touch dependency + * references or inject `optionalDependencies`. Safe to commit to git + * because it preserves `workspace:*` dep specs that the lockfile expects. + * + * When false (default), also rewrites `workspace:*` deps to the literal + * version and injects `optionalDependencies` on meta packages. This is + * the publish-time mode used by CI — never committed. + */ + versionOnly?: boolean; +} + +/** + * Rewrite every discovered package's `version` to the given string. + * + * In full mode (default, `versionOnly: false`): also injects + * `optionalDependencies` on meta packages and rewrites `workspace:*` + * dependency references to the literal version. This is the publish-time + * mode used by CI and must NOT be committed — it breaks + * `pnpm install --frozen-lockfile` because the lockfile expects + * `workspace:*`, not literal versions. + * + * In version-only mode (`versionOnly: true`): only rewrites the `version` + * field. Safe to commit. Used by `cut-release.ts` so the repo records the + * new version in package.jsons without breaking the lockfile. + * + * Returns the number of files written. + */ +export async function bumpPackageJsons( + repoRoot: string, + version: string, + opts: BumpOptions = {}, +): Promise { + const packages = discoverPackages(repoRoot); + const packageNames = new Set(packages.map((p) => p.name)); + const metaPlatformMap = buildMetaPlatformMap(packages); + const versionOnly = opts.versionOnly ?? false; + + let updated = 0; + for (const pkg of packages) { + const pkgJsonPath = join(pkg.dir, "package.json"); + const raw = await fs.readFile(pkgJsonPath, "utf8"); + const pkgJson: PackageJson = JSON.parse(raw); + + pkgJson.version = version; + + if (!versionOnly) { + // Inject optionalDependencies on meta packages so end users get the + // correct platform-specific binary via npm's os/cpu/libc resolution. + const platformPkgs = metaPlatformMap.get(pkg.name); + if (platformPkgs && platformPkgs.length > 0) { + pkgJson.optionalDependencies = pkgJson.optionalDependencies ?? {}; + for (const platPkg of platformPkgs) { + pkgJson.optionalDependencies[platPkg] = version; + } + } + + for (const field of DEP_FIELDS) { + const deps = pkgJson[field]; + if (!deps) continue; + for (const [dep, spec] of Object.entries(deps)) { + const isWorkspace = + typeof spec === "string" && spec.startsWith("workspace:"); + if (!isWorkspace) continue; + const isOurPkg = + packageNames.has(dep) || dep.startsWith("@rivet-dev/agent-os-"); + if (!isOurPkg) continue; + deps[dep] = version; + } + } + } + + // Tab-indented, trailing newline — matches the repo convention. + const newContent = `${JSON.stringify(pkgJson, null, "\t")}\n`; + if (opts.dryRun) { + log.info(`[dry-run] would update ${pkg.name} -> ${version}`); + } else { + await fs.writeFile(pkgJsonPath, newContent); + log.info(`updated ${pkg.name} -> ${version}`); + } + updated++; + } + + log.info(`total: ${updated} package.json files updated to ${version}`); + return updated; +} + +/** + * Rewrite the Rust workspace version and the lock-step internal crate versions. + * + * a6 declares internal crate deps as inline tables in `[workspace.dependencies]`, + * e.g. `agent-os-bridge = { path = "crates/bridge", version = "0.2.0-rc.3" }`, + * so a single regex rewrites every `version` field next to a `path`. + */ +export async function bumpCargoVersions( + repoRoot: string, + version: string, + opts: Pick = {}, +): Promise { + const cargoTomlPath = join(repoRoot, "Cargo.toml"); + const cargoToml = await fs.readFile(cargoTomlPath, "utf8"); + let next = cargoToml.replace( + /(\[workspace\.package\]\n(?:[^\n]*\n)*?[ \t]*version = )"[^"]+"/, + `$1"${version}"`, + ); + next = next.replace( + /(agent-os-[a-z0-9-]+ = \{ path = "[^"]+", version = ")[^"]+(" \})/g, + `$1${version}$2`, + ); + + if (next === cargoToml) { + log.info(`Cargo.toml Rust versions already set to ${version}`); + return; + } + + if (opts.dryRun) { + log.info(`[dry-run] would update Cargo.toml Rust versions -> ${version}`); + } else { + await fs.writeFile(cargoTomlPath, next); + log.info(`updated Cargo.toml Rust versions -> ${version}`); + } +} + +/** + * Rewrite non-package.json, non-Cargo source files to the given version. + * Called only by the local release cutter. Examples that pin `@rivet-dev/agent-os-*` + * to a literal version (rather than `workspace:*`) get updated so released + * examples carry the new version. `required: false` because a6 examples use + * `workspace:*` today, so a no-match is expected and not an error. + */ +export async function updateSourceFiles( + repoRoot: string, + version: string, +): Promise { + const findReplace: Array<{ + path: string; + find: RegExp; + replace: string; + required?: boolean; + }> = [ + { + path: "examples/**/package.json", + find: /"(@rivet-dev\/agent-os-[^"]+)": "\^?[0-9]+\.[0-9]+\.[0-9]+(?:-[^"]+)?"/g, + replace: `"$1": "^${version}"`, + required: false, + }, + ]; + + for (const { path: globPath, find, replace, required = true } of findReplace) { + const paths = await glob(globPath, { + cwd: repoRoot, + ignore: ["**/node_modules/**"], + }); + if (paths.length === 0) { + if (required) { + throw new Error(`no paths matched: ${globPath}`); + } + continue; + } + for (const fileRelPath of paths) { + const filePath = resolvePath(repoRoot, fileRelPath); + const file = await fs.readFile(filePath, "utf-8"); + + find.lastIndex = 0; + const hasMatch = find.test(file); + if (!hasMatch) continue; + + find.lastIndex = 0; + const newFile = file.replace(find, replace); + await fs.writeFile(filePath, newFile); + log.info(`updated ${fileRelPath}`); + } + } +} + +// ----------------------------------------------------------------------------- +// Local semver helpers — used only by `cut-release.ts`. +// ----------------------------------------------------------------------------- + +async function getAllGitVersions(): Promise { + try { + await $`git fetch --tags --force --quiet`; + } catch { + throw new Error( + "could not fetch git tags — refusing to compute latest flag from stale local tags", + ); + } + const result = await $`git tag -l v*`; + const tags = result.stdout.trim().split("\n").filter(Boolean); + if (tags.length === 0) return []; + return tags + .map((tag) => tag.replace(/^v/, "")) + .filter((v) => semver.valid(v)) + .sort((a, b) => semver.rcompare(a, b)); +} + +export async function getLatestGitVersion(): Promise { + const versions = await getAllGitVersions(); + const stable = versions.filter((v) => { + const p = semver.parse(v); + return p && p.prerelease.length === 0; + }); + return stable[0] ?? null; +} + +export async function listRecentVersions(limit = 10): Promise { + const all = await getAllGitVersions(); + return all.slice(0, limit); +} + +/** + * Auto-detect whether a version should be tagged as `latest`. A version is + * `latest` only if it has no prerelease identifier AND is greater than any + * existing stable git tag. + */ +export async function shouldTagAsLatest(version: string): Promise { + const parsed = semver.parse(version); + if (!parsed) throw new Error(`invalid semantic version: ${version}`); + if (parsed.prerelease.length > 0) return false; + const latest = await getLatestGitVersion(); + if (!latest) return true; + return semver.gt(version, latest); +} + +export interface ResolveVersionOpts { + version?: string; + major?: boolean; + minor?: boolean; + patch?: boolean; +} + +export async function resolveVersion( + opts: ResolveVersionOpts, +): Promise { + if (opts.version) { + if (!semver.valid(opts.version)) { + throw new Error(`invalid semantic version: ${opts.version}`); + } + return opts.version; + } + if (!opts.major && !opts.minor && !opts.patch) { + throw new Error("must provide --version, --major, --minor, or --patch"); + } + const latest = await getLatestGitVersion(); + if (!latest) { + throw new Error( + "no existing version tags found — use --version to set an explicit version", + ); + } + let next: string | null = null; + if (opts.major) next = semver.inc(latest, "major"); + else if (opts.minor) next = semver.inc(latest, "minor"); + else if (opts.patch) next = semver.inc(latest, "patch"); + if (!next) throw new Error("failed to compute next version"); + return next; +} diff --git a/scripts/publish/src/local/cut-release.ts b/scripts/publish/src/local/cut-release.ts new file mode 100644 index 000000000..532c85007 --- /dev/null +++ b/scripts/publish/src/local/cut-release.ts @@ -0,0 +1,197 @@ +#!/usr/bin/env tsx +/** + * Linear release cutter — called by humans, never by CI. + * + * Steps: + * 1. Resolve target version (flags → semver bump → error) + * 2. Auto-detect or confirm `latest` flag + * 3. Validate git working tree is clean + * 4. Print release plan and confirm + * 5. Rewrite Cargo.toml + example source files + * 6. Rewrite every publishable package.json version (version-only) + * 7. Optional local type-check fail-fast + * 8. Commit + push + * 9. Trigger the publish.yaml workflow + * + * Debugging: comment out any step. No `--only-steps`, no phases. + */ +import { existsSync } from "node:fs"; +import { dirname, join } from "node:path"; +import * as readline from "node:readline"; +import { fileURLToPath } from "node:url"; +import { Command } from "commander"; +import { $ } from "execa"; +import { validateClean } from "../lib/git.js"; +import { scoped } from "../lib/logger.js"; +import { + bumpCargoVersions, + bumpPackageJsons, + getLatestGitVersion, + listRecentVersions, + resolveVersion, + shouldTagAsLatest, + updateSourceFiles, +} from "../lib/version.js"; + +const log = scoped("release"); + +function findRepoRoot(): string { + let dir = dirname(fileURLToPath(import.meta.url)); + for (let i = 0; i < 10; i++) { + if (existsSync(join(dir, "pnpm-workspace.yaml"))) return dir; + dir = dirname(dir); + } + throw new Error("could not locate repo root"); +} + +async function confirmPrompt(question: string): Promise { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + const answer = await new Promise((resolve) => { + rl.question(question, resolve); + }); + rl.close(); + const a = answer.trim().toLowerCase(); + return a === "yes" || a === "y"; +} + +interface CliOpts { + version?: string; + major?: boolean; + minor?: boolean; + patch?: boolean; + latest?: boolean; + noLatest?: boolean; + dryRun?: boolean; + yes?: boolean; + skipChecks?: boolean; +} + +async function main() { + const program = new Command(); + program + .name("cut-release") + .description("Cut a new Agent OS release (local orchestrator)") + .option("--version ", "Explicit version (e.g. 0.2.0 or 0.2.0-rc.1)") + .option("--major", "Bump major") + .option("--minor", "Bump minor") + .option("--patch", "Bump patch") + .option("--latest", "Mark as latest dist-tag") + .option("--no-latest", "Do not mark as latest") + .option("--dry-run", "Do not commit/push/trigger (still mutates source files)") + .option("-y, --yes", "Skip interactive confirmation") + .option("--skip-checks", "Skip local type-check fail-fast") + .parse(); + + const opts = program.opts(); + const repoRoot = findRepoRoot(); + + // 1. Resolve version. + const version = await resolveVersion({ + version: opts.version, + major: opts.major, + minor: opts.minor, + patch: opts.patch, + }); + + // 2. Latest flag: explicit > auto > false. + let latest: boolean; + if (opts.latest === true) latest = true; + else if (opts.noLatest === true || opts.latest === false) latest = false; + else latest = await shouldTagAsLatest(version); + + // 3. Validate git clean. + await validateClean(); + + // 4. Print plan. + const { stdout: branch } = await $`git rev-parse --abbrev-ref HEAD`; + const latestGit = await getLatestGitVersion(); + const recent = await listRecentVersions(10); + console.log(""); + console.log("Release plan"); + console.log(` Version: ${version}`); + console.log(` Latest: ${latest}`); + console.log(` Branch: ${branch.trim()}`); + console.log(` Previous: ${latestGit ?? "(none)"}`); + if (opts.dryRun) console.log(" Dry run: no git commit / push / workflow trigger"); + console.log(""); + if (recent.length > 0) { + console.log("Recent versions:"); + for (const v of recent) { + const marker = v === latestGit ? " (latest)" : ""; + console.log(` - ${v}${marker}`); + } + console.log(""); + } + + if (!opts.yes) { + const ok = await confirmPrompt("Proceed with release? (yes/no): "); + if (!ok) { + log.info("release cancelled"); + process.exit(0); + } + } + + // 5. Update Cargo.toml + example source files. + log.info("updating Cargo.toml + example source files"); + await bumpCargoVersions(repoRoot, version); + await updateSourceFiles(repoRoot, version); + + // 6. Rewrite package.json version fields via discovery. Uses versionOnly + // mode so `workspace:*` dep specs are preserved — the lockfile depends on + // them. CI runs the full publish-time bump (with dep rewriting + + // optionalDependencies injection) after `pnpm install --frozen-lockfile`. + log.info("rewriting package.json versions"); + await bumpPackageJsons(repoRoot, version, { versionOnly: true }); + + // 7. Local type-check fail-fast. + if (!opts.skipChecks) { + log.info("running local core build + type-check (fail-fast)"); + await $({ stdio: "inherit", cwd: repoRoot })`pnpm --dir packages/core build`; + await $({ + stdio: "inherit", + cwd: repoRoot, + })`pnpm --dir packages/core check-types`; + } + + if (opts.dryRun) { + log.info("dry run complete — source files mutated, nothing committed"); + return; + } + + // 8. Commit + push. + log.info("committing version bump"); + await $({ stdio: "inherit", cwd: repoRoot })`git add -A`; + await $({ + stdio: "inherit", + cwd: repoRoot, + shell: true, + })`git commit --allow-empty -m "chore(release): update version to ${version}"`; + + const currentBranch = ( + await $`git rev-parse --abbrev-ref HEAD` + ).stdout.trim(); + await $({ stdio: "inherit", cwd: repoRoot })`git push origin ${currentBranch}`; + + // 9. Trigger the workflow. + log.info("triggering publish.yaml workflow"); + const latestFlag = latest ? "true" : "false"; + await $({ + stdio: "inherit", + cwd: repoRoot, + })`gh workflow run .github/workflows/publish.yaml -f version=${version} -f latest=${latestFlag} --ref ${currentBranch}`; + + const { stdout: repo } = + await $`gh repo view --json nameWithOwner -q .nameWithOwner`; + console.log(""); + console.log( + `Workflow triggered: https://github.com/${repo.trim()}/actions/workflows/publish.yaml`, + ); +} + +main().catch((err) => { + log.error(String(err?.stack ?? err)); + process.exit(1); +}); diff --git a/scripts/publish/tsconfig.json b/scripts/publish/tsconfig.json new file mode 100644 index 000000000..97047a4f4 --- /dev/null +++ b/scripts/publish/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "types": ["node"], + "rootDir": "src" + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/scripts/release.ts b/scripts/release.ts deleted file mode 100644 index 73fe5a68e..000000000 --- a/scripts/release.ts +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env npx tsx - -import { execSync } from "node:child_process"; -import { readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { createInterface } from "node:readline"; - -const ROOT = join(import.meta.dirname, ".."); - -// ── Helpers ── - -function run(cmd: string, opts?: { cwd?: string; stdio?: "pipe" | "inherit" }) { - const result = execSync(cmd, { - cwd: opts?.cwd ?? ROOT, - stdio: opts?.stdio ?? "pipe", - encoding: "utf-8", - }); - return result?.trim() ?? ""; -} - -function tryRun(cmd: string): { ok: boolean; output: string } { - try { - return { ok: true, output: run(cmd) }; - } catch { - return { ok: false, output: "" }; - } -} - -function fatal(msg: string): never { - console.error(`\x1b[31mError:\x1b[0m ${msg}`); - process.exit(1); -} - -async function confirm(msg: string): Promise { - const rl = createInterface({ input: process.stdin, output: process.stdout }); - return new Promise((resolve) => { - rl.question(`${msg} (y/N) `, (answer) => { - rl.close(); - resolve(answer.trim().toLowerCase() === "y"); - }); - }); -} - -function bumpVersion(current: string, type: "patch" | "minor" | "major"): string { - const base = current.replace(/-.*$/, ""); - const [major, minor, patch] = base.split(".").map(Number); - switch (type) { - case "patch": return `${major}.${minor}.${patch + 1}`; - case "minor": return `${major}.${minor + 1}.0`; - case "major": return `${major + 1}.0.0`; - } -} - -// ── Parse args ── - -function npmTag(version: string): "latest" | "rc" { - return version.includes("-") ? "rc" : "latest"; -} - -function parseArgs(): { - version: string; - tag: "latest" | "rc"; - noGitChecks: boolean; - noVcs: boolean; -} { - const args = process.argv.slice(2); - const noGitChecks = args.includes("--no-git-checks"); - // --no-vcs only rewrites versions in the working tree. The caller owns the - // commit/tag/push/dispatch (e.g. when driving the jj workflow by hand). - const noVcs = args.includes("--no-vcs"); - - if (args.includes("--version")) { - const idx = args.indexOf("--version"); - const ver = args[idx + 1]; - if (!ver || ver.startsWith("--")) { - fatal("--version requires an exact version (e.g. --version 0.1.0 or --version 0.2.0-rc.1)"); - } - if (!/^\d+\.\d+\.\d+(-[\w.]+)?$/.test(ver)) { - fatal(`Invalid version format: "${ver}"`); - } - return { version: ver, tag: npmTag(ver), noGitChecks, noVcs }; - } - - const rootPkg = JSON.parse(readFileSync(join(ROOT, "packages/core/package.json"), "utf-8")); - const current = rootPkg.version; - - for (const type of ["patch", "minor", "major"] as const) { - if (args.includes(`--${type}`)) { - return { version: bumpVersion(current, type), tag: "latest", noGitChecks, noVcs }; - } - } - - fatal("Usage: release --patch | --minor | --major | --version [--no-vcs]"); -} - -// ── Update version ── - -function findPublishablePackages(): string[] { - const output = run("pnpm -r ls --json --depth -1"); - const packages = JSON.parse(output) as Array<{ path: string; private?: boolean }>; - return packages - .filter((p) => !p.private && p.path !== ROOT && !p.path.includes("/registry/software/")) - .map((p) => join(p.path, "package.json")); -} - -// Platform binary packages live under the meta package's npm/ subdir and are -// NOT pnpm workspace members, so they are bumped explicitly. -const SIDECAR_PLATFORM_PACKAGES = [ - "packages/sidecar-binary/npm/linux-x64-gnu/package.json", - "packages/sidecar-binary/npm/linux-arm64-gnu/package.json", -] as const; - -function setVersion(version: string) { - const files = findPublishablePackages(); - for (const file of files) { - const content = readFileSync(file, "utf-8"); - const pkg = JSON.parse(content); - pkg.version = version; - const indent = content.match(/^(\s+)"/m)?.[1] ?? "\t"; - writeFileSync(file, JSON.stringify(pkg, null, indent) + "\n"); - console.log(` ${pkg.name} → ${version}`); - } - - for (const rel of SIDECAR_PLATFORM_PACKAGES) { - const file = join(ROOT, rel); - const content = readFileSync(file, "utf-8"); - const pkg = JSON.parse(content); - pkg.version = version; - const indent = content.match(/^(\s+)"/m)?.[1] ?? "\t"; - writeFileSync(file, JSON.stringify(pkg, null, indent) + "\n"); - console.log(` ${pkg.name} → ${version}`); - } - - setCargoVersion(version); -} - -// Bump the Rust workspace version and the lock-step internal crate versions in -// [workspace.dependencies] so the crates.io publish chain stays consistent. -function setCargoVersion(version: string) { - const file = join(ROOT, "Cargo.toml"); - let content = readFileSync(file, "utf-8"); - - content = content.replace( - /(\[workspace\.package\][\s\S]*?\nversion = ")[^"]+(")/, - `$1${version}$2`, - ); - content = content.replace( - /(agent-os-[a-z0-9-]+ = \{ path = "[^"]+", version = ")[^"]+(" \})/g, - `$1${version}$2`, - ); - - writeFileSync(file, content); - console.log(` Cargo workspace → ${version}`); -} - -// ── Main ── - -async function main() { - const { version, tag, noGitChecks, noVcs } = parseArgs(); - const branch = run("git branch --show-current"); - - if (!noGitChecks && !noVcs) { - if (branch !== "main") { - fatal(`Must be on main branch (currently on "${branch}")`); - } - - run("git fetch origin main"); - const local = run("git rev-parse HEAD"); - const remote = run("git rev-parse origin/main"); - if (local !== remote) { - fatal("Local main is not even with origin/main. Pull or push first."); - } - - const status = run("git status --porcelain"); - if (status) { - fatal("Working tree is not clean. Commit or stash changes first."); - } - } else { - console.log("\x1b[33m⚠ Skipping git checks (--no-git-checks)\x1b[0m"); - } - - const pkgFiles = findPublishablePackages(); - const pkgNames = pkgFiles.map((f) => JSON.parse(readFileSync(f, "utf-8")).name as string); - - console.log(`\n\x1b[1mRelease Plan\x1b[0m`); - console.log(` Version: \x1b[36m${version}\x1b[0m`); - console.log(` NPM tag: \x1b[36m${tag}\x1b[0m`); - console.log(` Git tag: \x1b[36mv${version}\x1b[0m`); - console.log(` Packages: \x1b[36m${pkgNames.length}\x1b[0m`); - for (const name of pkgNames) { - console.log(` - ${name}`); - } - console.log(); - - if (!noVcs && !(await confirm("Proceed?"))) { - console.log("Aborted."); - process.exit(0); - } - - // Bump version - console.log(`\n\x1b[1mBumping version to ${version}...\x1b[0m`); - setVersion(version); - - if (noVcs) { - console.log( - `\n\x1b[32m✓ Versions bumped to ${version} (--no-vcs: skipping commit/tag/push/dispatch).\x1b[0m`, - ); - return; - } - - // Commit & push - console.log("\n\x1b[1mCommitting version bump...\x1b[0m"); - run("git add -A"); - const staged = run("git diff --cached --name-only"); - if (staged) { - run(`git commit -m "release: v${version}"`); - run(`git push origin ${branch}`); - } else { - console.log(" No changes to commit, skipping."); - } - - // Git tag - console.log(`\n\x1b[1mCreating git tag v${version}...\x1b[0m`); - const tagExists = tryRun(`git rev-parse v${version}`).ok; - if (tagExists) { - console.log(` Tag v${version} already exists, skipping.`); - } else { - run(`git tag v${version}`); - run(`git push origin v${version}`); - } - - // Trigger CI release workflow - console.log(`\n\x1b[1mTriggering CI release workflow...\x1b[0m`); - run(`gh workflow run release.yml -f version=${version} -f npm-tag=${tag}`, { stdio: "inherit" }); - - console.log(`\n\x1b[32m✓ Tag v${version} pushed. CI will build and publish.\x1b[0m`); - console.log(` Watch progress: \x1b[36mhttps://github.com/rivet-dev/agent-os/actions/workflows/release.yml\x1b[0m`); -} - -main().catch((err) => { - console.error(err); - process.exit(1); -});