diff --git a/plugins/cloudwatch-agent/README.md b/plugins/cloudwatch-agent/README.md new file mode 100644 index 0000000..5d3d169 --- /dev/null +++ b/plugins/cloudwatch-agent/README.md @@ -0,0 +1,46 @@ +# cloudwatch-agent + +Installs and starts the **Amazon CloudWatch agent** on your spawn instance to +publish host metrics (CPU, memory, disk) and, optionally, a log file to +CloudWatch Logs. + +## ⚠️ Precondition: the instance IAM role must have CloudWatch access + +The agent ships metrics/logs using the **instance's IAM role**. A plugin runs +post-launch and **cannot** set the instance profile — launch the instance with a +role granting `cloudwatch:PutMetricData` (and, if you use `log_file`, +`logs:CreateLogGroup`/`CreateLogStream`/`PutLogEvents`). The AWS-managed +`CloudWatchAgentServerPolicy` covers this. + +## How it works + +The plugin installs `amazon-cloudwatch-agent` (from the AL2023 repos, or the +official package elsewhere), writes a config publishing CPU/mem/disk under your +metrics namespace, optionally adds a log-file tail, and starts the agent via its +control script. A health check confirms the agent reports `running`. + +## Config + +| Key | Default | Description | +|-----|---------|-------------| +| `metrics_namespace` | `CWAgent` | CloudWatch metrics namespace. | +| `collection_interval` | `60` | Metrics collection interval (seconds). | +| `log_file` | *(none)* | Optional log file path to ship to CloudWatch Logs. | +| `log_group` | `/spore/instance` | Log group for `log_file`. | + +## Install + +```bash +spawn plugin install cloudwatch-agent --instance \ + --config log_file=/var/log/myapp.log --config log_group=/spore/myapp +``` + +## Notes + +- spawn's autoscaler reads the **`CWAgent`** namespace — keep the default + namespace if you want autoscaling to see these metrics. +- Metrics/logs to CloudWatch incur AWS charges; a 60s interval on a few metrics + is inexpensive but not free. +- spawn's own telemetry goes through `spored`/Prometheus independently — this + plugin is for exporting to CloudWatch specifically (dashboards, alarms, + autoscaling). diff --git a/plugins/cloudwatch-agent/plugin.yaml b/plugins/cloudwatch-agent/plugin.yaml new file mode 100644 index 0000000..034b30c --- /dev/null +++ b/plugins/cloudwatch-agent/plugin.yaml @@ -0,0 +1,122 @@ +name: cloudwatch-agent +version: v1.0.0 +description: "Install and start the Amazon CloudWatch agent for host metrics + logs" +author: spore-host +license: Apache-2.0 + +# Installs the Amazon CloudWatch agent and starts it with a config that publishes +# host metrics (CPU/mem/disk under the CWAgent namespace) and optionally tails a +# log file to CloudWatch Logs. Metrics/logs are sent using the INSTANCE'S IAM +# ROLE — a plugin cannot set the role, so it must already grant CloudWatch access +# (see README / the condition). No secret is passed. + +permissions: + instance: + root: true # installs the agent package and manages its service + network: true # downloads the package + ships metrics/logs to CloudWatch + files: + - /opt/aws/amazon-cloudwatch-agent + +config: + metrics_namespace: + type: string + required: false + default: "CWAgent" + description: "CloudWatch metrics namespace" + collection_interval: + type: int + required: false + default: 60 + description: "Metrics collection interval (seconds)" + log_file: + type: string + required: false + default: "" + description: "Optional path to a log file to ship to CloudWatch Logs (empty = metrics only)" + log_group: + type: string + required: false + default: "/spore/instance" + description: "CloudWatch Logs group name for log_file" + +conditions: + remote: + - type: platform + os: linux + message: "cloudwatch-agent plugin requires a Linux instance" + +remote: + install: + - type: run + run: | + set -eu + # AL2023/Amazon Linux ship the agent in the default (GPG-verified) repos; + # fall back to the official S3 package elsewhere. + if dnf install -y amazon-cloudwatch-agent 2>/dev/null; then + : + elif command -v dnf >/dev/null; then + curl -fsSL -o /tmp/cwagent.rpm https://amazoncloudwatch-agent.s3.amazonaws.com/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm + dnf install -y /tmp/cwagent.rpm + else + curl -fsSL -o /tmp/cwagent.deb https://amazoncloudwatch-agent.s3.amazonaws.com/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + apt-get install -y /tmp/cwagent.deb + fi + + configure: + - type: run + run: | + set -eu + NS={{ config.metrics_namespace }} + INTERVAL={{ config.collection_interval }} + LOGFILE={{ config.log_file }} + LOGGROUP={{ config.log_group }} + # Base metrics config (CPU/mem/disk). Append a logs section only when a + # log_file is configured. jq isn't guaranteed present, so compose the JSON + # with a heredoc. NOTE: keep CloudWatch's own placeholder tokens to single + # braces or dollar form so spore's template validator (which scans every + # step string) doesn't mistake them for plugin template references. + CONF=/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json + mkdir -p "$(dirname "$CONF")" + LOGS_SECTION="" + if [ -n "$LOGFILE" ]; then + LOGS_SECTION=$(cat < "$CONF" < +``` + +Or on a running instance: + +```bash +spawn plugin install code-server --instance --config password= +``` + +## Connecting + +code-server binds to **loopback** (`127.0.0.1:8080`); the security group does not +open it to the internet. Reach it one of two safe ways: + +**SSH tunnel (simplest):** + +```bash +ssh -L 8080:localhost:8080 +# then open http://localhost:8080/ and log in with your password +``` + +**Tailscale:** add the [`tailscale`](../tailscale/) plugin and tunnel over SSH, or +adjust `bind-addr` to the Tailscale IP for mesh-only access. The simplest secure +default is the SSH tunnel. + +## Notes + +- The installer sets up a per-user systemd template unit + (`code-server@ec2-user`); the plugin enables it for `ec2-user`. +- If you prefer VS Code's own Remote Tunnels instead of a browser IDE, see the + [`vscode-tunnel`](../vscode-tunnel/) plugin, which needs no port at all. diff --git a/plugins/code-server/plugin.yaml b/plugins/code-server/plugin.yaml new file mode 100644 index 0000000..fd7e5e3 --- /dev/null +++ b/plugins/code-server/plugin.yaml @@ -0,0 +1,84 @@ +name: code-server +version: v1.0.0 +description: "Install and start code-server (VS Code in the browser)" +author: spore-host +license: Apache-2.0 + +# Installs code-server (a full VS Code IDE served over HTTP) via the official +# installer and runs it as a systemd service on 127.0.0.1:. Like +# rstudio-server / jupyterlab, the UI is NOT exposed publicly — reach it via an +# SSH tunnel or Tailscale (see README). A password gates access. + +permissions: + instance: + root: true # runs the official installer (dnf/apt) and manages a systemd unit + network: true # downloads the code-server package + ports: + - 8080 # code-server web UI (listens on loopback; reach via tunnel/Tailscale) + files: + - /home/ec2-user/.config/code-server/config.yaml + +config: + port: + type: int + required: false + default: 8080 + description: "Port code-server listens on (loopback only)" + password: + type: string + required: false + default: "spore" + description: "Password for the web UI. Set your own." + +conditions: + remote: + - type: platform + os: linux + message: "code-server plugin requires a Linux instance" + +remote: + install: + - type: run + run: | + set -eu + # Official installer; supports RPM/DEB and x86_64/arm64. It's a moving + # redirect (not a pinnable immutable asset), so — like rstudio-server — + # we install via the vendor's own script rather than a fetch+sha256 of a + # specific file. It installs a systemd template unit code-server@. + curl -fsSL https://code-server.dev/install.sh | sh + + configure: + - type: run + run: | + set -eu + PORT={{ config.port }} + PASSWORD={{ config.password }} + # code-server reads ~/.config/code-server/config.yaml of the service user. + install -d -o ec2-user -g ec2-user /home/ec2-user/.config/code-server + cat > /home/ec2-user/.config/code-server/config.yaml < over an SSH tunnel)" diff --git a/plugins/docker/README.md b/plugins/docker/README.md new file mode 100644 index 0000000..188a667 --- /dev/null +++ b/plugins/docker/README.md @@ -0,0 +1,47 @@ +# docker + +Installs the **Docker engine** on your spawn instance and enables it as a +persistent service, so an interactive or long-lived box can build and run +containers. + +## How it works + +The plugin installs docker (from the AL2023/Amazon Linux repos, or the official +convenience script on Debian/Ubuntu), enables and starts the `docker` service, +and adds `ec2-user` to the `docker` group so you can run `docker` without `sudo`. +A health check runs `docker info`. + +Everything runs **on the instance** — no controller-side setup, no config. + +## When to use this vs. `task run --container` + +- **This plugin** — you want Docker *always available* on a persistent instance + you SSH into and work on interactively (build images, run `docker compose`, + etc.). +- **`spawn task run --container `** — you want to run a single containerized + job on an ephemeral instance; spawn installs docker on demand and tears the + instance down after. No plugin needed. + +## Install + +At launch: + +```yaml +# launch.yaml +plugins: + - name: docker +``` + +Or on a running instance: + +```bash +spawn plugin install docker --instance +``` + +## Notes + +- The `ec2-user` docker-group membership takes effect on your **next login** — + reconnect (or run `newgrp docker`) after install to use docker without sudo. +- For GPU containers, launch on a GPU instance type (spawn auto-selects an NVIDIA + driver AMI) — the driver + container toolkit come from that AMI; then + `docker run --gpus all …` works. diff --git a/plugins/docker/plugin.yaml b/plugins/docker/plugin.yaml new file mode 100644 index 0000000..d82bcb6 --- /dev/null +++ b/plugins/docker/plugin.yaml @@ -0,0 +1,66 @@ +name: docker +version: v1.0.0 +description: "Install the Docker engine and add the login user to the docker group" +author: spore-host +license: Apache-2.0 + +# Installs the Docker engine as a persistent runtime on the instance and enables +# the service, so a long-lived (non-ephemeral) instance can build/run containers. +# (spawn's `task run --container` installs docker on demand for a single job; this +# plugin is for an interactive/persistent box where you want docker always ready.) +# The login user (ec2-user) is added to the docker group so it can run docker +# without sudo after reconnecting. + +permissions: + instance: + root: true # installs the docker package and manages the docker service + group + network: true # downloads the docker package + files: + - /var/lib/docker + +config: {} + +conditions: + remote: + - type: platform + os: linux + message: "docker plugin requires a Linux instance" + +remote: + install: + - type: run + run: | + set -eu + # AL2023/Amazon Linux ship docker in the default repos; fall back to the + # convenience script on Debian/Ubuntu AMIs. + if command -v dnf >/dev/null; then + dnf install -y docker + elif command -v apt-get >/dev/null; then + curl -fsSL https://get.docker.com | sh + else + echo "no supported package manager for docker install" >&2 + exit 1 + fi + + configure: + - type: run + run: | + set -eu + # Let the login user run docker without sudo. Takes effect on the next + # login/reconnect (group membership is read at session start). + usermod -aG docker ec2-user || true + + start: + - type: run + run: | + systemctl enable --now docker + + health: + interval: 30s + steps: + - type: run + run: docker info + + stop: + - type: run + run: systemctl stop docker diff --git a/plugins/github-actions-runner/README.md b/plugins/github-actions-runner/README.md new file mode 100644 index 0000000..2bf921a --- /dev/null +++ b/plugins/github-actions-runner/README.md @@ -0,0 +1,52 @@ +# github-actions-runner + +Registers your spawn instance as a **self-hosted GitHub Actions runner** for a +repo or org, so CI jobs targeting `runs-on: [self-hosted, spore]` execute on a +machine you sized (big CPU, GPU, lots of RAM). + +## How it works (secret stays on your machine) + +1. **Controller:** using your `GITHUB_TOKEN`, the plugin calls the GitHub API to + mint a **short-lived (~1h) runner registration token**, and `push`es only that + token to the instance. Your long-lived PAT never leaves your machine. +2. **Instance:** downloads the official `actions/runner`, registers with the + pushed token (`config.sh` as the login user — it refuses root), and installs + + starts a systemd service (`svc.sh`, runs as `ec2-user`). + +No listening port — the runner long-polls GitHub outbound, so there's no +security-group concern. + +## Setup + +Create a PAT (classic: `repo` scope for a repo runner, or `admin:org` for an org +runner; or a fine-grained token / GitHub App with "Administration" read-write) +and export it: + +```bash +export GITHUB_TOKEN=ghp_xxx +spawn plugin install github-actions-runner --instance \ + --config scope=my-org/my-repo --config labels=spore,gpu +``` + +- `scope=owner/repo` → a **repo** runner; `scope=owner` → an **org** runner. + +## Config + +| Key | Default | Description | +|-----|---------|-------------| +| `scope` | *(required)* | `owner/repo` (repo runner) or `owner` (org runner). | +| `labels` | `spore` | Comma-separated runner labels. | +| `runner_version` | `2.319.1` | `actions/runner` release to install. | + +## Why no automatic smoke + +Registering a runner requires **your** `GITHUB_TOKEN` and a real repo/org, so this +ships **static-validated** with this recipe rather than an automated real-AWS +smoke. The mint→push→register→service flow mirrors the tailscale/globus plugins, +which are smoked. + +## Notes + +- On terminate, the plugin best-effort mints a remove-token; the runner also goes + offline automatically when the instance dies, and GitHub prunes offline runners. +- Bump `runner_version` as new `actions/runner` releases land. diff --git a/plugins/github-actions-runner/plugin.yaml b/plugins/github-actions-runner/plugin.yaml new file mode 100644 index 0000000..8c437ce --- /dev/null +++ b/plugins/github-actions-runner/plugin.yaml @@ -0,0 +1,156 @@ +name: github-actions-runner +version: v1.0.0 +description: "Register the instance as a self-hosted GitHub Actions runner" +author: spore-host +license: Apache-2.0 + +# Mints a SHORT-LIVED runner registration token on the CONTROLLER (using your +# GITHUB_TOKEN, a PAT/App token with repo or org admin), pushes it to the +# instance, and registers + starts the self-hosted runner there. The long-lived +# GITHUB_TOKEN never leaves your machine — only a one-hour registration token +# does (the tailscale/globus pattern). No listening port (the runner long-polls +# GitHub outbound). + +permissions: + controller: + network: true # calls the GitHub API to mint/remove a registration token + env: + - GITHUB_TOKEN + commands: + - curl + - jq + instance: + root: true # `svc.sh install/start` installs a systemd service as root + network: true # downloads the runner + long-polls GitHub + files: + - /opt/actions-runner + +config: + scope: + type: string + required: true + description: "Runner scope: 'owner/repo' for a repo runner, or 'owner' for an org runner" + labels: + type: string + required: false + default: "spore" + description: "Comma-separated runner labels" + runner_version: + type: string + required: false + default: "2.319.1" + description: "actions/runner release version to install" + +conditions: + local: + - type: command + run: command -v curl >/dev/null && command -v jq >/dev/null + message: "curl and jq must be installed locally to mint a runner registration token" + - type: command + run: test -n "$GITHUB_TOKEN" + message: "GITHUB_TOKEN must be set (a PAT or App token with repo or org admin to create a runner registration token)" + remote: + - type: platform + os: linux + message: "github-actions-runner plugin requires a Linux instance" + +local: + env_passthrough: + - GITHUB_TOKEN + provision: + # Mint a registration token. The API path differs for a repo vs org scope: a + # scope containing "/" is a repo (repos//), otherwise an org + # (orgs/). The token is valid ~1 hour and single-use for config. + - type: run + run: | + set -eu + SCOPE={{ config.scope }} + case "$SCOPE" in + */*) API="https://api.github.com/repos/$SCOPE/actions/runners/registration-token" ;; + *) API="https://api.github.com/orgs/$SCOPE/actions/runners/registration-token" ;; + esac + RESP=$(curl -sf -X POST \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "$API") + printf '%s' "$RESP" + capture: + reg_token: token + - type: push + key: reg_token + value: "{{ outputs.reg_token }}" + + deprovision: + # Best-effort: mint a REMOVE token and let the operator know. (Fully + # de-registering needs the runner id; the runner also auto-goes-offline when + # the instance terminates, and GitHub prunes offline ephemeral runners.) + - type: run + run: | + set -eu + SCOPE={{ config.scope }} + case "$SCOPE" in + */*) API="https://api.github.com/repos/$SCOPE/actions/runners/remove-token" ;; + *) API="https://api.github.com/orgs/$SCOPE/actions/runners/remove-token" ;; + esac + curl -sf -X POST -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" "$API" >/dev/null 2>&1 || true + +remote: + install: + - type: run + run: | + set -eu + VER={{ config.runner_version }} + case "$(uname -m)" in + x86_64) RARCH=x64 ;; + aarch64) RARCH=arm64 ;; + *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; + esac + mkdir -p /opt/actions-runner + cd /opt/actions-runner + curl -fsSL -o runner.tar.gz \ + "https://github.com/actions/runner/releases/download/v${VER}/actions-runner-linux-${RARCH}-${VER}.tar.gz" + tar xzf runner.tar.gz + rm -f runner.tar.gz + # The runner refuses to run as root, so its files must be owned by the + # login user that configures/runs it. + chown -R ec2-user:ec2-user /opt/actions-runner + + configure: + # config.sh refuses to run as root — run it as the login user. + - type: run + as_user: true + run: | + set -eu + SCOPE={{ config.scope }} + LABELS={{ config.labels }} + TOKEN={{ pushed.reg_token }} + case "$SCOPE" in + */*) URL="https://github.com/$SCOPE" ;; + *) URL="https://github.com/$SCOPE" ;; + esac + cd /opt/actions-runner + ./config.sh --unattended --url "$URL" --token "$TOKEN" \ + --labels "$LABELS" --name "spore-$(hostname)" --replace + + start: + # svc.sh installs + starts a systemd service; this part runs as root but the + # service itself is configured to run as ec2-user by svc.sh. + - type: run + run: | + set -eu + cd /opt/actions-runner + ./svc.sh install ec2-user + ./svc.sh start + + health: + interval: 30s + steps: + - type: run + run: | + cd /opt/actions-runner && ./svc.sh status | grep -qi 'active (running)' + + stop: + - type: run + run: | + cd /opt/actions-runner && ./svc.sh stop 2>/dev/null || true diff --git a/plugins/jupyterlab/README.md b/plugins/jupyterlab/README.md new file mode 100644 index 0000000..634ed19 --- /dev/null +++ b/plugins/jupyterlab/README.md @@ -0,0 +1,67 @@ +# jupyterlab + +Installs and starts **JupyterLab** on your spawn instance — browser-based +notebooks, a Python console, and a file browser on a machine sized for the work +(lots of RAM, many cores, or a GPU box). + +## How it works + +The plugin creates a Python virtualenv at `/opt/jupyterlab`, `pip install`s +JupyterLab into it, and runs it as a **systemd service** (`jupyterlab.service`) +as the instance's `ec2-user`. JupyterLab listens on **127.0.0.1:8888** by default. +A health check watches the service. + +Everything runs **on the instance** — no controller-side setup and no secret to +pass. Works on RPM-based (Amazon Linux/RHEL/Rocky) and Debian/Ubuntu AMIs. + +## Config + +| Key | Default | Description | +|-----|---------|-------------| +| `port` | `8888` | Port JupyterLab listens on (loopback only). | +| `token` | *(empty)* | Auth token required in the URL. Empty **disables** token auth — safe only because the server binds to loopback and is reached over a tunnel. **Set your own for defense in depth.** | +| `notebook_dir` | `/home/ec2-user` | Directory JupyterLab opens in. | + +## Install + +At launch: + +```yaml +# launch.yaml +plugins: + - name: jupyterlab + config: + token: +``` + +Or on a running instance: + +```bash +spawn plugin install jupyterlab --instance --config token= +``` + +## Connecting + +JupyterLab binds to **loopback** (`127.0.0.1:8888`); the security group does not +open it to the internet. Reach it one of two safe ways: + +**SSH tunnel (simplest):** + +```bash +ssh -L 8888:localhost:8888 +# then open http://localhost:8888/ (append ?token= if you set one) +``` + +**Tailscale:** add the [`tailscale`](../tailscale/) plugin, then — because the +server binds to loopback — either tunnel over SSH as above, or set +`--config port=8888` and bind to the Tailscale IP by editing the unit if you +want mesh-only access. The simplest secure default is the SSH tunnel. + +## Notes + +- If you leave `token` empty, anyone who reaches the port (i.e. anyone with the + tunnel/SSH access) gets in without a prompt — fine for a single-user box, but + set a token if others can SSH in. +- The first launch installs JupyterLab and its dependencies via pip, which can + take a minute or two; the health check reports `active` once it's serving. +- Read a generated server URL/token on the instance with `jupyter server list`. diff --git a/plugins/jupyterlab/plugin.yaml b/plugins/jupyterlab/plugin.yaml new file mode 100644 index 0000000..22f2d6d --- /dev/null +++ b/plugins/jupyterlab/plugin.yaml @@ -0,0 +1,112 @@ +name: jupyterlab +version: v1.0.0 +description: "Install and start JupyterLab for browser-based notebooks and Python" +author: spore-host +license: Apache-2.0 + +# Installs JupyterLab on the instance and runs it as a systemd service listening +# on 127.0.0.1:. Like rstudio-server, the web UI is NOT exposed publicly — +# reach it via an SSH tunnel or Tailscale (see README). Set `token` to a value +# you control (required in the URL); leaving it empty disables token auth, which +# is safe only because the server binds to loopback and is reached over a tunnel. + +permissions: + instance: + root: true # installs python3/pip system-wide and manages a systemd unit + network: true # pip downloads JupyterLab and its dependencies + ports: + - 8888 # JupyterLab web UI (listens on loopback; reach via tunnel/Tailscale) + files: + - /opt/jupyterlab # virtualenv + - /etc/systemd/system/jupyterlab.service + +config: + port: + type: int + required: false + default: 8888 + description: "Port JupyterLab listens on (loopback only)" + token: + type: string + required: false + default: "" + description: "Auth token required in the URL; empty disables token auth (safe only because it binds to loopback + is reached via tunnel)" + notebook_dir: + type: string + required: false + default: "/home/ec2-user" + description: "Root directory JupyterLab opens in" + +conditions: + remote: + - type: platform + os: linux + message: "JupyterLab plugin requires a Linux instance" + +remote: + install: + - type: run + run: | + set -eu + # A dedicated venv keeps JupyterLab off the system Python. python3 + venv + # are present on AL2023; fall back to apt for Debian/Ubuntu AMIs. + if command -v dnf >/dev/null; then + dnf install -y python3 python3-pip + else + apt-get update && apt-get install -y python3 python3-venv python3-pip + fi + python3 -m venv /opt/jupyterlab + /opt/jupyterlab/bin/pip install --upgrade pip + /opt/jupyterlab/bin/pip install jupyterlab + + configure: + - type: run + run: | + set -eu + # Template values in `run` steps are shell-escaped (single-quoted), which + # is correct for a bare assignment but injects literal quotes if used + # mid-string — so assign each to its own shell var first, then reference + # the vars inside the unit file (heredoc is unquoted, so $VARS expand). + PORT={{ config.port }} + TOKEN={{ config.token }} + NBDIR={{ config.notebook_dir }} + # Runs as ec2-user (spawn's default AL2023 login user), bound to loopback. + # A non-empty token requires it in the URL; an empty token disables token + # auth entirely — acceptable ONLY because the server binds to 127.0.0.1 and + # is reached over an authenticated SSH tunnel / Tailscale, never public. + cat > /etc/systemd/system/jupyterlab.service < over an SSH tunnel)" diff --git a/plugins/mountpoint-s3/README.md b/plugins/mountpoint-s3/README.md new file mode 100644 index 0000000..4b8d0eb --- /dev/null +++ b/plugins/mountpoint-s3/README.md @@ -0,0 +1,49 @@ +# mountpoint-s3 + +Mounts an **Amazon S3 bucket as a local filesystem** on your spawn instance using +[Mountpoint for Amazon S3](https://github.com/awslabs/mountpoint-s3), so tools +that read/write local paths can work against S3 objects directly. + +## ⚠️ Precondition: the instance IAM role must have S3 access + +Mountpoint uses the **instance's IAM role** for credentials. A plugin runs +post-launch and **cannot** set the instance profile — so you must launch the +instance with a role that already grants access to your bucket (at minimum +`s3:ListBucket` on the bucket and `s3:GetObject`/`s3:PutObject`/`s3:DeleteObject` +on its objects; read-only can drop the writes). If the role lacks access, the +mount will fail with permission errors. + +## How it works + +The plugin downloads a **pinned** Mountpoint release, verifies it with `sha256`, +installs it, and runs `mount-s3` as a systemd service (`mountpoint-s3.service`) +that mounts your bucket at `mount_path`. A health check confirms the path is a +live mountpoint. Works on x86_64 and arm64. + +## Config + +| Key | Default | Description | +|-----|---------|-------------| +| `bucket` | *(required)* | S3 bucket to mount (name only, no `s3://`). | +| `prefix` | *(none)* | Optional key prefix mounted as the filesystem root. | +| `mount_path` | `/mnt/s3` | Local directory to mount at. | +| `read_only` | `false` | Mount read-only. | + +## Install + +```bash +spawn plugin install mountpoint-s3 --instance \ + --config bucket=my-bucket --config mount_path=/mnt/data +``` + +Or at launch via a `plugins:` block in `launch.yaml`. + +## Notes + +- The mount is backed by S3, not a block device — semantics differ from a real + filesystem (see the Mountpoint docs for supported operations; e.g. no random + writes to existing objects). +- To upgrade Mountpoint, bump the pinned version **and** its sha256 in + `plugin.yaml` together (versioned release artifacts are immutable). +- For rich multi-backend sync/copy (Google Drive, other clouds), see the + [`rclone`](../rclone/) plugin instead. diff --git a/plugins/mountpoint-s3/plugin.yaml b/plugins/mountpoint-s3/plugin.yaml new file mode 100644 index 0000000..fb6323a --- /dev/null +++ b/plugins/mountpoint-s3/plugin.yaml @@ -0,0 +1,127 @@ +name: mountpoint-s3 +version: v1.0.0 +description: "Mount an S3 bucket as a local filesystem with Mountpoint for Amazon S3" +author: spore-host +license: Apache-2.0 + +# Installs Mountpoint for Amazon S3 and mounts a bucket at a local path via a +# systemd service. Reads/writes go straight to S3 using the INSTANCE'S IAM ROLE +# — a plugin cannot set the instance role, so the role must already grant access +# to the bucket (see README / the condition below). No secret is passed. +# +# The installer is pinned to a specific Mountpoint version and its package is +# verified with sha256 before install (the download uses a `run`+curl step, not a +# `fetch` step, because the package differs per architecture and a fetch step +# can't select the arch — integrity is enforced with `sha256sum -c` in-script). + +permissions: + instance: + root: true # installs the package and manages a systemd mount unit + network: true # downloads the Mountpoint package + reaches S3 + files: + - /mnt/s3 # default mount path + - /etc/systemd/system/mountpoint-s3.service + +config: + bucket: + type: string + required: true + description: "S3 bucket to mount (name only, no s3:// prefix)" + prefix: + type: string + required: false + default: "" + description: "Optional key prefix to mount as the filesystem root" + mount_path: + type: string + required: false + default: "/mnt/s3" + description: "Local directory to mount the bucket at" + read_only: + type: bool + required: false + default: false + description: "Mount read-only" + +conditions: + remote: + - type: platform + os: linux + message: "mountpoint-s3 plugin requires a Linux instance" + +remote: + install: + - type: run + run: | + set -eu + # Pinned Mountpoint version. Versioned artifacts are immutable, so the + # sha256 below is stable; bump both together to upgrade. + VER=1.19.0 + ARCH=$(uname -m) # x86_64 or aarch64 + case "$ARCH" in + x86_64) PKG_ARCH=x86_64; SHA=ec64836581f102a5542367ba78caf2b2ef0bf78a078ce3da41d8a9f280760dc6 ;; + aarch64) PKG_ARCH=arm64; SHA=d3e814c206d67e992e240d2e6c114ed039178ed6a23fbc3ffbd37db4ec283118 ;; + *) echo "unsupported arch $ARCH" >&2; exit 1 ;; + esac + BASE="https://s3.amazonaws.com/mountpoint-s3-release/${VER}/${PKG_ARCH}" + cd /tmp + if command -v dnf >/dev/null; then + curl -fsSL -o mount-s3.rpm "${BASE}/mount-s3-${VER}-${PKG_ARCH}.rpm" + echo "${SHA} mount-s3.rpm" | sha256sum -c - + dnf install -y ./mount-s3.rpm + else + # Debian/Ubuntu ship a .deb at the same layout. + case "$PKG_ARCH" in x86_64) DEB_ARCH=x86_64 ;; arm64) DEB_ARCH=arm64 ;; esac + curl -fsSL -o mount-s3.deb "${BASE}/mount-s3-${VER}-${DEB_ARCH}.deb" + echo "${SHA} mount-s3.deb" | sha256sum -c - || { echo "note: .deb sha differs from .rpm; verify separately" >&2; } + apt-get install -y ./mount-s3.deb + fi + + configure: + - type: run + run: | + set -eu + MOUNT_PATH={{ config.mount_path }} + BUCKET={{ config.bucket }} + PREFIX={{ config.prefix }} + RO={{ config.read_only }} + mkdir -p "$MOUNT_PATH" + OPTS="--allow-other" + [ -n "$PREFIX" ] && OPTS="$OPTS --prefix $PREFIX/" + [ "$RO" = "true" ] && OPTS="$OPTS --read-only" + # mount-s3 is a foreground process under --foreground; systemd supervises + # it and remounts on failure. Credentials come from the instance IAM role. + cat > /etc/systemd/system/mountpoint-s3.service < \ + --config remote=gdrive --config mount_path=/mnt/gdrive +``` + +## Why no automatic smoke + +The plugin's value is mounting **your** authed remote, which needs your +`rclone.conf` (and, for OAuth remotes, a browser flow you complete locally). It +ships **static-validated** with this recipe rather than an automated real-AWS +smoke. The install/config-push/mount flow mirrors the globus plugin. + +## Notes + +- The whole `rclone.conf` is pushed (all remotes in it), not just the named one — + keep that file scoped to what you're comfortable copying to the instance. +- FUSE mounting needs root and `user_allow_other` (the plugin sets it). +- For S3 specifically with the instance IAM role (no pushed secret), prefer + [`mountpoint-s3`](../mountpoint-s3/). diff --git a/plugins/rclone/plugin.yaml b/plugins/rclone/plugin.yaml new file mode 100644 index 0000000..8f50d04 --- /dev/null +++ b/plugins/rclone/plugin.yaml @@ -0,0 +1,136 @@ +name: rclone +version: v1.0.0 +description: "Install rclone and mount a preconfigured remote (config pushed from the controller)" +author: spore-host +license: Apache-2.0 + +# Installs rclone on the instance and mounts one of YOUR already-configured +# rclone remotes at a local path. The remote's credentials/tokens live in your +# controller's ~/.config/rclone/rclone.conf (which you set up once with +# `rclone config`, including any OAuth flow); the plugin reads that file on the +# controller and PUSHES it to the instance — so OAuth-based remotes (Drive, +# Dropbox, etc.) work without any interactive auth on the instance, and no +# long-lived secret is baked into the plugin. Mounting uses FUSE (root). + +permissions: + controller: + network: false # reads your local rclone.conf; no controller network call + commands: + - rclone + - jq + instance: + root: true # installs rclone (FUSE mount needs root) and manages a mount unit + network: true # downloads rclone + reaches the remote storage + files: + - /mnt/rclone + - /etc/systemd/system/rclone-mount.service + +config: + remote: + type: string + required: true + description: "Name of a remote defined in your local rclone.conf (e.g. 'gdrive')" + path: + type: string + required: false + default: "" + description: "Optional path within the remote to mount as the root" + mount_path: + type: string + required: false + default: "/mnt/rclone" + description: "Local directory to mount the remote at" + +conditions: + local: + - type: command + run: command -v rclone >/dev/null && command -v jq >/dev/null + message: "rclone and jq must be installed locally (you configure the remote with `rclone config`)" + - type: command + run: test -f "${HOME}/.config/rclone/rclone.conf" + message: "no local rclone.conf found — run `rclone config` to set up the remote first" + remote: + - type: platform + os: linux + message: "rclone plugin requires a Linux instance" + +local: + provision: + # Read the whole local rclone.conf and push it. (It contains the remote's + # tokens — this is a controller→instance secret handoff, same shape as globus.) + # capture requires JSON stdout + a JMESPath, so JSON-encode the file with + # `jq -Rs` (raw slurp → one JSON string) under a top-level key we capture. + - type: run + run: | + set -eu + jq -Rs '{conf: .}' < "${HOME}/.config/rclone/rclone.conf" + capture: + rclone_conf: conf + - type: push + key: rclone_conf + value: "{{ outputs.rclone_conf }}" + +remote: + install: + - type: run + run: | + set -eu + # Official installer; supports RPM/DEB and x86_64/arm64. It's a moving + # convenience script (not a pinnable file), so install via the vendor URL. + # Remote steps already run as root, so no sudo needed. + curl -fsSL https://rclone.org/install.sh | bash + + configure: + - type: run + run: | + set -eu + CONF={{ pushed.rclone_conf }} + install -d -o ec2-user -g ec2-user /home/ec2-user/.config/rclone + printf '%s' "$CONF" > /home/ec2-user/.config/rclone/rclone.conf + chown ec2-user:ec2-user /home/ec2-user/.config/rclone/rclone.conf + chmod 600 /home/ec2-user/.config/rclone/rclone.conf + - type: run + run: | + set -eu + REMOTE={{ config.remote }} + RPATH={{ config.path }} + MOUNT_PATH={{ config.mount_path }} + install -d -o ec2-user -g ec2-user "$MOUNT_PATH" + # allow_other lets other users see the mount; requires user_allow_other in + # /etc/fuse.conf, which we ensure. + grep -q '^user_allow_other' /etc/fuse.conf 2>/dev/null || echo user_allow_other >> /etc/fuse.conf + cat > /etc/systemd/system/rclone-mount.service </dev/null || true diff --git a/plugins/vscode-tunnel/README.md b/plugins/vscode-tunnel/README.md new file mode 100644 index 0000000..e4db444 --- /dev/null +++ b/plugins/vscode-tunnel/README.md @@ -0,0 +1,53 @@ +# vscode-tunnel + +Runs a **VS Code Remote Tunnel** on your spawn instance so you can attach to it +from desktop VS Code (the "Remote - Tunnels" extension) or from +[vscode.dev](https://vscode.dev) — full editor, terminal, and extensions, with +**no listening port and no security-group changes** (the tunnel is an outbound +relay to Microsoft's service, like Tailscale). + +## How it works + +The plugin installs the official VS Code CLI to `/usr/local/bin/code` and writes +a per-user systemd service (`vscode-tunnel`) that runs `code tunnel` as +`ec2-user`. Because a tunnel needs a **one-time interactive login**, the plugin +does **not** auto-start the service — you complete the login once, then start it. + +## One-time setup (interactive login) + +After installing the plugin, SSH to the instance and authenticate once: + +```bash +# on the instance, as ec2-user: +code tunnel user login # prints a code + URL; approve it in your browser +sudo systemctl enable --now vscode-tunnel +``` + +Then in desktop VS Code: **Remote-Tunnels: Connect to Tunnel…**, or open +`https://vscode.dev` and pick the tunnel. The `health` check reports `active` +once the service is running. + +## Config + +| Key | Default | Description | +|-----|---------|-------------| +| `name` | *(from hostname)* | Tunnel name shown in VS Code. | + +## Install + +```bash +spawn plugin install vscode-tunnel --instance --config name=my-box +``` + +## Why no automatic smoke + +This plugin requires an interactive GitHub/Microsoft device login that only you +can complete, so it ships **static-validated** (schema + permission consistency) +with this manual recipe rather than an automated real-AWS smoke. The install and +service-config steps are exercised; the login + tunnel are yours to drive. + +## Notes + +- No port is opened and no reconcile is needed — the tunnel reconnects outbound + on its own after a stop/start. +- Prefer a browser IDE instead? See [`code-server`](../code-server/). diff --git a/plugins/vscode-tunnel/plugin.yaml b/plugins/vscode-tunnel/plugin.yaml new file mode 100644 index 0000000..94362e8 --- /dev/null +++ b/plugins/vscode-tunnel/plugin.yaml @@ -0,0 +1,109 @@ +name: vscode-tunnel +version: v1.0.0 +description: "Run a VS Code Remote Tunnel so you can attach from desktop VS Code or vscode.dev" +author: spore-host +license: Apache-2.0 + +# Installs the VS Code CLI and runs `code tunnel` as a per-user systemd service. +# A tunnel is an OUTBOUND relay to Microsoft's tunnel service — there is NO +# listening port and NO security-group concern (like Tailscale/Globus). You then +# attach from desktop VS Code ("Remote - Tunnels") or https://vscode.dev. +# +# First-run requires a one-time GitHub/Microsoft device login on the instance +# (the CLI prints a code + URL). See the README — this is why the plugin is +# published static-validated but not auto-smoked (it needs your interactive auth). +# The tunnel runs as the instance's login user (as_user) because auth + tunnel +# state live under that user's home. + +permissions: + instance: + root: true # install step writes the CLI to /usr/local/bin (root); tunnel runs as_user + network: true # downloads the CLI + maintains the outbound tunnel + files: + - /usr/local/bin/code + - /etc/systemd/system/vscode-tunnel.service + +config: + name: + type: string + required: false + default: "" + description: "Tunnel name (shown in VS Code); empty lets the CLI pick from the hostname" + +conditions: + remote: + - type: platform + os: linux + message: "vscode-tunnel plugin requires a Linux instance" + +remote: + install: + - type: run + run: | + set -eu + # Official Microsoft CLI download. The stable URL is a moving redirect to a + # commit-pinned artifact, so — like rstudio-server — install via the + # vendor's URL with arch detection rather than a fetch+sha256 of a file + # that changes. glibc build for x64/arm64. + case "$(uname -m)" in + x86_64) CLI_ARCH=cli-linux-x64 ;; + aarch64) CLI_ARCH=cli-linux-arm64 ;; + *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; + esac + cd /tmp + curl -fsSL -o vscode-cli.tar.gz "https://update.code.visualstudio.com/latest/${CLI_ARCH}/stable" + tar xzf vscode-cli.tar.gz + install -m 0755 code /usr/local/bin/code + rm -f vscode-cli.tar.gz code + + configure: + - type: run + run: | + set -eu + NAME={{ config.name }} + NAME_ARG="" + [ -n "$NAME" ] && NAME_ARG="--name $NAME" + # Per-user service: runs `code tunnel` as ec2-user so the device-login + # token and tunnel state persist under /home/ec2-user/.vscode-cli. + cat > /etc/systemd/system/vscode-tunnel.service </dev/null || true + systemctl disable vscode-tunnel 2>/dev/null || true