diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f463f5a..67306d9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -40,7 +40,9 @@ "source=${localEnv:HOME}/.config/gh,target=/home/vscode/.config/gh,type=bind,consistency=cached", "source=${localEnv:HOME}/.claude,target=/home/vscode/.claude,type=bind,consistency=cached", "source=${localEnv:HOME}/.aws,target=/home/vscode/.aws,type=bind,consistency=cached", - "source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind,consistency=cached" + "source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind,consistency=cached", + "source=${localWorkspaceFolder}/../site,target=/workspaces/site,type=bind,consistency=cached", + "source=${localWorkspaceFolder}/../remo-broker,target=/workspaces/remo-broker,type=bind,consistency=cached" ], "postCreateCommand": "python3 --version && node --version" } diff --git a/.dockerignore b/.dockerignore index 2071b76..1c0e518 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,14 +6,19 @@ .claude .ansible .remo +.git .venv venv dist build +node_modules +coverage *.egg-info *.egg __pycache__ *.pyc +*.log* +.env* *.pem id_rsa id_rsa.pub @@ -29,3 +34,5 @@ vault.yml tests LICENSE CLAUDE.md +Dockerfile* +.dockerignore diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 7d07250..c3188a4 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -803,6 +803,51 @@ jobs: $SSH_CMD "pkill -f 'python3 -m http.server 9090'" 2>/dev/null || true $SSH_CMD "pkill -f 'python3 -m http.server 3000'" 2>/dev/null || true + - name: Test remo shell -p / --exec / --detach (project-launch passthrough) + run: | + SSH_CMD="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=30 -o BatchMode=yes -i ~/.ssh/id_rsa remo@$HOST" + + # Plant a project dir without a .devcontainer so we exercise the + # host-side branches of project-launch (no docker dance needed in CI). + $SSH_CMD "mkdir -p ~/projects/smoke-proj && \ + echo 'placeholder' > ~/projects/smoke-proj/README" + + # --- Foreground --exec on a no-devcontainer project (Case 4) --- + OUT=$(uv run remo shell -p smoke-proj --exec 'echo hello-$REMO_PROJECT-$REMO_INSTANCE && pwd' 2>&1) + echo "$OUT" + echo "$OUT" | grep -q "hello-smoke-proj-" || { + echo "FAIL: --exec did not run / env vars missing"; exit 1; } + echo "$OUT" | grep -q "/projects/smoke-proj" || { + echo "FAIL: --exec did not cd into project dir"; exit 1; } + echo "remo shell -p --exec (foreground): PASS" + + # --- Detached --exec writes log file and returns immediately --- + DETACH_OUT=$(uv run remo shell -p smoke-proj --detach \ + --exec 'sleep 1 && echo detached-marker-$REMO_PROJECT' 2>&1) + echo "$DETACH_OUT" + echo "$DETACH_OUT" | grep -q "Launched detached" || { + echo "FAIL: detach didn't print launched message"; exit 1; } + # Give the background command a moment to finish + sleep 3 + LOG=$($SSH_CMD "cat ~/.local/state/remo/smoke-proj.log") + echo "log contents:"; echo "$LOG" + echo "$LOG" | grep -q "detached-marker-smoke-proj" || { + echo "FAIL: detached command did not run / log missing marker"; exit 1; } + echo "remo shell -p --detach --exec: PASS" + + # --- Validation: --exec without -p exits 2 --- + if uv run remo shell --exec 'true' 2>/dev/null; then + echo "FAIL: --exec without -p should have errored"; exit 1 + fi + echo "validation: --exec without -p errors as expected" + + # --- Validation: --detach without --exec exits 2 --- + if uv run remo shell -p smoke-proj --detach 2>/dev/null; then + echo "FAIL: --detach without --exec should have errored"; exit 1 + fi + echo "validation: --detach without --exec errors as expected" + - name: Teardown if: always() run: | diff --git a/.gitignore b/.gitignore index 20b5bb9..3c93f5f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ vault.yml # Local configuration *.local.yml +.env* # Remo state directory (legacy location, now uses ~/.config/remo) .remo/ @@ -45,3 +46,5 @@ env/ .maverick/* !.maverick/checkpoints/ .claude/settings.local.json +.DS_Store +Thumbs.db diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a0fde34 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,7 @@ +# Agent Instructions + +This repository's canonical agent guidance lives in [`CLAUDE.md`](./CLAUDE.md). + +Use `CLAUDE.md` as the source of truth for repository-specific conventions, architecture, commands, and implementation rules when working in this codebase. + +The current credential-broker feature provisions a managed `_remo-vault` project alongside normal workspaces; use `remo shell -p _remo-vault` for sidecar-specific flows. Normal projects get a generated devcontainer config at launch time so the checked-in config is not mutated when the read-only manifest mount and broker socket are injected. diff --git a/CLAUDE.md b/CLAUDE.md index 4292696..28fc340 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,6 +12,8 @@ See `.specify/memory/constitution.md` for project principles and non-negotiable - Python 3.11+ + Click (CLI framework), InquirerPy (interactive picker), boto3 (AWS, optional), hcloud (Hetzner, optional) (003-python-cli-rewrite) - Flat file (`~/.config/remo/known_hosts`, colon-delimited) (003-python-cli-rewrite) - Cross-provider snapshot model (`models/snapshot.py`) + shared helpers in `core/snapshot.py` (name generator, validator, table formatter, destroy-time cleanup hook). No new runtime deps. (005-provider-snapshots) +- Python 3.11+, Ansible Core 2.18+, Bash/Jinja2 remote host templates + Click 8.1+, InquirerPy, ansible-core, Docker CE + Compose plugin, `@devcontainers/cli`, systemd credentials on the LXC host, sibling `remo-broker` v2 binary/schema, and remote sidecar CLIs (`gh`, `aws`, `claude`, `fnox`) (006-credential-broker-laptop-push) +- Version-controlled `~/projects//.remo/manifest.toml`; sidecar Docker volume at `/var/lib/remo-vault/fnox.enc`; host-side systemd credential material for the sidecar decryption key; broker secrets in memory only; broker audit log on host disk (006-credential-broker-laptop-push) - Ansible 2.14+ / YAML + `ansible.builtin`, `community.general` (for zypper module) (001-bootstrap-incus-host) @@ -132,10 +134,12 @@ Provider SDKs (boto3, hcloud) are lazy-imported with clear error messages if mis - Ansible 2.14+ / YAML: Follow standard conventions plus Constitution principles ## Recent Changes +- 006-credential-broker-laptop-push: Added Python 3.11+, Ansible Core 2.18+, Bash/Jinja2 remote host templates + Click 8.1+, InquirerPy, ansible-core, Docker CE + Compose plugin, `@devcontainers/cli`, systemd credentials on the LXC host, sibling `remo-broker` v2 binary/schema, and remote sidecar CLIs (`gh`, `aws`, `claude`, `fnox`) - 005-provider-snapshots: Added cross-provider snapshot CLI (`remo

snapshot {create,list,restore,delete}`) + destroy-time cleanup hook across Incus / Proxmox / AWS / Hetzner. - 003-python-cli-rewrite: Added Python 3.11+ + Click (CLI framework), InquirerPy (interactive picker), boto3 (AWS, optional), hcloud (Hetzner, optional) -- 002-incus-container-support: Added Ansible 2.14+ / YAML + `ansible.builtin`, `community.general` (existing), Incus CLI (local) +- Project startup injects the secrets feature through a generated `devcontainer --config` file rather than mutating the repo's checked-in devcontainer config. +- `_remo-vault` is a reserved managed project: use `remo shell -p _remo-vault` for broker admin flows, but do not wrap its startup with `remo-fetch-secrets`. diff --git a/README.md b/README.md index dfa1504..11b6893 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,21 @@ You land in an interactive project menu. Pick a project, and you're in a persist [Exit to shell] ``` +### Managed credential sidecar + +Every `remo create` and `remo update` now reconciles a managed broker service plus a reserved `_remo-vault` workspace on the remote host. `--only` and `--skip` still control the optional tool bundle, but they do **not** disable broker or sidecar reconciliation. + +```bash +remo shell -p _remo-vault +``` + +Use `_remo-vault` to log into `gh` / `aws`, inspect broker state with `remo-vend-status`, and reload project allowlists after editing `~/projects//.remo/manifest.toml`: + +```bash +remo-reload +remo-test-project +``` + --- ## Choose Your Platform @@ -94,6 +109,44 @@ The `fzf`-powered menu shows your projects from `~/projects`: - **c**: Clone a new repository - **x**: Exit to shell +### Jump Straight to a Project + +Skip the menu and land directly in a project (devcontainer auto-launches): + +```bash +remo shell -p my-app +``` + +Run a one-shot command inside the project's devcontainer instead of opening +a shell — quote the command as a single string: + +```bash +remo shell -p my-app --exec 'pytest -x' +remo shell -p my-app --exec 'claude --remote-control' +``` + +Fire-and-forget — kick off a command on the remote and exit SSH immediately: + +```bash +remo shell -p my-app --detach --exec 'claude remote-control --name remo-rc' +remo shell -p my-app --detach --exec './long-build.sh' +``` + +Detached output is captured to `~/.local/state/remo/.log` on the +remote, so you can tail it later (`remo shell -p my-app --exec 'tail -f +~/.local/state/remo/my-app.log'`). The command's environment gets +`REMO_INSTANCE` and `REMO_PROJECT` exported automatically — handy for +deterministic naming, e.g.: + +```bash +remo shell -p my-app --detach --exec \ + 'claude remote-control --name "remo-$REMO_INSTANCE-$REMO_PROJECT"' +``` + +Then on your phone, open claude.ai/code and pick the session by name. + +For normal projects, Remo generates a temporary merged devcontainer config at startup so the checked-in `.devcontainer/devcontainer.json` stays untouched while the broker socket and read-only manifest mount are injected. Secrets are fetched before the interactive shell, `--exec`, or `--detach` command starts. The reserved `_remo-vault` project keeps its own sidecar devcontainer config and does not get the project-secrets wrapper. + ### Port Forwarding Forward remote ports to your local machine during SSH sessions: @@ -175,6 +228,9 @@ they continue to incur storage costs on AWS/Hetzner). # Connect to environment remo shell # Auto-connect (or picker if multiple) remo shell my-env # Connect to a specific environment +remo shell -p my-app # Skip the menu, jump to ~/projects/my-app +remo shell -p my-app --exec 'pytest -x' # Run command in devcontainer +remo shell -p my-app --detach --exec 'claude remote-control --name rc' # Fire and exit remo shell -L 8080 # Shell + forward remote :8080 to local :8080 remo shell -L 9000:8080 # Shell + forward remote :8080 to local :9000 remo shell -L 8080 -L 3000 # Shell + forward multiple ports diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 458d612..dc80030 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -27,3 +27,17 @@ zellij_session_name: "main" # Session name for SSH auto-attach # Node.js Configuration nodejs_version: "24" # LTS version + +# Credential broker configuration +remo_broker_user: "remo-broker" +remo_broker_group: "remo-broker" +remo_broker_protocol_version: 2 +remo_broker_binary_path: "/usr/local/bin/remo-broker" +remo_broker_config_dir: "/etc/remo-broker" +remo_broker_config_path: "/etc/remo-broker/config.toml" +remo_broker_projects_dir: "/etc/remo-broker/projects" +remo_broker_socket_dir: "/run/remo-broker" +remo_broker_admin_socket: "/run/remo-broker/admin.sock" +remo_broker_log_dir: "/var/log/remo-broker" +remo_broker_audit_log: "/var/log/remo-broker/audit.log" +remo_broker_bootstrap_token_path: "/etc/remo-broker/bootstrap-token" diff --git a/ansible/roles/broker_host/tasks/main.yml b/ansible/roles/broker_host/tasks/main.yml new file mode 100644 index 0000000..a50223e --- /dev/null +++ b/ansible/roles/broker_host/tasks/main.yml @@ -0,0 +1,44 @@ +--- +- name: Create broker host directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - /etc/remo-broker + - /etc/remo-broker/projects + - /etc/remo-broker/credentials + - /etc/systemd/system/remo-broker.service.d + - /var/log/remo-broker + +- name: Install broker manifest schema reference + ansible.builtin.template: + src: manifest.schema.toml.j2 + dest: /etc/remo-broker/manifest.schema.toml + owner: root + group: root + mode: '0644' + +- name: Install broker environment defaults + ansible.builtin.copy: + dest: /etc/default/remo-broker + owner: root + group: root + mode: '0644' + content: | + REMO_BROKER_PROTOCOL_VERSION=2 + REMO_BROKER_ADMIN_SOCKET=/run/remo-broker/admin.sock + REMO_BROKER_PROJECTS_DIR=/etc/remo-broker/projects + REMO_BROKER_AUDIT_LOG=/var/log/remo-broker/audit.log + +- name: Install broker service override + ansible.builtin.copy: + dest: /etc/systemd/system/remo-broker.service.d/override.conf + owner: root + group: root + mode: '0644' + content: | + [Service] + EnvironmentFile=-/etc/default/remo-broker diff --git a/ansible/roles/broker_host/templates/manifest.schema.toml.j2 b/ansible/roles/broker_host/templates/manifest.schema.toml.j2 new file mode 100644 index 0000000..4f0c453 --- /dev/null +++ b/ansible/roles/broker_host/templates/manifest.schema.toml.j2 @@ -0,0 +1,18 @@ +# Canonical host-side project manifest schema for remo broker v2 helpers. +schema_version = 1 + +[manifest] +required_fields = ["schema_version", "project"] +secret_table = "secrets" + +[secrets] +fetch_as_default = "env" +file_modes = ["0600", "0640"] + +[cache] +default_ttl_seconds = 900 +default_max_entries = 50 + +[broker] +protocol_version = 2 +admin_socket = "/run/remo-broker/admin.sock" diff --git a/ansible/roles/broker_sidecar/tasks/main.yml b/ansible/roles/broker_sidecar/tasks/main.yml new file mode 100644 index 0000000..9a9a6d2 --- /dev/null +++ b/ansible/roles/broker_sidecar/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: Create managed vault workspace directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0755' + loop: + - "{{ dev_workspace_dir }}/_remo-vault" + - "{{ dev_workspace_dir }}/_remo-vault/.remo" + - "{{ dev_workspace_dir }}/_remo-vault/bin" + +- name: Install managed vault manifest + ansible.builtin.template: + src: manifest.toml.j2 + dest: "{{ dev_workspace_dir }}/_remo-vault/.remo/manifest.toml" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0644' + +- name: Install managed vault README + ansible.builtin.copy: + dest: "{{ dev_workspace_dir }}/_remo-vault/README.txt" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0644' + content: | + _remo-vault is the remo-managed credential broker sidecar workspace. + Store login state and encrypted secret material here, then use: + + remo-list-creds + remo-vend-status + remo-reload + + The sidecar is created automatically and should not be deleted manually. + +- name: Install broker helper scripts + ansible.builtin.template: + src: "{{ item.src }}" + dest: "/home/{{ remo_user }}/.local/bin/{{ item.dest }}" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0755' + loop: + - src: remo-vend-status.sh.j2 + dest: remo-vend-status + - src: remo-list-creds.sh.j2 + dest: remo-list-creds + - src: remo-reload.sh.j2 + dest: remo-reload + - src: remo-fetch-secrets.sh.j2 + dest: remo-fetch-secrets diff --git a/ansible/roles/broker_sidecar/templates/manifest.toml.j2 b/ansible/roles/broker_sidecar/templates/manifest.toml.j2 new file mode 100644 index 0000000..8346ab2 --- /dev/null +++ b/ansible/roles/broker_sidecar/templates/manifest.toml.j2 @@ -0,0 +1,20 @@ +schema_version = 1 +project = "_remo-vault" + +[secrets.gh] +fetch_as = "env" +env_var = "GH_TOKEN" + +[secrets.aws] +fetch_as = "file" +file_path = "~/.aws/credentials" +file_mode = "0600" +template = """ +[default] +{% raw %}aws_access_key_id={{aws_access_key_id}} +aws_secret_access_key={{aws_secret_access_key}}{% endraw %} +""" + +[cache] +default_ttl_seconds = 900 +default_max_entries = 50 diff --git a/ansible/roles/broker_sidecar/templates/remo-fetch-secrets.sh.j2 b/ansible/roles/broker_sidecar/templates/remo-fetch-secrets.sh.j2 new file mode 100644 index 0000000..96f47e2 --- /dev/null +++ b/ansible/roles/broker_sidecar/templates/remo-fetch-secrets.sh.j2 @@ -0,0 +1,41 @@ +#!/bin/bash +set -euo pipefail + +PROJECT="${1:-${REMO_PROJECT:-}}" + +if [ -z "$PROJECT" ]; then + echo "Usage: remo-fetch-secrets " >&2 + echo " or set REMO_PROJECT inside a project session" >&2 + exit 2 +fi + +MANIFEST="{{ dev_workspace_dir }}/$PROJECT/.remo/manifest.toml" +if [ ! -f "$MANIFEST" ]; then + echo "remo-fetch-secrets: manifest not found: $MANIFEST" >&2 + exit 1 +fi + +python3 - "$MANIFEST" <<'PY' +from __future__ import annotations + +import sys +import tomllib +from pathlib import Path + +manifest_path = Path(sys.argv[1]) +data = tomllib.loads(manifest_path.read_text()) + +if data.get("schema_version") != 1: + raise SystemExit("remo-fetch-secrets: unsupported manifest schema version") + +project = data.get("project") +secrets = data.get("secrets", {}) + +print(f"project: {project}") +for name, spec in secrets.items(): + fetch_as = spec.get("fetch_as", "env") + if fetch_as == "file": + print(f"{name}\tfile\t{spec.get('file_path', '')}") + else: + print(f"{name}\tenv\t{spec.get('env_var', name.upper())}") +PY diff --git a/ansible/roles/broker_sidecar/templates/remo-list-creds.sh.j2 b/ansible/roles/broker_sidecar/templates/remo-list-creds.sh.j2 new file mode 100644 index 0000000..4002b5f --- /dev/null +++ b/ansible/roles/broker_sidecar/templates/remo-list-creds.sh.j2 @@ -0,0 +1,31 @@ +#!/bin/bash +set -euo pipefail + +if ! command -v fnox >/dev/null 2>&1; then + echo "remo-list-creds: fnox is not installed on this host" >&2 + exit 1 +fi + +if fnox list --json >/tmp/remo-fnox-list.$$ 2>/dev/null; then + python3 - /tmp/remo-fnox-list.$$ <<'PY' +from __future__ import annotations + +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +raw = json.loads(path.read_text()) + +items = raw if isinstance(raw, list) else raw.get("items", []) +for item in items: + if isinstance(item, dict): + name = item.get("name", "") + meta = item.get("updated_at") or item.get("created_at") or "metadata unavailable" + print(f"{name}\t{meta}") +PY + rm -f /tmp/remo-fnox-list.$$ + exit 0 +fi + +exec fnox list diff --git a/ansible/roles/broker_sidecar/templates/remo-reload.sh.j2 b/ansible/roles/broker_sidecar/templates/remo-reload.sh.j2 new file mode 100644 index 0000000..63f3950 --- /dev/null +++ b/ansible/roles/broker_sidecar/templates/remo-reload.sh.j2 @@ -0,0 +1,46 @@ +#!/bin/bash +set -euo pipefail + +if [ $# -ne 1 ]; then + echo "Usage: remo-reload " >&2 + exit 2 +fi + +PROJECT="$1" +MANIFEST="{{ dev_workspace_dir }}/$PROJECT/.remo/manifest.toml" +ADMIN_SOCKET="${REMO_BROKER_ADMIN_SOCKET:-/run/remo-broker/admin.sock}" + +if [ ! -f "$MANIFEST" ]; then + echo "remo-reload: manifest not found: $MANIFEST" >&2 + exit 1 +fi + +python3 - "$ADMIN_SOCKET" "$PROJECT" <<'PY' +from __future__ import annotations + +import json +import socket +import sys + +sock_path = sys.argv[1] +project = sys.argv[2] +payload = json.dumps({"op": "reload", "name": project}).encode() + b"\n" + +try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.connect(sock_path) + client.sendall(payload) + response = client.recv(65536) +except FileNotFoundError as exc: + raise SystemExit(f"remo-reload: admin socket not found: {sock_path}") from exc +except OSError as exc: + raise SystemExit(f"remo-reload: broker reload failed: {exc}") from exc + +data = json.loads(response.decode().strip()) +if not data.get("ok"): + raise SystemExit(f"remo-reload: broker returned error: {data}") + +allowlist = ", ".join(data.get("allowlist", [])) or "(empty)" +print(f"reloaded {project}") +print(f"allowlist: {allowlist}") +PY diff --git a/ansible/roles/broker_sidecar/templates/remo-vend-status.sh.j2 b/ansible/roles/broker_sidecar/templates/remo-vend-status.sh.j2 new file mode 100644 index 0000000..286185d --- /dev/null +++ b/ansible/roles/broker_sidecar/templates/remo-vend-status.sh.j2 @@ -0,0 +1,37 @@ +#!/bin/bash +set -euo pipefail + +ADMIN_SOCKET="${REMO_BROKER_ADMIN_SOCKET:-/run/remo-broker/admin.sock}" + +python3 - "$ADMIN_SOCKET" <<'PY' +from __future__ import annotations + +import json +import socket +import sys + +sock_path = sys.argv[1] +payload = json.dumps({"op": "status"}).encode() + b"\n" + +try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.connect(sock_path) + client.sendall(payload) + response = client.recv(65536) +except FileNotFoundError as exc: + raise SystemExit(f"remo-vend-status: admin socket not found: {sock_path}") from exc +except OSError as exc: + raise SystemExit(f"remo-vend-status: could not query broker status: {exc}") from exc + +data = json.loads(response.decode().strip()) +if not data.get("ok"): + raise SystemExit(f"remo-vend-status: broker returned error: {data}") +if data.get("protocol_version") != 2: + raise SystemExit( + f"remo-vend-status: incompatible broker protocol {data.get('protocol_version')}, expected 2" + ) + +print(f"protocol_version: {data['protocol_version']}") +print(f"secret_count: {data.get('secret_count', 0)}") +print(f"secrets_loaded_at: {data.get('secrets_loaded_at', 'n/a')}") +PY diff --git a/ansible/roles/devcontainers/tasks/main.yml b/ansible/roles/devcontainers/tasks/main.yml index c559d96..0dd711b 100644 --- a/ansible/roles/devcontainers/tasks/main.yml +++ b/ansible/roles/devcontainers/tasks/main.yml @@ -10,6 +10,19 @@ register: devcontainers_version changed_when: false +- name: Verify devcontainer CLI supports custom config injection + ansible.builtin.command: devcontainer up --help + register: devcontainers_up_help + changed_when: false + +- name: Assert devcontainer CLI supports --config + ansible.builtin.assert: + that: + - "'--config' in (devcontainers_up_help.stdout | default(''))" + fail_msg: >- + Installed devcontainer CLI must support --config so remo can inject the + read-only manifest and broker socket feature into project startups. + - name: Display devcontainer CLI version ansible.builtin.debug: msg: "devcontainer CLI: {{ devcontainers_version.stdout }}" diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index d38e39d..e96ae5b 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -59,14 +59,48 @@ ansible.builtin.apt: update_cache: true +- name: Check Docker repository package availability + ansible.builtin.command: + cmd: >- + apt-cache policy + docker-ce + docker-ce-cli + containerd.io + docker-buildx-plugin + docker-compose-plugin + register: docker_repo_package_policy + changed_when: false + +- name: Select Docker package source + ansible.builtin.set_fact: + docker_package_names: >- + {{ + [ + 'docker-ce', + 'docker-ce-cli', + 'containerd.io', + 'docker-buildx-plugin', + 'docker-compose-plugin', + ] + if 'Candidate: (none)' not in docker_repo_package_policy.stdout + else [ + 'docker.io', + 'containerd', + 'docker-buildx-plugin', + 'docker-compose-v2', + ] + }} + +- name: Report Docker package fallback + ansible.builtin.debug: + msg: >- + Docker's upstream apt repo does not publish the required packages for + this host; falling back to Ubuntu's docker.io package set. + when: "'Candidate: (none)' in docker_repo_package_policy.stdout" + - name: Install Docker packages ansible.builtin.apt: - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin + name: "{{ docker_package_names }}" state: present - name: Ensure Docker service is started and enabled diff --git a/ansible/roles/remo_broker/README.md b/ansible/roles/remo_broker/README.md new file mode 100644 index 0000000..27b62e0 --- /dev/null +++ b/ansible/roles/remo_broker/README.md @@ -0,0 +1,8 @@ +# remo_broker + +Installs and configures the on-instance `remo-broker` daemon assets: + +- system user/group +- config and environment files +- admin socket and audit-log directories +- systemd unit wiring diff --git a/ansible/roles/remo_broker/tasks/main.yml b/ansible/roles/remo_broker/tasks/main.yml new file mode 100644 index 0000000..083509e --- /dev/null +++ b/ansible/roles/remo_broker/tasks/main.yml @@ -0,0 +1,97 @@ +--- +- name: Ensure remo-broker group exists + ansible.builtin.group: + name: "{{ remo_broker_group }}" + system: true + state: present + +- name: Ensure remo-broker user exists + ansible.builtin.user: + name: "{{ remo_broker_user }}" + group: "{{ remo_broker_group }}" + system: true + shell: /usr/sbin/nologin + create_home: false + state: present + +- name: Create remo-broker directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: + - path: "{{ remo_broker_config_dir }}" + owner: root + group: root + mode: '0755' + - path: "{{ remo_broker_projects_dir }}" + owner: root + group: root + mode: '0755' + - path: "{{ remo_broker_log_dir }}" + owner: "{{ remo_broker_user }}" + group: "{{ remo_broker_group }}" + mode: '0750' + - path: "{{ remo_broker_socket_dir }}" + owner: "{{ remo_broker_user }}" + group: "{{ remo_broker_group }}" + mode: '0755' + +- name: Install remo-broker environment file + ansible.builtin.template: + src: remo-broker.env.j2 + dest: /etc/default/remo-broker + owner: root + group: root + mode: '0644' + +- name: Install remo-broker systemd unit + ansible.builtin.template: + src: remo-broker.service.j2 + dest: /etc/systemd/system/remo-broker.service + owner: root + group: root + mode: '0644' + register: remo_broker_unit + +- name: Install placeholder bootstrap token file + ansible.builtin.copy: + dest: "{{ remo_broker_bootstrap_token_path }}" + owner: root + group: root + mode: '0600' + force: false + content: | + set-this-to-a-real-bootstrap-token + +- name: Ensure remo user can inspect broker project sockets + ansible.builtin.user: + name: "{{ remo_user }}" + groups: "{{ remo_broker_group }}" + append: true + when: remo_user | default('') | length > 0 + +- name: Verify remo-broker assets are installed + ansible.builtin.command: + cmd: >- + /bin/bash -lc + 'test -f /etc/default/remo-broker && + test -f /etc/systemd/system/remo-broker.service && + test -f "{{ remo_broker_bootstrap_token_path }}"' + register: remo_broker_install_check + changed_when: false + failed_when: false + +- name: Fail when remo-broker assets are incomplete + ansible.builtin.fail: + msg: >- + Managed remo-broker provisioning is incomplete. Expected the broker + environment file, systemd unit, and bootstrap token to exist. + when: remo_broker_install_check.rc | default(1) != 0 + +- name: Reload systemd when remo-broker unit changed + ansible.builtin.systemd_service: + daemon_reload: true + when: remo_broker_unit.changed | default(false) diff --git a/ansible/roles/remo_broker/templates/remo-broker.env.j2 b/ansible/roles/remo_broker/templates/remo-broker.env.j2 new file mode 100644 index 0000000..451f89d --- /dev/null +++ b/ansible/roles/remo_broker/templates/remo-broker.env.j2 @@ -0,0 +1,7 @@ +REMO_BROKER_PROTOCOL_VERSION={{ remo_broker_protocol_version }} +REMO_BROKER_ADMIN_SOCKET={{ remo_broker_admin_socket }} +REMO_BROKER_SOCKET_DIR={{ remo_broker_socket_dir }} +REMO_BROKER_PROJECTS_DIR={{ remo_broker_projects_dir }} +REMO_BROKER_AUDIT_LOG={{ remo_broker_audit_log }} +REMO_BROKER_CONFIG={{ remo_broker_config_path }} +REMO_BROKER_BINARY={{ remo_broker_binary_path }} diff --git a/ansible/roles/remo_broker/templates/remo-broker.service.j2 b/ansible/roles/remo_broker/templates/remo-broker.service.j2 new file mode 100644 index 0000000..7cb06fc --- /dev/null +++ b/ansible/roles/remo_broker/templates/remo-broker.service.j2 @@ -0,0 +1,49 @@ +[Unit] +Description=Remo on-instance credential broker +Documentation=https://github.com/get2knowio/remo-broker +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +NotifyAccess=main +EnvironmentFile=-/etc/default/remo-broker +ExecStart={{ remo_broker_binary_path }} --bootstrap-token-path %d/bootstrap-token +TimeoutStopSec=10s +Restart=on-failure +RestartSec=5s +LimitCORE=0 +ProtectSystem=strict +ProtectHome=yes +NoNewPrivileges=yes +MemoryDenyWriteExecute=yes +RestrictSUIDSGID=yes +User={{ remo_broker_user }} +Group={{ remo_broker_group }} +RuntimeDirectory=remo-broker +RuntimeDirectoryMode=0755 +LogsDirectory=remo-broker +LogsDirectoryMode=0750 +ReadWritePaths={{ remo_broker_socket_dir }} {{ remo_broker_log_dir }} +LoadCredential=bootstrap-token:{{ remo_broker_bootstrap_token_path }} +PrivateTmp=yes +PrivateDevices=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes +ProtectProc=invisible +ProcSubset=pid +RestrictNamespaces=yes +RestrictRealtime=yes +LockPersonality=yes +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallFilter=~@privileged @resources +CapabilityBoundingSet= +AmbientCapabilities= + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/remo_secrets_feature/README.md b/ansible/roles/remo_secrets_feature/README.md new file mode 100644 index 0000000..3b004eb --- /dev/null +++ b/ansible/roles/remo_secrets_feature/README.md @@ -0,0 +1,4 @@ +# remo_secrets_feature + +Installs shared manifest-schema and project startup helper assets used by +broker-backed secret vending. diff --git a/ansible/roles/remo_secrets_feature/tasks/main.yml b/ansible/roles/remo_secrets_feature/tasks/main.yml new file mode 100644 index 0000000..30154e4 --- /dev/null +++ b/ansible/roles/remo_secrets_feature/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Create remo secrets feature directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0755' + loop: + - "/home/{{ remo_user }}/.local/share/remo-secrets" + - "/home/{{ remo_user }}/.cache/remo-secrets" + +- name: Install manifest schema reference + ansible.builtin.template: + src: manifest.schema.toml + dest: "/home/{{ remo_user }}/.local/share/remo-secrets/manifest.schema.toml" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0644' + +- name: Install secrets feature devcontainer snippet + ansible.builtin.template: + src: feature-devcontainer.json.j2 + dest: "/home/{{ remo_user }}/.local/share/remo-secrets/feature-devcontainer.json" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0644' + +- name: Install remo-fetch-secrets helper + ansible.builtin.template: + src: remo-fetch-secrets.sh.j2 + dest: "/home/{{ remo_user }}/.local/bin/remo-fetch-secrets" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0755' + +- name: Verify remo secrets feature assets are installed + ansible.builtin.command: + cmd: >- + /bin/bash -lc + 'test -f "/home/{{ remo_user }}/.local/share/remo-secrets/manifest.schema.toml" && + test -f "/home/{{ remo_user }}/.local/share/remo-secrets/feature-devcontainer.json" && + test -x "/home/{{ remo_user }}/.local/bin/remo-fetch-secrets"' + register: remo_secrets_feature_check + changed_when: false + failed_when: false + +- name: Fail when remo secrets feature assets are incomplete + ansible.builtin.fail: + msg: >- + Failed to install remo secrets feature assets completely. Expected the + manifest schema, feature devcontainer snippet, and remo-fetch-secrets + helper to exist for {{ remo_user }}. + when: remo_secrets_feature_check.rc | default(1) != 0 diff --git a/ansible/roles/remo_secrets_feature/templates/feature-devcontainer.json.j2 b/ansible/roles/remo_secrets_feature/templates/feature-devcontainer.json.j2 new file mode 100644 index 0000000..9636ac0 --- /dev/null +++ b/ansible/roles/remo_secrets_feature/templates/feature-devcontainer.json.j2 @@ -0,0 +1,12 @@ +{ + "mounts": [ + "source=/run/remo-broker/${localWorkspaceFolderBasename}.sock,target=/run/remo-broker/${localWorkspaceFolderBasename}.sock,type=bind", + "source=${localWorkspaceFolder}/.remo/manifest.toml,target=/workspace/.remo/manifest.toml,type=bind,readonly" + ], + "containerEnv": { + "REMO_PROJECT": "${localWorkspaceFolderBasename}", + "REMO_BROKER_PROJECT_SOCKET": "/run/remo-broker/${localWorkspaceFolderBasename}.sock", + "REMO_MANIFEST_PATH": "/workspace/.remo/manifest.toml", + "REMO_SECRETS_ENV_FILE": "/home/vscode/.cache/remo-secrets/${localWorkspaceFolderBasename}.env" + } +} diff --git a/ansible/roles/remo_secrets_feature/templates/manifest.schema.toml b/ansible/roles/remo_secrets_feature/templates/manifest.schema.toml new file mode 100644 index 0000000..45c4e03 --- /dev/null +++ b/ansible/roles/remo_secrets_feature/templates/manifest.schema.toml @@ -0,0 +1,15 @@ +schema_version = 1 + +[manifest] +required_fields = ["schema_version", "project", "secrets"] + +[secrets] +fetch_as = ["env", "file"] +default_fetch_as = "env" + +[file] +required_fields = ["file_path", "file_mode", "template"] + +[cache] +default_ttl_seconds = 900 +default_max_entries = 50 diff --git a/ansible/roles/remo_secrets_feature/templates/remo-fetch-secrets.sh.j2 b/ansible/roles/remo_secrets_feature/templates/remo-fetch-secrets.sh.j2 new file mode 100644 index 0000000..0183646 --- /dev/null +++ b/ansible/roles/remo_secrets_feature/templates/remo-fetch-secrets.sh.j2 @@ -0,0 +1,177 @@ +#!/bin/bash +set -euo pipefail + +PROJECT="${1:-${REMO_PROJECT:-}}" + +if [ -z "$PROJECT" ]; then + echo "Usage: remo-fetch-secrets " >&2 + echo " or set REMO_PROJECT inside a project session" >&2 + exit 2 +fi + +MANIFEST="${REMO_MANIFEST_PATH:-$PWD/.remo/manifest.toml}" +BROKER_SOCKET="${REMO_BROKER_PROJECT_SOCKET:-/run/remo-broker/${PROJECT}.sock}" +ENV_FILE="${REMO_SECRETS_ENV_FILE:-$HOME/.cache/remo-secrets/${PROJECT}.env}" +TMP_ROOT="${REMO_SECRETS_TMP_ROOT:-/dev/shm/remo-secrets/${PROJECT}}" +TIMEOUT_SECONDS="${REMO_FETCH_SECRETS_TIMEOUT_SECONDS:-15}" +DEADLINE=$((SECONDS + TIMEOUT_SECONDS)) + +mkdir -p "$(dirname "$ENV_FILE")" + +while true; do + set +e + python3 - "$PROJECT" "$MANIFEST" "$BROKER_SOCKET" "$ENV_FILE" "$TMP_ROOT" <<'PY' +from __future__ import annotations + +import base64 +import json +import os +import re +import shlex +import socket +import stat +import sys +import tomllib +from pathlib import Path + +project = sys.argv[1] +manifest_path = Path(sys.argv[2]) +socket_path = sys.argv[3] +env_file = Path(sys.argv[4]) +tmp_root = Path(sys.argv[5]) + +if not manifest_path.exists(): + raise SystemExit(f"manifest not found: {manifest_path}") + +data = tomllib.loads(manifest_path.read_text()) +if data.get("schema_version") != 1: + raise SystemExit("unsupported manifest schema version") +if data.get("project") != project: + raise SystemExit( + f"manifest project mismatch: expected {project}, got {data.get('project')}" + ) + +secrets = data.get("secrets", {}) +if not isinstance(secrets, dict): + raise SystemExit("manifest secrets must be a table") + +tmp_root.mkdir(parents=True, exist_ok=True) +os.chmod(tmp_root, stat.S_IRWXU) +env_lines: list[str] = [] + + +def broker_ping() -> None: + payload = json.dumps({"op": "ping", "project": project}).encode() + b"\n" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.connect(socket_path) + client.sendall(payload) + response = client.recv(65536) + ping = json.loads(response.decode().strip()) + if not ping.get("ok"): + raise RuntimeError(ping.get("message") or ping.get("code") or "broker ping failed") + if ping.get("protocol_version") != 2: + print( + f"incompatible broker protocol {ping.get('protocol_version')}, expected 2", + file=sys.stderr, + ) + raise SystemExit(3) + +def broker_get(secret_name: str) -> str: + request = json.dumps({"op": "get", "name": secret_name}).encode() + b"\n" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.connect(socket_path) + client.sendall(request) + response = client.recv(65536) + payload = json.loads(response.decode().strip()) + if not payload.get("ok"): + raise RuntimeError(payload.get("message") or payload.get("code") or "broker get failed") + if "value_b64" in payload: + return base64.b64decode(payload["value_b64"]).decode() + return payload["value"] + +def render_template(template: str, values: dict[str, str]) -> str: + rendered = template + for match in re.findall(r"\{\{\s*([a-zA-Z0-9_]+)\s*\}\}", template): + if match not in values: + raise RuntimeError(f"missing template field: {match}") + rendered = re.sub(r"\{\{\s*" + re.escape(match) + r"\s*\}\}", values[match], rendered) + return rendered + + +def materialize_file(target: Path, contents: str, mode: str) -> Path: + managed_root = tmp_root / "files" + managed_target = managed_root / target.as_posix().lstrip("/") + managed_target.parent.mkdir(parents=True, exist_ok=True) + managed_target.write_text(contents) + os.chmod(managed_target, int(mode, 8)) + + target.parent.mkdir(parents=True, exist_ok=True) + if target.is_symlink() or target.exists(): + target.unlink() + target.symlink_to(managed_target) + return managed_target + +broker_ping() +for secret_name, binding in secrets.items(): + if not isinstance(binding, dict): + raise SystemExit(f"secret binding for {secret_name} must be a table") + raw_value = broker_get(secret_name) + fetch_as = binding.get("fetch_as", "env") + + if fetch_as == "env": + env_var = binding.get("env_var") or secret_name.upper() + env_lines.append(f"export {env_var}={shlex.quote(raw_value)}") + continue + + if fetch_as != "file": + raise SystemExit(f"unsupported fetch_as for {secret_name}: {fetch_as}") + + file_path = binding.get("file_path") + file_mode = binding.get("file_mode") + template = binding.get("template") + if not file_path or not file_mode or not template: + raise SystemExit(f"file binding for {secret_name} is missing required fields") + + target = Path(os.path.expanduser(str(file_path))) + if not target.is_absolute(): + target = Path.home() / target + + try: + parsed_value = json.loads(raw_value) + except json.JSONDecodeError: + parsed_value = {"value": raw_value, secret_name: raw_value} + else: + if isinstance(parsed_value, str): + parsed_value = {"value": parsed_value, secret_name: parsed_value} + elif not isinstance(parsed_value, dict): + raise RuntimeError(f"unsupported structured payload for {secret_name}") + else: + parsed_value = {str(k): str(v) for k, v in parsed_value.items()} + + rendered = render_template(str(template), parsed_value) + materialize_file(target, rendered, str(file_mode)) + env_lines.append(f"export REMO_SECRET_FILE_{secret_name.upper()}={shlex.quote(str(target))}") + +env_file.write_text("\n".join(env_lines) + ("\n" if env_lines else "")) +os.chmod(env_file, stat.S_IRUSR | stat.S_IWUSR) +PY + _fetch_rc=$? + set -e + + if [ "$_fetch_rc" -eq 0 ]; then + break + fi + + if [ "$_fetch_rc" -eq 3 ]; then + exit 1 + fi + + if [ "$SECONDS" -ge "$DEADLINE" ]; then + echo "remo-fetch-secrets: required secrets for $PROJECT stayed unavailable for ${TIMEOUT_SECONDS} seconds" >&2 + exit 1 + fi + + sleep 1 +done + +echo "source \"$ENV_FILE\"" diff --git a/ansible/roles/user_setup/tasks/main.yml b/ansible/roles/user_setup/tasks/main.yml index 74d3323..c690ace 100644 --- a/ansible/roles/user_setup/tasks/main.yml +++ b/ansible/roles/user_setup/tasks/main.yml @@ -206,6 +206,8 @@ - .config/opencode - .config/gh - .coderabbit + - .cache/devcontainer-hashes + - .cache/remo-devcontainer-configs - .local/bin - name: Create projects directory @@ -296,6 +298,7 @@ "$_project_dir/.devcontainer.json" \ "$_project_dir/.devcontainer/Dockerfile" \ "$_project_dir/.devcontainer/docker-compose.yml" \ + "$HOME/.local/share/remo-secrets/feature-devcontainer.json" \ 2>/dev/null | sha256sum | cut -d' ' -f1) # Compare with stored hash @@ -314,9 +317,14 @@ _rebuild_flag="--remove-existing-container" rm -f "$_project_dir/.devcontainer-rebuild" fi - + + _config_args=() + if [[ -n "${REMO_DEVCONTAINER_CONFIG:-}" ]]; then + _config_args=(--config "$REMO_DEVCONTAINER_CONFIG") + fi + echo "Starting devcontainer for $ZELLIJ_SESSION_NAME..." - if ! devcontainer up --workspace-folder "$_project_dir" $_rebuild_flag; then + if ! devcontainer up --workspace-folder "$_project_dir" "${_config_args[@]}" $_rebuild_flag; then echo "" echo "devcontainer up failed. [Press any key to continue]" read -n 1 -s -r @@ -330,13 +338,34 @@ echo "Entering devcontainer (exit to return to host shell)..." echo "" - # Run devcontainer shell (not exec, so we can stop container after) - if ! devcontainer exec --workspace-folder "$_project_dir" /bin/bash -c \ - 'if command -v zsh &> /dev/null; then exec zsh; else exec bash; fi'; then + if [[ "${REMO_DEVCONTAINER_SECRETS:-0}" == "1" ]]; then + # Secrets must be vended before the user's shell or command + # starts inside the devcontainer. We keep the original command + # in an env var so quoting survives the host-side handoff. + if [[ "${REMO_DEVCONTAINER_EXEC_MODE:-shell}" == "command" ]]; then + _exec_cmd='set -euo pipefail; eval "$(remo-fetch-secrets "$REMO_PROJECT")"; exec bash -lc "$REMO_DEVCONTAINER_STARTUP_CMD"' + else + _exec_cmd='set -euo pipefail; eval "$(remo-fetch-secrets "$REMO_PROJECT")"; if command -v zsh &> /dev/null; then exec zsh -l; else exec bash -l; fi' + fi + else + if [[ -n "${REMO_DEVCONTAINER_STARTUP_CMD:-}" ]]; then + _exec_cmd='exec bash -lc "$REMO_DEVCONTAINER_STARTUP_CMD"' + else + _exec_cmd='if command -v zsh &> /dev/null; then exec zsh -l; else exec bash -l; fi' + fi + fi + if ! devcontainer exec --workspace-folder "$_project_dir" \ + "${_config_args[@]}" \ + env REMO_INSTANCE="${REMO_INSTANCE:-$(hostname)}" \ + REMO_PROJECT="$ZELLIJ_SESSION_NAME" \ + REMO_DEVCONTAINER_SECRETS="${REMO_DEVCONTAINER_SECRETS:-0}" \ + REMO_DEVCONTAINER_STARTUP_CMD="${REMO_DEVCONTAINER_STARTUP_CMD:-}" \ + /bin/bash -lc "$_exec_cmd"; then echo "" echo "devcontainer exec failed. [Press any key to continue]" read -n 1 -s -r fi + unset _exec_cmd # After exiting devcontainer, stop the container to free resources echo "" @@ -349,7 +378,7 @@ # Exit the zellij session to return to project menu exit 0 fi - unset _project_dir _hash_dir _hash_file _rebuild_flag _current_hash _stored_hash _container_id + unset _project_dir _hash_dir _hash_file _rebuild_flag _current_hash _stored_hash _container_id _config_args fi create: true owner: "{{ remo_user }}" @@ -393,6 +422,14 @@ group: "{{ remo_user }}" mode: '0755' +- name: Install project-launch script + ansible.builtin.template: + src: project-launch.sh.j2 + dest: "/home/{{ remo_user }}/.local/bin/project-launch" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0755' + - name: Verify remo user exists ansible.builtin.command: id {{ remo_user }} register: user_setup_user_info diff --git a/ansible/roles/user_setup/templates/devshell.sh.j2 b/ansible/roles/user_setup/templates/devshell.sh.j2 index b0f7565..66407ab 100644 --- a/ansible/roles/user_setup/templates/devshell.sh.j2 +++ b/ansible/roles/user_setup/templates/devshell.sh.j2 @@ -16,6 +16,7 @@ show_usage() { echo "Commands:" echo " devshell Start/attach to devcontainer for " echo " devshell clone Clone repo and start devcontainer" + echo " devshell _remo-vault Open the managed credential sidecar" echo "" echo "Available projects in $PROJECTS_DIR:" if [ -d "$PROJECTS_DIR" ] && [ "$(ls -A "$PROJECTS_DIR" 2>/dev/null)" ]; then diff --git a/ansible/roles/user_setup/templates/project-launch.sh.j2 b/ansible/roles/user_setup/templates/project-launch.sh.j2 new file mode 100644 index 0000000..42b85d9 --- /dev/null +++ b/ansible/roles/user_setup/templates/project-launch.sh.j2 @@ -0,0 +1,252 @@ +#!/bin/bash +# project-launch - Launch a project session non-interactively. +# Managed by Ansible - do not edit manually +# +# Usage: +# project-launch --project NAME [--detach] [--exec SHELL_COMMAND] +# +# SHELL_COMMAND is a single string run via `bash -lc`, so variable +# expansion, pipes, `&&`, etc. all work as written. +# +# Modes: +# Interactive (default): +# Acts like "pick NAME in project-menu". Drops into the project's +# zellij session; the existing .bashrc DEVCONTAINER block brings +# up the devcontainer and execs the user's shell (or SHELL_COMMAND +# when --exec is supplied — passed via REMO_DEVCONTAINER_CMD). +# +# Detached (--detach): +# Requires --exec. Brings up the devcontainer (or runs on the +# host project dir if no .devcontainer), launches the command in +# the background via nohup+setsid + `bash -lc`, captures +# stdout/stderr to ~/.local/state/remo/.log, and +# returns immediately. +# +# Environment passed to SHELL_COMMAND (both modes): +# REMO_INSTANCE - hostname of the remo instance +# REMO_PROJECT - the project name + +set -euo pipefail + +PROJECTS_DIR="{{ dev_workspace_dir }}" +MANAGED_VAULT_PROJECT="_remo-vault" + +PROJECT="" +DETACH=false +EXEC_CMD="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --project) + PROJECT="$2" + shift 2 + ;; + --detach) + DETACH=true + shift + ;; + --exec) + EXEC_CMD="$2" + shift 2 + ;; + -h|--help) + sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) + echo "project-launch: unknown argument: $1" >&2 + exit 2 + ;; + esac +done + +if [[ -z "$PROJECT" ]]; then + echo "project-launch: --project NAME is required" >&2 + exit 2 +fi + +if [[ "$PROJECT" != "$MANAGED_VAULT_PROJECT" ]] && [[ ! "$PROJECT" =~ ^[a-zA-Z0-9][a-zA-Z0-9._/-]*$ ]]; then + echo "project-launch: invalid project name: $PROJECT" >&2 + exit 2 +fi + +PROJECT_DIR="$PROJECTS_DIR/$PROJECT" +if [[ ! -d "$PROJECT_DIR" ]]; then + echo "project-launch: project not found: $PROJECT_DIR" >&2 + exit 1 +fi + +HAS_DC=false +if [[ -d "$PROJECT_DIR/.devcontainer" ]] || [[ -f "$PROJECT_DIR/.devcontainer.json" ]]; then + HAS_DC=true +fi + +prepare_devcontainer_config() { + local project_dir="$1" + local project_name="$2" + local base_config="" + local feature_config="$HOME/.local/share/remo-secrets/feature-devcontainer.json" + local generated_dir="$HOME/.cache/remo-devcontainer-configs" + local generated_config="$generated_dir/$project_name.json" + + if [[ -f "$project_dir/.devcontainer/devcontainer.json" ]]; then + base_config="$project_dir/.devcontainer/devcontainer.json" + elif [[ -f "$project_dir/.devcontainer.json" ]]; then + base_config="$project_dir/.devcontainer.json" + else + echo "project-launch: no devcontainer.json found for $project_name" >&2 + return 1 + fi + + if [[ ! -f "$feature_config" ]]; then + echo "project-launch: missing remo secrets feature config: $feature_config" >&2 + return 1 + fi + + mkdir -p "$generated_dir" + python3 - "$base_config" "$feature_config" "$generated_config" <<'PY' +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +def merge(base: object, feature: object) -> object: + if isinstance(base, dict) and isinstance(feature, dict): + merged = dict(base) + for key, value in feature.items(): + if key in {"mounts", "runArgs", "features"}: + existing = merged.get(key, [] if key != "features" else {}) + if isinstance(existing, list) and isinstance(value, list): + combined: list[object] = [] + for item in [*existing, *value]: + if item not in combined: + combined.append(item) + merged[key] = combined + else: + merged[key] = merge(existing, value) + elif key in merged: + merged[key] = merge(merged[key], value) + else: + merged[key] = value + return merged + if isinstance(base, list) and isinstance(feature, list): + combined: list[object] = [] + for item in [*base, *feature]: + if item not in combined: + combined.append(item) + return combined + return feature + + +base = json.loads(Path(sys.argv[1]).read_text()) +feature = json.loads(Path(sys.argv[2]).read_text()) +merged = merge(base, feature) +Path(sys.argv[3]).write_text(json.dumps(merged, indent=2) + "\n") +PY + printf '%s\n' "$generated_config" +} + +REMO_INSTANCE="$(hostname)" +export REMO_INSTANCE +export REMO_PROJECT="$PROJECT" + +# --------------------------------------------------------------------------- +# Detached mode: no zellij, no interactive shell. Run EXEC_CMD in background. +# --------------------------------------------------------------------------- +if $DETACH; then + if [[ -z "$EXEC_CMD" ]]; then + echo "project-launch: --detach requires --exec SHELL_COMMAND" >&2 + exit 2 + fi + + LOG_DIR="$HOME/.local/state/remo" + mkdir -p "$LOG_DIR" + LOG_FILE="$LOG_DIR/$PROJECT.log" + + cd "$PROJECT_DIR" + + if $HAS_DC && [[ "$PROJECT" != "$MANAGED_VAULT_PROJECT" ]]; then + REMO_DEVCONTAINER_CONFIG="$(prepare_devcontainer_config "$PROJECT_DIR" "$PROJECT")" + export REMO_DEVCONTAINER_CONFIG + echo "Bringing up devcontainer for $PROJECT..." + devcontainer up --workspace-folder "$PROJECT_DIR" --config "$REMO_DEVCONTAINER_CONFIG" >/dev/null + printf -- '----- %s detached launch: %s -----\n' "$(date -Iseconds)" "$EXEC_CMD" >>"$LOG_FILE" + nohup setsid env REMO_STARTUP_CMD="$EXEC_CMD" \ + devcontainer exec --workspace-folder "$PROJECT_DIR" \ + --config "$REMO_DEVCONTAINER_CONFIG" \ + env REMO_INSTANCE="$REMO_INSTANCE" REMO_PROJECT="$PROJECT" \ + REMO_STARTUP_CMD="$EXEC_CMD" \ + /bin/bash -lc 'set -euo pipefail; eval "$(remo-fetch-secrets "$REMO_PROJECT")"; exec bash -lc "$REMO_STARTUP_CMD"' >>"$LOG_FILE" 2>&1 /dev/null + printf -- '----- %s detached launch: %s -----\n' "$(date -Iseconds)" "$EXEC_CMD" >>"$LOG_FILE" + nohup setsid env REMO_STARTUP_CMD="$EXEC_CMD" \ + devcontainer exec --workspace-folder "$PROJECT_DIR" \ + env REMO_INSTANCE="$REMO_INSTANCE" REMO_PROJECT="$PROJECT" \ + REMO_STARTUP_CMD="$EXEC_CMD" \ + /bin/bash -lc 'set -euo pipefail; exec bash -lc "$REMO_STARTUP_CMD"' >>"$LOG_FILE" 2>&1 >"$LOG_FILE" + nohup setsid env REMO_INSTANCE="$REMO_INSTANCE" REMO_PROJECT="$PROJECT" \ + bash -lc "$EXEC_CMD" >>"$LOG_FILE" 2>&1 /dev/null | sed 's/\x1b\[[0-9;]*m//g' | grep -q "^$PROJECT.*EXITED"; then + zellij delete-session "$PROJECT" 2>/dev/null || true +fi + +exec zellij attach --create "$PROJECT" diff --git a/ansible/roles/user_setup/templates/project-menu.sh.j2 b/ansible/roles/user_setup/templates/project-menu.sh.j2 index c8a83a1..0cf31b3 100644 --- a/ansible/roles/user_setup/templates/project-menu.sh.j2 +++ b/ansible/roles/user_setup/templates/project-menu.sh.j2 @@ -5,6 +5,7 @@ set -e PROJECTS_DIR="{{ dev_workspace_dir }}" +MANAGED_VAULT_PROJECT="_remo-vault" {% if remo_version %} REMO_VERSION="{{ remo_version }}" {% else %} @@ -34,7 +35,18 @@ cleanup_exited_sessions() { # Get list of project directories get_project_dirs() { if [ -d "$PROJECTS_DIR" ]; then - find "$PROJECTS_DIR" -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort + local dirs=() + mapfile -t dirs < <(find "$PROJECTS_DIR" -maxdepth 1 -mindepth 1 -type d -exec basename {} \; 2>/dev/null | sort) + for dir in "${dirs[@]}"; do + if [ "$dir" = "$MANAGED_VAULT_PROJECT" ]; then + printf '%s\n' "$dir" + fi + done + for dir in "${dirs[@]}"; do + if [ "$dir" != "$MANAGED_VAULT_PROJECT" ]; then + printf '%s\n' "$dir" + fi + done fi } @@ -284,6 +296,12 @@ handle_delete() { echo "" + if [ "$project_name" = "$MANAGED_VAULT_PROJECT" ]; then + printf '\033[33m %s is managed by remo and cannot be deleted from project-menu.\033[0m\n' "$MANAGED_VAULT_PROJECT" + sleep 2 + return 1 + fi + if [ ! -d "$project_dir" ]; then echo " Not a project directory, skipping." sleep 1 diff --git a/ansible/roles/vault_devcontainer/README.md b/ansible/roles/vault_devcontainer/README.md new file mode 100644 index 0000000..5e60f87 --- /dev/null +++ b/ansible/roles/vault_devcontainer/README.md @@ -0,0 +1,4 @@ +# vault_devcontainer + +Creates the managed `_remo-vault` workspace and its supporting devcontainer, +watcher script, and helper commands. diff --git a/ansible/roles/vault_devcontainer/tasks/main.yml b/ansible/roles/vault_devcontainer/tasks/main.yml new file mode 100644 index 0000000..73f7262 --- /dev/null +++ b/ansible/roles/vault_devcontainer/tasks/main.yml @@ -0,0 +1,125 @@ +--- +- name: Create managed vault workspace directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0755' + loop: + - "{{ dev_workspace_dir }}/_remo-vault" + - "{{ dev_workspace_dir }}/_remo-vault/.devcontainer" + - "{{ dev_workspace_dir }}/_remo-vault/.remo" + - "{{ dev_workspace_dir }}/_remo-vault/.local" + - "{{ dev_workspace_dir }}/_remo-vault/.local/bin" + +- name: Install managed vault manifest + ansible.builtin.copy: + dest: "{{ dev_workspace_dir }}/_remo-vault/.remo/manifest.toml" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0644' + content: | + schema_version = 1 + project = "_remo-vault" + + [secrets.gh] + fetch_as = "env" + env_var = "GH_TOKEN" + + [secrets.aws] + fetch_as = "file" + file_path = "~/.aws/credentials" + file_mode = "0600" + template = """ + [default] + aws_access_key_id={{aws_access_key_id}} + aws_secret_access_key={{aws_secret_access_key}} + """ + + [cache] + default_ttl_seconds = 900 + default_max_entries = 50 + +- name: Install managed vault devcontainer Dockerfile + ansible.builtin.template: + src: Dockerfile.j2 + dest: "{{ dev_workspace_dir }}/_remo-vault/.devcontainer/Dockerfile" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0644' + +- name: Install managed vault compose definition + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ dev_workspace_dir }}/_remo-vault/.devcontainer/docker-compose.yml" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0644' + +- name: Install managed vault devcontainer definition + ansible.builtin.template: + src: devcontainer.json.j2 + dest: "{{ dev_workspace_dir }}/_remo-vault/.devcontainer/devcontainer.json" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0644' + +- name: Install managed vault helper scripts + ansible.builtin.template: + src: "{{ item.src }}" + dest: "/home/{{ remo_user }}/.local/bin/{{ item.dest }}" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0755' + loop: + - src: remo-list-creds.sh.j2 + dest: remo-list-creds + - src: remo-reload.sh.j2 + dest: remo-reload + - src: remo-test-project.sh.j2 + dest: remo-test-project + - src: remo-vend-status.sh.j2 + dest: remo-vend-status + - src: remo-vault-watcher.sh.j2 + dest: remo-vault-watcher + +- name: Verify managed vault helper scripts are installed + ansible.builtin.command: + cmd: >- + /bin/bash -lc + 'test -x "/home/{{ remo_user }}/.local/bin/remo-list-creds" && + test -x "/home/{{ remo_user }}/.local/bin/remo-reload" && + test -x "/home/{{ remo_user }}/.local/bin/remo-test-project" && + test -x "/home/{{ remo_user }}/.local/bin/remo-vend-status" && + test -x "/home/{{ remo_user }}/.local/bin/remo-vault-watcher" && + test -f "{{ dev_workspace_dir }}/_remo-vault/.devcontainer/devcontainer.json"' + register: vault_devcontainer_check + changed_when: false + failed_when: false + +- name: Fail when managed vault assets are incomplete + ansible.builtin.fail: + msg: >- + Managed _remo-vault provisioning is incomplete. Expected helper scripts + and devcontainer assets were not fully installed for {{ remo_user }}. + when: vault_devcontainer_check.rc | default(1) != 0 + +- name: Install managed vault MOTD + ansible.builtin.template: + src: motd.j2 + dest: "{{ dev_workspace_dir }}/_remo-vault/.remo/motd" + owner: "{{ remo_user }}" + group: "{{ remo_user }}" + mode: '0644' + +- name: Create vault state directories on the host + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: '0755' + loop: + - /var/lib/remo-vault + - /var/lib/remo-vault/watch diff --git a/ansible/roles/vault_devcontainer/templates/Dockerfile.j2 b/ansible/roles/vault_devcontainer/templates/Dockerfile.j2 new file mode 100644 index 0000000..7e551ac --- /dev/null +++ b/ansible/roles/vault_devcontainer/templates/Dockerfile.j2 @@ -0,0 +1,10 @@ +FROM mcr.microsoft.com/devcontainers/base:ubuntu + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + jq \ + python3 \ + && rm -rf /var/lib/apt/lists/* diff --git a/ansible/roles/vault_devcontainer/templates/devcontainer.json.j2 b/ansible/roles/vault_devcontainer/templates/devcontainer.json.j2 new file mode 100644 index 0000000..a42a12f --- /dev/null +++ b/ansible/roles/vault_devcontainer/templates/devcontainer.json.j2 @@ -0,0 +1,21 @@ +{ + "name": "remo-vault", + "dockerComposeFile": "docker-compose.yml", + "service": "remo-vault", + "remoteUser": "vscode", + "workspaceFolder": "/workspace", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind", + "mounts": [ + "source={{ dev_workspace_dir }}/_remo-vault/.remo/manifest.toml,target=/workspace/.remo/manifest.toml,type=bind,consistency=cached,readonly", + "source=/run/remo-broker/admin.sock,target=/run/remo-broker/admin.sock,type=bind", + "source=remo-vault-fnox,target=/var/lib/remo-vault,type=volume" + ], + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.defaultProfile.linux": "bash" + } + } + }, + "postAttachCommand": "cat /workspace/.remo/motd 2>/dev/null || true" +} diff --git a/ansible/roles/vault_devcontainer/templates/docker-compose.yml.j2 b/ansible/roles/vault_devcontainer/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..4d03029 --- /dev/null +++ b/ansible/roles/vault_devcontainer/templates/docker-compose.yml.j2 @@ -0,0 +1,15 @@ +services: + remo-vault: + build: + context: . + dockerfile: Dockerfile + volumes: + - remo-vault-fnox:/var/lib/remo-vault + - /run/remo-broker/admin.sock:/run/remo-broker/admin.sock + environment: + REMO_BROKER_ADMIN_SOCKET: /run/remo-broker/admin.sock + REMO_PROJECT: _remo-vault + +volumes: + remo-vault-fnox: + name: remo-vault-fnox diff --git a/ansible/roles/vault_devcontainer/templates/motd.j2 b/ansible/roles/vault_devcontainer/templates/motd.j2 new file mode 100644 index 0000000..309fe5e --- /dev/null +++ b/ansible/roles/vault_devcontainer/templates/motd.j2 @@ -0,0 +1,9 @@ +_remo-vault is the managed credential sidecar for this remo instance. + +Common commands: + remo-list-creds + remo-vend-status + remo-reload + remo-test-project + +Store credentials here, then reload project manifests from the host-side copy. diff --git a/ansible/roles/vault_devcontainer/templates/remo-list-creds.sh.j2 b/ansible/roles/vault_devcontainer/templates/remo-list-creds.sh.j2 new file mode 100644 index 0000000..f9f9217 --- /dev/null +++ b/ansible/roles/vault_devcontainer/templates/remo-list-creds.sh.j2 @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail + +if ! command -v fnox >/dev/null 2>&1; then + echo "remo-list-creds: fnox is not installed on this host" >&2 + exit 1 +fi + +exec fnox list diff --git a/ansible/roles/vault_devcontainer/templates/remo-reload.sh.j2 b/ansible/roles/vault_devcontainer/templates/remo-reload.sh.j2 new file mode 100644 index 0000000..f434637 --- /dev/null +++ b/ansible/roles/vault_devcontainer/templates/remo-reload.sh.j2 @@ -0,0 +1,36 @@ +#!/bin/bash +set -euo pipefail + +if [ $# -ne 1 ]; then + echo "Usage: remo-reload " >&2 + exit 2 +fi + +PROJECT="$1" +ADMIN_SOCKET="${REMO_BROKER_ADMIN_SOCKET:-/run/remo-broker/admin.sock}" + +python3 - "$ADMIN_SOCKET" "$PROJECT" <<'PY' +from __future__ import annotations + +import json +import socket +import sys + +sock_path = sys.argv[1] +project = sys.argv[2] + +payload = json.dumps({"op": "reload", "name": project}).encode() + b"\n" + +with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.connect(sock_path) + client.sendall(payload) + response = client.recv(65536) + +data = json.loads(response.decode().strip()) +if not data.get("ok"): + raise SystemExit(f"remo-reload: broker returned error: {data}") + +print(f"reloaded {project}") +print(f"allowlist: {', '.join(data.get('allowlist', [])) or '(empty)'}") +print("manifest_source: host-side project manifest") +PY diff --git a/ansible/roles/vault_devcontainer/templates/remo-test-project.sh.j2 b/ansible/roles/vault_devcontainer/templates/remo-test-project.sh.j2 new file mode 100644 index 0000000..bfd1b94 --- /dev/null +++ b/ansible/roles/vault_devcontainer/templates/remo-test-project.sh.j2 @@ -0,0 +1,18 @@ +#!/bin/bash +set -euo pipefail + +if [ $# -ne 1 ]; then + echo "Usage: remo-test-project " >&2 + exit 2 +fi + +PROJECT="$1" +AUDIT_LOG="${REMO_BROKER_AUDIT_LOG:-/var/log/remo-broker/audit.log}" + +echo "Testing manifest and broker access for $PROJECT..." +remo-reload "$PROJECT" +if ! REMO_PROJECT="$PROJECT" remo-fetch-secrets "$PROJECT"; then + echo "remo-test-project: secret vending failed for $PROJECT" >&2 + echo "Check broker audit log: $AUDIT_LOG" >&2 + exit 1 +fi diff --git a/ansible/roles/vault_devcontainer/templates/remo-vault-watcher.sh.j2 b/ansible/roles/vault_devcontainer/templates/remo-vault-watcher.sh.j2 new file mode 100644 index 0000000..d5ff1e1 --- /dev/null +++ b/ansible/roles/vault_devcontainer/templates/remo-vault-watcher.sh.j2 @@ -0,0 +1,51 @@ +#!/bin/bash +set -euo pipefail + +ADMIN_SOCKET="${REMO_BROKER_ADMIN_SOCKET:-/run/remo-broker/admin.sock}" +STATE_FILE="${REMO_VAULT_STATE_FILE:-/var/lib/remo-vault/watch/push-creds.json}" + +if [ ! -s "$STATE_FILE" ]; then + echo "remo-vault-watcher: no staged credential snapshot at $STATE_FILE" >&2 + exit 1 +fi + +python3 - "$ADMIN_SOCKET" "$STATE_FILE" <<'PY' +from __future__ import annotations + +import json +import socket +import sys +from pathlib import Path + +sock_path = sys.argv[1] +state_file = Path(sys.argv[2]) + + +def exchange(payload: dict[str, object]) -> dict[str, object]: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.connect(sock_path) + client.sendall(json.dumps(payload).encode() + b"\n") + response = client.recv(65536) + return json.loads(response.decode().strip()) + + +status = exchange({"op": "status"}) +if not status.get("ok"): + raise SystemExit(f"remo-vault-watcher: broker returned status error: {status}") +if status.get("protocol_version") != 2: + raise SystemExit( + "remo-vault-watcher: incompatible broker protocol " + f"{status.get('protocol_version')}, expected 2" + ) + +payload = {"op": "push-creds", "secrets": json.loads(state_file.read_text())} + +data = exchange(payload) +if not data.get("ok"): + raise SystemExit(f"remo-vault-watcher: broker returned error: {data}") + +print( + f"pushed {data.get('secret_count', 0)} secret(s) at " + f"{data.get('loaded_at', 'unknown time')}" +) +PY diff --git a/ansible/roles/vault_devcontainer/templates/remo-vend-status.sh.j2 b/ansible/roles/vault_devcontainer/templates/remo-vend-status.sh.j2 new file mode 100644 index 0000000..f757235 --- /dev/null +++ b/ansible/roles/vault_devcontainer/templates/remo-vend-status.sh.j2 @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail + +ADMIN_SOCKET="${REMO_BROKER_ADMIN_SOCKET:-/run/remo-broker/admin.sock}" +AUDIT_LOG="${REMO_BROKER_AUDIT_LOG:-/var/log/remo-broker/audit.log}" + +python3 - "$ADMIN_SOCKET" <<'PY' +from __future__ import annotations + +import json +import socket +import sys + +sock_path = sys.argv[1] +payload = json.dumps({"op": "status"}).encode() + b"\n" + +with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.connect(sock_path) + client.sendall(payload) + response = client.recv(65536) + +data = json.loads(response.decode().strip()) +if not data.get("ok"): + raise SystemExit(f"remo-vend-status: broker returned error: {data}") +if data.get("protocol_version") != 2: + raise SystemExit( + f"remo-vend-status: incompatible broker protocol {data.get('protocol_version')}, expected 2" + ) + +print(f"protocol_version: {data['protocol_version']}") +print(f"secret_count: {data.get('secret_count', 0)}") +print(f"secrets_loaded_at: {data.get('secrets_loaded_at', 'n/a')}") +print(f"audit_log: {AUDIT_LOG}") +PY diff --git a/ansible/tasks/configure_dev_tools.yml b/ansible/tasks/configure_dev_tools.yml index b9aad15..be2f203 100644 --- a/ansible/tasks/configure_dev_tools.yml +++ b/ansible/tasks/configure_dev_tools.yml @@ -61,6 +61,18 @@ name: devcontainers when: configure_devcontainers | default(true) | bool +- name: Configure credential broker host assets + ansible.builtin.include_role: + name: remo_broker + +- name: Install managed vault devcontainer + ansible.builtin.include_role: + name: vault_devcontainer + +- name: Install project secrets feature assets + ansible.builtin.include_role: + name: remo_secrets_feature + - name: Install GitHub CLI ansible.builtin.include_role: name: github_cli diff --git a/docs/aws.md b/docs/aws.md index 3ff810b..fe1520e 100644 --- a/docs/aws.md +++ b/docs/aws.md @@ -25,6 +25,9 @@ remo aws create # Connect remo shell + +# Open the managed credential sidecar +remo shell -p _remo-vault ``` ## Configuration @@ -116,6 +119,8 @@ remo aws snapshot delete [-y] # Remove snapshot Available tools: `docker`, `user_setup`, `nodejs`, `devcontainers`, `github_cli`, `fzf`, `zellij` +`remo aws create` and `remo aws update` always reconcile the managed broker service and `_remo-vault` sidecar. `--only` and `--skip` narrow the optional tool bundle only. + ### Destroy Options | Option | Description | @@ -166,6 +171,7 @@ Available tools: `docker`, `user_setup`, `nodejs`, `devcontainers`, `github_cli` | **Spot Instances** | Optional spot pricing for ~70% cost savings | | **Multi-user** | Resources namespaced by `--name` for shared AWS accounts | | **Patch Manager** | Automatic OS security patching via AWS SSM Patch Manager — daily scan, weekly install (Sunday 4 AM UTC) with auto-reboot | +| **Managed `_remo-vault` sidecar** | Broker admin workspace for `remo-vend-status`, `remo-list-creds`, and manifest reloads | ## Instance Types diff --git a/docs/hetzner.md b/docs/hetzner.md index 860d675..4e971b4 100644 --- a/docs/hetzner.md +++ b/docs/hetzner.md @@ -22,6 +22,9 @@ remo hetzner create # Connect remo shell + +# Open the managed credential sidecar +remo shell -p _remo-vault ``` ## Configuration @@ -86,6 +89,8 @@ remo hetzner destroy --yes --remove-volume Available tools: `docker`, `user_setup`, `nodejs`, `devcontainers`, `github_cli`, `fzf`, `zellij` +`remo hetzner create` and `remo hetzner update` always reconcile the managed broker service and `_remo-vault` sidecar. `--only` and `--skip` narrow the optional tool bundle only. + ### Destroy Options | Option | Description | @@ -100,6 +105,7 @@ Available tools: `docker`, `user_setup`, `nodejs`, `devcontainers`, `github_cli` | **Persistent Volume** | `/home/remo` mounted on a separate volume that survives server teardown | | **Strict Firewall** | SSH-only access (port 22) | | **Ubuntu 24.04** | Latest LTS with automatic security updates | +| **Managed `_remo-vault` sidecar** | Broker admin workspace for credential vending and manifest reloads | ## Server Types diff --git a/docs/incus.md b/docs/incus.md index f322559..ffc3b1f 100644 --- a/docs/incus.md +++ b/docs/incus.md @@ -18,6 +18,9 @@ remo incus create dev1 --host incus-host --user youruser # Connect remo shell + +# Open the managed credential sidecar +remo shell -p _remo-vault ``` ## Configuration @@ -96,6 +99,8 @@ remo incus info --name dev1 Available tools: `docker`, `user_setup`, `nodejs`, `devcontainers`, `github_cli`, `fzf`, `zellij` +`remo incus create` and `remo incus update` always reconcile the managed broker service and `_remo-vault` sidecar. `--only` and `--skip` narrow the optional tool bundle only. + ### Destroy Options | Option | Description | @@ -114,6 +119,7 @@ Available tools: `docker`, `user_setup`, `nodejs`, `devcontainers`, `github_cli` | **Hostname DNS** | Works if your router registers DHCP hostnames | | **Host Mounts** | Optional persistent data directories from the Incus host | | **macvlan Network** | Containers appear as separate devices on your LAN | +| **Managed `_remo-vault` sidecar** | Broker admin workspace for credential vending and manifest reloads | ## Bootstrap diff --git a/docs/proxmox.md b/docs/proxmox.md index 336fefa..866cb91 100644 --- a/docs/proxmox.md +++ b/docs/proxmox.md @@ -23,6 +23,9 @@ remo proxmox create --name dev1 --host prox01 --user root # Connect remo shell + +# Open the managed credential sidecar +remo shell -p _remo-vault ``` ## CLI Commands @@ -107,6 +110,8 @@ remo proxmox info --name dev1 Available tools: `docker`, `user_setup`, `nodejs`, `devcontainers`, `github_cli`, `fzf`, `zellij` +`remo proxmox create` and `remo proxmox update` always reconcile the managed broker service and `_remo-vault` sidecar. `--only` and `--skip` narrow the optional tool bundle only. + ### Destroy Options | Option | Description | @@ -125,6 +130,7 @@ Available tools: `docker`, `user_setup`, `nodejs`, `devcontainers`, `github_cli` | **Unprivileged + Nesting** | Default security posture; Docker-in-Docker works out of the box | | **Auto-start on boot** | `--onboot 1` — survives node reboots | | **Same dev tools as Incus/Hetzner** | Docker, Node.js, fzf, github_cli, devcontainers, zellij, user_setup | +| **Managed `_remo-vault` sidecar** | Broker admin workspace for credential vending and manifest reloads | ## Bootstrap diff --git a/docs/remo-fnox-spec.md b/docs/remo-fnox-spec.md new file mode 100644 index 0000000..aff5fa6 --- /dev/null +++ b/docs/remo-fnox-spec.md @@ -0,0 +1,323 @@ +# Feature Specification: Credential Broker + +**Feature Branch**: `005-credential-broker` +**Created**: 2026-05-23 +**Status**: Draft +**Input**: User description: "Defend against malicious-dependency supply-chain attacks by removing long-lived developer credentials from Remo instances. Replace ambient environment variables and dotfile-resident secrets with an on-instance broker process that surfaces narrowly-scoped, allowlisted credentials into each devcontainer via per-project Unix sockets." + +## Background and Motivation + +Recent supply-chain attacks against npm (Shai-Hulud, Mini Shai-Hulud against `@antv`), PyPI, and other ecosystems share a pattern: a malicious dependency's install or postinstall script runs with the developer's full ambient environment and reads `GITHUB_TOKEN`, `NPM_TOKEN`, `AWS_*`, `~/.aws/credentials`, `~/.netrc`, `~/.npmrc`, SSH private keys, and similar. The Remo instance — being where most active dev credentials accumulate — is a high-value target. + +Remo's existing isolation (one instance per developer, separate from the laptop) bounds blast radius, but the *contents* of the instance are exactly the secrets the attacker wants. Every `npm install`, `cargo build`, `pip install` of an untrusted package today runs with full credential access. + +The credential broker design closes this gap by ensuring: + +1. Long-lived developer credentials never live on the Remo instance at rest. +2. The instance fetches credentials on demand from an external backend (1Password, Vault, AWS Secrets Manager, etc.). +3. Each devcontainer sees only the credentials the project it hosts has explicitly declared a need for, enforced by kernel-level namespace separation, not just policy. + +## Current implementation notes + +- The canonical manifest path is `~/projects//.remo/manifest.toml`. +- Project startup generates a temporary merged devcontainer config and passes it with `devcontainer --config`, so checked-in devcontainer files stay unchanged. +- `remo-fetch-secrets` runs before the project shell, `--exec`, or `--detach` command begins and hard-fails on broker protocol mismatches. +- `_remo-vault` remains the reserved sidecar workspace for `remo-vend-status`, `remo-reload`, and `remo-test-project`. + +## Terms and Definitions + +These terms are used consistently throughout this spec and should be adopted in all related code, CLI surface, and documentation. + +| Term | Definition | +|---|---| +| **Backend** (L0) | The external secret store of record — 1Password, HashiCorp Vault, AWS Secrets Manager, Azure Key Vault, GCP Secret Manager, age-encrypted file in a private repo, or an OS keychain. Holds the user's actual credential values. Never directly accessed by devcontainers or untrusted code. | +| **Node** (L1) | A bare-metal or VM-based machine that hosts one or more Remo *instances*. Only applicable to self-hosted providers (Incus, Proxmox). For AWS and Hetzner, the cloud provider owns this layer and Remo does not address it. A node may host multiple instances belonging to one or many users. | +| **Instance** (L2) | The compute resource Remo creates per `remo create`. The SSH target tracked in `known_hosts.yml`. Called *instance* (AWS), *server* (Hetzner), or *container* (Incus, Proxmox) in per-provider CLI labels — but structurally a single concept. The *broker* runs here. | +| **Devcontainer** (L3) | A Docker container launched inside an *instance* by `devcontainer up`, one per *project*, where untrusted dependency code (npm install, etc.) actually executes. Consumes one *project socket*. | +| **Project** (L4) | A directory under `~/projects/` on an instance, usually a git repo, typically containing a `.devcontainer/` config and a *project manifest*. The unit selected in the project menu. | +| **Backend** identity | A credential the *broker* uses to authenticate upward to the backend (e.g., a 1Password Service Account token, Vault AppRole, AWS IAM instance profile). Scoped narrowly to a single *instance*'s allowed secrets. Not the same as the secrets it fetches. | +| **Bootstrap token** | The on-disk form of the backend identity stored on an instance (file at `/etc/remo-broker/bootstrap-token` on the instance). For self-hosted providers, originates on the *node* and is bind-mounted in. For Hetzner, SSH-pushed at create time. For AWS, replaced by an instance profile (no on-disk token). | +| **Broker** | The Remo-owned daemon (`remo-broker`) running on the instance as a systemd unit. Holds the bootstrap token, fetches credentials from the backend (using `fnox` internally as its multi-backend retrieval library), enforces per-project allowlists, and serves *project sockets* to devcontainers. One broker per instance. See "Component Sourcing" below for why this is built rather than adopted. | +| **Project socket** | A Unix domain socket at `/run/remo-broker/.sock` on the instance, one per active project, bind-mounted as `/run/remo-broker/sock` into the project's devcontainer. The broker enforces a per-project allowlist on each. | +| **Project manifest** | `.remo/broker.toml` (auto-synthesized by Remo) or `.devcontainer/remo-broker.toml` (committed to the repo) declaring the set of backend-resolvable secret *names* this project is permitted to fetch. The broker reads this when creating the project socket and uses it as the per-project allowlist. Backend-side mappings (which credential store each name lives in, what backend identity to use) are configured separately at the instance level via the embedded fnox layer's own configuration. | +| **Provisioning credential** | A credential Remo uses to call cloud APIs (HETZNER_API_TOKEN, AWS_ACCESS_KEY_ID, Incus/Proxmox API tokens, 1Password SA admin token). Lives only in the laptop's fnox configuration, fetched on demand per `remo` invocation, never persisted to an instance or node. | +| **User secret** | A credential a project needs at runtime (GITHUB_TOKEN, NPM_TOKEN, runtime AWS keys, ANTHROPIC_API_KEY). Fetched on demand by the broker via the bootstrap token, held in broker memory only, exposed to devcontainers via project sockets. Never written to disk on the instance. | + +### Layer diagram + +``` +L0 Backend + └── 1Password / Vault / AWS Secrets Manager / age+git / keychain + ▲ + │ broker authenticates with bootstrap token + │ +L1 Node (Incus/Proxmox only; absent on AWS/Hetzner) + ├── /var/lib/remo-broker/instance-tokens/ + └── bind-mounted read-only into the instance + │ + ▼ +L2 Instance + ├── /etc/remo-broker/bootstrap-token (file or IMDS-derived) + ├── remo-broker daemon (systemd unit; embeds fnox for backend retrieval) + ├── /run/remo-broker/projA.sock (allowlist: GITHUB_TOKEN, …) + └── /run/remo-broker/projB.sock (allowlist: NPM_TOKEN, …) + │ + ▼ (bind-mounted as /run/remo-broker/sock) +L3 Devcontainer (one per project) + └── tools fetch secrets via the mounted project socket only + │ + ▼ +L4 Project workspace (mounted from ~/projects/) +``` + +For AWS/Hetzner, L1 collapses: there is no Remo-addressable node, and the bootstrap is delivered either via cloud workload identity (AWS instance profile) or SSH push (Hetzner). + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Project creds available in devcontainer, never on the instance OS (Priority: P1) + +A developer SSHes into a Remo instance, picks a project from the menu, and lands in a devcontainer. Inside the devcontainer, `gh auth status` shows them authenticated, `npm publish` finds an NPM_TOKEN, `aws sts get-caller-identity` works. None of those credentials existed on the instance OS before the devcontainer started, and none persist after it stops. A worm in `npm install` running inside that devcontainer can only see the credentials the project's manifest declared a need for — not credentials from other projects, not the bootstrap token, not the developer's full secret backend. + +**Why this priority**: This is the entire point of the feature. Without it, no other behavior matters. + +**Independent Test**: Provision an instance, configure a project manifest declaring `GITHUB_TOKEN`, launch the devcontainer, verify `gh auth status` reports authenticated *and* `cat ~/.config/gh/hosts.yml` shows no on-disk token. From outside the devcontainer (on the instance OS), verify `printenv` shows no GITHUB_TOKEN and `~/.config/gh/` does not exist for the project's user. + +**Acceptance Scenarios**: + +1. **Given** an instance with the broker installed and a project with a manifest declaring `GITHUB_TOKEN`, **When** the user selects that project from the menu, **Then** the launched devcontainer can use `gh` against authenticated GitHub APIs. +2. **Given** the same setup, **When** the user runs `printenv` or inspects `~/.aws/`, `~/.npmrc`, `~/.netrc` on the instance OS (outside any devcontainer), **Then** none of the project's credentials appear. +3. **Given** two projects A and B where A's manifest declares `GITHUB_TOKEN` and B's declares `NPM_TOKEN`, **When** project A's devcontainer requests `NPM_TOKEN` via the broker, **Then** the request is denied. +4. **Given** a devcontainer is running with a project socket mounted, **When** the devcontainer process exits, **Then** the project socket is removed from `/run/remo-broker/` and the broker drops the project's allowlist from memory. + +--- + +### User Story 2 - Multi-device access to the same instance (Priority: P1) + +A developer creates an instance from their laptop, does some work, then later runs `remo shell` from a different machine (a second laptop, a web session, a phone-tethered tablet). The instance is reachable and project credentials work in devcontainers from any device, without re-bootstrapping the instance or re-unlocking anything device-specific. + +**Why this priority**: A broker design that only worked from the laptop that created the instance would fail Remo's core "remote dev environment" promise. The broker lives on the instance precisely to make this work. + +**Independent Test**: Create an instance from device A, run a devcontainer with broker creds successfully. Without reconnecting from device A, run `remo shell` from device B against the same instance, launch the same devcontainer, verify credentials still work. + +**Acceptance Scenarios**: + +1. **Given** an instance was created from device A, **When** the user runs `remo shell` from device B, **Then** the broker is already running and serves credentials to launched devcontainers without intervention. +2. **Given** the instance has been rebooted, **When** the broker comes back up, **Then** it re-reads its bootstrap token, re-authenticates to the backend, and resumes serving without any device needing to connect. +3. **Given** an autonomous Claude Code session was started in a devcontainer before the user disconnected, **When** the developer is offline overnight, **Then** the session continues to access its allowlisted credentials. + +--- + +### User Story 3 - Provisioning credentials never reach the instance (Priority: P1) + +A developer runs `remo hetzner create myproject`. Remo reads the Hetzner API token from the laptop's fnox (not from `HETZNER_API_TOKEN` in the laptop's shell env), uses it to provision the VM, and then forgets it. After provisioning, the Hetzner API token has never been written to the instance, never appeared in cloud-init user-data visible in the Hetzner console, and is not present in the laptop's process environment. + +**Why this priority**: Provisioning credentials are typically the most powerful (can create/destroy any resource in the account) and the easiest to leak. The instance has no need for them. + +**Independent Test**: With `HETZNER_API_TOKEN` *unset* in the laptop shell but stored in the laptop's fnox config, run `remo hetzner create test`, then `ssh remo@test "env | grep -i hetzner"` returns nothing and the Hetzner console's user-data field for the VM contains no token. + +**Acceptance Scenarios**: + +1. **Given** Hetzner API token is stored in the laptop's fnox and *not* exported in the laptop's environment, **When** `remo hetzner create` runs, **Then** provisioning succeeds. +2. **Given** an instance has been created, **When** the user inspects cloud provider metadata/user-data via the provider's console or API, **Then** no provisioning credentials are visible. +3. **Given** the same setup, **When** the user inspects the instance OS, **Then** no provisioning credentials are present in environment, dotfiles, or any disk location. + +--- + +### User Story 4 - Per-project credential allowlist via the manifest (Priority: P2) + +A developer clones a new repo into `~/projects/foo`. Selecting it from the menu either reads the committed `.devcontainer/remo-broker.toml`, or — if absent — synthesizes a default `.remo/broker.toml` declaring only `github_token` (enough for `git push`). The developer can broaden the allowlist by editing the manifest; secrets not in the manifest cannot be fetched, even by tools that know their names. + +**Why this priority**: The allowlist is the policy mechanism that makes the kernel-enforced isolation actually meaningful. Without it, every devcontainer would see every secret. + +**Independent Test**: Place a project with a manifest declaring only `github_token`. Inside the devcontainer, attempt to fetch `npm_token` via the broker socket (`remo-broker get npm_token`) — expect refusal. Add `npm_token` to the manifest, restart the devcontainer, retry — expect success. + +**Acceptance Scenarios**: + +1. **Given** a project with no manifest, **When** the user selects it from the menu, **Then** Remo synthesizes `.remo/broker.toml` with a minimal default allowlist (gitignored) and proceeds. +2. **Given** a project with `.devcontainer/remo-broker.toml` declaring `[mcp] secrets = ["x", "y"]`, **When** the devcontainer requests secret `z`, **Then** the broker returns a denial and logs the attempt. +3. **Given** the manifest is updated, **When** the devcontainer is rebuilt or restarted, **Then** the new allowlist takes effect. + +--- + +### User Story 5 - Bootstrap token rotation and instance destruction revoke access (Priority: P2) + +When `remo destroy` runs against any instance, Remo revokes the bootstrap token at the backend *before* destroying the instance. A long-running rotation policy also periodically replaces each instance's bootstrap token. A token leaked from a destroyed or rotated-away instance has no remaining backend access. + +**Why this priority**: Without revocation, the broker design just relocates the attack surface from "credentials on disk" to "bootstrap tokens that live until manually rotated." Lifecycle integration is what makes the threat model honest. + +**Independent Test**: Create an instance, save a copy of its bootstrap token externally. Destroy the instance. Use the saved token to attempt to fetch a secret — expect failure within the backend's revocation propagation window (typically seconds). + +**Acceptance Scenarios**: + +1. **Given** an instance with an active bootstrap token, **When** `remo destroy` runs, **Then** the token is revoked at the backend before any instance-deletion API call. +2. **Given** a rotation policy is configured, **When** the rotation interval elapses, **Then** a fresh bootstrap token is provisioned to the instance and the previous one is revoked. +3. **Given** rotation fails for any reason, **When** the broker detects auth failures against the backend, **Then** it surfaces an actionable error and continues serving in-memory-cached credentials until they expire. + +--- + +### User Story 6 - Devcontainer auto-synthesis for projects without one (Priority: P2) + +The current project menu launches `devcontainer up` only when a project contains `.devcontainer/devcontainer.json`; otherwise it falls back to a plain shell on the instance OS, defeating the credential boundary. With this feature, projects without a committed devcontainer get an auto-synthesized `.remo/devcontainer.json` based on simple language detection, so *every* project the user selects from the menu lands them in a devcontainer with a broker socket. + +**Why this priority**: The instance-OS fallback is the leak that defeats the rest of the design. Required for correctness. + +**Independent Test**: Clone a repo with no `.devcontainer/` and a `package.json`. Select it from the menu. Expect to land in a Node-based devcontainer with a project socket mounted, not in a shell on the instance OS. + +**Acceptance Scenarios**: + +1. **Given** a project with no devcontainer config and a recognized language marker (`package.json`, `Cargo.toml`, `pyproject.toml`, etc.), **When** the user selects it, **Then** Remo writes `.remo/devcontainer.json` matching the language and launches it. +2. **Given** a project with no language marker, **When** the user selects it, **Then** Remo uses a generic base image and proceeds. +3. **Given** the user explicitly wants a shell on the instance OS, **When** they pick the "exit to host shell" menu option, **Then** they get one — with a one-time warning that no broker is available there. + +--- + +### Edge Cases + +- **Backend unavailable**: broker holds in-memory-cached secrets until they expire, then surfaces clear errors to devcontainers. Does not silently return stale values past expiry. +- **Backend unreachable from instance (network issue)**: broker reports the error via the project socket; tools using the broker see a clear "credential unavailable" error rather than a generic auth failure. +- **Two projects with the same name in different parent dirs**: project socket naming uses the absolute project path's hash (truncated) as suffix to avoid collisions. +- **A devcontainer escape into the instance OS**: attacker gains access to every project socket currently mounted in the instance plus the bootstrap token. Rotation interval and per-instance scoping bound the damage; full-backend access is not possible. +- **A node-level compromise on Incus/Proxmox**: attacker gains every instance's bootstrap token on that node. Threat model treats nodes as critical assets requiring OS hardening separate from instances. +- **A user runs untrusted code on the instance OS rather than in a devcontainer** (via "exit to host shell"): no broker available, so user secrets are simply not present. The escape hatch is safe by absence. +- **A project manifest declares a secret that doesn't exist in the backend**: broker returns a "not found" error to the devcontainer; tool's behavior depends on the tool, but the broker itself does not crash or fall back to other secrets. +- **The user has chosen `age + git` as backend** (no per-instance scoping primitive): `remo init` warns that this backend does not support narrowly-scoped bootstrap tokens; either reject the combination or fall back to laptop-unlock-per-session for Hetzner/Incus/Proxmox. +- **AWS SSM access mode**: bootstrap (instance profile) is metadata-based, no SSH push needed; broker installation still proceeds via the standard `*_configure.yml` flow which already works over SSM. + +## Requirements *(mandatory)* + +### Functional Requirements + +**Backend integration** +- **FR-001**: System MUST support pluggable backends for the credential store, with first-class support for 1Password, HashiCorp Vault, AWS Secrets Manager, and age-encrypted git for the v1 cut. +- **FR-002**: System MUST allow per-installation backend choice via `remo init`, persisted to laptop-side fnox configuration. +- **FR-003**: System MUST warn the user at `init` time when their selected backend lacks per-instance scoping primitives (age + git) and offer a more secure alternative or a clearly-described downgrade. + +**Provisioning credentials** +- **FR-004**: System MUST read provisioning credentials (HETZNER_API_TOKEN, AWS access keys, Incus/Proxmox API tokens) from the laptop's fnox rather than the laptop's shell environment. +- **FR-005**: System MUST NOT write provisioning credentials to any instance's disk, environment, or cloud-init user-data. +- **FR-006**: System MUST replace `lookup('env', '')` patterns in `ansible/group_vars/all.yml` with `lookup('pipe', 'fnox get ...')` invocations against the laptop's fnox. + +**Bootstrap & broker installation** +- **FR-007**: System MUST install the broker (`remo-broker` daemon + systemd unit; Remo-owned, embeds fnox for backend retrieval) on every Remo instance as part of the standard `*_configure.yml` Ansible flow. +- **FR-008**: For AWS, system MUST attach an instance-scoped IAM role at create time and configure the broker to use IMDS for credentials. No bootstrap token shall be written to disk. +- **FR-009**: For Hetzner, system MUST mint an instance-scoped bootstrap token on the laptop, SSH-push it to `/etc/remo-broker/bootstrap-token` (mode 0400, root) after first boot, and not include the token in any cloud-init user-data. +- **FR-010**: For Incus and Proxmox, system MUST mint the bootstrap token on the laptop, place it under `/var/lib/remo-broker/instance-tokens/` on the *node* (not in any instance), and bind-mount it read-only into the instance at `/etc/remo-broker/bootstrap-token`. The node itself MUST NOT inspect or store project information. +- **FR-011**: System MUST register the node (one-time per Incus/Proxmox node) via a new `remo incus add-node` and `remo proxmox add-node` command, installing the token-manager helper. + +**Project sockets and manifests** +- **FR-012**: System MUST discover project manifests in priority order: `.devcontainer/remo-broker.toml` (committed) → `.remo/broker.toml` (auto-synthesized, gitignored). +- **FR-013**: System MUST auto-synthesize a minimal default manifest declaring `github_token` for projects with no existing manifest. +- **FR-014**: System MUST create one project socket per active project at `/run/remo-broker/.sock` on the instance, enforcing the project's manifest allowlist. +- **FR-015**: System MUST mount the appropriate project socket into the project's devcontainer at `/run/remo-broker/sock` via devcontainer bind-mount configuration. +- **FR-016**: System MUST remove a project socket when its associated devcontainer exits. + +**Devcontainer enforcement** +- **FR-017**: The project menu MUST launch every selected project inside a devcontainer (committed or auto-synthesized), with no fallback to running the project on the instance OS. +- **FR-018**: The project menu MUST provide an explicit "exit to instance shell" option that surfaces a one-time warning explaining the broker is not available outside a devcontainer. +- **FR-019**: The instance OS shell MUST NOT have a broker socket available by default. + +**Lifecycle** +- **FR-020**: `remo destroy` MUST revoke an instance's bootstrap token at the backend *before* destroying the instance. +- **FR-021**: System MUST support periodic rotation of bootstrap tokens via a `remo rotate-bootstrap [instance]` command, configurable to run automatically on a cadence. +- **FR-022**: The broker MUST hold user-secret values in memory only, never writing them to disk on the instance. + +**Auditability** +- **FR-023**: The broker MUST log every secret-access request (project, secret name, allowed/denied, timestamp) to a local log file readable only by root on the instance. +- **FR-024**: System MUST provide a `remo audit ` command that retrieves and displays the broker's access log. + +### Non-Functional Requirements + +- **NFR-001**: A broker-mediated secret fetch in the steady state (warm cache) MUST add no more than 50 ms latency over a direct env-var read. +- **NFR-002**: The broker MUST survive `systemd` restarts and instance reboots without manual reconfiguration. +- **NFR-003**: An instance's broker MUST function for all configured backends across a backend network outage by serving the last in-memory-cached value until that value's TTL expires. + +### Key Entities + +- **Node** (new model): represents an Incus or Proxmox node registered with Remo. Fields include `name`, `host` (SSH target), `provider`, `bootstrap_admin_identity` (an SA admin token capable of minting per-instance sub-tokens, stored in laptop's fnox). Not used for AWS or Hetzner. +- **Bootstrap token** (no Remo model — opaque string): the per-instance backend identity. Located at `/etc/remo-broker/bootstrap-token` on instance, `/var/lib/remo-broker/instance-tokens/` on node (self-hosted only). On instances with a TPM, the token SHOULD be sealed at rest via systemd's `LoadCredentialEncrypted` / TPM2 binding so that an offline disk read does not yield a usable token. +- **Project manifest** (TOML file in repo or `.remo/`): declares the set of backend secret names this project requires. Read by the broker when minting a project socket. +- **Project socket** (Unix domain socket): per-project, per-instance, ephemeral, enforces the manifest's allowlist. +- **Broker** (process): one per instance, runs as a systemd unit (`remo-broker.service`), Remo-owned code that holds the bootstrap token, embeds fnox as its multi-backend retrieval library, holds a memory-only TTL cache of recently-resolved user secrets, enforces per-project allowlists, and writes an append-only audit log. +- **KnownHost** (existing model, unchanged): continues to represent L2 instances and their SSH-target metadata. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: After feature is fully adopted, **zero long-lived user secrets** are written to any Remo instance's disk. Verified by auditing `~/.aws/`, `~/.config/gh/`, `~/.npmrc`, `~/.netrc`, and `env` across a representative sample of instances. +- **SC-002**: A simulated supply-chain attack — a malicious `postinstall` script running `printenv`, `cat ~/.aws/credentials`, `cat ~/.config/gh/hosts.yml`, and an outbound HTTP exfil call from inside a devcontainer — recovers **only** the secrets declared in that project's manifest, and no others. +- **SC-003**: `remo shell` works from any device the user has authenticated to the backend from, with **no per-device instance reconfiguration** required. +- **SC-004**: Provisioning a new instance and launching a devcontainer adds **no more than 30 seconds** to today's flow on a typical broadband connection (warm laptop fnox cache, warm backend). +- **SC-005**: After `remo destroy`, a copy of the destroyed instance's bootstrap token is **rejected by the backend within 60 seconds**. +- **SC-006**: When the user adds a secret to a project manifest, it becomes available inside that project's devcontainer **on the next devcontainer restart**, with no instance-level configuration changes. + +## Component Sourcing + +This section records why the broker is built rather than adopted, and which external components are leveraged. + +### Adopted: fnox (laptop CLI + on-instance retrieval library) + +[`fnox`](https://github.com/jdx/fnox) is a mature (v1.25.1, ~39 releases, post-1.0) multi-backend secret-fetching CLI that already abstracts 1Password, HashiCorp Vault, AWS Secrets Manager, age, and the OS keychain behind a single `fnox get ` interface. Remo adopts fnox in two roles: + +1. **Laptop-side provisioning credential lookup** — `lookup('pipe', 'fnox get …')` invocations in Ansible replace `lookup('env', …)` patterns (FR-006). The laptop already typically has fnox configured for the developer's day-to-day secret access. +2. **On-instance backend retrieval inside the broker** — the `remo-broker` daemon embeds fnox (as a subprocess or library, design-deferred) to resolve a name like `GITHUB_TOKEN` to a fetched value via whichever backend is configured for that instance. + +This sidesteps the largest single body of work in the spec — writing and maintaining adapters for five different secret stores, each with its own auth model and SDK. + +### Built: the `remo-broker` daemon + +A broader survey of credential-broker-adjacent tooling (Vault Agent, OpenBao Agent, SPIFFE/SPIRE, Teleport Machine ID, Infisical Agent, systemd credentials, 1Password Connect, EKS Pod Identity Agent, Doppler, sops, Bitwarden Secrets Manager, aws-vault) found **no single existing tool that combines** the four properties this spec requires: + +1. Multi-backend retrieval (1Password + Vault + AWS SM + age + keychain in one process). +2. Per-project Unix-socket serving (one socket per project, distinct allowlists). +3. Manifest-driven allowlists that the broker itself enforces. +4. Bootstrap modes spanning IMDS (AWS), token-file (Hetzner), and node-bind-mount (Incus/Proxmox). + +The closest single-tool fit is **Vault Agent / OpenBao Agent**, but it is single-backend (Vault-only) and its per-listener `role` / `require_request_header` knobs are anti-SSRF controls rather than per-project secret allowlists — true allowlisting would require one agent process per project, which defeats the operational model. + +The strongest patterns to *borrow* (not adopt wholesale) come from: + +- **EKS Pod Identity Agent** — proves the "token-file mounted into client → daemon returns only that client's allowed credentials" pattern in production. +- **SPIFFE/SPIRE Workload API** — proves that kernel-attested per-caller scoping over a Unix socket is practical. + +The broker therefore is Remo-owned code, with these influences guiding its IPC and attestation design. It is small (estimated low single thousands of lines), policy-driven, and depends on fnox-core for the genuinely complex backend-integration work. + +#### Repository layout and implementation language + +The `remo-broker` daemon lives in a separate repository (`get2knowio/remo-broker`) for reasons of language asymmetry (Rust vs. Remo's Python), distribution shape (signed binary releases vs. PyPI), release cadence (the daemon is install-once, the laptop CLI iterates), and audit surface. Implementation language: **Rust**, with [`fnox-core`](https://crates.io/crates/fnox-core) (MIT, published by the fnox author) as a Cargo dependency — closes OQ-7 in the "library" direction by avoiding subprocess fork/exec per uncached fetch and keeping secret values inside fnox-core's typed wrappers end-to-end. + +The new repo owns the broker's internal design and the two cross-repo contracts: + +- `specs/001-broker-daemon/spec.md` — full feature spec for the daemon +- `docs/manifest-schema.md` — versioned TOML schema for `remo-broker.toml` (cross-repo contract; source of truth here, JSON Schema published per release and consumed by Remo for laptop-side validation) +- `docs/wire-protocol.md` — project-socket and admin-socket wire protocol (cross-repo contract) + +Schema-drift mitigations between the two repos: (1) JSON Schema generated from Rust types and validated on the Remo side, (2) `schema_version` integer in every manifest with broker-side refusal of unknown versions, (3) end-to-end CI test exercising both repos against a real manifest + socket round-trip. + +### Future escape hatches + +- If fnox becomes unmaintained or unsuitable, the natural swap is **OpenBao Agent** (MPL-2.0) running one agent per project behind a thin Remo supervisor. Trade-off: loss of native 1Password / OS-keychain support unless those are proxied via OpenBao secret engines. +- If a generic broker daemon emerges upstream that meets all four requirements above, Remo's broker can be retired in favor of it. + +### Complementary, used underneath the broker + +- **systemd `LoadCredentialEncrypted` + TPM2** — used to seal the bootstrap-token file at rest on TPM-equipped instances (typically Incus / Proxmox nodes), so an offline disk read does not yield a usable token. Does not replace any broker function — it hardens token storage. + +## Out of Scope + +- **Sandboxing untrusted code beyond the devcontainer boundary**: this spec does not require additional in-devcontainer sandboxing (gVisor, firejail, network egress restrictions per-install-script). Those are complementary defenses worth considering separately. +- **Replacing existing SSH key management**: the broker handles user secrets fetched at runtime by devcontainer tooling. SSH key material for `remo shell` itself remains handled by the existing flow (laptop ssh-agent forwarding or instance-resident keys, depending on access mode). +- **Backend selection UI improvements** beyond the minimum needed at `remo init`. A full backend management TUI is future work. +- **Secret rotation at the user-secret level**: this spec rotates *bootstrap tokens* (the broker's identity). Rotation of the actual GitHub PAT, NPM token, etc. is the backend's concern and the user's policy choice. +- **Multi-user instances**: this spec assumes each instance has one developer user. Multi-tenant instances would need per-user project-socket isolation, deferred to future work. + +## Open Questions + +- **OQ-1**: Should the project socket be created per-devcontainer-lifetime or per-project-lifetime? The former gives strict ephemerality but may complicate background tasks like `cargo build` running across `devcontainer exec` invocations. The latter is operationally simpler. +- **OQ-2**: For Incus/Proxmox nodes that host instances from multiple developers, how is the node's admin identity (used to mint sub-tokens for each developer's instances) bootstrapped? Per-developer admin SAs, or a single node admin SA with logical sub-scoping? +- **OQ-3**: Should the broker also serve the `gh` git credential helper protocol natively, or is `gh auth login --with-token` against a broker-fetched token sufficient? +- **OQ-4**: Default rotation cadence for bootstrap tokens — 24h, 7d, or "never (revoke only on destroy)"? Trade-off is operational noise vs. exposure window. +- **OQ-5**: How should the broker behave when a backend requires interactive auth (e.g., 1Password biometric prompt) and the requesting context is non-interactive (autonomous AI agent at 3am)? Refuse and surface the requirement, or rely entirely on non-interactive backend identities (SA tokens) and forbid interactive ones for instance use? +- **OQ-6**: Should the broker make the bootstrap-token file's TPM2 sealing mandatory on instances where a TPM is available, or remain opt-in via configuration? Mandatory closes the offline-disk-read attack reliably; opt-in avoids surprises for users on older hardware or with custom systemd setups. +- **OQ-7**: Should the broker embed fnox as a Rust library (tight coupling, one process, faster) or shell out to the `fnox` binary as a subprocess (loose coupling, version-independent, slower)? Subprocess is simpler to ship and lets users upgrade fnox independently; library is faster and avoids a fork+exec per uncached fetch but locks the broker to a specific fnox version. +- **OQ-8**: What's the wire protocol on the project socket — a simple line-based `GET \n` / `\n` shape, a fnox-CLI-compatible protocol (so tools that already speak fnox work unchanged), or something richer (gRPC, JSON-RPC) that supports streaming, watches, and structured errors? Simpler is easier to audit; richer enables future features like credential-change notifications. diff --git a/specs/006-credential-broker-laptop-push/.github/agents/speckit.analyze.agent.md b/specs/006-credential-broker-laptop-push/.github/agents/speckit.analyze.agent.md new file mode 100644 index 0000000..1237e10 --- /dev/null +++ b/specs/006-credential-broker-laptop-push/.github/agents/speckit.analyze.agent.md @@ -0,0 +1,249 @@ +--- +description: Perform a non-destructive cross-artifact consistency and quality analysis across spec.md, plan.md, and tasks.md after task generation. +--- + +## User Input + +```text +$ARGUMENTS +``` + +You **MUST** consider the user input before proceeding (if not empty). + +## Pre-Execution Checks + +**Check for extension hooks (before analysis)**: +- Check if `.specify/extensions.yml` exists in the project root. +- If it exists, read it and look for entries under the `hooks.before_analyze` key +- If the YAML cannot be parsed or is invalid, skip hook checking silently and continue normally +- Filter out hooks where `enabled` is explicitly `false`. Treat hooks without an `enabled` field as enabled by default. +- For each remaining hook, do **not** attempt to interpret or evaluate hook `condition` expressions: + - If the hook has no `condition` field, or it is null/empty, treat the hook as executable + - If the hook defines a non-empty `condition`, skip the hook and leave condition evaluation to the HookExecutor implementation +- For each executable hook, output the following based on its `optional` flag: + - **Optional hook** (`optional: true`): + ``` + ## Extension Hooks + + **Optional Pre-Hook**: {extension} + Command: `/{command}` + Description: {description} + + Prompt: {prompt} + To execute: `/{command}` + ``` + - **Mandatory hook** (`optional: false`): + ``` + ## Extension Hooks + + **Automatic Pre-Hook**: {extension} + Executing: `/{command}` + EXECUTE_COMMAND: {command} + + Wait for the result of the hook command before proceeding to the Goal. + ``` +- If no hooks are registered or `.specify/extensions.yml` does not exist, skip silently + +## Goal + +Identify inconsistencies, duplications, ambiguities, and underspecified items across the three core artifacts (`spec.md`, `plan.md`, `tasks.md`) before implementation. This command MUST run only after `/speckit.tasks` has successfully produced a complete `tasks.md`. + +## Operating Constraints + +**STRICTLY READ-ONLY**: Do **not** modify any files. Output a structured analysis report. Offer an optional remediation plan (user must explicitly approve before any follow-up editing commands would be invoked manually). + +**Constitution Authority**: The project constitution (`.specify/memory/constitution.md`) is **non-negotiable** within this analysis scope. Constitution conflicts are automatically CRITICAL and require adjustment of the spec, plan, or tasks—not dilution, reinterpretation, or silent ignoring of the principle. If a principle itself needs to change, that must occur in a separate, explicit constitution update outside `/speckit.analyze`. + +## Execution Steps + +### 1. Initialize Analysis Context + +Run `.specify/scripts/bash/check-prerequisites.sh --json --require-tasks --include-tasks` once from repo root and parse JSON for FEATURE_DIR and AVAILABLE_DOCS. Derive absolute paths: + +- SPEC = FEATURE_DIR/spec.md +- PLAN = FEATURE_DIR/plan.md +- TASKS = FEATURE_DIR/tasks.md + +Abort with an error message if any required file is missing (instruct the user to run missing prerequisite command). +For single quotes in args like "I'm Groot", use escape syntax: e.g 'I'\''m Groot' (or double-quote if possible: "I'm Groot"). + +### 2. Load Artifacts (Progressive Disclosure) + +Load only the minimal necessary context from each artifact: + +**From spec.md:** + +- Overview/Context +- Functional Requirements +- Success Criteria (measurable outcomes — e.g., performance, security, availability, user success, business impact) +- User Stories +- Edge Cases (if present) + +**From plan.md:** + +- Architecture/stack choices +- Data Model references +- Phases +- Technical constraints + +**From tasks.md:** + +- Task IDs +- Descriptions +- Phase grouping +- Parallel markers [P] +- Referenced file paths + +**From constitution:** + +- Load `.specify/memory/constitution.md` for principle validation + +### 3. Build Semantic Models + +Create internal representations (do not include raw artifacts in output): + +- **Requirements inventory**: For each Functional Requirement (FR-###) and Success Criterion (SC-###), record a stable key. Use the explicit FR-/SC- identifier as the primary key when present, and optionally also derive an imperative-phrase slug for readability (e.g., "User can upload file" → `user-can-upload-file`). Include only Success Criteria items that require buildable work (e.g., load-testing infrastructure, security audit tooling), and exclude post-launch outcome metrics and business KPIs (e.g., "Reduce support tickets by 50%"). +- **User story/action inventory**: Discrete user actions with acceptance criteria +- **Task coverage mapping**: Map each task to one or more requirements or stories (inference by keyword / explicit reference patterns like IDs or key phrases) +- **Constitution rule set**: Extract principle names and MUST/SHOULD normative statements + +### 4. Detection Passes (Token-Efficient Analysis) + +Focus on high-signal findings. Limit to 50 findings total; aggregate remainder in overflow summary. + +#### A. Duplication Detection + +- Identify near-duplicate requirements +- Mark lower-quality phrasing for consolidation + +#### B. Ambiguity Detection + +- Flag vague adjectives (fast, scalable, secure, intuitive, robust) lacking measurable criteria +- Flag unresolved placeholders (TODO, TKTK, ???, ``, etc.) + +#### C. Underspecification + +- Requirements with verbs but missing object or measurable outcome +- User stories missing acceptance criteria alignment +- Tasks referencing files or components not defined in spec/plan + +#### D. Constitution Alignment + +- Any requirement or plan element conflicting with a MUST principle +- Missing mandated sections or quality gates from constitution + +#### E. Coverage Gaps + +- Requirements with zero associated tasks +- Tasks with no mapped requirement/story +- Success Criteria requiring buildable work (performance, security, availability) not reflected in tasks + +#### F. Inconsistency + +- Terminology drift (same concept named differently across files) +- Data entities referenced in plan but absent in spec (or vice versa) +- Task ordering contradictions (e.g., integration tasks before foundational setup tasks without dependency note) +- Conflicting requirements (e.g., one requires Next.js while other specifies Vue) + +### 5. Severity Assignment + +Use this heuristic to prioritize findings: + +- **CRITICAL**: Violates constitution MUST, missing core spec artifact, or requirement with zero coverage that blocks baseline functionality +- **HIGH**: Duplicate or conflicting requirement, ambiguous security/performance attribute, untestable acceptance criterion +- **MEDIUM**: Terminology drift, missing non-functional task coverage, underspecified edge case +- **LOW**: Style/wording improvements, minor redundancy not affecting execution order + +### 6. Produce Compact Analysis Report + +Output a Markdown report (no file writes) with the following structure: + +## Specification Analysis Report + +| ID | Category | Severity | Location(s) | Summary | Recommendation | +|----|----------|----------|-------------|---------|----------------| +| A1 | Duplication | HIGH | spec.md:L120-134 | Two similar requirements ... | Merge phrasing; keep clearer version | + +(Add one row per finding; generate stable IDs prefixed by category initial.) + +**Coverage Summary Table:** + +| Requirement Key | Has Task? | Task IDs | Notes | +|-----------------|-----------|----------|-------| + +**Constitution Alignment Issues:** (if any) + +**Unmapped Tasks:** (if any) + +**Metrics:** + +- Total Requirements +- Total Tasks +- Coverage % (requirements with >=1 task) +- Ambiguity Count +- Duplication Count +- Critical Issues Count + +### 7. Provide Next Actions + +At end of report, output a concise Next Actions block: + +- If CRITICAL issues exist: Recommend resolving before `/speckit.implement` +- If only LOW/MEDIUM: User may proceed, but provide improvement suggestions +- Provide explicit command suggestions: e.g., "Run /speckit.specify with refinement", "Run /speckit.plan to adjust architecture", "Manually edit tasks.md to add coverage for 'performance-metrics'" + +### 8. Offer Remediation + +Ask the user: "Would you like me to suggest concrete remediation edits for the top N issues?" (Do NOT apply them automatically.) + +### 9. Check for extension hooks + +After reporting, check if `.specify/extensions.yml` exists in the project root. +- If it exists, read it and look for entries under the `hooks.after_analyze` key +- If the YAML cannot be parsed or is invalid, skip hook checking silently and continue normally +- Filter out hooks where `enabled` is explicitly `false`. Treat hooks without an `enabled` field as enabled by default. +- For each remaining hook, do **not** attempt to interpret or evaluate hook `condition` expressions: + - If the hook has no `condition` field, or it is null/empty, treat the hook as executable + - If the hook defines a non-empty `condition`, skip the hook and leave condition evaluation to the HookExecutor implementation +- For each executable hook, output the following based on its `optional` flag: + - **Optional hook** (`optional: true`): + ``` + ## Extension Hooks + + **Optional Hook**: {extension} + Command: `/{command}` + Description: {description} + + Prompt: {prompt} + To execute: `/{command}` + ``` + - **Mandatory hook** (`optional: false`): + ``` + ## Extension Hooks + + **Automatic Hook**: {extension} + Executing: `/{command}` + EXECUTE_COMMAND: {command} + ``` +- If no hooks are registered or `.specify/extensions.yml` does not exist, skip silently + +## Operating Principles + +### Context Efficiency + +- **Minimal high-signal tokens**: Focus on actionable findings, not exhaustive documentation +- **Progressive disclosure**: Load artifacts incrementally; don't dump all content into analysis +- **Token-efficient output**: Limit findings table to 50 rows; summarize overflow +- **Deterministic results**: Rerunning without changes should produce consistent IDs and counts + +### Analysis Guidelines + +- **NEVER modify files** (this is read-only analysis) +- **NEVER hallucinate missing sections** (if absent, report them accurately) +- **Prioritize constitution violations** (these are always CRITICAL) +- **Use examples over exhaustive rules** (cite specific instances, not generic patterns) +- **Report zero issues gracefully** (emit success report with coverage statistics) + +## Context + +$ARGUMENTS diff --git a/specs/006-credential-broker-laptop-push/.github/agents/speckit.checklist.agent.md b/specs/006-credential-broker-laptop-push/.github/agents/speckit.checklist.agent.md new file mode 100644 index 0000000..93ec785 --- /dev/null +++ b/specs/006-credential-broker-laptop-push/.github/agents/speckit.checklist.agent.md @@ -0,0 +1,361 @@ +--- +description: Generate a custom checklist for the current feature based on user requirements. +--- + +## Checklist Purpose: "Unit Tests for English" + +**CRITICAL CONCEPT**: Checklists are **UNIT TESTS FOR REQUIREMENTS WRITING** - they validate the quality, clarity, and completeness of requirements in a given domain. + +**NOT for verification/testing**: + +- ❌ NOT "Verify the button clicks correctly" +- ❌ NOT "Test error handling works" +- ❌ NOT "Confirm the API returns 200" +- ❌ NOT checking if code/implementation matches the spec + +**FOR requirements quality validation**: + +- ✅ "Are visual hierarchy requirements defined for all card types?" (completeness) +- ✅ "Is 'prominent display' quantified with specific sizing/positioning?" (clarity) +- ✅ "Are hover state requirements consistent across all interactive elements?" (consistency) +- ✅ "Are accessibility requirements defined for keyboard navigation?" (coverage) +- ✅ "Does the spec define what happens when logo image fails to load?" (edge cases) + +**Metaphor**: If your spec is code written in English, the checklist is its unit test suite. You're testing whether the requirements are well-written, complete, unambiguous, and ready for implementation - NOT whether the implementation works. + +## User Input + +```text +$ARGUMENTS +``` + +You **MUST** consider the user input before proceeding (if not empty). + +## Pre-Execution Checks + +**Check for extension hooks (before checklist generation)**: +- Check if `.specify/extensions.yml` exists in the project root. +- If it exists, read it and look for entries under the `hooks.before_checklist` key +- If the YAML cannot be parsed or is invalid, skip hook checking silently and continue normally +- Filter out hooks where `enabled` is explicitly `false`. Treat hooks without an `enabled` field as enabled by default. +- For each remaining hook, do **not** attempt to interpret or evaluate hook `condition` expressions: + - If the hook has no `condition` field, or it is null/empty, treat the hook as executable + - If the hook defines a non-empty `condition`, skip the hook and leave condition evaluation to the HookExecutor implementation +- For each executable hook, output the following based on its `optional` flag: + - **Optional hook** (`optional: true`): + ``` + ## Extension Hooks + + **Optional Pre-Hook**: {extension} + Command: `/{command}` + Description: {description} + + Prompt: {prompt} + To execute: `/{command}` + ``` + - **Mandatory hook** (`optional: false`): + ``` + ## Extension Hooks + + **Automatic Pre-Hook**: {extension} + Executing: `/{command}` + EXECUTE_COMMAND: {command} + + Wait for the result of the hook command before proceeding to the Execution Steps. + ``` +- If no hooks are registered or `.specify/extensions.yml` does not exist, skip silently + +## Execution Steps + +1. **Setup**: Run `.specify/scripts/bash/check-prerequisites.sh --json` from repo root and parse JSON for FEATURE_DIR and AVAILABLE_DOCS list. + - All file paths must be absolute. + - For single quotes in args like "I'm Groot", use escape syntax: e.g 'I'\''m Groot' (or double-quote if possible: "I'm Groot"). + +2. **Clarify intent (dynamic)**: Derive up to THREE initial contextual clarifying questions (no pre-baked catalog). They MUST: + - Be generated from the user's phrasing + extracted signals from spec/plan/tasks + - Only ask about information that materially changes checklist content + - Be skipped individually if already unambiguous in `$ARGUMENTS` + - Prefer precision over breadth + + Generation algorithm: + 1. Extract signals: feature domain keywords (e.g., auth, latency, UX, API), risk indicators ("critical", "must", "compliance"), stakeholder hints ("QA", "review", "security team"), and explicit deliverables ("a11y", "rollback", "contracts"). + 2. Cluster signals into candidate focus areas (max 4) ranked by relevance. + 3. Identify probable audience & timing (author, reviewer, QA, release) if not explicit. + 4. Detect missing dimensions: scope breadth, depth/rigor, risk emphasis, exclusion boundaries, measurable acceptance criteria. + 5. Formulate questions chosen from these archetypes: + - Scope refinement (e.g., "Should this include integration touchpoints with X and Y or stay limited to local module correctness?") + - Risk prioritization (e.g., "Which of these potential risk areas should receive mandatory gating checks?") + - Depth calibration (e.g., "Is this a lightweight pre-commit sanity list or a formal release gate?") + - Audience framing (e.g., "Will this be used by the author only or peers during PR review?") + - Boundary exclusion (e.g., "Should we explicitly exclude performance tuning items this round?") + - Scenario class gap (e.g., "No recovery flows detected—are rollback / partial failure paths in scope?") + + Question formatting rules: + - If presenting options, generate a compact table with columns: Option | Candidate | Why It Matters + - Limit to A–E options maximum; omit table if a free-form answer is clearer + - Never ask the user to restate what they already said + - Avoid speculative categories (no hallucination). If uncertain, ask explicitly: "Confirm whether X belongs in scope." + + Defaults when interaction impossible: + - Depth: Standard + - Audience: Reviewer (PR) if code-related; Author otherwise + - Focus: Top 2 relevance clusters + + Output the questions (label Q1/Q2/Q3). After answers: if ≥2 scenario classes (Alternate / Exception / Recovery / Non-Functional domain) remain unclear, you MAY ask up to TWO more targeted follow‑ups (Q4/Q5) with a one-line justification each (e.g., "Unresolved recovery path risk"). Do not exceed five total questions. Skip escalation if user explicitly declines more. + +3. **Understand user request**: Combine `$ARGUMENTS` + clarifying answers: + - Derive checklist theme (e.g., security, review, deploy, ux) + - Consolidate explicit must-have items mentioned by user + - Map focus selections to category scaffolding + - Infer any missing context from spec/plan/tasks (do NOT hallucinate) + +4. **Load feature context**: Read from FEATURE_DIR: + - spec.md: Feature requirements and scope + - plan.md (if exists): Technical details, dependencies + - tasks.md (if exists): Implementation tasks + + **Context Loading Strategy**: + - Load only necessary portions relevant to active focus areas (avoid full-file dumping) + - Prefer summarizing long sections into concise scenario/requirement bullets + - Use progressive disclosure: add follow-on retrieval only if gaps detected + - If source docs are large, generate interim summary items instead of embedding raw text + +5. **Generate checklist** - Create "Unit Tests for Requirements": + - Create `FEATURE_DIR/checklists/` directory if it doesn't exist + - Generate unique checklist filename: + - Use short, descriptive name based on domain (e.g., `ux.md`, `api.md`, `security.md`) + - Format: `[domain].md` + - File handling behavior: + - If file does NOT exist: Create new file and number items starting from CHK001 + - If file exists: Append new items to existing file, continuing from the last CHK ID (e.g., if last item is CHK015, start new items at CHK016) + - Never delete or replace existing checklist content - always preserve and append + + **CORE PRINCIPLE - Test the Requirements, Not the Implementation**: + Every checklist item MUST evaluate the REQUIREMENTS THEMSELVES for: + - **Completeness**: Are all necessary requirements present? + - **Clarity**: Are requirements unambiguous and specific? + - **Consistency**: Do requirements align with each other? + - **Measurability**: Can requirements be objectively verified? + - **Coverage**: Are all scenarios/edge cases addressed? + + **Category Structure** - Group items by requirement quality dimensions: + - **Requirement Completeness** (Are all necessary requirements documented?) + - **Requirement Clarity** (Are requirements specific and unambiguous?) + - **Requirement Consistency** (Do requirements align without conflicts?) + - **Acceptance Criteria Quality** (Are success criteria measurable?) + - **Scenario Coverage** (Are all flows/cases addressed?) + - **Edge Case Coverage** (Are boundary conditions defined?) + - **Non-Functional Requirements** (Performance, Security, Accessibility, etc. - are they specified?) + - **Dependencies & Assumptions** (Are they documented and validated?) + - **Ambiguities & Conflicts** (What needs clarification?) + + **HOW TO WRITE CHECKLIST ITEMS - "Unit Tests for English"**: + + ❌ **WRONG** (Testing implementation): + - "Verify landing page displays 3 episode cards" + - "Test hover states work on desktop" + - "Confirm logo click navigates home" + + ✅ **CORRECT** (Testing requirements quality): + - "Are the exact number and layout of featured episodes specified?" [Completeness] + - "Is 'prominent display' quantified with specific sizing/positioning?" [Clarity] + - "Are hover state requirements consistent across all interactive elements?" [Consistency] + - "Are keyboard navigation requirements defined for all interactive UI?" [Coverage] + - "Is the fallback behavior specified when logo image fails to load?" [Edge Cases] + - "Are loading states defined for asynchronous episode data?" [Completeness] + - "Does the spec define visual hierarchy for competing UI elements?" [Clarity] + + **ITEM STRUCTURE**: + Each item should follow this pattern: + - Question format asking about requirement quality + - Focus on what's WRITTEN (or not written) in the spec/plan + - Include quality dimension in brackets [Completeness/Clarity/Consistency/etc.] + - Reference spec section `[Spec §X.Y]` when checking existing requirements + - Use `[Gap]` marker when checking for missing requirements + + **EXAMPLES BY QUALITY DIMENSION**: + + Completeness: + - "Are error handling requirements defined for all API failure modes? [Gap]" + - "Are accessibility requirements specified for all interactive elements? [Completeness]" + - "Are mobile breakpoint requirements defined for responsive layouts? [Gap]" + + Clarity: + - "Is 'fast loading' quantified with specific timing thresholds? [Clarity, Spec §NFR-2]" + - "Are 'related episodes' selection criteria explicitly defined? [Clarity, Spec §FR-5]" + - "Is 'prominent' defined with measurable visual properties? [Ambiguity, Spec §FR-4]" + + Consistency: + - "Do navigation requirements align across all pages? [Consistency, Spec §FR-10]" + - "Are card component requirements consistent between landing and detail pages? [Consistency]" + + Coverage: + - "Are requirements defined for zero-state scenarios (no episodes)? [Coverage, Edge Case]" + - "Are concurrent user interaction scenarios addressed? [Coverage, Gap]" + - "Are requirements specified for partial data loading failures? [Coverage, Exception Flow]" + + Measurability: + - "Are visual hierarchy requirements measurable/testable? [Acceptance Criteria, Spec §FR-1]" + - "Can 'balanced visual weight' be objectively verified? [Measurability, Spec §FR-2]" + + **Scenario Classification & Coverage** (Requirements Quality Focus): + - Check if requirements exist for: Primary, Alternate, Exception/Error, Recovery, Non-Functional scenarios + - For each scenario class, ask: "Are [scenario type] requirements complete, clear, and consistent?" + - If scenario class missing: "Are [scenario type] requirements intentionally excluded or missing? [Gap]" + - Include resilience/rollback when state mutation occurs: "Are rollback requirements defined for migration failures? [Gap]" + + **Traceability Requirements**: + - MINIMUM: ≥80% of items MUST include at least one traceability reference + - Each item should reference: spec section `[Spec §X.Y]`, or use markers: `[Gap]`, `[Ambiguity]`, `[Conflict]`, `[Assumption]` + - If no ID system exists: "Is a requirement & acceptance criteria ID scheme established? [Traceability]" + + **Surface & Resolve Issues** (Requirements Quality Problems): + Ask questions about the requirements themselves: + - Ambiguities: "Is the term 'fast' quantified with specific metrics? [Ambiguity, Spec §NFR-1]" + - Conflicts: "Do navigation requirements conflict between §FR-10 and §FR-10a? [Conflict]" + - Assumptions: "Is the assumption of 'always available podcast API' validated? [Assumption]" + - Dependencies: "Are external podcast API requirements documented? [Dependency, Gap]" + - Missing definitions: "Is 'visual hierarchy' defined with measurable criteria? [Gap]" + + **Content Consolidation**: + - Soft cap: If raw candidate items > 40, prioritize by risk/impact + - Merge near-duplicates checking the same requirement aspect + - If >5 low-impact edge cases, create one item: "Are edge cases X, Y, Z addressed in requirements? [Coverage]" + + **🚫 ABSOLUTELY PROHIBITED** - These make it an implementation test, not a requirements test: + - ❌ Any item starting with "Verify", "Test", "Confirm", "Check" + implementation behavior + - ❌ References to code execution, user actions, system behavior + - ❌ "Displays correctly", "works properly", "functions as expected" + - ❌ "Click", "navigate", "render", "load", "execute" + - ❌ Test cases, test plans, QA procedures + - ❌ Implementation details (frameworks, APIs, algorithms) + + **✅ REQUIRED PATTERNS** - These test requirements quality: + - ✅ "Are [requirement type] defined/specified/documented for [scenario]?" + - ✅ "Is [vague term] quantified/clarified with specific criteria?" + - ✅ "Are requirements consistent between [section A] and [section B]?" + - ✅ "Can [requirement] be objectively measured/verified?" + - ✅ "Are [edge cases/scenarios] addressed in requirements?" + - ✅ "Does the spec define [missing aspect]?" + +6. **Structure Reference**: Generate the checklist following the canonical template in `.specify/templates/checklist-template.md` for title, meta section, category headings, and ID formatting. If template is unavailable, use: H1 title, purpose/created meta lines, `##` category sections containing `- [ ] CHK### ` lines with globally incrementing IDs starting at CHK001. + +7. **Report**: Output full path to checklist file, item count, and summarize whether the run created a new file or appended to an existing one. Summarize: + - Focus areas selected + - Depth level + - Actor/timing + - Any explicit user-specified must-have items incorporated + +**Important**: Each `/speckit.checklist` command invocation uses a short, descriptive checklist filename and either creates a new file or appends to an existing one. This allows: + +- Multiple checklists of different types (e.g., `ux.md`, `test.md`, `security.md`) +- Simple, memorable filenames that indicate checklist purpose +- Easy identification and navigation in the `checklists/` folder + +To avoid clutter, use descriptive types and clean up obsolete checklists when done. + +## Example Checklist Types & Sample Items + +**UX Requirements Quality:** `ux.md` + +Sample items (testing the requirements, NOT the implementation): + +- "Are visual hierarchy requirements defined with measurable criteria? [Clarity, Spec §FR-1]" +- "Is the number and positioning of UI elements explicitly specified? [Completeness, Spec §FR-1]" +- "Are interaction state requirements (hover, focus, active) consistently defined? [Consistency]" +- "Are accessibility requirements specified for all interactive elements? [Coverage, Gap]" +- "Is fallback behavior defined when images fail to load? [Edge Case, Gap]" +- "Can 'prominent display' be objectively measured? [Measurability, Spec §FR-4]" + +**API Requirements Quality:** `api.md` + +Sample items: + +- "Are error response formats specified for all failure scenarios? [Completeness]" +- "Are rate limiting requirements quantified with specific thresholds? [Clarity]" +- "Are authentication requirements consistent across all endpoints? [Consistency]" +- "Are retry/timeout requirements defined for external dependencies? [Coverage, Gap]" +- "Is versioning strategy documented in requirements? [Gap]" + +**Performance Requirements Quality:** `performance.md` + +Sample items: + +- "Are performance requirements quantified with specific metrics? [Clarity]" +- "Are performance targets defined for all critical user journeys? [Coverage]" +- "Are performance requirements under different load conditions specified? [Completeness]" +- "Can performance requirements be objectively measured? [Measurability]" +- "Are degradation requirements defined for high-load scenarios? [Edge Case, Gap]" + +**Security Requirements Quality:** `security.md` + +Sample items: + +- "Are authentication requirements specified for all protected resources? [Coverage]" +- "Are data protection requirements defined for sensitive information? [Completeness]" +- "Is the threat model documented and requirements aligned to it? [Traceability]" +- "Are security requirements consistent with compliance obligations? [Consistency]" +- "Are security failure/breach response requirements defined? [Gap, Exception Flow]" + +## Anti-Examples: What NOT To Do + +**❌ WRONG - These test implementation, not requirements:** + +```markdown +- [ ] CHK001 - Verify landing page displays 3 episode cards [Spec §FR-001] +- [ ] CHK002 - Test hover states work correctly on desktop [Spec §FR-003] +- [ ] CHK003 - Confirm logo click navigates to home page [Spec §FR-010] +- [ ] CHK004 - Check that related episodes section shows 3-5 items [Spec §FR-005] +``` + +**✅ CORRECT - These test requirements quality:** + +```markdown +- [ ] CHK001 - Are the number and layout of featured episodes explicitly specified? [Completeness, Spec §FR-001] +- [ ] CHK002 - Are hover state requirements consistently defined for all interactive elements? [Consistency, Spec §FR-003] +- [ ] CHK003 - Are navigation requirements clear for all clickable brand elements? [Clarity, Spec §FR-010] +- [ ] CHK004 - Is the selection criteria for related episodes documented? [Gap, Spec §FR-005] +- [ ] CHK005 - Are loading state requirements defined for asynchronous episode data? [Gap] +- [ ] CHK006 - Can "visual hierarchy" requirements be objectively measured? [Measurability, Spec §FR-001] +``` + +**Key Differences:** + +- Wrong: Tests if the system works correctly +- Correct: Tests if the requirements are written correctly +- Wrong: Verification of behavior +- Correct: Validation of requirement quality +- Wrong: "Does it do X?" +- Correct: "Is X clearly specified?" + +## Post-Execution Checks + +**Check for extension hooks (after checklist generation)**: +Check if `.specify/extensions.yml` exists in the project root. +- If it exists, read it and look for entries under the `hooks.after_checklist` key +- If the YAML cannot be parsed or is invalid, skip hook checking silently and continue normally +- Filter out hooks where `enabled` is explicitly `false`. Treat hooks without an `enabled` field as enabled by default. +- For each remaining hook, do **not** attempt to interpret or evaluate hook `condition` expressions: + - If the hook has no `condition` field, or it is null/empty, treat the hook as executable + - If the hook defines a non-empty `condition`, skip the hook and leave condition evaluation to the HookExecutor implementation +- For each executable hook, output the following based on its `optional` flag: + - **Optional hook** (`optional: true`): + ``` + ## Extension Hooks + + **Optional Hook**: {extension} + Command: `/{command}` + Description: {description} + + Prompt: {prompt} + To execute: `/{command}` + ``` + - **Mandatory hook** (`optional: false`): + ``` + ## Extension Hooks + + **Automatic Hook**: {extension} + Executing: `/{command}` + EXECUTE_COMMAND: {command} + ``` +- If no hooks are registered or `.specify/extensions.yml` does not exist, skip silently diff --git a/specs/006-credential-broker-laptop-push/.github/agents/speckit.clarify.agent.md b/specs/006-credential-broker-laptop-push/.github/agents/speckit.clarify.agent.md new file mode 100644 index 0000000..9de13e1 --- /dev/null +++ b/specs/006-credential-broker-laptop-push/.github/agents/speckit.clarify.agent.md @@ -0,0 +1,247 @@ +--- +description: Identify underspecified areas in the current feature spec by asking up to 5 highly targeted clarification questions and encoding answers back into the spec. +handoffs: + - label: Build Technical Plan + agent: speckit.plan + prompt: Create a plan for the spec. I am building with... +--- + +## User Input + +```text +$ARGUMENTS +``` + +You **MUST** consider the user input before proceeding (if not empty). + +## Pre-Execution Checks + +**Check for extension hooks (before clarification)**: +- Check if `.specify/extensions.yml` exists in the project root. +- If it exists, read it and look for entries under the `hooks.before_clarify` key +- If the YAML cannot be parsed or is invalid, skip hook checking silently and continue normally +- Filter out hooks where `enabled` is explicitly `false`. Treat hooks without an `enabled` field as enabled by default. +- For each remaining hook, do **not** attempt to interpret or evaluate hook `condition` expressions: + - If the hook has no `condition` field, or it is null/empty, treat the hook as executable + - If the hook defines a non-empty `condition`, skip the hook and leave condition evaluation to the HookExecutor implementation +- For each executable hook, output the following based on its `optional` flag: + - **Optional hook** (`optional: true`): + ``` + ## Extension Hooks + + **Optional Pre-Hook**: {extension} + Command: `/{command}` + Description: {description} + + Prompt: {prompt} + To execute: `/{command}` + ``` + - **Mandatory hook** (`optional: false`): + ``` + ## Extension Hooks + + **Automatic Pre-Hook**: {extension} + Executing: `/{command}` + EXECUTE_COMMAND: {command} + + Wait for the result of the hook command before proceeding to the Outline. + ``` +- If no hooks are registered or `.specify/extensions.yml` does not exist, skip silently + +## Outline + +Goal: Detect and reduce ambiguity or missing decision points in the active feature specification and record the clarifications directly in the spec file. + +Note: This clarification workflow is expected to run (and be completed) BEFORE invoking `/speckit.plan`. If the user explicitly states they are skipping clarification (e.g., exploratory spike), you may proceed, but must warn that downstream rework risk increases. + +Execution steps: + +1. Run `.specify/scripts/bash/check-prerequisites.sh --json --paths-only` from repo root **once** (combined `--json --paths-only` mode / `-Json -PathsOnly`). Parse minimal JSON payload fields: + - `FEATURE_DIR` + - `FEATURE_SPEC` + - (Optionally capture `IMPL_PLAN`, `TASKS` for future chained flows.) + - If JSON parsing fails, abort and instruct user to re-run `/speckit.specify` or verify feature branch environment. + - For single quotes in args like "I'm Groot", use escape syntax: e.g 'I'\''m Groot' (or double-quote if possible: "I'm Groot"). + +2. Load the current spec file. Perform a structured ambiguity & coverage scan using this taxonomy. For each category, mark status: Clear / Partial / Missing. Produce an internal coverage map used for prioritization (do not output raw map unless no questions will be asked). + + Functional Scope & Behavior: + - Core user goals & success criteria + - Explicit out-of-scope declarations + - User roles / personas differentiation + + Domain & Data Model: + - Entities, attributes, relationships + - Identity & uniqueness rules + - Lifecycle/state transitions + - Data volume / scale assumptions + + Interaction & UX Flow: + - Critical user journeys / sequences + - Error/empty/loading states + - Accessibility or localization notes + + Non-Functional Quality Attributes: + - Performance (latency, throughput targets) + - Scalability (horizontal/vertical, limits) + - Reliability & availability (uptime, recovery expectations) + - Observability (logging, metrics, tracing signals) + - Security & privacy (authN/Z, data protection, threat assumptions) + - Compliance / regulatory constraints (if any) + + Integration & External Dependencies: + - External services/APIs and failure modes + - Data import/export formats + - Protocol/versioning assumptions + + Edge Cases & Failure Handling: + - Negative scenarios + - Rate limiting / throttling + - Conflict resolution (e.g., concurrent edits) + + Constraints & Tradeoffs: + - Technical constraints (language, storage, hosting) + - Explicit tradeoffs or rejected alternatives + + Terminology & Consistency: + - Canonical glossary terms + - Avoided synonyms / deprecated terms + + Completion Signals: + - Acceptance criteria testability + - Measurable Definition of Done style indicators + + Misc / Placeholders: + - TODO markers / unresolved decisions + - Ambiguous adjectives ("robust", "intuitive") lacking quantification + + For each category with Partial or Missing status, add a candidate question opportunity unless: + - Clarification would not materially change implementation or validation strategy + - Information is better deferred to planning phase (note internally) + +3. Generate (internally) a prioritized queue of candidate clarification questions (maximum 5). Do NOT output them all at once. Apply these constraints: + - Maximum of 5 total questions across the whole session. + - Each question must be answerable with EITHER: + - A short multiple‑choice selection (2–5 distinct, mutually exclusive options), OR + - A one-word / short‑phrase answer (explicitly constrain: "Answer in <=5 words"). + - Only include questions whose answers materially impact architecture, data modeling, task decomposition, test design, UX behavior, operational readiness, or compliance validation. + - Ensure category coverage balance: attempt to cover the highest impact unresolved categories first; avoid asking two low-impact questions when a single high-impact area (e.g., security posture) is unresolved. + - Exclude questions already answered, trivial stylistic preferences, or plan-level execution details (unless blocking correctness). + - Favor clarifications that reduce downstream rework risk or prevent misaligned acceptance tests. + - If more than 5 categories remain unresolved, select the top 5 by (Impact * Uncertainty) heuristic. + +4. Sequential questioning loop (interactive): + - Present EXACTLY ONE question at a time. + - For multiple‑choice questions: + - **Analyze all options** and determine the **most suitable option** based on: + - Best practices for the project type + - Common patterns in similar implementations + - Risk reduction (security, performance, maintainability) + - Alignment with any explicit project goals or constraints visible in the spec + - Present your **recommended option prominently** at the top with clear reasoning (1-2 sentences explaining why this is the best choice). + - Format as: `**Recommended:** Option [X] - ` + - Then render all options as a Markdown table: + + | Option | Description | + |--------|-------------| + | A |