From ae978cacd5f85f27893e7b9b974b998388446eb7 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Sun, 17 May 2026 12:43:34 +0200 Subject: [PATCH] ci: auto-update external-plugin pins via scheduled workflow External plugins (terraform-skill, future ones) were pinned by hand. Add a generic, repo-local updater. - .github/scripts/update_external_plugins.py: stdlib-only; auto- discovers external entries (object source == github) from .claude-plugin/marketplace.json, cross-checks the name-matching .agents/plugins entry, resolves the latest eligible upstream release (semver, no prereleases by default), and atomically rewrites source.ref in both manifests plus the mirrored .claude-plugin version. --dry-run supported; fails on rename / deleted tag / API error; "nothing to do" exits 0. - .github/workflows/update-external-plugins.yml: daily cron + workflow_dispatch; opens a reviewable chore(external-plugins) PR via peter-evans/create-pull-request pinned by SHA (v8.1.1). contents+pull-requests write only; no pull_request_target. - .github/external-plugin-updates.json: policy overlay (defaults + optional per-plugin overrides). JSON, not YAML, so the updater needs no third-party deps. - validate.yml: add .agents/plugins/** to trigger paths + a cross-manifest sync check (repo/ref match, version == v+ref). - CONTRIBUTING.md / CLAUDE.md: document the automation; do not hand-bump pins. Manifest/CI only (no plugins/ content), ci: type -> no release. Live dry-run detects the real pending terraform-skill v1.10.0 -> v1.11.0 bump. --- .github/external-plugin-updates.json | 9 + .github/scripts/update_external_plugins.py | 215 ++++++++++++++++++ .github/workflows/update-external-plugins.yml | 57 +++++ .github/workflows/validate.yml | 57 +++++ CLAUDE.md | 10 +- CONTRIBUTING.md | 24 +- 6 files changed, 363 insertions(+), 9 deletions(-) create mode 100644 .github/external-plugin-updates.json create mode 100644 .github/scripts/update_external_plugins.py create mode 100644 .github/workflows/update-external-plugins.yml diff --git a/.github/external-plugin-updates.json b/.github/external-plugin-updates.json new file mode 100644 index 0000000..434c82b --- /dev/null +++ b/.github/external-plugin-updates.json @@ -0,0 +1,9 @@ +{ + "_comment": "Policy overlay for .github/scripts/update_external_plugins.py. External plugins are auto-discovered from .claude-plugin/marketplace.json (object source == github); they do NOT need an entry here. Add an entry under 'plugins' ONLY to deviate from defaults (e.g. allow prereleases, a non-semver tag pattern, or tags instead of releases). JSON (not YAML) so the updater needs no third-party deps.", + "defaults": { + "includePrereleases": false, + "tagPattern": "^v?(\\d+\\.\\d+\\.\\d+)$", + "source": "github-releases" + }, + "plugins": {} +} diff --git a/.github/scripts/update_external_plugins.py b/.github/scripts/update_external_plugins.py new file mode 100644 index 0000000..375fd24 --- /dev/null +++ b/.github/scripts/update_external_plugins.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Auto-update external-plugin pins in agent-plugins. + +Source of truth: .claude-plugin/marketplace.json (external entries where +`source` is an object with source == "github"). The matching entry in +.agents/plugins/marketplace.json (keyed by `name`) is kept in sync. + +For each external plugin, resolve the latest eligible upstream release tag and, +if newer than the pinned ref, rewrite atomically: + - .claude-plugin: entry.source.ref = vX.Y.Z AND entry.version = X.Y.Z + - .agents: entry.source.ref = vX.Y.Z (no version field) + +No third-party deps (stdlib only). Real errors exit non-zero; "nothing to do" +exits 0. `--dry-run` prints intended changes without writing. + +Optional policy overlay: .github/external-plugin-updates.json + { + "defaults": {"includePrereleases": false, + "tagPattern": "^v?(\\\\d+\\\\.\\\\d+\\\\.\\\\d+)$", + "source": "github-releases"}, + "plugins": {"": {"source": "github-tags", ...}} + } +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import urllib.error +import urllib.request + +CLAUDE_MANIFEST = ".claude-plugin/marketplace.json" +AGENTS_MANIFEST = ".agents/plugins/marketplace.json" +POLICY_FILE = ".github/external-plugin-updates.json" + +DEFAULTS = { + "includePrereleases": False, + "tagPattern": r"^v?(\d+\.\d+\.\d+)$", + "source": "github-releases", # or "github-tags" +} +API = "https://api.github.com" + + +def fail(msg: str) -> None: + print(f"ERROR: {msg}", file=sys.stderr) + sys.exit(1) + + +def load_json(path: str): + try: + with open(path, encoding="utf-8") as f: + return json.load(f) + except FileNotFoundError: + fail(f"{path} not found") + except json.JSONDecodeError as e: + fail(f"{path}: invalid JSON: {e}") + + +def write_json(path: str, data) -> None: + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + f.write("\n") + + +def gh_get(url: str): + req = urllib.request.Request(url, headers={ + "Accept": "application/vnd.github+json", + "User-Agent": "agent-plugins-external-updater", + "X-GitHub-Api-Version": "2022-11-28", + }) + token = os.environ.get("GITHUB_TOKEN") + if token: + req.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")), resp.headers + except urllib.error.HTTPError as e: + fail(f"GitHub API {url} -> HTTP {e.code} {e.reason}") + except urllib.error.URLError as e: + fail(f"GitHub API {url} unreachable: {e.reason}") + + +def paginate(url: str): + items = [] + while url: + page, headers = gh_get(url) + if not isinstance(page, list): + fail(f"expected a list from {url}") + items.extend(page) + url = "" + link = headers.get("Link", "") + for part in link.split(","): + if 'rel="next"' in part: + url = part[part.find("<") + 1:part.find(">")] + return items + + +def semver(t: str, pat: re.Pattern): + m = pat.match(t) + if not m: + return None + return tuple(int(x) for x in m.group(1).split(".")) + + +def normalize_repo(agents_url: str) -> str: + # git@github.com:owner/repo.git -> owner/repo + m = re.match(r"^git@github\.com:(.+?)(?:\.git)?$", agents_url.strip()) + if m: + return m.group(1) + m = re.match(r"^https://github\.com/(.+?)(?:\.git)?$", agents_url.strip()) + if m: + return m.group(1) + return agents_url.strip() + + +def latest_version(repo: str, cfg: dict): + pat = re.compile(cfg["tagPattern"]) + # rename / existence guard + meta, _ = gh_get(f"{API}/repos/{repo}") + if meta.get("full_name", "").lower() != repo.lower(): + fail(f"{repo}: upstream full_name is " + f"{meta.get('full_name')!r} (renamed?) - update the manifest") + + candidates = [] + if cfg["source"] == "github-tags": + for t in paginate(f"{API}/repos/{repo}/tags?per_page=100"): + v = semver(t.get("name", ""), pat) + if v: + candidates.append((v, t["name"])) + else: + for r in paginate(f"{API}/repos/{repo}/releases?per_page=100"): + if r.get("draft"): + continue + if r.get("prerelease") and not cfg["includePrereleases"]: + continue + v = semver(r.get("tag_name", ""), pat) + if v: + candidates.append((v, r["tag_name"])) + if not candidates: + fail(f"{repo}: no eligible release/tag matching {cfg['tagPattern']}") + candidates.sort(key=lambda x: x[0]) + ver, tag = candidates[-1] + return ".".join(str(n) for n in ver), tag + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + policy = {} + if os.path.exists(POLICY_FILE): + policy = load_json(POLICY_FILE) + pdefaults = {**DEFAULTS, **(policy.get("defaults") or {})} + poverrides = policy.get("plugins") or {} + + claude = load_json(CLAUDE_MANIFEST) + agents = load_json(AGENTS_MANIFEST) + agents_by_name = {p.get("name"): p for p in agents.get("plugins", [])} + + changed = [] + for entry in claude.get("plugins", []): + src = entry.get("source") + if not isinstance(src, dict) or src.get("source") != "github": + continue # inline or non-github external + name = entry.get("name") + repo = src.get("repo", "") + if not re.match(r"^[\w.-]+/[\w.-]+$", repo): + fail(f"{name}: invalid source.repo {repo!r}") + + a = agents_by_name.get(name) + if a is None: + fail(f"{name}: no matching entry in {AGENTS_MANIFEST}") + a_src = a.get("source", {}) + a_repo = normalize_repo(a_src.get("url", "")) + if a_repo.lower() != repo.lower(): + fail(f"{name}: {AGENTS_MANIFEST} repo {a_repo!r} != " + f"{repo!r} ({CLAUDE_MANIFEST})") + + cfg = {**pdefaults, **(poverrides.get(name) or {})} + ver, tag = latest_version(repo, cfg) + cur_ref = src.get("ref", "") + if cur_ref == tag: + print(f" {name}: up to date ({tag})") + continue + + changed.append((name, repo, cur_ref, tag)) + if args.dry_run: + continue + src["ref"] = tag + entry["version"] = ver # mirrored, no leading v + a_src["ref"] = tag # .agents: ref only + + if not changed: + print("Nothing to update.") + return 0 + + print("\nplugin | repo | old -> new") + for n, r, o, t in changed: + print(f" {n} | {r} | {o or '(none)'} -> {t}") + + if args.dry_run: + print("\n(dry-run: no files written)") + return 0 + + write_json(CLAUDE_MANIFEST, claude) + write_json(AGENTS_MANIFEST, agents) + print(f"\nUpdated {CLAUDE_MANIFEST} and {AGENTS_MANIFEST}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/update-external-plugins.yml b/.github/workflows/update-external-plugins.yml new file mode 100644 index 0000000..df9a581 --- /dev/null +++ b/.github/workflows/update-external-plugins.yml @@ -0,0 +1,57 @@ +name: Update External Plugins + +# Auto-bump external-plugin pins (source.ref + mirrored version) when an +# upstream publishes a newer release. Opens a reviewable chore(...) PR; +# manifest-only chore commits do not trigger an agent-plugins release. + +on: + schedule: + - cron: '17 6 * * *' # daily, 06:17 UTC + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: update-external-plugins + cancel-in-progress: false + +jobs: + update: + name: Resolve and pin latest external releases + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Update external plugin pins + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 .github/scripts/update_external_plugins.py + + - name: Open PR + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + branch: automation/external-plugin-updates + base: master + commit-message: 'chore(external-plugins): update external plugin pins' + title: 'chore(external-plugins): update external plugin pins' + body: | + Automated external-plugin pin update. + + Bumps `source.ref` in both manifests (and the mirrored + `version` in `.claude-plugin/marketplace.json`) to the latest + eligible upstream release. Manifest-only `chore:` change - no + agent-plugins release is triggered. + + Review the diff against the upstream changelog before merging. + delete-branch: true + add-paths: | + .claude-plugin/marketplace.json + .agents/plugins/marketplace.json diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 301c23d..f912d43 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -5,11 +5,13 @@ on: paths: - 'plugins/**' - '.claude-plugin/**' + - '.agents/plugins/**' push: branches: [master, main] paths: - 'plugins/**' - '.claude-plugin/**' + - '.agents/plugins/**' workflow_dispatch: jobs: @@ -236,6 +238,61 @@ jobs: f"{n_local} local + {n_external} external plugin(s))") EOF + - name: Validate external manifest sync + run: | + python3 << 'EOF' + import json, re, sys + + print("🔍 Checking .claude-plugin <-> .agents external sync...") + claude = json.load(open('.claude-plugin/marketplace.json')) + agents = json.load(open('.agents/plugins/marketplace.json')) + a_by_name = {p.get('name'): p for p in agents.get('plugins', [])} + + def norm(url): + m = re.match(r'^git@github\.com:(.+?)(?:\.git)?$', (url or '').strip()) + if m: + return m.group(1).lower() + m = re.match(r'^https://github\.com/(.+?)(?:\.git)?$', + (url or '').strip()) + return m.group(1).lower() if m else (url or '').strip().lower() + + errors = [] + n = 0 + for e in claude.get('plugins', []): + s = e.get('source') + if not isinstance(s, dict) or s.get('source') != 'github': + continue + n += 1 + name = e.get('name') + repo = s.get('repo', '') + ref = s.get('ref', '') + ver = e.get('version') + if ver is not None and ref != f"v{ver}": + errors.append( + f"{name}: .claude-plugin ref {ref!r} != v+version " + f"(version {ver!r})") + a = a_by_name.get(name) + if a is None: + errors.append( + f"{name}: external plugin missing from " + f".agents/plugins/marketplace.json") + continue + a_src = a.get('source', {}) + if norm(a_src.get('url', '')) != repo.lower(): + errors.append( + f"{name}: .agents repo " + f"{norm(a_src.get('url',''))!r} != {repo.lower()!r}") + if a_src.get('ref', '') != ref: + errors.append( + f"{name}: ref mismatch - .claude-plugin {ref!r} vs " + f".agents {a_src.get('ref','')!r}") + + if errors: + print("\n".join(f"❌ {x}" for x in errors)) + sys.exit(1) + print(f"✅ {n} external plugin(s) in sync across both manifests") + EOF + - name: Check for Broken Links run: | echo "🔍 Checking internal reference links (inline plugins only)..." diff --git a/CLAUDE.md b/CLAUDE.md index e9865e1..f5a7ecb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -53,8 +53,14 @@ sources, auto-clones + caches the referenced repo (see | `version` field | Optional, manual mirror of `source.ref` (NOT CI-managed) | Required, CI-managed, must equal SKILL.md `metadata.version` | `terraform-skill` is external: `antonbabenko/terraform-skill`, pinned by -`source.ref`. Its content and tags (`vX.Y.Z`) live in that repo; to ship a -newer version, bump `source.ref` and the mirrored `version` in the manifest. +`source.ref`. Its content and tags (`vX.Y.Z`) live in that repo. Pins are +bumped automatically: the scheduled `Update External Plugins` workflow +(`.github/workflows/update-external-plugins.yml`) auto-discovers external +entries from `.claude-plugin/marketplace.json`, resolves the latest upstream +release, and opens a reviewable `chore(external-plugins): ...` PR updating +`source.ref` in both manifests plus the mirrored `version`. Per-plugin +overrides live in `.github/external-plugin-updates.json`; `validate.yml` +cross-checks the two manifests stay in sync. Do not hand-bump. ## Adding a Plugin diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2a69ff4..230d2e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,12 +33,20 @@ standards, and the per-plugin release model before contributing. **External** (content + releases in its own repo) — manifest entry only: -1. Add a `plugins[]` entry to `.claude-plugin/marketplace.json`: `name`, - `source: { "source": "github", "repo": "owner/repo", "ref": "vX.Y.Z" }`, - `description`, optional `category` / `keywords`, optional `version` - (manual mirror of the ref). -2. No local content, CHANGELOG, tests, or scoped-commit release. Update by - bumping `source.ref` (and the mirrored `version`). +1. Add a `plugins[]` entry to BOTH `.claude-plugin/marketplace.json` + (`source: { "source": "github", "repo": "owner/repo", "ref": "vX.Y.Z" }` + plus a mirrored top-level `version: "X.Y.Z"`) and + `.agents/plugins/marketplace.json` (Codex; same `name`, `ref`, no + `version`). +2. No local content, CHANGELOG, tests, or scoped-commit release. +3. **Do not hand-bump the pin.** The scheduled `Update External Plugins` + workflow (`.github/workflows/update-external-plugins.yml`) auto-discovers + every external entry, resolves the latest upstream release, and opens a + reviewable `chore(external-plugins): ...` PR that updates `source.ref` in + both manifests and the mirrored `version`. Override defaults (prereleases, + tag pattern, tags-vs-releases) per plugin in + `.github/external-plugin-updates.json`. CI cross-checks the two manifests + stay in sync. **Inline** (content lives here): @@ -64,7 +72,9 @@ qualifies for a plugin when it changes release-worthy content under `plugins//` (anything except `tests/` and the CI-managed `CHANGELOG.md`), or when its subject is scoped to that plugin (`feat(): ...`, back-compat). The commit **type** sets the bump. -External plugins release upstream; update them here by bumping `source.ref`. +External plugins release upstream; their pins are bumped automatically by the +`Update External Plugins` workflow (a reviewable `chore(external-plugins): ...` +PR), not by hand. | Qualifying commit type | Result | |------------------------|--------|