diff --git a/README.md b/README.md index 8736a7b..6fdc16f 100644 --- a/README.md +++ b/README.md @@ -40,15 +40,31 @@ contributions welcome for other platforms! ## usage -```python +pre-built wheels are served from a [PEP 503 index](https://commaai.github.io/dependencies/simple/) backed by GitHub Releases: + +```toml dependencies = [ - # use per-package release branches for pre-built wheels - "capnproto @ git+https://github.com/commaai/dependencies.git@release-capnproto#subdirectory=capnproto", - "ffmpeg @ git+https://github.com/commaai/dependencies.git@release-ffmpeg#subdirectory=ffmpeg", + "capnproto==1.0.1", + "ffmpeg==7.1.0", +] + +[[tool.uv.index]] +name = "comma-dependencies" +url = "https://commaai.github.io/dependencies/simple/" +explicit = true + +[tool.uv.sources] +capnproto = { index = "comma-dependencies" } +ffmpeg = { index = "comma-dependencies" } +``` + +with plain pip, pass the index explicitly: `pip install --extra-index-url https://commaai.github.io/dependencies/simple/ capnproto` + +to build a package from source instead, use the master branch directly: - # use the master branch to build the package on pip install +```python +dependencies = [ "capnproto @ git+https://github.com/commaai/dependencies.git@master#subdirectory=capnproto", - "ffmpeg @ git+https://github.com/commaai/dependencies.git@master#subdirectory=ffmpeg", ] ``` @@ -57,5 +73,6 @@ dependencies = [ to add a new package: * start a new top-level directory as a new package * `./test.sh` tests the building of all packages -* on pushes to `master`, wheels are built for our target platforms and pushed to a GitHub release -* each `release-` branch contains a single shim package, so old lockfiles keep resolving even as new packages are added +* on pushes to `master`, wheels are built for our target platforms and pushed to a GitHub release +* `make_index.py` regenerates the index from all GitHub releases and publishes it to the `gh-pages` branch, so old lockfiles keep resolving even as new packages are added +* each `release-` branch contains a single shim package (legacy; superseded by the index) diff --git a/make_index.py b/make_index.py new file mode 100644 index 0000000..9f94f9d --- /dev/null +++ b/make_index.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Generate a static PEP 503 "simple" index from this repo's GitHub Releases. + +The index pages link directly to the wheel assets on GitHub Releases, so the +index itself is just HTML. Served via GitHub Pages (gh-pages branch): + https://commaai.github.io/dependencies/simple/ +""" +import hashlib +import json +import os +import re +import sys +import urllib.request + +REPO = os.environ.get("GITHUB_REPOSITORY", "commaai/dependencies") +API = f"https://api.github.com/repos/{REPO}/releases" + + +def _get(url): + req = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"}) + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + req.add_header("Authorization", f"Bearer {token}") + return urllib.request.urlopen(req, timeout=60) + + +def get_releases(): + releases = [] + page = 1 + while True: + with _get(f"{API}?per_page=100&page={page}") as resp: + batch = json.load(resp) + releases += batch + if len(batch) < 100: + return releases + page += 1 + + +def normalize(name): + # PEP 503 name normalization + return re.sub(r"[-_.]+", "-", name).lower() + + +def sha256_of(asset): + digest = asset.get("digest") + if digest and digest.startswith("sha256:"): + return digest.removeprefix("sha256:") + print(f" no digest for {asset['name']}, downloading to hash ...") + with _get(asset["browser_download_url"]) as resp: + return hashlib.sha256(resp.read()).hexdigest() + + +def main(out_dir): + # project name -> list of (filename, url, sha256), from release tags like "capnproto/v1.0.1" + projects = {} + for release in get_releases(): + name = normalize(release["tag_name"].split("/")[0]) + for asset in release["assets"]: + if not asset["name"].endswith(".whl"): + continue + projects.setdefault(name, []).append((asset["name"], asset["browser_download_url"], sha256_of(asset))) + + os.makedirs(os.path.join(out_dir, "simple"), exist_ok=True) + root = ["", ""] + for name in sorted(projects): + root.append(f'{name}
') + page = ["", "", f"

{name}

"] + for filename, url, sha in sorted(projects[name]): + page.append(f'{filename}
') + page += ["", ""] + os.makedirs(os.path.join(out_dir, "simple", name), exist_ok=True) + with open(os.path.join(out_dir, "simple", name, "index.html"), "w") as f: + f.write("\n".join(page)) + root += ["", ""] + with open(os.path.join(out_dir, "simple", "index.html"), "w") as f: + f.write("\n".join(root)) + + # tell GitHub Pages not to run Jekyll + open(os.path.join(out_dir, ".nojekyll"), "w").close() + with open(os.path.join(out_dir, "index.html"), "w") as f: + f.write(f'\n

{REPO}

PEP 503 index: simple/\n') + + n = sum(len(v) for v in projects.values()) + print(f"wrote index for {len(projects)} projects ({n} wheels) to {out_dir}/simple") + + +if __name__ == "__main__": + main(sys.argv[1] if len(sys.argv) > 1 else "pages") diff --git a/release.sh b/release.sh index ad08fce..7c2bea9 100755 --- a/release.sh +++ b/release.sh @@ -175,3 +175,19 @@ done shopt -u nullglob rm -rf "$TMP_DIR" + +echo +echo "Publishing PEP 503 index to GitHub Pages" + +PAGES_DIR="$(mktemp -d)" +GH_TOKEN="$TOKEN" python3 make_index.py "$PAGES_DIR" +( + cd "$PAGES_DIR" + git init + git checkout -b gh-pages + git add -A + git -c user.name="github-actions[bot]" -c user.email="github-actions[bot]@users.noreply.github.com" commit -m "update simple index" + git remote add origin "https://x-access-token:${TOKEN}@github.com/${REPO}.git" + git push -f origin gh-pages +) +rm -rf "$PAGES_DIR"