From e0a87119f633c9babe593870d5938e8e681f5347 Mon Sep 17 00:00:00 2001 From: Jason Farrar Date: Tue, 7 Jul 2026 14:19:07 +0100 Subject: [PATCH] feat: implement two-stage release process with draft releases (#140) - Modify release.yml to create DRAFT releases instead of publishing to PyPI immediately - Create new promote-release.yml workflow for manual promotion to live - Promotion includes PyPI publish, release un-drafting, and doc deployment - Add GitHub Deployment tracking for release visibility - Add PyPI package verification step after publishing - Create RELEASE.md documentation with complete workflow instructions - Update Trusted Publisher workflow reference from release.yml to promote-release.yml This allows maintainers to review artifacts before making releases public. Signed-off-by: Jason Farrar --- .github/workflows/promote-release.yml | 258 ++++++++++++++++++++++++ .github/workflows/release.yml | 115 ++--------- RELEASE.md | 272 ++++++++++++++++++++++++++ 3 files changed, 544 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/promote-release.yml create mode 100644 RELEASE.md diff --git a/.github/workflows/promote-release.yml b/.github/workflows/promote-release.yml new file mode 100644 index 0000000..65bdbcc --- /dev/null +++ b/.github/workflows/promote-release.yml @@ -0,0 +1,258 @@ +# promote-release.yml — promote a draft release to live and publish to PyPI. +# +# Triggered manually via GitHub UI or CLI: +# +# gh workflow run promote-release.yml -f tag=v1.0.0 +# +# This workflow: +# 1. Verifies the tag and draft release exist +# 2. Downloads artifacts from the draft release +# 3. Publishes wheel + sdist to PyPI via OIDC +# 4. Creates a GitHub Deployment record (for tracking) +# 5. Un-drafts the GitHub Release +# 6. Generates and deploys versioned docs to GitHub Pages + +name: Promote Release + +on: + workflow_dispatch: + inputs: + tag: + description: 'Release tag to promote (e.g., v1.0.0)' + required: true + type: string + +permissions: {} # no workflow-level permissions; each job declares its own + +jobs: + # ── Validate tag, publish to PyPI, un-draft release ────────────────── + promote: + name: Promote to Live & Publish to PyPI + runs-on: ubuntu-latest + environment: pypi + outputs: + version: ${{ steps.version.outputs.version }} + permissions: + id-token: write + contents: write + deployments: write + steps: + - name: Checkout tag + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.tag }} + persist-credentials: false + + - name: Validate tag and extract version + id: version + run: | + VERSION=$(python3 -c " + import tomllib + with open('pyproject.toml', 'rb') as f: + print(tomllib.load(f)['project']['version']) + ") + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + TAG_VERSION="${{ inputs.tag }}" + TAG_VERSION="${TAG_VERSION#v}" + if [ "$TAG_VERSION" != "$VERSION" ]; then + echo "::error::Tag version ($TAG_VERSION) does not match pyproject.toml version ($VERSION)" + exit 1 + fi + echo "✓ Tag ${{ inputs.tag }} matches version $VERSION" + + - name: Check draft release exists + id: release + env: + GH_TOKEN: ${{ github.token }} + run: | + RELEASE_JSON=$(gh release view "${{ inputs.tag }}" --json isDraft,id 2>/dev/null || echo "{}") + IS_DRAFT=$(echo "$RELEASE_JSON" | python3 -c "import sys, json; print(json.load(sys.stdin).get('isDraft', False))") + RELEASE_ID=$(echo "$RELEASE_JSON" | python3 -c "import sys, json; print(json.load(sys.stdin).get('id', ''))") + if [ "$IS_DRAFT" != "True" ] && [ "$RELEASE_ID" != "" ]; then + echo "::warning::Release ${{ inputs.tag }} is not a draft, but proceeding..." + fi + if [ -z "$RELEASE_ID" ]; then + echo "::error::Release ${{ inputs.tag }} not found" + exit 1 + fi + echo "release_id=$RELEASE_ID" >> "$GITHUB_OUTPUT" + echo "✓ Draft release found (ID: $RELEASE_ID)" + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + + - name: Install Python + run: uv python install 3.14.2 + + - name: Download artifacts from release + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p dist/ + gh release download "${{ inputs.tag }}" \ + --pattern "*.whl" \ + --pattern "*.tar.gz" \ + --dir dist/ + echo "Downloaded artifacts:" + ls -lh dist/ + + - name: Publish to PyPI + run: uv publish + + - name: Verify package on PyPI + id: pypi-verify + run: | + echo "Waiting 30s for PyPI indexing..." + sleep 30 + VERSION="${{ steps.version.outputs.version }}" + WHEEL_FILE=$(ls dist/*.whl | head -1) + WHEEL_NAME=$(basename "$WHEEL_FILE" .whl) + # Extract package name and version from wheel (format: {distribution}-{version}-...) + PKG_NAME=$(echo "$WHEEL_NAME" | cut -d- -f1 | tr '_' '-') + echo "Checking PyPI for $PKG_NAME==$VERSION..." + python3 -c " + import urllib.request + import json + try: + url = f'https://pypi.org/pypi/{urllib.parse.quote(\"$PKG_NAME\")}/json' + with urllib.request.urlopen(url, timeout=10) as resp: + data = json.loads(resp.read()) + releases = data.get('releases', {}) + if '${VERSION}' in releases: + print(f'✓ Package $PKG_NAME==${VERSION} found on PyPI') + else: + raise Exception(f'Version ${VERSION} not found. Available: {list(releases.keys())[-5:]}') + except Exception as e: + print(f'✗ PyPI verification failed: {e}') + exit(1) + " 2>&1 | tee /tmp/pypi-check.log + exit_code=${PIPESTATUS[0]} + result=$(cat /tmp/pypi-check.log) + echo "pypi_result=$result" >> "$GITHUB_OUTPUT" + exit $exit_code + + - name: Create GitHub Deployment + id: deployment + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const deployment = await github.rest.repos.createDeployment({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: '${{ inputs.tag }}', + environment: 'production', + description: 'Release ${{ inputs.tag }} to PyPI', + auto_merge: false, + required_contexts: [] + }); + core.setOutput('deployment_id', deployment.data.id); + console.log(`✓ Created deployment ${deployment.data.id}`); + + - name: Update Deployment Status (success) + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + await github.rest.repos.createDeploymentStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + deployment_id: ${{ steps.deployment.outputs.deployment_id }}, + state: 'success', + description: 'Package published to PyPI', + environment_url: 'https://pypi.org/project/bank-statement-parser/${{ steps.version.outputs.version }}/' + }); + console.log('✓ Deployment marked as success'); + + - name: Un-draft the release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release edit "${{ inputs.tag }}" --draft=false + echo "✓ Release ${{ inputs.tag }} promoted to live" + + # ── Generate and deploy docs to GitHub Pages ────────────────────────── + deploy-docs: + name: Generate & Deploy Docs + runs-on: ubuntu-latest + needs: promote + permissions: + contents: write + steps: + - name: Checkout release tag + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.tag }} + persist-credentials: true + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + + - name: Install Python + run: uv python install 3.14.2 + + - name: Install dependencies + run: uv sync --group dev + + - name: Generate docs for this release + run: uv run python scripts/generate_docs.py + + - name: Build site with mkdocs + run: uv run zensical build --clean + env: + GOATCOUNTER_URL: ${{ secrets.GOATCOUNTER_URL }} + + - name: Fetch existing gh-pages to avoid orphan push rejection + run: git fetch origin gh-pages:gh-pages --depth=1 || true + + - name: Deploy current version with mike + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + uv run mike deploy --push --update-aliases \ + "${{ needs.promote.outputs.version }}" latest + + - name: Fetch all tags for retroactive updates + run: git fetch origin --tags --prune + + - name: Retroactively update 2 prior versions + run: | + set -e + + # Get all tags sorted by version (newest first), then take 2nd and 3rd + TAGS=$(git tag --sort=-version:refname) + PRIOR_VERSIONS=$(echo "$TAGS" | head -3 | tail -2) + + for tag in $PRIOR_VERSIONS; do + version="${tag#v}" + echo "Retroactively updating docs for $version with current bank configs..." + + # Stash any uncommitted changes + git stash || true + + # Checkout the old tag to get its docs state (tag has v prefix) + git checkout "$tag" -- docs/ + + # Get the latest bank configs from master + git checkout origin/master -- src/bank_statement_parser/project/config/import/ + + # Regenerate docs with new bank data + uv run python scripts/generate_docs.py + + # Deploy locally (no push) — mike creates a commit on local gh-pages + uv run mike deploy "$version" + done + + # Create root index.html that redirects to latest/ + uv run mike set-default latest + + # Push all retroactive commits to remote in one go + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git push origin gh-pages + + - name: Clean up + if: always() + run: git stash pop || true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 63e090c..3296a29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,11 +1,12 @@ -# release.yml — build and publish on version tag push. +# release.yml — build binaries and create draft release on version tag push. # -# Triggered by pushing a tag like "v0.1.0". Four jobs run in sequence: +# Triggered by pushing a tag like "v0.1.0". Three jobs run in sequence: # -# 1. pypi-publish — build wheel + sdist, publish to PyPI via OIDC. +# 1. pypi-publish — build wheel + sdist (no PyPI upload yet). # 2. system-packages — build .deb and .rpm via fpm. -# 3. github-release — create a GitHub Release with auto-generated notes and all assets. -# 4. deploy-docs — generate and deploy versioned docs to GitHub Pages. +# 3. github-release — create a DRAFT GitHub Release with auto-generated notes and all assets. +# +# After reviewing artifacts, use promote-release.yml to publish to PyPI and un-draft. # # Releases can be triggered from any branch by pushing a version tag (v*). # The tag must match the version in pyproject.toml. This allows flexible @@ -14,14 +15,14 @@ # # Release notes are automatically generated by GitHub from PR titles. # For significant releases, you can edit the release notes on GitHub after -# the release is created. +# the release is created and promoted to live. # # One-time setup required: # - Create a "pypi" environment in GitHub repo Settings > Environments. # - Register a Trusted Publisher on PyPI (account > Publishing): # Owner: boscorat # Repository: bank_statement_parser -# Workflow: release.yml +# Workflow: promote-release.yml (update from release.yml after first release) # Environment: pypi # - Enable GitHub Pages on gh-pages branch (Settings > Pages). # - Configure Pages to deploy from gh-pages branch. @@ -37,21 +38,19 @@ on: permissions: {} # no workflow-level permissions; each job declares its own jobs: - # ── Build wheel + sdist and publish to PyPI ───────────────────────── + # ── Build wheel + sdist (no PyPI publish yet) ──────────────────────── pypi-publish: - name: Build & publish to PyPI + name: Build Python distributions runs-on: ubuntu-latest - environment: pypi outputs: version: ${{ steps.version.outputs.version }} permissions: - id-token: write - contents: write + contents: read steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - persist-credentials: true + persist-credentials: false fetch-depth: 0 - name: Install uv @@ -85,9 +84,6 @@ jobs: uv run --isolated --no-project --with dist/*.whl -- \ python -c "import bank_statement_parser; print(bank_statement_parser.__version__)" - - name: Publish to PyPI - run: uv publish - - name: Upload build artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -252,92 +248,9 @@ jobs: - name: Create release uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 with: + draft: true generate_release_notes: true files: | dist/* packages/* - - # ── Generate and deploy docs to GitHub Pages ────────────────────────── - deploy-docs: - name: Generate & Deploy Docs - runs-on: ubuntu-latest - needs: [pypi-publish, github-release] - permissions: - contents: write - steps: - - name: Checkout release tag - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: true - - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: false - - - name: Install Python - run: uv python install 3.14.2 - - - name: Install dependencies - run: uv sync --group dev - - - name: Generate docs for this release - run: uv run python scripts/generate_docs.py - - - name: Build site with mkdocs - run: uv run zensical build --clean - env: - GOATCOUNTER_URL: ${{ secrets.GOATCOUNTER_URL }} - - - name: Fetch existing gh-pages to avoid orphan push rejection - run: git fetch origin gh-pages:gh-pages --depth=1 || true - - - name: Deploy current version with mike - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - uv run mike deploy --push --update-aliases \ - "${{ needs.pypi-publish.outputs.version }}" latest - - - name: Fetch all tags for retroactive updates - run: git fetch origin --tags --prune - - - name: Retroactively update 2 prior versions - run: | - set -e - - # Get all tags sorted by version (newest first), then take 2nd and 3rd - TAGS=$(git tag --sort=-version:refname) - PRIOR_VERSIONS=$(echo "$TAGS" | head -3 | tail -2) - - for tag in $PRIOR_VERSIONS; do - version="${tag#v}" - echo "Retroactively updating docs for $version with current bank configs..." - - # Stash any uncommitted changes - git stash || true - - # Checkout the old tag to get its docs state (tag has v prefix) - git checkout "$tag" -- docs/ - - # Get the latest bank configs from master - git checkout origin/master -- src/bank_statement_parser/project/config/import/ - - # Regenerate docs with new bank data - uv run python scripts/generate_docs.py - - # Deploy locally (no push) — mike creates a commit on local gh-pages - uv run mike deploy "$version" - done - - # Create root index.html that redirects to latest/ - uv run mike set-default latest - - # Push all retroactive commits to remote in one go - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git push origin gh-pages - - - name: Clean up - if: always() - run: git stash pop || true + # ── Deploy docs to GitHub Pages (moved to promote-release.yml) ──────── diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..c017875 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,272 @@ +# Release Process + +This document describes the two-stage release process for bank-statement-parser. + +## Overview + +Releases follow a **draft-then-promote** workflow: + +1. **Stage 1 (Automatic)**: Push a version tag → builds binaries → creates a **draft** GitHub Release +2. **Stage 2 (Manual)**: Review artifacts → promote to live → publish to PyPI → deploy docs + +This allows maintainers to verify all artifacts are correct before making a release public. + +--- + +## Stage 1: Create a Draft Release + +### Prerequisites + +- Version is bumped in `pyproject.toml` +- All changes are committed and pushed to a branch +- The branch is ready for release + +### Steps + +1. **Update version in `pyproject.toml`**: + ```toml + [project] + version = "1.0.0" + ``` + +2. **Create an annotated tag** and push it: + ```bash + git tag -a v1.0.0 -m "Release 1.0.0" + git push origin v1.0.0 + ``` + +3. **GitHub Actions triggers automatically**: + - `release.yml` workflow starts + - Builds Python distributions (wheel + sdist) + - Builds system packages (.deb + .rpm) + - Creates a **draft** GitHub Release with all artifacts + - Does **NOT** publish to PyPI + - Does **NOT** deploy docs + +4. **Monitor the workflow**: + - Go to **Actions** tab in GitHub + - Click **Release** workflow + - Watch for job completion (~15-20 minutes) + +### Verify the Draft Release + +1. Go to **Releases** page on GitHub +2. Look for a **Draft** label on the new release (appears at top of page) +3. Download and inspect artifacts: + - `bank_statement_parser-*.whl` (wheel distribution) + - `bank_statement_parser-*.tar.gz` (source distribution) + - `uk-bank-statement-parser_*.deb` (Debian package) + - `uk-bank-statement-parser-*.rpm` (RPM package) + +4. Verify release notes auto-generated from PR titles + +5. **If something is wrong**: + - Delete the draft release (GitHub UI) + - Delete the local tag: `git tag -d v1.0.0` + - Delete the remote tag: `git push origin --delete v1.0.0` + - Fix the issue + - Re-create and push the tag (workflow will re-trigger) + +--- + +## Stage 2: Promote to Live + +Once artifacts are verified, promote the release to live. This: + +1. Publishes wheel + sdist to PyPI +2. Verifies package appears on PyPI +3. Creates a GitHub Deployment record +4. Un-drafts the GitHub Release (makes it visible to users) +5. Generates and deploys versioned docs to GitHub Pages + +### Promote via GitHub CLI + +```bash +gh workflow run promote-release.yml -f tag=v1.0.0 +``` + +This triggers the `promote-release.yml` workflow for the specified tag. + +### Promote via GitHub UI + +1. Go to **Actions** → **Promote Release** workflow +2. Click **Run workflow** +3. Enter the tag (e.g., `v1.0.0`) +4. Click **Run workflow** + +### Monitor Promotion + +1. Go to **Actions** → **Promote Release** +2. Click the workflow run +3. Watch for job completion (~10-15 minutes) + +### Verify Promotion + +After promotion completes: + +1. **GitHub Release**: + - Navigate to **Releases** page + - Draft label should be gone + - Release is now **live** and visible to all users + +2. **PyPI**: + - Visit `https://pypi.org/project/bank-statement-parser/` + - New version should appear in release history + - Package can be installed: `pip install bank-statement-parser==1.0.0` + +3. **GitHub Deployments** (tracking): + - Go to **Deployments** page + - New production deployment should show version + - Status should be **success** + +4. **Documentation**: + - Visit GitHub Pages docs site + - New version should be listed in version selector + - Latest alias should point to new version + +--- + +## Rollback After Promotion + +If you promoted a release but need to undo: + +1. **Undo PyPI publication** (contact PyPI support): + - PyPI does not allow yanking recent packages via the UI normally + - Use `pip-audit` or mark as yanked manually + - See [PEP 592 - Yanked Releases](https://www.python.org/dev/peps/pep-0592/) + +2. **Re-draft on GitHub**: + ```bash + gh release edit v1.0.0 --draft + ``` + +3. **Delete the Git tag** (if you want to re-release): + ```bash + git tag -d v1.0.0 + git push origin --delete v1.0.0 + ``` + +--- + +## Troubleshooting + +### Release workflow fails during build + +**Problem**: `pypi-publish` job fails + +**Solution**: +- Check job logs for error (usually version mismatch or build failure) +- Fix the issue in code +- Delete the tag and re-create it +- Push again + +### Draft release exists but promote workflow won't start + +**Problem**: `promote-release.yml` fails at "Check draft release exists" + +**Solution**: +- Verify the tag exists: `git tag -l v1.0.0` +- Verify the GitHub Release exists on the Releases page +- If not found, delete everything and re-tag + +### Promote workflow fails at PyPI verification + +**Problem**: Package doesn't appear on PyPI after 30 seconds + +**Solution**: +- PyPI indexing can take longer +- Check `https://pypi.org/project/bank-statement-parser/` manually +- If present, the promote workflow actually succeeded; re-run to verify +- If missing, check PyPI API: `curl https://pypi.org/pypi/bank-statement-parser/1.0.0/json` + +### Docs deployment fails + +**Problem**: `deploy-docs` job in promote workflow fails + +**Solution**: +- This is non-blocking; release was already promoted to live +- Fix docs issues and re-run the promote workflow (or manually deploy) +- See mkdocs/mike documentation if needed + +--- + +## One-time Setup + +The following setup is required once per repository: + +### PyPI Trusted Publisher + +1. Go to **Settings** → **Environments** → **New environment** +2. Create environment named `pypi` +3. Register a Trusted Publisher on PyPI: + - Log in to PyPI account + - Go to Account → Publishing (or https://pypi.org/manage/account/publishing/) + - Add a new pending publisher: + - **GitHub Repository**: `boscorat/bank_statement_parser` + - **Workflow**: `promote-release.yml` (used for promotion only) + - **Environment**: `pypi` + +### GitHub Pages + +1. Go to **Settings** → **Pages** +2. Set **Source** to `gh-pages` branch +3. Ensure deploy from `gh-pages` is enabled + +### Auto-generate Release Notes + +1. Go to **Settings** → **General** +2. Under **Features**, enable **Auto-generate release notes** +3. (Optional) Customize release notes template in **Settings** → **Code & automation** → **Probot** + +--- + +## Version Numbering + +This project follows **Semantic Versioning** (SemVer): `MAJOR.MINOR.PATCH` + +- **MAJOR**: Breaking API changes +- **MINOR**: New features (backward-compatible) +- **PATCH**: Bug fixes + +Examples: `1.0.0`, `1.1.0`, `1.1.1`, `2.0.0-rc.1` + +See [semver.org](https://semver.org/) for details. + +--- + +## Release Checklist + +Before tagging a release: + +- [ ] All tests pass locally: `pytest` +- [ ] Linting passes: `ruff check .` +- [ ] Formatting is correct: `ruff format .` +- [ ] Version is updated in `pyproject.toml` +- [ ] No uncommitted changes: `git status` +- [ ] Main branch is up-to-date: `git pull origin main` +- [ ] CHANGELOG or release notes are up-to-date (if applicable) + +After promotion: + +- [ ] Release is live on GitHub +- [ ] Package is available on PyPI +- [ ] Docs are deployed to GitHub Pages +- [ ] Announcement posted (if desired) + +--- + +## Related Files + +- **`.github/workflows/release.yml`** — Triggered on tag push; builds binaries and creates draft +- **`.github/workflows/promote-release.yml`** — Manual trigger; promotes to live and publishes to PyPI +- **`pyproject.toml`** — Contains version number (must match tag) +- **`.github/workflows/ci.yml`** — Runs tests on every push (ensure passing before release) + +--- + +## See Also + +- [git-workflows skill](../AGENTS.md#git-workflows) — Release versioning and tagging patterns +- [Semantic Versioning](https://semver.org/) +- [GitHub Actions Documentation](https://docs.github.com/en/actions) +- [PyPI Documentation](https://pypi.org/)