diff --git a/.github/workflows/check-removed-urls.yml b/.github/workflows/check-removed-urls.yml index 22f72efe..e00d4ad7 100644 --- a/.github/workflows/check-removed-urls.yml +++ b/.github/workflows/check-removed-urls.yml @@ -2,6 +2,13 @@ name: Check for removed URLs on: workflow_call: + inputs: + base_ref: + required: false + type: string + base_repo: + required: false + type: string pull_request: branches: [main] @@ -21,8 +28,8 @@ jobs: - name: Checkout base branch uses: actions/checkout@v5 with: - ref: ${{ github.event.pull_request.base.ref }} - repository: ${{ github.event.pull_request.base.repo.full_name }} + ref: ${{ inputs.base_ref || github.event.pull_request.base.ref }} + repository: ${{ inputs.base_repo || github.event.pull_request.base.repo.full_name }} fetch-depth: 0 path: base - uses: actions/setup-python@v6 @@ -36,20 +43,17 @@ jobs: - name: Generate current URLs list run: | for dir in compare base; do - pushd ${dir}/docs - find ./_build/ -name '*.html' \ - | sed 's|/_build||;s|/index.html$|/|;s|.html$||' \ - | sort > urls.txt - popd + build_dir="${dir}/docs/_build" + urls_file="${dir}/docs/urls.txt" + + if [ ! -d "${build_dir}" ]; then + echo "Expected docs build directory not found: ${build_dir}" + exit 1 + fi + + find "${build_dir}" -name '*.html' \ + | sed "s|^${build_dir}||;s|^/html||;s|/index.html$|/|;s|.html$||" \ + | sort > "${urls_file}" done - name: Compare URLs - run: | - BASE_URLS_PATH="base/docs/urls.txt" - COMPARE_URLS_PATH="compare/docs/urls.txt" - removed=$(comm -23 ${BASE_URLS_PATH} ${COMPARE_URLS_PATH} ) - if [ -n "$removed" ]; then - echo "The following URLs were removed:" - echo "$removed" - echo "Please ensure removed pages are redirected" - exit 1 - fi + run: python3 compare/docs/_dev/check_removed_urls.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 711ead50..733701d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Upcoming +* Prevent Vale from processing Markdown files in the build directory +* Update link to documenation in README +* Make removed URL check redirect-aware and add support for reusable workflow contexts * Exclude utility directories from builds and checks * Update link to documentation in README @@ -11,6 +14,7 @@ * `docs/Makefile` [#605](https://github.com/canonical/sphinx-stack/pull/605), [#610](https://github.com/canonical/sphinx-stack/pull/610) * `README.md` [#603](https://github.com/canonical/sphinx-stack/pull/603) * `.github/workflows/cla-check.yml` [#606](https://github.com/canonical/sphinx-stack/pull/606) +* `.github/workflows/check-removed-urls.yml` [#PR_NUMBER](https://github.com/canonical/sphinx-stack/pull/#PR_NUMBER) ## 2.0 diff --git a/docs/_dev/check_removed_urls.py b/docs/_dev/check_removed_urls.py new file mode 100755 index 00000000..4158bf93 --- /dev/null +++ b/docs/_dev/check_removed_urls.py @@ -0,0 +1,120 @@ +#! /usr/bin/env python + +"""Check for removed URLs and verify if redirects exist.""" + +import argparse +import csv +import io +import sys +from pathlib import Path + + +def read_urls(path): + return { + line.strip() + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + } + + +def read_redirect_sources(path): + sources = set() + + if not path.exists(): + return sources + + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + + fields = next( + csv.reader( + io.StringIO(line), + delimiter=" ", + quotechar='"', + skipinitialspace=True, + ), + [], + ) + if fields: + sources.add(fields[0]) + + return sources + + +def source_candidates_for_url(url): + clean_path = url.strip() + clean_path = clean_path.removeprefix("./") + clean_path = clean_path.removeprefix("/") + clean_path = clean_path.removesuffix(".html") + clean_path = clean_path.rstrip("/") + + if not clean_path: + return {"index.md"} + + # A removed dirhtml URL can map back to either a page file or an + # index file. Directory-level redirects are stored with a trailing + # slash, so include that form too. + return { + f"{clean_path}.md", + f"{clean_path}/index.md", + f"{clean_path}/", + } + + +def main(): + parser = argparse.ArgumentParser( + description="Check for removed URLs and verify if redirects exist." + ) + parser.add_argument( + "--base-urls", + type=Path, + default=Path("base/docs/urls.txt"), + help="Path to base branch URLs file", + ) + parser.add_argument( + "--compare-urls", + type=Path, + default=Path("compare/docs/urls.txt"), + help="Path to compare branch URLs file", + ) + parser.add_argument( + "--redirects", + type=Path, + default=Path("compare/docs/redirects.txt"), + help="Path to redirects.txt file", + ) + args = parser.parse_args() + + if not args.base_urls.exists(): + print(f"Error: Base URLs file not found at {args.base_urls}") + sys.exit(1) + if not args.compare_urls.exists(): + print(f"Error: Compare URLs file not found at {args.compare_urls}") + sys.exit(1) + + removed_urls = sorted( + read_urls(args.base_urls) - read_urls(args.compare_urls) + ) + redirect_sources = read_redirect_sources(args.redirects) + + missing_redirects = [ + url + for url in removed_urls + if source_candidates_for_url(url).isdisjoint(redirect_sources) + ] + + if missing_redirects: + print("The following URLs were removed without redirects:") + print("\n".join(missing_redirects)) + print("Please ensure removed pages are redirected") + sys.exit(1) + + if removed_urls: + print("Removed URLs have redirects:") + print("\n".join(removed_urls)) + + +if __name__ == "__main__": + main()