diff --git a/.gitignore b/.gitignore index f9bc4010d69..ad550fc4359 100644 --- a/.gitignore +++ b/.gitignore @@ -301,6 +301,7 @@ assets/jsconfig.json # local build files .backup_path local/bin/py/build/* +!local/bin/py/build/configurations/ !local/bin/py/build/configurations/pull_config_preview.yaml !local/bin/py/build/configurations/pull_config.yaml !local/bin/py/build/configurations/integration_merge.yaml @@ -338,3 +339,4 @@ playwright-report/ .claude/*local* .superpowers/ data/site_region_usage.json + diff --git a/REPO_REORG.md b/REPO_REORG.md new file mode 100644 index 00000000000..4ba61c51a87 --- /dev/null +++ b/REPO_REORG.md @@ -0,0 +1,29 @@ +# `documentation` repository reorg + +# Reorg status + +Not yet merged. + +# Project overview + +We're gradually migrating the Datadog docs site from Hugo to Astro. To support this, we must reorganize the `documentation` repo into a monorepo that supports two sites (`hugo` and `astro`), instead of having the entire repo serve as our Hugo site. + +**This reorg causes conflicts in almost every open PR in the `documentation` repo.** + +## What does the reorg change? + +The most impactful change is that any Hugo-specific site files move to a `hugo` folder instead of being located at the top level of the repo. This change impacts thousands of files, and causes the majority of conflicts. + +For a full list of repo files and folders and their updated location, see [the configuration file for the reorg script](https://github.com/DataDog/documentation/blob/jen.gilbert/astro-reorg-scripts/astro_reorg/config.yaml). + +# Handling PR conflicts + +Once the reorg is merged to master, all open PRs are processed. Conflicts are fixed automatically where possible. When an automatic fix is not possible, we defer to the author's best judgment in resolving the conflicts, but offer escalation options in the PR comments in case they need help. + +## PR cases + +* **Skipped because it has no conflicts:**: We do not try to autofix the PR because it doesn't have any conflicts. We add a label as a guard against further processing. +* **Skipped as a WIP:**: We do not try to autofix the PR because it has the `WORK IN PROGRESS` label. We leave a comment describing how to queue the PR for autofix processing if desired. +* **Autofixed PR:** We fix the conflicts automatically, close the PR, and link a new PR for the author's review. +* **Stale PR:** We do not act on the PR, because it appears to be stale. We leave a comment describing how to queue the PR for autofix processing if desired. +* **Manual fix required:** We cannot resolve the conflicts automatically. We leave a comment describing why the conflicts are occurring, and how to ask for help if needed. \ No newline at end of file diff --git a/astro_reorg/CLAUDE.md b/astro_reorg/CLAUDE.md new file mode 100644 index 00000000000..fbb91b895a3 --- /dev/null +++ b/astro_reorg/CLAUDE.md @@ -0,0 +1,23 @@ +# Docs repository reorg context + +This repo is currently a Hugo site. We instead want it to contain a `hugo` and `astro` site side by side, with no overlap in their envs, `package.json` files, etc. + +`astro_reorg/config.yaml` describes the relocation target for every file and folder at the top level of the repo. + +`astro_reorg/execute_reorg.py` implements the file and folder path changes, and updates any dependencies on those paths, such as GitHub actions, CODEOWNERS, and Husky workflows. + +`astro_reorg/helpers.py` contains utilities (path manipulation, git/shell helpers, YAML config loading) used by `validate_reorg.py`. The other scripts predate it and each define their own small helpers inline. + +`astro_reorg/local_rollback.py` functions as an "undo" action for the reorg: removes `hugo/` and restores `.gitignore`, `.github/`, and `.husky/` from git. + +`astro_reorg/validate_reorg.py` verifies the functionality of affected entities where possible. + +`astro_reorg/resolve_pr_conflicts.py` finds open PRs with merge conflicts caused by the reorg, and either auto-fixes them (by replaying commits at post-reorg paths) or labels them for manual review. It defaults to a dry run against a mock base branch, so it can't touch real PRs by accident. + +`astro_reorg/create_test_prs.py` creates test PRs for exercising `resolve_pr_conflicts.py`, including the conflicts it can't auto-fix. Each spec in `TEST_PRS` becomes a PR that reproduces one conflict scenario. + +Both scripts share the `test:` section of `config.yaml`, which names the branches a test run uses: `mock_reorged_master_branch` (the stand-in for post-reorg `master` that test PRs target and the resolver treats as the base) and `branch_from` (the frozen snapshot the test PRs are cut from). + +For flags and the mechanics of each scenario, read the script's module docstring and the comments on `TEST_PRS` — that's the source of truth, so this file doesn't have to track them. + +You can ignore the `astro` folder, it's a remnant from another branch where an Astro site is being developed. It is completely out of scope. \ No newline at end of file diff --git a/astro_reorg/close_stale_prs.py b/astro_reorg/close_stale_prs.py new file mode 100644 index 00000000000..5c63de7c77d --- /dev/null +++ b/astro_reorg/close_stale_prs.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +Close open PRs that carry the `autolabeled-stale` label. +Posts a comment before closing. Defaults to a dry run where it just reports +what would be done. + +Use the GitHub label query to review which PRs are queued for closure: +https://github.com/DataDog/documentation/pulls?q=is%3Aopen+label%3Aautolabeled-stale + +Usage: + python3 astro_reorg/close_stale_prs.py [--no-dry-run] [--pr NUMBER ...] [--limit N] + +Flags: + --no-dry-run Actually close PRs instead of just reporting what would be done. + --pr NUMBER ... Only process the given PR number(s). + --limit N Stop after closing N PRs. PRs that are skipped don't count toward + the limit. Use it to roll out gradually: start with --limit 1, review + the results, then raise it as you gain confidence. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from datetime import datetime, timedelta, timezone + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +REPO = "DataDog/documentation" + +LABEL_STALE = "autolabeled-stale" + +CLOSE_COMMENT = ( + "Closing this PR because it has had no activity for six months. " + "If this was done in error, feel free to re-open the PR." +) + +# --------------------------------------------------------------------------- +# Shell helpers +# --------------------------------------------------------------------------- + +def run(cmd: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True) + + +def gh_json(*args: str) -> object: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return json.loads(result.stdout) if result.stdout.strip() else [] + + +def gh_run(*args: str) -> str: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return result.stdout + + +# --------------------------------------------------------------------------- +# GitHub helpers +# --------------------------------------------------------------------------- + +def post_comment(pr_number: int, body: str, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would comment on PR #{pr_number}:\n {body[:120]}") + return + gh_run("pr", "comment", str(pr_number), "--repo", REPO, "--body", body) + print(f" Posted comment on PR #{pr_number}") + + +def close_pr(pr_number: int, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would close PR #{pr_number}") + return + gh_run("pr", "close", str(pr_number), "--repo", REPO) + print(f" Closed PR #{pr_number}") + + +# --------------------------------------------------------------------------- +# Per-PR processing +# --------------------------------------------------------------------------- + +def process_pr(pr: dict, dry_run: bool) -> bool: + """Comment and close a labeled PR. Return True if we acted, False if we skipped.""" + pr_number = pr["number"] + title = pr["title"] + + print(f"\nPR #{pr_number}: {title}") + print(f" Last updated: {pr.get('updatedAt', 'unknown')}") + + post_comment(pr_number, CLOSE_COMMENT, dry_run) + close_pr(pr_number, dry_run) + return True + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def get_labeled_prs(only: list[int] | None = None, fetch_limit: int = 1000) -> list[dict]: + """Return open PRs carrying the stale label, optionally filtered to specific numbers.""" + fields = "number,title,updatedAt" + if only: + prs = [] + for n in only: + pr = gh_json("pr", "view", str(n), "--repo", REPO, "--json", fields) + prs.append(pr) + return prs # type: ignore[return-value] + return gh_json( # type: ignore[return-value] + "pr", "list", "--repo", REPO, "--state", "open", + "--label", LABEL_STALE, + "--json", fields, "--limit", str(fetch_limit), + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description=f"Close open PRs labeled {LABEL_STALE!r}.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "--dry-run", action=argparse.BooleanOptionalAction, default=True, + help="Report what would be done without making any changes (default: on). " + "Use --no-dry-run to apply changes.", + ) + parser.add_argument( + "--pr", type=int, action="append", dest="prs", metavar="NUMBER", + help="Only process this PR number (may be repeated).", + ) + parser.add_argument( + "--limit", type=int, default=None, metavar="N", + help="Stop after closing N PRs. Skipped PRs don't count toward the limit. " + "Use it to roll out gradually: start at 1, review, then raise it.", + ) + args = parser.parse_args() + + if args.limit is not None and args.limit < 1: + parser.error("--limit must be a positive integer.") + + if args.dry_run: + print("DRY-RUN mode — no PRs will be modified.\n") + + prs = get_labeled_prs(args.prs) + print(f"Found {len(prs)} labeled PR(s) to close.") + if args.limit is not None: + print(f"Limit: will stop after closing {args.limit} PR(s).") + + acted = 0 + for pr in prs: + if args.limit is not None and acted >= args.limit: + print(f"\nReached --limit of {args.limit} closed PR(s) — stopping.") + break + try: + if process_pr(pr, args.dry_run): + acted += 1 + except Exception as exc: + print(f"\nERROR processing PR #{pr.get('number', '?')}: {exc}", + file=sys.stderr) + print(" Skipping to the next PR.", file=sys.stderr) + + print(f"\nClosed {acted} PR(s). Done.") + + +if __name__ == "__main__": + main() diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml new file mode 100644 index 00000000000..bb46a5eae47 --- /dev/null +++ b/astro_reorg/config.yaml @@ -0,0 +1,139 @@ +# Files and folders that stay at the top level (not moved into hugo/). +# Everything else moves into hugo/. + +top_level: + # Sites + - astro + + # Repo scripts + - astro_reorg + - reorg_issues.md + + # Created by reorg.py as the move target + - hugo + + # Git / CI / repo-wide tooling + - .git + - .github + - .gitlab-ci.yml + - .gitignore + - .gitattributes + - .husky + - .claude + + # Editor / IDE config + - .editorconfig + - .vscode + + # Repo docs + - README.md + - LICENSE + - CONTRIBUTING.md + - CLAUDE.md + - REPO_REORG.md + + # Linting / formatting (shared) + - .eslintrc + - .eslintignore + - .vale.ini + - .rubocop.yml + - .prettierignore + - prettier.config.js + - static-analysis.datadog.yml + + # Datadog CI / synthetics + - datadog-ci.preview.json + - general.preview.synthetics.json + - repository.datadog.yml + + # Docker + - docker-compose-docs.yml + +moves_to_hugo: + # Hugo core + - archetypes + - assets + - config + - content + - data + - i18n + - layouts + - static + - resources + - public + + # Hugo build tooling + - Makefile + - Makefile.config + - Makefile.config.example + - local + - hugpython + - gradle + - _vendor + + # Hugo build artifacts / state + - .hugo_build.lock + - logs + - integrations_data + + # Node / JS (Hugo-specific) + - babel.config.js + - jest.config.js + - postcss.config.js + - markdoc.config.json + - customization_config + + # Go module — powers Hugo Modules (imports websites-modules, vendors into + # _vendor/). Hugo requires go.mod in the project root; Astro uses no Go. + - go.mod + - go.sum + + # Node / Yarn (Hugo site owns these; Astro has its own under astro/) + - package.json + - package-lock.json + - yarn.lock + - .yarnrc.yml + - .yarn + - node_modules + - .nvmrc + + # Testing (Hugo-specific) + - playwright.config.ts + - playwright-report + - test-results + - e2e + + # Content tooling + - translate.yaml + - .translate + - typesense.config.json + - .htmltest.yml + - .apigentools-info + - agent_config_types_list.txt + + # Examples / docs + - examples + - docs + - plans + + # Static assets (Hugo) + - usage-notifications-email.png + +ignore: + # macOS metadata + - .DS_Store + # Python bytecode cache + - __pycache__ + +# Branches used to exercise the reorg tooling against a fake main instead of the +# real master. create_test_prs.py opens test PRs against mock_reorged_master_branch, and +# resolve_pr_conflicts.py treats that same branch as the post-reorg base by +# default (pass --live to target the real master), so the two scripts always +# agree on which branches to use during a test run. +test: + # Stand-in for post-reorg master: the branch that has the reorg applied. Test + # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. + mock_reorged_master_branch: mock-reorged-master-7-20-1231 + # Frozen snapshot of master that test PRs branch off, so each PR's diff stays + # small (a PR shows every commit in its head that isn't in the base branch). + branch_from: jen.gilbert/astro-reorg-scripts diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py new file mode 100644 index 00000000000..52e69d78e83 --- /dev/null +++ b/astro_reorg/create_test_prs.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python3 +"""Create test PRs against a mock base branch for exercising the reorg tooling. + +For each spec in TEST_PRS this script: + 1. branches off BRANCH_FROM (a frozen snapshot of master), + 2. applies the spec's content change, + 3. commits and pushes the branch, + 4. opens a PR against the base branch (see below). + +BRANCH_FROM and MOCK_BASE_BRANCH are read from the `test:` section of +config.yaml (shared with resolve_pr_conflicts.py, which defaults to this same +mock base branch). Both must already exist on the remote before running. + +Testing the *unresolvable* conflict paths: + By default the mock base differs from the PRs only by pure file moves, so + resolve_pr_conflicts.py always replays them cleanly. To exercise the cases + where it CAN'T auto-fix, a spec may carry a `base_edit` describing a change + to make on the base itself — on the same line the PR touches, but at the + post-reorg (hugo/) path. When any spec has a base_edit, this script builds a + throwaway base branch off MOCK_BASE_BRANCH (unique per run, so no + force-push), applies every base_edit to it, and points ALL the PRs at that + branch. Then one resolve_pr_conflicts.py run against it exercises both a + clean auto-fix and the manual-review fallbacks. The exact command to run is + printed at the end. + + - A base_edit on a moved (hugo/) file makes git am --3way fail: the fix is + classified reorg-caused but can't be replayed (manual-review fallback). + - A base_edit on a top-level (never-moved) file surfaces as a plain + non-reorg conflict, which is sent straight to manual review. + +View all test PRs: + https://github.com/DataDog/documentation/pulls?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-testing +""" +from __future__ import annotations + +import subprocess +import sys +import uuid +import webbrowser +from datetime import datetime, timedelta, timezone +from pathlib import Path + +try: + import yaml +except ImportError: + print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +REPO = "DataDog/documentation" +REPO_ROOT = Path(__file__).parent.parent +CONFIG_PATH = Path(__file__).parent / "config.yaml" + +# The test branches live in config.yaml under `test:` so this script and +# resolve_pr_conflicts.py always agree on which branches a test run uses. +with CONFIG_PATH.open() as f: + _test_config = yaml.safe_load(f).get("test", {}) + +# Branch the test PRs target. Create this on the remote before running. +MOCK_BASE_BRANCH = _test_config["mock_reorged_master_branch"] + +# Branch the test PRs are cut from. This is a frozen snapshot of master (created +# manually) so PR diffs stay small: a PR shows every commit in the head branch +# that isn't in the base, so branching off a moving master would drag in every +# new master commit as noise. Branching off this frozen point keeps each PR to +# just its own change. Must already exist on the remote. +BRANCH_FROM = _test_config["branch_from"] + +# Distinctive prefix so branches created here are obvious and easy to clean up. +BRANCH_PREFIX = "jen.gilbert/astro-reorg-test" + +# Label applied to every PR so they can all be found together. Created by the +# script if missing (see ensure_label). +LABEL = "astro-reorg-testing" +LABEL_COLOR = "e4e669" +LABEL_DESCRIPTION = "Test PRs for exercising the astro reorg tooling" + +# Existing repo label applied to every test PR to guard against accidental +# merges. Assumed to already exist in the repo. +DO_NOT_MERGE_LABEL = "Do Not Merge" + +# Each spec describes one PR: the branch to create, the file to edit, the exact +# text to replace (old -> new, must match once), and the PR title/body. +# +# An optional `base_edit` makes the conflict unresolvable by having the BASE +# branch also change the same line (at the post-reorg hugo/ path). See the +# module docstring. Specs without a base_edit still auto-fix cleanly, so a run +# that mixes both kinds exercises every resolve_pr_conflicts.py outcome at once. +# +# An optional `add_file` ({path, content}) also creates a brand-new page at a +# pre-reorg path, on top of the old->new edit, to model "added a page and linked +# it in the nav menu." +TEST_PRS = [ + { + # Resolvable: reorg-only conflict, auto-fix replays it cleanly. + # + # IMPORTANT: this edit must target a different LINE than any base_edit + # in the other specs. Git merges at line granularity, so even two edits + # to different words on the same long line will conflict. The description + # frontmatter field is on its own line and no base_edit touches it. + "branch": f"{BRANCH_PREFIX}-wording", + "file": "content/en/getting_started/_index.md", + "old": "with guides for installation, configuration, and getting started with key features.", + "new": "with guides for setup, configuration, and getting started with key features.", + "commit": "Test PR: minor wording tweak in getting started description", + "title": "[TEST] Minor wording tweak in getting started intro (auto-fixable)", + "body": ( + "Test PR for exercising the astro reorg tooling. Makes a minor, " + "non-material wording change in the getting started page description.\n\n" + "Do not merge." + ), + "expected_outcome": ( + "**Auto-fixed.** The conflict is purely reorg-caused (file moved from " + "`content/` to `hugo/content/`). The resolver opens a new PR with the " + "commit replayed at the post-reorg path, closes this PR, and labels it " + "`astro-reorg-autofixed`." + ), + }, + { + # Auto-fixable, but marked WIP — resolve_pr_conflicts.py should skip it + # and leave a comment telling the author to add astro-reorg-help-requested. + # Uses a different file from the other specs to avoid line-level collisions. + "branch": f"{BRANCH_PREFIX}-wip", + "file": "content/en/getting_started/learning_center.md", + "old": "Access hands-on courses and tutorials to learn Datadog platform features, best practices, and implementation strategies.", + "new": "Access hands-on courses and tutorials to learn Datadog platform features, best practices, and deployment strategies.", + "commit": "Test PR: minor wording tweak in learning center description (WIP)", + "title": "[TEST] WIP PR with reorg conflict (should be skipped by resolver)", + "body": ( + "Test PR for exercising the astro reorg tooling. Auto-fixable reorg conflict, " + "but marked WIP so the resolver should leave a comment and skip it.\n\n" + "Do not merge." + ), + "extra_labels": ["WORK IN PROGRESS"], + "expected_outcome": ( + "**Skipped (WIP).** The conflict is reorg-caused and would otherwise be " + "auto-fixable, but the PR carries the `WORK IN PROGRESS` label. The resolver " + "posts a comment explaining the situation, applies `astro-reorg-skip`, and " + "leaves the PR open for the author to act on." + ), + }, + { + # Unresolvable (Case 2): reorg-classified, but the base changed the same + # line at the hugo/ path, so git am --3way can't replay it. Falls back + # to the astro-reorg-manual-review label. + "branch": f"{BRANCH_PREFIX}-unresolvable-reorg", + "file": "content/en/getting_started/_index.md", + "old": "combined into a customized solution", + "new": "combined into a unified solution", + "commit": "Test PR: edit a line the reorged base also changed", + "title": "[TEST] Unresolvable reorg conflict (base edited the same line) (not auto-fixable)", + "body": ( + "Test PR for exercising the astro reorg tooling. Edits a line that " + "the reorged base branch also changed, so the auto-fix can't replay " + "it and the PR should be labeled for manual review.\n\n" + "Do not merge." + ), + "base_edit": { + "file": "hugo/content/en/getting_started/_index.md", + "old": "combined into a customized solution", + "new": "combined into a tailored solution", + }, + "expected_outcome": ( + "**Manual review (unresolvable reorg conflict).** The conflict is classified " + "as reorg-caused, but the base branch also edited the same line at the " + "post-reorg path (`hugo/content/en/getting_started/_index.md`), so " + "`git am --3way` cannot replay the patch. The resolver labels this PR " + "`astro-reorg-manual-review` and posts a comment." + ), + }, + { + # Unresolvable (Case 1): the conflict is on a top-level file the reorg + # never moves, so it isn't reorg-caused at all — sent straight to manual + # review with no auto-fix attempt. + "branch": f"{BRANCH_PREFIX}-non-reorg-conflict", + "file": "CONTRIBUTING.md", + "old": "how to write and edit content", + "new": "how to write and revise content", + "commit": "Test PR: edit a top-level file the base also changed", + "title": "[TEST] Non-reorg conflict on a top-level file (not auto-fixable)", + "body": ( + "Test PR for exercising the astro reorg tooling. Creates a conflict " + "on a top-level file that the reorg doesn't move, so it isn't a " + "reorg conflict and should go straight to manual review.\n\n" + "Do not merge." + ), + "base_edit": { + "file": "CONTRIBUTING.md", + "old": "how to write and edit content", + "new": "how to author and edit content", + }, + "expected_outcome": ( + "**Manual review (non-reorg conflict).** The conflict is in `CONTRIBUTING.md`, " + "a top-level file the reorg never moves. The resolver classifies it as an " + "unrelated conflict and immediately labels this PR `astro-reorg-manual-review` " + "without attempting an auto-fix." + ), + }, + { + # Auto-fixable: a new page plus a nav-menu link to it, where nobody else + # touched that menu line. Exercises the "wrong-path addition + menu edit" + # case where both changes can be replayed cleanly by the auto-fix. + # Uses a different menu insertion point ("Agent" heading) than the + # unresolvable spec below ("Essentials" heading) to avoid line collisions. + "branch": f"{BRANCH_PREFIX}-new-page-nav-autofixable", + "add_file": { + "path": "content/en/getting_started/example_auto_fix/_index.md", + "content": ( + "---\n" + "title: Example Auto Fix\n" + "---\n\n" + "Placeholder page for exercising the astro reorg tooling.\n" + ), + }, + "file": "config/_default/menus/main.en.yaml", + "old": " - name: Agent\n identifier: getting_started_agent", + "new": ( + " - name: Example Auto Fix\n" + " identifier: example_auto_fix_heading\n" + " url: /getting_started/example_auto_fix/\n" + " weight: 500000\n" + " - name: Agent\n" + " identifier: getting_started_agent" + ), + "commit": "Test PR: add a page and a nav menu link (no base conflict)", + "title": "[TEST] New page with nav link, no base conflict (auto-fixable)", + "body": ( + "Test PR for exercising the astro reorg tooling. Adds a new page and " + "a nav menu entry linking to it. The base branch does not touch that " + "menu line, so the auto-fix should replay both changes cleanly.\n\n" + "Do not merge." + ), + "expected_outcome": ( + "**Auto-fixed.** The PR adds a new page at a pre-reorg path (a " + "\"wrong-path addition\") and links it in the nav menu. Both changes are " + "reorg-caused and the base branch did not touch that menu line, so the " + "resolver replays both commits cleanly at the post-reorg paths, opens a " + "new PR, closes this one, and labels it `astro-reorg-autofixed`." + ), + }, + { + # Unresolvable (Case 3): a new page plus a nav-menu link to it, where + # the base branch edited that same menu line. The new page is a + # wrong-path addition (added at a pre-reorg content/ path) AND the menu + # edit conflicts with the base, so the fix is classified reorg-caused + # but git am --3way can't replay the menu change -> manual review. + "branch": f"{BRANCH_PREFIX}-new-page-nav-conflict", + "add_file": { + "path": "content/en/getting_started/example_feature/_index.md", + "content": ( + "---\n" + "title: Example Feature\n" + "---\n\n" + "Placeholder page for exercising the astro reorg tooling.\n" + ), + }, + "file": "config/_default/menus/main.en.yaml", + # Insert a nav entry for the new page just before the Essentials heading. + "old": " - name: Essentials", + "new": ( + " - name: Example Feature\n" + " identifier: example_feature_heading\n" + " url: /getting_started/example_feature/\n" + " weight: 500000\n" + " - name: Essentials" + ), + "commit": "Test PR: add a page and a nav menu link to it", + "title": "[TEST] New page with nav link (base edited the same menu line) (not auto-fixable)", + "body": ( + "Test PR for exercising the astro reorg tooling. Adds a new page and " + "a nav menu entry linking to it. The base branch renames that same " + "menu heading, so the auto-fix can't replay the menu change and the " + "PR should be labeled for manual review.\n\n" + "Do not merge." + ), + # Someone else renamed the same heading the PR inserts in front of. + "base_edit": { + "file": "hugo/config/_default/menus/main.en.yaml", + "old": " - name: Essentials", + "new": " - name: Essential Features", + }, + "expected_outcome": ( + "**Manual review (unresolvable reorg conflict).** The PR adds a new page " + "at a pre-reorg path and inserts a nav menu entry. The conflict is classified " + "as reorg-caused, but the base branch also renamed the same menu heading, so " + "`git am --3way` cannot replay the nav edit. The resolver labels this PR " + "`astro-reorg-manual-review` and posts a comment." + ), + }, +] + +# --------------------------------------------------------------------------- +# Shell helpers +# --------------------------------------------------------------------------- + +def run(cmd: list[str], *, cwd: Path | None = None) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True, cwd=cwd or REPO_ROOT) + + +def git(*args: str) -> subprocess.CompletedProcess: + return run(["git", *args]) + + +def gh(*args: str) -> subprocess.CompletedProcess: + return run(["gh", *args]) + + +def die(message: str) -> None: + print(f"ERROR: {message}", file=sys.stderr) + sys.exit(1) + + +def ensure_label() -> None: + """Create the shared label, treating "already exists" as success. + + We don't pre-check with `gh label list`: it defaults to 30 labels, so in a + repo with many labels an existing one may not show up, and we'd then fail on + create. Instead we just create and accept the "already exists" error. + """ + create = gh("label", "create", LABEL, "--repo", REPO, + "--color", LABEL_COLOR, "--description", LABEL_DESCRIPTION) + if create.returncode == 0: + print(f"Created label: {LABEL!r}") + return + if "already exists" in create.stderr.lower(): + return + die(f"could not create label {LABEL!r}: {create.stderr.strip()}") + + +# --------------------------------------------------------------------------- +# Preflight +# --------------------------------------------------------------------------- + +def preflight() -> None: + """Fail fast on uncommitted tracked changes, or if the mock base is missing. + + Only TRACKED changes matter: the script switches branches and builds a + commit, so staged/unstaged edits could be carried across or swept in. + Untracked files (e.g. a folder from another branch that doesn't exist here) + are ignored — they don't block checkouts and never enter the commit, which + only `git add`s the one target file. + """ + unstaged = git("diff", "--quiet").returncode != 0 + staged = git("diff", "--cached", "--quiet").returncode != 0 + if unstaged or staged: + die("you have uncommitted changes to tracked files; " + "commit or stash them first. (Untracked files are fine.)") + + # Both the branch-off point and the PR target must exist on the remote. + for name in (BRANCH_FROM, MOCK_BASE_BRANCH): + ls = git("ls-remote", "--heads", "origin", name) + if not ls.stdout.strip(): + die(f"branch '{name}' not found on origin. " + f"Create and push it before running.") + + +def current_branch() -> str: + return git("rev-parse", "--abbrev-ref", "HEAD").stdout.strip() + + +# --------------------------------------------------------------------------- +# PR creation +# --------------------------------------------------------------------------- + +def build_conflicting_base(specs: list[dict]) -> str | None: + """Build a throwaway base branch that conflicts with the base_edit specs. + + Some specs carry a `base_edit` describing a change to make ON THE BASE, at a + post-reorg (hugo/) path, on the same line the PR touches at the pre-reorg + path. That divergent edit is what makes resolve_pr_conflicts.py's auto-fix + fail (git am --3way can't reconcile the line) or — for a top-level file — + surfaces as a plain non-reorg conflict. Without it the mock base differs + from the PRs only by pure file moves, which always replay cleanly. + + Builds a fresh branch off origin/MOCK_BASE_BRANCH, unique per run so a plain + push always fast-forwards (no force-push, per repo policy), applies every + base_edit, commits, and pushes. Returns the branch name, or None if no spec + needs a conflicting base (in which case callers use MOCK_BASE_BRANCH). + """ + edits = [s["base_edit"] for s in specs if s.get("base_edit")] + if not edits: + return None + + branch = f"{MOCK_BASE_BRANCH}-conflict-{uuid.uuid4().hex[:8]}" + print(f"\n=== building conflicting base {branch} ===") + + if git("fetch", "origin", MOCK_BASE_BRANCH).returncode != 0: + die(f"git fetch origin {MOCK_BASE_BRANCH} failed.") + + checkout = git("checkout", "-b", branch, f"origin/{MOCK_BASE_BRANCH}") + if checkout.returncode != 0: + die(f"could not create branch {branch}: {checkout.stderr.strip()}") + + # base_edit paths are post-reorg (hugo/...), so they exist on this branch. + for edit in edits: + target = REPO_ROOT / edit["file"] + if not target.exists(): + die(f"base_edit target does not exist on the base branch: {edit['file']}") + text = target.read_text() + count = text.count(edit["old"]) + if count != 1: + die(f"expected {edit['old']!r} exactly once in {edit['file']}, " + f"found {count}.") + target.write_text(text.replace(edit["old"], edit["new"])) + git("add", edit["file"]) + + commit = git("commit", "-m", + "Test setup: edit reorged files so open test PRs conflict") + if commit.returncode != 0: + die(f"base commit failed: {commit.stderr.strip()}") + + push = git("push", "--set-upstream", "origin", branch) + if push.returncode != 0: + die(f"push of conflicting base failed: {push.stderr.strip()}") + + print(f" Pushed conflicting base: {branch}") + return branch + + +def create_pr(spec: dict, base: str) -> str | None: + """Create the branch, apply the change, push, and open a PR. Returns URL.""" + # Append a short unique suffix so every run creates a fresh branch rather + # than colliding with one from a previous run. + branch = f"{spec['branch']}-{uuid.uuid4().hex[:8]}" + target = REPO_ROOT / spec["file"] + print(f"\n=== {branch} ===") + + # Start from the frozen snapshot so the PR diff is just this change. + if git("fetch", "origin", BRANCH_FROM).returncode != 0: + die(f"git fetch origin {BRANCH_FROM} failed.") + + checkout = git("checkout", "-b", branch, f"origin/{BRANCH_FROM}") + if checkout.returncode != 0: + die(f"could not create branch {branch}: {checkout.stderr.strip()}") + + # Check after checkout: the file exists on BRANCH_FROM (pre-reorg) but not + # on the conflicting base branch (post-reorg), so the check must run here. + if not target.exists(): + die(f"target file does not exist: {spec['file']}") + + # Optionally add a brand-new page at a pre-reorg path (content/...). The + # resolver detects this as a "wrong-path addition" that belongs under hugo/. + add_file = spec.get("add_file") + if add_file: + new_page = REPO_ROOT / add_file["path"] + if new_page.exists(): + die(f"add_file target already exists: {add_file['path']}") + new_page.parent.mkdir(parents=True, exist_ok=True) + new_page.write_text(add_file["content"]) + git("add", add_file["path"]) + + # Apply the wording change. + text = target.read_text() + count = text.count(spec["old"]) + if count != 1: + die(f"expected to find {spec['old']!r} exactly once in {spec['file']}, " + f"found {count}.") + target.write_text(text.replace(spec["old"], spec["new"])) + + git("add", spec["file"]) + commit = git("commit", "-m", spec["commit"]) + if commit.returncode != 0: + die(f"commit failed: {commit.stderr.strip()}") + + # Branch name is unique per run, so a plain push always fast-forwards. + push = git("push", "--set-upstream", "origin", branch) + if push.returncode != 0: + die(f"push failed: {push.stderr.strip()}") + + label_args: list[str] = ["--label", LABEL, "--label", DO_NOT_MERGE_LABEL] + for extra in spec.get("extra_labels", []): + label_args += ["--label", extra] + pr = gh( + "pr", "create", + "--repo", REPO, + "--head", branch, + "--base", base, + "--title", spec["title"], + "--body", spec["body"], + *label_args, + ) + if pr.returncode != 0: + die(f"gh pr create failed: {pr.stderr.strip()}") + + url = pr.stdout.strip().splitlines()[-1] + print(f" Opened PR against {base}: {url}") + return url + + +def write_pr_list( + pr_pairs: list[tuple[dict, str]], + base: str, + conflicting_base: str | None, + safe_after: datetime, +) -> None: + """Write test_pr_list.md at the repo root listing each PR and its expected outcome.""" + base_flag = f"--base-branch {conflicting_base}" if conflicting_base else "--live" + n = len(pr_pairs) + safe_after_str = safe_after.strftime("%Y-%m-%d %H:%M %Z") + lines = [ + "# Test PR list", + "", + "Generated by `astro_reorg/create_test_prs.py`. Do not edit by hand.", + "", + f"**Base branch:** `{base}` ", + f"**Branch PRs are cut from:** `{BRANCH_FROM}` ", + f"**Safe to resolve after:** {safe_after_str} (GitHub needs ~5 min to compute mergeability)", + "", + "## Expected outcomes", + "", + "Run the resolver against these PRs to verify each outcome:", + "", + f"```", + f"# Dry run (no changes):", + f"python3 astro_reorg/resolve_pr_conflicts.py {base_flag} --limit {n}", + f"", + f"# Live run (applies fixes and labels):", + f"python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run {base_flag} --limit {n}", + f"```", + "", + "## PRs", + "", + ] + for spec, url in pr_pairs: + lines += [ + f"### [{spec['title']}]({url})", + "", + spec["expected_outcome"], + "", + ] + + output_path = REPO_ROOT / "astro_reorg" / "docs" / "test_pr_list.md" + output_path.write_text("\n".join(lines)) + print(f"\nWrote {output_path}") + + +def main() -> None: + preflight() + ensure_label() + original_branch = current_branch() + + pr_pairs: list[tuple[dict, str]] = [] + conflicting_base: str | None = None + base: str = MOCK_BASE_BRANCH + try: + # If any spec needs the base to diverge, build one throwaway conflicting + # base and point every PR at it, so a single resolve_pr_conflicts.py run + # (--base-branch ) exercises the clean auto-fix and the + # manual-review fallbacks together. Build it before cutting PR branches; + # this leaves the working tree on the base branch, but each create_pr + # checks out fresh off BRANCH_FROM anyway. + conflicting_base = build_conflicting_base(TEST_PRS) + base = conflicting_base or MOCK_BASE_BRANCH + + for spec in TEST_PRS: + url = create_pr(spec, base) + if url: + pr_pairs.append((spec, url)) + finally: + # Return to wherever we started, whatever happened. + git("checkout", original_branch) + + if not pr_pairs: + print("\nNo PRs created.") + return + + urls = [url for _, url in pr_pairs] + print(f"\nCreated {len(urls)} PR(s):") + for url in urls: + print(f" {url}") + + from zoneinfo import ZoneInfo + central = ZoneInfo("America/Chicago") + safe_after = datetime.now(central) + timedelta(minutes=5) + write_pr_list(pr_pairs, base, conflicting_base, safe_after) + + base_flag = f"--base-branch {conflicting_base}" if conflicting_base else "--live" + n = len(urls) + print( + f"\nView all test PRs:\n" + f" https://github.com/DataDog/documentation/pulls" + f"?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-testing\n" + f"\nDry-run conflict resolution (no changes made):\n" + f" python3 astro_reorg/resolve_pr_conflicts.py {base_flag} --limit {n}\n" + f"\nLive conflict resolution (applies fixes and labels):\n" + f" python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run {base_flag} --limit {n}" + ) + + +if __name__ == "__main__": + main() diff --git a/astro_reorg/docs/.gitignore b/astro_reorg/docs/.gitignore new file mode 100644 index 00000000000..cc822d70466 --- /dev/null +++ b/astro_reorg/docs/.gitignore @@ -0,0 +1 @@ +test_pr_list.md diff --git a/astro_reorg/docs/pr_handling_details.md b/astro_reorg/docs/pr_handling_details.md new file mode 100644 index 00000000000..d444caf6b3d --- /dev/null +++ b/astro_reorg/docs/pr_handling_details.md @@ -0,0 +1,181 @@ +# PR handling details + +How the conflict-resolution script decides what to do with each open PR after the repo reorg (which moved root files and folders into `hugo/`). This describes the behavior in plain terms; the script itself is the source of truth. + +## Labels used + +| Label | Applied to | Meaning | +|-------|-----------|---------| +| `LABEL_NO_CONFLICTS` | original PR | PR merges cleanly after the reorg; ignored from then on. | +| `LABEL_MANUAL_REVIEW` | original PR | Conflicts that can't be fixed automatically; queued for a person. Ignored from then on. Applied together with `LABEL_WIP` and a comment (see [Sending a PR to manual review](#sending-a-pr-to-manual-review)). | +| `LABEL_STALE` | original PR | PR had no activity for more than `STALE_DAYS`; skipped instead of auto-fixed. | +| `LABEL_AUTOFIXED` | original PR | PR was auto-fixed; the original is closed and points to a new fix PR. | +| `LABEL_AUTO_PR` | new fix PR | Marks the automatically created fix PR. | +| `LABEL_HELP_REQUESTED` | original PR (added by the author) | The author is asking a person to step in on an already-processed PR they're still stuck on. The script leaves these PRs alone. | +| `LABEL_WIP` | original PR (added by the author) | Author marked the PR as not ready; it gets a comment and is then skipped. | +| `LABEL_SKIP` | original PR | Applied by the script after commenting on a work-in-progress PR, so later runs don't pick it up again. | +| `LABEL_PROCESSED` | every PR the script acts on | Visibility aid so all reorg-affected PRs are findable with one filter, regardless of outcome. Not used for idempotency and deliberately absent from the query. | +| `DO_NOT_MERGE_LABEL` | new fix PR (test runs only) | Keeps auto-created fix PRs out of other teams' review queues. | + +Key thresholds: +- `STALE_DAYS` (31) — a PR with no activity for longer than this is treated as stale (roughly one calendar month). +- `STALE_REACTIVATE_GRACE` (60 seconds) — a small allowance so the script's own labeling of a PR is never mistaken for the author reactivating it. + +## Which PRs are considered + +By default the script looks at every **open** PR targeting the base branch (`master`, or a mock base branch during test runs), but **ignores** any PR that already carries `LABEL_NO_CONFLICTS`, `LABEL_MANUAL_REVIEW`, `LABEL_HELP_REQUESTED`, or `LABEL_SKIP`. `LABEL_NO_CONFLICTS` and `LABEL_MANUAL_REVIEW` are final: once a PR has one, it's never looked at again. `LABEL_HELP_REQUESTED` is added by an author who wants a person to take over, so the script stops touching the PR and leaves it for manual handling. `LABEL_SKIP` is applied by the script after it comments on a work-in-progress PR, so it isn't picked up and re-commented on later. + +You can also point the script at specific PR numbers instead of the whole open list. In that mode it will also consider closed PRs, but it still skips any PR that targets a different base branch. + +## Decision flow + +For each PR, in order. The first case that matches wins. + +### 1. The PR has no conflicts + +Label it `LABEL_NO_CONFLICTS` and move on. This is how PRs that are opened *after* the reorg reaches master get ignored going forward. Labeling a clean PR doesn't count against the run's processing limit. + +### 2. GitHub hasn't figured out yet whether the PR conflicts + +Skip it for now, with no label or comment. GitHub computes this in the background, so a later run will pick it up once the answer is available. + +### 3. The PR has conflicts + +The script does a trial merge against the post-reorg base and sorts each conflict into two buckets: conflicts **caused by the reorg** (a file the reorg moved) and conflicts **unrelated to the reorg**. It also catches files the PR adds at an old, pre-reorg location that should now live under `hugo/`, and counts those as reorg conflicts. + +What happens next depends on what it found: + +#### 3a. No conflicts could actually be pinned down +- If the trial merge failed but the script couldn't identify which files conflicted (an unusual case), it plays it safe and [sends the PR to manual review](#sending-a-pr-to-manual-review) rather than guessing. +- If the trial merge was clean after all, GitHub's earlier "conflicting" status was probably out of date; the PR is skipped so a later run can reassess. + +#### 3b. Some conflicts are unrelated to the reorg + +The script never touches conflicts it didn't cause. It [sends the PR to manual review](#sending-a-pr-to-manual-review) and leaves it for a person. + +#### 3c. All conflicts were caused by the reorg + +Before fixing anything, the PR has to be "ready." These checks run in order. + +**Work in progress** — If the PR is marked `LABEL_WIP`, the script posts a comment telling the author how to ask for help, adds `LABEL_SKIP`, and skips the fix. Because `LABEL_SKIP` is excluded from the query, later runs won't pick the PR up again, so the comment is posted only once. + +**Stale** — See [Stale handling](#stale-handling) below. + +**Auto-fix** — If the PR is ready and every conflict came from the reorg, the script fixes it: +- It replays the PR's commits onto a new fix branch with all file paths moved to the new `hugo/` layout, keeping the original authorship and commit messages. +- If that succeeds, it opens a new fix PR (labeled `LABEL_AUTO_PR` and `LABEL_WIP`, plus `DO_NOT_MERGE_LABEL` on test runs), then closes the original PR with a comment linking to the fix and labels the original `LABEL_AUTOFIXED`. Every PR the script acts on — including this one — also receives `LABEL_PROCESSED`. The fix PR's description **@mentions the original author**, so they're notified the moment it's opened (which is before the original is closed — see [Resuming an interrupted fix](#resuming-an-interrupted-fix)). +- If the fix can't be applied — for example the PR comes from a fork the script can't push to, or the replay fails — it [sends the PR to manual review](#sending-a-pr-to-manual-review). + +### Resuming an interrupted fix + +The auto-fix happens in stages: push the fix branch, open the fix PR, then close the original. If a run dies in between (a network blip, an interrupted process), a later run detects the partial state and **finishes the job** instead of bailing: + +- **Fix PR already open:** the script reuses it — re-applies its labels, then closes the original pointing at it and labels the original `LABEL_AUTOFIXED`. +- **Fix branch pushed but no PR:** the script opens the PR from the existing branch, then finishes as above. + +It never rebuilds or force-pushes an existing fix branch, so a fix already in review is never clobbered. If it truly can't build the branch (for instance it's checked out in a lingering worktree), it falls back to [manual review](#sending-a-pr-to-manual-review). Because the fix PR's description @mentions the author, they learn about the fix even if the original close step never ran. + +### Sending a PR to manual review + +Every non-auto-fixable outcome (a non-reorg conflict, an unclassifiable merge failure, or a failed replay) is handled the same way: the script applies `LABEL_MANUAL_REVIEW` (which excludes the PR from all future runs), applies `LABEL_WIP` (which keeps it out of the docs team's review queue until the author acts), and posts a single comment explaining the situation. The comment tells the author to resolve the conflicts and then remove the `WORK IN PROGRESS` label, or to add `LABEL_HELP_REQUESTED` if they'd rather a person take over. + +## Stale handling + +A PR is **stale** when its last activity was more than `STALE_DAYS` ago. Stale handling only applies to PRs whose conflicts all came from the reorg. + +**If a stale PR isn't labeled stale yet:** the script posts a comment explaining that no action was taken because the PR is stale (and how to ask for help), then adds the `LABEL_STALE` label. + +**If a PR already has the `LABEL_STALE` label:** the label serves two purposes — it marks the PR as stale, and it prevents the same comment from being posted again. Deciding whether the PR has "woken up" is based on whether there's been activity *since the label was applied* — not simply on how old the last activity is. This matters because the script's own comment and label count as activity; without this rule, a PR the script just labeled would look active again on the very next run and get fixed anyway. + +So for a PR that already has the stale label: +- The PR is considered **reactivated** if there was genuine activity (a new commit, a comment, or removing the stale label) after the label was applied — allowing for the `STALE_REACTIVATE_GRACE` window so the labeling itself doesn't count. +- **If it hasn't been reactivated:** skip it quietly, without commenting again. The skip lasts across runs until a person interacts with the PR. +- **If it has been reactivated:** remove the `LABEL_STALE` label and handle the PR normally (it goes on to the auto-fix step). +- **If the activity history can't be read** for some reason, the PR is treated as *not* reactivated and stays skipped, so a temporary glitch won't cause an unexpected auto-fix. + +### How an author reactivates a stale PR +- Push a new commit or leave a comment, **or** +- Remove the `LABEL_STALE` label (removing a label also counts as activity). + +To instead ask a person to step in rather than have the script retry, the author adds `LABEL_HELP_REQUESTED`; the script then leaves the PR alone (see [Labels used](#labels-used)). + +## Limiting a run and dry runs +- A run can be capped to act on only a set number of PRs. A PR counts against that cap when it's auto-fixed or gets a review/stale label and comment. PRs that need no action — clean PRs, ones GitHub hasn't evaluated, or ones that turn out clean on the trial merge — don't count. +- By default the script only reports what it *would* do, without changing anything, and runs against a mock base branch for safety. There are flags to apply changes for real and to run against the real `master`. + +## State machine diagram + +```mermaid +flowchart TD + START([Query open PRs targeting base branch]) --> FILTER + + FILTER{Already has
NO_CONFLICTS label,
MANUAL_REVIEW label,
HELP_REQUESTED label,
or SKIP label?} + FILTER -- Yes --> IGNORE([⏭️  Ignore — skip permanently]) + FILTER -- No --> CONFLICT_STATUS + + CONFLICT_STATUS{Conflict
status?} + CONFLICT_STATUS -- No conflicts --> CLEAN[Add NO_CONFLICTS label] + CONFLICT_STATUS -- Unknown / not yet computed --> SKIP_PENDING([⏭️  Skip — no label
reassess next run]) + CONFLICT_STATUS -- Has conflicts --> TRIAL_MERGE + + CLEAN --> DONE_CLEAN([✅  Done — doesn't count against cap]) + + TRIAL_MERGE[Trial merge
against post-reorg base] + TRIAL_MERGE --> TRIAL_RESULT + + TRIAL_RESULT{Trial merge
result?} + TRIAL_RESULT -- "Can't identify conflicting files" --> MANUAL_REVIEW + TRIAL_RESULT -- "Trial merge clean
(stale GitHub status)" --> SKIP_STALE_STATUS([⏭️  Skip — no label
reassess next run]) + TRIAL_RESULT -- "Non-reorg conflicts
exist" --> MANUAL_REVIEW + TRIAL_RESULT -- "All conflicts are
reorg-caused" --> CHECK_WIP + + CHECK_WIP{PR has
WIP label?} + CHECK_WIP -- Yes --> WIP_ACTION["Comment: how to ask for help
Add SKIP label"] + CHECK_WIP -- No --> CHECK_STALE + + WIP_ACTION --> DONE_WIP([✅  Done — counts against cap]) + + CHECK_STALE{Is PR
stale?} + CHECK_STALE -- No --> AUTOFIX + CHECK_STALE -- "Yes, not yet
labeled stale" --> STALE_LABEL["Comment: PR is stale
Add STALE label"] + CHECK_STALE -- "Yes, already
has STALE label" --> CHECK_REACTIVATED + + STALE_LABEL --> DONE_STALE([✅  Done — counts against cap]) + + CHECK_REACTIVATED{Reactivated since
label was applied?} + CHECK_REACTIVATED -- "Can't read
activity history" --> SKIP_STALE([⏭️  Skip quietly — no label
no count against cap]) + CHECK_REACTIVATED -- Not reactivated --> SKIP_STALE + CHECK_REACTIVATED -- Reactivated --> REMOVE_STALE["Remove STALE label"] + + REMOVE_STALE --> AUTOFIX + + AUTOFIX{Partial fix
already in progress?} + AUTOFIX -- "Fix PR already open" --> REUSE["Reuse fix PR
Re-apply labels"] + AUTOFIX -- "Fix branch pushed
but no PR" --> OPEN_PR_FROM_BRANCH["Open fix PR
from existing branch"] + AUTOFIX -- "No partial state" --> REPLAY + + REPLAY[Replay commits onto
new fix branch with
updated hugo/ paths] + REPLAY -- "Replay fails
(fork, locked branch,
or other error)" --> MANUAL_REVIEW + REPLAY -- Success --> PUSH_AND_OPEN["Push fix branch
Open fix PR
(AUTO_PR + WIP labels;
DO_NOT_MERGE label in test runs)
@mention original author"] + + PUSH_AND_OPEN --> CLOSE_ORIGINAL + REUSE --> CLOSE_ORIGINAL + OPEN_PR_FROM_BRANCH --> CLOSE_ORIGINAL + + CLOSE_ORIGINAL["Close original PR
with link to fix PR
Add AUTOFIXED label to original
Add PROCESSED label to both"] + CLOSE_ORIGINAL --> DONE_FIXED([✅  Done — counts against cap]) + + MANUAL_REVIEW["🔴  Add MANUAL_REVIEW + WIP labels
Post comment:
resolve conflicts or add
HELP_REQUESTED label
(excluded from all future runs)"] + MANUAL_REVIEW --> DONE_MANUAL([✅  Done — counts against cap]) + + style MANUAL_REVIEW fill:#fff,stroke:#000,color:#000 + style DONE_FIXED fill:#fff,stroke:#000,color:#000 + style DONE_CLEAN fill:#fff,stroke:#000,color:#000 + style IGNORE fill:#fff,stroke:#000,color:#000 + style SKIP_PENDING fill:#fff,stroke:#000,color:#000 + style SKIP_STALE fill:#fff,stroke:#000,color:#000 + style SKIP_STALE_STATUS fill:#fff,stroke:#000,color:#000 + style DONE_WIP fill:#fff,stroke:#000,color:#000 + style DONE_STALE fill:#fff,stroke:#000,color:#000 + style DONE_MANUAL fill:#fff,stroke:#000,color:#000 +``` diff --git a/astro_reorg/docs/reorg_execution_steps.md b/astro_reorg/docs/reorg_execution_steps.md new file mode 100644 index 00000000000..710d48eeef6 --- /dev/null +++ b/astro_reorg/docs/reorg_execution_steps.md @@ -0,0 +1,155 @@ +# Reorg execution steps + +## When a date has been set + +### 1. Update the reorg README + +- [ ] Add the target date to the reorg README. + +### 2. Publish the README + +- [ ] Publish [REPO_REORG.md](../../REPO_REORG.md) on master, so you can link to it. +- [ ] Set the `REORG_README_BRANCH` constant in `resolve_pr_conflicts.py` to `"master"` (it defaults to the scripts branch), commit, and push. Leave `CONFIG_LINK_BRANCH` pointing at the scripts branch — the reorg deletes `astro_reorg/` from master, so that link must stay on a branch that retains it. + +### 3. Announce the reorg + +Announce the reorg: + +- [ ] To the #documentation channel () +- [ ] To the #docs-backroom channel () +- [ ] In a banner in the `documentation` README + +## The day before + +### 1. Bump the announcement in the #documentation channel + +See link you saved above. + +### 2. Remind the docs on-call of the reorg + +- [ ] Post the message below in #docs-backroom, tagging the current on-call folks, and put the URL here: + +``` +Hi , + +Tomorrow at about , I'll declare a code freeze in order to reorganize the docs repo. These are the steps I'll take: + +1. Start the code freeze. +2. Reorg the repo and verify the build. +3. Make any necessary last-minute fixes to GitHub actions, etc. broken by the reorg. +4. Merge the reorg to master, and verify the build. +5. Run a script that processes every open PR in the docs repo: + - PRs with no conflicts: Label them to avoid reprocessing them. + - PRs marked `WORK IN PROGRESS`: Skip, but leave a comment on them describing how to opt in to the auto fix. + - PRs with no recent activity (1 month): Skip, but leave a comment on them describing how to opt in to the auto fix. + - PRs with conflicts that can be automatically fixed: Close the original PR and link a new autofix PR. + - PRs with conflicts that CANNOT be automatically fixed: Put a label on it to prevent repeat processing, and comment on the PR to delegate conflict resolution to the author (with an escalation path if they need it). +6. Lift the code freeze. + +Auto-fixable PRs are closed, manual-intervention PRs are automatically labeled `WORK IN PROGRESS`, and auto-created PRs are labeled `WORK IN PROGRESS` so their authors can review them. This means that your review queue will be strangely quiet at first, and should not ever contain weird noise from the reorg. + +Once an author has reviewed an auto-created PR and removed the `WORK IN PROGRESS` label, you can treat it as any other PR. + +**You are not expected to provide support for any issues arising from the reorg.** I'll be here to help all day, and for longer than that if needed. If someone raises an issue in #documentation, you can just tag me or cross-post it to #docs-repo-reorg-support. + +If you have questions, please ask them on this thread or reach out in #docs-repo-reorg-support. +``` + +## The day of + +NOTE: This file will get deleted from your reorg branch, so don't edit it there. You can push what you have, then check it off [here](https://github.com/DataDog/documentation/blob/jen.gilbert/astro-reorg-scripts/astro_reorg/docs/reorg_execution_steps.md) instead. + +### 1. Bump the announcement in the #documentation channel + +See link you saved above. + +### 2. Bump the detailed message in #docs-backroom + +See link you saved above. + +### 3. Declare a code freeze + +See the [Confluence page](https://datadoghq.atlassian.net/wiki/spaces/WEB/pages/5286757291/Code+Freeze+Workflows). + +Announce the freeze in `#documentation` with a link to the reorg README. + +### 4. Create a reorged `master` branch + +- [ ] Merge master into the scripts branch: + ```bash + git checkout jen.gilbert/astro-reorg-scripts + git merge master + ``` +- [ ] Cut a reorg branch from the reorg scripts branch: + ```bash + git checkout -b jen.gilbert/hugo-site-folder + ``` +- [ ] Run the reorg script: + ```bash + python3 astro_reorg/execute_reorg.py + ``` +- [ ] Delete the `astro_reorg` folder: + ```bash + rm -rf astro_reorg/ + ``` +- [ ] Stage and commit the result (the push happens in the next step): + ```bash + git add -A -- ':!astro/' + git commit -m "Create hugo site folder" + ``` + +### 5. Push and open a PR against master + +```bash +git push -u origin jen.gilbert/hugo-site-folder && +gh pr create --base master --title "Create Hugo site folder" +``` + +### 6. Verify that the GitHub actions etc. are working, and make any necessary fixes + +- [ ] Verify that GitHub actions aren't erring out from bad paths. +- [ ] While the preview is building, run some local checks: + - [ ] The Cdocs e2e tests are passing. + - [ ] Integrations pages are working correctly. + - [ ] API pages are working correctly. + - [ ] Single sourced content is working correctly. + +### 7. Verify the preview build + +- [ ] The GitLab artifacts for the preview build should be similar to those of the last build on `master`: same number of HTML files in `public`, for example. +- [ ] + +### 7. Merge to master and verify the build + +### 8. Run the PR resolution script + +Run the script in batches, building confidence before scaling up. The script defaults to a dry run against the mock base branch — pass `--live` to target real master, and `--no-dry-run` to apply changes. + +- [ ] Start with a single PR as a dry run to confirm the output looks right: + ```bash + python3 astro_reorg/resolve_pr_conflicts.py --live --limit 1 + ``` +- [ ] If that looks good, run one PR for real: + ```bash + python3 astro_reorg/resolve_pr_conflicts.py --live --no-dry-run --limit 1 + ``` +- [ ] Review the result on GitHub. If it looks good, increase the limit and repeat until all PRs are processed: + ```bash + python3 astro_reorg/resolve_pr_conflicts.py --live --no-dry-run --limit 10 + python3 astro_reorg/resolve_pr_conflicts.py --live --no-dry-run --limit 50 + # ... and so on until all PRs are processed + ``` + +You can view [all processed PRs here](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-processed), regardless of their outcome. + +### 9. End the code freeze + +### 10. Post in #documentation and #docs-backroom + +### 11. Monitor the queues + +- [All processed PRs](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-processed) +- [Fixed-and-closed PRs](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-autofixed) +- [Auto-generated PRs](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-auto-pr) +- [Manual intervention PRs](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-manual-review) +- [PRs escalated with a help request](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-help-requested) diff --git a/astro_reorg/docs/test_setup.md b/astro_reorg/docs/test_setup.md new file mode 100644 index 00000000000..aba0fcf3b66 --- /dev/null +++ b/astro_reorg/docs/test_setup.md @@ -0,0 +1,84 @@ +# Test setup for resolve_pr_conflicts.py + +## Prerequisites + +Two branches must exist on the remote, named in `config.yaml` under `test:`: + +- `branch_from` — a non-reorged snapshot of master that the mock contributor PRs are cut from. The scripts branch (`jen.gilbert/astro-reorg-scripts`) works and already exists, so you usually don't need to create this one. +- `mock_reorged_master_branch` — a reorged stand-in for post-reorg master that the test PRs target and the resolver treats as the base. You create this below. It must have the reorg applied, so the scripts branch can't stand in for it. + +## Setup steps + +### 1. Choose a branch name for the reorged mock master + +**Do not create the branch yet.** + +Choose a name for the reorged mock master you intend to create, such as `mock-reorged-master-7-14-1258` (this example just used the date and current time to make the branch name unique). + +### 2. Update the script config + +Update `mock_reorged_master_branch` in `config.yaml` to match your intended branch name. Commit the change. + +### 3. Create the mock reorged master branch + +Replace `` with the branch name you chose above. + +```bash +git checkout jen.gilbert/astro-reorg-scripts +git checkout -b +python3 astro_reorg/execute_reorg.py +git add -- . ':!astro' +git commit -m "Apply reorg" +git push --set-upstream origin +``` + +### 4. Switch back to `jen.gilbert/astro-reorg-scripts` + +Switch back to the Astro reorg scripts branch: + +```bash +git checkout jen.gilbert/astro-reorg-scripts +``` + +Revert any lingering (untracked) reorg changes: + +```bash +python3 astro_reorg/local_rollback.py +``` + +## Running the test + +### 1. Create the test PRs + +```bash +python3 astro_reorg/create_test_prs.py +``` + +This opens one PR per spec in `TEST_PRS`. Because some specs carry a `base_edit`, the script also builds a throwaway conflicting base branch (unique per run, so no force-push) and points every PR at it, so a single resolver run exercises both the clean auto-fixes and the manual-review fallbacks together. On completion it: + +- writes `test_pr_list.md` at the repo root, listing every PR and its expected outcome, and +- prints the exact dry-run and real-run commands to use next, already filled in with the right `--base-branch` and `--limit`. + +### 2. Execute a dry run + +Run the dry-run command printed by `create_test_prs.py`. It looks like: + +```bash +python3 astro_reorg/resolve_pr_conflicts.py --base-branch --limit +``` + +`--limit` is required, so don't drop it. + +### 3. Execute a real run + +Run the real-run command printed by `create_test_prs.py`. It looks like: + +```bash +python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run --base-branch --limit +``` + +Expected outcomes (`test_pr_list.md` is the authoritative list): + +- **Auto-fixed** — original closed and replaced with a `[reorg fix]` PR (`astro-reorg-autofixed`): the wording-tweak PR and the new-page-with-nav PR. +- **Skipped as WIP** — `astro-reorg-skip` label plus a comment: the `WORK IN PROGRESS` PR. +- **Manual review** — `astro-reorg-manual-review` and `WORK IN PROGRESS` labels plus a comment: the non-reorg-conflict PR and the two unresolvable-reorg-conflict PRs. diff --git a/astro_reorg/execute_reorg.py b/astro_reorg/execute_reorg.py new file mode 100644 index 00000000000..ce2470d411a --- /dev/null +++ b/astro_reorg/execute_reorg.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +import os +import shutil +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + +repo_root = Path(__file__).parent.parent +config_path = Path(__file__).parent / "config.yaml" + +with config_path.open() as f: + config = yaml.safe_load(f) + +repo_level_files = set(config.get("top_level", [])) +hugo_level_files = set(config.get("moves_to_hugo", [])) +ignore = set(config.get("ignore", [])) + +# Sanity-check the config itself for conflicts. +config_inconsistencies = repo_level_files & hugo_level_files +if config_inconsistencies: + for name in sorted(config_inconsistencies): + print(f"ERROR: '{name}' appears in both top_level and moves_to_hugo", file=sys.stderr) + sys.exit(1) + +errors = [] + +for name in os.listdir(repo_root): + if name in ignore: + continue + + if name in repo_level_files: + pass # keep in place + elif name in hugo_level_files: + pass # will move below + else: + errors.append(name) + +if errors: + for name in sorted(errors): + print(f"ERROR: '{name}' is not listed in astro_reorg/config.yaml", file=sys.stderr) + print(f"{len(errors)} error(s) found. Update astro_reorg/config.yaml before running.", file=sys.stderr) + sys.exit(1) + +hugo_dir = repo_root / "hugo" +hugo_dir.mkdir(exist_ok=True) + +moved = 0 +deleted = 0 +for name in sorted(os.listdir(repo_root)): + if name in ignore or name in repo_level_files or name == "hugo": + continue + src = repo_root / name + dst = hugo_dir / name + + # A Python virtualenv bakes its own ABSOLUTE path into bin/activate, the + # bin/python* symlinks, and pyvenv.cfg, so relocating the directory leaves + # those pointing at the old path — activation then falls back to the system + # Python (no project deps). A venv is a regenerable artifact, so delete it + # rather than move it; `make` rebuilds it in place with correct paths. + if (src / "pyvenv.cfg").is_file(): + print(f"Deleting venv {name} (regenerated by make in hugo/{name})") + shutil.rmtree(src) + deleted += 1 + continue + + print(f"Moving {name} -> hugo/{name}") + src.rename(dst) + moved += 1 + +print(f"Done. {moved} item(s) moved into hugo/, {deleted} venv(s) deleted for regeneration.") + +# Split .gitignore between the root and hugo/ instead of copying it wholesale. +# +# A .gitignore is interpreted RELATIVE TO ITS OWN DIRECTORY, so — unlike +# CODEOWNERS — a moved rule needs NO "hugo/" prefix; it simply belongs in +# hugo/.gitignore. We therefore only ROUTE each line, never rewrite it, by its +# FIRST PATH SEGMENT (driven entirely by astro_reorg/config.yaml, same as CODEOWNERS): +# first segment in moves_to_hugo -> hugo/.gitignore only +# first segment in top_level -> root .gitignore only +# first segment in neither -> generic/pure glob, kept in BOTH +# Comments and blank lines are kept in both to preserve section readability. +def route_gitignore_segment(line): + """Return the first path segment of a .gitignore pattern, or None for a + comment/blank line. A leading '!' (negation) and '/' (root anchor) are + stripped before taking segment 0; the line text itself is never altered. + """ + stripped = line.strip() + if not stripped or stripped.startswith("#"): + return None + body = stripped[1:] if stripped.startswith("!") else stripped # un-negate + body = body.lstrip("/") # un-anchor + return body.split("/", 1)[0] + + +print("\nSplitting .gitignore between root and hugo/...") +gitignore = repo_root / ".gitignore" +original_gitignore_lines = gitignore.read_text().splitlines(keepends=True) + +root_lines = [] +hugo_lines = [] +hugo_only_segments = set() # routed off root into hugo/ (for the summary) +both_segments = set() # in neither config list -> kept in both (surfaced) + +for raw in original_gitignore_lines: + segment = route_gitignore_segment(raw) + if segment is None: # comment / blank -> keep in both + root_lines.append(raw) + hugo_lines.append(raw) + elif segment in hugo_level_files: + hugo_lines.append(raw) + hugo_only_segments.add(segment) + elif segment in repo_level_files: + root_lines.append(raw) + else: + root_lines.append(raw) + hugo_lines.append(raw) + both_segments.add(segment) + +gitignore.write_text("".join(root_lines)) +(hugo_dir / ".gitignore").write_text("".join(hugo_lines)) +print(f" Written: .gitignore (root, pruned) and hugo/.gitignore " + f"({len(hugo_only_segments)} segment(s) -> hugo/ only, {len(both_segments)} generic kept in both)") +if both_segments: + print(" NOTE: kept in both (first path segment not in astro_reorg/config.yaml): " + + ", ".join(sorted(both_segments))) + +# Update .github/workflows/ files to reference paths under hugo/. +WORKFLOW_SUBSTITUTIONS = [ + ("- 'content/en/", "- 'hugo/content/en/"), + ("- 'layouts/shortcodes/", "- 'hugo/layouts/shortcodes/"), + ("- 'static/images/", "- 'hugo/static/images/"), + ("python local/bin/", "python hugo/local/bin/"), + # vale_linter.yml passes the template path to vale via --output= + ("--output=local/bin/", "--output=hugo/local/bin/"), + (" static |", " hugo/static |"), + ("^static/images/", "^hugo/static/images/"), + ("-- 'content/en/**/*.md'", "-- 'hugo/content/en/**/*.md'"), + # bump_* and version_getter_shared workflows cp/commit data files by ./ path. + # Match without a trailing slash so a bare `mkdir -p ./data` is repathed too, + # not just `./data/` references. + ("./data", "./hugo/data"), +] + +print("\nUpdating .github/workflows/...") +workflows_dir = repo_root / ".github" / "workflows" +for yml_file in sorted(workflows_dir.glob("*.yml")): + original = yml_file.read_text() + updated = original + for old, new in WORKFLOW_SUBSTITUTIONS: + updated = updated.replace(old, new) + if updated != original: + yml_file.write_text(updated) + print(f" Written: {yml_file.name}") + +# Update .husky/ hook scripts to reference paths under hugo/content/. +HUSKY_SUBSTITUTIONS = [ + ("content/en/{dir_name}/{name}", "hugo/content/en/{dir_name}/{name}"), + ("content/en/{dir_name}/", "hugo/content/en/{dir_name}/"), + ("Path('content/en')", "Path('hugo/content/en')"), + ('Path("content/.gitignore")', 'Path("hugo/content/.gitignore")'), + ("f.startswith('content/en/')", "f.startswith('hugo/content/en/')"), + (".replace('content/en/', '')", ".replace('hugo/content/en/', '')"), + ("repo_root / 'content' / 'en'", "repo_root / 'hugo' / 'content' / 'en'"), + ('repo_pattern = "content"', 'repo_pattern = "hugo/content"'), +] + +print("\nUpdating .husky/...") +husky_dir = repo_root / ".husky" +for py_file in sorted(husky_dir.glob("*.py")): + original = py_file.read_text() + updated = original + for old, new in HUSKY_SUBSTITUTIONS: + updated = updated.replace(old, new) + if updated != original: + py_file.write_text(updated) + print(f" Written: {py_file.name}") + +# Update .github/CODEOWNERS to reference paths under hugo/. +# +# CODEOWNERS is not YAML; it is a line-based format where each rule is +# " ". Rather than blanket str.replace() over the whole +# file (which let a short pattern like "config/" corrupt a longer one like +# "customization_config/"), we parse each line and route its pattern by its +# FIRST PATH SEGMENT. The decision is made on a whole segment, never a raw +# substring, so "config" can never match inside "customization_config" and the +# substitution list no longer has to be hand-ordered or kept in sync with the +# move list — it is derived entirely from astro_reorg/config.yaml. +def route_codeowners_pattern(pattern): + """Rewrite one CODEOWNERS pattern for the hugo/ move. + + Returns (segment, new_pattern). new_pattern is None when the line is left + untouched; segment is None for the global '*' owner. A leading '/' + (repo-root anchor) is preserved; owners are never touched because we only + ever rewrite the pattern token. + """ + if pattern == "*": + return None, None + + anchored = pattern.startswith("/") # leading '/' = repo-root-anchored + body = pattern[1:] if anchored else pattern + segment = body.split("/", 1)[0] + + # Carried over from the original script: ".local/" is a misspelling of + # "local/" (which moves into hugo/). Normalize the segment before routing + # so the typo'd entry is repathed alongside the real one. + if segment == ".local": + body = "local" + body[len(".local"):] + segment = "local" + + if segment not in hugo_level_files: + return segment, None + + new_body = "hugo/" + body + return segment, (("/" + new_body) if anchored else new_body) + + +print("\nUpdating .github/CODEOWNERS...") +codeowners = repo_root / ".github" / "CODEOWNERS" +lines = codeowners.read_text().splitlines(keepends=True) +left_alone = set() # first segments in neither config list (surfaced below) + +changed = False +for i, raw in enumerate(lines): + newline = "\n" if raw.endswith("\n") else "" + line = raw[:-len(newline)] if newline else raw + stripped = line.lstrip() + if not stripped or stripped.startswith("#"): + continue + indent = line[:len(line) - len(stripped)] + pattern = stripped.split(maxsplit=1)[0] + rest = stripped[len(pattern):] # original spacing + owners, verbatim + segment, new_pattern = route_codeowners_pattern(pattern) + if new_pattern is None: + if segment is not None and segment not in repo_level_files: + left_alone.add(segment) + continue + lines[i] = indent + new_pattern + rest + newline + changed = True + +if left_alone: + print("\n NOTE: left unchanged (first path segment not in astro_reorg/config.yaml): " + + ", ".join(sorted(left_alone))) + +if changed: + codeowners.write_text("".join(lines)) + print(" Written: CODEOWNERS") diff --git a/astro_reorg/helpers.py b/astro_reorg/helpers.py new file mode 100644 index 00000000000..40bef6311da --- /dev/null +++ b/astro_reorg/helpers.py @@ -0,0 +1,665 @@ +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +try: + import yaml +except ImportError: + print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + +repo_root = Path(__file__).parent.parent +hugo_dir = repo_root / "hugo" + +# Distinctive marker so anything we create is obvious and easy to clean up. +MARKER = "__reorg_harness__" + +# (status, name, detail) tuples collected as checks run. +results = [] + + +def record(status, name, detail=""): + """Record a check outcome. status is one of PASS, FAIL, SKIP, WARN.""" + results.append((status, name, detail)) + symbol = {"PASS": "✅", "FAIL": "❌", "SKIP": "⏭ ", "WARN": "⚠ "}[status] + line = f"{symbol} {status:4} {name}" + if detail: + line += f"\n {detail}" + print(line) + + +def git(*args, check=False): + """Run a git command from the repo root and return the CompletedProcess.""" + return subprocess.run( + ["git", *args], + cwd=repo_root, + capture_output=True, + text=True, + check=check, + ) + + +def load_config(): + """Load astro_reorg/config.yaml and return (top_level, moves_to_hugo) name sets.""" + config_path = Path(__file__).parent / "config.yaml" + with config_path.open() as f: + config = yaml.safe_load(f) + return set(config.get("top_level", [])), set(config.get("moves_to_hugo", [])) + + +def check_layout(top_level, moves_to_hugo): + """hugo/ holds the moved dirs; nothing that moved is left at the root.""" + # Moved items that the reorg actually had to relocate (present before). + expected_in_hugo = sorted(n for n in moves_to_hugo if (hugo_dir / n).exists()) + missing_from_hugo = sorted( + n for n in moves_to_hugo + if (repo_root / n).exists() and not (hugo_dir / n).exists() + ) + + # A moved name still sitting at the root means the move didn't happen. + still_at_root = sorted( + n for n in moves_to_hugo if (repo_root / n).exists() + ) + if still_at_root: + record("FAIL", "layout: moved items remain at repo root", + ", ".join(still_at_root)) + else: + record("PASS", "layout: no moved items remain at repo root", + f"{len(expected_in_hugo)} item(s) confirmed under hugo/") + + if missing_from_hugo: + record("FAIL", "layout: moved items missing under hugo/", + ", ".join(missing_from_hugo)) + + # No top_level item should have leaked into hugo/. Two names are expected to + # appear under hugo/ and are NOT leaks: 'hugo' itself (the move target), and + # '.gitignore' — execute_reorg.py deliberately SPLITS the root .gitignore, + # writing a routed subset to hugo/.gitignore (verified by check_gitignore_split + # and the "present at root and in hugo/" check below). + expected_under_hugo = {"hugo", ".gitignore"} + leaked = sorted( + n for n in top_level + if n not in expected_under_hugo and (hugo_dir / n).exists() + ) + if leaked: + record("FAIL", "layout: top-level items leaked into hugo/", + ", ".join(leaked)) + else: + record("PASS", "layout: no top-level items leaked into hugo/") + + # Critical anchors that must still exist at the root after the move. + # package.json/node_modules/yarn.lock and go.mod/go.sum belong under hugo/ + # (each site owns its own Node setup; the Go module powers Hugo Modules), + # so they are deliberately NOT listed here. + must_stay = ["astro", ".husky", ".github", ".vale.ini"] + absent = [n for n in must_stay if not (repo_root / n).exists()] + if absent: + record("FAIL", "layout: expected top-level items are missing", + ", ".join(absent)) + else: + record("PASS", "layout: top-level anchors intact", + ", ".join(must_stay)) + + # Both .gitignore files should exist (execute_reorg.py splits one into two; + # check_gitignore_split below verifies the split routed correctly). + if (repo_root / ".gitignore").exists() and (hugo_dir / ".gitignore").exists(): + record("PASS", "layout: .gitignore present at root and in hugo/") + else: + record("FAIL", "layout: missing a .gitignore copy", + "expected both ./.gitignore and ./hugo/.gitignore") + + +def check_gitignore_split(top_level, moves_to_hugo): + """Verify execute_reorg.py routed .gitignore rules to the correct side. + + A .gitignore is relative to its own directory, so after the split no + surviving rule should point at a path that lives on the OTHER side: + - root .gitignore must not carry a rule whose first segment moved to hugo/ + - hugo/.gitignore must not carry a rule whose first segment stays at root + Generic globs (first segment in neither config list) legitimately live in + both files, so they are ignored here. + """ + def first_segment(line): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + return None + body = stripped[1:] if stripped.startswith("!") else stripped + return body.lstrip("/").split("/", 1)[0] + + def leaks(path, wrong_side): + out = [] + for raw in path.read_text().splitlines(): + seg = first_segment(raw) + if seg in wrong_side: + out.append(f"{path.name}: {raw.strip()} (segment {seg!r})") + return out + + root_gi = repo_root / ".gitignore" + hugo_gi = hugo_dir / ".gitignore" + if not (root_gi.exists() and hugo_gi.exists()): + record("SKIP", "gitignore: split not verifiable (a .gitignore is missing)") + return + + problems = leaks(root_gi, moves_to_hugo) + leaks(hugo_gi, top_level - {"hugo"}) + if problems: + record("FAIL", "gitignore: rules survive on the wrong side of the split", + f"{len(problems)}:\n " + "\n ".join(problems[:20])) + else: + record("PASS", "gitignore: no moved-path rule left at root, " + "no root-path rule left in hugo/") + + +def check_workflows(moves_to_hugo): + """Flag references to moved paths (directories AND files) lacking hugo/. + + Routes every path-like token by its first segment, mirroring execute_reorg.py: + a token whose first segment moved into hugo/ must be hugo/-prefixed (after a + correct reorg its first segment is 'hugo', so it no longer matches). A token + counts as a path reference when it contains a '/', or when the whole token is + the exact name of a moved file — so a bare word like 'content' in prose isn't + flagged, but a standalone 'babel.config.js' is. + + Moved files whose names also exist under astro/ (package.json, yarn.lock, + node_modules, .nvmrc) are excluded from bare-name matching: a standalone + 'package.json' is genuinely ambiguous. They are still validated when they + appear inside a slashed path routed by first segment. + """ + workflows_dir = repo_root / ".github" / "workflows" + if not workflows_dir.exists(): + record("SKIP", "workflows: .github/workflows/ not found") + return + + moved_files = { + n for n in moves_to_hugo + if (hugo_dir / n).is_file() and not (repo_root / "astro" / n).exists() + } + + # Lines that legitimately reference a moved name but are NOT paths to fix + # (illustrative prose, external URLs). Keyed by workflow filename; a line is + # exempt if it contains any of the listed markers. + allowlist = { + # Security-doc example of how untrusted paths are reported, not a real path. + "claude_review.yml": ["__untrusted/content/"], + } + + # A path-like token: a maximal run of path/glob characters. Whitespace, + # quotes, ':', '@' and '!' all act as boundaries, so a leading '!' negation + # or surrounding quotes fall away naturally. + token_re = re.compile(r"[A-Za-z0-9_.\-*/]+") + + def first_segment(token): + """(first path segment, ./-stripped token) for routing.""" + t = token + while t.startswith("./"): + t = t[2:] + return t.split("/", 1)[0], t + + suspects = [] + for yml in sorted(workflows_dir.glob("*.yml")): + exempt = allowlist.get(yml.name, []) + for lineno, line in enumerate(yml.read_text().splitlines(), 1): + if any(marker in line for marker in exempt): + continue + for token in token_re.findall(line): + seg, normalized = first_segment(token) + # Only treat a token as a path reference if it's an actual path + # or is exactly the name of a moved file. Test the RAW token for + # the slash, not the ./-stripped form: `./data` is a genuine path + # reference, but normalizing it to `data` would drop the slash and + # make it read as a bare prose word. + if "/" not in token and normalized not in moved_files: + continue + if seg in moves_to_hugo: + suspects.append(f"{yml.name}:{lineno}: {line.strip()}") + break + + # De-duplicate while keeping order. + seen = set() + unique = [s for s in suspects if not (s in seen or seen.add(s))] + + if unique: + record("FAIL", "workflows: unprefixed moved paths found", + f"{len(unique)} line(s):\n " + "\n ".join(unique[:20])) + else: + record("PASS", "workflows: all moved paths (dirs and files) are hugo/-prefixed") + + +def check_workflow_path_filters(moves_to_hugo): + """Parse each workflow's on.*.paths filters and assert moved paths are prefixed. + + Unlike paths embedded in `run:` shell (which are opaque string scalars to a + parser), `on..paths` / `paths-ignore` are structured YAML list values, + so we can validate them precisely. Each entry is routed by its first path + segment: if that segment moved into hugo/, the entry must be hugo/-prefixed. + + Footgun handled: under YAML 1.1, PyYAML loads the `on:` key as the boolean + True, not the string "on" — so we look it up under both keys. + """ + workflows_dir = repo_root / ".github" / "workflows" + if not workflows_dir.exists(): + record("SKIP", "workflows: .github/workflows/ not found (path filters)") + return + + def first_segment(entry): + # Strip a leading '!' negation and any root anchor, then take segment 0. + p = entry[1:] if entry.startswith("!") else entry + return p.lstrip("/").split("/", 1)[0] + + problems = [] + for yml in sorted(workflows_dir.glob("*.yml")): + try: + doc = yaml.safe_load(yml.read_text()) + except yaml.YAMLError as exc: + record("WARN", f"workflows: {yml.name} did not parse as YAML", + str(exc).splitlines()[0][:120]) + continue + if not isinstance(doc, dict): + continue + triggers = doc.get("on", doc.get(True)) # `on:` -> True under YAML 1.1 + if not isinstance(triggers, dict): + continue + for event, spec in triggers.items(): + if not isinstance(spec, dict): + continue + for key in ("paths", "paths-ignore"): + for entry in spec.get(key) or []: + if not isinstance(entry, str): + continue + seg = first_segment(entry) + anchored = entry.lstrip("!").lstrip("/") + if seg in moves_to_hugo and not anchored.startswith("hugo/"): + problems.append(f"{yml.name}: on.{event}.{key}: {entry}") + + if problems: + record("FAIL", "workflows: on.*.paths filters missing hugo/ prefix", + f"{len(problems)}:\n " + "\n ".join(problems[:20])) + else: + record("PASS", "workflows: on.*.paths filters are hugo/-prefixed") + + +def check_codeowners(): + """Flag concrete (non-glob) CODEOWNERS patterns that don't resolve correctly. + + Stale entries pointing at files absent from both hugo/ and the repo root are + reported as informational notes but never failed. The two cases for a + non-resolving concrete pattern: + + - hugo/-prefixed AND its de-prefixed path resolves at the repo root + -> REGRESSION: the pattern was prefixed but the file is still at + root (the move didn't happen / went to the wrong place). FAIL. + - anything else that doesn't resolve -> absent from both hugo/ and root. + Reported as an informational note, never a failure. + """ + codeowners = repo_root / ".github" / "CODEOWNERS" + if not codeowners.exists(): + record("SKIP", "codeowners: .github/CODEOWNERS not found") + return + + regressions = [] + preexisting = [] + for line in codeowners.read_text().splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + pattern = stripped.split()[0] + if pattern == "*": + continue + # Skip glob patterns; we can't resolve them to a single path. + if any(ch in pattern for ch in "*?[]"): + continue + # Leading slash in CODEOWNERS is repo-root-anchored. + rel = pattern.lstrip("/") + if (repo_root / rel).exists(): + continue + # Doesn't resolve. Is it something the reorg broke, or pre-existing rot? + if rel.startswith("hugo/") and (repo_root / rel[len("hugo/"):]).exists(): + regressions.append(pattern) + else: + preexisting.append(pattern) + + if regressions: + record("FAIL", "codeowners: patterns the reorg moved to hugo/ but whose " + "file is still at root", + f"{len(regressions)}:\n " + "\n ".join(regressions[:20])) + elif preexisting: + record("PASS", "codeowners: no reorg-introduced breakage", + f"{len(preexisting)} pre-existing dangling entry(ies) ignored " + f"(absent on master too): e.g. {', '.join(preexisting[:3])}") + else: + record("PASS", "codeowners: all concrete patterns resolve on disk") + + +def check_codeowners_prefixing(moves_to_hugo): + """Every CODEOWNERS pattern whose first segment moved into hugo/ must carry + the hugo/ prefix — globs included. + + check_codeowners() above only proves that *concrete* paths still RESOLVE on + disk; it skips every glob (layouts/shortcodes/**/*.md) and never confirms a + moved pattern was actually repathed. Here we route each pattern by its first + path segment exactly as execute_reorg.py's route_codeowners_pattern does and assert + that a moved segment is prefixed. After a correct reorg the pattern's first + segment is 'hugo', so a flagged pattern means a substitution was missed. + """ + codeowners = repo_root / ".github" / "CODEOWNERS" + if not codeowners.exists(): + record("SKIP", "codeowners: .github/CODEOWNERS not found (prefixing)") + return + + def first_segment(pattern): + body = pattern[1:] if pattern.startswith("/") else pattern # un-anchor + seg = body.split("/", 1)[0] + # execute_reorg.py normalizes the '.local' typo to 'local' before routing. + return "local" if seg == ".local" else seg + + unprefixed = [] + for line in codeowners.read_text().splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + pattern = stripped.split()[0] + if pattern == "*": + continue + if first_segment(pattern) not in moves_to_hugo: + continue + if not pattern.lstrip("/").startswith("hugo/"): + unprefixed.append(pattern) + + if unprefixed: + record("FAIL", "codeowners: moved patterns missing the hugo/ prefix", + f"{len(unprefixed)} (globs included):\n " + + "\n ".join(unprefixed[:20])) + else: + record("PASS", "codeowners: every moved pattern is hugo/-prefixed " + "(globs included)") + + +def run_hook(script_name): + """Run a .husky hook script from the repo root; return CompletedProcess.""" + return subprocess.run( + ["python3", str(repo_root / ".husky" / script_name)], + cwd=repo_root, + capture_output=True, + text=True, + ) + + +def check_husky_circular_aliases(): + """Plant a self-aliasing page and confirm the hook rejects it.""" + rel = f"hugo/content/en/{MARKER}_alias/selftest.md" + target = repo_root / rel + if target.exists(): + record("WARN", "husky: circular-aliases test skipped (temp path exists)", rel) + return + + # An alias equal to the file's own location is the circular case. + body = ( + "---\n" + "title: Reorg Harness Self Test\n" + "aliases:\n" + f" - /{MARKER}_alias/selftest\n" + "---\n\n" + "Temporary file created by astro_reorg/validate_reorg.py.\n" + ) + try: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + git("add", "-f", rel) + proc = run_hook("check-circular-aliases.py") + if proc.returncode != 0 and "circular" in (proc.stdout + proc.stderr).lower(): + record("PASS", "husky: circular-aliases hook rejects bad input") + else: + record("FAIL", "husky: circular-aliases hook did NOT reject bad input", + "hook may not be referencing hugo/content/en/ " + f"(exit={proc.returncode})") + finally: + git("reset", "-q", "HEAD", rel) + if target.exists(): + target.unlink() + _rmdir_if_empty(target.parent) + + +def check_husky_section_index(): + """Plant a new top-level section with no _index.md and confirm rejection.""" + section = f"{MARKER}_section" + rel = f"hugo/content/en/{section}/page.md" + target = repo_root / rel + if target.exists() or (hugo_dir / "content" / "en" / section).exists(): + record("WARN", "husky: section-index test skipped (temp path exists)", rel) + return + + body = "---\ntitle: Reorg Harness Page\nprivate: true\n---\n\nTemp page.\n" + try: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + git("add", "-f", rel) + proc = run_hook("check-section-index.py") + if proc.returncode != 0 and "_index.md" in (proc.stdout + proc.stderr): + record("PASS", "husky: section-index hook rejects missing _index.md") + else: + record("FAIL", "husky: section-index hook did NOT reject bad input", + "hook may not be referencing hugo/content/en/ " + f"(exit={proc.returncode})") + finally: + git("reset", "-q", "HEAD", rel) + if target.exists(): + target.unlink() + _rmdir_if_empty(target.parent) + + +def check_husky_cdocs_gitignore(): + """ + Append a harness pattern to hugo/content/.gitignore, force-track a matching + compiled file (with a .mdoc.md sibling), and confirm the hook flags it. + """ + content_gitignore = hugo_dir / "content" / ".gitignore" + if not content_gitignore.exists(): + record("SKIP", "husky: cdocs-gitignore test skipped (no hugo/content/.gitignore)") + return + + pattern = f"/en/{MARKER}_cdocs.md" + compiled_rel = f"hugo/content/en/{MARKER}_cdocs.md" + source_rel = f"hugo/content/en/{MARKER}_cdocs.mdoc.md" + compiled = repo_root / compiled_rel + source = repo_root / source_rel + if compiled.exists() or source.exists(): + record("WARN", "husky: cdocs-gitignore test skipped (temp path exists)", + compiled_rel) + return + + original_gitignore = content_gitignore.read_text() + try: + content_gitignore.write_text( + original_gitignore.rstrip("\n") + f"\n{pattern}\n" + ) + source.write_text("Temp Cdocs source.\n") + compiled.write_text("Temp compiled Cdocs output.\n") + git("add", "-f", compiled_rel) # force past the gitignore to simulate the mistake + proc = run_hook("check-cdocs-gitignore.py") + if proc.returncode != 0 and MARKER in (proc.stdout + proc.stderr): + record("PASS", "husky: cdocs-gitignore hook flags tracked compiled file") + else: + record("FAIL", "husky: cdocs-gitignore hook did NOT flag tracked file", + "hook may not be referencing hugo/content/.gitignore " + f"(exit={proc.returncode})") + finally: + git("reset", "-q", "HEAD", compiled_rel) + for p in (compiled, source): + if p.exists(): + p.unlink() + content_gitignore.write_text(original_gitignore) + + +def _rmdir_if_empty(path): + """Remove a directory only if it is empty (safety guard).""" + try: + path.rmdir() + except OSError: + pass + + +def check_build_presence(): + """Static check; the real build is the manual `make start` in todo #5.""" + # Hugo's build entrypoints all need to be co-located under hugo/: the Makefile, + # the Node manifest, and go.mod (required by Hugo Modules at the project root). + required = ["Makefile", "package.json", "go.mod"] + missing = [n for n in required if not (hugo_dir / n).exists()] + if not missing: + record("PASS", "build: hugo/{Makefile,package.json,go.mod} present", + "run `cd hugo && make start` (todo #5) to verify the full build") + else: + record("FAIL", "build: missing Hugo build entrypoint(s) under hugo/", + ", ".join(f"hugo/{n}" for n in missing)) + + +def check_rollback_roundtrip(): + """execute_reorg.py then local_rollback.py must restore the tree byte-for-byte. + + This is the only check that exercises local_rollback.py at all. Rather than + mutate the live repo (and risk leaving it half-reorganized if a step fails), + it builds a small throwaway git repo holding one representative item for + every code path the two scripts touch — a mixed .gitignore (rules that route + to hugo/, to root, and to both), a workflow with substitutable paths, a + CODEOWNERS with a moved rule, a husky hook with a substitutable path, plus a + few moved/stayed files — then: + + snapshot -> run execute_reorg.py (answering y to every prompt) + -> run local_rollback.py -> snapshot again + -> assert byte-identical. + + It specifically catches the easy-to-miss case where execute_reorg.py edits a file in + place (the root .gitignore split) that rollback must restore from git rather + than merely delete. + """ + if shutil.which("python3") is None or shutil.which("git") is None: + record("SKIP", "rollback: python3/git not both available") + return + + workdir = tempfile.mkdtemp(prefix=f"{MARKER}_rollback_") + work = Path(workdir) + + def write(rel, text): + p = work / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + + def git_work(*args): + return subprocess.run(["git", *args], cwd=work, + capture_output=True, text=True) + + def snapshot(): + """relpath -> bytes for every file under work/, excluding .git/.""" + tree = {} + for path in sorted(work.rglob("*")): + if path.is_dir(): + continue + rel = path.relative_to(work) + if rel.parts and rel.parts[0] == ".git": + continue + tree[str(rel)] = path.read_bytes() + return tree + + try: + # Representative fixture. Every top-level name here must appear in + # astro_reorg/config.yaml or execute_reorg.py refuses to run; the mix below routes + # across moves_to_hugo, top_level, and generic-glob cases. + write(".gitignore", + "# build\n" + "public/*\n" + "data/generated\n" + "content/en/api/**/*.go\n" + "node_modules\n" + "\n" + "# generic (kept in both)\n" + "*.log\n" + "\n" + "# root-only tooling\n" + ".github/preview-links-template.md\n") + write(".github/workflows/sample.yml", + "name: sample\n" + "on:\n" + " pull_request:\n" + " paths:\n" + " - 'content/en/**/*.md'\n" + "jobs:\n" + " build:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: python local/bin/foo.py\n") + write(".github/CODEOWNERS", + "* @DataDog/documentation\n" + "/content/ @DataDog/team-a\n" + "data/ @DataDog/team-b\n" + "README.md @DataDog/team-c\n") + write(".husky/check-sample.py", + "from pathlib import Path\n" + 'repo_pattern = "content"\n' + "p = Path('content/en')\n") + write("README.md", "# Fixture\n") + write("astro/package.json", '{"name": "astro-fixture"}\n') + write("content/en/page.md", "---\ntitle: Page\n---\n\nBody.\n") + write("data/sample.yaml", "key: value\n") + write("Makefile", "start:\n\techo build\n") + write("package.json", '{"name": "hugo-fixture"}\n') + write("go.mod", "module example.com/fixture\n\ngo 1.21\n") + write("babel.config.js", "module.exports = {};\n") + + # The scripts resolve repo_root as their parent's parent, so place them + # under an astro_reorg/ subfolder that mirrors the real repo layout. + (work / "astro_reorg").mkdir() + for tool in ("execute_reorg.py", "local_rollback.py", "config.yaml"): + shutil.copy2(repo_root / "astro_reorg" / tool, work / "astro_reorg" / tool) + + git_work("init", "-q") + git_work("add", "-A") + commit = git_work("-c", "user.email=harness@example.com", + "-c", "user.name=reorg harness", + "-c", "commit.gpgsign=false", + "commit", "-q", "-m", "fixture") + if commit.returncode != 0: + record("FAIL", "rollback: could not commit fixture repo", + (commit.stderr or commit.stdout).strip()[:200]) + return + + before = snapshot() + + # execute_reorg.py is interactive (single y/N per mutation section); answer y to + # all. Far more lines than prompts is fine — the extras are ignored. + reorg = subprocess.run( + ["python3", str(work / "astro_reorg" / "execute_reorg.py")], + cwd=work, capture_output=True, text=True, input="y\n" * 100, + ) + if reorg.returncode != 0 or not (work / "hugo").exists(): + record("FAIL", "rollback: execute_reorg.py failed on the fixture", + (reorg.stderr or reorg.stdout).strip()[:200]) + return + + rollback = subprocess.run( + ["python3", str(work / "astro_reorg" / "local_rollback.py")], + cwd=work, capture_output=True, text=True, + ) + if rollback.returncode != 0 or (work / "hugo").exists(): + record("FAIL", "rollback: local_rollback.py failed on the fixture", + (rollback.stderr or rollback.stdout).strip()[:200]) + return + + after = snapshot() + + added = sorted(set(after) - set(before)) + removed = sorted(set(before) - set(after)) + changed = sorted(p for p in before.keys() & after.keys() + if before[p] != after[p]) + if added or removed or changed: + diffs = ([f"+ {p}" for p in added] + + [f"- {p}" for p in removed] + + [f"~ {p} (in-place edit not reverted)" for p in changed]) + record("FAIL", "rollback: tree not byte-identical after reorg + rollback", + f"{len(diffs)} diff(s):\n " + "\n ".join(diffs[:20])) + else: + record("PASS", "rollback: reorg + rollback restores the tree byte-for-byte", + f"{len(before)} file(s) round-tripped, incl. the .gitignore split") + finally: + shutil.rmtree(work, ignore_errors=True) diff --git a/astro_reorg/label_old_prs.py b/astro_reorg/label_old_prs.py new file mode 100644 index 00000000000..c3dac2c7210 --- /dev/null +++ b/astro_reorg/label_old_prs.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +""" +Label open PRs that have had no activity for six months or more with `autolabeled-stale`. +Already-labeled PRs are excluded from the query so they are never processed twice. +Defaults to a dry run where it just reports what would be done. + +Use the GitHub label query to review labeled PRs: +https://github.com/DataDog/documentation/pulls?q=is%3Aopen+label%3Aautolabeled-stale + +Usage: + python3 astro_reorg/label_old_prs.py [--no-dry-run] [--pr NUMBER ...] [--limit N] + +Flags: + --no-dry-run Actually add labels instead of just reporting what would be done. + --pr NUMBER ... Only process the given PR number(s). + --limit N Stop after labeling N PRs. PRs that are skipped don't count toward + the limit. Use it to roll out gradually: start with --limit 1, review + the results, then raise it as you gain confidence. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from datetime import datetime, timedelta, timezone + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +REPO = "DataDog/documentation" + +INACTIVITY_DAYS = 180 # ~6 calendar months + +LABEL_STALE = "autolabeled-stale" +LABEL_COLOR = "e4e669" + +# --------------------------------------------------------------------------- +# Shell helpers +# --------------------------------------------------------------------------- + +def run(cmd: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True) + + +def gh_json(*args: str) -> object: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return json.loads(result.stdout) if result.stdout.strip() else [] + + +def gh_run(*args: str) -> str: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return result.stdout + + +# --------------------------------------------------------------------------- +# Staleness helpers +# --------------------------------------------------------------------------- + +def _parse_ts(ts: str) -> datetime: + """Parse a GitHub ISO-8601 timestamp (e.g. '2026-07-16T12:00:00Z').""" + return datetime.fromisoformat(ts.replace("Z", "+00:00")) + + +def is_inactive(pr: dict) -> bool: + """Return True if the PR has had no activity in the last INACTIVITY_DAYS days.""" + updated_at = pr.get("updatedAt") + if not updated_at: + return False + cutoff = datetime.now(timezone.utc) - timedelta(days=INACTIVITY_DAYS) + return _parse_ts(updated_at) < cutoff + + +def days_since_update(pr: dict) -> int | None: + """Return the number of days since the PR was last updated, or None.""" + updated_at = pr.get("updatedAt") + if not updated_at: + return None + delta = datetime.now(timezone.utc) - _parse_ts(updated_at) + return delta.days + + +# --------------------------------------------------------------------------- +# GitHub helpers +# --------------------------------------------------------------------------- + +def ensure_label_exists(label: str, dry_run: bool) -> None: + """Create the GitHub label if it doesn't already exist.""" + existing = gh_json("label", "list", "--repo", REPO, "--search", label, "--json", "name") + if any(l["name"] == label for l in existing): # type: ignore[index] + return + if dry_run: + print(f" [dry-run] would create label: {label!r}") + return + gh_run("label", "create", label, "--repo", REPO, + "--color", LABEL_COLOR, "--description", "PR labeled due to inactivity") + print(f" Created label: {label!r}") + + +def add_label(pr_number: int, label: str, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would add label {label!r} to PR #{pr_number}") + return + gh_run("api", f"repos/{REPO}/issues/{pr_number}/labels", + "-f", f"labels[]={label}") + print(f" Added label {label!r} to PR #{pr_number}") + + +# --------------------------------------------------------------------------- +# Per-PR processing +# --------------------------------------------------------------------------- + +def process_pr(pr: dict, dry_run: bool) -> bool: + """Label a stale PR. Return True if we acted, False if we skipped.""" + pr_number = pr["number"] + title = pr["title"] + days = days_since_update(pr) + + print(f"\nPR #{pr_number}: {title}") + print(f" Last updated: {pr.get('updatedAt', 'unknown')} ({days} days ago)") + + if not is_inactive(pr): + print(f" Active within the last {INACTIVITY_DAYS} days — skipping.") + return False + + print(f" Inactive for {days} days — labeling.") + add_label(pr_number, LABEL_STALE, dry_run) + return True + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def get_open_prs(only: list[int] | None = None, fetch_limit: int = 1000) -> list[dict]: + """Return open PRs without the stale label, optionally filtered to specific numbers.""" + fields = "number,title,updatedAt" + if only: + prs = [] + for n in only: + pr = gh_json("pr", "view", str(n), "--repo", REPO, "--json", fields) + prs.append(pr) + return prs # type: ignore[return-value] + return gh_json( # type: ignore[return-value] + "pr", "list", "--repo", REPO, "--state", "open", + "--search", f"-label:{LABEL_STALE} sort:updated-asc", + "--json", fields, "--limit", str(fetch_limit), + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Label open PRs with no activity for six months or more.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "--dry-run", action=argparse.BooleanOptionalAction, default=True, + help="Report what would be done without making any changes (default: on). " + "Use --no-dry-run to apply changes.", + ) + parser.add_argument( + "--pr", type=int, action="append", dest="prs", metavar="NUMBER", + help="Only check this PR number (may be repeated).", + ) + parser.add_argument( + "--limit", type=int, default=None, metavar="N", + help="Stop after labeling N PRs. Skipped PRs don't count toward the limit. " + "Use it to roll out gradually: start at 1, review, then raise it.", + ) + args = parser.parse_args() + + if args.limit is not None and args.limit < 1: + parser.error("--limit must be a positive integer.") + + if args.dry_run: + print("DRY-RUN mode — no PRs will be modified.\n") + + ensure_label_exists(LABEL_STALE, args.dry_run) + + cutoff_date = datetime.now(timezone.utc) - timedelta(days=INACTIVITY_DAYS) + print(f"Targeting PRs with no activity since: {cutoff_date.date()} ({INACTIVITY_DAYS} days ago)\n") + + prs = get_open_prs(args.prs) + print(f"Found {len(prs)} open PR(s) to check.") + if args.limit is not None: + print(f"Limit: will stop after labeling {args.limit} PR(s).") + + acted = 0 + for pr in prs: + if args.limit is not None and acted >= args.limit: + print(f"\nReached --limit of {args.limit} labeled PR(s) — stopping.") + break + try: + if process_pr(pr, args.dry_run): + acted += 1 + except Exception as exc: + print(f"\nERROR processing PR #{pr.get('number', '?')}: {exc}", + file=sys.stderr) + print(" Skipping to the next PR.", file=sys.stderr) + + print(f"\nLabeled {acted} PR(s). Done.") + print( + "\nView labeled PRs:\n" + " https://github.com/DataDog/documentation/pulls" + "?q=is%3Aopen+label%3Aautolabeled-stale" + ) + + +if __name__ == "__main__": + main() diff --git a/astro_reorg/local_rollback.py b/astro_reorg/local_rollback.py new file mode 100644 index 00000000000..06bd6bbbda4 --- /dev/null +++ b/astro_reorg/local_rollback.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +import shutil +import subprocess +import sys +from pathlib import Path + +repo_root = Path(__file__).parent.parent +hugo_dir = repo_root / "hugo" + +if not hugo_dir.exists(): + print("Nothing to roll back: hugo/ does not exist.", file=sys.stderr) + sys.exit(1) + +makefile_config_src = hugo_dir / "Makefile.config" +makefile_config_dst = repo_root / "Makefile.config" +if makefile_config_src.exists(): + shutil.copy2(makefile_config_src, makefile_config_dst) + print("Restored Makefile.config to repo root.") + +shutil.rmtree(hugo_dir) +print("Removed hugo/") + +subprocess.run( + ["git", "checkout", "--", + ".gitignore", ".github/workflows/", ".github/CODEOWNERS", ".husky/"], + cwd=repo_root, + check=True, +) +print("Restored .gitignore, .github/workflows/, .github/CODEOWNERS, and .husky/ from git.") diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py new file mode 100644 index 00000000000..9378b3a4405 --- /dev/null +++ b/astro_reorg/resolve_pr_conflicts.py @@ -0,0 +1,1171 @@ +#!/usr/bin/env python3 +""" +Find open PRs with merge conflicts caused by the reorg. +Defaults to a dry run where it just reports the changes +that would be made. + +Usage: + python3 astro_reorg/resolve_pr_conflicts.py --limit N [--no-dry-run] [--live | --base-branch BRANCH] [--pr NUMBER ...] + +Flags: + --no-dry-run Actually apply fixes and labels instead of just reporting what would be done. + --live Run against the real master. Without it the script + defaults to the mock base branch from config.yaml's + `test:` section (the branch create_test_prs.py opens + its test PRs against) for safety. + --base-branch BRANCH Branch to treat as the post-reorg base. Overrides the + default mock base branch and --live. + --pr NUMBER ... Only process the given PR number(s) instead of all open + PRs against the base branch. PRs targeting a different + base are skipped. + --limit N Stop after acting on N PRs (auto-fixing or labeling). + PRs that need no action — mergeable, already fixed, or + not-yet-computed — don't count toward the limit. Use it + to roll out gradually: start with --limit 1, review the + results, then raise it as you gain confidence. + +Background: + The reorg moves every entry in `moves_to_hugo` (astro_reorg/config.yaml) + from the repo root into hugo/. For example, content/ → hugo/content/, + layouts/ → hugo/layouts/, etc. PRs opened before the reorg was merged to + master will have their branches pointing at the old paths. When github + tries to compute mergeability, those PRs show as CONFLICTING. + +How we decide whether a conflict came from the reorg: + When git merges a PR branch into post-reorg master it uses rename detection + to pair the PR's pre-reorg file path (e.g. content/en/foo.md) with the + corresponding post-reorg path (hugo/content/en/foo.md) in master. If both + sides modified the file, git reports a conflict: Rename detection usually + places the conflict at the POST-reorg path (hugo/content/en/foo.md), because + that is where the file lives in master. Occasionally, when rename detection + fails (the file was heavily edited or the threshold wasn't met), git instead + reports a "deleted by them" conflict at the PRE-reorg path. + + A conflict is "from the reorg" if its path maps to a reorg-moved location: + a. hugo//... where is in moves_to_hugo (post-reorg path, + rename detected — the common case) + b. /... where is in moves_to_hugo (pre-reorg path, rename + NOT detected — git sees it as "they deleted the file") + + Any conflict at a path NOT matching either pattern is unrelated to the + reorg and must be resolved manually. + + We also detect a subtler case: files ADDED by the PR at a pre-reorg path + (e.g. content/en/brand_new.md). These produce no conflict marker — git + happily merges them at the wrong path. We catch them by scanning all + paths staged in the test merge and flagging any whose first segment is in + moves_to_hugo. + +Auto-fix strategy (reorg-only PRs): + For PRs where every conflict is a reorg conflict, the fix is to replay the + PR's commits at the post-reorg paths: + + 1. Find the merge base between the PR branch and the base branch (where + the PR diverged from master, before the reorg landed). + 2. Export each PR commit as its own patch with `git format-patch`, + preserving the original author and commit message. + 3. Rewrite every file path in the patches whose first segment is in + moves_to_hugo to be prefixed with hugo/ (content/en/ → hugo/content/en/). + 4. Replay the series onto a fresh branch off the base branch with + `git am --3way`. --3way falls back to a per-patch 3-way merge when + context lines have drifted because master made unrelated edits between + the PR's base and today. + 5. Push as `reorg-fix/pr-`, open a new PR for it (whose body @mentions + the original author so they're notified immediately), close the original + PR with a comment pointing to the fix PR, and label the original + astro-reorg-autofixed. + + If a run dies partway through step 5 (branch pushed and/or fix PR opened, but + the original never closed), a later run detects the existing branch/PR and + finishes the job rather than bailing — it never rebuilds or force-pushes an + existing fix branch, so a fix already in review is safe. + + PRs from forks cannot be auto-fixed (we don't have push access to the fork). + They receive the astro-reorg-manual-review label. +""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from datetime import datetime, timedelta, timezone +from pathlib import Path + +try: + import yaml +except ImportError: + print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +SCRIPT_DIR = Path(__file__).parent +REPO_ROOT = SCRIPT_DIR.parent +CONFIG_PATH = SCRIPT_DIR / "config.yaml" + +with CONFIG_PATH.open() as f: + _config = yaml.safe_load(f) + +HUGO_FOLDER_FILES: set[str] = set(_config.get("moves_to_hugo", [])) +TOP_LEVEL_FILES: set[str] = set(_config.get("top_level", [])) + +# Shared with create_test_prs.py: the mock base branch used for test runs. When +# invoked without --live or --base-branch, the script defaults to this branch +# as the post-reorg base so it operates on the same branch the test PRs were +# opened against. +_TEST_CONFIG = _config.get("test", {}) +TEST_BASE_BRANCH: str | None = _TEST_CONFIG.get("mock_reorged_master_branch") + +REPO = "DataDog/documentation" + +# The author-facing comments link to two files that live on DIFFERENT branches +# after the reorg, so they need separate branch constants — do not merge them. +# +# REORG_README_BRANCH: branch hosting REPO_REORG.md. Before the reorg lands this +# is the scripts branch; once REPO_REORG.md is published on master (see +# docs/reorg_execution_steps.md), set this to "master". +# CONFIG_LINK_BRANCH: branch hosting astro_reorg/config.yaml. The reorg DELETES +# astro_reorg/ from master, so this link must point at a branch that RETAINS it +# (the scripts branch) — it can never be "master". +REORG_README_BRANCH = "jen.gilbert/astro-reorg-scripts" +CONFIG_LINK_BRANCH = "jen.gilbert/astro-reorg-scripts" +REPO_REORG_README_LINK = f"https://github.com/DataDog/documentation/blob/{REORG_README_BRANCH}/REPO_REORG.md" +LABEL_MANUAL_REVIEW = "astro-reorg-manual-review" +LABEL_STALE = "astro-reorg-stale" +LABEL_AUTOFIXED = "astro-reorg-autofixed" +LABEL_AUTO_PR = "astro-reorg-auto-pr" +LABEL_NO_CONFLICTS = "astro-reorg-no-conflicts" +# Author-applied on an already-processed PR to request manual intervention. The +# script does not act on these PRs — they're excluded from the query and left +# for a person to handle. +LABEL_HELP_REQUESTED = "astro-reorg-help-requested" +# Applied by the script to a work-in-progress PR after it comments once. Excluded +# from the query so the PR isn't picked up (and re-commented on) on later runs. +LABEL_SKIP = "astro-reorg-skip" +LABEL_COLOR = "e4e669" +LABEL_DESCRIPTION = "Needs manual conflict resolution after replatforming reorg" +# Applied to every PR the script acts on, regardless of outcome (auto-fixed, +# manual review, WIP, stale, ...). Purely a visibility aid so all affected PRs +# can be found in GitHub with one label filter — it is NOT used for idempotency +# and is deliberately absent from the get_open_prs query. +LABEL_PROCESSED = "astro-reorg-processed" + +# PRs with no activity in this many days are treated as stale and receive a +# comment + label instead of an auto-fix attempt. +STALE_DAYS = 31 # ~1 calendar month + +# A stale-labeled PR is considered reactivated only if its updatedAt is later +# than when we applied the label by more than this grace window. The label +# event and the resulting updatedAt bump share a timestamp; the grace absorbs +# any skew so our own labeling never counts as fresh activity. +STALE_REACTIVATE_GRACE = timedelta(seconds=60) + +# Appended to every author-facing comment so readers know how to follow up. +AUTOMATED_COMMENT_FOOTER = ( + "\n\nThis is an automated comment, but if you have a question, you can mention me in this PR " + "(external contributors) or reach out in #docs-repo-reorg-support on Slack (internal contributors)." +) + +# --------------------------------------------------------------------------- +# Comment builders — one function per user-facing message +# --------------------------------------------------------------------------- + +def build_manual_review_comment() -> str: + """Posted on PRs with non-reorg conflicts, or when auto-fix fails.""" + return ( + f"This PR has merge conflicts from a [recent repo reorg]({REPO_REORG_README_LINK}) that could not be resolved automatically.\n\n" + "If you feel comfortable resolving the conflicts yourself:\n\n" + "1. Resolve the conflicts. For a full list of repo files and folders and their updated location, " + "see [the configuration file for the reorg script]" + f"(https://github.com/DataDog/documentation/blob/{CONFIG_LINK_BRANCH}/astro_reorg/config.yaml).\n" + f"2. When your PR is ready for merge, remove the `{LABEL_WIP}` label.\n" + "3. Wait for the standard docs team approval before merging. " + "Optionally, you can check the 'ready for merge' checkbox in the PR description " + "if you would like the docs team to merge it for you.\n\n" + f"If you need assistance resolving your conflicts, add the label `{LABEL_HELP_REQUESTED}` to your PR. " + "This will add it to our support queue, and we will reach out to you as soon as possible." + + AUTOMATED_COMMENT_FOOTER + ) + + +def build_stale_comment() -> str: + """Posted on PRs with no activity in the last STALE_DAYS days.""" + return ( + f"This PR has conflicts created by the [docs repo reorg project]({REPO_REORG_README_LINK}). " + f"Because this PR is stale (more than {STALE_DAYS} days old), no attempt was made to auto-resolve the conflicts. " + "If you still intend to use this PR, remove the label " + f"`{LABEL_STALE}`. Your PR will be processed in the next batch of attempted auto-fixes." + + AUTOMATED_COMMENT_FOOTER + ) + + +def build_wip_comment() -> str: + """Posted on PRs carrying the WORK IN PROGRESS label.""" + return ( + f"This PR has merge conflicts created by the [docs repo reorg project]({REPO_REORG_README_LINK}). " + "Because this PR is marked as a work in progress, no attempt was made to auto-resolve the conflicts. " + f"When your PR is finished, you can queue your PR for an auto-fix by removing the `{LABEL_WIP}` label and the `{LABEL_SKIP}` label. " + "Your PR will be processed in the next batch of attempted auto-fixes." + + AUTOMATED_COMMENT_FOOTER + ) + + +def build_autofix_close_comment(new_pr_number: int | str) -> str: + """Posted on the original PR when it is closed in favor of a fix PR.""" + return ( + f"🤖 **Reorg conflict auto-fix:**\n\n" + f"This PR has merge conflicts caused by the [recent docs repo reorg]({REPO_REORG_README_LINK}) " + f"(files moved from the repo root into `hugo/`). " + f"A new PR with your commits translated to the correct paths " + f"has been opened: #{new_pr_number}\n\n" + f"Please follow the instructions in the PR description." + + AUTOMATED_COMMENT_FOOTER + ) + + +def build_autofix_pr_body( + original_pr_number: int | str, + original_body: str, + author_login: str | None = None, +) -> str: + """Body of the auto-generated fix PR. + + When author_login is given, the body opens with an @mention so the original + author is notified the moment this PR is created. This matters because the + fix PR is opened before the original is closed: if a run is interrupted in + between, the @mention is the author's only signal that the fix exists. + """ + mention = ( + f"@{author_login} — the [docs repo reorg]({REPO_REORG_README_LINK}) created merge conflicts in your " + f"PR #{original_pr_number}. This is an auto-generated replacement with the " + f"file paths fixed; please use it instead of the original.\n\n" + if author_login else "" + ) + return ( + f"🤖 Auto-generated fix for #{original_pr_number}.\n\n" + f"{mention}" + f"This PR replays the commits from #{original_pr_number} with file paths " + f"translated to the post-reorg `hugo/` layout. The original commits " + f"are preserved — same messages and authorship.\n\n" + f"The original PR (#{original_pr_number}) will be closed in favor of this one.\n\n" + f"**Next steps:**\n\n" + f"1. Verify that this PR looks correct in the browser.\n" + f"2. Remove the `{LABEL_WIP}` label from this PR.\n" + f"3. Wait for the standard docs team approval before merging. " + f"Optionally, you can check the 'ready for merge' checkbox below " + f"if you would like the docs team to merge it for you.\n\n" + f"- [ ] Ready for merge\n\n" + f"---\n\n" + f"**Original PR description:**\n\n{original_body}" + ) + +# Existing repo labels assumed to already exist; the script checks for them but +# does not create them. +LABEL_WIP = "WORK IN PROGRESS" +LABEL_DO_NOT_MERGE = "Do Not Merge" # test mode only — keeps fix PRs out of review queues + +# Set in main() from --base-branch; everything else reads this. +BASE_BRANCH = "master" + +# True when running against the mock base branch rather than real master. Set +# in main(); gates test-only behavior like labeling fix PRs "Do Not Merge". +IS_TEST_MODE = False + +# --------------------------------------------------------------------------- +# Shell helpers +# --------------------------------------------------------------------------- + +def run(cmd: list[str], *, cwd: Path | None = None, input: str | None = None) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True, cwd=cwd, input=input) + + +def run_bytes(cmd: list[str], *, cwd: Path | None = None) -> subprocess.CompletedProcess: + """Run a command and capture output as bytes (needed for binary file content).""" + return subprocess.run(cmd, capture_output=True, cwd=cwd) + + +def gh_json(*args: str) -> object: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return json.loads(result.stdout) + + +def gh_run(*args: str) -> str: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return result.stdout + + +def git(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess: + return run(["git", *args], cwd=cwd or REPO_ROOT) + + +# --------------------------------------------------------------------------- +# Staleness helpers +# --------------------------------------------------------------------------- + +def _parse_ts(ts: str) -> datetime: + """Parse a GitHub ISO-8601 timestamp (e.g. '2026-07-16T12:00:00Z').""" + return datetime.fromisoformat(ts.replace("Z", "+00:00")) + + +def is_pr_stale(pr: dict) -> bool: + """Return True if the PR has had no activity in the last STALE_DAYS days.""" + updated_at = pr.get("updatedAt") + if not updated_at: + return False + cutoff = datetime.now(timezone.utc) - timedelta(days=STALE_DAYS) + return _parse_ts(updated_at) < cutoff + + +def stale_labeled_at(pr_number: int) -> datetime | None: + """Return when LABEL_STALE was most recently added to the PR, or None if it + was never labeled or the timeline can't be read. Reads the issue timeline, + which records a 'labeled' event (with the label name and a created_at) for + every label addition.""" + try: + events = gh_json("api", "--paginate", + f"repos/{REPO}/issues/{pr_number}/timeline") + except RuntimeError as exc: + print(f" Could not read timeline for PR #{pr_number}: {exc}", + file=sys.stderr) + return None + times = [e["created_at"] for e in events # type: ignore[union-attr] + if e.get("event") == "labeled" + and e.get("label", {}).get("name") == LABEL_STALE + and e.get("created_at")] + return max((_parse_ts(t) for t in times), default=None) + + +# --------------------------------------------------------------------------- +# Reorg path helpers +# --------------------------------------------------------------------------- + +def path_first_segment(file_path: str) -> str: + """Return the first directory segment of a path string.""" + return file_path.lstrip("/").split("/", 1)[0] + + +def is_reorg_path(file_path: str) -> bool: + """ + Return True if this path maps to a location moved by the reorg. + + Covers both forms that git can report: + - Pre-reorg path: content/en/foo.md (first segment in moves_to_hugo) + - Post-reorg path: hugo/content/en/foo.md (second segment in moves_to_hugo) + """ + parts = Path(file_path).parts + if not parts: + return False + if parts[0] in HUGO_FOLDER_FILES: + return True + if parts[0] == "hugo" and len(parts) > 1 and parts[1] in HUGO_FOLDER_FILES: + return True + return False + + +def to_post_reorg_path(file_path: str) -> str: + """ + Convert a file path to its post-reorg location. + + content/en/foo.md → hugo/content/en/foo.md + hugo/content/en/foo.md → hugo/content/en/foo.md (already correct) + README.md → README.md (not a reorg-moved path) + """ + parts = Path(file_path).parts + if not parts: + return file_path + if parts[0] == "hugo": + return file_path + if parts[0] in HUGO_FOLDER_FILES: + return "hugo/" + file_path + return file_path + + +def to_pre_reorg_path(file_path: str) -> str | None: + """ + Convert a post-reorg path back to its pre-reorg location, or None if not + applicable. + + hugo/content/en/foo.md → content/en/foo.md + """ + parts = Path(file_path).parts + if len(parts) > 1 and parts[0] == "hugo" and parts[1] in HUGO_FOLDER_FILES: + return "/".join(parts[1:]) + return None + + +# --------------------------------------------------------------------------- +# Merge conflict analysis (run inside a temp worktree) +# --------------------------------------------------------------------------- + +def get_conflict_classification(worktree: Path) -> tuple[list[str], list[str]]: + """ + Parse git status inside a worktree after a --no-commit merge attempt. + + Returns (reorg_conflicts, other_conflicts) — lists of conflicted file paths. + + git status --porcelain conflict codes (XY): + UU both modified + AA both added + DD both deleted + AU added by us, not staged on their side + UA added by them + DU deleted by us + UD deleted by them (common for rename/delete: the reorg "deleted" the + file from the old path by renaming it; the PR still has the old path) + + For rename conflicts git shows "old -> new" in the path field. We use + the FINAL path (after the arrow) for classification, because that is the + path that needs to be resolved in the working tree. + """ + result = run(["git", "status", "--porcelain"], cwd=worktree) + reorg: list[str] = [] + other: list[str] = [] + for line in result.stdout.splitlines(): + if len(line) < 4: + continue + xy = line[:2] + path = line[3:] + # Only unmerged (conflict) entries have U in XY or are AA/DD. + if "U" not in xy and xy not in ("AA", "DD"): + continue + # Rename entries show "old -> new". Treat a rename conflict as + # reorg-caused only when BOTH endpoints map to reorg-moved paths; if + # either side is unrelated, classify it as "other" so the PR goes to + # manual review rather than getting an auto-fix we can't be sure about. + if " -> " in path: + src, dst = (p.strip() for p in path.split(" -> ", 1)) + both_reorg = is_reorg_path(src) and is_reorg_path(dst) + (reorg if both_reorg else other).append(dst) + else: + path = path.strip() + (reorg if is_reorg_path(path) else other).append(path) + return reorg, other + + +def get_wrong_path_additions(worktree: Path) -> list[str]: + """ + Find files staged in the test merge at PRE-REORG paths that should instead + live under hugo/. + + These do not cause merge conflict markers — git happily adds the file at + the old path with no complaint. We catch them here so the caller can + include them in the "reorg conflict" count. + + Specifically: any file with status 'A' (added) in the staged diff against + HEAD whose first path segment is in moves_to_hugo is a "wrong path + addition" caused by the PR adding a brand-new file at a pre-reorg path. + """ + result = run( + ["git", "diff", "--cached", "--name-status", "--diff-filter=A", "HEAD"], + cwd=worktree, + ) + wrong: list[str] = [] + for line in result.stdout.splitlines(): + parts = line.split("\t", 1) + if len(parts) != 2: + continue + path = parts[1].strip() + if path_first_segment(path) in HUGO_FOLDER_FILES: + wrong.append(path) + return wrong + + +# --------------------------------------------------------------------------- +# Diff path transformation +# --------------------------------------------------------------------------- + +def transform_diff_paths(diff_text: str) -> str: + """ + Rewrite file paths in a unified diff to use post-reorg (hugo/-prefixed) paths. + + Only paths whose first segment is in moves_to_hugo are changed; all other + paths, and all diff hunk content lines, are left untouched. + + Handles the following diff header forms: + diff --git a/ b/ + --- a/ + +++ b/ + rename from + rename to + + The hunk bodies (+/- content lines) are never touched because they contain + file content, not file names. + """ + out: list[str] = [] + for line in diff_text.splitlines(keepends=True): + if line.startswith("diff --git "): + # "diff --git a/content/en/foo.md b/content/en/foo.md" + # Rewrite both the a/ and b/ path tokens. + tokens = line.rstrip("\n").split(" ") + new_tokens = [] + for tok in tokens: + if tok.startswith("a/") or tok.startswith("b/"): + side, rest = tok[:2], tok[2:] + new_tokens.append(side + to_post_reorg_path(rest)) + else: + new_tokens.append(tok) + out.append(" ".join(new_tokens) + "\n") + elif line.startswith("--- a/") or line.startswith("+++ b/"): + side = line[:6] # "--- a/" or "+++ b/" + rest = line[6:].rstrip("\n") + out.append(side + to_post_reorg_path(rest) + "\n") + elif line.startswith("rename from ") or line.startswith("rename to "): + keyword_end = line.index(" ", line.index(" ") + 1) + 1 + keyword = line[:keyword_end] + path = line[keyword_end:].rstrip("\n") + out.append(keyword + to_post_reorg_path(path) + "\n") + else: + out.append(line) + return "".join(out) + + +# --------------------------------------------------------------------------- +# GitHub label helpers +# --------------------------------------------------------------------------- + +def ensure_label_exists(label: str, dry_run: bool) -> None: + """Create the GitHub label if it doesn't already exist.""" + existing = gh_json("label", "list", "--repo", REPO, "--search", label, "--json", "name") + if any(l["name"] == label for l in existing): # type: ignore[index] + return + if dry_run: + print(f" [dry-run] would create label: {label!r}") + return + gh_run("label", "create", label, "--repo", REPO, + "--color", LABEL_COLOR, "--description", LABEL_DESCRIPTION) + print(f" Created label: {label!r}") + + +def add_label(pr_number: int, label: str, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would add label {label!r} to PR #{pr_number}") + return + # Use the REST API rather than `gh pr edit --add-label`. `gh pr edit` + # fetches the PR's projectCards to preserve them, but that field is part of + # Projects (classic), now deprecated — so the call fails with a GraphQL + # error even though only a label is being added. The REST labels endpoint + # touches nothing else and appends the label. + gh_run("api", f"repos/{REPO}/issues/{pr_number}/labels", + "-f", f"labels[]={label}") + print(f" Added label {label!r} to PR #{pr_number}") + + +def remove_label(pr_number: int, label: str, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would remove label {label!r} from PR #{pr_number}") + return + # Mirror add_label: hit the REST labels endpoint directly to avoid + # `gh pr edit`'s deprecated projectCards fetch. DELETE is a no-op (404) + # if the label isn't present, so callers can call it unconditionally. + gh_run("api", "--method", "DELETE", + f"repos/{REPO}/issues/{pr_number}/labels/{label}") + print(f" Removed label {label!r} from PR #{pr_number}") + + +def post_comment(pr_number: int, body: str, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would comment on PR #{pr_number}:\n {body[:120]}...") + return + gh_run("pr", "comment", str(pr_number), "--repo", REPO, "--body", body) + print(f" Posted comment on PR #{pr_number}") + + +def send_to_manual_review(pr_number: int, dry_run: bool) -> None: + """Route a PR to manual review, consistently, wherever we can't auto-fix. + + Applies both the durable manual-review label (which excludes the PR from all + future runs) and WORK IN PROGRESS (which keeps it out of the docs team's + review queue until the author has resolved the conflicts), then comments once + explaining what to do. build_manual_review_comment() already tells the author + to remove the WORK IN PROGRESS label once the conflicts are resolved. + + Used for every non-auto-fixable outcome: non-reorg conflicts, unclassifiable + merge failures, and failed replays. + """ + add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + add_label(pr_number, LABEL_WIP, dry_run) + post_comment(pr_number, build_manual_review_comment(), dry_run) + + +# --------------------------------------------------------------------------- +# Auto-fix +# --------------------------------------------------------------------------- + +def find_open_fix_pr(fix_branch: str) -> dict | None: + """Return an open PR opened from fix_branch on a prior run, or None. + + A run that dies after opening the fix PR but before closing the original + leaves this PR behind. Finding it lets a later run finish the job instead of + bailing to manual review. + """ + prs = gh_json("pr", "list", "--repo", REPO, "--state", "open", + "--head", fix_branch, "--json", "number,url") + return prs[0] if prs else None # type: ignore[index] + + +def fix_branch_on_origin(fix_branch: str) -> bool: + """True if fix_branch already exists on origin (pushed by a prior run).""" + return bool(git("ls-remote", "--heads", "origin", fix_branch).stdout.strip()) + + +def finalize_autofix(pr: dict, fix_branch: str, existing_fix_pr: dict | None) -> bool: + """Finish an auto-fix once the fix branch exists on origin: ensure the fix PR + is open and labeled, then close the original PR pointing at it and label the + original autofixed. + + Kept separate from the branch-building steps so it can complete a run that + was interrupted partway through. It is idempotent: label adds are no-ops when + the label is already present, and the original PR is only ever closed once + (it is still open whenever we reach here, since a closed PR drops out of the + open-PR query). `existing_fix_pr` is a prior run's fix PR when one was already + opened; otherwise it is None and we open the PR now. + """ + pr_number = pr["number"] + + if existing_fix_pr: + new_pr_number = existing_fix_pr["number"] + print(f" Reusing existing fix PR #{new_pr_number}: {existing_fix_pr['url']}") + else: + original_body = pr.get("body") or "" + author_login = (pr.get("author") or {}).get("login") + new_pr_body = build_autofix_pr_body(pr_number, original_body, author_login) + new_pr_title = f"[reorg fix] {pr['title']}" + pr_create = gh_run( + "pr", "create", + "--repo", REPO, + "--head", fix_branch, + "--base", pr["baseRefName"], + "--title", new_pr_title, + "--body", new_pr_body, + ) + new_pr_url = pr_create.strip() + new_pr_number = int(new_pr_url.rstrip("/").split("/")[-1]) + print(f" Opened fix PR: {new_pr_url}") + + # Labels are a set on GitHub's side, so re-adding an existing one is a + # harmless no-op — safe when finishing a run that applied some already. + add_label(int(new_pr_number), LABEL_WIP, dry_run=False) + add_label(int(new_pr_number), LABEL_AUTO_PR, dry_run=False) + # In test mode, keep the auto-created PR out of other teams' review queues by + # marking it "Do Not Merge". Never do this on real master. + if IS_TEST_MODE: + add_label(int(new_pr_number), LABEL_DO_NOT_MERGE, dry_run=False) + + gh_run( + "pr", "close", str(pr_number), "--repo", REPO, + "--comment", build_autofix_close_comment(new_pr_number), + ) + print(f" Closed PR #{pr_number} with comment pointing to fix PR #{new_pr_number}") + add_label(pr_number, LABEL_AUTOFIXED, dry_run=False) + return True + + +def attempt_fix(pr: dict, dry_run: bool) -> bool: + """ + Attempt to auto-fix a reorg-conflict PR by re-applying its commits at the + post-reorg paths. + + Strategy: format-patch + am, preserving the PR's individual commits. + + 1. Find the merge base between the PR branch and BASE_BRANCH (the commit + where the PR diverged from master before the reorg landed). + 2. Use `git format-patch` to export each PR commit as its own patch, + including the original author name, email, and commit message. + 3. Rewrite file paths in every patch: anything whose first segment is in + moves_to_hugo gets a hugo/ prefix (content/ → hugo/content/, etc.). + 4. Apply the series with `git am --3way` onto a fresh branch off + BASE_BRANCH. --3way falls back to a 3-way merge per patch when + context lines have drifted due to unrelated master changes, so the + PR's individual commits land cleanly even if master moved on. + 5. Push as `reorg-fix/pr-` and post a comment on the PR. + + Using format-patch/am rather than a single squashed diff means the fix + branch has the same commit history as the original PR — same messages, + same authorship, same granularity — making it easy to review and to revert + individual commits if needed. + + Returns True if the fix was applied (or would be in dry-run mode), False + if we gave up and the caller should fall back to labeling. + + PRs from forks are not auto-fixed: we cannot push to a fork branch, so we + return False immediately and let the caller add the manual-review label. + """ + pr_number = pr["number"] + head_ref = pr["headRefName"] + pr_base = pr["baseRefName"] + is_fork = pr.get("isCrossRepository", False) + + if is_fork: + print(f" PR #{pr_number} is from a fork — cannot push; will label instead.") + return False + + # Fetch the PR branch so we can reference it locally. + pr_remote_ref = f"refs/remotes/origin/{head_ref}" + fetch = git("fetch", "origin", f"{head_ref}:{pr_remote_ref.replace('refs/remotes/', '')}") + if fetch.returncode != 0: + print(f" fetch failed: {fetch.stderr.strip()[:120]}", file=sys.stderr) + return False + + # The merge base is the last common ancestor of the PR branch and the PR's + # actual base branch — the point where the PR diverged from master before + # the reorg commit landed. In production pr_base == BASE_BRANCH (master); + # in test mode it may be a per-run conflicting-base branch. + merge_base = git("merge-base", f"origin/{pr_base}", pr_remote_ref) + if merge_base.returncode != 0: + print(f" could not find merge base: {merge_base.stderr.strip()[:80]}", file=sys.stderr) + return False + base_sha = merge_base.stdout.strip() + + # Export each PR commit as a separate mbox-format patch. stdout gives us + # all patches concatenated; git am can consume this directly. + format_patch = git("format-patch", "--stdout", f"{base_sha}..{pr_remote_ref}") + if format_patch.returncode != 0: + print(f" format-patch failed: {format_patch.stderr.strip()[:80]}", file=sys.stderr) + return False + + patches = format_patch.stdout + if not patches.strip(): + print(" no patches between merge base and PR HEAD — nothing to apply.") + return False + + transformed = transform_diff_paths(patches) + + fix_branch = f"reorg-fix/pr-{pr_number}" + + # A prior run may have died partway through: the fix branch was pushed and/or + # the fix PR was opened, but the original PR was never closed. Detect that so + # we FINISH the interrupted job rather than bail to manual review — which + # would post a misleading "resolve it yourself" comment on a PR that already + # has a working fix. Both checks are read-only, so they're safe in dry-run. + existing_fix_pr = find_open_fix_pr(fix_branch) + branch_pushed = existing_fix_pr is not None or fix_branch_on_origin(fix_branch) + + if dry_run: + if existing_fix_pr: + print(f" [dry-run] fix PR #{existing_fix_pr['number']} already exists — " + f"would finish the interrupted auto-fix: label it, close " + f"#{pr_number} pointing to it, and label it {LABEL_AUTOFIXED!r}.") + return True + if branch_pushed: + print(f" [dry-run] fix branch {fix_branch!r} exists on origin with no open " + f"PR — would open a PR from it and finish the auto-fix.") + return True + # Count patches and show per-patch file summaries. + subjects = [l[len("Subject: "):] for l in transformed.splitlines() + if l.startswith("Subject: ")] + print(f" [dry-run] would apply {len(subjects)} commit(s) to {fix_branch}:") + for s in subjects[:10]: + print(f" {s}") + if len(subjects) > 10: + print(f" ... and {len(subjects) - 10} more") + would_be_title = f"[reorg fix] {pr['title']}" + print(f" [dry-run] would open PR: {would_be_title!r}") + print(f" [dry-run] would label fix PR {LABEL_WIP!r}, {LABEL_AUTO_PR!r}") + if IS_TEST_MODE: + print(f" [dry-run] would label fix PR {LABEL_DO_NOT_MERGE!r} (test mode)") + print(f" [dry-run] would close PR #{pr_number} with comment pointing to fix PR, " + f"and label it {LABEL_AUTOFIXED!r}") + return True + + # Finish an interrupted run without rebuilding (and never force-pushing) the + # branch, so a fix already in review is never clobbered. + if existing_fix_pr: + print(f" Fix PR #{existing_fix_pr['number']} already exists — " + f"finishing the interrupted auto-fix.") + return finalize_autofix(pr, fix_branch, existing_fix_pr) + if branch_pushed: + print(f" Fix branch {fix_branch!r} exists on origin but has no open PR — " + f"opening it and finishing the interrupted auto-fix.") + return finalize_autofix(pr, fix_branch, None) + + tmpdir = tempfile.mkdtemp(prefix=f"reorg_fix_{pr_number}_") + try: + # A stale LOCAL branch can linger when a prior run's worktree was cleaned + # up but its branch wasn't (worktree removal doesn't delete the branch). + # We've already confirmed there's no fix PR and no origin branch, so any + # local reorg-fix/pr- is a safe-to-drop leftover. Prune first so the + # delete isn't blocked by a registration for an already-deleted worktree. + git("worktree", "prune") + git("branch", "-D", fix_branch) # no-op if it doesn't exist + + add_wt = git("worktree", "add", "-b", fix_branch, tmpdir, f"origin/{pr_base}") + if add_wt.returncode != 0: + # Still couldn't create the branch (e.g. it's checked out in a + # lingering worktree we couldn't prune). Bail to manual review rather + # than risk clobbering anything. + print(f" could not create fix branch {fix_branch!r} " + f"— leaving for manual review: " + f"{add_wt.stderr.strip()[:120]}", file=sys.stderr) + return False + worktree = Path(tmpdir) + + # git am replays each patch as its own commit, preserving the original + # author and message. --3way enables per-patch 3-way merging so that + # patches whose context drifted (because master made unrelated edits + # between the PR base and now) still apply cleanly. + am = run(["git", "am", "--3way"], cwd=worktree, input=transformed) + if am.returncode != 0: + print(f" git am failed:\n{am.stderr[:400]}", file=sys.stderr) + run(["git", "am", "--abort"], cwd=worktree) + return False + + push = run(["git", "push", "origin", fix_branch], cwd=worktree) + if push.returncode != 0: + # The branch appeared on origin since our earlier check (a concurrent + # run, most likely). We don't force-push; finish via whatever is + # already there rather than clobber it. + print(f" push failed (fix branch appeared on origin): " + f"{push.stderr.strip()[:120]} — finishing via the existing branch.", + file=sys.stderr) + return finalize_autofix(pr, fix_branch, find_open_fix_pr(fix_branch)) + + print(f" Pushed fix to branch {fix_branch!r}") + return finalize_autofix(pr, fix_branch, None) + + finally: + git("worktree", "remove", "--force", tmpdir) + shutil.rmtree(tmpdir, ignore_errors=True) + + +# --------------------------------------------------------------------------- +# Per-PR analysis +# --------------------------------------------------------------------------- + +def analyze_pr(pr: dict, dry_run: bool) -> bool: + """Process one PR. Return True if we acted on it (auto-fixed or labeled in + a way that counts toward --limit), False otherwise (mergeability not yet + computed, no conflicts found locally, or MERGEABLE — which gets a label but + doesn't consume --limit). The return value drives --limit. + """ + pr_number = pr["number"] + title = pr["title"] + mergeable = pr.get("mergeable", "UNKNOWN") + + print(f"\nPR #{pr_number}: {title}") + print(f" mergeable: {mergeable}") + + if mergeable == "MERGEABLE": + # No conflicts — label so future runs exclude it from the query. This + # lets us ignore PRs that land after the reorg reaches master. Labeling + # doesn't count as "acting" on the PR, so it never consumes --limit. + print(f" No conflicts — labeling {LABEL_NO_CONFLICTS!r} and skipping.") + add_label(pr_number, LABEL_NO_CONFLICTS, dry_run) + return False + + if mergeable == "UNKNOWN": + # GitHub computes mergeability lazily; try again later if needed. + print(" Mergeability not yet computed by GitHub — skipping.") + return False + + # CONFLICTING: fetch the branch and do a local merge test to classify + # conflicts as reorg-caused vs unrelated. + pr_ref = f"refs/remotes/origin/{pr['headRefName']}" + fetch = git("fetch", "origin", + f"refs/pull/{pr_number}/head:{pr_ref.replace('refs/remotes/', '')}") + if fetch.returncode != 0: + print(f" fetch failed: {fetch.stderr.strip()[:120]}", file=sys.stderr) + return False + + # Use the PR's actual base branch for the merge test so the local result + # matches what GitHub computes. In production this is always BASE_BRANCH + # (master); in test mode the conflicting-base branch may differ because + # create_test_prs.py builds a per-run branch with extra base_edits baked in. + pr_base = pr["baseRefName"] + if pr_base != BASE_BRANCH: + fetch_base = git("fetch", "origin", pr_base) + if fetch_base.returncode != 0: + print(f" fetch of PR base {pr_base!r} failed: " + f"{fetch_base.stderr.strip()[:120]}", file=sys.stderr) + return False + + tmpdir = tempfile.mkdtemp(prefix=f"reorg_check_{pr_number}_") + try: + add_wt = git("worktree", "add", "--detach", tmpdir, f"origin/{pr_base}") + if add_wt.returncode != 0: + print(f" worktree add failed: {add_wt.stderr.strip()[:120]}", file=sys.stderr) + return False + worktree = Path(tmpdir) + + # Attempt the merge without committing so we can inspect the conflicts. + # We do NOT use --no-ff here; the default is fine since we're only + # inspecting, not keeping the result. + # + # merge.renames=false disables git's rename detection for this test. The + # reorg only MOVES files (content/foo -> hugo/content/foo); it never + # edits their content. With rename detection ON (git's default), git + # pairs a PR's edit at the old path with the moved file on the base and + # merges cleanly, hiding the conflict entirely — so the script would see + # nothing to fix. GitHub's mergeability check does no rename detection + # and reports these as CONFLICTING (modify/delete). Disabling it here + # makes the local test match what GitHub sees: the moved file surfaces + # as a conflict at its pre-reorg path, which is_reorg_path() recognizes. + merge = run(["git", "-c", "merge.renames=false", "merge", "--no-commit", pr_ref], cwd=worktree) + + # Classify any files that have conflict markers. + reorg_conflicts, other_conflicts = get_conflict_classification(worktree) + + # Even if the merge succeeded cleanly (returncode 0), the PR may have + # added files at pre-reorg paths with no conflict marker. Check for + # these "wrong path additions" in the staged result. + wrong_additions: list[str] = [] + if merge.returncode == 0: + wrong_additions = get_wrong_path_additions(worktree) + # Treat wrong-path additions as reorg conflicts: the PR added a + # file at a pre-reorg path that should be under hugo/. + reorg_conflicts.extend(wrong_additions) + + # Always abort the test merge before leaving the worktree. + run(["git", "merge", "--abort"], cwd=worktree) + + print(f" Reorg-caused conflicts : {reorg_conflicts or 'none'}") + print(f" Unrelated conflicts : {other_conflicts or 'none'}") + if wrong_additions: + print(f" Wrong-path additions : {wrong_additions}") + + if not reorg_conflicts and not other_conflicts: + if merge.returncode != 0: + # The merge failed but we couldn't classify any conflicted path + # (an unusual conflict type we don't parse). Don't guess that + # it's reorg-caused — flag it for a human rather than skip a + # real conflict. + print(" Merge failed but no conflicts could be classified " + "— labeling for manual review.") + send_to_manual_review(pr_number, dry_run) + return True + print(" No conflicts found locally (GitHub mergeability may be stale).") + return False + + if other_conflicts: + # The PR has conflicts that are NOT from the reorg. We must not + # touch it — just label it so a human can resolve it manually. + print(" Non-reorg conflicts present — labeling for manual review.") + send_to_manual_review(pr_number, dry_run) + return True + + # All conflicts are reorg-caused. Before attempting an auto-fix, skip + # PRs that aren't ready: WIP and stale PRs both get a comment pointing + # the author to astro-reorg-help-requested when they want manual help. + is_wip = any(l["name"] == LABEL_WIP for l in pr.get("labels", [])) + if is_wip: + print(" PR is marked WIP — skipping auto-fix, commenting and labeling.") + # Comment once, then apply the skip label. The label is excluded from + # the query, so later runs won't pick this PR up and re-comment. + post_comment(pr_number, build_wip_comment(), dry_run) + add_label(pr_number, LABEL_SKIP, dry_run) + return True + + # Stale handling. The stale label is both the durable "we deemed this + # stale" marker and the guard against re-commenting. We can't use + # is_pr_stale() to decide whether a *labeled* PR has been reactivated: + # our own label + comment bump updatedAt, so the PR would read as active + # on the very next run. Instead we compare updatedAt against when we + # applied the label — only activity after that (author pushes, comments, + # or removing the label) counts as genuine reactivation. + already_stale = any(l["name"] == LABEL_STALE for l in pr.get("labels", [])) + if already_stale: + labeled_at = stale_labeled_at(pr_number) + updated_at = _parse_ts(pr["updatedAt"]) if pr.get("updatedAt") else None + reactivated = ( + labeled_at is not None and updated_at is not None + and updated_at > labeled_at + STALE_REACTIVATE_GRACE + ) + if not reactivated: + print(" Already labeled stale, no activity since — " + "skipping without re-commenting.") + return True + print(" Reactivated since we labeled it stale — removing stale label.") + remove_label(pr_number, LABEL_STALE, dry_run) + # Fall through and process the PR normally. + elif is_pr_stale(pr): + print(f" PR is stale (no activity in >{STALE_DAYS} days) — " + "commenting and labeling without auto-fix.") + # Comment first, then label so the label is our last action: the + # labeled-event timestamp then matches the PR's updatedAt, giving + # the reactivation check on later runs a clean baseline. + post_comment(pr_number, build_stale_comment(), dry_run) + add_label(pr_number, LABEL_STALE, dry_run) + return True + + # All conflicts are reorg-caused. Attempt the auto-fix. + print(" All conflicts are reorg-caused — attempting auto-fix.") + success = attempt_fix(pr, dry_run) + if success: + # attempt_fix closed the PR and labeled it LABEL_AUTOFIXED. The + # caller adds LABEL_PROCESSED on top for the affected-PR filter. + return True + # Auto-fix failed (fork, apply error, etc.) — fall back to labeling. + print(" Auto-fix failed or not applicable — labeling for manual review.") + send_to_manual_review(pr_number, dry_run) + return True + + finally: + run(["git", "worktree", "remove", "--force", tmpdir]) + shutil.rmtree(tmpdir, ignore_errors=True) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def get_open_prs(only: list[int] | None = None, limit: int = 50) -> list[dict]: + """Return open PRs targeting BASE_BRANCH, optionally filtered to specific numbers. + + The --base filter is what keeps a run scoped: in test mode only PRs opened + against the mock base branch are considered, so real PRs against master are + never fetched, merged, labeled, or commented on. In live mode it scopes to + PRs against master. Without it, `gh pr list` would return every open PR in + the repo regardless of target. + """ + # Only fields actually consumed below. Note baseRefOid/headRefOid are NOT + # requested: `gh pr list` doesn't support them (only `gh pr view` does), and + # nothing here uses them. + fields = ("number,title,body,author,labels,headRefName," + "baseRefName,isCrossRepository,mergeable,updatedAt") + if only: + # Explicitly named PRs are an intentional override, but still guard + # against acting on a PR that targets a different base than this run. + prs = [] + for n in only: + pr = gh_json("pr", "view", str(n), "--repo", REPO, "--json", fields) + if pr.get("baseRefName") != BASE_BRANCH: # type: ignore[union-attr] + print(f" Skipping PR #{n}: targets " + f"{pr.get('baseRefName')!r}, not base {BASE_BRANCH!r}.", # type: ignore[union-attr] + file=sys.stderr) + continue + prs.append(pr) + return prs + return gh_json( # type: ignore[return-value] + "pr", "list", "--repo", REPO, "--state", "open", + "--base", BASE_BRANCH, + "--search", f"-label:{LABEL_NO_CONFLICTS} -label:{LABEL_MANUAL_REVIEW} " + f"-label:{LABEL_HELP_REQUESTED} -label:{LABEL_SKIP}", + "--json", fields, "--limit", str(limit), + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Check open PRs for reorg-caused merge conflicts.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "--dry-run", action=argparse.BooleanOptionalAction, default=True, + help="Report what would be done without making any changes (default: on). Use --no-dry-run to apply changes.", + ) + parser.add_argument( + "--pr", type=int, action="append", dest="prs", metavar="NUMBER", + help="Only check this PR number (may be repeated).", + ) + parser.add_argument( + "--limit", type=int, required=True, metavar="N", + help="Fetch and act on at most N PRs. PRs needing no action don't count " + "toward the limit. Use it to roll out gradually: start at 1, " + "review, then raise it.", + ) + parser.add_argument( + "--base-branch", default=None, metavar="BRANCH", + help="Branch to treat as the post-reorg base. Overrides the default " + "(the mock base branch from config.yaml) and --live.", + ) + parser.add_argument( + "--live", action="store_true", + help="Run against the real master. Without this the script defaults to " + "the mock base branch from config.yaml (the same branch " + "create_test_prs.py opens its test PRs against) for safety.", + ) + args = parser.parse_args() + + if args.limit < 1: + parser.error("--limit must be a positive integer.") + + if args.live and args.base_branch: + parser.error("--live and --base-branch are mutually exclusive; " + "--live already selects master.") + + global BASE_BRANCH, IS_TEST_MODE + if args.base_branch: + BASE_BRANCH = args.base_branch + # A custom base is a test/dev scenario unless it's the real master. + IS_TEST_MODE = args.base_branch != "master" + elif args.live: + BASE_BRANCH = "master" + print("LIVE mode — running against master.\n") + else: + # Default to the mock base branch so an accidental run can't touch + # real PRs against master. + if not TEST_BASE_BRANCH: + parser.error("no base branch: set `test.mock_reorged_master_branch` in " + "config.yaml, or pass --live or --base-branch.") + BASE_BRANCH = TEST_BASE_BRANCH + IS_TEST_MODE = True + print(f"TEST mode (default) — using mock base branch {BASE_BRANCH!r} " + f"from config.yaml. Pass --live to run against master.\n") + + if args.dry_run: + print("DRY-RUN mode — no branches or PRs will be modified.\n") + + print(f"Fetching origin/{BASE_BRANCH}...") + fetch_master = git("fetch", "origin", BASE_BRANCH) + if fetch_master.returncode != 0: + print(f"Warning: could not update {BASE_BRANCH}: {fetch_master.stderr.strip()[:80]}", + file=sys.stderr) + + ensure_label_exists(LABEL_MANUAL_REVIEW, args.dry_run) + ensure_label_exists(LABEL_STALE, args.dry_run) + ensure_label_exists(LABEL_AUTOFIXED, args.dry_run) + ensure_label_exists(LABEL_AUTO_PR, args.dry_run) + ensure_label_exists(LABEL_NO_CONFLICTS, args.dry_run) + ensure_label_exists(LABEL_SKIP, args.dry_run) + ensure_label_exists(LABEL_PROCESSED, args.dry_run) + + existing_labels = gh_json("label", "list", "--repo", REPO, "--search", LABEL_WIP, "--json", "name") + if not any(l["name"] == LABEL_WIP for l in existing_labels): # type: ignore[index] + print(f"Error: label {LABEL_WIP!r} does not exist in the repo. " + f"Check that the label name is correct.", file=sys.stderr) + sys.exit(1) + + prs = get_open_prs(args.prs, limit=args.limit) + print(f"Found {len(prs)} open PR(s) to check.") + print(f"Limit: will stop after acting on {args.limit} PR(s).") + + acted = 0 + for pr in prs: + if acted >= args.limit: + print(f"\nReached --limit of {args.limit} acted-on PR(s) — stopping.") + break + # Isolate failures: one PR raising (e.g. a transient gh/network error) + # shouldn't abort the whole batch. A half-finished fix is safe to + # retry — if the fix branch already exists, the re-run falls back to + # manual review rather than clobbering it. + try: + if analyze_pr(pr, args.dry_run): + # Mark every acted-on PR (however it was handled) so all + # reorg-affected PRs are findable with one label filter. + add_label(pr["number"], LABEL_PROCESSED, args.dry_run) + acted += 1 + except Exception as exc: + print(f"\nERROR processing PR #{pr.get('number', '?')}: {exc}", + file=sys.stderr) + print(" Skipping to the next PR.", file=sys.stderr) + + print(f"\nActed on {acted} PR(s). Done.") + print( + "\nView auto-fix PRs:\n" + " https://github.com/DataDog/documentation/pulls" + "?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-auto-pr" + ) + + +if __name__ == "__main__": + main() diff --git a/astro_reorg/validate_reorg.py b/astro_reorg/validate_reorg.py new file mode 100644 index 00000000000..3515631a258 --- /dev/null +++ b/astro_reorg/validate_reorg.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +""" +Post-reorg validation harness. + +Run this AFTER execute_reorg.py, from the repo root, on a feature branch (not master). +The harness is non-destructive: every file it creates lives under a '__reorg_harness__' +prefix, and every change (staged paths, gitignore edits) is reverted in a finally block. +It never commits. +""" + +import sys + +from helpers import ( + hugo_dir, + git, + load_config, + results, + check_layout, + check_gitignore_split, + check_workflows, + check_workflow_path_filters, + check_codeowners, + check_codeowners_prefixing, + check_husky_circular_aliases, + check_husky_section_index, + check_husky_cdocs_gitignore, + check_build_presence, + check_rollback_roundtrip, +) + + +def main(): + if not hugo_dir.exists(): + print("hugo/ does not exist — run astro_reorg/execute_reorg.py first.", file=sys.stderr) + sys.exit(1) + + branch = git("branch", "--show-current").stdout.strip() + if branch == "master": + print("Refusing to run on master; check out a feature branch.", file=sys.stderr) + sys.exit(1) + + top_level, moves_to_hugo = load_config() + + # hugo/ holds the moved dirs; nothing moved is left at root; + # the root .gitignore split routed each rule to the right side. + print("== Layout ==") + check_layout(top_level, moves_to_hugo) + check_gitignore_split(top_level, moves_to_hugo) + + # No .github/workflows/ file references a moved path without the hugo/ prefix, + # in shell scalars and in structured on.*.paths filters. + print("\n== Workflow paths ==") + check_workflows(moves_to_hugo) + check_workflow_path_filters(moves_to_hugo) + + # No pattern was moved to hugo/ while its file stayed at root, and every + # moved pattern (globs included) carries the hugo/ prefix. Pre-existing + # dangling entries are reported but not failed. + print("\n== CODEOWNERS ==") + check_codeowners() + check_codeowners_prefixing(moves_to_hugo) + + # Each pre-commit check still REJECTS a known-bad input planted at the + # hugo/ path. A hook that still points at the old path passes vacuously -> we fail it. + print("\n== Husky hooks ==") + check_husky_circular_aliases() + check_husky_section_index() + check_husky_cdocs_gitignore() + + # Static presence check only; run `make start` manually. + print("\n== Hugo build ==") + check_build_presence() + + # On a throwaway repo, execute_reorg.py then local_rollback.py must restore the + # tree byte-for-byte (the only test of rollback). + print("\n== Rollback round-trip ==") + check_rollback_roundtrip() + + # Summary + counts = {"PASS": 0, "FAIL": 0, "SKIP": 0, "WARN": 0} + for status, _, _ in results: + counts[status] += 1 + print("\n" + "=" * 50) + print(f"PASS {counts['PASS']} FAIL {counts['FAIL']} " + f"WARN {counts['WARN']} SKIP {counts['SKIP']}") + + sys.exit(1 if counts["FAIL"] else 0) + + +if __name__ == "__main__": + main()