From 80b2cbf4abf2806a72a1cae7d209b193f869e070 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 14:02:12 -0500 Subject: [PATCH 01/70] Add reorg scripts and config --- reorg.py | 61 +++++++++++++++++++++++ reorg_config.yaml | 123 ++++++++++++++++++++++++++++++++++++++++++++++ reorg_rollback.py | 23 +++++++++ 3 files changed, 207 insertions(+) create mode 100644 reorg.py create mode 100644 reorg_config.yaml create mode 100644 reorg_rollback.py diff --git a/reorg.py b/reorg.py new file mode 100644 index 00000000000..0eaa68d4fc6 --- /dev/null +++ b/reorg.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +import os +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + +repo_root = Path(__file__).parent +config_path = repo_root / "reorg_config.yaml" + +with config_path.open() as f: + config = yaml.safe_load(f) + +top_level = set(config.get("top_level", [])) +moves_to_hugo = set(config.get("moves_to_hugo", [])) +ignore = set(config.get("ignore", [])) + +# Sanity-check the config itself for conflicts. +conflicts = top_level & moves_to_hugo +if conflicts: + for name in sorted(conflicts): + print(f"ERROR: '{name}' appears in both top_level and moves_to_hugo", file=sys.stderr) + sys.exit(1) + +errors = [] + +for name in os.listdir(repo_root): + if name in ignore: + continue + + if name in top_level: + pass # keep in place + elif name in moves_to_hugo: + pass # will move below + else: + errors.append(name) + +if errors: + for name in sorted(errors): + print(f"ERROR: '{name}' is not listed in reorg_config.yaml", file=sys.stderr) + print(f"{len(errors)} error(s) found. Update reorg_config.yaml before running.", file=sys.stderr) + sys.exit(1) + +hugo_dir = repo_root / "hugo" +hugo_dir.mkdir(exist_ok=True) + +moved = 0 +for name in sorted(os.listdir(repo_root)): + if name in ignore or name in top_level or name == "hugo": + continue + src = repo_root / name + dst = hugo_dir / name + print(f"Moving {name} -> hugo/{name}") + src.rename(dst) + moved += 1 + +print(f"Done. {moved} item(s) moved into hugo/.") diff --git a/reorg_config.yaml b/reorg_config.yaml new file mode 100644 index 00000000000..befcf6030a1 --- /dev/null +++ b/reorg_config.yaml @@ -0,0 +1,123 @@ +# Files and folders that stay at the top level (not moved into hugo/). +# Everything else moves into hugo/. + +top_level: + # Sites + - astro + + # Repo scripts + - reorg.py + - reorg_rollback.py + - reorg_config.yaml + + # Created by reorg.py as the move target + - hugo + + # Git / CI / repo-wide tooling + - .git + - .github + - .gitlab-ci.yml + - .gitignore + - .gitattributes + - .husky + - .claude + + # Editor / IDE config + - .editorconfig + - .vscode + + # Repo docs + - README.md + - LICENSE + - CONTRIBUTING.md + - CLAUDE.md + + # Node / Yarn (shared by both sites) + - package.json + - package-lock.json + - yarn.lock + - .yarnrc.yml + - .yarn + - node_modules + - .nvmrc + + # Linting / formatting (shared) + - .eslintrc + - .eslintignore + - .vale.ini + - .rubocop.yml + - .prettierignore + - prettier.config.js + - static-analysis.datadog.yml + + # Datadog CI / synthetics + - datadog-ci.preview.json + - general.preview.synthetics.json + - repository.datadog.yml + + # Docker + - docker-compose-docs.yml + + # Go module (top-level) + - go.mod + - go.sum + +moves_to_hugo: + # Hugo core + - archetypes + - assets + - config + - content + - data + - i18n + - layouts + - static + - resources + - public + + # Hugo build tooling + - Makefile + - Makefile.config + - Makefile.config.example + - local + - hugpython + - gradle + - _vendor + + # Hugo build artifacts / state + - .hugo_build.lock + - logs + - integrations_data + + # Node / JS (Hugo-specific) + - babel.config.js + - jest.config.js + - postcss.config.js + - markdoc.config.json + - customization_config + + # Testing (Hugo-specific) + - playwright.config.ts + - playwright-report + - test-results + - e2e + + # Content tooling + - translate.yaml + - .translate + - typesense.config.json + - .htmltest.yml + - .apigentools-info + - agent_config_types_list.txt + + # Examples / docs + - examples + - docs + - plans + + # Static assets (Hugo) + - usage-notifications-email.png + +ignore: + # macOS metadata + - .DS_Store diff --git a/reorg_rollback.py b/reorg_rollback.py new file mode 100644 index 00000000000..050433f7e1d --- /dev/null +++ b/reorg_rollback.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import os +import sys +from pathlib import Path + +repo_root = Path(__file__).parent +hugo_dir = repo_root / "hugo" + +if not hugo_dir.exists(): + print("Nothing to roll back: hugo/ does not exist.", file=sys.stderr) + sys.exit(1) + +moved = 0 +for name in sorted(os.listdir(hugo_dir)): + src = hugo_dir / name + dst = repo_root / name + print(f"Moving hugo/{name} -> {name}") + src.rename(dst) + moved += 1 + +# Only succeeds if hugo/ is now empty, preventing accidental data loss. +hugo_dir.rmdir() +print(f"Done. {moved} item(s) restored to repo root.") From fb3d0a6797a98919da5092a2f6f9af2fd5f208b5 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 16:07:28 -0500 Subject: [PATCH 02/70] Review code and fill gaps --- reorg.py | 206 ++++++++++++ reorg_config.yaml | 30 +- reorg_context.md | 11 + reorg_harness.py | 837 ++++++++++++++++++++++++++++++++++++++++++++++ reorg_rollback.py | 25 ++ 5 files changed, 1096 insertions(+), 13 deletions(-) create mode 100644 reorg_context.md create mode 100644 reorg_harness.py diff --git a/reorg.py b/reorg.py index 0eaa68d4fc6..f50390583c2 100644 --- a/reorg.py +++ b/reorg.py @@ -59,3 +59,209 @@ moved += 1 print(f"Done. {moved} item(s) moved into hugo/.") + +# Split .gitignore between the root and hugo/ instead of copying it wholesale. +# +# A .gitignore is interpreted RELATIVE TO ITS OWN DIRECTORY, so — unlike +# CODEOWNERS — a moved rule needs NO "hugo/" prefix; it simply belongs in +# hugo/.gitignore. We therefore only ROUTE each line, never rewrite it, by its +# FIRST PATH SEGMENT (driven entirely by reorg_config.yaml, same as CODEOWNERS): +# first segment in moves_to_hugo -> hugo/.gitignore only +# first segment in top_level -> root .gitignore only +# first segment in neither -> generic/pure glob, kept in BOTH +# Comments and blank lines are kept in both to preserve section readability. +def route_gitignore_segment(line): + """Return the first path segment of a .gitignore pattern, or None for a + comment/blank line. A leading '!' (negation) and '/' (root anchor) are + stripped before taking segment 0; the line text itself is never altered. + """ + stripped = line.strip() + if not stripped or stripped.startswith("#"): + return None + body = stripped[1:] if stripped.startswith("!") else stripped # un-negate + body = body.lstrip("/") # un-anchor + return body.split("/", 1)[0] + + +print("\nSplitting .gitignore between root and hugo/...") +gitignore = repo_root / ".gitignore" +gi_lines = gitignore.read_text().splitlines(keepends=True) + +root_lines = [] +hugo_lines = [] +hugo_only_segments = set() # routed off root into hugo/ (for the summary) +both_segments = set() # in neither config list -> kept in both (surfaced) + +for raw in gi_lines: + segment = route_gitignore_segment(raw) + if segment is None: # comment / blank -> keep in both + root_lines.append(raw) + hugo_lines.append(raw) + elif segment in moves_to_hugo: + hugo_lines.append(raw) + hugo_only_segments.add(segment) + elif segment in top_level: + root_lines.append(raw) + else: + root_lines.append(raw) + hugo_lines.append(raw) + both_segments.add(segment) + +print(f"\n .gitignore: {len(hugo_only_segments)} segment(s) -> hugo/.gitignore only, " + f"{len(both_segments)} generic kept in both") +print(f" root: {len(root_lines)} line(s), hugo/: {len(hugo_lines)} line(s) " + f"(was {len(gi_lines)} duplicated wholesale)") +answer = input(" Apply? [y/N] ").strip().lower() +if answer == "y": + gitignore.write_text("".join(root_lines)) + (hugo_dir / ".gitignore").write_text("".join(hugo_lines)) + print(" Written: .gitignore (root, pruned) and hugo/.gitignore") + if both_segments: + print(" NOTE: kept in both (first path segment not in reorg_config.yaml): " + + ", ".join(sorted(both_segments))) + +# Update .github/workflows/ files to reference paths under hugo/. +WORKFLOW_SUBSTITUTIONS = [ + ("- 'content/en/", "- 'hugo/content/en/"), + ("- 'layouts/shortcodes/", "- 'hugo/layouts/shortcodes/"), + ("- 'static/images/", "- 'hugo/static/images/"), + ("python local/bin/", "python hugo/local/bin/"), + # vale_linter.yml passes the template path to vale via --output= + ("--output=local/bin/", "--output=hugo/local/bin/"), + (" static |", " hugo/static |"), + ("^static/images/", "^hugo/static/images/"), + ("-- 'content/en/**/*.md'", "-- 'hugo/content/en/**/*.md'"), + # bump_* and version_getter_shared workflows cp/commit data files by ./ path + ("./data/", "./hugo/data/"), +] + +print("\nUpdating .github/workflows/...") +workflows_dir = repo_root / ".github" / "workflows" +for yml_file in sorted(workflows_dir.glob("*.yml")): + original = yml_file.read_text() + updated = original + for old, new in WORKFLOW_SUBSTITUTIONS: + if old not in updated: + continue + print(f"\n {yml_file.name}: {old!r} → {new!r}") + answer = input(" Apply? [y/N] ").strip().lower() + if answer == "y": + updated = updated.replace(old, new) + if updated != original: + yml_file.write_text(updated) + print(f" Written: {yml_file.name}") + +# Update .husky/ hook scripts to reference paths under hugo/content/. +HUSKY_SUBSTITUTIONS = [ + ("content/en/{dir_name}/{name}", "hugo/content/en/{dir_name}/{name}"), + ("content/en/{dir_name}/", "hugo/content/en/{dir_name}/"), + ("Path('content/en')", "Path('hugo/content/en')"), + ('Path("content/.gitignore")', 'Path("hugo/content/.gitignore")'), + ("f.startswith('content/en/')", "f.startswith('hugo/content/en/')"), + (".replace('content/en/', '')", ".replace('hugo/content/en/', '')"), + ("repo_root / 'content' / 'en'", "repo_root / 'hugo' / 'content' / 'en'"), + ('repo_pattern = "content"', 'repo_pattern = "hugo/content"'), +] + +print("\nUpdating .husky/...") +husky_dir = repo_root / ".husky" +for py_file in sorted(husky_dir.glob("*.py")): + original = py_file.read_text() + updated = original + for old, new in HUSKY_SUBSTITUTIONS: + if old not in updated: + continue + print(f"\n {py_file.name}: {old!r} → {new!r}") + answer = input(" Apply? [y/N] ").strip().lower() + if answer == "y": + updated = updated.replace(old, new) + if updated != original: + py_file.write_text(updated) + print(f" Written: {py_file.name}") + +# Update .github/CODEOWNERS to reference paths under hugo/. +# +# CODEOWNERS is not YAML; it is a line-based format where each rule is +# " ". Rather than blanket str.replace() over the whole +# file (which let a short pattern like "config/" corrupt a longer one like +# "customization_config/"), we parse each line and route its pattern by its +# FIRST PATH SEGMENT. The decision is made on a whole segment, never a raw +# substring, so "config" can never match inside "customization_config" and the +# substitution list no longer has to be hand-ordered or kept in sync with the +# move list — it is derived entirely from reorg_config.yaml. +def route_codeowners_pattern(pattern): + """Rewrite one CODEOWNERS pattern for the hugo/ move. + + Returns (segment, new_pattern). new_pattern is None when the line is left + untouched; segment is None for the global '*' owner. A leading '/' + (repo-root anchor) is preserved; owners are never touched because we only + ever rewrite the pattern token. + """ + if pattern == "*": + return None, None + + anchored = pattern.startswith("/") # leading '/' = repo-root-anchored + body = pattern[1:] if anchored else pattern + segment = body.split("/", 1)[0] + + # Carried over from the original script: ".local/" is a misspelling of + # "local/" (which moves into hugo/). Normalize the segment before routing + # so the typo'd entry is repathed alongside the real one. + if segment == ".local": + body = "local" + body[len(".local"):] + segment = "local" + + if segment not in moves_to_hugo: + return segment, None + + new_body = "hugo/" + body + return segment, (("/" + new_body) if anchored else new_body) + + +print("\nUpdating .github/CODEOWNERS...") +codeowners = repo_root / ".github" / "CODEOWNERS" +lines = codeowners.read_text().splitlines(keepends=True) + +# Group rewritable lines by first path segment so we prompt once per segment +# (one y/N covers every "content/..." rule) instead of once per line. +changes_by_segment = {} # segment -> list of (line_index, old_pattern, new_line) +left_alone = set() # first segments in neither config list (surfaced below) + +for i, raw in enumerate(lines): + newline = "\n" if raw.endswith("\n") else "" + line = raw[:-len(newline)] if newline else raw + stripped = line.lstrip() + if not stripped or stripped.startswith("#"): + continue + indent = line[:len(line) - len(stripped)] + pattern = stripped.split(maxsplit=1)[0] + rest = stripped[len(pattern):] # original spacing + owners, verbatim + segment, new_pattern = route_codeowners_pattern(pattern) + if new_pattern is None: + if segment is not None and segment not in top_level: + left_alone.add(segment) + continue + changes_by_segment.setdefault(segment, []).append( + (i, pattern, indent + new_pattern + rest + newline) + ) + +applied = False +for segment in sorted(changes_by_segment): + entries = changes_by_segment[segment] + old_pattern = entries[0][1] + new_pattern = entries[0][2].lstrip().split(maxsplit=1)[0] + print(f"\n CODEOWNERS: prefix {len(entries)} pattern(s) under {segment!r} " + f"with hugo/ (e.g. {old_pattern!r} → {new_pattern!r})") + answer = input(" Apply? [y/N] ").strip().lower() + if answer == "y": + for idx, _, new_line in entries: + lines[idx] = new_line + applied = True + +if left_alone: + print("\n NOTE: left unchanged (first path segment not in reorg_config.yaml): " + + ", ".join(sorted(left_alone))) + +if applied: + codeowners.write_text("".join(lines)) + print(" Written: CODEOWNERS") diff --git a/reorg_config.yaml b/reorg_config.yaml index befcf6030a1..83f5d0eb6af 100644 --- a/reorg_config.yaml +++ b/reorg_config.yaml @@ -9,6 +9,9 @@ top_level: - reorg.py - reorg_rollback.py - reorg_config.yaml + - reorg_harness.py + - reorg_context.md + - reorg_issues.md # Created by reorg.py as the move target - hugo @@ -32,15 +35,6 @@ top_level: - CONTRIBUTING.md - CLAUDE.md - # Node / Yarn (shared by both sites) - - package.json - - package-lock.json - - yarn.lock - - .yarnrc.yml - - .yarn - - node_modules - - .nvmrc - # Linting / formatting (shared) - .eslintrc - .eslintignore @@ -58,10 +52,6 @@ top_level: # Docker - docker-compose-docs.yml - # Go module (top-level) - - go.mod - - go.sum - moves_to_hugo: # Hugo core - archetypes @@ -96,6 +86,20 @@ moves_to_hugo: - markdoc.config.json - customization_config + # Go module — powers Hugo Modules (imports websites-modules, vendors into + # _vendor/). Hugo requires go.mod in the project root; Astro uses no Go. + - go.mod + - go.sum + + # Node / Yarn (Hugo site owns these; Astro has its own under astro/) + - package.json + - package-lock.json + - yarn.lock + - .yarnrc.yml + - .yarn + - node_modules + - .nvmrc + # Testing (Hugo-specific) - playwright.config.ts - playwright-report diff --git a/reorg_context.md b/reorg_context.md new file mode 100644 index 00000000000..55c39a900e5 --- /dev/null +++ b/reorg_context.md @@ -0,0 +1,11 @@ +# Docs repository reorg context + +This repo is currently a Hugo site. We instead want it to contain a `hugo` and `astro` site side by side, with no overlap in their envs, `package.json` files, etc. + +The `reorg_config.yaml` describes the relocation target for every file and folder at the top level of the repo. + +`reorg.py` implements the file and folder path changes, and updates any dependencies on those paths, such as GitHub actions, CODEOWNERS, and Husky workflows. + +`reorg_rollback.py` functions as an "undo" action for the reorg. + +`reorg_harness.py` verifies the functionality of affected entities where possible. \ No newline at end of file diff --git a/reorg_harness.py b/reorg_harness.py new file mode 100644 index 00000000000..c2e908bd9df --- /dev/null +++ b/reorg_harness.py @@ -0,0 +1,837 @@ +#!/usr/bin/env python3 +""" +Post-reorg validation harness. + +Run this AFTER reorg.py, from the repo root, on a feature branch (not master). +It verifies that the things the reorg could silently break still work: + + A. Layout - hugo/ holds the moved dirs; nothing moved is left at root; + hugo/ and astro/ each self-own a package.json and no Hugo + Node/build file lingers at the root competing with Astro. + B. Workflows - no .github/workflows/ file references a moved path (dir OR + file) without the hugo/ prefix, in shell scalars and in + structured on.*.paths filters (catches gaps in reorg.py). + C. CODEOWNERS - every concrete path pattern still resolves on disk, and + every moved pattern (globs included) carries the hugo/ prefix. + D. Husky hooks - each pre-commit check still REJECTS a known-bad input planted + at the new hugo/ path. If a hook wasn't repathed it inspects + the now-missing content/en/ and passes vacuously -> we fail it. + E. Vale - vale still flags a known violation using the Datadog style, + proving StylesPath resolves from the new content location. + F. Hugo build - static presence check only; run `make start` manually (todo #5). + G. Rollback - on a throwaway repo, reorg.py then reorg_rollback.py must + restore the tree byte-for-byte (the only test of rollback). + +The harness is non-destructive. Every file it creates lives under a +'__reorg_harness__' prefix, and every change (staged paths, gitignore edits) is +reverted in a finally block. It never commits. +""" + +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +try: + import yaml +except ImportError: + print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + +repo_root = Path(__file__).parent +hugo_dir = repo_root / "hugo" + +# Distinctive marker so anything we create is obvious and easy to clean up. +MARKER = "__reorg_harness__" + +# (status, name, detail) tuples collected as checks run. +results = [] + + +def record(status, name, detail=""): + """Record a check outcome. status is one of PASS, FAIL, SKIP, WARN.""" + results.append((status, name, detail)) + symbol = {"PASS": "✅", "FAIL": "❌", "SKIP": "⏭ ", "WARN": "⚠ "}[status] + line = f"{symbol} {status:4} {name}" + if detail: + line += f"\n {detail}" + print(line) + + +def git(*args, check=False): + """Run a git command from the repo root and return the CompletedProcess.""" + return subprocess.run( + ["git", *args], + cwd=repo_root, + capture_output=True, + text=True, + check=check, + ) + + +def load_config(): + """Load reorg_config.yaml and return (top_level, moves_to_hugo) name sets.""" + config_path = repo_root / "reorg_config.yaml" + with config_path.open() as f: + config = yaml.safe_load(f) + return set(config.get("top_level", [])), set(config.get("moves_to_hugo", [])) + + +# -------------------------------------------------------------------------- +# A. Layout +# -------------------------------------------------------------------------- + +def check_layout(top_level, moves_to_hugo): + """hugo/ holds the moved dirs; nothing that moved is left at the root.""" + # Moved items that the reorg actually had to relocate (present before). + expected_in_hugo = sorted(n for n in moves_to_hugo if (hugo_dir / n).exists()) + missing_from_hugo = sorted( + n for n in moves_to_hugo + if (repo_root / n).exists() and not (hugo_dir / n).exists() + ) + + # A moved name still sitting at the root means the move didn't happen. + still_at_root = sorted( + n for n in moves_to_hugo if (repo_root / n).exists() + ) + if still_at_root: + record("FAIL", "layout: moved items remain at repo root", + ", ".join(still_at_root)) + else: + record("PASS", "layout: no moved items remain at repo root", + f"{len(expected_in_hugo)} item(s) confirmed under hugo/") + + if missing_from_hugo: + record("FAIL", "layout: moved items missing under hugo/", + ", ".join(missing_from_hugo)) + + # No top_level item should have leaked into hugo/. 'hugo' itself is listed + # in top_level (it IS the move target) so it is excluded. + leaked = sorted( + n for n in top_level + if n != "hugo" and (hugo_dir / n).exists() + ) + if leaked: + record("FAIL", "layout: top-level items leaked into hugo/", + ", ".join(leaked)) + else: + record("PASS", "layout: no top-level items leaked into hugo/") + + # Critical anchors that must still exist at the root after the move. + # NB: package.json/node_modules/yarn.lock and go.mod/go.sum now move into + # hugo/ (each site owns its own Node setup; the Go module powers Hugo + # Modules), so they are deliberately NOT listed here. + must_stay = ["astro", ".husky", ".github", ".vale.ini"] + absent = [n for n in must_stay if not (repo_root / n).exists()] + if absent: + record("FAIL", "layout: expected top-level items are missing", + ", ".join(absent)) + else: + record("PASS", "layout: top-level anchors intact", + ", ".join(must_stay)) + + # Both .gitignore files should exist (reorg.py splits one into two; + # check_gitignore_split below verifies the split routed correctly). + if (repo_root / ".gitignore").exists() and (hugo_dir / ".gitignore").exists(): + record("PASS", "layout: .gitignore present at root and in hugo/") + else: + record("FAIL", "layout: missing a .gitignore copy", + "expected both ./.gitignore and ./hugo/.gitignore") + + +def check_gitignore_split(top_level, moves_to_hugo): + """Verify reorg.py routed .gitignore rules to the correct side. + + A .gitignore is relative to its own directory, so after the split no + surviving rule should point at a path that lives on the OTHER side: + - root .gitignore must not carry a rule whose first segment moved to hugo/ + - hugo/.gitignore must not carry a rule whose first segment stays at root + Generic globs (first segment in neither config list) legitimately live in + both files, so they are ignored here. + """ + def first_segment(line): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + return None + body = stripped[1:] if stripped.startswith("!") else stripped + return body.lstrip("/").split("/", 1)[0] + + def leaks(path, wrong_side): + out = [] + for raw in path.read_text().splitlines(): + seg = first_segment(raw) + if seg in wrong_side: + out.append(f"{path.name}: {raw.strip()} (segment {seg!r})") + return out + + root_gi = repo_root / ".gitignore" + hugo_gi = hugo_dir / ".gitignore" + if not (root_gi.exists() and hugo_gi.exists()): + record("SKIP", "gitignore: split not verifiable (a .gitignore is missing)") + return + + problems = leaks(root_gi, moves_to_hugo) + leaks(hugo_gi, top_level - {"hugo"}) + if problems: + record("FAIL", "gitignore: rules survive on the wrong side of the split", + f"{len(problems)}:\n " + "\n ".join(problems[:20])) + else: + record("PASS", "gitignore: no moved-path rule left at root, " + "no root-path rule left in hugo/") + + +def check_site_separation(moves_to_hugo): + """The two sites must own non-overlapping Node/build setups. + + The point of the reorg is a clean hugo/ vs astro/ split: each site has its + OWN Node manifest, and any name the two sites share must not also linger at + the root where their copies would collide. check_layout already fails on ANY + moved item left at root (against the full config), so this check adds only + what that doesn't cover — and derives its clash set from the config rather + than a hand-maintained list: + - astro/package.json and hugo/package.json both exist (each self-owns) + - no name that moved into hugo/ AND is also owned by astro/ remains at root + """ + problems = [] + + # Each site self-owns its Node manifest. (package.json is in moves_to_hugo, + # so it belongs to Hugo; astro/ must carry its own copy.) + for site in ("hugo", "astro"): + if not (repo_root / site / "package.json").exists(): + problems.append(f"missing {site}/package.json") + + # A name that moved into hugo/ AND that astro/ also owns is a shared + # toolchain file (package.json, node_modules, yarn.lock, ...). If such a + # name is ALSO present at the root, the two sites' copies collide. The clash + # set is derived from moves_to_hugo intersected with astro/'s actual + # contents — no hand-maintained file list to drift from the config. + shared_with_astro = sorted( + n for n in moves_to_hugo if (repo_root / "astro" / n).exists() + ) + leaked = [n for n in shared_with_astro if (repo_root / n).exists()] + if leaked: + problems.append("left at root, colliding with astro/: " + ", ".join(leaked)) + + if problems: + record("FAIL", "separation: hugo/ and astro/ Node setups overlap or leak", + "; ".join(problems)) + else: + record("PASS", "separation: hugo/ and astro/ each self-own package.json; " + "no shared toolchain file left at root") + + +# -------------------------------------------------------------------------- +# B. Workflow paths +# -------------------------------------------------------------------------- + +def check_workflows(moves_to_hugo): + """Flag references to moved paths (directories AND files) lacking hugo/. + + This previously scanned only directory names ('.' not in n) using a + trailing-slash token, so a workflow that referenced a moved *file* — + babel.config.js, markdoc.config.json, the Makefile, go.mod — was never + validated. We now route every path-like token by its first segment exactly + as reorg.py does: a token whose first segment moved into hugo/ must be + hugo/-prefixed (after a correct reorg its first segment is 'hugo', so it no + longer matches). A token counts as a path reference when it contains a '/', + or when the whole token is the exact name of a moved file — so a bare word + like 'content' in prose isn't flagged, but a standalone 'babel.config.js' is. + + A moved file whose name ALSO exists under astro/ (package.json, yarn.lock, + node_modules, .nvmrc) is left out of the bare-name match: a standalone + 'package.json' is genuinely ambiguous (it may be Astro's), which is exactly + why reorg.py never blind-substitutes those names. They are still validated + when they appear inside a slashed path routed by its first segment. + """ + workflows_dir = repo_root / ".github" / "workflows" + if not workflows_dir.exists(): + record("SKIP", "workflows: .github/workflows/ not found") + return + + moved_files = { + n for n in moves_to_hugo + if (hugo_dir / n).is_file() and not (repo_root / "astro" / n).exists() + } + + # Lines that legitimately reference a moved name but are NOT paths to fix + # (illustrative prose, external URLs). Keyed by workflow filename; a line is + # exempt if it contains any of the listed markers. + allowlist = { + # Security-doc example of how untrusted paths are reported, not a real path. + "claude_review.yml": ["__untrusted/content/"], + } + + # A path-like token: a maximal run of path/glob characters. Whitespace, + # quotes, ':', '@' and '!' all act as boundaries, so a leading '!' negation + # or surrounding quotes fall away naturally. + token_re = re.compile(r"[A-Za-z0-9_.\-*/]+") + + def first_segment(token): + """(first path segment, ./-stripped token) for routing.""" + t = token + while t.startswith("./"): + t = t[2:] + return t.split("/", 1)[0], t + + suspects = [] + for yml in sorted(workflows_dir.glob("*.yml")): + exempt = allowlist.get(yml.name, []) + for lineno, line in enumerate(yml.read_text().splitlines(), 1): + if any(marker in line for marker in exempt): + continue + for token in token_re.findall(line): + seg, normalized = first_segment(token) + # Only treat a token as a path reference if it's an actual path + # (has a '/') or is exactly the name of a moved file. + if "/" not in normalized and normalized not in moved_files: + continue + if seg in moves_to_hugo: + suspects.append(f"{yml.name}:{lineno}: {line.strip()}") + break + + # De-duplicate while keeping order. + seen = set() + unique = [s for s in suspects if not (s in seen or seen.add(s))] + + if unique: + record("FAIL", "workflows: unprefixed moved paths found", + f"{len(unique)} line(s):\n " + "\n ".join(unique[:20])) + else: + record("PASS", "workflows: all moved paths (dirs and files) are hugo/-prefixed") + + +def check_workflow_path_filters(moves_to_hugo): + """Parse each workflow's on.*.paths filters and assert moved paths are prefixed. + + Unlike paths embedded in `run:` shell (which are opaque string scalars to a + parser), `on..paths` / `paths-ignore` are structured YAML list values, + so we can validate them precisely. Each entry is routed by its first path + segment: if that segment moved into hugo/, the entry must be hugo/-prefixed. + + Footgun handled: under YAML 1.1, PyYAML loads the `on:` key as the boolean + True, not the string "on" — so we look it up under both keys. + """ + workflows_dir = repo_root / ".github" / "workflows" + if not workflows_dir.exists(): + record("SKIP", "workflows: .github/workflows/ not found (path filters)") + return + + def first_segment(entry): + # Strip a leading '!' negation and any root anchor, then take segment 0. + p = entry[1:] if entry.startswith("!") else entry + return p.lstrip("/").split("/", 1)[0] + + problems = [] + for yml in sorted(workflows_dir.glob("*.yml")): + try: + doc = yaml.safe_load(yml.read_text()) + except yaml.YAMLError as exc: + record("WARN", f"workflows: {yml.name} did not parse as YAML", + str(exc).splitlines()[0][:120]) + continue + if not isinstance(doc, dict): + continue + triggers = doc.get("on", doc.get(True)) # `on:` -> True under YAML 1.1 + if not isinstance(triggers, dict): + continue + for event, spec in triggers.items(): + if not isinstance(spec, dict): + continue + for key in ("paths", "paths-ignore"): + for entry in spec.get(key) or []: + if not isinstance(entry, str): + continue + seg = first_segment(entry) + anchored = entry.lstrip("!").lstrip("/") + if seg in moves_to_hugo and not anchored.startswith("hugo/"): + problems.append(f"{yml.name}: on.{event}.{key}: {entry}") + + if problems: + record("FAIL", "workflows: on.*.paths filters missing hugo/ prefix", + f"{len(problems)}:\n " + "\n ".join(problems[:20])) + else: + record("PASS", "workflows: on.*.paths filters are hugo/-prefixed") + + +# -------------------------------------------------------------------------- +# C. CODEOWNERS +# -------------------------------------------------------------------------- + +def check_codeowners(): + """Every concrete (non-glob) path pattern should resolve on disk.""" + codeowners = repo_root / ".github" / "CODEOWNERS" + if not codeowners.exists(): + record("SKIP", "codeowners: .github/CODEOWNERS not found") + return + + missing = [] + for line in codeowners.read_text().splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + pattern = stripped.split()[0] + if pattern == "*": + continue + # Skip glob patterns; we can't resolve them to a single path. + if any(ch in pattern for ch in "*?[]"): + continue + # Leading slash in CODEOWNERS is repo-root-anchored. + rel = pattern.lstrip("/") + if not (repo_root / rel).exists(): + missing.append(pattern) + + if missing: + record("FAIL", "codeowners: patterns that no longer resolve", + f"{len(missing)}:\n " + "\n ".join(missing[:20])) + else: + record("PASS", "codeowners: all concrete patterns resolve on disk") + + +def check_codeowners_prefixing(moves_to_hugo): + """Every CODEOWNERS pattern whose first segment moved into hugo/ must carry + the hugo/ prefix — globs included. + + check_codeowners() above only proves that *concrete* paths still RESOLVE on + disk; it skips every glob (layouts/shortcodes/**/*.md) and never confirms a + moved pattern was actually repathed. Here we route each pattern by its first + path segment exactly as reorg.py's route_codeowners_pattern does and assert + that a moved segment is prefixed. After a correct reorg the pattern's first + segment is 'hugo', so a flagged pattern means a substitution was missed. + """ + codeowners = repo_root / ".github" / "CODEOWNERS" + if not codeowners.exists(): + record("SKIP", "codeowners: .github/CODEOWNERS not found (prefixing)") + return + + def first_segment(pattern): + body = pattern[1:] if pattern.startswith("/") else pattern # un-anchor + seg = body.split("/", 1)[0] + # reorg.py normalizes the '.local' typo to 'local' before routing. + return "local" if seg == ".local" else seg + + unprefixed = [] + for line in codeowners.read_text().splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + pattern = stripped.split()[0] + if pattern == "*": + continue + if first_segment(pattern) not in moves_to_hugo: + continue + if not pattern.lstrip("/").startswith("hugo/"): + unprefixed.append(pattern) + + if unprefixed: + record("FAIL", "codeowners: moved patterns missing the hugo/ prefix", + f"{len(unprefixed)} (globs included):\n " + + "\n ".join(unprefixed[:20])) + else: + record("PASS", "codeowners: every moved pattern is hugo/-prefixed " + "(globs included)") + + +# -------------------------------------------------------------------------- +# D. Husky behavioral checks +# -------------------------------------------------------------------------- + +def run_hook(script_name): + """Run a .husky hook script from the repo root; return CompletedProcess.""" + return subprocess.run( + ["python3", str(repo_root / ".husky" / script_name)], + cwd=repo_root, + capture_output=True, + text=True, + ) + + +def check_husky_circular_aliases(): + """Plant a self-aliasing page and confirm the hook rejects it.""" + rel = f"hugo/content/en/{MARKER}_alias/selftest.md" + target = repo_root / rel + if target.exists(): + record("WARN", "husky: circular-aliases test skipped (temp path exists)", rel) + return + + # An alias equal to the file's own location is the circular case. + body = ( + "---\n" + "title: Reorg Harness Self Test\n" + "aliases:\n" + f" - /{MARKER}_alias/selftest\n" + "---\n\n" + "Temporary file created by reorg_harness.py.\n" + ) + try: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + git("add", "-f", rel) + proc = run_hook("check-circular-aliases.py") + if proc.returncode != 0 and "circular" in (proc.stdout + proc.stderr).lower(): + record("PASS", "husky: circular-aliases hook rejects bad input") + else: + record("FAIL", "husky: circular-aliases hook did NOT reject bad input", + "hook may still point at the old content/en/ path " + f"(exit={proc.returncode})") + finally: + git("reset", "-q", "HEAD", rel) + if target.exists(): + target.unlink() + _rmdir_if_empty(target.parent) + + +def check_husky_section_index(): + """Plant a new top-level section with no _index.md and confirm rejection.""" + section = f"{MARKER}_section" + rel = f"hugo/content/en/{section}/page.md" + target = repo_root / rel + if target.exists() or (hugo_dir / "content" / "en" / section).exists(): + record("WARN", "husky: section-index test skipped (temp path exists)", rel) + return + + body = "---\ntitle: Reorg Harness Page\nprivate: true\n---\n\nTemp page.\n" + try: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + git("add", "-f", rel) + proc = run_hook("check-section-index.py") + if proc.returncode != 0 and "_index.md" in (proc.stdout + proc.stderr): + record("PASS", "husky: section-index hook rejects missing _index.md") + else: + record("FAIL", "husky: section-index hook did NOT reject bad input", + "hook may still point at the old content/en/ path " + f"(exit={proc.returncode})") + finally: + git("reset", "-q", "HEAD", rel) + if target.exists(): + target.unlink() + _rmdir_if_empty(target.parent) + + +def check_husky_cdocs_gitignore(): + """ + Append a harness pattern to hugo/content/.gitignore, force-track a matching + compiled file (with a .mdoc.md sibling), and confirm the hook flags it. + """ + content_gitignore = hugo_dir / "content" / ".gitignore" + if not content_gitignore.exists(): + record("SKIP", "husky: cdocs-gitignore test skipped (no hugo/content/.gitignore)") + return + + pattern = f"/en/{MARKER}_cdocs.md" + compiled_rel = f"hugo/content/en/{MARKER}_cdocs.md" + source_rel = f"hugo/content/en/{MARKER}_cdocs.mdoc.md" + compiled = repo_root / compiled_rel + source = repo_root / source_rel + if compiled.exists() or source.exists(): + record("WARN", "husky: cdocs-gitignore test skipped (temp path exists)", + compiled_rel) + return + + original_gitignore = content_gitignore.read_text() + try: + content_gitignore.write_text( + original_gitignore.rstrip("\n") + f"\n{pattern}\n" + ) + source.write_text("Temp Cdocs source.\n") + compiled.write_text("Temp compiled Cdocs output.\n") + git("add", "-f", compiled_rel) # force past the gitignore to simulate the mistake + proc = run_hook("check-cdocs-gitignore.py") + if proc.returncode != 0 and MARKER in (proc.stdout + proc.stderr): + record("PASS", "husky: cdocs-gitignore hook flags tracked compiled file") + else: + record("FAIL", "husky: cdocs-gitignore hook did NOT flag tracked file", + "hook may still read the old content/.gitignore path " + f"(exit={proc.returncode})") + finally: + git("reset", "-q", "HEAD", compiled_rel) + for p in (compiled, source): + if p.exists(): + p.unlink() + content_gitignore.write_text(original_gitignore) + + +def _rmdir_if_empty(path): + """Remove a directory only if it is empty (safety guard).""" + try: + path.rmdir() + except OSError: + pass + + +# -------------------------------------------------------------------------- +# E. Vale +# -------------------------------------------------------------------------- + +def check_vale(): + """Confirm vale still flags a known violation using the Datadog style.""" + if subprocess.run(["which", "vale"], capture_output=True).returncode != 0: + record("SKIP", "vale: vale not installed") + return + + styles = (repo_root / ".vale.ini") + if not styles.exists(): + record("SKIP", "vale: .vale.ini not found") + return + + rel = f"hugo/content/en/{MARKER}_vale.md" + target = repo_root / rel + if target.exists(): + record("WARN", "vale: test skipped (temp path exists)", rel) + return + + # Each of these trips a Datadog substitution rule. + body = ( + "---\ntitle: Reorg Harness Vale Test\n---\n\n" + "Simply leverage this feature to ensure it works.\n" + ) + try: + target.write_text(body) + proc = subprocess.run( + ["vale", rel], + cwd=repo_root, + capture_output=True, + text=True, + ) + output = proc.stdout + proc.stderr + if "Datadog." in output: + record("PASS", "vale: Datadog style flags violations from new path") + elif "StylesPath" in output or "ExecError" in output or not output.strip(): + record("FAIL", "vale: ran but produced no Datadog findings", + "StylesPath may not resolve from the new content location") + else: + record("WARN", "vale: ran but no Datadog.* rule fired", + output.strip()[:200]) + finally: + if target.exists(): + target.unlink() + + +# -------------------------------------------------------------------------- +# F. Hugo build (static presence only) +# -------------------------------------------------------------------------- + +def check_build_presence(): + """Static check; the real build is the manual `make start` in todo #5.""" + # Hugo's build entrypoints all need to be co-located under hugo/: the Makefile, + # the Node manifest, and go.mod (required by Hugo Modules at the project root). + required = ["Makefile", "package.json", "go.mod"] + missing = [n for n in required if not (hugo_dir / n).exists()] + if not missing: + record("PASS", "build: hugo/{Makefile,package.json,go.mod} present", + "run `cd hugo && make start` (todo #5) to verify the full build") + else: + record("FAIL", "build: missing Hugo build entrypoint(s) under hugo/", + ", ".join(f"hugo/{n}" for n in missing)) + + +# -------------------------------------------------------------------------- +# G. Rollback round-trip +# -------------------------------------------------------------------------- + +def check_rollback_roundtrip(): + """reorg.py then reorg_rollback.py must restore the tree byte-for-byte. + + This is the only check that exercises reorg_rollback.py at all. Rather than + mutate the live repo (and risk leaving it half-reorganized if a step fails), + it builds a small throwaway git repo holding one representative item for + every code path the two scripts touch — a mixed .gitignore (rules that route + to hugo/, to root, and to both), a workflow with substitutable paths, a + CODEOWNERS with a moved rule, a husky hook with a substitutable path, plus a + few moved/stayed files — then: + + snapshot -> run reorg.py (answering y to every prompt) + -> run reorg_rollback.py -> snapshot again + -> assert byte-identical. + + It specifically catches the easy-to-miss case where reorg.py edits a file in + place (the root .gitignore split) that rollback must restore from git rather + than merely delete. + """ + if shutil.which("python3") is None or shutil.which("git") is None: + record("SKIP", "rollback: python3/git not both available") + return + + workdir = tempfile.mkdtemp(prefix=f"{MARKER}_rollback_") + work = Path(workdir) + + def write(rel, text): + p = work / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + + def git_work(*args): + return subprocess.run(["git", *args], cwd=work, + capture_output=True, text=True) + + def snapshot(): + """relpath -> bytes for every file under work/, excluding .git/.""" + tree = {} + for path in sorted(work.rglob("*")): + if path.is_dir(): + continue + rel = path.relative_to(work) + if rel.parts and rel.parts[0] == ".git": + continue + tree[str(rel)] = path.read_bytes() + return tree + + try: + # Representative fixture. Every top-level name here must appear in + # reorg_config.yaml or reorg.py refuses to run; the mix below routes + # across moves_to_hugo, top_level, and generic-glob cases. + write(".gitignore", + "# build\n" + "public/*\n" + "data/generated\n" + "content/en/api/**/*.go\n" + "node_modules\n" + "\n" + "# generic (kept in both)\n" + "*.log\n" + "\n" + "# root-only tooling\n" + ".github/preview-links-template.md\n") + write(".github/workflows/sample.yml", + "name: sample\n" + "on:\n" + " pull_request:\n" + " paths:\n" + " - 'content/en/**/*.md'\n" + "jobs:\n" + " build:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: python local/bin/foo.py\n") + write(".github/CODEOWNERS", + "* @DataDog/documentation\n" + "/content/ @DataDog/team-a\n" + "data/ @DataDog/team-b\n" + "README.md @DataDog/team-c\n") + write(".husky/check-sample.py", + "from pathlib import Path\n" + 'repo_pattern = "content"\n' + "p = Path('content/en')\n") + write("README.md", "# Fixture\n") + write("astro/package.json", '{"name": "astro-fixture"}\n') + write("content/en/page.md", "---\ntitle: Page\n---\n\nBody.\n") + write("data/sample.yaml", "key: value\n") + write("Makefile", "start:\n\techo build\n") + write("package.json", '{"name": "hugo-fixture"}\n') + write("go.mod", "module example.com/fixture\n\ngo 1.21\n") + write("babel.config.js", "module.exports = {};\n") + + # The scripts resolve repo_root from their own location, so copy them in. + for tool in ("reorg.py", "reorg_rollback.py", "reorg_config.yaml"): + shutil.copy2(repo_root / tool, work / tool) + + git_work("init", "-q") + git_work("add", "-A") + commit = git_work("-c", "user.email=harness@example.com", + "-c", "user.name=reorg harness", + "-c", "commit.gpgsign=false", + "commit", "-q", "-m", "fixture") + if commit.returncode != 0: + record("FAIL", "rollback: could not commit fixture repo", + (commit.stderr or commit.stdout).strip()[:200]) + return + + before = snapshot() + + # reorg.py is interactive (single y/N per mutation section); answer y to + # all. Far more lines than prompts is fine — the extras are ignored. + reorg = subprocess.run( + ["python3", str(work / "reorg.py")], + cwd=work, capture_output=True, text=True, input="y\n" * 100, + ) + if reorg.returncode != 0 or not (work / "hugo").exists(): + record("FAIL", "rollback: reorg.py failed on the fixture", + (reorg.stderr or reorg.stdout).strip()[:200]) + return + + rollback = subprocess.run( + ["python3", str(work / "reorg_rollback.py")], + cwd=work, capture_output=True, text=True, + ) + if rollback.returncode != 0 or (work / "hugo").exists(): + record("FAIL", "rollback: reorg_rollback.py failed on the fixture", + (rollback.stderr or rollback.stdout).strip()[:200]) + return + + after = snapshot() + + added = sorted(set(after) - set(before)) + removed = sorted(set(before) - set(after)) + changed = sorted(p for p in before.keys() & after.keys() + if before[p] != after[p]) + if added or removed or changed: + diffs = ([f"+ {p}" for p in added] + + [f"- {p}" for p in removed] + + [f"~ {p} (in-place edit not reverted)" for p in changed]) + record("FAIL", "rollback: tree not byte-identical after reorg + rollback", + f"{len(diffs)} diff(s):\n " + "\n ".join(diffs[:20])) + else: + record("PASS", "rollback: reorg + rollback restores the tree byte-for-byte", + f"{len(before)} file(s) round-tripped, incl. the .gitignore split") + finally: + shutil.rmtree(work, ignore_errors=True) + + +# -------------------------------------------------------------------------- +# Main +# -------------------------------------------------------------------------- + +def main(): + if not hugo_dir.exists(): + print("hugo/ does not exist — run reorg.py first.", file=sys.stderr) + sys.exit(1) + + branch = git("branch", "--show-current").stdout.strip() + if branch == "master": + print("Refusing to run on master; check out a feature branch.", file=sys.stderr) + sys.exit(1) + + top_level, moves_to_hugo = load_config() + + print("== A. Layout ==") + check_layout(top_level, moves_to_hugo) + check_gitignore_split(top_level, moves_to_hugo) + check_site_separation(moves_to_hugo) + + print("\n== B. Workflow paths ==") + check_workflows(moves_to_hugo) + check_workflow_path_filters(moves_to_hugo) + + print("\n== C. CODEOWNERS ==") + check_codeowners() + check_codeowners_prefixing(moves_to_hugo) + + print("\n== D. Husky hooks ==") + check_husky_circular_aliases() + check_husky_section_index() + check_husky_cdocs_gitignore() + + print("\n== E. Vale ==") + check_vale() + + print("\n== F. Hugo build ==") + check_build_presence() + + print("\n== G. Rollback round-trip ==") + check_rollback_roundtrip() + + # Summary. + counts = {"PASS": 0, "FAIL": 0, "SKIP": 0, "WARN": 0} + for status, _, _ in results: + counts[status] += 1 + print("\n" + "=" * 50) + print(f"PASS {counts['PASS']} FAIL {counts['FAIL']} " + f"WARN {counts['WARN']} SKIP {counts['SKIP']}") + + sys.exit(1 if counts["FAIL"] else 0) + + +if __name__ == "__main__": + main() diff --git a/reorg_rollback.py b/reorg_rollback.py index 050433f7e1d..4c061adfbfd 100644 --- a/reorg_rollback.py +++ b/reorg_rollback.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import os +import subprocess import sys from pathlib import Path @@ -12,12 +13,36 @@ moved = 0 for name in sorted(os.listdir(hugo_dir)): + # reorg.py SPLITS .gitignore in place (it prunes the root copy and writes a + # routed subset to hugo/.gitignore). Skip it here so this rename can't clobber + # the pruned root copy with the hugo subset; the root copy is restored from + # git below, and hugo/.gitignore is discarded with the now-empty directory. + if name == ".gitignore": + continue src = hugo_dir / name dst = repo_root / name print(f"Moving hugo/{name} -> {name}") src.rename(dst) moved += 1 +# Discard the hugo/.gitignore the split created before emptying the directory. +gitignore_copy = hugo_dir / ".gitignore" +if gitignore_copy.exists(): + gitignore_copy.unlink() + print("Removed hugo/.gitignore (created by the split)") + # Only succeeds if hugo/ is now empty, preventing accidental data loss. hugo_dir.rmdir() print(f"Done. {moved} item(s) restored to repo root.") + +# Restore the files reorg.py edited in place to their committed state. The root +# .gitignore was pruned by the split, and reorg.py may have applied partial +# workflow/CODEOWNERS/husky substitutions interactively, so reversing them +# individually isn't reliable — let git restore them wholesale. +subprocess.run( + ["git", "checkout", "--", + ".gitignore", ".github/workflows/", ".github/CODEOWNERS", ".husky/"], + cwd=repo_root, + check=True, +) +print("Restored .gitignore, .github/workflows/, .github/CODEOWNERS, and .husky/ from git.") From d1401437663cce2d92388695fbd709abc05c3548 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 16:14:45 -0500 Subject: [PATCH 03/70] Organize reorg files --- reorg_config.yaml => reorg/config.yaml | 6 +-- reorg/context.md | 11 +++++ reorg.py => reorg/execute_reorg.py | 16 +++---- reorg_harness.py => reorg/harness.py | 58 +++++++++++++------------- reorg_rollback.py => reorg/rollback.py | 2 +- reorg_context.md | 11 ----- 6 files changed, 51 insertions(+), 53 deletions(-) rename reorg_config.yaml => reorg/config.yaml (95%) create mode 100644 reorg/context.md rename reorg.py => reorg/execute_reorg.py (96%) rename reorg_harness.py => reorg/harness.py (93%) rename reorg_rollback.py => reorg/rollback.py (97%) delete mode 100644 reorg_context.md diff --git a/reorg_config.yaml b/reorg/config.yaml similarity index 95% rename from reorg_config.yaml rename to reorg/config.yaml index 83f5d0eb6af..e1e489238f4 100644 --- a/reorg_config.yaml +++ b/reorg/config.yaml @@ -6,11 +6,7 @@ top_level: - astro # Repo scripts - - reorg.py - - reorg_rollback.py - - reorg_config.yaml - - reorg_harness.py - - reorg_context.md + - reorg - reorg_issues.md # Created by reorg.py as the move target diff --git a/reorg/context.md b/reorg/context.md new file mode 100644 index 00000000000..ea141f6122e --- /dev/null +++ b/reorg/context.md @@ -0,0 +1,11 @@ +# Docs repository reorg context + +This repo is currently a Hugo site. We instead want it to contain a `hugo` and `astro` site side by side, with no overlap in their envs, `package.json` files, etc. + +`reorg/config.yaml` describes the relocation target for every file and folder at the top level of the repo. + +`reorg/execute_reorg.py` implements the file and folder path changes, and updates any dependencies on those paths, such as GitHub actions, CODEOWNERS, and Husky workflows. + +`reorg/rollback.py` functions as an "undo" action for the reorg. + +`reorg/harness.py` verifies the functionality of affected entities where possible. \ No newline at end of file diff --git a/reorg.py b/reorg/execute_reorg.py similarity index 96% rename from reorg.py rename to reorg/execute_reorg.py index f50390583c2..1a71b9dfb99 100644 --- a/reorg.py +++ b/reorg/execute_reorg.py @@ -9,8 +9,8 @@ print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) sys.exit(1) -repo_root = Path(__file__).parent -config_path = repo_root / "reorg_config.yaml" +repo_root = Path(__file__).parent.parent +config_path = Path(__file__).parent / "config.yaml" with config_path.open() as f: config = yaml.safe_load(f) @@ -41,8 +41,8 @@ if errors: for name in sorted(errors): - print(f"ERROR: '{name}' is not listed in reorg_config.yaml", file=sys.stderr) - print(f"{len(errors)} error(s) found. Update reorg_config.yaml before running.", file=sys.stderr) + print(f"ERROR: '{name}' is not listed in reorg/config.yaml", file=sys.stderr) + print(f"{len(errors)} error(s) found. Update reorg/config.yaml before running.", file=sys.stderr) sys.exit(1) hugo_dir = repo_root / "hugo" @@ -65,7 +65,7 @@ # A .gitignore is interpreted RELATIVE TO ITS OWN DIRECTORY, so — unlike # CODEOWNERS — a moved rule needs NO "hugo/" prefix; it simply belongs in # hugo/.gitignore. We therefore only ROUTE each line, never rewrite it, by its -# FIRST PATH SEGMENT (driven entirely by reorg_config.yaml, same as CODEOWNERS): +# FIRST PATH SEGMENT (driven entirely by reorg/config.yaml, same as CODEOWNERS): # first segment in moves_to_hugo -> hugo/.gitignore only # first segment in top_level -> root .gitignore only # first segment in neither -> generic/pure glob, kept in BOTH @@ -117,7 +117,7 @@ def route_gitignore_segment(line): (hugo_dir / ".gitignore").write_text("".join(hugo_lines)) print(" Written: .gitignore (root, pruned) and hugo/.gitignore") if both_segments: - print(" NOTE: kept in both (first path segment not in reorg_config.yaml): " + print(" NOTE: kept in both (first path segment not in reorg/config.yaml): " + ", ".join(sorted(both_segments))) # Update .github/workflows/ files to reference paths under hugo/. @@ -188,7 +188,7 @@ def route_gitignore_segment(line): # FIRST PATH SEGMENT. The decision is made on a whole segment, never a raw # substring, so "config" can never match inside "customization_config" and the # substitution list no longer has to be hand-ordered or kept in sync with the -# move list — it is derived entirely from reorg_config.yaml. +# move list — it is derived entirely from reorg/config.yaml. def route_codeowners_pattern(pattern): """Rewrite one CODEOWNERS pattern for the hugo/ move. @@ -259,7 +259,7 @@ def route_codeowners_pattern(pattern): applied = True if left_alone: - print("\n NOTE: left unchanged (first path segment not in reorg_config.yaml): " + print("\n NOTE: left unchanged (first path segment not in reorg/config.yaml): " + ", ".join(sorted(left_alone))) if applied: diff --git a/reorg_harness.py b/reorg/harness.py similarity index 93% rename from reorg_harness.py rename to reorg/harness.py index c2e908bd9df..f49a6b754ae 100644 --- a/reorg_harness.py +++ b/reorg/harness.py @@ -2,7 +2,7 @@ """ Post-reorg validation harness. -Run this AFTER reorg.py, from the repo root, on a feature branch (not master). +Run this AFTER execute_reorg.py, from the repo root, on a feature branch (not master). It verifies that the things the reorg could silently break still work: A. Layout - hugo/ holds the moved dirs; nothing moved is left at root; @@ -10,7 +10,7 @@ Node/build file lingers at the root competing with Astro. B. Workflows - no .github/workflows/ file references a moved path (dir OR file) without the hugo/ prefix, in shell scalars and in - structured on.*.paths filters (catches gaps in reorg.py). + structured on.*.paths filters (catches gaps in execute_reorg.py). C. CODEOWNERS - every concrete path pattern still resolves on disk, and every moved pattern (globs included) carries the hugo/ prefix. D. Husky hooks - each pre-commit check still REJECTS a known-bad input planted @@ -19,7 +19,7 @@ E. Vale - vale still flags a known violation using the Datadog style, proving StylesPath resolves from the new content location. F. Hugo build - static presence check only; run `make start` manually (todo #5). - G. Rollback - on a throwaway repo, reorg.py then reorg_rollback.py must + G. Rollback - on a throwaway repo, execute_reorg.py then rollback.py must restore the tree byte-for-byte (the only test of rollback). The harness is non-destructive. Every file it creates lives under a @@ -41,7 +41,7 @@ print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) sys.exit(1) -repo_root = Path(__file__).parent +repo_root = Path(__file__).parent.parent hugo_dir = repo_root / "hugo" # Distinctive marker so anything we create is obvious and easy to clean up. @@ -73,8 +73,8 @@ def git(*args, check=False): def load_config(): - """Load reorg_config.yaml and return (top_level, moves_to_hugo) name sets.""" - config_path = repo_root / "reorg_config.yaml" + """Load reorg/config.yaml and return (top_level, moves_to_hugo) name sets.""" + config_path = Path(__file__).parent / "config.yaml" with config_path.open() as f: config = yaml.safe_load(f) return set(config.get("top_level", [])), set(config.get("moves_to_hugo", [])) @@ -133,7 +133,7 @@ def check_layout(top_level, moves_to_hugo): record("PASS", "layout: top-level anchors intact", ", ".join(must_stay)) - # Both .gitignore files should exist (reorg.py splits one into two; + # Both .gitignore files should exist (execute_reorg.py splits one into two; # check_gitignore_split below verifies the split routed correctly). if (repo_root / ".gitignore").exists() and (hugo_dir / ".gitignore").exists(): record("PASS", "layout: .gitignore present at root and in hugo/") @@ -143,7 +143,7 @@ def check_layout(top_level, moves_to_hugo): def check_gitignore_split(top_level, moves_to_hugo): - """Verify reorg.py routed .gitignore rules to the correct side. + """Verify execute_reorg.py routed .gitignore rules to the correct side. A .gitignore is relative to its own directory, so after the split no surviving rule should point at a path that lives on the OTHER side: @@ -233,7 +233,7 @@ def check_workflows(moves_to_hugo): trailing-slash token, so a workflow that referenced a moved *file* — babel.config.js, markdoc.config.json, the Makefile, go.mod — was never validated. We now route every path-like token by its first segment exactly - as reorg.py does: a token whose first segment moved into hugo/ must be + as execute_reorg.py does: a token whose first segment moved into hugo/ must be hugo/-prefixed (after a correct reorg its first segment is 'hugo', so it no longer matches). A token counts as a path reference when it contains a '/', or when the whole token is the exact name of a moved file — so a bare word @@ -242,7 +242,7 @@ def check_workflows(moves_to_hugo): A moved file whose name ALSO exists under astro/ (package.json, yarn.lock, node_modules, .nvmrc) is left out of the bare-name match: a standalone 'package.json' is genuinely ambiguous (it may be Astro's), which is exactly - why reorg.py never blind-substitutes those names. They are still validated + why execute_reorg.py never blind-substitutes those names. They are still validated when they appear inside a slashed path routed by its first segment. """ workflows_dir = repo_root / ".github" / "workflows" @@ -396,7 +396,7 @@ def check_codeowners_prefixing(moves_to_hugo): check_codeowners() above only proves that *concrete* paths still RESOLVE on disk; it skips every glob (layouts/shortcodes/**/*.md) and never confirms a moved pattern was actually repathed. Here we route each pattern by its first - path segment exactly as reorg.py's route_codeowners_pattern does and assert + path segment exactly as execute_reorg.py's route_codeowners_pattern does and assert that a moved segment is prefixed. After a correct reorg the pattern's first segment is 'hugo', so a flagged pattern means a substitution was missed. """ @@ -408,7 +408,7 @@ def check_codeowners_prefixing(moves_to_hugo): def first_segment(pattern): body = pattern[1:] if pattern.startswith("/") else pattern # un-anchor seg = body.split("/", 1)[0] - # reorg.py normalizes the '.local' typo to 'local' before routing. + # execute_reorg.py normalizes the '.local' typo to 'local' before routing. return "local" if seg == ".local" else seg unprefixed = [] @@ -462,7 +462,7 @@ def check_husky_circular_aliases(): "aliases:\n" f" - /{MARKER}_alias/selftest\n" "---\n\n" - "Temporary file created by reorg_harness.py.\n" + "Temporary file created by reorg/harness.py.\n" ) try: target.parent.mkdir(parents=True, exist_ok=True) @@ -632,9 +632,9 @@ def check_build_presence(): # -------------------------------------------------------------------------- def check_rollback_roundtrip(): - """reorg.py then reorg_rollback.py must restore the tree byte-for-byte. + """execute_reorg.py then rollback.py must restore the tree byte-for-byte. - This is the only check that exercises reorg_rollback.py at all. Rather than + This is the only check that exercises rollback.py at all. Rather than mutate the live repo (and risk leaving it half-reorganized if a step fails), it builds a small throwaway git repo holding one representative item for every code path the two scripts touch — a mixed .gitignore (rules that route @@ -642,11 +642,11 @@ def check_rollback_roundtrip(): CODEOWNERS with a moved rule, a husky hook with a substitutable path, plus a few moved/stayed files — then: - snapshot -> run reorg.py (answering y to every prompt) - -> run reorg_rollback.py -> snapshot again + snapshot -> run execute_reorg.py (answering y to every prompt) + -> run rollback.py -> snapshot again -> assert byte-identical. - It specifically catches the easy-to-miss case where reorg.py edits a file in + It specifically catches the easy-to-miss case where execute_reorg.py edits a file in place (the root .gitignore split) that rollback must restore from git rather than merely delete. """ @@ -680,7 +680,7 @@ def snapshot(): try: # Representative fixture. Every top-level name here must appear in - # reorg_config.yaml or reorg.py refuses to run; the mix below routes + # reorg/config.yaml or execute_reorg.py refuses to run; the mix below routes # across moves_to_hugo, top_level, and generic-glob cases. write(".gitignore", "# build\n" @@ -723,9 +723,11 @@ def snapshot(): write("go.mod", "module example.com/fixture\n\ngo 1.21\n") write("babel.config.js", "module.exports = {};\n") - # The scripts resolve repo_root from their own location, so copy them in. - for tool in ("reorg.py", "reorg_rollback.py", "reorg_config.yaml"): - shutil.copy2(repo_root / tool, work / tool) + # The scripts resolve repo_root as their parent's parent, so place them + # under a reorg/ subfolder that mirrors the real repo layout. + (work / "reorg").mkdir() + for tool in ("execute_reorg.py", "rollback.py", "config.yaml"): + shutil.copy2(repo_root / "reorg" / tool, work / "reorg" / tool) git_work("init", "-q") git_work("add", "-A") @@ -740,23 +742,23 @@ def snapshot(): before = snapshot() - # reorg.py is interactive (single y/N per mutation section); answer y to + # execute_reorg.py is interactive (single y/N per mutation section); answer y to # all. Far more lines than prompts is fine — the extras are ignored. reorg = subprocess.run( - ["python3", str(work / "reorg.py")], + ["python3", str(work / "reorg" / "execute_reorg.py")], cwd=work, capture_output=True, text=True, input="y\n" * 100, ) if reorg.returncode != 0 or not (work / "hugo").exists(): - record("FAIL", "rollback: reorg.py failed on the fixture", + record("FAIL", "rollback: execute_reorg.py failed on the fixture", (reorg.stderr or reorg.stdout).strip()[:200]) return rollback = subprocess.run( - ["python3", str(work / "reorg_rollback.py")], + ["python3", str(work / "reorg" / "rollback.py")], cwd=work, capture_output=True, text=True, ) if rollback.returncode != 0 or (work / "hugo").exists(): - record("FAIL", "rollback: reorg_rollback.py failed on the fixture", + record("FAIL", "rollback: rollback.py failed on the fixture", (rollback.stderr or rollback.stdout).strip()[:200]) return @@ -785,7 +787,7 @@ def snapshot(): def main(): if not hugo_dir.exists(): - print("hugo/ does not exist — run reorg.py first.", file=sys.stderr) + print("hugo/ does not exist — run reorg/execute_reorg.py first.", file=sys.stderr) sys.exit(1) branch = git("branch", "--show-current").stdout.strip() diff --git a/reorg_rollback.py b/reorg/rollback.py similarity index 97% rename from reorg_rollback.py rename to reorg/rollback.py index 4c061adfbfd..b2bda7fd1c2 100644 --- a/reorg_rollback.py +++ b/reorg/rollback.py @@ -4,7 +4,7 @@ import sys from pathlib import Path -repo_root = Path(__file__).parent +repo_root = Path(__file__).parent.parent hugo_dir = repo_root / "hugo" if not hugo_dir.exists(): diff --git a/reorg_context.md b/reorg_context.md deleted file mode 100644 index 55c39a900e5..00000000000 --- a/reorg_context.md +++ /dev/null @@ -1,11 +0,0 @@ -# Docs repository reorg context - -This repo is currently a Hugo site. We instead want it to contain a `hugo` and `astro` site side by side, with no overlap in their envs, `package.json` files, etc. - -The `reorg_config.yaml` describes the relocation target for every file and folder at the top level of the repo. - -`reorg.py` implements the file and folder path changes, and updates any dependencies on those paths, such as GitHub actions, CODEOWNERS, and Husky workflows. - -`reorg_rollback.py` functions as an "undo" action for the reorg. - -`reorg_harness.py` verifies the functionality of affected entities where possible. \ No newline at end of file From d511ddb66e92e7b15747957a86f8a401d7f86c45 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 16:18:33 -0500 Subject: [PATCH 04/70] Rename file --- reorg/context.md | 2 +- reorg/{harness.py => validate_reorg.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename reorg/{harness.py => validate_reorg.py} (99%) diff --git a/reorg/context.md b/reorg/context.md index ea141f6122e..49e59ebc781 100644 --- a/reorg/context.md +++ b/reorg/context.md @@ -8,4 +8,4 @@ This repo is currently a Hugo site. We instead want it to contain a `hugo` and ` `reorg/rollback.py` functions as an "undo" action for the reorg. -`reorg/harness.py` verifies the functionality of affected entities where possible. \ No newline at end of file +`reorg/validate_reorg.py` verifies the functionality of affected entities where possible. \ No newline at end of file diff --git a/reorg/harness.py b/reorg/validate_reorg.py similarity index 99% rename from reorg/harness.py rename to reorg/validate_reorg.py index f49a6b754ae..b998dbc606e 100644 --- a/reorg/harness.py +++ b/reorg/validate_reorg.py @@ -462,7 +462,7 @@ def check_husky_circular_aliases(): "aliases:\n" f" - /{MARKER}_alias/selftest\n" "---\n\n" - "Temporary file created by reorg/harness.py.\n" + "Temporary file created by reorg/validate_reorg.py.\n" ) try: target.parent.mkdir(parents=True, exist_ok=True) From 5ee8f2689c94450f0eeaa5193720516169f3a8d0 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 16:21:12 -0500 Subject: [PATCH 05/70] Rename files --- {reorg => astro_reorg}/config.yaml | 2 +- astro_reorg/context.md | 11 +++++++++++ {reorg => astro_reorg}/execute_reorg.py | 12 ++++++------ {reorg => astro_reorg}/rollback.py | 0 {reorg => astro_reorg}/validate_reorg.py | 18 +++++++++--------- reorg/context.md | 11 ----------- 6 files changed, 27 insertions(+), 27 deletions(-) rename {reorg => astro_reorg}/config.yaml (99%) create mode 100644 astro_reorg/context.md rename {reorg => astro_reorg}/execute_reorg.py (94%) rename {reorg => astro_reorg}/rollback.py (100%) rename {reorg => astro_reorg}/validate_reorg.py (98%) delete mode 100644 reorg/context.md diff --git a/reorg/config.yaml b/astro_reorg/config.yaml similarity index 99% rename from reorg/config.yaml rename to astro_reorg/config.yaml index e1e489238f4..b8a182be4fb 100644 --- a/reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -6,7 +6,7 @@ top_level: - astro # Repo scripts - - reorg + - astro_reorg - reorg_issues.md # Created by reorg.py as the move target diff --git a/astro_reorg/context.md b/astro_reorg/context.md new file mode 100644 index 00000000000..31c2924595d --- /dev/null +++ b/astro_reorg/context.md @@ -0,0 +1,11 @@ +# Docs repository reorg context + +This repo is currently a Hugo site. We instead want it to contain a `hugo` and `astro` site side by side, with no overlap in their envs, `package.json` files, etc. + +`astro_reorg/config.yaml` describes the relocation target for every file and folder at the top level of the repo. + +`astro_reorg/execute_reorg.py` implements the file and folder path changes, and updates any dependencies on those paths, such as GitHub actions, CODEOWNERS, and Husky workflows. + +`astro_reorg/rollback.py` functions as an "undo" action for the reorg. + +`astro_reorg/validate_reorg.py` verifies the functionality of affected entities where possible. \ No newline at end of file diff --git a/reorg/execute_reorg.py b/astro_reorg/execute_reorg.py similarity index 94% rename from reorg/execute_reorg.py rename to astro_reorg/execute_reorg.py index 1a71b9dfb99..53524555e8b 100644 --- a/reorg/execute_reorg.py +++ b/astro_reorg/execute_reorg.py @@ -41,8 +41,8 @@ if errors: for name in sorted(errors): - print(f"ERROR: '{name}' is not listed in reorg/config.yaml", file=sys.stderr) - print(f"{len(errors)} error(s) found. Update reorg/config.yaml before running.", file=sys.stderr) + print(f"ERROR: '{name}' is not listed in astro_reorg/config.yaml", file=sys.stderr) + print(f"{len(errors)} error(s) found. Update astro_reorg/config.yaml before running.", file=sys.stderr) sys.exit(1) hugo_dir = repo_root / "hugo" @@ -65,7 +65,7 @@ # A .gitignore is interpreted RELATIVE TO ITS OWN DIRECTORY, so — unlike # CODEOWNERS — a moved rule needs NO "hugo/" prefix; it simply belongs in # hugo/.gitignore. We therefore only ROUTE each line, never rewrite it, by its -# FIRST PATH SEGMENT (driven entirely by reorg/config.yaml, same as CODEOWNERS): +# FIRST PATH SEGMENT (driven entirely by astro_reorg/config.yaml, same as CODEOWNERS): # first segment in moves_to_hugo -> hugo/.gitignore only # first segment in top_level -> root .gitignore only # first segment in neither -> generic/pure glob, kept in BOTH @@ -117,7 +117,7 @@ def route_gitignore_segment(line): (hugo_dir / ".gitignore").write_text("".join(hugo_lines)) print(" Written: .gitignore (root, pruned) and hugo/.gitignore") if both_segments: - print(" NOTE: kept in both (first path segment not in reorg/config.yaml): " + print(" NOTE: kept in both (first path segment not in astro_reorg/config.yaml): " + ", ".join(sorted(both_segments))) # Update .github/workflows/ files to reference paths under hugo/. @@ -188,7 +188,7 @@ def route_gitignore_segment(line): # FIRST PATH SEGMENT. The decision is made on a whole segment, never a raw # substring, so "config" can never match inside "customization_config" and the # substitution list no longer has to be hand-ordered or kept in sync with the -# move list — it is derived entirely from reorg/config.yaml. +# move list — it is derived entirely from astro_reorg/config.yaml. def route_codeowners_pattern(pattern): """Rewrite one CODEOWNERS pattern for the hugo/ move. @@ -259,7 +259,7 @@ def route_codeowners_pattern(pattern): applied = True if left_alone: - print("\n NOTE: left unchanged (first path segment not in reorg/config.yaml): " + print("\n NOTE: left unchanged (first path segment not in astro_reorg/config.yaml): " + ", ".join(sorted(left_alone))) if applied: diff --git a/reorg/rollback.py b/astro_reorg/rollback.py similarity index 100% rename from reorg/rollback.py rename to astro_reorg/rollback.py diff --git a/reorg/validate_reorg.py b/astro_reorg/validate_reorg.py similarity index 98% rename from reorg/validate_reorg.py rename to astro_reorg/validate_reorg.py index b998dbc606e..8f1c00e7a00 100644 --- a/reorg/validate_reorg.py +++ b/astro_reorg/validate_reorg.py @@ -73,7 +73,7 @@ def git(*args, check=False): def load_config(): - """Load reorg/config.yaml and return (top_level, moves_to_hugo) name sets.""" + """Load astro_reorg/config.yaml and return (top_level, moves_to_hugo) name sets.""" config_path = Path(__file__).parent / "config.yaml" with config_path.open() as f: config = yaml.safe_load(f) @@ -462,7 +462,7 @@ def check_husky_circular_aliases(): "aliases:\n" f" - /{MARKER}_alias/selftest\n" "---\n\n" - "Temporary file created by reorg/validate_reorg.py.\n" + "Temporary file created by astro_reorg/validate_reorg.py.\n" ) try: target.parent.mkdir(parents=True, exist_ok=True) @@ -680,7 +680,7 @@ def snapshot(): try: # Representative fixture. Every top-level name here must appear in - # reorg/config.yaml or execute_reorg.py refuses to run; the mix below routes + # astro_reorg/config.yaml or execute_reorg.py refuses to run; the mix below routes # across moves_to_hugo, top_level, and generic-glob cases. write(".gitignore", "# build\n" @@ -724,10 +724,10 @@ def snapshot(): write("babel.config.js", "module.exports = {};\n") # The scripts resolve repo_root as their parent's parent, so place them - # under a reorg/ subfolder that mirrors the real repo layout. - (work / "reorg").mkdir() + # under an astro_reorg/ subfolder that mirrors the real repo layout. + (work / "astro_reorg").mkdir() for tool in ("execute_reorg.py", "rollback.py", "config.yaml"): - shutil.copy2(repo_root / "reorg" / tool, work / "reorg" / tool) + shutil.copy2(repo_root / "astro_reorg" / tool, work / "astro_reorg" / tool) git_work("init", "-q") git_work("add", "-A") @@ -745,7 +745,7 @@ def snapshot(): # execute_reorg.py is interactive (single y/N per mutation section); answer y to # all. Far more lines than prompts is fine — the extras are ignored. reorg = subprocess.run( - ["python3", str(work / "reorg" / "execute_reorg.py")], + ["python3", str(work / "astro_reorg" / "execute_reorg.py")], cwd=work, capture_output=True, text=True, input="y\n" * 100, ) if reorg.returncode != 0 or not (work / "hugo").exists(): @@ -754,7 +754,7 @@ def snapshot(): return rollback = subprocess.run( - ["python3", str(work / "reorg" / "rollback.py")], + ["python3", str(work / "astro_reorg" / "rollback.py")], cwd=work, capture_output=True, text=True, ) if rollback.returncode != 0 or (work / "hugo").exists(): @@ -787,7 +787,7 @@ def snapshot(): def main(): if not hugo_dir.exists(): - print("hugo/ does not exist — run reorg/execute_reorg.py first.", file=sys.stderr) + print("hugo/ does not exist — run astro_reorg/execute_reorg.py first.", file=sys.stderr) sys.exit(1) branch = git("branch", "--show-current").stdout.strip() diff --git a/reorg/context.md b/reorg/context.md deleted file mode 100644 index 49e59ebc781..00000000000 --- a/reorg/context.md +++ /dev/null @@ -1,11 +0,0 @@ -# Docs repository reorg context - -This repo is currently a Hugo site. We instead want it to contain a `hugo` and `astro` site side by side, with no overlap in their envs, `package.json` files, etc. - -`reorg/config.yaml` describes the relocation target for every file and folder at the top level of the repo. - -`reorg/execute_reorg.py` implements the file and folder path changes, and updates any dependencies on those paths, such as GitHub actions, CODEOWNERS, and Husky workflows. - -`reorg/rollback.py` functions as an "undo" action for the reorg. - -`reorg/validate_reorg.py` verifies the functionality of affected entities where possible. \ No newline at end of file From 93e4efb05906519fab796f6ee65da2be842002a2 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 16:22:56 -0500 Subject: [PATCH 06/70] Add file to config --- astro_reorg/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index b8a182be4fb..47bc16e6276 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -121,3 +121,5 @@ moves_to_hugo: ignore: # macOS metadata - .DS_Store + # Python bytecode cache + - __pycache__ From 7e4fef0f5af51d2c48b95f8e3105c80b3316b767 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 16:31:09 -0500 Subject: [PATCH 07/70] Add visual diffs for file updates --- astro_reorg/execute_reorg.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/astro_reorg/execute_reorg.py b/astro_reorg/execute_reorg.py index 53524555e8b..4dfc925df39 100644 --- a/astro_reorg/execute_reorg.py +++ b/astro_reorg/execute_reorg.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import difflib import os import sys from pathlib import Path @@ -19,6 +20,28 @@ moves_to_hugo = set(config.get("moves_to_hugo", [])) ignore = set(config.get("ignore", [])) + +def show_diff(original: str, updated: str, filename: str = "") -> None: + """Print a colored unified diff between original and updated text.""" + orig_lines = original.splitlines(keepends=True) + upd_lines = updated.splitlines(keepends=True) + diff = list(difflib.unified_diff( + orig_lines, upd_lines, + fromfile=f"a/{filename}", tofile=f"b/{filename}", + n=2, + )) + if not diff: + return + for line in diff: + if line.startswith("+") and not line.startswith("+++"): + print(f"\033[32m{line}\033[0m", end="") + elif line.startswith("-") and not line.startswith("---"): + print(f"\033[31m{line}\033[0m", end="") + elif line.startswith("@@"): + print(f"\033[36m{line}\033[0m", end="") + else: + print(line, end="") + # Sanity-check the config itself for conflicts. conflicts = top_level & moves_to_hugo if conflicts: @@ -111,6 +134,10 @@ def route_gitignore_segment(line): f"{len(both_segments)} generic kept in both") print(f" root: {len(root_lines)} line(s), hugo/: {len(hugo_lines)} line(s) " f"(was {len(gi_lines)} duplicated wholesale)") +original_gi = "".join(gi_lines) +show_diff(original_gi, "".join(root_lines), ".gitignore") +print(" --- new file: hugo/.gitignore ---") +show_diff("", "".join(hugo_lines), "hugo/.gitignore") answer = input(" Apply? [y/N] ").strip().lower() if answer == "y": gitignore.write_text("".join(root_lines)) @@ -144,6 +171,7 @@ def route_gitignore_segment(line): if old not in updated: continue print(f"\n {yml_file.name}: {old!r} → {new!r}") + show_diff(updated, updated.replace(old, new), yml_file.name) answer = input(" Apply? [y/N] ").strip().lower() if answer == "y": updated = updated.replace(old, new) @@ -172,6 +200,7 @@ def route_gitignore_segment(line): if old not in updated: continue print(f"\n {py_file.name}: {old!r} → {new!r}") + show_diff(updated, updated.replace(old, new), py_file.name) answer = input(" Apply? [y/N] ").strip().lower() if answer == "y": updated = updated.replace(old, new) @@ -252,6 +281,10 @@ def route_codeowners_pattern(pattern): new_pattern = entries[0][2].lstrip().split(maxsplit=1)[0] print(f"\n CODEOWNERS: prefix {len(entries)} pattern(s) under {segment!r} " f"with hugo/ (e.g. {old_pattern!r} → {new_pattern!r})") + preview = lines[:] + for idx, _, new_line in entries: + preview[idx] = new_line + show_diff("".join(lines), "".join(preview), "CODEOWNERS") answer = input(" Apply? [y/N] ").strip().lower() if answer == "y": for idx, _, new_line in entries: From 6092c839925fb1b91afaa8f91560dd49e6c0cf1c Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 16:37:54 -0500 Subject: [PATCH 08/70] Skip .gitignore previews --- astro_reorg/execute_reorg.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/astro_reorg/execute_reorg.py b/astro_reorg/execute_reorg.py index 4dfc925df39..d28f1358ad7 100644 --- a/astro_reorg/execute_reorg.py +++ b/astro_reorg/execute_reorg.py @@ -134,10 +134,6 @@ def route_gitignore_segment(line): f"{len(both_segments)} generic kept in both") print(f" root: {len(root_lines)} line(s), hugo/: {len(hugo_lines)} line(s) " f"(was {len(gi_lines)} duplicated wholesale)") -original_gi = "".join(gi_lines) -show_diff(original_gi, "".join(root_lines), ".gitignore") -print(" --- new file: hugo/.gitignore ---") -show_diff("", "".join(hugo_lines), "hugo/.gitignore") answer = input(" Apply? [y/N] ").strip().lower() if answer == "y": gitignore.write_text("".join(root_lines)) From 4ff7a3204bac911abd7980fad48eab4363807550 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 16:41:43 -0500 Subject: [PATCH 09/70] Simplify script --- astro_reorg/execute_reorg.py | 87 ++++++------------------------------ 1 file changed, 13 insertions(+), 74 deletions(-) diff --git a/astro_reorg/execute_reorg.py b/astro_reorg/execute_reorg.py index d28f1358ad7..146792b032b 100644 --- a/astro_reorg/execute_reorg.py +++ b/astro_reorg/execute_reorg.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -import difflib import os import sys from pathlib import Path @@ -20,28 +19,6 @@ moves_to_hugo = set(config.get("moves_to_hugo", [])) ignore = set(config.get("ignore", [])) - -def show_diff(original: str, updated: str, filename: str = "") -> None: - """Print a colored unified diff between original and updated text.""" - orig_lines = original.splitlines(keepends=True) - upd_lines = updated.splitlines(keepends=True) - diff = list(difflib.unified_diff( - orig_lines, upd_lines, - fromfile=f"a/{filename}", tofile=f"b/{filename}", - n=2, - )) - if not diff: - return - for line in diff: - if line.startswith("+") and not line.startswith("+++"): - print(f"\033[32m{line}\033[0m", end="") - elif line.startswith("-") and not line.startswith("---"): - print(f"\033[31m{line}\033[0m", end="") - elif line.startswith("@@"): - print(f"\033[36m{line}\033[0m", end="") - else: - print(line, end="") - # Sanity-check the config itself for conflicts. conflicts = top_level & moves_to_hugo if conflicts: @@ -130,18 +107,13 @@ def route_gitignore_segment(line): hugo_lines.append(raw) both_segments.add(segment) -print(f"\n .gitignore: {len(hugo_only_segments)} segment(s) -> hugo/.gitignore only, " - f"{len(both_segments)} generic kept in both") -print(f" root: {len(root_lines)} line(s), hugo/: {len(hugo_lines)} line(s) " - f"(was {len(gi_lines)} duplicated wholesale)") -answer = input(" Apply? [y/N] ").strip().lower() -if answer == "y": - gitignore.write_text("".join(root_lines)) - (hugo_dir / ".gitignore").write_text("".join(hugo_lines)) - print(" Written: .gitignore (root, pruned) and hugo/.gitignore") - if both_segments: - print(" NOTE: kept in both (first path segment not in astro_reorg/config.yaml): " - + ", ".join(sorted(both_segments))) +gitignore.write_text("".join(root_lines)) +(hugo_dir / ".gitignore").write_text("".join(hugo_lines)) +print(f" Written: .gitignore (root, pruned) and hugo/.gitignore " + f"({len(hugo_only_segments)} segment(s) -> hugo/ only, {len(both_segments)} generic kept in both)") +if both_segments: + print(" NOTE: kept in both (first path segment not in astro_reorg/config.yaml): " + + ", ".join(sorted(both_segments))) # Update .github/workflows/ files to reference paths under hugo/. WORKFLOW_SUBSTITUTIONS = [ @@ -164,13 +136,7 @@ def route_gitignore_segment(line): original = yml_file.read_text() updated = original for old, new in WORKFLOW_SUBSTITUTIONS: - if old not in updated: - continue - print(f"\n {yml_file.name}: {old!r} → {new!r}") - show_diff(updated, updated.replace(old, new), yml_file.name) - answer = input(" Apply? [y/N] ").strip().lower() - if answer == "y": - updated = updated.replace(old, new) + updated = updated.replace(old, new) if updated != original: yml_file.write_text(updated) print(f" Written: {yml_file.name}") @@ -193,13 +159,7 @@ def route_gitignore_segment(line): original = py_file.read_text() updated = original for old, new in HUSKY_SUBSTITUTIONS: - if old not in updated: - continue - print(f"\n {py_file.name}: {old!r} → {new!r}") - show_diff(updated, updated.replace(old, new), py_file.name) - answer = input(" Apply? [y/N] ").strip().lower() - if answer == "y": - updated = updated.replace(old, new) + updated = updated.replace(old, new) if updated != original: py_file.write_text(updated) print(f" Written: {py_file.name}") @@ -246,12 +206,9 @@ def route_codeowners_pattern(pattern): print("\nUpdating .github/CODEOWNERS...") codeowners = repo_root / ".github" / "CODEOWNERS" lines = codeowners.read_text().splitlines(keepends=True) - -# Group rewritable lines by first path segment so we prompt once per segment -# (one y/N covers every "content/..." rule) instead of once per line. -changes_by_segment = {} # segment -> list of (line_index, old_pattern, new_line) left_alone = set() # first segments in neither config list (surfaced below) +changed = False for i, raw in enumerate(lines): newline = "\n" if raw.endswith("\n") else "" line = raw[:-len(newline)] if newline else raw @@ -266,31 +223,13 @@ def route_codeowners_pattern(pattern): if segment is not None and segment not in top_level: left_alone.add(segment) continue - changes_by_segment.setdefault(segment, []).append( - (i, pattern, indent + new_pattern + rest + newline) - ) - -applied = False -for segment in sorted(changes_by_segment): - entries = changes_by_segment[segment] - old_pattern = entries[0][1] - new_pattern = entries[0][2].lstrip().split(maxsplit=1)[0] - print(f"\n CODEOWNERS: prefix {len(entries)} pattern(s) under {segment!r} " - f"with hugo/ (e.g. {old_pattern!r} → {new_pattern!r})") - preview = lines[:] - for idx, _, new_line in entries: - preview[idx] = new_line - show_diff("".join(lines), "".join(preview), "CODEOWNERS") - answer = input(" Apply? [y/N] ").strip().lower() - if answer == "y": - for idx, _, new_line in entries: - lines[idx] = new_line - applied = True + lines[i] = indent + new_pattern + rest + newline + changed = True if left_alone: print("\n NOTE: left unchanged (first path segment not in astro_reorg/config.yaml): " + ", ".join(sorted(left_alone))) -if applied: +if changed: codeowners.write_text("".join(lines)) print(" Written: CODEOWNERS") From b896be101529d1bd512ef80697a510927eb8cdc3 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 17:18:45 -0500 Subject: [PATCH 10/70] Code tweaks --- astro_reorg/context.md | 4 +- astro_reorg/helpers.py | 662 +++++++++++++++++++++++++++ astro_reorg/validate_reorg.py | 826 ++-------------------------------- 3 files changed, 704 insertions(+), 788 deletions(-) create mode 100644 astro_reorg/helpers.py diff --git a/astro_reorg/context.md b/astro_reorg/context.md index 31c2924595d..f92cc1951ac 100644 --- a/astro_reorg/context.md +++ b/astro_reorg/context.md @@ -8,4 +8,6 @@ This repo is currently a Hugo site. We instead want it to contain a `hugo` and ` `astro_reorg/rollback.py` functions as an "undo" action for the reorg. -`astro_reorg/validate_reorg.py` verifies the functionality of affected entities where possible. \ No newline at end of file +`astro_reorg/validate_reorg.py` verifies the functionality of affected entities where possible. + +You can ignore the `astro` folder, it's a remnant from another branch where an Astro site is being developed. It is completely out of scope. \ No newline at end of file diff --git a/astro_reorg/helpers.py b/astro_reorg/helpers.py new file mode 100644 index 00000000000..cd4ec79c455 --- /dev/null +++ b/astro_reorg/helpers.py @@ -0,0 +1,662 @@ +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +try: + import yaml +except ImportError: + print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + +repo_root = Path(__file__).parent.parent +hugo_dir = repo_root / "hugo" + +# Distinctive marker so anything we create is obvious and easy to clean up. +MARKER = "__reorg_harness__" + +# (status, name, detail) tuples collected as checks run. +results = [] + + +def record(status, name, detail=""): + """Record a check outcome. status is one of PASS, FAIL, SKIP, WARN.""" + results.append((status, name, detail)) + symbol = {"PASS": "✅", "FAIL": "❌", "SKIP": "⏭ ", "WARN": "⚠ "}[status] + line = f"{symbol} {status:4} {name}" + if detail: + line += f"\n {detail}" + print(line) + + +def git(*args, check=False): + """Run a git command from the repo root and return the CompletedProcess.""" + return subprocess.run( + ["git", *args], + cwd=repo_root, + capture_output=True, + text=True, + check=check, + ) + + +def load_config(): + """Load astro_reorg/config.yaml and return (top_level, moves_to_hugo) name sets.""" + config_path = Path(__file__).parent / "config.yaml" + with config_path.open() as f: + config = yaml.safe_load(f) + return set(config.get("top_level", [])), set(config.get("moves_to_hugo", [])) + + +def check_layout(top_level, moves_to_hugo): + """hugo/ holds the moved dirs; nothing that moved is left at the root.""" + # Moved items that the reorg actually had to relocate (present before). + expected_in_hugo = sorted(n for n in moves_to_hugo if (hugo_dir / n).exists()) + missing_from_hugo = sorted( + n for n in moves_to_hugo + if (repo_root / n).exists() and not (hugo_dir / n).exists() + ) + + # A moved name still sitting at the root means the move didn't happen. + still_at_root = sorted( + n for n in moves_to_hugo if (repo_root / n).exists() + ) + if still_at_root: + record("FAIL", "layout: moved items remain at repo root", + ", ".join(still_at_root)) + else: + record("PASS", "layout: no moved items remain at repo root", + f"{len(expected_in_hugo)} item(s) confirmed under hugo/") + + if missing_from_hugo: + record("FAIL", "layout: moved items missing under hugo/", + ", ".join(missing_from_hugo)) + + # No top_level item should have leaked into hugo/. Two names are expected to + # appear under hugo/ and are NOT leaks: 'hugo' itself (the move target), and + # '.gitignore' — execute_reorg.py deliberately SPLITS the root .gitignore, + # writing a routed subset to hugo/.gitignore (verified by check_gitignore_split + # and the "present at root and in hugo/" check below). + expected_under_hugo = {"hugo", ".gitignore"} + leaked = sorted( + n for n in top_level + if n not in expected_under_hugo and (hugo_dir / n).exists() + ) + if leaked: + record("FAIL", "layout: top-level items leaked into hugo/", + ", ".join(leaked)) + else: + record("PASS", "layout: no top-level items leaked into hugo/") + + # Critical anchors that must still exist at the root after the move. + # package.json/node_modules/yarn.lock and go.mod/go.sum belong under hugo/ + # (each site owns its own Node setup; the Go module powers Hugo Modules), + # so they are deliberately NOT listed here. + must_stay = ["astro", ".husky", ".github", ".vale.ini"] + absent = [n for n in must_stay if not (repo_root / n).exists()] + if absent: + record("FAIL", "layout: expected top-level items are missing", + ", ".join(absent)) + else: + record("PASS", "layout: top-level anchors intact", + ", ".join(must_stay)) + + # Both .gitignore files should exist (execute_reorg.py splits one into two; + # check_gitignore_split below verifies the split routed correctly). + if (repo_root / ".gitignore").exists() and (hugo_dir / ".gitignore").exists(): + record("PASS", "layout: .gitignore present at root and in hugo/") + else: + record("FAIL", "layout: missing a .gitignore copy", + "expected both ./.gitignore and ./hugo/.gitignore") + + +def check_gitignore_split(top_level, moves_to_hugo): + """Verify execute_reorg.py routed .gitignore rules to the correct side. + + A .gitignore is relative to its own directory, so after the split no + surviving rule should point at a path that lives on the OTHER side: + - root .gitignore must not carry a rule whose first segment moved to hugo/ + - hugo/.gitignore must not carry a rule whose first segment stays at root + Generic globs (first segment in neither config list) legitimately live in + both files, so they are ignored here. + """ + def first_segment(line): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + return None + body = stripped[1:] if stripped.startswith("!") else stripped + return body.lstrip("/").split("/", 1)[0] + + def leaks(path, wrong_side): + out = [] + for raw in path.read_text().splitlines(): + seg = first_segment(raw) + if seg in wrong_side: + out.append(f"{path.name}: {raw.strip()} (segment {seg!r})") + return out + + root_gi = repo_root / ".gitignore" + hugo_gi = hugo_dir / ".gitignore" + if not (root_gi.exists() and hugo_gi.exists()): + record("SKIP", "gitignore: split not verifiable (a .gitignore is missing)") + return + + problems = leaks(root_gi, moves_to_hugo) + leaks(hugo_gi, top_level - {"hugo"}) + if problems: + record("FAIL", "gitignore: rules survive on the wrong side of the split", + f"{len(problems)}:\n " + "\n ".join(problems[:20])) + else: + record("PASS", "gitignore: no moved-path rule left at root, " + "no root-path rule left in hugo/") + + +def check_workflows(moves_to_hugo): + """Flag references to moved paths (directories AND files) lacking hugo/. + + Routes every path-like token by its first segment, mirroring execute_reorg.py: + a token whose first segment moved into hugo/ must be hugo/-prefixed (after a + correct reorg its first segment is 'hugo', so it no longer matches). A token + counts as a path reference when it contains a '/', or when the whole token is + the exact name of a moved file — so a bare word like 'content' in prose isn't + flagged, but a standalone 'babel.config.js' is. + + Moved files whose names also exist under astro/ (package.json, yarn.lock, + node_modules, .nvmrc) are excluded from bare-name matching: a standalone + 'package.json' is genuinely ambiguous. They are still validated when they + appear inside a slashed path routed by first segment. + """ + workflows_dir = repo_root / ".github" / "workflows" + if not workflows_dir.exists(): + record("SKIP", "workflows: .github/workflows/ not found") + return + + moved_files = { + n for n in moves_to_hugo + if (hugo_dir / n).is_file() and not (repo_root / "astro" / n).exists() + } + + # Lines that legitimately reference a moved name but are NOT paths to fix + # (illustrative prose, external URLs). Keyed by workflow filename; a line is + # exempt if it contains any of the listed markers. + allowlist = { + # Security-doc example of how untrusted paths are reported, not a real path. + "claude_review.yml": ["__untrusted/content/"], + } + + # A path-like token: a maximal run of path/glob characters. Whitespace, + # quotes, ':', '@' and '!' all act as boundaries, so a leading '!' negation + # or surrounding quotes fall away naturally. + token_re = re.compile(r"[A-Za-z0-9_.\-*/]+") + + def first_segment(token): + """(first path segment, ./-stripped token) for routing.""" + t = token + while t.startswith("./"): + t = t[2:] + return t.split("/", 1)[0], t + + suspects = [] + for yml in sorted(workflows_dir.glob("*.yml")): + exempt = allowlist.get(yml.name, []) + for lineno, line in enumerate(yml.read_text().splitlines(), 1): + if any(marker in line for marker in exempt): + continue + for token in token_re.findall(line): + seg, normalized = first_segment(token) + # Only treat a token as a path reference if it's an actual path + # (has a '/') or is exactly the name of a moved file. + if "/" not in normalized and normalized not in moved_files: + continue + if seg in moves_to_hugo: + suspects.append(f"{yml.name}:{lineno}: {line.strip()}") + break + + # De-duplicate while keeping order. + seen = set() + unique = [s for s in suspects if not (s in seen or seen.add(s))] + + if unique: + record("FAIL", "workflows: unprefixed moved paths found", + f"{len(unique)} line(s):\n " + "\n ".join(unique[:20])) + else: + record("PASS", "workflows: all moved paths (dirs and files) are hugo/-prefixed") + + +def check_workflow_path_filters(moves_to_hugo): + """Parse each workflow's on.*.paths filters and assert moved paths are prefixed. + + Unlike paths embedded in `run:` shell (which are opaque string scalars to a + parser), `on..paths` / `paths-ignore` are structured YAML list values, + so we can validate them precisely. Each entry is routed by its first path + segment: if that segment moved into hugo/, the entry must be hugo/-prefixed. + + Footgun handled: under YAML 1.1, PyYAML loads the `on:` key as the boolean + True, not the string "on" — so we look it up under both keys. + """ + workflows_dir = repo_root / ".github" / "workflows" + if not workflows_dir.exists(): + record("SKIP", "workflows: .github/workflows/ not found (path filters)") + return + + def first_segment(entry): + # Strip a leading '!' negation and any root anchor, then take segment 0. + p = entry[1:] if entry.startswith("!") else entry + return p.lstrip("/").split("/", 1)[0] + + problems = [] + for yml in sorted(workflows_dir.glob("*.yml")): + try: + doc = yaml.safe_load(yml.read_text()) + except yaml.YAMLError as exc: + record("WARN", f"workflows: {yml.name} did not parse as YAML", + str(exc).splitlines()[0][:120]) + continue + if not isinstance(doc, dict): + continue + triggers = doc.get("on", doc.get(True)) # `on:` -> True under YAML 1.1 + if not isinstance(triggers, dict): + continue + for event, spec in triggers.items(): + if not isinstance(spec, dict): + continue + for key in ("paths", "paths-ignore"): + for entry in spec.get(key) or []: + if not isinstance(entry, str): + continue + seg = first_segment(entry) + anchored = entry.lstrip("!").lstrip("/") + if seg in moves_to_hugo and not anchored.startswith("hugo/"): + problems.append(f"{yml.name}: on.{event}.{key}: {entry}") + + if problems: + record("FAIL", "workflows: on.*.paths filters missing hugo/ prefix", + f"{len(problems)}:\n " + "\n ".join(problems[:20])) + else: + record("PASS", "workflows: on.*.paths filters are hugo/-prefixed") + + +def check_codeowners(): + """Flag concrete (non-glob) CODEOWNERS patterns that don't resolve correctly. + + Stale entries pointing at files absent from both hugo/ and the repo root are + reported as informational notes but never failed. The two cases for a + non-resolving concrete pattern: + + - hugo/-prefixed AND its de-prefixed path resolves at the repo root + -> REGRESSION: the pattern was prefixed but the file is still at + root (the move didn't happen / went to the wrong place). FAIL. + - anything else that doesn't resolve -> absent from both hugo/ and root. + Reported as an informational note, never a failure. + """ + codeowners = repo_root / ".github" / "CODEOWNERS" + if not codeowners.exists(): + record("SKIP", "codeowners: .github/CODEOWNERS not found") + return + + regressions = [] + preexisting = [] + for line in codeowners.read_text().splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + pattern = stripped.split()[0] + if pattern == "*": + continue + # Skip glob patterns; we can't resolve them to a single path. + if any(ch in pattern for ch in "*?[]"): + continue + # Leading slash in CODEOWNERS is repo-root-anchored. + rel = pattern.lstrip("/") + if (repo_root / rel).exists(): + continue + # Doesn't resolve. Is it something the reorg broke, or pre-existing rot? + if rel.startswith("hugo/") and (repo_root / rel[len("hugo/"):]).exists(): + regressions.append(pattern) + else: + preexisting.append(pattern) + + if regressions: + record("FAIL", "codeowners: patterns the reorg moved to hugo/ but whose " + "file is still at root", + f"{len(regressions)}:\n " + "\n ".join(regressions[:20])) + elif preexisting: + record("PASS", "codeowners: no reorg-introduced breakage", + f"{len(preexisting)} pre-existing dangling entry(ies) ignored " + f"(absent on master too): e.g. {', '.join(preexisting[:3])}") + else: + record("PASS", "codeowners: all concrete patterns resolve on disk") + + +def check_codeowners_prefixing(moves_to_hugo): + """Every CODEOWNERS pattern whose first segment moved into hugo/ must carry + the hugo/ prefix — globs included. + + check_codeowners() above only proves that *concrete* paths still RESOLVE on + disk; it skips every glob (layouts/shortcodes/**/*.md) and never confirms a + moved pattern was actually repathed. Here we route each pattern by its first + path segment exactly as execute_reorg.py's route_codeowners_pattern does and assert + that a moved segment is prefixed. After a correct reorg the pattern's first + segment is 'hugo', so a flagged pattern means a substitution was missed. + """ + codeowners = repo_root / ".github" / "CODEOWNERS" + if not codeowners.exists(): + record("SKIP", "codeowners: .github/CODEOWNERS not found (prefixing)") + return + + def first_segment(pattern): + body = pattern[1:] if pattern.startswith("/") else pattern # un-anchor + seg = body.split("/", 1)[0] + # execute_reorg.py normalizes the '.local' typo to 'local' before routing. + return "local" if seg == ".local" else seg + + unprefixed = [] + for line in codeowners.read_text().splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + pattern = stripped.split()[0] + if pattern == "*": + continue + if first_segment(pattern) not in moves_to_hugo: + continue + if not pattern.lstrip("/").startswith("hugo/"): + unprefixed.append(pattern) + + if unprefixed: + record("FAIL", "codeowners: moved patterns missing the hugo/ prefix", + f"{len(unprefixed)} (globs included):\n " + + "\n ".join(unprefixed[:20])) + else: + record("PASS", "codeowners: every moved pattern is hugo/-prefixed " + "(globs included)") + + +def run_hook(script_name): + """Run a .husky hook script from the repo root; return CompletedProcess.""" + return subprocess.run( + ["python3", str(repo_root / ".husky" / script_name)], + cwd=repo_root, + capture_output=True, + text=True, + ) + + +def check_husky_circular_aliases(): + """Plant a self-aliasing page and confirm the hook rejects it.""" + rel = f"hugo/content/en/{MARKER}_alias/selftest.md" + target = repo_root / rel + if target.exists(): + record("WARN", "husky: circular-aliases test skipped (temp path exists)", rel) + return + + # An alias equal to the file's own location is the circular case. + body = ( + "---\n" + "title: Reorg Harness Self Test\n" + "aliases:\n" + f" - /{MARKER}_alias/selftest\n" + "---\n\n" + "Temporary file created by astro_reorg/validate_reorg.py.\n" + ) + try: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + git("add", "-f", rel) + proc = run_hook("check-circular-aliases.py") + if proc.returncode != 0 and "circular" in (proc.stdout + proc.stderr).lower(): + record("PASS", "husky: circular-aliases hook rejects bad input") + else: + record("FAIL", "husky: circular-aliases hook did NOT reject bad input", + "hook may not be referencing hugo/content/en/ " + f"(exit={proc.returncode})") + finally: + git("reset", "-q", "HEAD", rel) + if target.exists(): + target.unlink() + _rmdir_if_empty(target.parent) + + +def check_husky_section_index(): + """Plant a new top-level section with no _index.md and confirm rejection.""" + section = f"{MARKER}_section" + rel = f"hugo/content/en/{section}/page.md" + target = repo_root / rel + if target.exists() or (hugo_dir / "content" / "en" / section).exists(): + record("WARN", "husky: section-index test skipped (temp path exists)", rel) + return + + body = "---\ntitle: Reorg Harness Page\nprivate: true\n---\n\nTemp page.\n" + try: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + git("add", "-f", rel) + proc = run_hook("check-section-index.py") + if proc.returncode != 0 and "_index.md" in (proc.stdout + proc.stderr): + record("PASS", "husky: section-index hook rejects missing _index.md") + else: + record("FAIL", "husky: section-index hook did NOT reject bad input", + "hook may not be referencing hugo/content/en/ " + f"(exit={proc.returncode})") + finally: + git("reset", "-q", "HEAD", rel) + if target.exists(): + target.unlink() + _rmdir_if_empty(target.parent) + + +def check_husky_cdocs_gitignore(): + """ + Append a harness pattern to hugo/content/.gitignore, force-track a matching + compiled file (with a .mdoc.md sibling), and confirm the hook flags it. + """ + content_gitignore = hugo_dir / "content" / ".gitignore" + if not content_gitignore.exists(): + record("SKIP", "husky: cdocs-gitignore test skipped (no hugo/content/.gitignore)") + return + + pattern = f"/en/{MARKER}_cdocs.md" + compiled_rel = f"hugo/content/en/{MARKER}_cdocs.md" + source_rel = f"hugo/content/en/{MARKER}_cdocs.mdoc.md" + compiled = repo_root / compiled_rel + source = repo_root / source_rel + if compiled.exists() or source.exists(): + record("WARN", "husky: cdocs-gitignore test skipped (temp path exists)", + compiled_rel) + return + + original_gitignore = content_gitignore.read_text() + try: + content_gitignore.write_text( + original_gitignore.rstrip("\n") + f"\n{pattern}\n" + ) + source.write_text("Temp Cdocs source.\n") + compiled.write_text("Temp compiled Cdocs output.\n") + git("add", "-f", compiled_rel) # force past the gitignore to simulate the mistake + proc = run_hook("check-cdocs-gitignore.py") + if proc.returncode != 0 and MARKER in (proc.stdout + proc.stderr): + record("PASS", "husky: cdocs-gitignore hook flags tracked compiled file") + else: + record("FAIL", "husky: cdocs-gitignore hook did NOT flag tracked file", + "hook may not be referencing hugo/content/.gitignore " + f"(exit={proc.returncode})") + finally: + git("reset", "-q", "HEAD", compiled_rel) + for p in (compiled, source): + if p.exists(): + p.unlink() + content_gitignore.write_text(original_gitignore) + + +def _rmdir_if_empty(path): + """Remove a directory only if it is empty (safety guard).""" + try: + path.rmdir() + except OSError: + pass + + +def check_build_presence(): + """Static check; the real build is the manual `make start` in todo #5.""" + # Hugo's build entrypoints all need to be co-located under hugo/: the Makefile, + # the Node manifest, and go.mod (required by Hugo Modules at the project root). + required = ["Makefile", "package.json", "go.mod"] + missing = [n for n in required if not (hugo_dir / n).exists()] + if not missing: + record("PASS", "build: hugo/{Makefile,package.json,go.mod} present", + "run `cd hugo && make start` (todo #5) to verify the full build") + else: + record("FAIL", "build: missing Hugo build entrypoint(s) under hugo/", + ", ".join(f"hugo/{n}" for n in missing)) + + +def check_rollback_roundtrip(): + """execute_reorg.py then rollback.py must restore the tree byte-for-byte. + + This is the only check that exercises rollback.py at all. Rather than + mutate the live repo (and risk leaving it half-reorganized if a step fails), + it builds a small throwaway git repo holding one representative item for + every code path the two scripts touch — a mixed .gitignore (rules that route + to hugo/, to root, and to both), a workflow with substitutable paths, a + CODEOWNERS with a moved rule, a husky hook with a substitutable path, plus a + few moved/stayed files — then: + + snapshot -> run execute_reorg.py (answering y to every prompt) + -> run rollback.py -> snapshot again + -> assert byte-identical. + + It specifically catches the easy-to-miss case where execute_reorg.py edits a file in + place (the root .gitignore split) that rollback must restore from git rather + than merely delete. + """ + if shutil.which("python3") is None or shutil.which("git") is None: + record("SKIP", "rollback: python3/git not both available") + return + + workdir = tempfile.mkdtemp(prefix=f"{MARKER}_rollback_") + work = Path(workdir) + + def write(rel, text): + p = work / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text) + + def git_work(*args): + return subprocess.run(["git", *args], cwd=work, + capture_output=True, text=True) + + def snapshot(): + """relpath -> bytes for every file under work/, excluding .git/.""" + tree = {} + for path in sorted(work.rglob("*")): + if path.is_dir(): + continue + rel = path.relative_to(work) + if rel.parts and rel.parts[0] == ".git": + continue + tree[str(rel)] = path.read_bytes() + return tree + + try: + # Representative fixture. Every top-level name here must appear in + # astro_reorg/config.yaml or execute_reorg.py refuses to run; the mix below routes + # across moves_to_hugo, top_level, and generic-glob cases. + write(".gitignore", + "# build\n" + "public/*\n" + "data/generated\n" + "content/en/api/**/*.go\n" + "node_modules\n" + "\n" + "# generic (kept in both)\n" + "*.log\n" + "\n" + "# root-only tooling\n" + ".github/preview-links-template.md\n") + write(".github/workflows/sample.yml", + "name: sample\n" + "on:\n" + " pull_request:\n" + " paths:\n" + " - 'content/en/**/*.md'\n" + "jobs:\n" + " build:\n" + " runs-on: ubuntu-latest\n" + " steps:\n" + " - run: python local/bin/foo.py\n") + write(".github/CODEOWNERS", + "* @DataDog/documentation\n" + "/content/ @DataDog/team-a\n" + "data/ @DataDog/team-b\n" + "README.md @DataDog/team-c\n") + write(".husky/check-sample.py", + "from pathlib import Path\n" + 'repo_pattern = "content"\n' + "p = Path('content/en')\n") + write("README.md", "# Fixture\n") + write("astro/package.json", '{"name": "astro-fixture"}\n') + write("content/en/page.md", "---\ntitle: Page\n---\n\nBody.\n") + write("data/sample.yaml", "key: value\n") + write("Makefile", "start:\n\techo build\n") + write("package.json", '{"name": "hugo-fixture"}\n') + write("go.mod", "module example.com/fixture\n\ngo 1.21\n") + write("babel.config.js", "module.exports = {};\n") + + # The scripts resolve repo_root as their parent's parent, so place them + # under an astro_reorg/ subfolder that mirrors the real repo layout. + (work / "astro_reorg").mkdir() + for tool in ("execute_reorg.py", "rollback.py", "config.yaml"): + shutil.copy2(repo_root / "astro_reorg" / tool, work / "astro_reorg" / tool) + + git_work("init", "-q") + git_work("add", "-A") + commit = git_work("-c", "user.email=harness@example.com", + "-c", "user.name=reorg harness", + "-c", "commit.gpgsign=false", + "commit", "-q", "-m", "fixture") + if commit.returncode != 0: + record("FAIL", "rollback: could not commit fixture repo", + (commit.stderr or commit.stdout).strip()[:200]) + return + + before = snapshot() + + # execute_reorg.py is interactive (single y/N per mutation section); answer y to + # all. Far more lines than prompts is fine — the extras are ignored. + reorg = subprocess.run( + ["python3", str(work / "astro_reorg" / "execute_reorg.py")], + cwd=work, capture_output=True, text=True, input="y\n" * 100, + ) + if reorg.returncode != 0 or not (work / "hugo").exists(): + record("FAIL", "rollback: execute_reorg.py failed on the fixture", + (reorg.stderr or reorg.stdout).strip()[:200]) + return + + rollback = subprocess.run( + ["python3", str(work / "astro_reorg" / "rollback.py")], + cwd=work, capture_output=True, text=True, + ) + if rollback.returncode != 0 or (work / "hugo").exists(): + record("FAIL", "rollback: rollback.py failed on the fixture", + (rollback.stderr or rollback.stdout).strip()[:200]) + return + + after = snapshot() + + added = sorted(set(after) - set(before)) + removed = sorted(set(before) - set(after)) + changed = sorted(p for p in before.keys() & after.keys() + if before[p] != after[p]) + if added or removed or changed: + diffs = ([f"+ {p}" for p in added] + + [f"- {p}" for p in removed] + + [f"~ {p} (in-place edit not reverted)" for p in changed]) + record("FAIL", "rollback: tree not byte-identical after reorg + rollback", + f"{len(diffs)} diff(s):\n " + "\n ".join(diffs[:20])) + else: + record("PASS", "rollback: reorg + rollback restores the tree byte-for-byte", + f"{len(before)} file(s) round-tripped, incl. the .gitignore split") + finally: + shutil.rmtree(work, ignore_errors=True) diff --git a/astro_reorg/validate_reorg.py b/astro_reorg/validate_reorg.py index 8f1c00e7a00..68521c5ac57 100644 --- a/astro_reorg/validate_reorg.py +++ b/astro_reorg/validate_reorg.py @@ -3,787 +3,31 @@ Post-reorg validation harness. Run this AFTER execute_reorg.py, from the repo root, on a feature branch (not master). -It verifies that the things the reorg could silently break still work: - - A. Layout - hugo/ holds the moved dirs; nothing moved is left at root; - hugo/ and astro/ each self-own a package.json and no Hugo - Node/build file lingers at the root competing with Astro. - B. Workflows - no .github/workflows/ file references a moved path (dir OR - file) without the hugo/ prefix, in shell scalars and in - structured on.*.paths filters (catches gaps in execute_reorg.py). - C. CODEOWNERS - every concrete path pattern still resolves on disk, and - every moved pattern (globs included) carries the hugo/ prefix. - D. Husky hooks - each pre-commit check still REJECTS a known-bad input planted - at the new hugo/ path. If a hook wasn't repathed it inspects - the now-missing content/en/ and passes vacuously -> we fail it. - E. Vale - vale still flags a known violation using the Datadog style, - proving StylesPath resolves from the new content location. - F. Hugo build - static presence check only; run `make start` manually (todo #5). - G. Rollback - on a throwaway repo, execute_reorg.py then rollback.py must - restore the tree byte-for-byte (the only test of rollback). - -The harness is non-destructive. Every file it creates lives under a -'__reorg_harness__' prefix, and every change (staged paths, gitignore edits) is -reverted in a finally block. It never commits. +The harness is non-destructive: every file it creates lives under a '__reorg_harness__' +prefix, and every change (staged paths, gitignore edits) is reverted in a finally block. +It never commits. """ -import os -import re -import shutil -import subprocess import sys -import tempfile -from pathlib import Path - -try: - import yaml -except ImportError: - print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) - sys.exit(1) - -repo_root = Path(__file__).parent.parent -hugo_dir = repo_root / "hugo" - -# Distinctive marker so anything we create is obvious and easy to clean up. -MARKER = "__reorg_harness__" - -# (status, name, detail) tuples collected as checks run. -results = [] - - -def record(status, name, detail=""): - """Record a check outcome. status is one of PASS, FAIL, SKIP, WARN.""" - results.append((status, name, detail)) - symbol = {"PASS": "✅", "FAIL": "❌", "SKIP": "⏭ ", "WARN": "⚠ "}[status] - line = f"{symbol} {status:4} {name}" - if detail: - line += f"\n {detail}" - print(line) - - -def git(*args, check=False): - """Run a git command from the repo root and return the CompletedProcess.""" - return subprocess.run( - ["git", *args], - cwd=repo_root, - capture_output=True, - text=True, - check=check, - ) - - -def load_config(): - """Load astro_reorg/config.yaml and return (top_level, moves_to_hugo) name sets.""" - config_path = Path(__file__).parent / "config.yaml" - with config_path.open() as f: - config = yaml.safe_load(f) - return set(config.get("top_level", [])), set(config.get("moves_to_hugo", [])) - - -# -------------------------------------------------------------------------- -# A. Layout -# -------------------------------------------------------------------------- - -def check_layout(top_level, moves_to_hugo): - """hugo/ holds the moved dirs; nothing that moved is left at the root.""" - # Moved items that the reorg actually had to relocate (present before). - expected_in_hugo = sorted(n for n in moves_to_hugo if (hugo_dir / n).exists()) - missing_from_hugo = sorted( - n for n in moves_to_hugo - if (repo_root / n).exists() and not (hugo_dir / n).exists() - ) - - # A moved name still sitting at the root means the move didn't happen. - still_at_root = sorted( - n for n in moves_to_hugo if (repo_root / n).exists() - ) - if still_at_root: - record("FAIL", "layout: moved items remain at repo root", - ", ".join(still_at_root)) - else: - record("PASS", "layout: no moved items remain at repo root", - f"{len(expected_in_hugo)} item(s) confirmed under hugo/") - - if missing_from_hugo: - record("FAIL", "layout: moved items missing under hugo/", - ", ".join(missing_from_hugo)) - - # No top_level item should have leaked into hugo/. 'hugo' itself is listed - # in top_level (it IS the move target) so it is excluded. - leaked = sorted( - n for n in top_level - if n != "hugo" and (hugo_dir / n).exists() - ) - if leaked: - record("FAIL", "layout: top-level items leaked into hugo/", - ", ".join(leaked)) - else: - record("PASS", "layout: no top-level items leaked into hugo/") - - # Critical anchors that must still exist at the root after the move. - # NB: package.json/node_modules/yarn.lock and go.mod/go.sum now move into - # hugo/ (each site owns its own Node setup; the Go module powers Hugo - # Modules), so they are deliberately NOT listed here. - must_stay = ["astro", ".husky", ".github", ".vale.ini"] - absent = [n for n in must_stay if not (repo_root / n).exists()] - if absent: - record("FAIL", "layout: expected top-level items are missing", - ", ".join(absent)) - else: - record("PASS", "layout: top-level anchors intact", - ", ".join(must_stay)) - - # Both .gitignore files should exist (execute_reorg.py splits one into two; - # check_gitignore_split below verifies the split routed correctly). - if (repo_root / ".gitignore").exists() and (hugo_dir / ".gitignore").exists(): - record("PASS", "layout: .gitignore present at root and in hugo/") - else: - record("FAIL", "layout: missing a .gitignore copy", - "expected both ./.gitignore and ./hugo/.gitignore") - - -def check_gitignore_split(top_level, moves_to_hugo): - """Verify execute_reorg.py routed .gitignore rules to the correct side. - - A .gitignore is relative to its own directory, so after the split no - surviving rule should point at a path that lives on the OTHER side: - - root .gitignore must not carry a rule whose first segment moved to hugo/ - - hugo/.gitignore must not carry a rule whose first segment stays at root - Generic globs (first segment in neither config list) legitimately live in - both files, so they are ignored here. - """ - def first_segment(line): - stripped = line.strip() - if not stripped or stripped.startswith("#"): - return None - body = stripped[1:] if stripped.startswith("!") else stripped - return body.lstrip("/").split("/", 1)[0] - - def leaks(path, wrong_side): - out = [] - for raw in path.read_text().splitlines(): - seg = first_segment(raw) - if seg in wrong_side: - out.append(f"{path.name}: {raw.strip()} (segment {seg!r})") - return out - - root_gi = repo_root / ".gitignore" - hugo_gi = hugo_dir / ".gitignore" - if not (root_gi.exists() and hugo_gi.exists()): - record("SKIP", "gitignore: split not verifiable (a .gitignore is missing)") - return - - problems = leaks(root_gi, moves_to_hugo) + leaks(hugo_gi, top_level - {"hugo"}) - if problems: - record("FAIL", "gitignore: rules survive on the wrong side of the split", - f"{len(problems)}:\n " + "\n ".join(problems[:20])) - else: - record("PASS", "gitignore: no moved-path rule left at root, " - "no root-path rule left in hugo/") - - -def check_site_separation(moves_to_hugo): - """The two sites must own non-overlapping Node/build setups. - - The point of the reorg is a clean hugo/ vs astro/ split: each site has its - OWN Node manifest, and any name the two sites share must not also linger at - the root where their copies would collide. check_layout already fails on ANY - moved item left at root (against the full config), so this check adds only - what that doesn't cover — and derives its clash set from the config rather - than a hand-maintained list: - - astro/package.json and hugo/package.json both exist (each self-owns) - - no name that moved into hugo/ AND is also owned by astro/ remains at root - """ - problems = [] - - # Each site self-owns its Node manifest. (package.json is in moves_to_hugo, - # so it belongs to Hugo; astro/ must carry its own copy.) - for site in ("hugo", "astro"): - if not (repo_root / site / "package.json").exists(): - problems.append(f"missing {site}/package.json") - - # A name that moved into hugo/ AND that astro/ also owns is a shared - # toolchain file (package.json, node_modules, yarn.lock, ...). If such a - # name is ALSO present at the root, the two sites' copies collide. The clash - # set is derived from moves_to_hugo intersected with astro/'s actual - # contents — no hand-maintained file list to drift from the config. - shared_with_astro = sorted( - n for n in moves_to_hugo if (repo_root / "astro" / n).exists() - ) - leaked = [n for n in shared_with_astro if (repo_root / n).exists()] - if leaked: - problems.append("left at root, colliding with astro/: " + ", ".join(leaked)) - - if problems: - record("FAIL", "separation: hugo/ and astro/ Node setups overlap or leak", - "; ".join(problems)) - else: - record("PASS", "separation: hugo/ and astro/ each self-own package.json; " - "no shared toolchain file left at root") - - -# -------------------------------------------------------------------------- -# B. Workflow paths -# -------------------------------------------------------------------------- - -def check_workflows(moves_to_hugo): - """Flag references to moved paths (directories AND files) lacking hugo/. - - This previously scanned only directory names ('.' not in n) using a - trailing-slash token, so a workflow that referenced a moved *file* — - babel.config.js, markdoc.config.json, the Makefile, go.mod — was never - validated. We now route every path-like token by its first segment exactly - as execute_reorg.py does: a token whose first segment moved into hugo/ must be - hugo/-prefixed (after a correct reorg its first segment is 'hugo', so it no - longer matches). A token counts as a path reference when it contains a '/', - or when the whole token is the exact name of a moved file — so a bare word - like 'content' in prose isn't flagged, but a standalone 'babel.config.js' is. - - A moved file whose name ALSO exists under astro/ (package.json, yarn.lock, - node_modules, .nvmrc) is left out of the bare-name match: a standalone - 'package.json' is genuinely ambiguous (it may be Astro's), which is exactly - why execute_reorg.py never blind-substitutes those names. They are still validated - when they appear inside a slashed path routed by its first segment. - """ - workflows_dir = repo_root / ".github" / "workflows" - if not workflows_dir.exists(): - record("SKIP", "workflows: .github/workflows/ not found") - return - - moved_files = { - n for n in moves_to_hugo - if (hugo_dir / n).is_file() and not (repo_root / "astro" / n).exists() - } - - # Lines that legitimately reference a moved name but are NOT paths to fix - # (illustrative prose, external URLs). Keyed by workflow filename; a line is - # exempt if it contains any of the listed markers. - allowlist = { - # Security-doc example of how untrusted paths are reported, not a real path. - "claude_review.yml": ["__untrusted/content/"], - } - - # A path-like token: a maximal run of path/glob characters. Whitespace, - # quotes, ':', '@' and '!' all act as boundaries, so a leading '!' negation - # or surrounding quotes fall away naturally. - token_re = re.compile(r"[A-Za-z0-9_.\-*/]+") - - def first_segment(token): - """(first path segment, ./-stripped token) for routing.""" - t = token - while t.startswith("./"): - t = t[2:] - return t.split("/", 1)[0], t - - suspects = [] - for yml in sorted(workflows_dir.glob("*.yml")): - exempt = allowlist.get(yml.name, []) - for lineno, line in enumerate(yml.read_text().splitlines(), 1): - if any(marker in line for marker in exempt): - continue - for token in token_re.findall(line): - seg, normalized = first_segment(token) - # Only treat a token as a path reference if it's an actual path - # (has a '/') or is exactly the name of a moved file. - if "/" not in normalized and normalized not in moved_files: - continue - if seg in moves_to_hugo: - suspects.append(f"{yml.name}:{lineno}: {line.strip()}") - break - - # De-duplicate while keeping order. - seen = set() - unique = [s for s in suspects if not (s in seen or seen.add(s))] - - if unique: - record("FAIL", "workflows: unprefixed moved paths found", - f"{len(unique)} line(s):\n " + "\n ".join(unique[:20])) - else: - record("PASS", "workflows: all moved paths (dirs and files) are hugo/-prefixed") - - -def check_workflow_path_filters(moves_to_hugo): - """Parse each workflow's on.*.paths filters and assert moved paths are prefixed. - - Unlike paths embedded in `run:` shell (which are opaque string scalars to a - parser), `on..paths` / `paths-ignore` are structured YAML list values, - so we can validate them precisely. Each entry is routed by its first path - segment: if that segment moved into hugo/, the entry must be hugo/-prefixed. - - Footgun handled: under YAML 1.1, PyYAML loads the `on:` key as the boolean - True, not the string "on" — so we look it up under both keys. - """ - workflows_dir = repo_root / ".github" / "workflows" - if not workflows_dir.exists(): - record("SKIP", "workflows: .github/workflows/ not found (path filters)") - return - - def first_segment(entry): - # Strip a leading '!' negation and any root anchor, then take segment 0. - p = entry[1:] if entry.startswith("!") else entry - return p.lstrip("/").split("/", 1)[0] - - problems = [] - for yml in sorted(workflows_dir.glob("*.yml")): - try: - doc = yaml.safe_load(yml.read_text()) - except yaml.YAMLError as exc: - record("WARN", f"workflows: {yml.name} did not parse as YAML", - str(exc).splitlines()[0][:120]) - continue - if not isinstance(doc, dict): - continue - triggers = doc.get("on", doc.get(True)) # `on:` -> True under YAML 1.1 - if not isinstance(triggers, dict): - continue - for event, spec in triggers.items(): - if not isinstance(spec, dict): - continue - for key in ("paths", "paths-ignore"): - for entry in spec.get(key) or []: - if not isinstance(entry, str): - continue - seg = first_segment(entry) - anchored = entry.lstrip("!").lstrip("/") - if seg in moves_to_hugo and not anchored.startswith("hugo/"): - problems.append(f"{yml.name}: on.{event}.{key}: {entry}") - - if problems: - record("FAIL", "workflows: on.*.paths filters missing hugo/ prefix", - f"{len(problems)}:\n " + "\n ".join(problems[:20])) - else: - record("PASS", "workflows: on.*.paths filters are hugo/-prefixed") - - -# -------------------------------------------------------------------------- -# C. CODEOWNERS -# -------------------------------------------------------------------------- -def check_codeowners(): - """Every concrete (non-glob) path pattern should resolve on disk.""" - codeowners = repo_root / ".github" / "CODEOWNERS" - if not codeowners.exists(): - record("SKIP", "codeowners: .github/CODEOWNERS not found") - return +from helpers import ( + hugo_dir, + git, + load_config, + results, + check_layout, + check_gitignore_split, + check_workflows, + check_workflow_path_filters, + check_codeowners, + check_codeowners_prefixing, + check_husky_circular_aliases, + check_husky_section_index, + check_husky_cdocs_gitignore, + check_build_presence, + check_rollback_roundtrip, +) - missing = [] - for line in codeowners.read_text().splitlines(): - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - pattern = stripped.split()[0] - if pattern == "*": - continue - # Skip glob patterns; we can't resolve them to a single path. - if any(ch in pattern for ch in "*?[]"): - continue - # Leading slash in CODEOWNERS is repo-root-anchored. - rel = pattern.lstrip("/") - if not (repo_root / rel).exists(): - missing.append(pattern) - - if missing: - record("FAIL", "codeowners: patterns that no longer resolve", - f"{len(missing)}:\n " + "\n ".join(missing[:20])) - else: - record("PASS", "codeowners: all concrete patterns resolve on disk") - - -def check_codeowners_prefixing(moves_to_hugo): - """Every CODEOWNERS pattern whose first segment moved into hugo/ must carry - the hugo/ prefix — globs included. - - check_codeowners() above only proves that *concrete* paths still RESOLVE on - disk; it skips every glob (layouts/shortcodes/**/*.md) and never confirms a - moved pattern was actually repathed. Here we route each pattern by its first - path segment exactly as execute_reorg.py's route_codeowners_pattern does and assert - that a moved segment is prefixed. After a correct reorg the pattern's first - segment is 'hugo', so a flagged pattern means a substitution was missed. - """ - codeowners = repo_root / ".github" / "CODEOWNERS" - if not codeowners.exists(): - record("SKIP", "codeowners: .github/CODEOWNERS not found (prefixing)") - return - - def first_segment(pattern): - body = pattern[1:] if pattern.startswith("/") else pattern # un-anchor - seg = body.split("/", 1)[0] - # execute_reorg.py normalizes the '.local' typo to 'local' before routing. - return "local" if seg == ".local" else seg - - unprefixed = [] - for line in codeowners.read_text().splitlines(): - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - pattern = stripped.split()[0] - if pattern == "*": - continue - if first_segment(pattern) not in moves_to_hugo: - continue - if not pattern.lstrip("/").startswith("hugo/"): - unprefixed.append(pattern) - - if unprefixed: - record("FAIL", "codeowners: moved patterns missing the hugo/ prefix", - f"{len(unprefixed)} (globs included):\n " - + "\n ".join(unprefixed[:20])) - else: - record("PASS", "codeowners: every moved pattern is hugo/-prefixed " - "(globs included)") - - -# -------------------------------------------------------------------------- -# D. Husky behavioral checks -# -------------------------------------------------------------------------- - -def run_hook(script_name): - """Run a .husky hook script from the repo root; return CompletedProcess.""" - return subprocess.run( - ["python3", str(repo_root / ".husky" / script_name)], - cwd=repo_root, - capture_output=True, - text=True, - ) - - -def check_husky_circular_aliases(): - """Plant a self-aliasing page and confirm the hook rejects it.""" - rel = f"hugo/content/en/{MARKER}_alias/selftest.md" - target = repo_root / rel - if target.exists(): - record("WARN", "husky: circular-aliases test skipped (temp path exists)", rel) - return - - # An alias equal to the file's own location is the circular case. - body = ( - "---\n" - "title: Reorg Harness Self Test\n" - "aliases:\n" - f" - /{MARKER}_alias/selftest\n" - "---\n\n" - "Temporary file created by astro_reorg/validate_reorg.py.\n" - ) - try: - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(body) - git("add", "-f", rel) - proc = run_hook("check-circular-aliases.py") - if proc.returncode != 0 and "circular" in (proc.stdout + proc.stderr).lower(): - record("PASS", "husky: circular-aliases hook rejects bad input") - else: - record("FAIL", "husky: circular-aliases hook did NOT reject bad input", - "hook may still point at the old content/en/ path " - f"(exit={proc.returncode})") - finally: - git("reset", "-q", "HEAD", rel) - if target.exists(): - target.unlink() - _rmdir_if_empty(target.parent) - - -def check_husky_section_index(): - """Plant a new top-level section with no _index.md and confirm rejection.""" - section = f"{MARKER}_section" - rel = f"hugo/content/en/{section}/page.md" - target = repo_root / rel - if target.exists() or (hugo_dir / "content" / "en" / section).exists(): - record("WARN", "husky: section-index test skipped (temp path exists)", rel) - return - - body = "---\ntitle: Reorg Harness Page\nprivate: true\n---\n\nTemp page.\n" - try: - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(body) - git("add", "-f", rel) - proc = run_hook("check-section-index.py") - if proc.returncode != 0 and "_index.md" in (proc.stdout + proc.stderr): - record("PASS", "husky: section-index hook rejects missing _index.md") - else: - record("FAIL", "husky: section-index hook did NOT reject bad input", - "hook may still point at the old content/en/ path " - f"(exit={proc.returncode})") - finally: - git("reset", "-q", "HEAD", rel) - if target.exists(): - target.unlink() - _rmdir_if_empty(target.parent) - - -def check_husky_cdocs_gitignore(): - """ - Append a harness pattern to hugo/content/.gitignore, force-track a matching - compiled file (with a .mdoc.md sibling), and confirm the hook flags it. - """ - content_gitignore = hugo_dir / "content" / ".gitignore" - if not content_gitignore.exists(): - record("SKIP", "husky: cdocs-gitignore test skipped (no hugo/content/.gitignore)") - return - - pattern = f"/en/{MARKER}_cdocs.md" - compiled_rel = f"hugo/content/en/{MARKER}_cdocs.md" - source_rel = f"hugo/content/en/{MARKER}_cdocs.mdoc.md" - compiled = repo_root / compiled_rel - source = repo_root / source_rel - if compiled.exists() or source.exists(): - record("WARN", "husky: cdocs-gitignore test skipped (temp path exists)", - compiled_rel) - return - - original_gitignore = content_gitignore.read_text() - try: - content_gitignore.write_text( - original_gitignore.rstrip("\n") + f"\n{pattern}\n" - ) - source.write_text("Temp Cdocs source.\n") - compiled.write_text("Temp compiled Cdocs output.\n") - git("add", "-f", compiled_rel) # force past the gitignore to simulate the mistake - proc = run_hook("check-cdocs-gitignore.py") - if proc.returncode != 0 and MARKER in (proc.stdout + proc.stderr): - record("PASS", "husky: cdocs-gitignore hook flags tracked compiled file") - else: - record("FAIL", "husky: cdocs-gitignore hook did NOT flag tracked file", - "hook may still read the old content/.gitignore path " - f"(exit={proc.returncode})") - finally: - git("reset", "-q", "HEAD", compiled_rel) - for p in (compiled, source): - if p.exists(): - p.unlink() - content_gitignore.write_text(original_gitignore) - - -def _rmdir_if_empty(path): - """Remove a directory only if it is empty (safety guard).""" - try: - path.rmdir() - except OSError: - pass - - -# -------------------------------------------------------------------------- -# E. Vale -# -------------------------------------------------------------------------- - -def check_vale(): - """Confirm vale still flags a known violation using the Datadog style.""" - if subprocess.run(["which", "vale"], capture_output=True).returncode != 0: - record("SKIP", "vale: vale not installed") - return - - styles = (repo_root / ".vale.ini") - if not styles.exists(): - record("SKIP", "vale: .vale.ini not found") - return - - rel = f"hugo/content/en/{MARKER}_vale.md" - target = repo_root / rel - if target.exists(): - record("WARN", "vale: test skipped (temp path exists)", rel) - return - - # Each of these trips a Datadog substitution rule. - body = ( - "---\ntitle: Reorg Harness Vale Test\n---\n\n" - "Simply leverage this feature to ensure it works.\n" - ) - try: - target.write_text(body) - proc = subprocess.run( - ["vale", rel], - cwd=repo_root, - capture_output=True, - text=True, - ) - output = proc.stdout + proc.stderr - if "Datadog." in output: - record("PASS", "vale: Datadog style flags violations from new path") - elif "StylesPath" in output or "ExecError" in output or not output.strip(): - record("FAIL", "vale: ran but produced no Datadog findings", - "StylesPath may not resolve from the new content location") - else: - record("WARN", "vale: ran but no Datadog.* rule fired", - output.strip()[:200]) - finally: - if target.exists(): - target.unlink() - - -# -------------------------------------------------------------------------- -# F. Hugo build (static presence only) -# -------------------------------------------------------------------------- - -def check_build_presence(): - """Static check; the real build is the manual `make start` in todo #5.""" - # Hugo's build entrypoints all need to be co-located under hugo/: the Makefile, - # the Node manifest, and go.mod (required by Hugo Modules at the project root). - required = ["Makefile", "package.json", "go.mod"] - missing = [n for n in required if not (hugo_dir / n).exists()] - if not missing: - record("PASS", "build: hugo/{Makefile,package.json,go.mod} present", - "run `cd hugo && make start` (todo #5) to verify the full build") - else: - record("FAIL", "build: missing Hugo build entrypoint(s) under hugo/", - ", ".join(f"hugo/{n}" for n in missing)) - - -# -------------------------------------------------------------------------- -# G. Rollback round-trip -# -------------------------------------------------------------------------- - -def check_rollback_roundtrip(): - """execute_reorg.py then rollback.py must restore the tree byte-for-byte. - - This is the only check that exercises rollback.py at all. Rather than - mutate the live repo (and risk leaving it half-reorganized if a step fails), - it builds a small throwaway git repo holding one representative item for - every code path the two scripts touch — a mixed .gitignore (rules that route - to hugo/, to root, and to both), a workflow with substitutable paths, a - CODEOWNERS with a moved rule, a husky hook with a substitutable path, plus a - few moved/stayed files — then: - - snapshot -> run execute_reorg.py (answering y to every prompt) - -> run rollback.py -> snapshot again - -> assert byte-identical. - - It specifically catches the easy-to-miss case where execute_reorg.py edits a file in - place (the root .gitignore split) that rollback must restore from git rather - than merely delete. - """ - if shutil.which("python3") is None or shutil.which("git") is None: - record("SKIP", "rollback: python3/git not both available") - return - - workdir = tempfile.mkdtemp(prefix=f"{MARKER}_rollback_") - work = Path(workdir) - - def write(rel, text): - p = work / rel - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(text) - - def git_work(*args): - return subprocess.run(["git", *args], cwd=work, - capture_output=True, text=True) - - def snapshot(): - """relpath -> bytes for every file under work/, excluding .git/.""" - tree = {} - for path in sorted(work.rglob("*")): - if path.is_dir(): - continue - rel = path.relative_to(work) - if rel.parts and rel.parts[0] == ".git": - continue - tree[str(rel)] = path.read_bytes() - return tree - - try: - # Representative fixture. Every top-level name here must appear in - # astro_reorg/config.yaml or execute_reorg.py refuses to run; the mix below routes - # across moves_to_hugo, top_level, and generic-glob cases. - write(".gitignore", - "# build\n" - "public/*\n" - "data/generated\n" - "content/en/api/**/*.go\n" - "node_modules\n" - "\n" - "# generic (kept in both)\n" - "*.log\n" - "\n" - "# root-only tooling\n" - ".github/preview-links-template.md\n") - write(".github/workflows/sample.yml", - "name: sample\n" - "on:\n" - " pull_request:\n" - " paths:\n" - " - 'content/en/**/*.md'\n" - "jobs:\n" - " build:\n" - " runs-on: ubuntu-latest\n" - " steps:\n" - " - run: python local/bin/foo.py\n") - write(".github/CODEOWNERS", - "* @DataDog/documentation\n" - "/content/ @DataDog/team-a\n" - "data/ @DataDog/team-b\n" - "README.md @DataDog/team-c\n") - write(".husky/check-sample.py", - "from pathlib import Path\n" - 'repo_pattern = "content"\n' - "p = Path('content/en')\n") - write("README.md", "# Fixture\n") - write("astro/package.json", '{"name": "astro-fixture"}\n') - write("content/en/page.md", "---\ntitle: Page\n---\n\nBody.\n") - write("data/sample.yaml", "key: value\n") - write("Makefile", "start:\n\techo build\n") - write("package.json", '{"name": "hugo-fixture"}\n') - write("go.mod", "module example.com/fixture\n\ngo 1.21\n") - write("babel.config.js", "module.exports = {};\n") - - # The scripts resolve repo_root as their parent's parent, so place them - # under an astro_reorg/ subfolder that mirrors the real repo layout. - (work / "astro_reorg").mkdir() - for tool in ("execute_reorg.py", "rollback.py", "config.yaml"): - shutil.copy2(repo_root / "astro_reorg" / tool, work / "astro_reorg" / tool) - - git_work("init", "-q") - git_work("add", "-A") - commit = git_work("-c", "user.email=harness@example.com", - "-c", "user.name=reorg harness", - "-c", "commit.gpgsign=false", - "commit", "-q", "-m", "fixture") - if commit.returncode != 0: - record("FAIL", "rollback: could not commit fixture repo", - (commit.stderr or commit.stdout).strip()[:200]) - return - - before = snapshot() - - # execute_reorg.py is interactive (single y/N per mutation section); answer y to - # all. Far more lines than prompts is fine — the extras are ignored. - reorg = subprocess.run( - ["python3", str(work / "astro_reorg" / "execute_reorg.py")], - cwd=work, capture_output=True, text=True, input="y\n" * 100, - ) - if reorg.returncode != 0 or not (work / "hugo").exists(): - record("FAIL", "rollback: execute_reorg.py failed on the fixture", - (reorg.stderr or reorg.stdout).strip()[:200]) - return - - rollback = subprocess.run( - ["python3", str(work / "astro_reorg" / "rollback.py")], - cwd=work, capture_output=True, text=True, - ) - if rollback.returncode != 0 or (work / "hugo").exists(): - record("FAIL", "rollback: rollback.py failed on the fixture", - (rollback.stderr or rollback.stdout).strip()[:200]) - return - - after = snapshot() - - added = sorted(set(after) - set(before)) - removed = sorted(set(before) - set(after)) - changed = sorted(p for p in before.keys() & after.keys() - if before[p] != after[p]) - if added or removed or changed: - diffs = ([f"+ {p}" for p in added] - + [f"- {p}" for p in removed] - + [f"~ {p} (in-place edit not reverted)" for p in changed]) - record("FAIL", "rollback: tree not byte-identical after reorg + rollback", - f"{len(diffs)} diff(s):\n " + "\n ".join(diffs[:20])) - else: - record("PASS", "rollback: reorg + rollback restores the tree byte-for-byte", - f"{len(before)} file(s) round-tripped, incl. the .gitignore split") - finally: - shutil.rmtree(work, ignore_errors=True) - - -# -------------------------------------------------------------------------- -# Main -# -------------------------------------------------------------------------- def main(): if not hugo_dir.exists(): @@ -797,34 +41,42 @@ def main(): top_level, moves_to_hugo = load_config() - print("== A. Layout ==") + # hugo/ holds the moved dirs; nothing moved is left at root; + # the root .gitignore split routed each rule to the right side. + print("== Layout ==") check_layout(top_level, moves_to_hugo) check_gitignore_split(top_level, moves_to_hugo) - check_site_separation(moves_to_hugo) - print("\n== B. Workflow paths ==") + # No .github/workflows/ file references a moved path without the hugo/ prefix, + # in shell scalars and in structured on.*.paths filters. + print("\n== Workflow paths ==") check_workflows(moves_to_hugo) check_workflow_path_filters(moves_to_hugo) - print("\n== C. CODEOWNERS ==") + # No pattern was moved to hugo/ while its file stayed at root, and every + # moved pattern (globs included) carries the hugo/ prefix. Pre-existing + # dangling entries are reported but not failed. + print("\n== CODEOWNERS ==") check_codeowners() check_codeowners_prefixing(moves_to_hugo) - print("\n== D. Husky hooks ==") + # Each pre-commit check still REJECTS a known-bad input planted at the + # hugo/ path. A hook that still points at the old path passes vacuously -> we fail it. + print("\n== Husky hooks ==") check_husky_circular_aliases() check_husky_section_index() check_husky_cdocs_gitignore() - print("\n== E. Vale ==") - check_vale() - - print("\n== F. Hugo build ==") + # Static presence check only; run `make start` manually. + print("\n== Hugo build ==") check_build_presence() - print("\n== G. Rollback round-trip ==") + # On a throwaway repo, execute_reorg.py then rollback.py must restore the + # tree byte-for-byte (the only test of rollback). + print("\n== Rollback round-trip ==") check_rollback_roundtrip() - # Summary. + # Summary counts = {"PASS": 0, "FAIL": 0, "SKIP": 0, "WARN": 0} for status, _, _ in results: counts[status] += 1 From 883496eef0389c68ba008384fa52f82836708989 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Wed, 10 Jun 2026 17:42:28 -0500 Subject: [PATCH 11/70] Handle Python env --- astro_reorg/execute_reorg.py | 16 +++++++++++++++- astro_reorg/rollback.py | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/astro_reorg/execute_reorg.py b/astro_reorg/execute_reorg.py index 146792b032b..29dea2b3c6d 100644 --- a/astro_reorg/execute_reorg.py +++ b/astro_reorg/execute_reorg.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import os +import shutil import sys from pathlib import Path @@ -49,16 +50,29 @@ hugo_dir.mkdir(exist_ok=True) moved = 0 +deleted = 0 for name in sorted(os.listdir(repo_root)): if name in ignore or name in top_level or name == "hugo": continue src = repo_root / name dst = hugo_dir / name + + # A Python virtualenv bakes its own ABSOLUTE path into bin/activate, the + # bin/python* symlinks, and pyvenv.cfg, so relocating the directory leaves + # those pointing at the old path — activation then falls back to the system + # Python (no project deps). A venv is a regenerable artifact, so delete it + # rather than move it; `make` rebuilds it in place with correct paths. + if (src / "pyvenv.cfg").is_file(): + print(f"Deleting venv {name} (regenerated by make in hugo/{name})") + shutil.rmtree(src) + deleted += 1 + continue + print(f"Moving {name} -> hugo/{name}") src.rename(dst) moved += 1 -print(f"Done. {moved} item(s) moved into hugo/.") +print(f"Done. {moved} item(s) moved into hugo/, {deleted} venv(s) deleted for regeneration.") # Split .gitignore between the root and hugo/ instead of copying it wholesale. # diff --git a/astro_reorg/rollback.py b/astro_reorg/rollback.py index b2bda7fd1c2..4404265d5ba 100644 --- a/astro_reorg/rollback.py +++ b/astro_reorg/rollback.py @@ -12,6 +12,7 @@ sys.exit(1) moved = 0 +moved_names = [] for name in sorted(os.listdir(hugo_dir)): # reorg.py SPLITS .gitignore in place (it prunes the root copy and writes a # routed subset to hugo/.gitignore). Skip it here so this rename can't clobber @@ -24,6 +25,7 @@ print(f"Moving hugo/{name} -> {name}") src.rename(dst) moved += 1 + moved_names.append(name) # Discard the hugo/.gitignore the split created before emptying the directory. gitignore_copy = hugo_dir / ".gitignore" @@ -46,3 +48,27 @@ check=True, ) print("Restored .gitignore, .github/workflows/, .github/CODEOWNERS, and .husky/ from git.") + +# The move-back above restores whatever was in hugo/ verbatim — but a build run +# between execute_reorg.py and this rollback (e.g. `make start`) can clean +# committed-but-generated files (API code examples, service_checks JSON, +# integration pages) and, if interrupted, never regenerate them. That leaves the +# moved tree INCOMPLETE, so rollback alone wouldn't restore a clean working tree. +# Restore the moved, git-tracked paths to their committed state to absorb that. +# +# Scope strictly to the names we moved: astro_reorg/ and other untouched root +# paths are never passed to checkout. Filter to paths git actually tracks so +# untracked/ignored moves (node_modules, public, _vendor, resources) are skipped +# — passing one of those to `git checkout` would match no tracked files and +# abort the whole restore. +tracked = subprocess.run( + ["git", "ls-files", "--", *moved_names], + cwd=repo_root, capture_output=True, text=True, check=True, +).stdout.split() +tracked_top = sorted({Path(p).parts[0] for p in tracked}) +if tracked_top: + subprocess.run( + ["git", "checkout", "--", *tracked_top], + cwd=repo_root, check=True, + ) + print(f"Restored moved tracked paths to HEAD: {', '.join(tracked_top)}") From 52473320d5429575beb60ca429f09433c4aeaa12 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 11 Jun 2026 09:28:33 -0500 Subject: [PATCH 12/70] Simplify rollback script --- astro_reorg/rollback.py | 57 +++-------------------------------------- 1 file changed, 3 insertions(+), 54 deletions(-) diff --git a/astro_reorg/rollback.py b/astro_reorg/rollback.py index 4404265d5ba..6ad6da5ab7a 100644 --- a/astro_reorg/rollback.py +++ b/astro_reorg/rollback.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import os +import shutil import subprocess import sys from pathlib import Path @@ -11,36 +11,9 @@ print("Nothing to roll back: hugo/ does not exist.", file=sys.stderr) sys.exit(1) -moved = 0 -moved_names = [] -for name in sorted(os.listdir(hugo_dir)): - # reorg.py SPLITS .gitignore in place (it prunes the root copy and writes a - # routed subset to hugo/.gitignore). Skip it here so this rename can't clobber - # the pruned root copy with the hugo subset; the root copy is restored from - # git below, and hugo/.gitignore is discarded with the now-empty directory. - if name == ".gitignore": - continue - src = hugo_dir / name - dst = repo_root / name - print(f"Moving hugo/{name} -> {name}") - src.rename(dst) - moved += 1 - moved_names.append(name) +shutil.rmtree(hugo_dir) +print("Removed hugo/") -# Discard the hugo/.gitignore the split created before emptying the directory. -gitignore_copy = hugo_dir / ".gitignore" -if gitignore_copy.exists(): - gitignore_copy.unlink() - print("Removed hugo/.gitignore (created by the split)") - -# Only succeeds if hugo/ is now empty, preventing accidental data loss. -hugo_dir.rmdir() -print(f"Done. {moved} item(s) restored to repo root.") - -# Restore the files reorg.py edited in place to their committed state. The root -# .gitignore was pruned by the split, and reorg.py may have applied partial -# workflow/CODEOWNERS/husky substitutions interactively, so reversing them -# individually isn't reliable — let git restore them wholesale. subprocess.run( ["git", "checkout", "--", ".gitignore", ".github/workflows/", ".github/CODEOWNERS", ".husky/"], @@ -48,27 +21,3 @@ check=True, ) print("Restored .gitignore, .github/workflows/, .github/CODEOWNERS, and .husky/ from git.") - -# The move-back above restores whatever was in hugo/ verbatim — but a build run -# between execute_reorg.py and this rollback (e.g. `make start`) can clean -# committed-but-generated files (API code examples, service_checks JSON, -# integration pages) and, if interrupted, never regenerate them. That leaves the -# moved tree INCOMPLETE, so rollback alone wouldn't restore a clean working tree. -# Restore the moved, git-tracked paths to their committed state to absorb that. -# -# Scope strictly to the names we moved: astro_reorg/ and other untouched root -# paths are never passed to checkout. Filter to paths git actually tracks so -# untracked/ignored moves (node_modules, public, _vendor, resources) are skipped -# — passing one of those to `git checkout` would match no tracked files and -# abort the whole restore. -tracked = subprocess.run( - ["git", "ls-files", "--", *moved_names], - cwd=repo_root, capture_output=True, text=True, check=True, -).stdout.split() -tracked_top = sorted({Path(p).parts[0] for p in tracked}) -if tracked_top: - subprocess.run( - ["git", "checkout", "--", *tracked_top], - cwd=repo_root, check=True, - ) - print(f"Restored moved tracked paths to HEAD: {', '.join(tracked_top)}") From 902a46b44cd21743c0cfc2fdb03b48d493463880 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 11 Jun 2026 09:33:53 -0500 Subject: [PATCH 13/70] Tweak script name --- astro_reorg/{rollback.py => local_rollback.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename astro_reorg/{rollback.py => local_rollback.py} (100%) diff --git a/astro_reorg/rollback.py b/astro_reorg/local_rollback.py similarity index 100% rename from astro_reorg/rollback.py rename to astro_reorg/local_rollback.py From fc68cb10e92cd4fbcfe4334072b83bebee9c9158 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 11 Jun 2026 10:59:19 -0500 Subject: [PATCH 14/70] Add test plan --- astro_reorg/{context.md => CLAUDE.md} | 6 +- astro_reorg/pr_conflicts_test_plan.md | 484 ++++++++++++++++++ astro_reorg/resolve_pr_conflicts.py | 704 ++++++++++++++++++++++++++ 3 files changed, 1193 insertions(+), 1 deletion(-) rename astro_reorg/{context.md => CLAUDE.md} (56%) create mode 100644 astro_reorg/pr_conflicts_test_plan.md create mode 100644 astro_reorg/resolve_pr_conflicts.py diff --git a/astro_reorg/context.md b/astro_reorg/CLAUDE.md similarity index 56% rename from astro_reorg/context.md rename to astro_reorg/CLAUDE.md index f92cc1951ac..0f67727fffd 100644 --- a/astro_reorg/context.md +++ b/astro_reorg/CLAUDE.md @@ -6,8 +6,12 @@ This repo is currently a Hugo site. We instead want it to contain a `hugo` and ` `astro_reorg/execute_reorg.py` implements the file and folder path changes, and updates any dependencies on those paths, such as GitHub actions, CODEOWNERS, and Husky workflows. -`astro_reorg/rollback.py` functions as an "undo" action for the reorg. +`astro_reorg/helpers.py` contains shared utilities used by the other scripts (path manipulation, git/shell helpers, YAML config loading). + +`astro_reorg/local_rollback.py` functions as an "undo" action for the reorg: removes `hugo/` and restores `.gitignore`, `.github/`, and `.husky/` from git. `astro_reorg/validate_reorg.py` verifies the functionality of affected entities where possible. +`astro_reorg/resolve_pr_conflicts.py` finds open PRs with merge conflicts caused by the reorg, and either auto-fixes them (by replaying commits at post-reorg paths) or labels them for manual review. Defaults to dry-run mode; use `--no-dry-run` to apply changes. + You can ignore the `astro` folder, it's a remnant from another branch where an Astro site is being developed. It is completely out of scope. \ No newline at end of file diff --git a/astro_reorg/pr_conflicts_test_plan.md b/astro_reorg/pr_conflicts_test_plan.md new file mode 100644 index 00000000000..0c66f7c5fc9 --- /dev/null +++ b/astro_reorg/pr_conflicts_test_plan.md @@ -0,0 +1,484 @@ +# Manual test plan: resolve_pr_conflicts.py + +Uses `astro-reorg-master` as a fake post-reorg base branch throughout, so no +real master is touched and all test PRs can be cleaned up afterward. + +Script invocation shorthand used below: +```bash +python3 astro_reorg/resolve_pr_conflicts.py --base-branch astro-reorg-master +``` + +--- + +## Part 0: One-time setup + +These steps create the fake base branch and the labels the script manages. +Do them once before running any test case. + +### 0.1 Create `astro-reorg-master` + +This simulates post-reorg master: it contains files at their new `hugo/` +paths. The script treats this branch as the merge target for all PRs under +test. + +The key point is that the file must be **moved**, not just copied: the old +path (`content/en/getting_started/_index.md`) must no longer exist on this +branch, and the new path (`hugo/content/en/getting_started/_index.md`) must. +A PR that edits the old path then conflicts (modify/delete) exactly the way a +real pre-reorg PR does. A simple copy would leave the old path in place and +the merge would succeed cleanly — no conflict to test. + +```bash +# Start from current master +git fetch origin +git checkout -b astro-reorg-master origin/master + +# Simulate the reorg: MOVE the test file into hugo/ (git mv = delete old + +# add new), so PRs touching the old path produce a real reorg conflict. +mkdir -p hugo/content/en/getting_started +git mv content/en/getting_started/_index.md hugo/content/en/getting_started/_index.md +git commit -m "test: simulate reorg — move getting_started/_index.md into hugo/" + +git push origin astro-reorg-master +``` + +- [ ] Branch `astro-reorg-master` exists on origin +- [ ] `hugo/content/en/getting_started/_index.md` is present on that branch +- [ ] `content/en/getting_started/_index.md` no longer exists on that branch + +### 0.2 Verify labels are created on first run + +```bash +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --dry-run \ + --pr 1 # any real PR number; it won't be modified in dry-run +``` + +- [ ] Script prints `[dry-run] would create label: 'astro-reorg-manual-review'` +- [ ] Script prints `[dry-run] would create label: 'astro-reorg-stale'` + +Run once without `--dry-run` (still with `--pr 1`) to actually create them: +```bash +python3 astro_reorg/resolve_pr_conflicts.py --base-branch astro-reorg-master --pr 1 +``` + +- [ ] Both labels now exist in the repo (check via GitHub Labels page or + `gh label list --repo DataDog/documentation`) +- [ ] Labels are not re-created on a second run (script is idempotent) + +--- + +## Part 1: MERGEABLE PR — skipped + +**Behavior being verified:** PRs that have no conflicts are detected and +skipped immediately without creating any branches, labels, or comments. + +### 1.1 Create a PR with no conflicts + +```bash +git checkout -b test/no-conflict origin/master +# Edit a file that does NOT exist at a reorg-moved path, e.g. README.md +echo "" >> README.md +git add README.md +git commit -m "test: no-conflict PR" +git push origin test/no-conflict +gh pr create --repo DataDog/documentation \ + --head test/no-conflict \ + --base astro-reorg-master \ + --title "TEST no-conflict PR" \ + --body "Test PR for resolve_pr_conflicts.py — delete after testing." +``` + +Note the PR number (referred to as `` below). + +### 1.2 Run the script against it + +```bash +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --pr +``` + +- [ ] Output says `mergeable: MERGEABLE` and `No conflicts — skipping.` +- [ ] No `reorg-fix/` branch was pushed +- [ ] No labels added to the PR +- [ ] No comment posted on the PR + +### 1.3 Cleanup + +```bash +gh pr close --repo DataDog/documentation +git push origin --delete test/no-conflict +``` + +--- + +## Part 2: Reorg-only conflicts — auto-fix, single commit + +**Behavior being verified:** A PR that edits a file at a pre-reorg path +(e.g. `content/en/`) conflicts only because the reorg moved that file to +`hugo/content/en/`. The script classifies all conflicts as reorg-caused, +runs `format-patch` + `git am`, opens a fix PR, labels the original PR +`astro-reorg-stale`, and posts a comment pointing to the fix PR. + +### 2.1 Create the PR + +```bash +# Branch off a commit BEFORE the reorg file was added to astro-reorg-master +git checkout -b test/reorg-only-conflict origin/master +# Edit the same file that the "reorg" touched, but at its old path +echo "" >> content/en/getting_started/_index.md +git add content/en/getting_started/_index.md +git commit -m "test: edit getting_started at old path" +git push origin test/reorg-only-conflict +gh pr create --repo DataDog/documentation \ + --head test/reorg-only-conflict \ + --base astro-reorg-master \ + --title "TEST reorg-only conflict PR" \ + --body "Test PR — single commit, reorg-only conflict." +``` + +Note the PR number as ``. + +### 2.2 Dry-run first + +```bash +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --dry-run \ + --pr +``` + +- [ ] Output classifies the conflict as reorg-caused (not in "Unrelated conflicts") +- [ ] Output prints `[dry-run] would apply 1 commit(s) to reorg-fix/pr-` +- [ ] Output prints the commit subject line +- [ ] Output prints `[dry-run] would open PR: '[reorg fix] TEST reorg-only conflict PR'` +- [ ] No branch was pushed, no PR opened, no labels added (dry-run) + +### 2.3 Run for real + +```bash +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --pr +``` + +- [ ] Script prints `Pushed fix to branch 'reorg-fix/pr-'` +- [ ] Script prints `Opened fix PR: https://github.com/DataDog/documentation/pull/` +- [ ] Branch `reorg-fix/pr-` exists on origin + +On the fix PR: +- [ ] Title is `[reorg fix] TEST reorg-only conflict PR` +- [ ] Body references the original PR number and contains the original PR description +- [ ] Base branch is `astro-reorg-master` +- [ ] Commits on the fix PR have the **original author name/email** (not `reorg-fix-script`) +- [ ] Commit messages match the original PR's commits exactly +- [ ] The file path in the fix PR is `hugo/content/en/getting_started/_index.md` + (not `content/en/...`) +- [ ] Fix PR is mergeable (no conflicts) + +On the original PR ``: +- [ ] Label `astro-reorg-stale` has been added +- [ ] A comment was posted referencing the fix PR number +- [ ] No `astro-reorg-manual-review` label was added + +### 2.4 Verify re-run is idempotent + +Run the script against the same PR a second time. Because the original PR was +labeled `astro-reorg-stale` on the first run, it is now skipped before any +merge test, fix branch, or comment. + +```bash +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --pr +``` + +- [ ] Output says `Already has a fix PR (astro-reorg-stale) — skipping.` +- [ ] Script exits without error (no `gh pr create` failure) +- [ ] No second fix PR is opened +- [ ] No duplicate comment is posted on the original PR +- [ ] The fix branch is NOT re-pushed (no force-push) + +### 2.5 Cleanup + +```bash +gh pr close --repo DataDog/documentation +gh pr close --repo DataDog/documentation +git push origin --delete test/reorg-only-conflict +git push origin --delete reorg-fix/pr- +``` + +--- + +## Part 3: Reorg-only conflicts — auto-fix, multiple commits + +**Behavior being verified:** A PR with multiple commits all touching +reorg-moved paths produces a fix PR with the same number of commits, each +with the original message and authorship. This verifies `format-patch`/`am` +replay rather than squashing. + +### 3.1 Create the PR + +```bash +git checkout -b test/reorg-multi-commit origin/master +echo "" >> content/en/getting_started/_index.md +git add content/en/getting_started/_index.md +git commit -m "test: first edit at old path" + +echo "" >> content/en/getting_started/_index.md +git add content/en/getting_started/_index.md +git commit -m "test: second edit at old path" + +git push origin test/reorg-multi-commit +gh pr create --repo DataDog/documentation \ + --head test/reorg-multi-commit \ + --base astro-reorg-master \ + --title "TEST multi-commit reorg PR" \ + --body "Two commits, both at pre-reorg paths." +``` + +Note the PR number as ``. + +### 3.2 Run for real + +```bash +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --pr +``` + +- [ ] Output says `would apply 2 commit(s)` (or applies 2 commits when not dry-run) +- [ ] Fix PR has exactly 2 commits +- [ ] Commit 1 message: `test: first edit at old path` +- [ ] Commit 2 message: `test: second edit at old path` +- [ ] Both commits have the original author, not `reorg-fix-script` +- [ ] Original PR labeled `astro-reorg-stale` + +### 3.3 Cleanup + +```bash +gh pr close --repo DataDog/documentation +gh pr close --repo DataDog/documentation +git push origin --delete test/reorg-multi-commit +git push origin --delete reorg-fix/pr- +``` + +--- + +## Part 4: Mixed conflicts — reorg + unrelated + +**Behavior being verified:** When a PR has at least one conflict that is NOT +at a reorg-moved path, the script must not touch the PR content. It labels +the PR `astro-reorg-manual-review` only, opens no fix PR, and adds no +`astro-reorg-stale` label. + +### 4.1 Create the PR + +```bash +git checkout -b test/mixed-conflict origin/master +# Reorg-path edit (will conflict with astro-reorg-master) +echo "" >> content/en/getting_started/_index.md +git add content/en/getting_started/_index.md +# Non-reorg-path edit: edit README.md too, and make astro-reorg-master conflict it +echo "" >> README.md +git add README.md +git commit -m "test: mixed reorg + non-reorg edits" +git push origin test/mixed-conflict +``` + +Now create a conflicting edit to README.md on `astro-reorg-master`: +```bash +git checkout astro-reorg-master +echo "" >> README.md +git add README.md +git commit -m "test: conflicting README edit on base branch" +git push origin astro-reorg-master +``` + +```bash +gh pr create --repo DataDog/documentation \ + --head test/mixed-conflict \ + --base astro-reorg-master \ + --title "TEST mixed conflict PR" \ + --body "Has both reorg and non-reorg conflicts." +``` + +Note the PR number as ``. + +### 4.2 Run the script + +```bash +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --pr +``` + +- [ ] Output lists both a reorg conflict (`hugo/content/en/...` or `content/en/...`) + and an unrelated conflict (`README.md`) +- [ ] Output says `Non-reorg conflicts present — labeling for manual review.` +- [ ] Label `astro-reorg-manual-review` added to `` +- [ ] Label `astro-reorg-stale` NOT added +- [ ] No fix PR opened +- [ ] No comment posted on the PR + +### 4.3 Cleanup + +```bash +gh pr close --repo DataDog/documentation +git push origin --delete test/mixed-conflict +# Revert the README edit on astro-reorg-master if desired +``` + +--- + +## Part 5: Wrong-path addition (no conflict marker) + +**Behavior being verified:** When a PR adds a brand-new file at a pre-reorg +path (e.g. `content/en/new_file.md`), git merges it silently at the wrong +path with no conflict marker. The script detects this via +`get_wrong_path_additions()` and treats it as a reorg conflict, triggering +the auto-fix. + +### 5.1 Create the PR + +```bash +git checkout -b test/wrong-path-addition origin/master +# Add a brand-new file that doesn't exist anywhere yet +echo "# New page" > content/en/brand_new_test_page.md +git add content/en/brand_new_test_page.md +git commit -m "test: add new page at pre-reorg path" +git push origin test/wrong-path-addition +gh pr create --repo DataDog/documentation \ + --head test/wrong-path-addition \ + --base astro-reorg-master \ + --title "TEST wrong-path addition PR" \ + --body "Adds a new file at a pre-reorg path — no conflict marker expected." +``` + +Note the PR number as ``. + +### 5.2 Run the script + +```bash +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --pr +``` + +- [ ] Output shows `Wrong-path additions: ['content/en/brand_new_test_page.md']` +- [ ] Output shows this under reorg-caused conflicts (not unrelated) +- [ ] Fix PR is opened +- [ ] On the fix PR, the new file appears at `hugo/content/en/brand_new_test_page.md` + (not `content/en/...`) +- [ ] Original PR labeled `astro-reorg-stale` + +### 5.3 Cleanup + +```bash +gh pr close --repo DataDog/documentation +gh pr close --repo DataDog/documentation +git push origin --delete test/wrong-path-addition +git push origin --delete reorg-fix/pr- +``` + +--- + +## Part 6: UNKNOWN mergeability — skipped + +**Behavior being verified:** GitHub computes mergeability lazily. PRs that +return `UNKNOWN` are skipped without error so they can be re-checked later. + +This state is hard to manufacture reliably, but can be observed naturally on +a freshly pushed PR before GitHub has computed its mergeability. Watch the +output of a run against a PR you just created: + +```bash +# Run the script immediately after opening a PR, before GitHub has processed it +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --pr +``` + +- [ ] If mergeability is `UNKNOWN`, output says + `Mergeability not yet computed by GitHub — skipping.` +- [ ] No changes made to the PR + +--- + +## Part 7: `--dry-run` flag + +**Behavior being verified:** `--dry-run` reports all intended actions without +making any changes — no branches pushed, no PRs opened, no labels applied, +no comments posted. + +Use `` from Part 2 (re-create it if already cleaned up). + +```bash +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --dry-run \ + --pr +``` + +- [ ] Output starts with `DRY-RUN mode — no branches or PRs will be modified.` +- [ ] Output shows `[dry-run] would apply N commit(s)` +- [ ] Output shows `[dry-run] would open PR: '[reorg fix] ...'` +- [ ] No branch `reorg-fix/pr-` exists on origin after the run +- [ ] No labels added to the PR +- [ ] No comment posted on the PR + +--- + +## Part 8: `--pr` flag (targeted run) + +**Behavior being verified:** Without `--pr`, the script queries all open PRs. +With `--pr`, it checks only the specified PR(s). This is the main way to +avoid processing hundreds of PRs when testing. + +```bash +# Run against two specific PRs +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --dry-run \ + --pr --pr +``` + +- [ ] Output says `Found 2 open PR(s) to check.` +- [ ] Only `` and `` appear in the output + +--- + +## Part 9: Full scan (no `--pr` filter) + +**Behavior being verified:** Without `--pr`, the script lists all open PRs +from the repo and processes each one. Run this only when ready, as it will +make real changes to conflicting PRs. + +```bash +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch astro-reorg-master \ + --dry-run +``` + +- [ ] Output says `Found N open PR(s) to check.` for the real open PR count +- [ ] Each PR appears in the output with its mergeability status +- [ ] No changes made (dry-run) + +--- + +## Part 10: Teardown + +After all tests, clean up the fake base branch: + +```bash +git push origin --delete astro-reorg-master +git branch -D astro-reorg-master +``` + +If you created the labels in Part 0 and want to remove them: +```bash +gh label delete astro-reorg-manual-review --repo DataDog/documentation --yes +gh label delete astro-reorg-stale --repo DataDog/documentation --yes +``` diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py new file mode 100644 index 00000000000..6b68456e1f7 --- /dev/null +++ b/astro_reorg/resolve_pr_conflicts.py @@ -0,0 +1,704 @@ +#!/usr/bin/env python3 +""" +Find open PRs with merge conflicts caused by the reorg. +Defaults to a dry run where it just reports the changes +that would be made. + +Usage: + python3 astro_reorg/resolve_pr_conflicts.py [--dry-run] [--pr NUMBER ...] + +Flags: + --no-dry-run Actually apply fixes and labels instead of just reporting what would be done. + --base-branch BRANCH Branch to treat as the post-reorg base (default: master). + +Background: + The reorg moves every entry in `moves_to_hugo` (astro_reorg/config.yaml) + from the repo root into hugo/. For example, content/ → hugo/content/, + layouts/ → hugo/layouts/, etc. PRs opened before the reorg was merged to + master will have their branches pointing at the old paths. When github + tries to compute mergeability, those PRs show as CONFLICTING. + +How we decide whether a conflict came from the reorg: + When git merges a PR branch into post-reorg master it uses rename detection + to pair the PR's pre-reorg file path (e.g. content/en/foo.md) with the + corresponding post-reorg path (hugo/content/en/foo.md) in master. If both + sides modified the file, git reports a conflict: Rename detection usually + places the conflict at the POST-reorg path (hugo/content/en/foo.md), because + that is where the file lives in master. Occasionally, when rename detection + fails (the file was heavily edited or the threshold wasn't met), git instead + reports a "deleted by them" conflict at the PRE-reorg path. + + A conflict is "from the reorg" if its path maps to a reorg-moved location: + a. hugo//... where is in moves_to_hugo (post-reorg path, + rename detected — the common case) + b. /... where is in moves_to_hugo (pre-reorg path, rename + NOT detected — git sees it as "they deleted the file") + + Any conflict at a path NOT matching either pattern is unrelated to the + reorg and must be resolved manually. + + We also detect a subtler case: files ADDED by the PR at a pre-reorg path + (e.g. content/en/brand_new.md). These produce no conflict marker — git + happily merges them at the wrong path. We catch them by scanning all + paths staged in the test merge and flagging any whose first segment is in + moves_to_hugo. + +Auto-fix strategy (reorg-only PRs): + For PRs where every conflict is a reorg conflict, the fix is to replay the + PR's commits at the post-reorg paths: + + 1. Find the merge base between the PR branch and the base branch (where + the PR diverged from master, before the reorg landed). + 2. Export each PR commit as its own patch with `git format-patch`, + preserving the original author and commit message. + 3. Rewrite every file path in the patches whose first segment is in + moves_to_hugo to be prefixed with hugo/ (content/en/ → hugo/content/en/). + 4. Replay the series onto a fresh branch off the base branch with + `git am --3way`. --3way falls back to a per-patch 3-way merge when + context lines have drifted because master made unrelated edits between + the PR's base and today. + 5. Push as `reorg-fix/pr-`, open a new PR for it, comment on the + original PR pointing to the fix PR, and label the original + astro-reorg-stale. + + A PR that already carries the astro-reorg-stale label (a fix PR was opened + on a prior run) is skipped, so re-running the script is safe. + + PRs from forks cannot be auto-fixed (we don't have push access to the fork). + They receive the astro-reorg-manual-review label. +""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +try: + import yaml +except ImportError: + print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +SCRIPT_DIR = Path(__file__).parent +REPO_ROOT = SCRIPT_DIR.parent +CONFIG_PATH = SCRIPT_DIR / "config.yaml" + +with CONFIG_PATH.open() as f: + _config = yaml.safe_load(f) + +MOVES_TO_HUGO: set[str] = set(_config.get("moves_to_hugo", [])) +TOP_LEVEL: set[str] = set(_config.get("top_level", [])) + +REPO = "DataDog/documentation" +LABEL_MANUAL_REVIEW = "astro-reorg-manual-review" +LABEL_STALE = "astro-reorg-stale" +LABEL_COLOR = "e4e669" +LABEL_DESCRIPTION = "Needs manual conflict resolution after replatforming reorg" + +# Set in main() from --base-branch; everything else reads this. +BASE_BRANCH = "master" + +# --------------------------------------------------------------------------- +# Shell helpers +# --------------------------------------------------------------------------- + +def run(cmd: list[str], *, cwd: Path | None = None, input: str | None = None) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True, cwd=cwd, input=input) + + +def run_bytes(cmd: list[str], *, cwd: Path | None = None) -> subprocess.CompletedProcess: + """Run a command and capture output as bytes (needed for binary file content).""" + return subprocess.run(cmd, capture_output=True, cwd=cwd) + + +def gh_json(*args: str) -> object: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return json.loads(result.stdout) + + +def gh_run(*args: str) -> str: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return result.stdout + + +def git(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess: + return run(["git", *args], cwd=cwd or REPO_ROOT) + + +# --------------------------------------------------------------------------- +# Reorg path helpers +# --------------------------------------------------------------------------- + +def path_first_segment(file_path: str) -> str: + """Return the first directory segment of a path string.""" + return file_path.lstrip("/").split("/", 1)[0] + + +def is_reorg_path(file_path: str) -> bool: + """ + Return True if this path maps to a location moved by the reorg. + + Covers both forms that git can report: + - Pre-reorg path: content/en/foo.md (first segment in moves_to_hugo) + - Post-reorg path: hugo/content/en/foo.md (second segment in moves_to_hugo) + """ + parts = Path(file_path).parts + if not parts: + return False + if parts[0] in MOVES_TO_HUGO: + return True + if parts[0] == "hugo" and len(parts) > 1 and parts[1] in MOVES_TO_HUGO: + return True + return False + + +def to_post_reorg_path(file_path: str) -> str: + """ + Convert a file path to its post-reorg location. + + content/en/foo.md → hugo/content/en/foo.md + hugo/content/en/foo.md → hugo/content/en/foo.md (already correct) + README.md → README.md (not a reorg-moved path) + """ + parts = Path(file_path).parts + if not parts: + return file_path + if parts[0] == "hugo": + return file_path + if parts[0] in MOVES_TO_HUGO: + return "hugo/" + file_path + return file_path + + +def to_pre_reorg_path(file_path: str) -> str | None: + """ + Convert a post-reorg path back to its pre-reorg location, or None if not + applicable. + + hugo/content/en/foo.md → content/en/foo.md + """ + parts = Path(file_path).parts + if len(parts) > 1 and parts[0] == "hugo" and parts[1] in MOVES_TO_HUGO: + return "/".join(parts[1:]) + return None + + +# --------------------------------------------------------------------------- +# Merge conflict analysis (run inside a temp worktree) +# --------------------------------------------------------------------------- + +def get_conflict_classification(worktree: Path) -> tuple[list[str], list[str]]: + """ + Parse git status inside a worktree after a --no-commit merge attempt. + + Returns (reorg_conflicts, other_conflicts) — lists of conflicted file paths. + + git status --porcelain conflict codes (XY): + UU both modified + AA both added + DD both deleted + AU added by us, not staged on their side + UA added by them + DU deleted by us + UD deleted by them (common for rename/delete: the reorg "deleted" the + file from the old path by renaming it; the PR still has the old path) + + For rename conflicts git shows "old -> new" in the path field. We use + the FINAL path (after the arrow) for classification, because that is the + path that needs to be resolved in the working tree. + """ + result = run(["git", "status", "--porcelain"], cwd=worktree) + reorg: list[str] = [] + other: list[str] = [] + for line in result.stdout.splitlines(): + if len(line) < 4: + continue + xy = line[:2] + path = line[3:] + # Only unmerged (conflict) entries have U in XY or are AA/DD. + if "U" not in xy and xy not in ("AA", "DD"): + continue + # Rename entries show "old -> new". Treat a rename conflict as + # reorg-caused only when BOTH endpoints map to reorg-moved paths; if + # either side is unrelated, classify it as "other" so the PR goes to + # manual review rather than getting an auto-fix we can't be sure about. + if " -> " in path: + src, dst = (p.strip() for p in path.split(" -> ", 1)) + both_reorg = is_reorg_path(src) and is_reorg_path(dst) + (reorg if both_reorg else other).append(dst) + else: + path = path.strip() + (reorg if is_reorg_path(path) else other).append(path) + return reorg, other + + +def get_wrong_path_additions(worktree: Path) -> list[str]: + """ + Find files staged in the test merge at PRE-REORG paths that should instead + live under hugo/. + + These do not cause merge conflict markers — git happily adds the file at + the old path with no complaint. We catch them here so the caller can + include them in the "reorg conflict" count. + + Specifically: any file with status 'A' (added) in the staged diff against + HEAD whose first path segment is in moves_to_hugo is a "wrong path + addition" caused by the PR adding a brand-new file at a pre-reorg path. + """ + result = run( + ["git", "diff", "--cached", "--name-status", "--diff-filter=A", "HEAD"], + cwd=worktree, + ) + wrong: list[str] = [] + for line in result.stdout.splitlines(): + parts = line.split("\t", 1) + if len(parts) != 2: + continue + path = parts[1].strip() + if path_first_segment(path) in MOVES_TO_HUGO: + wrong.append(path) + return wrong + + +# --------------------------------------------------------------------------- +# Diff path transformation +# --------------------------------------------------------------------------- + +def transform_diff_paths(diff_text: str) -> str: + """ + Rewrite file paths in a unified diff to use post-reorg (hugo/-prefixed) paths. + + Only paths whose first segment is in moves_to_hugo are changed; all other + paths, and all diff hunk content lines, are left untouched. + + Handles the following diff header forms: + diff --git a/ b/ + --- a/ + +++ b/ + rename from + rename to + + The hunk bodies (+/- content lines) are never touched because they contain + file content, not file names. + """ + out: list[str] = [] + for line in diff_text.splitlines(keepends=True): + if line.startswith("diff --git "): + # "diff --git a/content/en/foo.md b/content/en/foo.md" + # Rewrite both the a/ and b/ path tokens. + tokens = line.rstrip("\n").split(" ") + new_tokens = [] + for tok in tokens: + if tok.startswith("a/") or tok.startswith("b/"): + side, rest = tok[:2], tok[2:] + new_tokens.append(side + to_post_reorg_path(rest)) + else: + new_tokens.append(tok) + out.append(" ".join(new_tokens) + "\n") + elif line.startswith("--- a/") or line.startswith("+++ b/"): + side = line[:6] # "--- a/" or "+++ b/" + rest = line[6:].rstrip("\n") + out.append(side + to_post_reorg_path(rest) + "\n") + elif line.startswith("rename from ") or line.startswith("rename to "): + keyword_end = line.index(" ", line.index(" ") + 1) + 1 + keyword = line[:keyword_end] + path = line[keyword_end:].rstrip("\n") + out.append(keyword + to_post_reorg_path(path) + "\n") + else: + out.append(line) + return "".join(out) + + +# --------------------------------------------------------------------------- +# GitHub label helpers +# --------------------------------------------------------------------------- + +def ensure_label_exists(label: str, dry_run: bool) -> None: + """Create the GitHub label if it doesn't already exist.""" + existing = gh_json("label", "list", "--repo", REPO, "--json", "name") + if any(l["name"] == label for l in existing): # type: ignore[index] + return + if dry_run: + print(f" [dry-run] would create label: {label!r}") + return + gh_run("label", "create", label, "--repo", REPO, + "--color", LABEL_COLOR, "--description", LABEL_DESCRIPTION) + print(f" Created label: {label!r}") + + +def add_label(pr_number: int, label: str, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would add label {label!r} to PR #{pr_number}") + return + gh_run("pr", "edit", str(pr_number), "--repo", REPO, "--add-label", label) + print(f" Added label {label!r} to PR #{pr_number}") + + +def post_comment(pr_number: int, body: str, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would comment on PR #{pr_number}:\n {body[:120]}...") + return + gh_run("pr", "comment", str(pr_number), "--repo", REPO, "--body", body) + print(f" Posted comment on PR #{pr_number}") + + +# --------------------------------------------------------------------------- +# Auto-fix +# --------------------------------------------------------------------------- + +def attempt_fix(pr: dict, dry_run: bool) -> bool: + """ + Attempt to auto-fix a reorg-conflict PR by re-applying its commits at the + post-reorg paths. + + Strategy: format-patch + am, preserving the PR's individual commits. + + 1. Find the merge base between the PR branch and BASE_BRANCH (the commit + where the PR diverged from master before the reorg landed). + 2. Use `git format-patch` to export each PR commit as its own patch, + including the original author name, email, and commit message. + 3. Rewrite file paths in every patch: anything whose first segment is in + moves_to_hugo gets a hugo/ prefix (content/ → hugo/content/, etc.). + 4. Apply the series with `git am --3way` onto a fresh branch off + BASE_BRANCH. --3way falls back to a 3-way merge per patch when + context lines have drifted due to unrelated master changes, so the + PR's individual commits land cleanly even if master moved on. + 5. Push as `reorg-fix/pr-` and post a comment on the PR. + + Using format-patch/am rather than a single squashed diff means the fix + branch has the same commit history as the original PR — same messages, + same authorship, same granularity — making it easy to review and to revert + individual commits if needed. + + Returns True if the fix was applied (or would be in dry-run mode), False + if we gave up and the caller should fall back to labeling. + + PRs from forks are not auto-fixed: we cannot push to a fork branch, so we + return False immediately and let the caller add the manual-review label. + """ + pr_number = pr["number"] + head_ref = pr["headRefName"] + is_fork = pr.get("isCrossRepository", False) + + if is_fork: + print(f" PR #{pr_number} is from a fork — cannot push; will label instead.") + return False + + # Fetch the PR branch so we can reference it locally. + pr_remote_ref = f"refs/remotes/origin/{head_ref}" + fetch = git("fetch", "origin", f"{head_ref}:{pr_remote_ref.replace('refs/remotes/', '')}") + if fetch.returncode != 0: + print(f" fetch failed: {fetch.stderr.strip()[:120]}", file=sys.stderr) + return False + + # The merge base is the last common ancestor of the PR branch and + # BASE_BRANCH — the point where the PR diverged from master before the + # reorg commit landed. + merge_base = git("merge-base", f"origin/{BASE_BRANCH}", pr_remote_ref) + if merge_base.returncode != 0: + print(f" could not find merge base: {merge_base.stderr.strip()[:80]}", file=sys.stderr) + return False + base_sha = merge_base.stdout.strip() + + # Export each PR commit as a separate mbox-format patch. stdout gives us + # all patches concatenated; git am can consume this directly. + format_patch = git("format-patch", "--stdout", f"{base_sha}..{pr_remote_ref}") + if format_patch.returncode != 0: + print(f" format-patch failed: {format_patch.stderr.strip()[:80]}", file=sys.stderr) + return False + + patches = format_patch.stdout + if not patches.strip(): + print(" no patches between merge base and PR HEAD — nothing to apply.") + return False + + transformed = transform_diff_paths(patches) + + if dry_run: + # Count patches and show per-patch file summaries. + subjects = [l[len("Subject: "):] for l in transformed.splitlines() + if l.startswith("Subject: ")] + print(f" [dry-run] would apply {len(subjects)} commit(s) to reorg-fix/pr-{pr_number}:") + for s in subjects[:10]: + print(f" {s}") + if len(subjects) > 10: + print(f" ... and {len(subjects) - 10} more") + would_be_title = f"[reorg fix] {pr['title']}" + print(f" [dry-run] would open PR: {would_be_title!r}") + return True + + fix_branch = f"reorg-fix/pr-{pr_number}" + tmpdir = tempfile.mkdtemp(prefix=f"reorg_fix_{pr_number}_") + try: + add_wt = git("worktree", "add", "-b", fix_branch, tmpdir, f"origin/{BASE_BRANCH}") + if add_wt.returncode != 0: + # Creating the branch failed — most likely a reorg-fix/pr- branch + # from a prior run already exists. We do NOT reset and reuse it: + # that could clobber a fix that's already in review. Bail and let + # the PR fall back to manual review. (A fully-applied fix carries + # astro-reorg-stale, so that PR would have been skipped earlier.) + print(f" could not create fix branch {fix_branch!r} " + f"(may already exist) — leaving for manual review: " + f"{add_wt.stderr.strip()[:120]}", file=sys.stderr) + return False + worktree = Path(tmpdir) + + # git am replays each patch as its own commit, preserving the original + # author and message. --3way enables per-patch 3-way merging so that + # patches whose context drifted (because master made unrelated edits + # between the PR base and now) still apply cleanly. + am = run(["git", "am", "--3way"], cwd=worktree, input=transformed) + if am.returncode != 0: + print(f" git am failed:\n{am.stderr[:400]}", file=sys.stderr) + run(["git", "am", "--abort"], cwd=worktree) + return False + + push = run(["git", "push", "origin", fix_branch], cwd=worktree) + if push.returncode != 0: + # A reorg-fix/pr- branch from a prior run already exists on + # origin. We don't force-push (repo policy), and a fix PR for it + # most likely already exists, so bail to manual review rather than + # clobber it. (PRs that were fully fixed carry astro-reorg-stale + # and are skipped before we ever get here.) + print(f" push failed (fix branch may already exist): " + f"{push.stderr.strip()[:120]}", file=sys.stderr) + return False + + print(f" Pushed fix to branch {fix_branch!r}") + + # Open a new PR for the fix branch so the author can preview, review, + # and merge it directly — then close the original conflicting PR. + original_body = pr.get("body") or "" + new_pr_body = ( + f"🤖 Auto-generated fix for #{pr_number}.\n\n" + f"This PR replays the commits from #{pr_number} with file paths " + f"translated to the post-reorg `hugo/` layout. The original commits " + f"are preserved — same messages and authorship.\n\n" + f"If this looks correct, merge this PR and close #{pr_number}.\n\n" + f"---\n\n" + f"**Original PR description:**\n\n{original_body}" + ) + # We only reach this point on a real run; dry-run returned earlier. + new_pr_title = f"[reorg fix] {pr['title']}" + pr_create = gh_run( + "pr", "create", + "--repo", REPO, + "--head", fix_branch, + "--base", BASE_BRANCH, + "--title", new_pr_title, + "--body", new_pr_body, + ) + new_pr_url = pr_create.strip() + new_pr_number = new_pr_url.rstrip("/").split("/")[-1] + print(f" Opened fix PR: {new_pr_url}") + + post_comment( + pr_number, + f"🤖 **Reorg conflict auto-fix: #{new_pr_number}**\n\n" + f"This PR has merge conflicts caused by the astro reorg " + f"(files moved from the repo root into `hugo/`). " + f"A new PR with your commits translated to the correct paths " + f"has been opened: #{new_pr_number}\n\n" + f"If #{new_pr_number} looks correct, merge it and close this PR.", + dry_run=False, + ) + add_label(pr_number, LABEL_STALE, dry_run) + return True + + finally: + git("worktree", "remove", "--force", tmpdir) + shutil.rmtree(tmpdir, ignore_errors=True) + + +# --------------------------------------------------------------------------- +# Per-PR analysis +# --------------------------------------------------------------------------- + +def analyze_pr(pr: dict, dry_run: bool) -> None: + pr_number = pr["number"] + title = pr["title"] + mergeable = pr.get("mergeable", "UNKNOWN") + + print(f"\nPR #{pr_number}: {title}") + print(f" mergeable: {mergeable}") + + # A fix PR was already opened for this one on a prior run — skip so we + # don't re-push the fix branch or post a duplicate comment. This makes + # the whole script safe to re-run. + if any(l["name"] == LABEL_STALE for l in pr.get("labels", [])): + print(f" Already has a fix PR ({LABEL_STALE}) — skipping.") + return + + if mergeable == "MERGEABLE": + print(" No conflicts — skipping.") + return + + if mergeable == "UNKNOWN": + # GitHub computes mergeability lazily; try again later if needed. + print(" Mergeability not yet computed by GitHub — skipping.") + return + + # CONFLICTING: fetch the branch and do a local merge test to classify + # conflicts as reorg-caused vs unrelated. + pr_ref = f"refs/remotes/origin/{pr['headRefName']}" + fetch = git("fetch", "origin", + f"refs/pull/{pr_number}/head:{pr_ref.replace('refs/remotes/', '')}") + if fetch.returncode != 0: + print(f" fetch failed: {fetch.stderr.strip()[:120]}", file=sys.stderr) + return + + tmpdir = tempfile.mkdtemp(prefix=f"reorg_check_{pr_number}_") + try: + add_wt = git("worktree", "add", "--detach", tmpdir, f"origin/{BASE_BRANCH}") + if add_wt.returncode != 0: + print(f" worktree add failed: {add_wt.stderr.strip()[:120]}", file=sys.stderr) + return + worktree = Path(tmpdir) + + # Attempt the merge without committing so we can inspect the conflicts. + # We do NOT use --no-ff here; the default is fine since we're only + # inspecting, not keeping the result. + merge = run(["git", "merge", "--no-commit", pr_ref], cwd=worktree) + + # Classify any files that have conflict markers. + reorg_conflicts, other_conflicts = get_conflict_classification(worktree) + + # Even if the merge succeeded cleanly (returncode 0), the PR may have + # added files at pre-reorg paths with no conflict marker. Check for + # these "wrong path additions" in the staged result. + wrong_additions: list[str] = [] + if merge.returncode == 0: + wrong_additions = get_wrong_path_additions(worktree) + # Treat wrong-path additions as reorg conflicts: the PR added a + # file at a pre-reorg path that should be under hugo/. + reorg_conflicts.extend(wrong_additions) + + # Always abort the test merge before leaving the worktree. + run(["git", "merge", "--abort"], cwd=worktree) + + print(f" Reorg-caused conflicts : {reorg_conflicts or 'none'}") + print(f" Unrelated conflicts : {other_conflicts or 'none'}") + if wrong_additions: + print(f" Wrong-path additions : {wrong_additions}") + + if not reorg_conflicts and not other_conflicts: + if merge.returncode != 0: + # The merge failed but we couldn't classify any conflicted path + # (an unusual conflict type we don't parse). Don't guess that + # it's reorg-caused — flag it for a human rather than skip a + # real conflict. + print(" Merge failed but no conflicts could be classified " + "— labeling for manual review.") + add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + else: + print(" No conflicts found locally (GitHub mergeability may be stale).") + return + + if other_conflicts: + # The PR has conflicts that are NOT from the reorg. We must not + # touch it — just label it so a human can resolve it manually. + print(" Non-reorg conflicts present — labeling for manual review.") + add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + else: + # All conflicts are reorg-caused. Attempt the auto-fix. + print(" All conflicts are reorg-caused — attempting auto-fix.") + success = attempt_fix(pr, dry_run) + if not success: + # Auto-fix failed (fork, apply error, etc.) — fall back to labeling. + print(" Auto-fix failed or not applicable — labeling for manual review.") + add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + + finally: + run(["git", "worktree", "remove", "--force", tmpdir]) + shutil.rmtree(tmpdir, ignore_errors=True) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def get_open_prs(only: list[int] | None = None) -> list[dict]: + """Return open PRs, optionally filtered to specific numbers.""" + fields = ("number,title,body,labels,headRefName,headRepositoryOwner," + "baseRefName,headRefOid,baseRefOid,isCrossRepository,mergeable") + if only: + prs = [] + for n in only: + pr = gh_json("pr", "view", str(n), "--repo", REPO, "--json", fields) + prs.append(pr) + return prs + return gh_json( # type: ignore[return-value] + "pr", "list", "--repo", REPO, "--state", "open", + "--json", fields, "--limit", "300", + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Check open PRs for reorg-caused merge conflicts.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "--dry-run", action=argparse.BooleanOptionalAction, default=True, + help="Report what would be done without making any changes (default: on). Use --no-dry-run to apply changes.", + ) + parser.add_argument( + "--pr", type=int, action="append", dest="prs", metavar="NUMBER", + help="Only check this PR number (may be repeated).", + ) + parser.add_argument( + "--base-branch", default="master", metavar="BRANCH", + help="Branch to treat as the post-reorg base (default: master). " + "Set to a test branch to run against a fake main.", + ) + args = parser.parse_args() + + global BASE_BRANCH + BASE_BRANCH = args.base_branch + + if args.dry_run: + print("DRY-RUN mode — no branches or PRs will be modified.\n") + + print(f"Fetching origin/{BASE_BRANCH}...") + fetch_master = git("fetch", "origin", BASE_BRANCH) + if fetch_master.returncode != 0: + print(f"Warning: could not update {BASE_BRANCH}: {fetch_master.stderr.strip()[:80]}", + file=sys.stderr) + + ensure_label_exists(LABEL_MANUAL_REVIEW, args.dry_run) + ensure_label_exists(LABEL_STALE, args.dry_run) + + prs = get_open_prs(args.prs) + print(f"Found {len(prs)} open PR(s) to check.") + + for pr in prs: + # Isolate failures: one PR raising (e.g. a transient gh/network error) + # shouldn't abort the whole batch. A half-finished fix is safe to + # retry — a re-run either skips it (astro-reorg-stale) or, if the fix + # branch already exists, falls back to manual review. + try: + analyze_pr(pr, args.dry_run) + except Exception as exc: + print(f"\nERROR processing PR #{pr.get('number', '?')}: {exc}", + file=sys.stderr) + print(" Skipping to the next PR.", file=sys.stderr) + + print("\nDone.") + + +if __name__ == "__main__": + main() From 87841fd8d0c4e61636b70ad68b8885f9797aa579 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 11:03:43 -0500 Subject: [PATCH 15/70] Tweak rollback script --- astro_reorg/local_rollback.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/astro_reorg/local_rollback.py b/astro_reorg/local_rollback.py index 6ad6da5ab7a..06bd6bbbda4 100644 --- a/astro_reorg/local_rollback.py +++ b/astro_reorg/local_rollback.py @@ -11,6 +11,12 @@ print("Nothing to roll back: hugo/ does not exist.", file=sys.stderr) sys.exit(1) +makefile_config_src = hugo_dir / "Makefile.config" +makefile_config_dst = repo_root / "Makefile.config" +if makefile_config_src.exists(): + shutil.copy2(makefile_config_src, makefile_config_dst) + print("Restored Makefile.config to repo root.") + shutil.rmtree(hugo_dir) print("Removed hugo/") From 7251884b35298f05e83ac0a26b4fdd7b52a71fc8 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 11:34:47 -0500 Subject: [PATCH 16/70] Tweak test PR script --- astro_reorg/CLAUDE.md | 2 + astro_reorg/create_test_prs.py | 208 +++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 astro_reorg/create_test_prs.py diff --git a/astro_reorg/CLAUDE.md b/astro_reorg/CLAUDE.md index 0f67727fffd..8a5ddef623e 100644 --- a/astro_reorg/CLAUDE.md +++ b/astro_reorg/CLAUDE.md @@ -14,4 +14,6 @@ This repo is currently a Hugo site. We instead want it to contain a `hugo` and ` `astro_reorg/resolve_pr_conflicts.py` finds open PRs with merge conflicts caused by the reorg, and either auto-fixes them (by replaying commits at post-reorg paths) or labels them for manual review. Defaults to dry-run mode; use `--no-dry-run` to apply changes. +`astro_reorg/create_test_prs.py` creates test PRs against a mock base branch (set `MOCK_BASE_BRANCH` at the top of the file) for exercising the reorg tooling. For each spec in `TEST_PRS` it branches off `master`, applies a content change, pushes, and opens a PR, then opens the PRs in the browser. + You can ignore the `astro` folder, it's a remnant from another branch where an Astro site is being developed. It is completely out of scope. \ No newline at end of file diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py new file mode 100644 index 00000000000..6ed7adfc556 --- /dev/null +++ b/astro_reorg/create_test_prs.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Create test PRs against a mock base branch for exercising the reorg tooling. + +For each spec in TEST_PRS this script: + 1. branches off master, + 2. applies the spec's content change, + 3. commits and pushes the branch, + 4. opens a PR against MOCK_BASE_BRANCH. + +On completion it opens each new PR in the browser (falling back to printing +the URLs if a browser can't be launched). + +The mock base branch must already exist on the remote. Set MOCK_BASE_BRANCH +below to point at it before running. +""" +from __future__ import annotations + +import subprocess +import sys +import webbrowser +from pathlib import Path + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +# Branch the test PRs target. Create this on the remote before running. +MOCK_BASE_BRANCH = "jen.gilbert/astro-reorg-mock-base" + +REPO = "DataDog/documentation" +REPO_ROOT = Path(__file__).parent.parent + +# Distinctive prefix so branches created here are obvious and easy to clean up. +BRANCH_PREFIX = "jen.gilbert/astro-reorg-test" + +# Label applied to every PR so they can all be found together. +LABEL = "astro-reorg-testing" +LABEL_COLOR = "e4e669" +LABEL_DESCRIPTION = "Test PRs for exercising the astro reorg tooling" + +# Each spec describes one PR: the branch to create, the file to edit, the exact +# text to replace (old -> new, must match once), and the PR title/body. +TEST_PRS = [ + { + "branch": f"{BRANCH_PREFIX}-wording", + "file": "content/en/getting_started/_index.md", + "old": "supports every phase of software development", + "new": "supports each phase of software development", + "commit": "Test PR: minor wording tweak in getting started intro", + "title": "[TEST] Minor wording tweak in getting started intro", + "body": ( + "Test PR for exercising the astro reorg tooling. Makes a minor, " + "non-material wording change in the getting started intro paragraph.\n\n" + "Do not merge." + ), + }, +] + +# --------------------------------------------------------------------------- +# Shell helpers +# --------------------------------------------------------------------------- + +def run(cmd: list[str], *, cwd: Path | None = None) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True, cwd=cwd or REPO_ROOT) + + +def git(*args: str) -> subprocess.CompletedProcess: + return run(["git", *args]) + + +def gh(*args: str) -> subprocess.CompletedProcess: + return run(["gh", *args]) + + +def die(message: str) -> None: + print(f"ERROR: {message}", file=sys.stderr) + sys.exit(1) + + +def ensure_label() -> None: + """Create the shared label if it doesn't already exist.""" + existing = gh("label", "list", "--repo", REPO, "--json", "name", "--jq", ".[].name") + if LABEL in existing.stdout.split(): + return + create = gh("label", "create", LABEL, "--repo", REPO, + "--color", LABEL_COLOR, "--description", LABEL_DESCRIPTION) + if create.returncode != 0: + die(f"could not create label {LABEL!r}: {create.stderr.strip()}") + print(f"Created label: {LABEL!r}") + + +# --------------------------------------------------------------------------- +# Preflight +# --------------------------------------------------------------------------- + +def preflight() -> None: + """Fail fast if the working tree is dirty or the mock base is missing.""" + status = git("status", "--porcelain") + if status.stdout.strip(): + die("working tree is not clean; commit or stash your changes first.") + + # The mock base branch must exist on the remote so PRs can target it. + ls = git("ls-remote", "--heads", "origin", MOCK_BASE_BRANCH) + if not ls.stdout.strip(): + die(f"mock base branch '{MOCK_BASE_BRANCH}' not found on origin. " + f"Create and push it before running.") + + +def current_branch() -> str: + return git("rev-parse", "--abbrev-ref", "HEAD").stdout.strip() + + +# --------------------------------------------------------------------------- +# PR creation +# --------------------------------------------------------------------------- + +def create_pr(spec: dict) -> str | None: + """Create the branch, apply the change, push, and open a PR. Returns URL.""" + branch = spec["branch"] + target = REPO_ROOT / spec["file"] + print(f"\n=== {branch} ===") + + if not target.exists(): + die(f"target file does not exist: {spec['file']}") + + # Start from an up-to-date master so the change is a clean diff. + if git("fetch", "origin", "master").returncode != 0: + die("git fetch origin master failed.") + + # Recreate the branch from scratch so reruns are idempotent. + git("branch", "-D", branch) # ignore failure: branch may not exist yet + checkout = git("checkout", "-b", branch, "origin/master") + if checkout.returncode != 0: + die(f"could not create branch {branch}: {checkout.stderr.strip()}") + + # Apply the wording change. + text = target.read_text() + count = text.count(spec["old"]) + if count != 1: + die(f"expected to find {spec['old']!r} exactly once in {spec['file']}, " + f"found {count}.") + target.write_text(text.replace(spec["old"], spec["new"])) + + git("add", spec["file"]) + commit = git("commit", "-m", spec["commit"]) + if commit.returncode != 0: + die(f"commit failed: {commit.stderr.strip()}") + + # Delete any stale remote copy of this test branch first so a plain (never + # forced) push always fast-forwards. Only ever touches our own test branch. + git("push", "origin", "--delete", branch) # ignore failure: may not exist + push = git("push", "--set-upstream", "origin", branch) + if push.returncode != 0: + die(f"push failed: {push.stderr.strip()}") + + pr = gh( + "pr", "create", + "--repo", REPO, + "--head", branch, + "--base", MOCK_BASE_BRANCH, + "--title", spec["title"], + "--body", spec["body"], + "--label", LABEL, + ) + if pr.returncode != 0: + die(f"gh pr create failed: {pr.stderr.strip()}") + + url = pr.stdout.strip().splitlines()[-1] + print(f" Opened PR: {url}") + return url + + +def main() -> None: + preflight() + ensure_label() + original_branch = current_branch() + + urls: list[str] = [] + try: + for spec in TEST_PRS: + url = create_pr(spec) + if url: + urls.append(url) + finally: + # Return to wherever we started, whatever happened. + git("checkout", original_branch) + + if not urls: + print("\nNo PRs created.") + return + + print(f"\nCreated {len(urls)} PR(s):") + for url in urls: + print(f" {url}") + + # Open in the browser; webbrowser.open returns False if it can't. + opened_any = False + for url in urls: + try: + opened_any = webbrowser.open(url) or opened_any + except Exception: + pass + if not opened_any: + print("\nCould not open a browser. Use the links above.") + + +if __name__ == "__main__": + main() From 218f70d7cd29c2b2656a78bcf4017ccff0c6941e Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 11:42:28 -0500 Subject: [PATCH 17/70] Tweak script --- astro_reorg/create_test_prs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index 6ed7adfc556..e0efcc65892 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -25,7 +25,7 @@ # --------------------------------------------------------------------------- # Branch the test PRs target. Create this on the remote before running. -MOCK_BASE_BRANCH = "jen.gilbert/astro-reorg-mock-base" +MOCK_BASE_BRANCH = "jen.gilbert/astro-reorg-demo-7-10-114044" REPO = "DataDog/documentation" REPO_ROOT = Path(__file__).parent.parent From 86ca7b408c7cc375cef07f29064e1f3d23e90b09 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 11:45:27 -0500 Subject: [PATCH 18/70] Ignore untracked astro folder --- astro_reorg/create_test_prs.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index e0efcc65892..f706a974af0 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -94,10 +94,19 @@ def ensure_label() -> None: # --------------------------------------------------------------------------- def preflight() -> None: - """Fail fast if the working tree is dirty or the mock base is missing.""" - status = git("status", "--porcelain") - if status.stdout.strip(): - die("working tree is not clean; commit or stash your changes first.") + """Fail fast on uncommitted tracked changes, or if the mock base is missing. + + Only TRACKED changes matter: the script switches branches and builds a + commit, so staged/unstaged edits could be carried across or swept in. + Untracked files (e.g. a folder from another branch that doesn't exist here) + are ignored — they don't block checkouts and never enter the commit, which + only `git add`s the one target file. + """ + unstaged = git("diff", "--quiet").returncode != 0 + staged = git("diff", "--cached", "--quiet").returncode != 0 + if unstaged or staged: + die("you have uncommitted changes to tracked files; " + "commit or stash them first. (Untracked files are fine.)") # The mock base branch must exist on the remote so PRs can target it. ls = git("ls-remote", "--heads", "origin", MOCK_BASE_BRANCH) From 18259b98131e9556fa78957f995808b4f29862f9 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 12:05:12 -0500 Subject: [PATCH 19/70] Tweak test PR setup --- astro_reorg/create_test_prs.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index f706a974af0..968762d46ed 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -17,6 +17,7 @@ import subprocess import sys +import uuid import webbrowser from pathlib import Path @@ -33,11 +34,16 @@ # Distinctive prefix so branches created here are obvious and easy to clean up. BRANCH_PREFIX = "jen.gilbert/astro-reorg-test" -# Label applied to every PR so they can all be found together. +# Label applied to every PR so they can all be found together. Created by the +# script if missing (see ensure_label). LABEL = "astro-reorg-testing" LABEL_COLOR = "e4e669" LABEL_DESCRIPTION = "Test PRs for exercising the astro reorg tooling" +# Existing repo label applied to every test PR to guard against accidental +# merges. Assumed to already exist in the repo. +DO_NOT_MERGE_LABEL = "Do Not Merge" + # Each spec describes one PR: the branch to create, the file to edit, the exact # text to replace (old -> new, must match once), and the PR title/body. TEST_PRS = [ @@ -125,7 +131,9 @@ def current_branch() -> str: def create_pr(spec: dict) -> str | None: """Create the branch, apply the change, push, and open a PR. Returns URL.""" - branch = spec["branch"] + # Append a short unique suffix so every run creates a fresh branch rather + # than colliding with one from a previous run. + branch = f"{spec['branch']}-{uuid.uuid4().hex[:8]}" target = REPO_ROOT / spec["file"] print(f"\n=== {branch} ===") @@ -136,8 +144,6 @@ def create_pr(spec: dict) -> str | None: if git("fetch", "origin", "master").returncode != 0: die("git fetch origin master failed.") - # Recreate the branch from scratch so reruns are idempotent. - git("branch", "-D", branch) # ignore failure: branch may not exist yet checkout = git("checkout", "-b", branch, "origin/master") if checkout.returncode != 0: die(f"could not create branch {branch}: {checkout.stderr.strip()}") @@ -155,9 +161,7 @@ def create_pr(spec: dict) -> str | None: if commit.returncode != 0: die(f"commit failed: {commit.stderr.strip()}") - # Delete any stale remote copy of this test branch first so a plain (never - # forced) push always fast-forwards. Only ever touches our own test branch. - git("push", "origin", "--delete", branch) # ignore failure: may not exist + # Branch name is unique per run, so a plain push always fast-forwards. push = git("push", "--set-upstream", "origin", branch) if push.returncode != 0: die(f"push failed: {push.stderr.strip()}") @@ -170,6 +174,7 @@ def create_pr(spec: dict) -> str | None: "--title", spec["title"], "--body", spec["body"], "--label", LABEL, + "--label", DO_NOT_MERGE_LABEL, ) if pr.returncode != 0: die(f"gh pr create failed: {pr.stderr.strip()}") From a5cffa5a7ad679b5903d68fc1c5b4140e36377bc Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 12:14:39 -0500 Subject: [PATCH 20/70] Use mock-master instead of master to yield small PR diffs --- astro_reorg/CLAUDE.md | 2 +- astro_reorg/create_test_prs.py | 32 ++++++++++++++++++++------------ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/astro_reorg/CLAUDE.md b/astro_reorg/CLAUDE.md index 8a5ddef623e..51f8887c5db 100644 --- a/astro_reorg/CLAUDE.md +++ b/astro_reorg/CLAUDE.md @@ -14,6 +14,6 @@ This repo is currently a Hugo site. We instead want it to contain a `hugo` and ` `astro_reorg/resolve_pr_conflicts.py` finds open PRs with merge conflicts caused by the reorg, and either auto-fixes them (by replaying commits at post-reorg paths) or labels them for manual review. Defaults to dry-run mode; use `--no-dry-run` to apply changes. -`astro_reorg/create_test_prs.py` creates test PRs against a mock base branch (set `MOCK_BASE_BRANCH` at the top of the file) for exercising the reorg tooling. For each spec in `TEST_PRS` it branches off `master`, applies a content change, pushes, and opens a PR, then opens the PRs in the browser. +`astro_reorg/create_test_prs.py` creates test PRs against a mock base branch (set `MOCK_BASE_BRANCH` at the top of the file) for exercising the reorg tooling. For each spec in `TEST_PRS` it branches off `BRANCH_FROM` (a frozen snapshot of `master`, so PR diffs stay small), applies a content change, pushes, and opens a PR, then opens the PRs in the browser. You can ignore the `astro` folder, it's a remnant from another branch where an Astro site is being developed. It is completely out of scope. \ No newline at end of file diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index 968762d46ed..100234303ae 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -2,7 +2,7 @@ """Create test PRs against a mock base branch for exercising the reorg tooling. For each spec in TEST_PRS this script: - 1. branches off master, + 1. branches off BRANCH_FROM (a frozen snapshot of master), 2. applies the spec's content change, 3. commits and pushes the branch, 4. opens a PR against MOCK_BASE_BRANCH. @@ -10,8 +10,8 @@ On completion it opens each new PR in the browser (falling back to printing the URLs if a browser can't be launched). -The mock base branch must already exist on the remote. Set MOCK_BASE_BRANCH -below to point at it before running. +Both BRANCH_FROM and MOCK_BASE_BRANCH must already exist on the remote. Set +them below before running. """ from __future__ import annotations @@ -28,6 +28,13 @@ # Branch the test PRs target. Create this on the remote before running. MOCK_BASE_BRANCH = "jen.gilbert/astro-reorg-demo-7-10-114044" +# Branch the test PRs are cut from. This is a frozen snapshot of master (created +# manually) so PR diffs stay small: a PR shows every commit in the head branch +# that isn't in the base, so branching off a moving master would drag in every +# new master commit as noise. Branching off this frozen point keeps each PR to +# just its own change. Must already exist on the remote. +BRANCH_FROM = "mock-master" + REPO = "DataDog/documentation" REPO_ROOT = Path(__file__).parent.parent @@ -114,11 +121,12 @@ def preflight() -> None: die("you have uncommitted changes to tracked files; " "commit or stash them first. (Untracked files are fine.)") - # The mock base branch must exist on the remote so PRs can target it. - ls = git("ls-remote", "--heads", "origin", MOCK_BASE_BRANCH) - if not ls.stdout.strip(): - die(f"mock base branch '{MOCK_BASE_BRANCH}' not found on origin. " - f"Create and push it before running.") + # Both the branch-off point and the PR target must exist on the remote. + for name in (BRANCH_FROM, MOCK_BASE_BRANCH): + ls = git("ls-remote", "--heads", "origin", name) + if not ls.stdout.strip(): + die(f"branch '{name}' not found on origin. " + f"Create and push it before running.") def current_branch() -> str: @@ -140,11 +148,11 @@ def create_pr(spec: dict) -> str | None: if not target.exists(): die(f"target file does not exist: {spec['file']}") - # Start from an up-to-date master so the change is a clean diff. - if git("fetch", "origin", "master").returncode != 0: - die("git fetch origin master failed.") + # Start from the frozen snapshot so the PR diff is just this change. + if git("fetch", "origin", BRANCH_FROM).returncode != 0: + die(f"git fetch origin {BRANCH_FROM} failed.") - checkout = git("checkout", "-b", branch, "origin/master") + checkout = git("checkout", "-b", branch, f"origin/{BRANCH_FROM}") if checkout.returncode != 0: die(f"could not create branch {branch}: {checkout.stderr.strip()}") From 1c8ff95293473f4dc0afc08c907723b81a26c16c Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 14:16:03 -0500 Subject: [PATCH 21/70] Move test branch names to config file and increase overall safety --- astro_reorg/CLAUDE.md | 6 +- astro_reorg/config.yaml | 13 + astro_reorg/create_test_prs.py | 27 +- astro_reorg/pr_conflicts_test_plan.md | 484 -------------------------- astro_reorg/resolve_pr_conflicts.py | 73 +++- 5 files changed, 101 insertions(+), 502 deletions(-) delete mode 100644 astro_reorg/pr_conflicts_test_plan.md diff --git a/astro_reorg/CLAUDE.md b/astro_reorg/CLAUDE.md index 51f8887c5db..b80cc796e99 100644 --- a/astro_reorg/CLAUDE.md +++ b/astro_reorg/CLAUDE.md @@ -12,8 +12,10 @@ This repo is currently a Hugo site. We instead want it to contain a `hugo` and ` `astro_reorg/validate_reorg.py` verifies the functionality of affected entities where possible. -`astro_reorg/resolve_pr_conflicts.py` finds open PRs with merge conflicts caused by the reorg, and either auto-fixes them (by replaying commits at post-reorg paths) or labels them for manual review. Defaults to dry-run mode; use `--no-dry-run` to apply changes. +`astro_reorg/resolve_pr_conflicts.py` finds open PRs with merge conflicts caused by the reorg, and either auto-fixes them (by replaying commits at post-reorg paths) or labels them for manual review. Defaults to dry-run mode; use `--no-dry-run` to apply changes. For safety it defaults to the mock base branch (see below); pass `--live` to run against the real `master`. -`astro_reorg/create_test_prs.py` creates test PRs against a mock base branch (set `MOCK_BASE_BRANCH` at the top of the file) for exercising the reorg tooling. For each spec in `TEST_PRS` it branches off `BRANCH_FROM` (a frozen snapshot of `master`, so PR diffs stay small), applies a content change, pushes, and opens a PR, then opens the PRs in the browser. +`astro_reorg/create_test_prs.py` creates test PRs for exercising the reorg tooling. For each spec in `TEST_PRS` it branches off `branch_from` (a frozen snapshot of `master`, so PR diffs stay small), applies a content change, pushes, and opens a PR against `mock_base_branch`, then opens the PRs in the browser. + +The two scripts above share the `test:` section of `config.yaml`, which names the branches a test run uses: `mock_base_branch` (the stand-in for post-reorg `master` that test PRs target) and `branch_from` (the frozen snapshot they're cut from). `create_test_prs.py` reads both; `resolve_pr_conflicts.py` defaults to `mock_base_branch` so it operates on the same branch the test PRs were opened against — no need to pass `--base-branch` by hand. Pass `--live` to that script to run against the real `master` instead. You can ignore the `astro` folder, it's a remnant from another branch where an Astro site is being developed. It is completely out of scope. \ No newline at end of file diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index 47bc16e6276..f67f4050208 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -123,3 +123,16 @@ ignore: - .DS_Store # Python bytecode cache - __pycache__ + +# Branches used to exercise the reorg tooling against a fake main instead of the +# real master. create_test_prs.py opens test PRs against mock_base_branch, and +# resolve_pr_conflicts.py treats that same branch as the post-reorg base by +# default (pass --live to target the real master), so the two scripts always +# agree on which branches to use during a test run. +test: + # Stand-in for post-reorg master: the branch that has the reorg applied. Test + # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. + mock_base_branch: jen.gilbert/astro-reorg-demo-7-10-114044 + # Frozen snapshot of master that test PRs branch off, so each PR's diff stays + # small (a PR shows every commit in its head that isn't in the base branch). + branch_from: mock-master diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index 100234303ae..e121c9f6bae 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -10,8 +10,9 @@ On completion it opens each new PR in the browser (falling back to printing the URLs if a browser can't be launched). -Both BRANCH_FROM and MOCK_BASE_BRANCH must already exist on the remote. Set -them below before running. +BRANCH_FROM and MOCK_BASE_BRANCH are read from the `test:` section of +config.yaml (shared with resolve_pr_conflicts.py, which defaults to this same +mock base branch). Both must already exist on the remote before running. """ from __future__ import annotations @@ -21,22 +22,34 @@ import webbrowser from pathlib import Path +try: + import yaml +except ImportError: + print("Error: PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- +REPO = "DataDog/documentation" +REPO_ROOT = Path(__file__).parent.parent +CONFIG_PATH = Path(__file__).parent / "config.yaml" + +# The test branches live in config.yaml under `test:` so this script and +# resolve_pr_conflicts.py always agree on which branches a test run uses. +with CONFIG_PATH.open() as f: + _test_config = yaml.safe_load(f).get("test", {}) + # Branch the test PRs target. Create this on the remote before running. -MOCK_BASE_BRANCH = "jen.gilbert/astro-reorg-demo-7-10-114044" +MOCK_BASE_BRANCH = _test_config["mock_base_branch"] # Branch the test PRs are cut from. This is a frozen snapshot of master (created # manually) so PR diffs stay small: a PR shows every commit in the head branch # that isn't in the base, so branching off a moving master would drag in every # new master commit as noise. Branching off this frozen point keeps each PR to # just its own change. Must already exist on the remote. -BRANCH_FROM = "mock-master" - -REPO = "DataDog/documentation" -REPO_ROOT = Path(__file__).parent.parent +BRANCH_FROM = _test_config["branch_from"] # Distinctive prefix so branches created here are obvious and easy to clean up. BRANCH_PREFIX = "jen.gilbert/astro-reorg-test" diff --git a/astro_reorg/pr_conflicts_test_plan.md b/astro_reorg/pr_conflicts_test_plan.md deleted file mode 100644 index 0c66f7c5fc9..00000000000 --- a/astro_reorg/pr_conflicts_test_plan.md +++ /dev/null @@ -1,484 +0,0 @@ -# Manual test plan: resolve_pr_conflicts.py - -Uses `astro-reorg-master` as a fake post-reorg base branch throughout, so no -real master is touched and all test PRs can be cleaned up afterward. - -Script invocation shorthand used below: -```bash -python3 astro_reorg/resolve_pr_conflicts.py --base-branch astro-reorg-master -``` - ---- - -## Part 0: One-time setup - -These steps create the fake base branch and the labels the script manages. -Do them once before running any test case. - -### 0.1 Create `astro-reorg-master` - -This simulates post-reorg master: it contains files at their new `hugo/` -paths. The script treats this branch as the merge target for all PRs under -test. - -The key point is that the file must be **moved**, not just copied: the old -path (`content/en/getting_started/_index.md`) must no longer exist on this -branch, and the new path (`hugo/content/en/getting_started/_index.md`) must. -A PR that edits the old path then conflicts (modify/delete) exactly the way a -real pre-reorg PR does. A simple copy would leave the old path in place and -the merge would succeed cleanly — no conflict to test. - -```bash -# Start from current master -git fetch origin -git checkout -b astro-reorg-master origin/master - -# Simulate the reorg: MOVE the test file into hugo/ (git mv = delete old + -# add new), so PRs touching the old path produce a real reorg conflict. -mkdir -p hugo/content/en/getting_started -git mv content/en/getting_started/_index.md hugo/content/en/getting_started/_index.md -git commit -m "test: simulate reorg — move getting_started/_index.md into hugo/" - -git push origin astro-reorg-master -``` - -- [ ] Branch `astro-reorg-master` exists on origin -- [ ] `hugo/content/en/getting_started/_index.md` is present on that branch -- [ ] `content/en/getting_started/_index.md` no longer exists on that branch - -### 0.2 Verify labels are created on first run - -```bash -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --dry-run \ - --pr 1 # any real PR number; it won't be modified in dry-run -``` - -- [ ] Script prints `[dry-run] would create label: 'astro-reorg-manual-review'` -- [ ] Script prints `[dry-run] would create label: 'astro-reorg-stale'` - -Run once without `--dry-run` (still with `--pr 1`) to actually create them: -```bash -python3 astro_reorg/resolve_pr_conflicts.py --base-branch astro-reorg-master --pr 1 -``` - -- [ ] Both labels now exist in the repo (check via GitHub Labels page or - `gh label list --repo DataDog/documentation`) -- [ ] Labels are not re-created on a second run (script is idempotent) - ---- - -## Part 1: MERGEABLE PR — skipped - -**Behavior being verified:** PRs that have no conflicts are detected and -skipped immediately without creating any branches, labels, or comments. - -### 1.1 Create a PR with no conflicts - -```bash -git checkout -b test/no-conflict origin/master -# Edit a file that does NOT exist at a reorg-moved path, e.g. README.md -echo "" >> README.md -git add README.md -git commit -m "test: no-conflict PR" -git push origin test/no-conflict -gh pr create --repo DataDog/documentation \ - --head test/no-conflict \ - --base astro-reorg-master \ - --title "TEST no-conflict PR" \ - --body "Test PR for resolve_pr_conflicts.py — delete after testing." -``` - -Note the PR number (referred to as `` below). - -### 1.2 Run the script against it - -```bash -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --pr -``` - -- [ ] Output says `mergeable: MERGEABLE` and `No conflicts — skipping.` -- [ ] No `reorg-fix/` branch was pushed -- [ ] No labels added to the PR -- [ ] No comment posted on the PR - -### 1.3 Cleanup - -```bash -gh pr close --repo DataDog/documentation -git push origin --delete test/no-conflict -``` - ---- - -## Part 2: Reorg-only conflicts — auto-fix, single commit - -**Behavior being verified:** A PR that edits a file at a pre-reorg path -(e.g. `content/en/`) conflicts only because the reorg moved that file to -`hugo/content/en/`. The script classifies all conflicts as reorg-caused, -runs `format-patch` + `git am`, opens a fix PR, labels the original PR -`astro-reorg-stale`, and posts a comment pointing to the fix PR. - -### 2.1 Create the PR - -```bash -# Branch off a commit BEFORE the reorg file was added to astro-reorg-master -git checkout -b test/reorg-only-conflict origin/master -# Edit the same file that the "reorg" touched, but at its old path -echo "" >> content/en/getting_started/_index.md -git add content/en/getting_started/_index.md -git commit -m "test: edit getting_started at old path" -git push origin test/reorg-only-conflict -gh pr create --repo DataDog/documentation \ - --head test/reorg-only-conflict \ - --base astro-reorg-master \ - --title "TEST reorg-only conflict PR" \ - --body "Test PR — single commit, reorg-only conflict." -``` - -Note the PR number as ``. - -### 2.2 Dry-run first - -```bash -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --dry-run \ - --pr -``` - -- [ ] Output classifies the conflict as reorg-caused (not in "Unrelated conflicts") -- [ ] Output prints `[dry-run] would apply 1 commit(s) to reorg-fix/pr-` -- [ ] Output prints the commit subject line -- [ ] Output prints `[dry-run] would open PR: '[reorg fix] TEST reorg-only conflict PR'` -- [ ] No branch was pushed, no PR opened, no labels added (dry-run) - -### 2.3 Run for real - -```bash -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --pr -``` - -- [ ] Script prints `Pushed fix to branch 'reorg-fix/pr-'` -- [ ] Script prints `Opened fix PR: https://github.com/DataDog/documentation/pull/` -- [ ] Branch `reorg-fix/pr-` exists on origin - -On the fix PR: -- [ ] Title is `[reorg fix] TEST reorg-only conflict PR` -- [ ] Body references the original PR number and contains the original PR description -- [ ] Base branch is `astro-reorg-master` -- [ ] Commits on the fix PR have the **original author name/email** (not `reorg-fix-script`) -- [ ] Commit messages match the original PR's commits exactly -- [ ] The file path in the fix PR is `hugo/content/en/getting_started/_index.md` - (not `content/en/...`) -- [ ] Fix PR is mergeable (no conflicts) - -On the original PR ``: -- [ ] Label `astro-reorg-stale` has been added -- [ ] A comment was posted referencing the fix PR number -- [ ] No `astro-reorg-manual-review` label was added - -### 2.4 Verify re-run is idempotent - -Run the script against the same PR a second time. Because the original PR was -labeled `astro-reorg-stale` on the first run, it is now skipped before any -merge test, fix branch, or comment. - -```bash -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --pr -``` - -- [ ] Output says `Already has a fix PR (astro-reorg-stale) — skipping.` -- [ ] Script exits without error (no `gh pr create` failure) -- [ ] No second fix PR is opened -- [ ] No duplicate comment is posted on the original PR -- [ ] The fix branch is NOT re-pushed (no force-push) - -### 2.5 Cleanup - -```bash -gh pr close --repo DataDog/documentation -gh pr close --repo DataDog/documentation -git push origin --delete test/reorg-only-conflict -git push origin --delete reorg-fix/pr- -``` - ---- - -## Part 3: Reorg-only conflicts — auto-fix, multiple commits - -**Behavior being verified:** A PR with multiple commits all touching -reorg-moved paths produces a fix PR with the same number of commits, each -with the original message and authorship. This verifies `format-patch`/`am` -replay rather than squashing. - -### 3.1 Create the PR - -```bash -git checkout -b test/reorg-multi-commit origin/master -echo "" >> content/en/getting_started/_index.md -git add content/en/getting_started/_index.md -git commit -m "test: first edit at old path" - -echo "" >> content/en/getting_started/_index.md -git add content/en/getting_started/_index.md -git commit -m "test: second edit at old path" - -git push origin test/reorg-multi-commit -gh pr create --repo DataDog/documentation \ - --head test/reorg-multi-commit \ - --base astro-reorg-master \ - --title "TEST multi-commit reorg PR" \ - --body "Two commits, both at pre-reorg paths." -``` - -Note the PR number as ``. - -### 3.2 Run for real - -```bash -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --pr -``` - -- [ ] Output says `would apply 2 commit(s)` (or applies 2 commits when not dry-run) -- [ ] Fix PR has exactly 2 commits -- [ ] Commit 1 message: `test: first edit at old path` -- [ ] Commit 2 message: `test: second edit at old path` -- [ ] Both commits have the original author, not `reorg-fix-script` -- [ ] Original PR labeled `astro-reorg-stale` - -### 3.3 Cleanup - -```bash -gh pr close --repo DataDog/documentation -gh pr close --repo DataDog/documentation -git push origin --delete test/reorg-multi-commit -git push origin --delete reorg-fix/pr- -``` - ---- - -## Part 4: Mixed conflicts — reorg + unrelated - -**Behavior being verified:** When a PR has at least one conflict that is NOT -at a reorg-moved path, the script must not touch the PR content. It labels -the PR `astro-reorg-manual-review` only, opens no fix PR, and adds no -`astro-reorg-stale` label. - -### 4.1 Create the PR - -```bash -git checkout -b test/mixed-conflict origin/master -# Reorg-path edit (will conflict with astro-reorg-master) -echo "" >> content/en/getting_started/_index.md -git add content/en/getting_started/_index.md -# Non-reorg-path edit: edit README.md too, and make astro-reorg-master conflict it -echo "" >> README.md -git add README.md -git commit -m "test: mixed reorg + non-reorg edits" -git push origin test/mixed-conflict -``` - -Now create a conflicting edit to README.md on `astro-reorg-master`: -```bash -git checkout astro-reorg-master -echo "" >> README.md -git add README.md -git commit -m "test: conflicting README edit on base branch" -git push origin astro-reorg-master -``` - -```bash -gh pr create --repo DataDog/documentation \ - --head test/mixed-conflict \ - --base astro-reorg-master \ - --title "TEST mixed conflict PR" \ - --body "Has both reorg and non-reorg conflicts." -``` - -Note the PR number as ``. - -### 4.2 Run the script - -```bash -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --pr -``` - -- [ ] Output lists both a reorg conflict (`hugo/content/en/...` or `content/en/...`) - and an unrelated conflict (`README.md`) -- [ ] Output says `Non-reorg conflicts present — labeling for manual review.` -- [ ] Label `astro-reorg-manual-review` added to `` -- [ ] Label `astro-reorg-stale` NOT added -- [ ] No fix PR opened -- [ ] No comment posted on the PR - -### 4.3 Cleanup - -```bash -gh pr close --repo DataDog/documentation -git push origin --delete test/mixed-conflict -# Revert the README edit on astro-reorg-master if desired -``` - ---- - -## Part 5: Wrong-path addition (no conflict marker) - -**Behavior being verified:** When a PR adds a brand-new file at a pre-reorg -path (e.g. `content/en/new_file.md`), git merges it silently at the wrong -path with no conflict marker. The script detects this via -`get_wrong_path_additions()` and treats it as a reorg conflict, triggering -the auto-fix. - -### 5.1 Create the PR - -```bash -git checkout -b test/wrong-path-addition origin/master -# Add a brand-new file that doesn't exist anywhere yet -echo "# New page" > content/en/brand_new_test_page.md -git add content/en/brand_new_test_page.md -git commit -m "test: add new page at pre-reorg path" -git push origin test/wrong-path-addition -gh pr create --repo DataDog/documentation \ - --head test/wrong-path-addition \ - --base astro-reorg-master \ - --title "TEST wrong-path addition PR" \ - --body "Adds a new file at a pre-reorg path — no conflict marker expected." -``` - -Note the PR number as ``. - -### 5.2 Run the script - -```bash -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --pr -``` - -- [ ] Output shows `Wrong-path additions: ['content/en/brand_new_test_page.md']` -- [ ] Output shows this under reorg-caused conflicts (not unrelated) -- [ ] Fix PR is opened -- [ ] On the fix PR, the new file appears at `hugo/content/en/brand_new_test_page.md` - (not `content/en/...`) -- [ ] Original PR labeled `astro-reorg-stale` - -### 5.3 Cleanup - -```bash -gh pr close --repo DataDog/documentation -gh pr close --repo DataDog/documentation -git push origin --delete test/wrong-path-addition -git push origin --delete reorg-fix/pr- -``` - ---- - -## Part 6: UNKNOWN mergeability — skipped - -**Behavior being verified:** GitHub computes mergeability lazily. PRs that -return `UNKNOWN` are skipped without error so they can be re-checked later. - -This state is hard to manufacture reliably, but can be observed naturally on -a freshly pushed PR before GitHub has computed its mergeability. Watch the -output of a run against a PR you just created: - -```bash -# Run the script immediately after opening a PR, before GitHub has processed it -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --pr -``` - -- [ ] If mergeability is `UNKNOWN`, output says - `Mergeability not yet computed by GitHub — skipping.` -- [ ] No changes made to the PR - ---- - -## Part 7: `--dry-run` flag - -**Behavior being verified:** `--dry-run` reports all intended actions without -making any changes — no branches pushed, no PRs opened, no labels applied, -no comments posted. - -Use `` from Part 2 (re-create it if already cleaned up). - -```bash -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --dry-run \ - --pr -``` - -- [ ] Output starts with `DRY-RUN mode — no branches or PRs will be modified.` -- [ ] Output shows `[dry-run] would apply N commit(s)` -- [ ] Output shows `[dry-run] would open PR: '[reorg fix] ...'` -- [ ] No branch `reorg-fix/pr-` exists on origin after the run -- [ ] No labels added to the PR -- [ ] No comment posted on the PR - ---- - -## Part 8: `--pr` flag (targeted run) - -**Behavior being verified:** Without `--pr`, the script queries all open PRs. -With `--pr`, it checks only the specified PR(s). This is the main way to -avoid processing hundreds of PRs when testing. - -```bash -# Run against two specific PRs -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --dry-run \ - --pr --pr -``` - -- [ ] Output says `Found 2 open PR(s) to check.` -- [ ] Only `` and `` appear in the output - ---- - -## Part 9: Full scan (no `--pr` filter) - -**Behavior being verified:** Without `--pr`, the script lists all open PRs -from the repo and processes each one. Run this only when ready, as it will -make real changes to conflicting PRs. - -```bash -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch astro-reorg-master \ - --dry-run -``` - -- [ ] Output says `Found N open PR(s) to check.` for the real open PR count -- [ ] Each PR appears in the output with its mergeability status -- [ ] No changes made (dry-run) - ---- - -## Part 10: Teardown - -After all tests, clean up the fake base branch: - -```bash -git push origin --delete astro-reorg-master -git branch -D astro-reorg-master -``` - -If you created the labels in Part 0 and want to remove them: -```bash -gh label delete astro-reorg-manual-review --repo DataDog/documentation --yes -gh label delete astro-reorg-stale --repo DataDog/documentation --yes -``` diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 6b68456e1f7..4dc03a75876 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -5,11 +5,19 @@ that would be made. Usage: - python3 astro_reorg/resolve_pr_conflicts.py [--dry-run] [--pr NUMBER ...] + python3 astro_reorg/resolve_pr_conflicts.py [--no-dry-run] [--live | --base-branch BRANCH] [--pr NUMBER ...] Flags: --no-dry-run Actually apply fixes and labels instead of just reporting what would be done. - --base-branch BRANCH Branch to treat as the post-reorg base (default: master). + --live Run against the real master. Without it the script + defaults to the mock base branch from config.yaml's + `test:` section (the branch create_test_prs.py opens + its test PRs against) for safety. + --base-branch BRANCH Branch to treat as the post-reorg base. Overrides the + default mock base branch and --live. + --pr NUMBER ... Only process the given PR number(s) instead of all open + PRs against the base branch. PRs targeting a different + base are skipped. Background: The reorg moves every entry in `moves_to_hugo` (astro_reorg/config.yaml) @@ -98,6 +106,12 @@ MOVES_TO_HUGO: set[str] = set(_config.get("moves_to_hugo", [])) TOP_LEVEL: set[str] = set(_config.get("top_level", [])) +# Shared with create_test_prs.py: the mock base branch used for test runs. When +# invoked with --test, the script treats this branch as the post-reorg base so +# it operates on the same branch the test PRs were opened against. +_TEST_CONFIG = _config.get("test", {}) +TEST_BASE_BRANCH: str | None = _TEST_CONFIG.get("mock_base_branch") + REPO = "DataDog/documentation" LABEL_MANUAL_REVIEW = "astro-reorg-manual-review" LABEL_STALE = "astro-reorg-stale" @@ -631,17 +645,35 @@ def analyze_pr(pr: dict, dry_run: bool) -> None: # --------------------------------------------------------------------------- def get_open_prs(only: list[int] | None = None) -> list[dict]: - """Return open PRs, optionally filtered to specific numbers.""" - fields = ("number,title,body,labels,headRefName,headRepositoryOwner," - "baseRefName,headRefOid,baseRefOid,isCrossRepository,mergeable") + """Return open PRs targeting BASE_BRANCH, optionally filtered to specific numbers. + + The --base filter is what keeps a run scoped: in test mode only PRs opened + against the mock base branch are considered, so real PRs against master are + never fetched, merged, labeled, or commented on. In live mode it scopes to + PRs against master. Without it, `gh pr list` would return every open PR in + the repo regardless of target. + """ + # Only fields actually consumed below. Note baseRefOid/headRefOid are NOT + # requested: `gh pr list` doesn't support them (only `gh pr view` does), and + # nothing here uses them. + fields = ("number,title,body,labels,headRefName," + "baseRefName,isCrossRepository,mergeable") if only: + # Explicitly named PRs are an intentional override, but still guard + # against acting on a PR that targets a different base than this run. prs = [] for n in only: pr = gh_json("pr", "view", str(n), "--repo", REPO, "--json", fields) + if pr.get("baseRefName") != BASE_BRANCH: # type: ignore[union-attr] + print(f" Skipping PR #{n}: targets " + f"{pr.get('baseRefName')!r}, not base {BASE_BRANCH!r}.", # type: ignore[union-attr] + file=sys.stderr) + continue prs.append(pr) return prs return gh_json( # type: ignore[return-value] "pr", "list", "--repo", REPO, "--state", "open", + "--base", BASE_BRANCH, "--json", fields, "--limit", "300", ) @@ -661,14 +693,37 @@ def main() -> None: help="Only check this PR number (may be repeated).", ) parser.add_argument( - "--base-branch", default="master", metavar="BRANCH", - help="Branch to treat as the post-reorg base (default: master). " - "Set to a test branch to run against a fake main.", + "--base-branch", default=None, metavar="BRANCH", + help="Branch to treat as the post-reorg base. Overrides the default " + "(the mock base branch from config.yaml) and --live.", + ) + parser.add_argument( + "--live", action="store_true", + help="Run against the real master. Without this the script defaults to " + "the mock base branch from config.yaml (the same branch " + "create_test_prs.py opens its test PRs against) for safety.", ) args = parser.parse_args() + if args.live and args.base_branch: + parser.error("--live and --base-branch are mutually exclusive; " + "--live already selects master.") + global BASE_BRANCH - BASE_BRANCH = args.base_branch + if args.base_branch: + BASE_BRANCH = args.base_branch + elif args.live: + BASE_BRANCH = "master" + print("LIVE mode — running against master.\n") + else: + # Default to the mock base branch so an accidental run can't touch + # real PRs against master. + if not TEST_BASE_BRANCH: + parser.error("no base branch: set `test.mock_base_branch` in " + "config.yaml, or pass --live or --base-branch.") + BASE_BRANCH = TEST_BASE_BRANCH + print(f"TEST mode (default) — using mock base branch {BASE_BRANCH!r} " + f"from config.yaml. Pass --live to run against master.\n") if args.dry_run: print("DRY-RUN mode — no branches or PRs will be modified.\n") From 94fcd9ab7bb23b6dcd7c29cd9d9a097121993cae Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 14:17:43 -0500 Subject: [PATCH 22/70] Make label creation idempotent --- astro_reorg/create_test_prs.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index e121c9f6bae..6656b5823eb 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -104,15 +104,20 @@ def die(message: str) -> None: def ensure_label() -> None: - """Create the shared label if it doesn't already exist.""" - existing = gh("label", "list", "--repo", REPO, "--json", "name", "--jq", ".[].name") - if LABEL in existing.stdout.split(): - return + """Create the shared label, treating "already exists" as success. + + We don't pre-check with `gh label list`: it defaults to 30 labels, so in a + repo with many labels an existing one may not show up, and we'd then fail on + create. Instead we just create and accept the "already exists" error. + """ create = gh("label", "create", LABEL, "--repo", REPO, "--color", LABEL_COLOR, "--description", LABEL_DESCRIPTION) - if create.returncode != 0: - die(f"could not create label {LABEL!r}: {create.stderr.strip()}") - print(f"Created label: {LABEL!r}") + if create.returncode == 0: + print(f"Created label: {LABEL!r}") + return + if "already exists" in create.stderr.lower(): + return + die(f"could not create label {LABEL!r}: {create.stderr.strip()}") # --------------------------------------------------------------------------- From 363e9b82a39b51489133bbbd4006989c8e9425a3 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 14:34:24 -0500 Subject: [PATCH 23/70] Tweak config naming --- astro_reorg/CLAUDE.md | 4 ++-- astro_reorg/config.yaml | 6 +++--- astro_reorg/create_test_prs.py | 2 +- astro_reorg/resolve_pr_conflicts.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/astro_reorg/CLAUDE.md b/astro_reorg/CLAUDE.md index b80cc796e99..7aef39c773d 100644 --- a/astro_reorg/CLAUDE.md +++ b/astro_reorg/CLAUDE.md @@ -14,8 +14,8 @@ This repo is currently a Hugo site. We instead want it to contain a `hugo` and ` `astro_reorg/resolve_pr_conflicts.py` finds open PRs with merge conflicts caused by the reorg, and either auto-fixes them (by replaying commits at post-reorg paths) or labels them for manual review. Defaults to dry-run mode; use `--no-dry-run` to apply changes. For safety it defaults to the mock base branch (see below); pass `--live` to run against the real `master`. -`astro_reorg/create_test_prs.py` creates test PRs for exercising the reorg tooling. For each spec in `TEST_PRS` it branches off `branch_from` (a frozen snapshot of `master`, so PR diffs stay small), applies a content change, pushes, and opens a PR against `mock_base_branch`, then opens the PRs in the browser. +`astro_reorg/create_test_prs.py` creates test PRs for exercising the reorg tooling. For each spec in `TEST_PRS` it branches off `branch_from` (a frozen snapshot of `master`, so PR diffs stay small), applies a content change, pushes, and opens a PR against `mock_reorged_master_branch`, then opens the PRs in the browser. -The two scripts above share the `test:` section of `config.yaml`, which names the branches a test run uses: `mock_base_branch` (the stand-in for post-reorg `master` that test PRs target) and `branch_from` (the frozen snapshot they're cut from). `create_test_prs.py` reads both; `resolve_pr_conflicts.py` defaults to `mock_base_branch` so it operates on the same branch the test PRs were opened against — no need to pass `--base-branch` by hand. Pass `--live` to that script to run against the real `master` instead. +The two scripts above share the `test:` section of `config.yaml`, which names the branches a test run uses: `mock_reorged_master_branch` (the stand-in for post-reorg `master` that test PRs target) and `branch_from` (the frozen snapshot they're cut from). `create_test_prs.py` reads both; `resolve_pr_conflicts.py` defaults to `mock_reorged_master_branch` so it operates on the same branch the test PRs were opened against — no need to pass `--base-branch` by hand. Pass `--live` to that script to run against the real `master` instead. You can ignore the `astro` folder, it's a remnant from another branch where an Astro site is being developed. It is completely out of scope. \ No newline at end of file diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index f67f4050208..8cd8c9daf7d 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -125,14 +125,14 @@ ignore: - __pycache__ # Branches used to exercise the reorg tooling against a fake main instead of the -# real master. create_test_prs.py opens test PRs against mock_base_branch, and +# real master. create_test_prs.py opens test PRs against mock_reorged_master_branch, and # resolve_pr_conflicts.py treats that same branch as the post-reorg base by # default (pass --live to target the real master), so the two scripts always # agree on which branches to use during a test run. test: # Stand-in for post-reorg master: the branch that has the reorg applied. Test # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. - mock_base_branch: jen.gilbert/astro-reorg-demo-7-10-114044 + mock_reorged_master_branch: jen.gilbert/astro-reorg-demo-7-10-22047 # Frozen snapshot of master that test PRs branch off, so each PR's diff stays # small (a PR shows every commit in its head that isn't in the base branch). - branch_from: mock-master + branch_from: jen.gilbert/astro-reorg-scripts diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index 6656b5823eb..d7afa59806a 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -42,7 +42,7 @@ _test_config = yaml.safe_load(f).get("test", {}) # Branch the test PRs target. Create this on the remote before running. -MOCK_BASE_BRANCH = _test_config["mock_base_branch"] +MOCK_BASE_BRANCH = _test_config["mock_reorged_master_branch"] # Branch the test PRs are cut from. This is a frozen snapshot of master (created # manually) so PR diffs stay small: a PR shows every commit in the head branch diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 4dc03a75876..1d595ddb953 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -110,7 +110,7 @@ # invoked with --test, the script treats this branch as the post-reorg base so # it operates on the same branch the test PRs were opened against. _TEST_CONFIG = _config.get("test", {}) -TEST_BASE_BRANCH: str | None = _TEST_CONFIG.get("mock_base_branch") +TEST_BASE_BRANCH: str | None = _TEST_CONFIG.get("mock_reorged_master_branch") REPO = "DataDog/documentation" LABEL_MANUAL_REVIEW = "astro-reorg-manual-review" @@ -719,7 +719,7 @@ def main() -> None: # Default to the mock base branch so an accidental run can't touch # real PRs against master. if not TEST_BASE_BRANCH: - parser.error("no base branch: set `test.mock_base_branch` in " + parser.error("no base branch: set `test.mock_reorged_master_branch` in " "config.yaml, or pass --live or --base-branch.") BASE_BRANCH = TEST_BASE_BRANCH print(f"TEST mode (default) — using mock base branch {BASE_BRANCH!r} " From b0725abbd2f104b648cdcfa6358797489a65c0b9 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 14:45:05 -0500 Subject: [PATCH 24/70] Turn rename detection off in the mergeability check --- astro_reorg/resolve_pr_conflicts.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 1d595ddb953..7343bc7607e 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -585,7 +585,17 @@ def analyze_pr(pr: dict, dry_run: bool) -> None: # Attempt the merge without committing so we can inspect the conflicts. # We do NOT use --no-ff here; the default is fine since we're only # inspecting, not keeping the result. - merge = run(["git", "merge", "--no-commit", pr_ref], cwd=worktree) + # + # merge.renames=false disables git's rename detection for this test. The + # reorg only MOVES files (content/foo -> hugo/content/foo); it never + # edits their content. With rename detection ON (git's default), git + # pairs a PR's edit at the old path with the moved file on the base and + # merges cleanly, hiding the conflict entirely — so the script would see + # nothing to fix. GitHub's mergeability check does no rename detection + # and reports these as CONFLICTING (modify/delete). Disabling it here + # makes the local test match what GitHub sees: the moved file surfaces + # as a conflict at its pre-reorg path, which is_reorg_path() recognizes. + merge = run(["git", "-c", "merge.renames=false", "merge", "--no-commit", pr_ref], cwd=worktree) # Classify any files that have conflict markers. reorg_conflicts, other_conflicts = get_conflict_classification(worktree) From cb33f3c3eccd3851354e1c72876585196d3b05b9 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 10 Jul 2026 15:02:11 -0500 Subject: [PATCH 25/70] Add do not merge label to all test PRs --- astro_reorg/resolve_pr_conflicts.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 7343bc7607e..01343a3c497 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -118,9 +118,17 @@ LABEL_COLOR = "e4e669" LABEL_DESCRIPTION = "Needs manual conflict resolution after replatforming reorg" +# Existing repo label applied to auto-created fix PRs in test mode so they stay +# out of other teams' review queues. Assumed to already exist in the repo. +DO_NOT_MERGE_LABEL = "Do Not Merge" + # Set in main() from --base-branch; everything else reads this. BASE_BRANCH = "master" +# True when running against the mock base branch rather than real master. Set +# in main(); gates test-only behavior like labeling fix PRs "Do Not Merge". +IS_TEST_MODE = False + # --------------------------------------------------------------------------- # Shell helpers # --------------------------------------------------------------------------- @@ -357,7 +365,13 @@ def add_label(pr_number: int, label: str, dry_run: bool) -> None: if dry_run: print(f" [dry-run] would add label {label!r} to PR #{pr_number}") return - gh_run("pr", "edit", str(pr_number), "--repo", REPO, "--add-label", label) + # Use the REST API rather than `gh pr edit --add-label`. `gh pr edit` + # fetches the PR's projectCards to preserve them, but that field is part of + # Projects (classic), now deprecated — so the call fails with a GraphQL + # error even though only a label is being added. The REST labels endpoint + # touches nothing else and appends the label. + gh_run("api", f"repos/{REPO}/issues/{pr_number}/labels", + "-f", f"labels[]={label}") print(f" Added label {label!r} to PR #{pr_number}") @@ -452,6 +466,8 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: print(f" ... and {len(subjects) - 10} more") would_be_title = f"[reorg fix] {pr['title']}" print(f" [dry-run] would open PR: {would_be_title!r}") + if IS_TEST_MODE: + print(f" [dry-run] would label fix PR {DO_NOT_MERGE_LABEL!r} (test mode)") return True fix_branch = f"reorg-fix/pr-{pr_number}" @@ -519,6 +535,11 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: new_pr_number = new_pr_url.rstrip("/").split("/")[-1] print(f" Opened fix PR: {new_pr_url}") + # In test mode, keep the auto-created PR out of other teams' review + # queues by marking it "Do Not Merge". Never do this on real master. + if IS_TEST_MODE: + add_label(int(new_pr_number), DO_NOT_MERGE_LABEL, dry_run=False) + post_comment( pr_number, f"🤖 **Reorg conflict auto-fix: #{new_pr_number}**\n\n" @@ -719,9 +740,11 @@ def main() -> None: parser.error("--live and --base-branch are mutually exclusive; " "--live already selects master.") - global BASE_BRANCH + global BASE_BRANCH, IS_TEST_MODE if args.base_branch: BASE_BRANCH = args.base_branch + # A custom base is a test/dev scenario unless it's the real master. + IS_TEST_MODE = args.base_branch != "master" elif args.live: BASE_BRANCH = "master" print("LIVE mode — running against master.\n") @@ -732,6 +755,7 @@ def main() -> None: parser.error("no base branch: set `test.mock_reorged_master_branch` in " "config.yaml, or pass --live or --base-branch.") BASE_BRANCH = TEST_BASE_BRANCH + IS_TEST_MODE = True print(f"TEST mode (default) — using mock base branch {BASE_BRANCH!r} " f"from config.yaml. Pass --live to run against master.\n") From f453624bdcb34cd3f9e6bc02a33048e252419886 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Mon, 13 Jul 2026 15:24:59 -0500 Subject: [PATCH 26/70] Tweaks --- astro_reorg/CLAUDE.md | 8 +- astro_reorg/create_test_prs.py | 206 +++++++++++++++++++++++++++- astro_reorg/resolve_pr_conflicts.py | 68 ++++++--- 3 files changed, 254 insertions(+), 28 deletions(-) diff --git a/astro_reorg/CLAUDE.md b/astro_reorg/CLAUDE.md index 7aef39c773d..f824f735aca 100644 --- a/astro_reorg/CLAUDE.md +++ b/astro_reorg/CLAUDE.md @@ -12,10 +12,12 @@ This repo is currently a Hugo site. We instead want it to contain a `hugo` and ` `astro_reorg/validate_reorg.py` verifies the functionality of affected entities where possible. -`astro_reorg/resolve_pr_conflicts.py` finds open PRs with merge conflicts caused by the reorg, and either auto-fixes them (by replaying commits at post-reorg paths) or labels them for manual review. Defaults to dry-run mode; use `--no-dry-run` to apply changes. For safety it defaults to the mock base branch (see below); pass `--live` to run against the real `master`. +`astro_reorg/resolve_pr_conflicts.py` finds open PRs with merge conflicts caused by the reorg, and either auto-fixes them (by replaying commits at post-reorg paths) or labels them for manual review. It defaults to a dry run against a mock base branch, so it can't touch real PRs by accident. -`astro_reorg/create_test_prs.py` creates test PRs for exercising the reorg tooling. For each spec in `TEST_PRS` it branches off `branch_from` (a frozen snapshot of `master`, so PR diffs stay small), applies a content change, pushes, and opens a PR against `mock_reorged_master_branch`, then opens the PRs in the browser. +`astro_reorg/create_test_prs.py` creates test PRs for exercising `resolve_pr_conflicts.py`, including the conflicts it can't auto-fix. Each spec in `TEST_PRS` becomes a PR that reproduces one conflict scenario. -The two scripts above share the `test:` section of `config.yaml`, which names the branches a test run uses: `mock_reorged_master_branch` (the stand-in for post-reorg `master` that test PRs target) and `branch_from` (the frozen snapshot they're cut from). `create_test_prs.py` reads both; `resolve_pr_conflicts.py` defaults to `mock_reorged_master_branch` so it operates on the same branch the test PRs were opened against — no need to pass `--base-branch` by hand. Pass `--live` to that script to run against the real `master` instead. +Both scripts share the `test:` section of `config.yaml`, which names the branches a test run uses: `mock_reorged_master_branch` (the stand-in for post-reorg `master` that test PRs target and the resolver treats as the base) and `branch_from` (the frozen snapshot the test PRs are cut from). + +For flags and the mechanics of each scenario, read the script's module docstring and the comments on `TEST_PRS` — that's the source of truth, so this file doesn't have to track them. You can ignore the `astro` folder, it's a remnant from another branch where an Astro site is being developed. It is completely out of scope. \ No newline at end of file diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index d7afa59806a..913c1fd8a6d 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -5,7 +5,7 @@ 1. branches off BRANCH_FROM (a frozen snapshot of master), 2. applies the spec's content change, 3. commits and pushes the branch, - 4. opens a PR against MOCK_BASE_BRANCH. + 4. opens a PR against the base branch (see below). On completion it opens each new PR in the browser (falling back to printing the URLs if a browser can't be launched). @@ -13,6 +13,23 @@ BRANCH_FROM and MOCK_BASE_BRANCH are read from the `test:` section of config.yaml (shared with resolve_pr_conflicts.py, which defaults to this same mock base branch). Both must already exist on the remote before running. + +Testing the *unresolvable* conflict paths: + By default the mock base differs from the PRs only by pure file moves, so + resolve_pr_conflicts.py always replays them cleanly. To exercise the cases + where it CAN'T auto-fix, a spec may carry a `base_edit` describing a change + to make on the base itself — on the same line the PR touches, but at the + post-reorg (hugo/) path. When any spec has a base_edit, this script builds a + throwaway base branch off MOCK_BASE_BRANCH (unique per run, so no + force-push), applies every base_edit to it, and points ALL the PRs at that + branch. Then one resolve_pr_conflicts.py run against it exercises both a + clean auto-fix and the manual-review fallbacks. The exact command to run is + printed at the end. + + - A base_edit on a moved (hugo/) file makes git am --3way fail: the fix is + classified reorg-caused but can't be replayed (manual-review fallback). + - A base_edit on a top-level (never-moved) file surfaces as a plain + non-reorg conflict, which is sent straight to manual review. """ from __future__ import annotations @@ -66,8 +83,18 @@ # Each spec describes one PR: the branch to create, the file to edit, the exact # text to replace (old -> new, must match once), and the PR title/body. +# +# An optional `base_edit` makes the conflict unresolvable by having the BASE +# branch also change the same line (at the post-reorg hugo/ path). See the +# module docstring. Specs without a base_edit still auto-fix cleanly, so a run +# that mixes both kinds exercises every resolve_pr_conflicts.py outcome at once. +# +# An optional `add_file` ({path, content}) also creates a brand-new page at a +# pre-reorg path, on top of the old->new edit, to model "added a page and linked +# it in the nav menu." TEST_PRS = [ { + # Resolvable: reorg-only conflict, auto-fix replays it cleanly. "branch": f"{BRANCH_PREFIX}-wording", "file": "content/en/getting_started/_index.md", "old": "supports every phase of software development", @@ -80,6 +107,92 @@ "Do not merge." ), }, + { + # Unresolvable (Case 2): reorg-classified, but the base changed the same + # line at the hugo/ path, so git am --3way can't replay it. Falls back + # to the astro-reorg-manual-review label. + "branch": f"{BRANCH_PREFIX}-unresolvable-reorg", + "file": "content/en/getting_started/_index.md", + "old": "combined into a customized solution", + "new": "combined into a unified solution", + "commit": "Test PR: edit a line the reorged base also changed", + "title": "[TEST] Unresolvable reorg conflict (base edited the same line)", + "body": ( + "Test PR for exercising the astro reorg tooling. Edits a line that " + "the reorged base branch also changed, so the auto-fix can't replay " + "it and the PR should be labeled for manual review.\n\n" + "Do not merge." + ), + "base_edit": { + "file": "hugo/content/en/getting_started/_index.md", + "old": "combined into a customized solution", + "new": "combined into a tailored solution", + }, + }, + { + # Unresolvable (Case 1): the conflict is on a top-level file the reorg + # never moves, so it isn't reorg-caused at all — sent straight to manual + # review with no auto-fix attempt. + "branch": f"{BRANCH_PREFIX}-non-reorg-conflict", + "file": "CONTRIBUTING.md", + "old": "how to write and edit content", + "new": "how to write and revise content", + "commit": "Test PR: edit a top-level file the base also changed", + "title": "[TEST] Non-reorg conflict on a top-level file", + "body": ( + "Test PR for exercising the astro reorg tooling. Creates a conflict " + "on a top-level file that the reorg doesn't move, so it isn't a " + "reorg conflict and should go straight to manual review.\n\n" + "Do not merge." + ), + "base_edit": { + "file": "CONTRIBUTING.md", + "old": "how to write and edit content", + "new": "how to author and edit content", + }, + }, + { + # Unresolvable (Case 3): a new page plus a nav-menu link to it, where + # the base branch edited that same menu line. The new page is a + # wrong-path addition (added at a pre-reorg content/ path) AND the menu + # edit conflicts with the base, so the fix is classified reorg-caused + # but git am --3way can't replay the menu change -> manual review. + "branch": f"{BRANCH_PREFIX}-new-page-nav-conflict", + "add_file": { + "path": "content/en/getting_started/example_feature/_index.md", + "content": ( + "---\n" + "title: Example Feature\n" + "---\n\n" + "Placeholder page for exercising the astro reorg tooling.\n" + ), + }, + "file": "config/_default/menus/main.en.yaml", + # Insert a nav entry for the new page just before the Essentials heading. + "old": " - name: Essentials", + "new": ( + " - name: Example Feature\n" + " identifier: example_feature_heading\n" + " url: /getting_started/example_feature/\n" + " weight: 500000\n" + " - name: Essentials" + ), + "commit": "Test PR: add a page and a nav menu link to it", + "title": "[TEST] New page with nav link (base edited the same menu line)", + "body": ( + "Test PR for exercising the astro reorg tooling. Adds a new page and " + "a nav menu entry linking to it. The base branch renames that same " + "menu heading, so the auto-fix can't replay the menu change and the " + "PR should be labeled for manual review.\n\n" + "Do not merge." + ), + # Someone else renamed the same heading the PR inserts in front of. + "base_edit": { + "file": "hugo/config/_default/menus/main.en.yaml", + "old": " - name: Essentials", + "new": " - name: Essential Features", + }, + }, ] # --------------------------------------------------------------------------- @@ -155,7 +268,62 @@ def current_branch() -> str: # PR creation # --------------------------------------------------------------------------- -def create_pr(spec: dict) -> str | None: +def build_conflicting_base(specs: list[dict]) -> str | None: + """Build a throwaway base branch that conflicts with the base_edit specs. + + Some specs carry a `base_edit` describing a change to make ON THE BASE, at a + post-reorg (hugo/) path, on the same line the PR touches at the pre-reorg + path. That divergent edit is what makes resolve_pr_conflicts.py's auto-fix + fail (git am --3way can't reconcile the line) or — for a top-level file — + surfaces as a plain non-reorg conflict. Without it the mock base differs + from the PRs only by pure file moves, which always replay cleanly. + + Builds a fresh branch off origin/MOCK_BASE_BRANCH, unique per run so a plain + push always fast-forwards (no force-push, per repo policy), applies every + base_edit, commits, and pushes. Returns the branch name, or None if no spec + needs a conflicting base (in which case callers use MOCK_BASE_BRANCH). + """ + edits = [s["base_edit"] for s in specs if s.get("base_edit")] + if not edits: + return None + + branch = f"{MOCK_BASE_BRANCH}-conflict-{uuid.uuid4().hex[:8]}" + print(f"\n=== building conflicting base {branch} ===") + + if git("fetch", "origin", MOCK_BASE_BRANCH).returncode != 0: + die(f"git fetch origin {MOCK_BASE_BRANCH} failed.") + + checkout = git("checkout", "-b", branch, f"origin/{MOCK_BASE_BRANCH}") + if checkout.returncode != 0: + die(f"could not create branch {branch}: {checkout.stderr.strip()}") + + # base_edit paths are post-reorg (hugo/...), so they exist on this branch. + for edit in edits: + target = REPO_ROOT / edit["file"] + if not target.exists(): + die(f"base_edit target does not exist on the base branch: {edit['file']}") + text = target.read_text() + count = text.count(edit["old"]) + if count != 1: + die(f"expected {edit['old']!r} exactly once in {edit['file']}, " + f"found {count}.") + target.write_text(text.replace(edit["old"], edit["new"])) + git("add", edit["file"]) + + commit = git("commit", "-m", + "Test setup: edit reorged files so open test PRs conflict") + if commit.returncode != 0: + die(f"base commit failed: {commit.stderr.strip()}") + + push = git("push", "--set-upstream", "origin", branch) + if push.returncode != 0: + die(f"push of conflicting base failed: {push.stderr.strip()}") + + print(f" Pushed conflicting base: {branch}") + return branch + + +def create_pr(spec: dict, base: str) -> str | None: """Create the branch, apply the change, push, and open a PR. Returns URL.""" # Append a short unique suffix so every run creates a fresh branch rather # than colliding with one from a previous run. @@ -174,6 +342,17 @@ def create_pr(spec: dict) -> str | None: if checkout.returncode != 0: die(f"could not create branch {branch}: {checkout.stderr.strip()}") + # Optionally add a brand-new page at a pre-reorg path (content/...). The + # resolver detects this as a "wrong-path addition" that belongs under hugo/. + add_file = spec.get("add_file") + if add_file: + new_page = REPO_ROOT / add_file["path"] + if new_page.exists(): + die(f"add_file target already exists: {add_file['path']}") + new_page.parent.mkdir(parents=True, exist_ok=True) + new_page.write_text(add_file["content"]) + git("add", add_file["path"]) + # Apply the wording change. text = target.read_text() count = text.count(spec["old"]) @@ -196,7 +375,7 @@ def create_pr(spec: dict) -> str | None: "pr", "create", "--repo", REPO, "--head", branch, - "--base", MOCK_BASE_BRANCH, + "--base", base, "--title", spec["title"], "--body", spec["body"], "--label", LABEL, @@ -206,7 +385,7 @@ def create_pr(spec: dict) -> str | None: die(f"gh pr create failed: {pr.stderr.strip()}") url = pr.stdout.strip().splitlines()[-1] - print(f" Opened PR: {url}") + print(f" Opened PR against {base}: {url}") return url @@ -216,9 +395,19 @@ def main() -> None: original_branch = current_branch() urls: list[str] = [] + conflicting_base: str | None = None try: + # If any spec needs the base to diverge, build one throwaway conflicting + # base and point every PR at it, so a single resolve_pr_conflicts.py run + # (--base-branch ) exercises the clean auto-fix and the + # manual-review fallbacks together. Build it before cutting PR branches; + # this leaves the working tree on the base branch, but each create_pr + # checks out fresh off BRANCH_FROM anyway. + conflicting_base = build_conflicting_base(TEST_PRS) + base = conflicting_base or MOCK_BASE_BRANCH + for spec in TEST_PRS: - url = create_pr(spec) + url = create_pr(spec, base) if url: urls.append(url) finally: @@ -233,6 +422,13 @@ def main() -> None: for url in urls: print(f" {url}") + if conflicting_base: + print(f"\nThese PRs target the throwaway conflicting base " + f"{conflicting_base!r}.\nResolve them (and see the manual-review " + f"fallbacks fire) with:\n" + f" python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run " + f"--base-branch {conflicting_base}") + # Open in the browser; webbrowser.open returns False if it can't. opened_any = False for url in urls: diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 01343a3c497..def051d0b07 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -5,7 +5,7 @@ that would be made. Usage: - python3 astro_reorg/resolve_pr_conflicts.py [--no-dry-run] [--live | --base-branch BRANCH] [--pr NUMBER ...] + python3 astro_reorg/resolve_pr_conflicts.py [--no-dry-run] [--live | --base-branch BRANCH] [--pr NUMBER ...] [--limit N] Flags: --no-dry-run Actually apply fixes and labels instead of just reporting what would be done. @@ -18,6 +18,11 @@ --pr NUMBER ... Only process the given PR number(s) instead of all open PRs against the base branch. PRs targeting a different base are skipped. + --limit N Stop after acting on N PRs (auto-fixing or labeling). + PRs that need no action — mergeable, already fixed, or + not-yet-computed — don't count toward the limit. Use it + to roll out gradually: start with --limit 1, review the + results, then raise it as you gain confidence. Background: The reorg moves every entry in `moves_to_hugo` (astro_reorg/config.yaml) @@ -562,7 +567,12 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: # Per-PR analysis # --------------------------------------------------------------------------- -def analyze_pr(pr: dict, dry_run: bool) -> None: +def analyze_pr(pr: dict, dry_run: bool) -> bool: + """Process one PR. Return True if we acted on it (auto-fixed or labeled), + False if it needed no action (mergeable, already fixed, mergeability not yet + computed, or no conflicts found locally). The return value drives --limit: + only acted-on PRs count toward it. + """ pr_number = pr["number"] title = pr["title"] mergeable = pr.get("mergeable", "UNKNOWN") @@ -575,16 +585,16 @@ def analyze_pr(pr: dict, dry_run: bool) -> None: # the whole script safe to re-run. if any(l["name"] == LABEL_STALE for l in pr.get("labels", [])): print(f" Already has a fix PR ({LABEL_STALE}) — skipping.") - return + return False if mergeable == "MERGEABLE": print(" No conflicts — skipping.") - return + return False if mergeable == "UNKNOWN": # GitHub computes mergeability lazily; try again later if needed. print(" Mergeability not yet computed by GitHub — skipping.") - return + return False # CONFLICTING: fetch the branch and do a local merge test to classify # conflicts as reorg-caused vs unrelated. @@ -593,14 +603,14 @@ def analyze_pr(pr: dict, dry_run: bool) -> None: f"refs/pull/{pr_number}/head:{pr_ref.replace('refs/remotes/', '')}") if fetch.returncode != 0: print(f" fetch failed: {fetch.stderr.strip()[:120]}", file=sys.stderr) - return + return False tmpdir = tempfile.mkdtemp(prefix=f"reorg_check_{pr_number}_") try: add_wt = git("worktree", "add", "--detach", tmpdir, f"origin/{BASE_BRANCH}") if add_wt.returncode != 0: print(f" worktree add failed: {add_wt.stderr.strip()[:120]}", file=sys.stderr) - return + return False worktree = Path(tmpdir) # Attempt the merge without committing so we can inspect the conflicts. @@ -648,23 +658,25 @@ def analyze_pr(pr: dict, dry_run: bool) -> None: print(" Merge failed but no conflicts could be classified " "— labeling for manual review.") add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) - else: - print(" No conflicts found locally (GitHub mergeability may be stale).") - return + return True + print(" No conflicts found locally (GitHub mergeability may be stale).") + return False if other_conflicts: # The PR has conflicts that are NOT from the reorg. We must not # touch it — just label it so a human can resolve it manually. print(" Non-reorg conflicts present — labeling for manual review.") add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) - else: - # All conflicts are reorg-caused. Attempt the auto-fix. - print(" All conflicts are reorg-caused — attempting auto-fix.") - success = attempt_fix(pr, dry_run) - if not success: - # Auto-fix failed (fork, apply error, etc.) — fall back to labeling. - print(" Auto-fix failed or not applicable — labeling for manual review.") - add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + return True + + # All conflicts are reorg-caused. Attempt the auto-fix. + print(" All conflicts are reorg-caused — attempting auto-fix.") + success = attempt_fix(pr, dry_run) + if not success: + # Auto-fix failed (fork, apply error, etc.) — fall back to labeling. + print(" Auto-fix failed or not applicable — labeling for manual review.") + add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + return True finally: run(["git", "worktree", "remove", "--force", tmpdir]) @@ -723,6 +735,12 @@ def main() -> None: "--pr", type=int, action="append", dest="prs", metavar="NUMBER", help="Only check this PR number (may be repeated).", ) + parser.add_argument( + "--limit", type=int, default=None, metavar="N", + help="Stop after acting on N PRs (auto-fixing or labeling). PRs needing " + "no action don't count. Use it to roll out gradually: start at 1, " + "review, then raise it.", + ) parser.add_argument( "--base-branch", default=None, metavar="BRANCH", help="Branch to treat as the post-reorg base. Overrides the default " @@ -736,6 +754,9 @@ def main() -> None: ) args = parser.parse_args() + if args.limit is not None and args.limit < 1: + parser.error("--limit must be a positive integer.") + if args.live and args.base_branch: parser.error("--live and --base-branch are mutually exclusive; " "--live already selects master.") @@ -773,20 +794,27 @@ def main() -> None: prs = get_open_prs(args.prs) print(f"Found {len(prs)} open PR(s) to check.") + if args.limit is not None: + print(f"Limit: will stop after acting on {args.limit} PR(s).") + acted = 0 for pr in prs: + if args.limit is not None and acted >= args.limit: + print(f"\nReached --limit of {args.limit} acted-on PR(s) — stopping.") + break # Isolate failures: one PR raising (e.g. a transient gh/network error) # shouldn't abort the whole batch. A half-finished fix is safe to # retry — a re-run either skips it (astro-reorg-stale) or, if the fix # branch already exists, falls back to manual review. try: - analyze_pr(pr, args.dry_run) + if analyze_pr(pr, args.dry_run): + acted += 1 except Exception as exc: print(f"\nERROR processing PR #{pr.get('number', '?')}: {exc}", file=sys.stderr) print(" Skipping to the next PR.", file=sys.stderr) - print("\nDone.") + print(f"\nActed on {acted} PR(s). Done.") if __name__ == "__main__": From f5ddaeff3011a8bcc1f95e4d07d459ed8944b49c Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Mon, 13 Jul 2026 15:40:47 -0500 Subject: [PATCH 27/70] Tweak handling of GitHub actions --- astro_reorg/execute_reorg.py | 6 ++++-- astro_reorg/helpers.py | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/astro_reorg/execute_reorg.py b/astro_reorg/execute_reorg.py index 29dea2b3c6d..8ab62868f03 100644 --- a/astro_reorg/execute_reorg.py +++ b/astro_reorg/execute_reorg.py @@ -140,8 +140,10 @@ def route_gitignore_segment(line): (" static |", " hugo/static |"), ("^static/images/", "^hugo/static/images/"), ("-- 'content/en/**/*.md'", "-- 'hugo/content/en/**/*.md'"), - # bump_* and version_getter_shared workflows cp/commit data files by ./ path - ("./data/", "./hugo/data/"), + # bump_* and version_getter_shared workflows cp/commit data files by ./ path. + # Match without a trailing slash so a bare `mkdir -p ./data` is repathed too, + # not just `./data/` references. + ("./data", "./hugo/data"), ] print("\nUpdating .github/workflows/...") diff --git a/astro_reorg/helpers.py b/astro_reorg/helpers.py index cd4ec79c455..2617cba5b7f 100644 --- a/astro_reorg/helpers.py +++ b/astro_reorg/helpers.py @@ -207,8 +207,11 @@ def first_segment(token): for token in token_re.findall(line): seg, normalized = first_segment(token) # Only treat a token as a path reference if it's an actual path - # (has a '/') or is exactly the name of a moved file. - if "/" not in normalized and normalized not in moved_files: + # or is exactly the name of a moved file. Test the RAW token for + # the slash, not the ./-stripped form: `./data` is a genuine path + # reference, but normalizing it to `data` would drop the slash and + # make it read as a bare prose word. + if "/" not in token and normalized not in moved_files: continue if seg in moves_to_hugo: suspects.append(f"{yml.name}:{lineno}: {line.strip()}") From d13d13273c8f7104d6a52144410bc0c8fcbb34a9 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Mon, 13 Jul 2026 15:44:24 -0500 Subject: [PATCH 28/70] Sync validation script --- astro_reorg/helpers.py | 12 ++++++------ astro_reorg/validate_reorg.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/astro_reorg/helpers.py b/astro_reorg/helpers.py index 2617cba5b7f..40bef6311da 100644 --- a/astro_reorg/helpers.py +++ b/astro_reorg/helpers.py @@ -516,9 +516,9 @@ def check_build_presence(): def check_rollback_roundtrip(): - """execute_reorg.py then rollback.py must restore the tree byte-for-byte. + """execute_reorg.py then local_rollback.py must restore the tree byte-for-byte. - This is the only check that exercises rollback.py at all. Rather than + This is the only check that exercises local_rollback.py at all. Rather than mutate the live repo (and risk leaving it half-reorganized if a step fails), it builds a small throwaway git repo holding one representative item for every code path the two scripts touch — a mixed .gitignore (rules that route @@ -527,7 +527,7 @@ def check_rollback_roundtrip(): few moved/stayed files — then: snapshot -> run execute_reorg.py (answering y to every prompt) - -> run rollback.py -> snapshot again + -> run local_rollback.py -> snapshot again -> assert byte-identical. It specifically catches the easy-to-miss case where execute_reorg.py edits a file in @@ -610,7 +610,7 @@ def snapshot(): # The scripts resolve repo_root as their parent's parent, so place them # under an astro_reorg/ subfolder that mirrors the real repo layout. (work / "astro_reorg").mkdir() - for tool in ("execute_reorg.py", "rollback.py", "config.yaml"): + for tool in ("execute_reorg.py", "local_rollback.py", "config.yaml"): shutil.copy2(repo_root / "astro_reorg" / tool, work / "astro_reorg" / tool) git_work("init", "-q") @@ -638,11 +638,11 @@ def snapshot(): return rollback = subprocess.run( - ["python3", str(work / "astro_reorg" / "rollback.py")], + ["python3", str(work / "astro_reorg" / "local_rollback.py")], cwd=work, capture_output=True, text=True, ) if rollback.returncode != 0 or (work / "hugo").exists(): - record("FAIL", "rollback: rollback.py failed on the fixture", + record("FAIL", "rollback: local_rollback.py failed on the fixture", (rollback.stderr or rollback.stdout).strip()[:200]) return diff --git a/astro_reorg/validate_reorg.py b/astro_reorg/validate_reorg.py index 68521c5ac57..3515631a258 100644 --- a/astro_reorg/validate_reorg.py +++ b/astro_reorg/validate_reorg.py @@ -71,7 +71,7 @@ def main(): print("\n== Hugo build ==") check_build_presence() - # On a throwaway repo, execute_reorg.py then rollback.py must restore the + # On a throwaway repo, execute_reorg.py then local_rollback.py must restore the # tree byte-for-byte (the only test of rollback). print("\n== Rollback round-trip ==") check_rollback_roundtrip() From e3725d63b4e0080c4c7690727d14255e7bdfa7c5 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 12:16:32 -0500 Subject: [PATCH 29/70] Align resolution script with contributor instructions --- astro_reorg/resolve_pr_conflicts.py | 82 ++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index def051d0b07..8dce9b7d8d9 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -72,10 +72,10 @@ the PR's base and today. 5. Push as `reorg-fix/pr-`, open a new PR for it, comment on the original PR pointing to the fix PR, and label the original - astro-reorg-stale. + has-astro-reorg-conflicts. - A PR that already carries the astro-reorg-stale label (a fix PR was opened - on a prior run) is skipped, so re-running the script is safe. + A PR that already carries the has-astro-reorg-conflicts label (a fix PR was + opened on a prior run) is skipped, so re-running the script is safe. PRs from forks cannot be auto-fixed (we don't have push access to the fork). They receive the astro-reorg-manual-review label. @@ -89,6 +89,7 @@ import subprocess import sys import tempfile +from datetime import datetime, timedelta, timezone from pathlib import Path try: @@ -119,10 +120,29 @@ REPO = "DataDog/documentation" LABEL_MANUAL_REVIEW = "astro-reorg-manual-review" -LABEL_STALE = "astro-reorg-stale" +LABEL_STALE = "has-astro-reorg-conflicts" +LABEL_HELP_REQUESTED = "astro-reorg-help-requested" +LABEL_WIP = "WORK IN PROGRESS" LABEL_COLOR = "e4e669" LABEL_DESCRIPTION = "Needs manual conflict resolution after replatforming reorg" +# PRs with no activity in this many days are treated as stale and receive a +# comment + label instead of an auto-fix attempt. +STALE_DAYS = 62 # ~2 calendar months + +MANUAL_REVIEW_COMMENT = ( + "This PR has merge conflicts from a recent repo reorg that could not be resolved automatically. " + "It has been queued for manual review by the Webops-Platform team." +) + +STALE_COMMENT = ( + "This PR has conflicts created by the docs repo reorg project. " + "Because this PR is stale (more than two months old), no action was taken. " + "If you still intend to use this PR and would like assistance resolving the conflicts, " + "add the label `astro-reorg-help-requested` to your PR to add it to our help queue. " + "We will reach out to you as soon as we can." +) + # Existing repo label applied to auto-created fix PRs in test mode so they stay # out of other teams' review queues. Assumed to already exist in the repo. DO_NOT_MERGE_LABEL = "Do Not Merge" @@ -165,6 +185,20 @@ def git(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess: return run(["git", *args], cwd=cwd or REPO_ROOT) +# --------------------------------------------------------------------------- +# Staleness helpers +# --------------------------------------------------------------------------- + +def is_pr_stale(pr: dict) -> bool: + """Return True if the PR has had no activity in the last STALE_DAYS days.""" + updated_at = pr.get("updatedAt") + if not updated_at: + return False + last_update = datetime.fromisoformat(updated_at.replace("Z", "+00:00")) + cutoff = datetime.now(timezone.utc) - timedelta(days=STALE_DAYS) + return last_update < cutoff + + # --------------------------------------------------------------------------- # Reorg path helpers # --------------------------------------------------------------------------- @@ -471,6 +505,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: print(f" ... and {len(subjects) - 10} more") would_be_title = f"[reorg fix] {pr['title']}" print(f" [dry-run] would open PR: {would_be_title!r}") + print(f" [dry-run] would label fix PR {LABEL_WIP!r}") if IS_TEST_MODE: print(f" [dry-run] would label fix PR {DO_NOT_MERGE_LABEL!r} (test mode)") return True @@ -540,6 +575,8 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: new_pr_number = new_pr_url.rstrip("/").split("/")[-1] print(f" Opened fix PR: {new_pr_url}") + add_label(int(new_pr_number), LABEL_WIP, dry_run=False) + # In test mode, keep the auto-created PR out of other teams' review # queues by marking it "Do Not Merge". Never do this on real master. if IS_TEST_MODE: @@ -547,12 +584,12 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: post_comment( pr_number, - f"🤖 **Reorg conflict auto-fix: #{new_pr_number}**\n\n" - f"This PR has merge conflicts caused by the astro reorg " + f"🤖 **Reorg conflict auto-fix:**\n\n" + f"This PR has merge conflicts caused by the recent docs repo reorg " f"(files moved from the repo root into `hugo/`). " f"A new PR with your commits translated to the correct paths " f"has been opened: #{new_pr_number}\n\n" - f"If #{new_pr_number} looks correct, merge it and close this PR.", + f"If the new PR looks correct, merge it and close this PR.", dry_run=False, ) add_label(pr_number, LABEL_STALE, dry_run) @@ -580,11 +617,11 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: print(f"\nPR #{pr_number}: {title}") print(f" mergeable: {mergeable}") - # A fix PR was already opened for this one on a prior run — skip so we - # don't re-push the fix branch or post a duplicate comment. This makes - # the whole script safe to re-run. + # A fix PR was already opened for this one (or it was previously noted as + # stale) on a prior run — skip so we don't re-push the fix branch or post + # a duplicate comment. This makes the whole script safe to re-run. if any(l["name"] == LABEL_STALE for l in pr.get("labels", [])): - print(f" Already has a fix PR ({LABEL_STALE}) — skipping.") + print(f" Already handled ({LABEL_STALE}) — skipping.") return False if mergeable == "MERGEABLE": @@ -669,6 +706,21 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) return True + # All conflicts are reorg-caused. Check staleness before attempting + # an auto-fix: if the PR has had no activity in >2 months, note it and + # leave it for the author to decide whether they still need it. + # Exception: if the author already added `astro-reorg-help-requested`, + # they have explicitly opted back in — treat it like a fresh PR. + help_requested = any( + l["name"] == LABEL_HELP_REQUESTED for l in pr.get("labels", []) + ) + if is_pr_stale(pr) and not help_requested: + print(f" PR is stale (no activity in >{STALE_DAYS} days) — " + "labeling and commenting without auto-fix.") + add_label(pr_number, LABEL_STALE, dry_run) + post_comment(pr_number, STALE_COMMENT, dry_run) + return True + # All conflicts are reorg-caused. Attempt the auto-fix. print(" All conflicts are reorg-caused — attempting auto-fix.") success = attempt_fix(pr, dry_run) @@ -700,7 +752,7 @@ def get_open_prs(only: list[int] | None = None) -> list[dict]: # requested: `gh pr list` doesn't support them (only `gh pr view` does), and # nothing here uses them. fields = ("number,title,body,labels,headRefName," - "baseRefName,isCrossRepository,mergeable") + "baseRefName,isCrossRepository,mergeable,updatedAt") if only: # Explicitly named PRs are an intentional override, but still guard # against acting on a PR that targets a different base than this run. @@ -792,6 +844,12 @@ def main() -> None: ensure_label_exists(LABEL_MANUAL_REVIEW, args.dry_run) ensure_label_exists(LABEL_STALE, args.dry_run) + existing_labels = gh_json("label", "list", "--repo", REPO, "--json", "name") + if not any(l["name"] == LABEL_WIP for l in existing_labels): # type: ignore[index] + print(f"Error: label {LABEL_WIP!r} does not exist in the repo. " + f"Check that the label name is correct.", file=sys.stderr) + sys.exit(1) + prs = get_open_prs(args.prs) print(f"Found {len(prs)} open PR(s) to check.") if args.limit is not None: From 02b9fcc75fc045ccb01069ea92d12d897bda6d70 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 12:26:58 -0500 Subject: [PATCH 30/70] Update mock master branch name --- astro_reorg/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index 8cd8c9daf7d..1418f0a44cc 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -132,7 +132,7 @@ ignore: test: # Stand-in for post-reorg master: the branch that has the reorg applied. Test # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. - mock_reorged_master_branch: jen.gilbert/astro-reorg-demo-7-10-22047 + mock_reorged_master_branch: mock-reorged-master-122011 # Frozen snapshot of master that test PRs branch off, so each PR's diff stays # small (a PR shows every commit in its head that isn't in the base branch). branch_from: jen.gilbert/astro-reorg-scripts From 8f46746ba6c9a38ae94147e8aa301d7cadb7f50f Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 12:33:29 -0500 Subject: [PATCH 31/70] Resolve script errors --- astro_reorg/create_test_prs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index 913c1fd8a6d..e9f0ae9cc34 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -30,6 +30,8 @@ classified reorg-caused but can't be replayed (manual-review fallback). - A base_edit on a top-level (never-moved) file surfaces as a plain non-reorg conflict, which is sent straight to manual review. + +View all test PRs: https://github.com/DataDog/documentation/pulls?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-testing """ from __future__ import annotations @@ -331,9 +333,6 @@ def create_pr(spec: dict, base: str) -> str | None: target = REPO_ROOT / spec["file"] print(f"\n=== {branch} ===") - if not target.exists(): - die(f"target file does not exist: {spec['file']}") - # Start from the frozen snapshot so the PR diff is just this change. if git("fetch", "origin", BRANCH_FROM).returncode != 0: die(f"git fetch origin {BRANCH_FROM} failed.") @@ -342,6 +341,11 @@ def create_pr(spec: dict, base: str) -> str | None: if checkout.returncode != 0: die(f"could not create branch {branch}: {checkout.stderr.strip()}") + # Check after checkout: the file exists on BRANCH_FROM (pre-reorg) but not + # on the conflicting base branch (post-reorg), so the check must run here. + if not target.exists(): + die(f"target file does not exist: {spec['file']}") + # Optionally add a brand-new page at a pre-reorg path (content/...). The # resolver detects this as a "wrong-path addition" that belongs under hugo/. add_file = spec.get("add_file") From b509301868314ed740cd7215013a03beb2c1911d Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 12:42:19 -0500 Subject: [PATCH 32/70] Resolve script errors --- astro_reorg/resolve_pr_conflicts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 8dce9b7d8d9..3159a5cc140 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -389,7 +389,7 @@ def transform_diff_paths(diff_text: str) -> str: def ensure_label_exists(label: str, dry_run: bool) -> None: """Create the GitHub label if it doesn't already exist.""" - existing = gh_json("label", "list", "--repo", REPO, "--json", "name") + existing = gh_json("label", "list", "--repo", REPO, "--search", label, "--json", "name") if any(l["name"] == label for l in existing): # type: ignore[index] return if dry_run: @@ -844,7 +844,7 @@ def main() -> None: ensure_label_exists(LABEL_MANUAL_REVIEW, args.dry_run) ensure_label_exists(LABEL_STALE, args.dry_run) - existing_labels = gh_json("label", "list", "--repo", REPO, "--json", "name") + existing_labels = gh_json("label", "list", "--repo", REPO, "--search", LABEL_WIP, "--json", "name") if not any(l["name"] == LABEL_WIP for l in existing_labels): # type: ignore[index] print(f"Error: label {LABEL_WIP!r} does not exist in the repo. " f"Check that the label name is correct.", file=sys.stderr) From 1d681f92e91189f5913706389237f18b43f15a12 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 13:08:48 -0500 Subject: [PATCH 33/70] Add test setup instructions --- astro_reorg/test_setup.md | 51 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 astro_reorg/test_setup.md diff --git a/astro_reorg/test_setup.md b/astro_reorg/test_setup.md new file mode 100644 index 00000000000..0915bb5636d --- /dev/null +++ b/astro_reorg/test_setup.md @@ -0,0 +1,51 @@ +# Test setup for resolve_pr_conflicts.py + +## Prerequisites + +Two branches must exist on the remote (configured in `config.yaml` under `test:`): + +- A non-reorged mock master branch for the mock contributors to branch from. +- A reorged mock master branch that acts as a base for the PR. This usually doesn't need to be created, since we can just use `jen.gilbert/astro-reorg-scripts` as the base. + +## Setup steps + +### 1. Choose a branch name for the reorged mock master + +**Do not create the branch yet.** + +Choose a name for the reorged mock master you intend to create, such as `mock-reorged-master-7-14-1258` (this example just used the date and current time to make the branch name unique). + +### 2. Update the script config + +Update `mock_reorged_master_branch` in `config.yaml` to match your intended branch name. Commit the change. + +### 3. Create the mock reorged master branch + +Replace `` with the branch name you chose above. + +```bash +git checkout jen.gilbert/astro-reorg-scripts +git checkout -b +python3 astro_reorg/execute_reorg.py +git add -- . ':!astro' +git commit -m "Apply reorg" +git push --set-upstream origin +``` + +## Running the test + +```bash +# 1. Create the four test PRs. Copy the --base-branch command from the output. +python3 astro_reorg/create_test_prs.py + +# 2. Run the resolver against them (dry run by default). +python3 astro_reorg/resolve_pr_conflicts.py \ + --base-branch mock-reorged-master-122011-conflict- + +# 3. Run for real. +python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run \ + --base-branch mock-reorged-master-122011-conflict- +``` + +Expected outcomes: the wording-tweak PR gets an auto-fix; the other three get +the `astro-reorg-manual-review` label. From 4f495250137a3f1216dcbaa93c9ce0c4f13ff9e8 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 13:14:47 -0500 Subject: [PATCH 34/70] Tweak test setup instructions --- astro_reorg/test_setup.md | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/astro_reorg/test_setup.md b/astro_reorg/test_setup.md index 0915bb5636d..9ddee8fc7b0 100644 --- a/astro_reorg/test_setup.md +++ b/astro_reorg/test_setup.md @@ -32,19 +32,41 @@ git commit -m "Apply reorg" git push --set-upstream origin ``` +### 4. Switch back to `jen.gilbert/astro-reorg-scripts` + +Switch back to the Astro reorg scripts branch: + +```bash +git checkout jen.gilbert/astro-reorg-scripts +``` + +Revert any lingering (untracked) reorg changes: + +```bash +python3 astro_reorg/local_rollback.py +``` + ## Running the test +### 1. Create the test PRs + +Create the test PRs: + ```bash -# 1. Create the four test PRs. Copy the --base-branch command from the output. python3 astro_reorg/create_test_prs.py +``` + +This script will output the commands you should use to execute a dry run and a real run of the automatic conflict resolution script. + +### 2. Execute a dry run -# 2. Run the resolver against them (dry run by default). python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch mock-reorged-master-122011-conflict- + --base-branch + +### 3. Execute a real run -# 3. Run for real. python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run \ - --base-branch mock-reorged-master-122011-conflict- + --base-branch ``` Expected outcomes: the wording-tweak PR gets an auto-fix; the other three get From 5fa9ce9f552a125c61345dcbff66759142c65ccf Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 13:18:22 -0500 Subject: [PATCH 35/70] Set new mock reorged master branch --- astro_reorg/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index 1418f0a44cc..a7042407ae4 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -132,7 +132,7 @@ ignore: test: # Stand-in for post-reorg master: the branch that has the reorg applied. Test # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. - mock_reorged_master_branch: mock-reorged-master-122011 + mock_reorged_master_branch: mock-reorged-master-7-14-118 # Frozen snapshot of master that test PRs branch off, so each PR's diff stays # small (a PR shows every commit in its head that isn't in the base branch). branch_from: jen.gilbert/astro-reorg-scripts From 1b9ad6d5c4780d075248938ae854dd1f6921ca45 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 13:44:43 -0500 Subject: [PATCH 36/70] Tweak test cases --- astro_reorg/create_test_prs.py | 39 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index e9f0ae9cc34..55313bbdd06 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -97,15 +97,20 @@ TEST_PRS = [ { # Resolvable: reorg-only conflict, auto-fix replays it cleanly. + # + # IMPORTANT: this edit must target a different LINE than any base_edit + # in the other specs. Git merges at line granularity, so even two edits + # to different words on the same long line will conflict. The description + # frontmatter field is on its own line and no base_edit touches it. "branch": f"{BRANCH_PREFIX}-wording", "file": "content/en/getting_started/_index.md", - "old": "supports every phase of software development", - "new": "supports each phase of software development", - "commit": "Test PR: minor wording tweak in getting started intro", + "old": "with guides for installation, configuration, and getting started with key features.", + "new": "with guides for setup, configuration, and getting started with key features.", + "commit": "Test PR: minor wording tweak in getting started description", "title": "[TEST] Minor wording tweak in getting started intro", "body": ( "Test PR for exercising the astro reorg tooling. Makes a minor, " - "non-material wording change in the getting started intro paragraph.\n\n" + "non-material wording change in the getting started page description.\n\n" "Do not merge." ), }, @@ -426,22 +431,16 @@ def main() -> None: for url in urls: print(f" {url}") - if conflicting_base: - print(f"\nThese PRs target the throwaway conflicting base " - f"{conflicting_base!r}.\nResolve them (and see the manual-review " - f"fallbacks fire) with:\n" - f" python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run " - f"--base-branch {conflicting_base}") - - # Open in the browser; webbrowser.open returns False if it can't. - opened_any = False - for url in urls: - try: - opened_any = webbrowser.open(url) or opened_any - except Exception: - pass - if not opened_any: - print("\nCould not open a browser. Use the links above.") + base_flag = f"--base-branch {conflicting_base}" if conflicting_base else "--live" + print( + f"\nView all test PRs:\n" + f" https://github.com/DataDog/documentation/pulls" + f"?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-testing\n" + f"\nDry-run conflict resolution (no changes made):\n" + f" python3 astro_reorg/resolve_pr_conflicts.py {base_flag}\n" + f"\nLive conflict resolution (applies fixes and labels):\n" + f" python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run {base_flag}" + ) if __name__ == "__main__": From d447fb7dd821e68295d623dd1305f98223bbc72d Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 13:49:00 -0500 Subject: [PATCH 37/70] Update mock reorged master branch --- astro_reorg/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index a7042407ae4..bd1854c1ae4 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -132,7 +132,7 @@ ignore: test: # Stand-in for post-reorg master: the branch that has the reorg applied. Test # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. - mock_reorged_master_branch: mock-reorged-master-7-14-118 + mock_reorged_master_branch: mock-reorged-master-7-14-148 # Frozen snapshot of master that test PRs branch off, so each PR's diff stays # small (a PR shows every commit in its head that isn't in the base branch). branch_from: jen.gilbert/astro-reorg-scripts From 4f793fdc2c11299f8ed2d14075f0c1668745118a Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 14:19:11 -0500 Subject: [PATCH 38/70] Add clearer labels/PR titles --- astro_reorg/create_test_prs.py | 8 ++++---- astro_reorg/resolve_pr_conflicts.py | 9 ++++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index 55313bbdd06..ecac49d9606 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -107,7 +107,7 @@ "old": "with guides for installation, configuration, and getting started with key features.", "new": "with guides for setup, configuration, and getting started with key features.", "commit": "Test PR: minor wording tweak in getting started description", - "title": "[TEST] Minor wording tweak in getting started intro", + "title": "[TEST] Minor wording tweak in getting started intro (auto-fixable)", "body": ( "Test PR for exercising the astro reorg tooling. Makes a minor, " "non-material wording change in the getting started page description.\n\n" @@ -123,7 +123,7 @@ "old": "combined into a customized solution", "new": "combined into a unified solution", "commit": "Test PR: edit a line the reorged base also changed", - "title": "[TEST] Unresolvable reorg conflict (base edited the same line)", + "title": "[TEST] Unresolvable reorg conflict (base edited the same line) (not auto-fixable)", "body": ( "Test PR for exercising the astro reorg tooling. Edits a line that " "the reorged base branch also changed, so the auto-fix can't replay " @@ -145,7 +145,7 @@ "old": "how to write and edit content", "new": "how to write and revise content", "commit": "Test PR: edit a top-level file the base also changed", - "title": "[TEST] Non-reorg conflict on a top-level file", + "title": "[TEST] Non-reorg conflict on a top-level file (not auto-fixable)", "body": ( "Test PR for exercising the astro reorg tooling. Creates a conflict " "on a top-level file that the reorg doesn't move, so it isn't a " @@ -185,7 +185,7 @@ " - name: Essentials" ), "commit": "Test PR: add a page and a nav menu link to it", - "title": "[TEST] New page with nav link (base edited the same menu line)", + "title": "[TEST] New page with nav link (base edited the same menu line) (not auto-fixable)", "body": ( "Test PR for exercising the astro reorg tooling. Adds a new page and " "a nav menu entry linking to it. The base branch renames that same " diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 3159a5cc140..4725041f47a 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -121,6 +121,8 @@ REPO = "DataDog/documentation" LABEL_MANUAL_REVIEW = "astro-reorg-manual-review" LABEL_STALE = "has-astro-reorg-conflicts" +LABEL_AUTOFIXED = "astro-reorg-autofixed" +LABEL_AUTO_PR = "astro-reorg-auto-pr" LABEL_HELP_REQUESTED = "astro-reorg-help-requested" LABEL_WIP = "WORK IN PROGRESS" LABEL_COLOR = "e4e669" @@ -505,7 +507,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: print(f" ... and {len(subjects) - 10} more") would_be_title = f"[reorg fix] {pr['title']}" print(f" [dry-run] would open PR: {would_be_title!r}") - print(f" [dry-run] would label fix PR {LABEL_WIP!r}") + print(f" [dry-run] would label fix PR {LABEL_WIP!r}, {LABEL_AUTO_PR!r}") if IS_TEST_MODE: print(f" [dry-run] would label fix PR {DO_NOT_MERGE_LABEL!r} (test mode)") return True @@ -576,6 +578,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: print(f" Opened fix PR: {new_pr_url}") add_label(int(new_pr_number), LABEL_WIP, dry_run=False) + add_label(int(new_pr_number), LABEL_AUTO_PR, dry_run=False) # In test mode, keep the auto-created PR out of other teams' review # queues by marking it "Do Not Merge". Never do this on real master. @@ -593,6 +596,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: dry_run=False, ) add_label(pr_number, LABEL_STALE, dry_run) + add_label(pr_number, LABEL_AUTOFIXED, dry_run) return True finally: @@ -728,6 +732,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: # Auto-fix failed (fork, apply error, etc.) — fall back to labeling. print(" Auto-fix failed or not applicable — labeling for manual review.") add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + post_comment(pr_number, MANUAL_REVIEW_COMMENT, dry_run) return True finally: @@ -843,6 +848,8 @@ def main() -> None: ensure_label_exists(LABEL_MANUAL_REVIEW, args.dry_run) ensure_label_exists(LABEL_STALE, args.dry_run) + ensure_label_exists(LABEL_AUTOFIXED, args.dry_run) + ensure_label_exists(LABEL_AUTO_PR, args.dry_run) existing_labels = gh_json("label", "list", "--repo", REPO, "--search", LABEL_WIP, "--json", "name") if not any(l["name"] == LABEL_WIP for l in existing_labels): # type: ignore[index] From 2dec23fe8af6eff36ca8a8b8cf510dd4a6183780 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 14:22:03 -0500 Subject: [PATCH 39/70] Add link for viewing auto-fixed PRs --- astro_reorg/resolve_pr_conflicts.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 4725041f47a..378e0314d6b 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -880,6 +880,11 @@ def main() -> None: print(" Skipping to the next PR.", file=sys.stderr) print(f"\nActed on {acted} PR(s). Done.") + print( + "\nView auto-fix PRs:\n" + " https://github.com/DataDog/documentation/pulls" + "?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-auto-pr" + ) if __name__ == "__main__": From 83375b1b5273e70416b364f808305e90fb597842 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 14:24:26 -0500 Subject: [PATCH 40/70] Add link for viewing auto-fixed PRs --- astro_reorg/create_test_prs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index ecac49d9606..333da0e481b 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -31,8 +31,8 @@ - A base_edit on a top-level (never-moved) file surfaces as a plain non-reorg conflict, which is sent straight to manual review. -View all test PRs: https://github.com/DataDog/documentation/pulls?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-testing -""" +View all test PRs: +"""https://github.com/DataDog/documentation/pulls?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-testing from __future__ import annotations import subprocess From c3f96ced1f006db2ea648d293e97ca68bc8e85f4 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 14:26:33 -0500 Subject: [PATCH 41/70] Update the mock reorged master branch name --- astro_reorg/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index bd1854c1ae4..f8d456784c6 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -132,7 +132,7 @@ ignore: test: # Stand-in for post-reorg master: the branch that has the reorg applied. Test # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. - mock_reorged_master_branch: mock-reorged-master-7-14-148 + mock_reorged_master_branch: mock-reorged-master-7-14-226 # Frozen snapshot of master that test PRs branch off, so each PR's diff stays # small (a PR shows every commit in its head that isn't in the base branch). branch_from: jen.gilbert/astro-reorg-scripts From a9801308d51211f6c5c07ea36b63a40b3297c553 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 14:35:30 -0500 Subject: [PATCH 42/70] Fix syntax error --- astro_reorg/create_test_prs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index 333da0e481b..8b174742255 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -31,8 +31,9 @@ - A base_edit on a top-level (never-moved) file surfaces as a plain non-reorg conflict, which is sent straight to manual review. -View all test PRs: -"""https://github.com/DataDog/documentation/pulls?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-testing +View all test PRs: + https://github.com/DataDog/documentation/pulls?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-testing +""" from __future__ import annotations import subprocess From cbafe2977e61e63765f738e65b1bcc8049df2e1e Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 14:36:13 -0500 Subject: [PATCH 43/70] Update mock reorged master branch name --- astro_reorg/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index f8d456784c6..99b47d8c78d 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -132,7 +132,7 @@ ignore: test: # Stand-in for post-reorg master: the branch that has the reorg applied. Test # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. - mock_reorged_master_branch: mock-reorged-master-7-14-226 + mock_reorged_master_branch: mock-reorged-master-7-14-235 # Frozen snapshot of master that test PRs branch off, so each PR's diff stays # small (a PR shows every commit in its head that isn't in the base branch). branch_from: jen.gilbert/astro-reorg-scripts From bf015bfe8ccdc591d9f37334538f44b9d6e126e1 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 15:11:09 -0500 Subject: [PATCH 44/70] Tweak labeling approach --- astro_reorg/resolve_pr_conflicts.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 378e0314d6b..824f4c36f7a 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -124,6 +124,7 @@ LABEL_AUTOFIXED = "astro-reorg-autofixed" LABEL_AUTO_PR = "astro-reorg-auto-pr" LABEL_HELP_REQUESTED = "astro-reorg-help-requested" +LABEL_PROCESSED = "astro-reorg-processed" LABEL_WIP = "WORK IN PROGRESS" LABEL_COLOR = "e4e669" LABEL_DESCRIPTION = "Needs manual conflict resolution after replatforming reorg" @@ -621,11 +622,11 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: print(f"\nPR #{pr_number}: {title}") print(f" mergeable: {mergeable}") - # A fix PR was already opened for this one (or it was previously noted as - # stale) on a prior run — skip so we don't re-push the fix branch or post - # a duplicate comment. This makes the whole script safe to re-run. - if any(l["name"] == LABEL_STALE for l in pr.get("labels", [])): - print(f" Already handled ({LABEL_STALE}) — skipping.") + # Already processed on a prior run — skip regardless of other labels + # (including astro-reorg-help-requested). This makes the script safe to + # re-run without re-processing or double-commenting. + if any(l["name"] == LABEL_PROCESSED for l in pr.get("labels", [])): + print(f" Already processed ({LABEL_PROCESSED}) — skipping.") return False if mergeable == "MERGEABLE": @@ -699,6 +700,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: print(" Merge failed but no conflicts could be classified " "— labeling for manual review.") add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + add_label(pr_number, LABEL_PROCESSED, dry_run) return True print(" No conflicts found locally (GitHub mergeability may be stale).") return False @@ -708,6 +710,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: # touch it — just label it so a human can resolve it manually. print(" Non-reorg conflicts present — labeling for manual review.") add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + add_label(pr_number, LABEL_PROCESSED, dry_run) return True # All conflicts are reorg-caused. Check staleness before attempting @@ -722,6 +725,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: print(f" PR is stale (no activity in >{STALE_DAYS} days) — " "labeling and commenting without auto-fix.") add_label(pr_number, LABEL_STALE, dry_run) + add_label(pr_number, LABEL_PROCESSED, dry_run) post_comment(pr_number, STALE_COMMENT, dry_run) return True @@ -733,6 +737,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: print(" Auto-fix failed or not applicable — labeling for manual review.") add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) post_comment(pr_number, MANUAL_REVIEW_COMMENT, dry_run) + add_label(pr_number, LABEL_PROCESSED, dry_run) return True finally: @@ -774,6 +779,7 @@ def get_open_prs(only: list[int] | None = None) -> list[dict]: return gh_json( # type: ignore[return-value] "pr", "list", "--repo", REPO, "--state", "open", "--base", BASE_BRANCH, + "--search", f"-label:{LABEL_PROCESSED}", "--json", fields, "--limit", "300", ) @@ -850,6 +856,7 @@ def main() -> None: ensure_label_exists(LABEL_STALE, args.dry_run) ensure_label_exists(LABEL_AUTOFIXED, args.dry_run) ensure_label_exists(LABEL_AUTO_PR, args.dry_run) + ensure_label_exists(LABEL_PROCESSED, args.dry_run) existing_labels = gh_json("label", "list", "--repo", REPO, "--search", LABEL_WIP, "--json", "name") if not any(l["name"] == LABEL_WIP for l in existing_labels): # type: ignore[index] From 60e693a758a9780ce35883090ccff6935c9a834e Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 15:14:00 -0500 Subject: [PATCH 45/70] Require a PR limit --- astro_reorg/create_test_prs.py | 5 +++-- astro_reorg/resolve_pr_conflicts.py | 19 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index 8b174742255..555a9e266a0 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -433,14 +433,15 @@ def main() -> None: print(f" {url}") base_flag = f"--base-branch {conflicting_base}" if conflicting_base else "--live" + n = len(urls) print( f"\nView all test PRs:\n" f" https://github.com/DataDog/documentation/pulls" f"?q=is%3Apr+is%3Aopen+label%3Aastro-reorg-testing\n" f"\nDry-run conflict resolution (no changes made):\n" - f" python3 astro_reorg/resolve_pr_conflicts.py {base_flag}\n" + f" python3 astro_reorg/resolve_pr_conflicts.py {base_flag} --limit {n}\n" f"\nLive conflict resolution (applies fixes and labels):\n" - f" python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run {base_flag}" + f" python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run {base_flag} --limit {n}" ) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 824f4c36f7a..3f204ee9004 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -749,7 +749,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: # Entry point # --------------------------------------------------------------------------- -def get_open_prs(only: list[int] | None = None) -> list[dict]: +def get_open_prs(only: list[int] | None = None, limit: int = 50) -> list[dict]: """Return open PRs targeting BASE_BRANCH, optionally filtered to specific numbers. The --base filter is what keeps a run scoped: in test mode only PRs opened @@ -780,7 +780,7 @@ def get_open_prs(only: list[int] | None = None) -> list[dict]: "pr", "list", "--repo", REPO, "--state", "open", "--base", BASE_BRANCH, "--search", f"-label:{LABEL_PROCESSED}", - "--json", fields, "--limit", "300", + "--json", fields, "--limit", str(limit), ) @@ -799,9 +799,9 @@ def main() -> None: help="Only check this PR number (may be repeated).", ) parser.add_argument( - "--limit", type=int, default=None, metavar="N", - help="Stop after acting on N PRs (auto-fixing or labeling). PRs needing " - "no action don't count. Use it to roll out gradually: start at 1, " + "--limit", type=int, required=True, metavar="N", + help="Fetch and act on at most N PRs. PRs needing no action don't count " + "toward the limit. Use it to roll out gradually: start at 1, " "review, then raise it.", ) parser.add_argument( @@ -817,7 +817,7 @@ def main() -> None: ) args = parser.parse_args() - if args.limit is not None and args.limit < 1: + if args.limit < 1: parser.error("--limit must be a positive integer.") if args.live and args.base_branch: @@ -864,14 +864,13 @@ def main() -> None: f"Check that the label name is correct.", file=sys.stderr) sys.exit(1) - prs = get_open_prs(args.prs) + prs = get_open_prs(args.prs, limit=args.limit) print(f"Found {len(prs)} open PR(s) to check.") - if args.limit is not None: - print(f"Limit: will stop after acting on {args.limit} PR(s).") + print(f"Limit: will stop after acting on {args.limit} PR(s).") acted = 0 for pr in prs: - if args.limit is not None and acted >= args.limit: + if acted >= args.limit: print(f"\nReached --limit of {args.limit} acted-on PR(s) — stopping.") break # Isolate failures: one PR raising (e.g. a transient gh/network error) From 1a940410d2c65ee23ec330feb64e6f8aa9f084bd Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 15:21:15 -0500 Subject: [PATCH 46/70] Tweak skip cases --- astro_reorg/resolve_pr_conflicts.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 3f204ee9004..e7edf9e45dc 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -146,6 +146,14 @@ "We will reach out to you as soon as we can." ) +WIP_COMMENT = ( + "This PR has merge conflicts created by the docs repo reorg project. " + "Because this PR is marked as a work in progress, no action was taken. " + "When you're ready and would like assistance resolving the conflicts, " + "add the label `astro-reorg-help-requested` to your PR to add it to our help queue. " + "We will reach out to you as soon as we can." +) + # Existing repo label applied to auto-created fix PRs in test mode so they stay # out of other teams' review queues. Assumed to already exist in the repo. DO_NOT_MERGE_LABEL = "Do Not Merge" @@ -713,14 +721,21 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: add_label(pr_number, LABEL_PROCESSED, dry_run) return True - # All conflicts are reorg-caused. Check staleness before attempting - # an auto-fix: if the PR has had no activity in >2 months, note it and - # leave it for the author to decide whether they still need it. + # All conflicts are reorg-caused. Before attempting an auto-fix, skip + # PRs that aren't ready: WIP and stale PRs both get a comment pointing + # the author to astro-reorg-help-requested when they want help. # Exception: if the author already added `astro-reorg-help-requested`, # they have explicitly opted back in — treat it like a fresh PR. help_requested = any( l["name"] == LABEL_HELP_REQUESTED for l in pr.get("labels", []) ) + is_wip = any(l["name"] == LABEL_WIP for l in pr.get("labels", [])) + if is_wip and not help_requested: + print(" PR is marked WIP — skipping auto-fix, leaving comment.") + add_label(pr_number, LABEL_PROCESSED, dry_run) + post_comment(pr_number, WIP_COMMENT, dry_run) + return True + if is_pr_stale(pr) and not help_requested: print(f" PR is stale (no activity in >{STALE_DAYS} days) — " "labeling and commenting without auto-fix.") From 7c038cb28d2ce2e9b85001ddf52c5a29386e3058 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Tue, 14 Jul 2026 15:21:26 -0500 Subject: [PATCH 47/70] Add test cases --- astro_reorg/create_test_prs.py | 58 ++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index 555a9e266a0..bd927ff8907 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -115,6 +115,23 @@ "Do not merge." ), }, + { + # Auto-fixable, but marked WIP — resolve_pr_conflicts.py should skip it + # and leave a comment telling the author to add astro-reorg-help-requested. + # Uses a different file from the other specs to avoid line-level collisions. + "branch": f"{BRANCH_PREFIX}-wip", + "file": "content/en/getting_started/learning_center.md", + "old": "Access hands-on courses and tutorials to learn Datadog platform features, best practices, and implementation strategies.", + "new": "Access hands-on courses and tutorials to learn Datadog platform features, best practices, and deployment strategies.", + "commit": "Test PR: minor wording tweak in learning center description (WIP)", + "title": "[TEST] WIP PR with reorg conflict (should be skipped by resolver)", + "body": ( + "Test PR for exercising the astro reorg tooling. Auto-fixable reorg conflict, " + "but marked WIP so the resolver should leave a comment and skip it.\n\n" + "Do not merge." + ), + "extra_labels": ["WORK IN PROGRESS"], + }, { # Unresolvable (Case 2): reorg-classified, but the base changed the same # line at the hugo/ path, so git am --3way can't replay it. Falls back @@ -159,6 +176,41 @@ "new": "how to author and edit content", }, }, + { + # Auto-fixable: a new page plus a nav-menu link to it, where nobody else + # touched that menu line. Exercises the "wrong-path addition + menu edit" + # case where both changes can be replayed cleanly by the auto-fix. + # Uses a different menu insertion point ("Agent" heading) than the + # unresolvable spec below ("Essentials" heading) to avoid line collisions. + "branch": f"{BRANCH_PREFIX}-new-page-nav-autofixable", + "add_file": { + "path": "content/en/getting_started/example_auto_fix/_index.md", + "content": ( + "---\n" + "title: Example Auto Fix\n" + "---\n\n" + "Placeholder page for exercising the astro reorg tooling.\n" + ), + }, + "file": "config/_default/menus/main.en.yaml", + "old": " - name: Agent\n identifier: getting_started_agent", + "new": ( + " - name: Example Auto Fix\n" + " identifier: example_auto_fix_heading\n" + " url: /getting_started/example_auto_fix/\n" + " weight: 500000\n" + " - name: Agent\n" + " identifier: getting_started_agent" + ), + "commit": "Test PR: add a page and a nav menu link (no base conflict)", + "title": "[TEST] New page with nav link, no base conflict (auto-fixable)", + "body": ( + "Test PR for exercising the astro reorg tooling. Adds a new page and " + "a nav menu entry linking to it. The base branch does not touch that " + "menu line, so the auto-fix should replay both changes cleanly.\n\n" + "Do not merge." + ), + }, { # Unresolvable (Case 3): a new page plus a nav-menu link to it, where # the base branch edited that same menu line. The new page is a @@ -381,6 +433,9 @@ def create_pr(spec: dict, base: str) -> str | None: if push.returncode != 0: die(f"push failed: {push.stderr.strip()}") + label_args: list[str] = ["--label", LABEL, "--label", DO_NOT_MERGE_LABEL] + for extra in spec.get("extra_labels", []): + label_args += ["--label", extra] pr = gh( "pr", "create", "--repo", REPO, @@ -388,8 +443,7 @@ def create_pr(spec: dict, base: str) -> str | None: "--base", base, "--title", spec["title"], "--body", spec["body"], - "--label", LABEL, - "--label", DO_NOT_MERGE_LABEL, + *label_args, ) if pr.returncode != 0: die(f"gh pr create failed: {pr.stderr.strip()}") From ae93469034a7b699de6233b90286beacd32a2527 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 09:32:02 -0500 Subject: [PATCH 48/70] Reduce staleness to 1 month and close autofixed PRs --- astro_reorg/resolve_pr_conflicts.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index e7edf9e45dc..75bde7db8eb 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -70,8 +70,8 @@ `git am --3way`. --3way falls back to a per-patch 3-way merge when context lines have drifted because master made unrelated edits between the PR's base and today. - 5. Push as `reorg-fix/pr-`, open a new PR for it, comment on the - original PR pointing to the fix PR, and label the original + 5. Push as `reorg-fix/pr-`, open a new PR for it, close the original + PR with a comment pointing to the fix PR, and label the original has-astro-reorg-conflicts. A PR that already carries the has-astro-reorg-conflicts label (a fix PR was @@ -131,7 +131,7 @@ # PRs with no activity in this many days are treated as stale and receive a # comment + label instead of an auto-fix attempt. -STALE_DAYS = 62 # ~2 calendar months +STALE_DAYS = 31 # ~1 calendar month MANUAL_REVIEW_COMMENT = ( "This PR has merge conflicts from a recent repo reorg that could not be resolved automatically. " @@ -140,7 +140,7 @@ STALE_COMMENT = ( "This PR has conflicts created by the docs repo reorg project. " - "Because this PR is stale (more than two months old), no action was taken. " + "Because this PR is stale (more than one month old), no action was taken. " "If you still intend to use this PR and would like assistance resolving the conflicts, " "add the label `astro-reorg-help-requested` to your PR to add it to our help queue. " "We will reach out to you as soon as we can." @@ -519,6 +519,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: print(f" [dry-run] would label fix PR {LABEL_WIP!r}, {LABEL_AUTO_PR!r}") if IS_TEST_MODE: print(f" [dry-run] would label fix PR {DO_NOT_MERGE_LABEL!r} (test mode)") + print(f" [dry-run] would close PR #{pr_number} with comment pointing to fix PR") return True fix_branch = f"reorg-fix/pr-{pr_number}" @@ -594,16 +595,17 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: if IS_TEST_MODE: add_label(int(new_pr_number), DO_NOT_MERGE_LABEL, dry_run=False) - post_comment( - pr_number, + gh_run( + "pr", "close", str(pr_number), "--repo", REPO, + "--comment", f"🤖 **Reorg conflict auto-fix:**\n\n" f"This PR has merge conflicts caused by the recent docs repo reorg " f"(files moved from the repo root into `hugo/`). " f"A new PR with your commits translated to the correct paths " f"has been opened: #{new_pr_number}\n\n" - f"If the new PR looks correct, merge it and close this PR.", - dry_run=False, + f"If the new PR looks correct, merge it.", ) + print(f" Closed PR #{pr_number} with comment pointing to fix PR #{new_pr_number}") add_label(pr_number, LABEL_STALE, dry_run) add_label(pr_number, LABEL_AUTOFIXED, dry_run) return True From 889525d70e8539758a40eb44a1c12cc3ca476bd1 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 09:34:32 -0500 Subject: [PATCH 49/70] Tweak comment verbiage --- astro_reorg/resolve_pr_conflicts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 75bde7db8eb..fb174a9c05d 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -569,7 +569,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: f"This PR replays the commits from #{pr_number} with file paths " f"translated to the post-reorg `hugo/` layout. The original commits " f"are preserved — same messages and authorship.\n\n" - f"If this looks correct, merge this PR and close #{pr_number}.\n\n" + f"The original PR (#{pr_number}) has been closed.\n\n" f"---\n\n" f"**Original PR description:**\n\n{original_body}" ) From 0627c6b72ab9460a6d5f670df016094940831ba8 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 09:45:58 -0500 Subject: [PATCH 50/70] Add repo reorg readme --- REPO_REORG.md | 1 + astro_reorg/config.yaml | 1 + astro_reorg/resolve_pr_conflicts.py | 102 +++++++++++++++++----------- 3 files changed, 65 insertions(+), 39 deletions(-) create mode 100644 REPO_REORG.md diff --git a/REPO_REORG.md b/REPO_REORG.md new file mode 100644 index 00000000000..118e11743a1 --- /dev/null +++ b/REPO_REORG.md @@ -0,0 +1 @@ +The repo reorg info will go here. \ No newline at end of file diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index 99b47d8c78d..7be03e5a3dc 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -30,6 +30,7 @@ top_level: - LICENSE - CONTRIBUTING.md - CLAUDE.md + - REPO_REORG.md # Linting / formatting (shared) - .eslintrc diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index fb174a9c05d..29b7be57777 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -119,6 +119,7 @@ TEST_BASE_BRANCH: str | None = _TEST_CONFIG.get("mock_reorged_master_branch") REPO = "DataDog/documentation" +REPO_REORG_README_LINK = "https://github.com/DataDog/documentation/blob/jen.gilbert/astro-reorg-scripts/REPO_REORG.md" LABEL_MANUAL_REVIEW = "astro-reorg-manual-review" LABEL_STALE = "has-astro-reorg-conflicts" LABEL_AUTOFIXED = "astro-reorg-autofixed" @@ -133,26 +134,63 @@ # comment + label instead of an auto-fix attempt. STALE_DAYS = 31 # ~1 calendar month -MANUAL_REVIEW_COMMENT = ( - "This PR has merge conflicts from a recent repo reorg that could not be resolved automatically. " - "It has been queued for manual review by the Webops-Platform team." -) - -STALE_COMMENT = ( - "This PR has conflicts created by the docs repo reorg project. " - "Because this PR is stale (more than one month old), no action was taken. " - "If you still intend to use this PR and would like assistance resolving the conflicts, " - "add the label `astro-reorg-help-requested` to your PR to add it to our help queue. " - "We will reach out to you as soon as we can." -) - -WIP_COMMENT = ( - "This PR has merge conflicts created by the docs repo reorg project. " - "Because this PR is marked as a work in progress, no action was taken. " - "When you're ready and would like assistance resolving the conflicts, " - "add the label `astro-reorg-help-requested` to your PR to add it to our help queue. " - "We will reach out to you as soon as we can." -) +# --------------------------------------------------------------------------- +# Comment builders — one function per user-facing message +# --------------------------------------------------------------------------- + +def build_manual_review_comment() -> str: + """Posted on PRs with non-reorg conflicts, or when auto-fix fails.""" + return ( + "This PR has merge conflicts from a recent repo reorg that could not be resolved automatically. " + "It has been queued for manual review by the Webops-Platform team." + ) + + +def build_stale_comment() -> str: + """Posted on PRs with no activity in the last STALE_DAYS days.""" + return ( + "This PR has conflicts created by the docs repo reorg project. " + f"Because this PR is stale (more than {STALE_DAYS} days old), no action was taken. " + "If you still intend to use this PR and would like assistance resolving the conflicts, " + f"add the label `{LABEL_HELP_REQUESTED}` to your PR to add it to our help queue. " + "We will reach out to you as soon as we can." + ) + + +def build_wip_comment() -> str: + """Posted on PRs carrying the WORK IN PROGRESS label.""" + return ( + "This PR has merge conflicts created by the docs repo reorg project. " + "Because this PR is marked as a work in progress, no action was taken. " + "When you're ready and would like assistance resolving the conflicts, " + f"add the label `{LABEL_HELP_REQUESTED}` to your PR to add it to our help queue. " + "We will reach out to you as soon as we can." + ) + + +def build_autofix_close_comment(new_pr_number: int | str) -> str: + """Posted on the original PR when it is closed in favour of a fix PR.""" + return ( + f"🤖 **Reorg conflict auto-fix:**\n\n" + f"This PR has merge conflicts caused by the recent docs repo reorg " + f"(files moved from the repo root into `hugo/`). " + f"A new PR with your commits translated to the correct paths " + f"has been opened: #{new_pr_number}\n\n" + f"If the new PR looks correct, merge it." + ) + + +def build_autofix_pr_body(original_pr_number: int | str, original_body: str) -> str: + """Body of the auto-generated fix PR.""" + return ( + f"🤖 Auto-generated fix for #{original_pr_number}.\n\n" + f"This PR replays the commits from #{original_pr_number} with file paths " + f"translated to the post-reorg `hugo/` layout. The original commits " + f"are preserved — same messages and authorship.\n\n" + f"The original PR (#{original_pr_number}) has been closed.\n\n" + f"---\n\n" + f"**Original PR description:**\n\n{original_body}" + ) # Existing repo label applied to auto-created fix PRs in test mode so they stay # out of other teams' review queues. Assumed to already exist in the repo. @@ -564,15 +602,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: # Open a new PR for the fix branch so the author can preview, review, # and merge it directly — then close the original conflicting PR. original_body = pr.get("body") or "" - new_pr_body = ( - f"🤖 Auto-generated fix for #{pr_number}.\n\n" - f"This PR replays the commits from #{pr_number} with file paths " - f"translated to the post-reorg `hugo/` layout. The original commits " - f"are preserved — same messages and authorship.\n\n" - f"The original PR (#{pr_number}) has been closed.\n\n" - f"---\n\n" - f"**Original PR description:**\n\n{original_body}" - ) + new_pr_body = build_autofix_pr_body(pr_number, original_body) # We only reach this point on a real run; dry-run returned earlier. new_pr_title = f"[reorg fix] {pr['title']}" pr_create = gh_run( @@ -597,13 +627,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: gh_run( "pr", "close", str(pr_number), "--repo", REPO, - "--comment", - f"🤖 **Reorg conflict auto-fix:**\n\n" - f"This PR has merge conflicts caused by the recent docs repo reorg " - f"(files moved from the repo root into `hugo/`). " - f"A new PR with your commits translated to the correct paths " - f"has been opened: #{new_pr_number}\n\n" - f"If the new PR looks correct, merge it.", + "--comment", build_autofix_close_comment(new_pr_number), ) print(f" Closed PR #{pr_number} with comment pointing to fix PR #{new_pr_number}") add_label(pr_number, LABEL_STALE, dry_run) @@ -735,7 +759,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: if is_wip and not help_requested: print(" PR is marked WIP — skipping auto-fix, leaving comment.") add_label(pr_number, LABEL_PROCESSED, dry_run) - post_comment(pr_number, WIP_COMMENT, dry_run) + post_comment(pr_number, build_wip_comment(), dry_run) return True if is_pr_stale(pr) and not help_requested: @@ -743,7 +767,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: "labeling and commenting without auto-fix.") add_label(pr_number, LABEL_STALE, dry_run) add_label(pr_number, LABEL_PROCESSED, dry_run) - post_comment(pr_number, STALE_COMMENT, dry_run) + post_comment(pr_number, build_stale_comment(), dry_run) return True # All conflicts are reorg-caused. Attempt the auto-fix. @@ -753,7 +777,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: # Auto-fix failed (fork, apply error, etc.) — fall back to labeling. print(" Auto-fix failed or not applicable — labeling for manual review.") add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) - post_comment(pr_number, MANUAL_REVIEW_COMMENT, dry_run) + post_comment(pr_number, build_manual_review_comment(), dry_run) add_label(pr_number, LABEL_PROCESSED, dry_run) return True From c4fc90c942902e42ad456d889a465662498b50e4 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 11:56:54 -0500 Subject: [PATCH 51/70] Add generic stale PR handling scripts --- astro_reorg/close_stale_prs.py | 170 ++++++++++++++++++++++++++ astro_reorg/label_old_prs.py | 216 +++++++++++++++++++++++++++++++++ 2 files changed, 386 insertions(+) create mode 100644 astro_reorg/close_stale_prs.py create mode 100644 astro_reorg/label_old_prs.py diff --git a/astro_reorg/close_stale_prs.py b/astro_reorg/close_stale_prs.py new file mode 100644 index 00000000000..5c63de7c77d --- /dev/null +++ b/astro_reorg/close_stale_prs.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +Close open PRs that carry the `autolabeled-stale` label. +Posts a comment before closing. Defaults to a dry run where it just reports +what would be done. + +Use the GitHub label query to review which PRs are queued for closure: +https://github.com/DataDog/documentation/pulls?q=is%3Aopen+label%3Aautolabeled-stale + +Usage: + python3 astro_reorg/close_stale_prs.py [--no-dry-run] [--pr NUMBER ...] [--limit N] + +Flags: + --no-dry-run Actually close PRs instead of just reporting what would be done. + --pr NUMBER ... Only process the given PR number(s). + --limit N Stop after closing N PRs. PRs that are skipped don't count toward + the limit. Use it to roll out gradually: start with --limit 1, review + the results, then raise it as you gain confidence. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from datetime import datetime, timedelta, timezone + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +REPO = "DataDog/documentation" + +LABEL_STALE = "autolabeled-stale" + +CLOSE_COMMENT = ( + "Closing this PR because it has had no activity for six months. " + "If this was done in error, feel free to re-open the PR." +) + +# --------------------------------------------------------------------------- +# Shell helpers +# --------------------------------------------------------------------------- + +def run(cmd: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True) + + +def gh_json(*args: str) -> object: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return json.loads(result.stdout) if result.stdout.strip() else [] + + +def gh_run(*args: str) -> str: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return result.stdout + + +# --------------------------------------------------------------------------- +# GitHub helpers +# --------------------------------------------------------------------------- + +def post_comment(pr_number: int, body: str, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would comment on PR #{pr_number}:\n {body[:120]}") + return + gh_run("pr", "comment", str(pr_number), "--repo", REPO, "--body", body) + print(f" Posted comment on PR #{pr_number}") + + +def close_pr(pr_number: int, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would close PR #{pr_number}") + return + gh_run("pr", "close", str(pr_number), "--repo", REPO) + print(f" Closed PR #{pr_number}") + + +# --------------------------------------------------------------------------- +# Per-PR processing +# --------------------------------------------------------------------------- + +def process_pr(pr: dict, dry_run: bool) -> bool: + """Comment and close a labeled PR. Return True if we acted, False if we skipped.""" + pr_number = pr["number"] + title = pr["title"] + + print(f"\nPR #{pr_number}: {title}") + print(f" Last updated: {pr.get('updatedAt', 'unknown')}") + + post_comment(pr_number, CLOSE_COMMENT, dry_run) + close_pr(pr_number, dry_run) + return True + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def get_labeled_prs(only: list[int] | None = None, fetch_limit: int = 1000) -> list[dict]: + """Return open PRs carrying the stale label, optionally filtered to specific numbers.""" + fields = "number,title,updatedAt" + if only: + prs = [] + for n in only: + pr = gh_json("pr", "view", str(n), "--repo", REPO, "--json", fields) + prs.append(pr) + return prs # type: ignore[return-value] + return gh_json( # type: ignore[return-value] + "pr", "list", "--repo", REPO, "--state", "open", + "--label", LABEL_STALE, + "--json", fields, "--limit", str(fetch_limit), + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description=f"Close open PRs labeled {LABEL_STALE!r}.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "--dry-run", action=argparse.BooleanOptionalAction, default=True, + help="Report what would be done without making any changes (default: on). " + "Use --no-dry-run to apply changes.", + ) + parser.add_argument( + "--pr", type=int, action="append", dest="prs", metavar="NUMBER", + help="Only process this PR number (may be repeated).", + ) + parser.add_argument( + "--limit", type=int, default=None, metavar="N", + help="Stop after closing N PRs. Skipped PRs don't count toward the limit. " + "Use it to roll out gradually: start at 1, review, then raise it.", + ) + args = parser.parse_args() + + if args.limit is not None and args.limit < 1: + parser.error("--limit must be a positive integer.") + + if args.dry_run: + print("DRY-RUN mode — no PRs will be modified.\n") + + prs = get_labeled_prs(args.prs) + print(f"Found {len(prs)} labeled PR(s) to close.") + if args.limit is not None: + print(f"Limit: will stop after closing {args.limit} PR(s).") + + acted = 0 + for pr in prs: + if args.limit is not None and acted >= args.limit: + print(f"\nReached --limit of {args.limit} closed PR(s) — stopping.") + break + try: + if process_pr(pr, args.dry_run): + acted += 1 + except Exception as exc: + print(f"\nERROR processing PR #{pr.get('number', '?')}: {exc}", + file=sys.stderr) + print(" Skipping to the next PR.", file=sys.stderr) + + print(f"\nClosed {acted} PR(s). Done.") + + +if __name__ == "__main__": + main() diff --git a/astro_reorg/label_old_prs.py b/astro_reorg/label_old_prs.py new file mode 100644 index 00000000000..c3dac2c7210 --- /dev/null +++ b/astro_reorg/label_old_prs.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +""" +Label open PRs that have had no activity for six months or more with `autolabeled-stale`. +Already-labeled PRs are excluded from the query so they are never processed twice. +Defaults to a dry run where it just reports what would be done. + +Use the GitHub label query to review labeled PRs: +https://github.com/DataDog/documentation/pulls?q=is%3Aopen+label%3Aautolabeled-stale + +Usage: + python3 astro_reorg/label_old_prs.py [--no-dry-run] [--pr NUMBER ...] [--limit N] + +Flags: + --no-dry-run Actually add labels instead of just reporting what would be done. + --pr NUMBER ... Only process the given PR number(s). + --limit N Stop after labeling N PRs. PRs that are skipped don't count toward + the limit. Use it to roll out gradually: start with --limit 1, review + the results, then raise it as you gain confidence. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from datetime import datetime, timedelta, timezone + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +REPO = "DataDog/documentation" + +INACTIVITY_DAYS = 180 # ~6 calendar months + +LABEL_STALE = "autolabeled-stale" +LABEL_COLOR = "e4e669" + +# --------------------------------------------------------------------------- +# Shell helpers +# --------------------------------------------------------------------------- + +def run(cmd: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(cmd, capture_output=True, text=True) + + +def gh_json(*args: str) -> object: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return json.loads(result.stdout) if result.stdout.strip() else [] + + +def gh_run(*args: str) -> str: + result = run(["gh", *args]) + if result.returncode != 0: + raise RuntimeError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return result.stdout + + +# --------------------------------------------------------------------------- +# Staleness helpers +# --------------------------------------------------------------------------- + +def _parse_ts(ts: str) -> datetime: + """Parse a GitHub ISO-8601 timestamp (e.g. '2026-07-16T12:00:00Z').""" + return datetime.fromisoformat(ts.replace("Z", "+00:00")) + + +def is_inactive(pr: dict) -> bool: + """Return True if the PR has had no activity in the last INACTIVITY_DAYS days.""" + updated_at = pr.get("updatedAt") + if not updated_at: + return False + cutoff = datetime.now(timezone.utc) - timedelta(days=INACTIVITY_DAYS) + return _parse_ts(updated_at) < cutoff + + +def days_since_update(pr: dict) -> int | None: + """Return the number of days since the PR was last updated, or None.""" + updated_at = pr.get("updatedAt") + if not updated_at: + return None + delta = datetime.now(timezone.utc) - _parse_ts(updated_at) + return delta.days + + +# --------------------------------------------------------------------------- +# GitHub helpers +# --------------------------------------------------------------------------- + +def ensure_label_exists(label: str, dry_run: bool) -> None: + """Create the GitHub label if it doesn't already exist.""" + existing = gh_json("label", "list", "--repo", REPO, "--search", label, "--json", "name") + if any(l["name"] == label for l in existing): # type: ignore[index] + return + if dry_run: + print(f" [dry-run] would create label: {label!r}") + return + gh_run("label", "create", label, "--repo", REPO, + "--color", LABEL_COLOR, "--description", "PR labeled due to inactivity") + print(f" Created label: {label!r}") + + +def add_label(pr_number: int, label: str, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would add label {label!r} to PR #{pr_number}") + return + gh_run("api", f"repos/{REPO}/issues/{pr_number}/labels", + "-f", f"labels[]={label}") + print(f" Added label {label!r} to PR #{pr_number}") + + +# --------------------------------------------------------------------------- +# Per-PR processing +# --------------------------------------------------------------------------- + +def process_pr(pr: dict, dry_run: bool) -> bool: + """Label a stale PR. Return True if we acted, False if we skipped.""" + pr_number = pr["number"] + title = pr["title"] + days = days_since_update(pr) + + print(f"\nPR #{pr_number}: {title}") + print(f" Last updated: {pr.get('updatedAt', 'unknown')} ({days} days ago)") + + if not is_inactive(pr): + print(f" Active within the last {INACTIVITY_DAYS} days — skipping.") + return False + + print(f" Inactive for {days} days — labeling.") + add_label(pr_number, LABEL_STALE, dry_run) + return True + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def get_open_prs(only: list[int] | None = None, fetch_limit: int = 1000) -> list[dict]: + """Return open PRs without the stale label, optionally filtered to specific numbers.""" + fields = "number,title,updatedAt" + if only: + prs = [] + for n in only: + pr = gh_json("pr", "view", str(n), "--repo", REPO, "--json", fields) + prs.append(pr) + return prs # type: ignore[return-value] + return gh_json( # type: ignore[return-value] + "pr", "list", "--repo", REPO, "--state", "open", + "--search", f"-label:{LABEL_STALE} sort:updated-asc", + "--json", fields, "--limit", str(fetch_limit), + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Label open PRs with no activity for six months or more.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "--dry-run", action=argparse.BooleanOptionalAction, default=True, + help="Report what would be done without making any changes (default: on). " + "Use --no-dry-run to apply changes.", + ) + parser.add_argument( + "--pr", type=int, action="append", dest="prs", metavar="NUMBER", + help="Only check this PR number (may be repeated).", + ) + parser.add_argument( + "--limit", type=int, default=None, metavar="N", + help="Stop after labeling N PRs. Skipped PRs don't count toward the limit. " + "Use it to roll out gradually: start at 1, review, then raise it.", + ) + args = parser.parse_args() + + if args.limit is not None and args.limit < 1: + parser.error("--limit must be a positive integer.") + + if args.dry_run: + print("DRY-RUN mode — no PRs will be modified.\n") + + ensure_label_exists(LABEL_STALE, args.dry_run) + + cutoff_date = datetime.now(timezone.utc) - timedelta(days=INACTIVITY_DAYS) + print(f"Targeting PRs with no activity since: {cutoff_date.date()} ({INACTIVITY_DAYS} days ago)\n") + + prs = get_open_prs(args.prs) + print(f"Found {len(prs)} open PR(s) to check.") + if args.limit is not None: + print(f"Limit: will stop after labeling {args.limit} PR(s).") + + acted = 0 + for pr in prs: + if args.limit is not None and acted >= args.limit: + print(f"\nReached --limit of {args.limit} labeled PR(s) — stopping.") + break + try: + if process_pr(pr, args.dry_run): + acted += 1 + except Exception as exc: + print(f"\nERROR processing PR #{pr.get('number', '?')}: {exc}", + file=sys.stderr) + print(" Skipping to the next PR.", file=sys.stderr) + + print(f"\nLabeled {acted} PR(s). Done.") + print( + "\nView labeled PRs:\n" + " https://github.com/DataDog/documentation/pulls" + "?q=is%3Aopen+label%3Aautolabeled-stale" + ) + + +if __name__ == "__main__": + main() From f4b93f3dece2e6f777bb9024fec0a9272a226d7c Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 11:57:12 -0500 Subject: [PATCH 52/70] Tweak automatic conflict resolution flow --- REPO_REORG.md | 64 ++++++++++- astro_reorg/pr_handling_details.md | 90 ++++++++++++++++ astro_reorg/resolve_pr_conflicts.py | 160 +++++++++++++++++++--------- 3 files changed, 264 insertions(+), 50 deletions(-) create mode 100644 astro_reorg/pr_handling_details.md diff --git a/REPO_REORG.md b/REPO_REORG.md index 118e11743a1..0ae95c557cd 100644 --- a/REPO_REORG.md +++ b/REPO_REORG.md @@ -1 +1,63 @@ -The repo reorg info will go here. \ No newline at end of file +# `documentation` repository reorg + +# Project overview + +We're gradually migrating the Datadog docs site from Hugo to Astro. To support this, we must reorganize the `documentation` repo to support two sites (`hugo` and `astro`, with shared resources in a `shared` folder). + +**This reorg causes conflicts in almost every open PR in the `documentation` repo.** + +## What does the reorg change? + +For a full list of repo files and folders and their updated location, see [the configuration file for the reorg script](https://github.com/DataDog/documentation/blob/jen.gilbert/astro-reorg-scripts/astro_reorg/config.yaml). + +# Handling PR conflicts + +To support contributors, we provide an automatic fix where possible. When an automatic fix is not possible, we'll provide any support needed to contributors as they manually resolve the conflicts. + +## Resolution steps for contributors + +If your PR is open at the time of the reorg, and has conflicts related to the reorg, your PR will receive a comment describing one of four scenarios: + +* **Skipped as a WIP:**: We did not try to autofix your PR because it has the `WORK IN PROGRESS` label, so we assume you're still working on it. +* **Autofixed PR:** We fixed your conflicts automatically, closed your PR, and linked a new PR for your review. +* **Stale PR:** We did not act on your PR, because it appears to be stale. +* **Manual fix required:** We could not resolve your conflicts automatically. + +Based on the comment you received, follow the relevant instructions below. + +### If we skipped your PR as a WIP, but your PR is ready now + +1. Remove the `WORK IN PROGRESS` label. +2. Remove the `astro-reorg-skip` label. + +Your PR will be processed in the next batch. + +### If we autofixed your PR + +1. Verify that the new PR looks correct in the browser. +2. Remove the `WORK IN PROGRESS` label from the new PR. +3. Wait for the standard docs team approval before merging. (Optionally, you can check the provided "ready for merge" checkbox in the PR description if you would like the docs team to merge it for you.) + +### If your PR was skipped due to staleness, but you want it to be processed + +Remove the label `astro-reorg-stale`. Your PR will be processed the next time we run a batch. + +### If we could not resolve the conflicts automatically + +If you feel comfortable resolving the conflicts yourself manually: + +1. Resolve the conflicts. For a full list of repo files and folders and their updated location, see [the configuration file for the reorg script](https://github.com/DataDog/documentation/blob/jen.gilbert/astro-reorg-scripts/astro_reorg/config.yaml). +2. When your PR is ready for merge, remove the `WORK IN PROGRESS` label. +3. Wait for the standard docs team approval before merging. (Optionally, you can check the provided "ready for merge" checkbox in the PR description if you would like the docs team to merge it for you.) + +If you prefer that we resolve your conflicts, add the label `astro-reorg-help-requested` to your PR. This will add it to our support queue, and we will reach out to you as soon as possible. + +## Need help? {#need-help?} + +### Datadog employees + +Reach out in [\#docs-repo-reorg-support](https://dd.enterprise.slack.com/archives/C0BJ3MJDY5N) on Slack. + +### Other contributors + +Tag the `@datadog/webops-platform` team in a comment on your PR. \ No newline at end of file diff --git a/astro_reorg/pr_handling_details.md b/astro_reorg/pr_handling_details.md new file mode 100644 index 00000000000..0b0ceb996fe --- /dev/null +++ b/astro_reorg/pr_handling_details.md @@ -0,0 +1,90 @@ +# PR handling details + +How the conflict-resolution script decides what to do with each open PR after the repo reorg (which moved root files and folders into `hugo/`). This describes the behavior in plain terms; the script itself is the source of truth. + +## Labels used + +| Label | Applied to | Meaning | +|-------|-----------|---------| +| `LABEL_NO_CONFLICTS` | original PR | PR merges cleanly after the reorg; ignored from then on. | +| `LABEL_MANUAL_REVIEW` | original PR | Conflicts that can't be fixed automatically; queued for a person. Ignored from then on. | +| `LABEL_STALE` | original PR | PR had no activity for more than `STALE_DAYS`; skipped instead of auto-fixed. | +| `LABEL_AUTOFIXED` | original PR | PR was auto-fixed; the original is closed and points to a new fix PR. | +| `LABEL_AUTO_PR` | new fix PR | Marks the automatically created fix PR. | +| `LABEL_HELP_REQUESTED` | original PR (added by the author) | The author is asking a person to step in on an already-processed PR they're still stuck on. The script leaves these PRs alone. | +| `LABEL_WIP` | original PR (added by the author) | Author marked the PR as not ready; it gets a comment and is then skipped. | +| `LABEL_SKIP` | original PR | Applied by the script after commenting on a work-in-progress PR, so later runs don't pick it up again. | +| `DO_NOT_MERGE_LABEL` | new fix PR (test runs only) | Keeps auto-created fix PRs out of other teams' review queues. | + +Key thresholds: +- `STALE_DAYS` (31) — a PR with no activity for longer than this is treated as stale (roughly one calendar month). +- `STALE_REACTIVATE_GRACE` (60 seconds) — a small allowance so the script's own labeling of a PR is never mistaken for the author reactivating it. + +## Which PRs are considered + +By default the script looks at every **open** PR targeting the base branch (`master`, or a mock base branch during test runs), but **ignores** any PR that already carries `LABEL_NO_CONFLICTS`, `LABEL_MANUAL_REVIEW`, `LABEL_HELP_REQUESTED`, or `LABEL_SKIP`. `LABEL_NO_CONFLICTS` and `LABEL_MANUAL_REVIEW` are final: once a PR has one, it's never looked at again. `LABEL_HELP_REQUESTED` is added by an author who wants a person to take over, so the script stops touching the PR and leaves it for manual handling. `LABEL_SKIP` is applied by the script after it comments on a work-in-progress PR, so it isn't picked up and re-commented on later. + +You can also point the script at specific PR numbers instead of the whole open list. In that mode it will also consider closed PRs, but it still skips any PR that targets a different base branch. + +## Decision flow + +For each PR, in order. The first case that matches wins. + +### 1. The PR has no conflicts + +Label it `LABEL_NO_CONFLICTS` and move on. This is how PRs that are opened *after* the reorg reaches master get ignored going forward. Labeling a clean PR doesn't count against the run's processing limit. + +### 2. GitHub hasn't figured out yet whether the PR conflicts + +Skip it for now, with no label or comment. GitHub computes this in the background, so a later run will pick it up once the answer is available. + +### 3. The PR has conflicts + +The script does a trial merge against the post-reorg base and sorts each conflict into two buckets: conflicts **caused by the reorg** (a file the reorg moved) and conflicts **unrelated to the reorg**. It also catches files the PR adds at an old, pre-reorg location that should now live under `hugo/`, and counts those as reorg conflicts. + +What happens next depends on what it found: + +#### 3a. No conflicts could actually be pinned down +- If the trial merge failed but the script couldn't identify which files conflicted (an unusual case), it plays it safe and labels the PR `LABEL_MANUAL_REVIEW` rather than guessing. +- If the trial merge was clean after all, GitHub's earlier "conflicting" status was probably out of date; the PR is skipped so a later run can reassess. + +#### 3b. Some conflicts are unrelated to the reorg + +The script never touches conflicts it didn't cause. It labels the PR `LABEL_MANUAL_REVIEW` and leaves it for a person. + +#### 3c. All conflicts were caused by the reorg + +Before fixing anything, the PR has to be "ready." These checks run in order. + +**Work in progress** — If the PR is marked `LABEL_WIP`, the script posts a comment telling the author how to ask for help, adds `LABEL_SKIP`, and skips the fix. Because `LABEL_SKIP` is excluded from the query, later runs won't pick the PR up again, so the comment is posted only once. + +**Stale** — See [Stale handling](#stale-handling) below. + +**Auto-fix** — If the PR is ready and every conflict came from the reorg, the script fixes it: +- It replays the PR's commits onto a new fix branch with all file paths moved to the new `hugo/` layout, keeping the original authorship and commit messages. +- If that succeeds, it opens a new fix PR (labeled `LABEL_AUTO_PR` and `LABEL_WIP`, plus `DO_NOT_MERGE_LABEL` on test runs), then closes the original PR with a comment linking to the fix and labels the original `LABEL_AUTOFIXED`. An auto-fixed PR carries only that one label. +- If the fix can't be applied — for example the PR comes from a fork the script can't push to, or the replay fails — it labels the PR `LABEL_MANUAL_REVIEW` and posts a comment saying so. + +## Stale handling + +A PR is **stale** when its last activity was more than `STALE_DAYS` ago. Stale handling only applies to PRs whose conflicts all came from the reorg. + +**If a stale PR isn't labeled stale yet:** the script posts a comment explaining that no action was taken because the PR is stale (and how to ask for help), then adds the `LABEL_STALE` label. + +**If a PR already has the `LABEL_STALE` label:** the label serves two purposes — it marks the PR as stale, and it prevents the same comment from being posted again. Deciding whether the PR has "woken up" is based on whether there's been activity *since the label was applied* — not simply on how old the last activity is. This matters because the script's own comment and label count as activity; without this rule, a PR the script just labeled would look active again on the very next run and get fixed anyway. + +So for a PR that already has the stale label: +- The PR is considered **reactivated** if there was genuine activity (a new commit, a comment, or removing the stale label) after the label was applied — allowing for the `STALE_REACTIVATE_GRACE` window so the labeling itself doesn't count. +- **If it hasn't been reactivated:** skip it quietly, without commenting again. The skip lasts across runs until a person interacts with the PR. +- **If it has been reactivated:** remove the `LABEL_STALE` label and handle the PR normally (it goes on to the auto-fix step). +- **If the activity history can't be read** for some reason, the PR is treated as *not* reactivated and stays skipped, so a temporary glitch won't cause an unexpected auto-fix. + +### How an author reactivates a stale PR +- Push a new commit or leave a comment, **or** +- Remove the `LABEL_STALE` label (removing a label also counts as activity). + +To instead ask a person to step in rather than have the script retry, the author adds `LABEL_HELP_REQUESTED`; the script then leaves the PR alone (see [Labels used](#labels-used)). + +## Limiting a run and dry runs +- A run can be capped to act on only a set number of PRs. A PR counts against that cap when it's auto-fixed or gets a review/stale label and comment. PRs that need no action — clean PRs, ones GitHub hasn't evaluated, or ones that turn out clean on the trial merge — don't count. +- By default the script only reports what it *would* do, without changing anything, and runs against a mock base branch for safety. There are flags to apply changes for real and to run against the real `master`. diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 29b7be57777..37a8a40dd57 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -72,10 +72,7 @@ the PR's base and today. 5. Push as `reorg-fix/pr-`, open a new PR for it, close the original PR with a comment pointing to the fix PR, and label the original - has-astro-reorg-conflicts. - - A PR that already carries the has-astro-reorg-conflicts label (a fix PR was - opened on a prior run) is skipped, so re-running the script is safe. + astro-reorg-autofixed. PRs from forks cannot be auto-fixed (we don't have push access to the fork). They receive the astro-reorg-manual-review label. @@ -121,11 +118,17 @@ REPO = "DataDog/documentation" REPO_REORG_README_LINK = "https://github.com/DataDog/documentation/blob/jen.gilbert/astro-reorg-scripts/REPO_REORG.md" LABEL_MANUAL_REVIEW = "astro-reorg-manual-review" -LABEL_STALE = "has-astro-reorg-conflicts" +LABEL_STALE = "astro-reorg-stale" LABEL_AUTOFIXED = "astro-reorg-autofixed" LABEL_AUTO_PR = "astro-reorg-auto-pr" +LABEL_NO_CONFLICTS = "astro-reorg-no-conflicts" +# Author-applied on an already-processed PR to request manual intervention. The +# script does not act on these PRs — they're excluded from the query and left +# for a person to handle. LABEL_HELP_REQUESTED = "astro-reorg-help-requested" -LABEL_PROCESSED = "astro-reorg-processed" +# Applied by the script to a work-in-progress PR after it comments once. Excluded +# from the query so the PR isn't picked up (and re-commented on) on later runs. +LABEL_SKIP = "astro-reorg-skip" LABEL_WIP = "WORK IN PROGRESS" LABEL_COLOR = "e4e669" LABEL_DESCRIPTION = "Needs manual conflict resolution after replatforming reorg" @@ -134,6 +137,12 @@ # comment + label instead of an auto-fix attempt. STALE_DAYS = 31 # ~1 calendar month +# A stale-labeled PR is considered reactivated only if its updatedAt is later +# than when we applied the label by more than this grace window. The label +# event and the resulting updatedAt bump share a timestamp; the grace absorbs +# any skew so our own labeling never counts as fresh activity. +STALE_REACTIVATE_GRACE = timedelta(seconds=60) + # --------------------------------------------------------------------------- # Comment builders — one function per user-facing message # --------------------------------------------------------------------------- @@ -238,14 +247,37 @@ def git(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess: # Staleness helpers # --------------------------------------------------------------------------- +def _parse_ts(ts: str) -> datetime: + """Parse a GitHub ISO-8601 timestamp (e.g. '2026-07-16T12:00:00Z').""" + return datetime.fromisoformat(ts.replace("Z", "+00:00")) + + def is_pr_stale(pr: dict) -> bool: """Return True if the PR has had no activity in the last STALE_DAYS days.""" updated_at = pr.get("updatedAt") if not updated_at: return False - last_update = datetime.fromisoformat(updated_at.replace("Z", "+00:00")) cutoff = datetime.now(timezone.utc) - timedelta(days=STALE_DAYS) - return last_update < cutoff + return _parse_ts(updated_at) < cutoff + + +def stale_labeled_at(pr_number: int) -> datetime | None: + """Return when LABEL_STALE was most recently added to the PR, or None if it + was never labeled or the timeline can't be read. Reads the issue timeline, + which records a 'labeled' event (with the label name and a created_at) for + every label addition.""" + try: + events = gh_json("api", "--paginate", + f"repos/{REPO}/issues/{pr_number}/timeline") + except RuntimeError as exc: + print(f" Could not read timeline for PR #{pr_number}: {exc}", + file=sys.stderr) + return None + times = [e["created_at"] for e in events # type: ignore[union-attr] + if e.get("event") == "labeled" + and e.get("label", {}).get("name") == LABEL_STALE + and e.get("created_at")] + return max((_parse_ts(t) for t in times), default=None) # --------------------------------------------------------------------------- @@ -463,6 +495,18 @@ def add_label(pr_number: int, label: str, dry_run: bool) -> None: print(f" Added label {label!r} to PR #{pr_number}") +def remove_label(pr_number: int, label: str, dry_run: bool) -> None: + if dry_run: + print(f" [dry-run] would remove label {label!r} from PR #{pr_number}") + return + # Mirror add_label: hit the REST labels endpoint directly to avoid + # `gh pr edit`'s deprecated projectCards fetch. DELETE is a no-op (404) + # if the label isn't present, so callers can call it unconditionally. + gh_run("api", "--method", "DELETE", + f"repos/{REPO}/issues/{pr_number}/labels/{label}") + print(f" Removed label {label!r} from PR #{pr_number}") + + def post_comment(pr_number: int, body: str, dry_run: bool) -> None: if dry_run: print(f" [dry-run] would comment on PR #{pr_number}:\n {body[:120]}...") @@ -557,7 +601,8 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: print(f" [dry-run] would label fix PR {LABEL_WIP!r}, {LABEL_AUTO_PR!r}") if IS_TEST_MODE: print(f" [dry-run] would label fix PR {DO_NOT_MERGE_LABEL!r} (test mode)") - print(f" [dry-run] would close PR #{pr_number} with comment pointing to fix PR") + print(f" [dry-run] would close PR #{pr_number} with comment pointing to fix PR, " + f"and label it {LABEL_AUTOFIXED!r}") return True fix_branch = f"reorg-fix/pr-{pr_number}" @@ -568,8 +613,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: # Creating the branch failed — most likely a reorg-fix/pr- branch # from a prior run already exists. We do NOT reset and reuse it: # that could clobber a fix that's already in review. Bail and let - # the PR fall back to manual review. (A fully-applied fix carries - # astro-reorg-stale, so that PR would have been skipped earlier.) + # the PR fall back to manual review. print(f" could not create fix branch {fix_branch!r} " f"(may already exist) — leaving for manual review: " f"{add_wt.stderr.strip()[:120]}", file=sys.stderr) @@ -591,8 +635,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: # A reorg-fix/pr- branch from a prior run already exists on # origin. We don't force-push (repo policy), and a fix PR for it # most likely already exists, so bail to manual review rather than - # clobber it. (PRs that were fully fixed carry astro-reorg-stale - # and are skipped before we ever get here.) + # clobber it. print(f" push failed (fix branch may already exist): " f"{push.stderr.strip()[:120]}", file=sys.stderr) return False @@ -630,7 +673,6 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: "--comment", build_autofix_close_comment(new_pr_number), ) print(f" Closed PR #{pr_number} with comment pointing to fix PR #{new_pr_number}") - add_label(pr_number, LABEL_STALE, dry_run) add_label(pr_number, LABEL_AUTOFIXED, dry_run) return True @@ -645,8 +687,8 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: def analyze_pr(pr: dict, dry_run: bool) -> bool: """Process one PR. Return True if we acted on it (auto-fixed or labeled), - False if it needed no action (mergeable, already fixed, mergeability not yet - computed, or no conflicts found locally). The return value drives --limit: + False if it needed no action (mergeable, mergeability not yet computed, or + no conflicts found locally). The return value drives --limit: only acted-on PRs count toward it. """ pr_number = pr["number"] @@ -656,15 +698,12 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: print(f"\nPR #{pr_number}: {title}") print(f" mergeable: {mergeable}") - # Already processed on a prior run — skip regardless of other labels - # (including astro-reorg-help-requested). This makes the script safe to - # re-run without re-processing or double-commenting. - if any(l["name"] == LABEL_PROCESSED for l in pr.get("labels", [])): - print(f" Already processed ({LABEL_PROCESSED}) — skipping.") - return False - if mergeable == "MERGEABLE": - print(" No conflicts — skipping.") + # No conflicts — label so future runs exclude it from the query. This + # lets us ignore PRs that land after the reorg reaches master. Labeling + # doesn't count as "acting" on the PR, so it never consumes --limit. + print(f" No conflicts — labeling {LABEL_NO_CONFLICTS!r} and skipping.") + add_label(pr_number, LABEL_NO_CONFLICTS, dry_run) return False if mergeable == "UNKNOWN": @@ -734,7 +773,6 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: print(" Merge failed but no conflicts could be classified " "— labeling for manual review.") add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) - add_label(pr_number, LABEL_PROCESSED, dry_run) return True print(" No conflicts found locally (GitHub mergeability may be stale).") return False @@ -744,41 +782,63 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: # touch it — just label it so a human can resolve it manually. print(" Non-reorg conflicts present — labeling for manual review.") add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) - add_label(pr_number, LABEL_PROCESSED, dry_run) return True # All conflicts are reorg-caused. Before attempting an auto-fix, skip # PRs that aren't ready: WIP and stale PRs both get a comment pointing - # the author to astro-reorg-help-requested when they want help. - # Exception: if the author already added `astro-reorg-help-requested`, - # they have explicitly opted back in — treat it like a fresh PR. - help_requested = any( - l["name"] == LABEL_HELP_REQUESTED for l in pr.get("labels", []) - ) + # the author to astro-reorg-help-requested when they want manual help. is_wip = any(l["name"] == LABEL_WIP for l in pr.get("labels", [])) - if is_wip and not help_requested: - print(" PR is marked WIP — skipping auto-fix, leaving comment.") - add_label(pr_number, LABEL_PROCESSED, dry_run) + if is_wip: + print(" PR is marked WIP — skipping auto-fix, commenting and labeling.") + # Comment once, then apply the skip label. The label is excluded from + # the query, so later runs won't pick this PR up and re-comment. post_comment(pr_number, build_wip_comment(), dry_run) + add_label(pr_number, LABEL_SKIP, dry_run) return True - if is_pr_stale(pr) and not help_requested: + # Stale handling. The stale label is both the durable "we deemed this + # stale" marker and the guard against re-commenting. We can't use + # is_pr_stale() to decide whether a *labeled* PR has been reactivated: + # our own label + comment bump updatedAt, so the PR would read as active + # on the very next run. Instead we compare updatedAt against when we + # applied the label — only activity after that (author pushes, comments, + # or removing the label) counts as genuine reactivation. + already_stale = any(l["name"] == LABEL_STALE for l in pr.get("labels", [])) + if already_stale: + labeled_at = stale_labeled_at(pr_number) + updated_at = _parse_ts(pr["updatedAt"]) if pr.get("updatedAt") else None + reactivated = ( + labeled_at is not None and updated_at is not None + and updated_at > labeled_at + STALE_REACTIVATE_GRACE + ) + if not reactivated: + print(" Already labeled stale, no activity since — " + "skipping without re-commenting.") + return True + print(" Reactivated since we labeled it stale — removing stale label.") + remove_label(pr_number, LABEL_STALE, dry_run) + # Fall through and process the PR normally. + elif is_pr_stale(pr): print(f" PR is stale (no activity in >{STALE_DAYS} days) — " - "labeling and commenting without auto-fix.") - add_label(pr_number, LABEL_STALE, dry_run) - add_label(pr_number, LABEL_PROCESSED, dry_run) + "commenting and labeling without auto-fix.") + # Comment first, then label so the label is our last action: the + # labeled-event timestamp then matches the PR's updatedAt, giving + # the reactivation check on later runs a clean baseline. post_comment(pr_number, build_stale_comment(), dry_run) + add_label(pr_number, LABEL_STALE, dry_run) return True # All conflicts are reorg-caused. Attempt the auto-fix. print(" All conflicts are reorg-caused — attempting auto-fix.") success = attempt_fix(pr, dry_run) - if not success: - # Auto-fix failed (fork, apply error, etc.) — fall back to labeling. - print(" Auto-fix failed or not applicable — labeling for manual review.") - add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) - post_comment(pr_number, build_manual_review_comment(), dry_run) - add_label(pr_number, LABEL_PROCESSED, dry_run) + if success: + # attempt_fix closed the PR and labeled it LABEL_AUTOFIXED — that's + # the only label an autofixed PR carries. + return True + # Auto-fix failed (fork, apply error, etc.) — fall back to labeling. + print(" Auto-fix failed or not applicable — labeling for manual review.") + add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + post_comment(pr_number, build_manual_review_comment(), dry_run) return True finally: @@ -820,7 +880,8 @@ def get_open_prs(only: list[int] | None = None, limit: int = 50) -> list[dict]: return gh_json( # type: ignore[return-value] "pr", "list", "--repo", REPO, "--state", "open", "--base", BASE_BRANCH, - "--search", f"-label:{LABEL_PROCESSED}", + "--search", f"-label:{LABEL_NO_CONFLICTS} -label:{LABEL_MANUAL_REVIEW} " + f"-label:{LABEL_HELP_REQUESTED} -label:{LABEL_SKIP}", "--json", fields, "--limit", str(limit), ) @@ -897,7 +958,8 @@ def main() -> None: ensure_label_exists(LABEL_STALE, args.dry_run) ensure_label_exists(LABEL_AUTOFIXED, args.dry_run) ensure_label_exists(LABEL_AUTO_PR, args.dry_run) - ensure_label_exists(LABEL_PROCESSED, args.dry_run) + ensure_label_exists(LABEL_NO_CONFLICTS, args.dry_run) + ensure_label_exists(LABEL_SKIP, args.dry_run) existing_labels = gh_json("label", "list", "--repo", REPO, "--search", LABEL_WIP, "--json", "name") if not any(l["name"] == LABEL_WIP for l in existing_labels): # type: ignore[index] @@ -916,8 +978,8 @@ def main() -> None: break # Isolate failures: one PR raising (e.g. a transient gh/network error) # shouldn't abort the whole batch. A half-finished fix is safe to - # retry — a re-run either skips it (astro-reorg-stale) or, if the fix - # branch already exists, falls back to manual review. + # retry — if the fix branch already exists, the re-run falls back to + # manual review rather than clobbering it. try: if analyze_pr(pr, args.dry_run): acted += 1 From d1f7da70d7e0d5894c4b11d17cb086746400e9fb Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 12:27:56 -0500 Subject: [PATCH 53/70] Tweak reorg UI --- REPO_REORG.md | 62 +++++++---------------------- astro_reorg/resolve_pr_conflicts.py | 43 +++++++++++++------- 2 files changed, 43 insertions(+), 62 deletions(-) diff --git a/REPO_REORG.md b/REPO_REORG.md index 0ae95c557cd..a9b208c3817 100644 --- a/REPO_REORG.md +++ b/REPO_REORG.md @@ -1,63 +1,29 @@ # `documentation` repository reorg +# Reorg status + +Not yet merged. + # Project overview -We're gradually migrating the Datadog docs site from Hugo to Astro. To support this, we must reorganize the `documentation` repo to support two sites (`hugo` and `astro`, with shared resources in a `shared` folder). +We're gradually migrating the Datadog docs site from Hugo to Astro. To support this, we must reorganize the `documentation` repo into a monorepo that supports two sites (`hugo` and `astro`), instead of having the entire repo serve as our Hugo site. **This reorg causes conflicts in almost every open PR in the `documentation` repo.** ## What does the reorg change? +The most impactful change is that any Hugo-specific site files move to a `hugo` folder instead of being located at the top level of the repo. This change impacts thousands of files, and causes the majority of conflicts. + For a full list of repo files and folders and their updated location, see [the configuration file for the reorg script](https://github.com/DataDog/documentation/blob/jen.gilbert/astro-reorg-scripts/astro_reorg/config.yaml). # Handling PR conflicts -To support contributors, we provide an automatic fix where possible. When an automatic fix is not possible, we'll provide any support needed to contributors as they manually resolve the conflicts. - -## Resolution steps for contributors - -If your PR is open at the time of the reorg, and has conflicts related to the reorg, your PR will receive a comment describing one of four scenarios: - -* **Skipped as a WIP:**: We did not try to autofix your PR because it has the `WORK IN PROGRESS` label, so we assume you're still working on it. -* **Autofixed PR:** We fixed your conflicts automatically, closed your PR, and linked a new PR for your review. -* **Stale PR:** We did not act on your PR, because it appears to be stale. -* **Manual fix required:** We could not resolve your conflicts automatically. - -Based on the comment you received, follow the relevant instructions below. - -### If we skipped your PR as a WIP, but your PR is ready now - -1. Remove the `WORK IN PROGRESS` label. -2. Remove the `astro-reorg-skip` label. - -Your PR will be processed in the next batch. - -### If we autofixed your PR - -1. Verify that the new PR looks correct in the browser. -2. Remove the `WORK IN PROGRESS` label from the new PR. -3. Wait for the standard docs team approval before merging. (Optionally, you can check the provided "ready for merge" checkbox in the PR description if you would like the docs team to merge it for you.) - -### If your PR was skipped due to staleness, but you want it to be processed - -Remove the label `astro-reorg-stale`. Your PR will be processed the next time we run a batch. - -### If we could not resolve the conflicts automatically - -If you feel comfortable resolving the conflicts yourself manually: - -1. Resolve the conflicts. For a full list of repo files and folders and their updated location, see [the configuration file for the reorg script](https://github.com/DataDog/documentation/blob/jen.gilbert/astro-reorg-scripts/astro_reorg/config.yaml). -2. When your PR is ready for merge, remove the `WORK IN PROGRESS` label. -3. Wait for the standard docs team approval before merging. (Optionally, you can check the provided "ready for merge" checkbox in the PR description if you would like the docs team to merge it for you.) - -If you prefer that we resolve your conflicts, add the label `astro-reorg-help-requested` to your PR. This will add it to our support queue, and we will reach out to you as soon as possible. - -## Need help? {#need-help?} - -### Datadog employees - -Reach out in [\#docs-repo-reorg-support](https://dd.enterprise.slack.com/archives/C0BJ3MJDY5N) on Slack. +Once the reorg is merged to master, all open PRs are processed. Conflicts are fixed automatically where possible. When an automatic fix is not possible, we defer to the author's best judgment in resolving the conflicts, but offer escalation options in the PR comments in case they need help. -### Other contributors +## PR cases -Tag the `@datadog/webops-platform` team in a comment on your PR. \ No newline at end of file +* **Skipped because it has no conflicts:**: We did not try to autofix the PR because it doesn't have any conflicts. +* **Skipped as a WIP:**: We did not try to autofix the PR because it has the `WORK IN PROGRESS` label, but left a comment describing how to invoke the autofix. +* **Autofixed PR:** We fixed the conflicts automatically, closed the PR, and linked a new PR for the author's review. +* **Stale PR:** We did not act on the PR, because it appears to be stale. We left a comment describing how to invoke the autofix. +* **Manual fix required:** We could not resolve the conflicts automatically. We left a comment describing how to ask for help if needed. \ No newline at end of file diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 37a8a40dd57..955b4cdf705 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -116,7 +116,8 @@ TEST_BASE_BRANCH: str | None = _TEST_CONFIG.get("mock_reorged_master_branch") REPO = "DataDog/documentation" -REPO_REORG_README_LINK = "https://github.com/DataDog/documentation/blob/jen.gilbert/astro-reorg-scripts/REPO_REORG.md" +MASTER_BRANCH_NAME = "jen.gilbert/astro-reorg-scripts" +REPO_REORG_README_LINK = f"https://github.com/DataDog/documentation/blob/{MASTER_BRANCH_NAME}/REPO_REORG.md" LABEL_MANUAL_REVIEW = "astro-reorg-manual-review" LABEL_STALE = "astro-reorg-stale" LABEL_AUTOFIXED = "astro-reorg-autofixed" @@ -150,30 +151,37 @@ def build_manual_review_comment() -> str: """Posted on PRs with non-reorg conflicts, or when auto-fix fails.""" return ( - "This PR has merge conflicts from a recent repo reorg that could not be resolved automatically. " - "It has been queued for manual review by the Webops-Platform team." + f"This PR has merge conflicts from a [recent repo reorg]({REPO_REORG_README_LINK}) that could not be resolved automatically.\n\n" + "If you feel comfortable resolving the conflicts yourself:\n\n" + "1. Resolve the conflicts. For a full list of repo files and folders and their updated location, " + "see [the configuration file for the reorg script]" + f"(https://github.com/DataDog/documentation/blob/{MASTER_BRANCH_NAME}/astro_reorg/config.yaml).\n" + f"2. When your PR is ready for merge, remove the `{LABEL_WIP}` label.\n" + "3. Wait for the standard docs team approval before merging. " + "Optionally, you can check the 'ready for merge' checkbox in the PR description " + "if you would like the docs team to merge it for you.\n\n" + f"If you prefer that we resolve your conflicts, add the label `{LABEL_HELP_REQUESTED}` to your PR. " + "This will add it to our support queue, and we will reach out to you as soon as possible." ) def build_stale_comment() -> str: """Posted on PRs with no activity in the last STALE_DAYS days.""" return ( - "This PR has conflicts created by the docs repo reorg project. " - f"Because this PR is stale (more than {STALE_DAYS} days old), no action was taken. " - "If you still intend to use this PR and would like assistance resolving the conflicts, " - f"add the label `{LABEL_HELP_REQUESTED}` to your PR to add it to our help queue. " - "We will reach out to you as soon as we can." + f"This PR has conflicts created by the [docs repo reorg project]({REPO_REORG_README_LINK}). " + f"Because this PR is stale (more than {STALE_DAYS} days old), no attempt was made to auto-resolve the conflicts. " + "If you still intend to use this PR, remove the label " + f"`{LABEL_STALE}`. Your PR will be processed in the next batch of attempted auto-fixes. " ) def build_wip_comment() -> str: """Posted on PRs carrying the WORK IN PROGRESS label.""" return ( - "This PR has merge conflicts created by the docs repo reorg project. " - "Because this PR is marked as a work in progress, no action was taken. " - "When you're ready and would like assistance resolving the conflicts, " - f"add the label `{LABEL_HELP_REQUESTED}` to your PR to add it to our help queue. " - "We will reach out to you as soon as we can." + f"This PR has merge conflicts created by the [docs repo reorg project]({REPO_REORG_README_LINK}). " + "Because this PR is marked as a work in progress, no attempt was made to auto-resolve the conflicts. " + f"When your PR is ready, remove the `{LABEL_WIP}` label and the `{LABEL_SKIP}` label. " + "Your PR will be processed in the next batch of attempted auto-fixes." ) @@ -181,7 +189,7 @@ def build_autofix_close_comment(new_pr_number: int | str) -> str: """Posted on the original PR when it is closed in favour of a fix PR.""" return ( f"🤖 **Reorg conflict auto-fix:**\n\n" - f"This PR has merge conflicts caused by the recent docs repo reorg " + f"This PR has merge conflicts caused by the [recent docs repo reorg]({REPO_REORG_README_LINK}) " f"(files moved from the repo root into `hugo/`). " f"A new PR with your commits translated to the correct paths " f"has been opened: #{new_pr_number}\n\n" @@ -197,6 +205,13 @@ def build_autofix_pr_body(original_pr_number: int | str, original_body: str) -> f"translated to the post-reorg `hugo/` layout. The original commits " f"are preserved — same messages and authorship.\n\n" f"The original PR (#{original_pr_number}) has been closed.\n\n" + f"**Next steps:**\n\n" + f"1. Verify that this PR looks correct in the browser.\n" + f"2. Remove the `{LABEL_WIP}` label from this PR.\n" + f"3. Wait for the standard docs team approval before merging. " + f"Optionally, you can check the 'ready for merge' checkbox below " + f"if you would like the docs team to merge it for you.\n\n" + f"- [ ] Ready for merge\n\n" f"---\n\n" f"**Original PR description:**\n\n{original_body}" ) From 58e9f5d974f9fe3b0530baf999e1a6ca2cc35e2c Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 12:30:31 -0500 Subject: [PATCH 54/70] Update reorg docs --- REPO_REORG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/REPO_REORG.md b/REPO_REORG.md index a9b208c3817..4ba61c51a87 100644 --- a/REPO_REORG.md +++ b/REPO_REORG.md @@ -22,8 +22,8 @@ Once the reorg is merged to master, all open PRs are processed. Conflicts are fi ## PR cases -* **Skipped because it has no conflicts:**: We did not try to autofix the PR because it doesn't have any conflicts. -* **Skipped as a WIP:**: We did not try to autofix the PR because it has the `WORK IN PROGRESS` label, but left a comment describing how to invoke the autofix. -* **Autofixed PR:** We fixed the conflicts automatically, closed the PR, and linked a new PR for the author's review. -* **Stale PR:** We did not act on the PR, because it appears to be stale. We left a comment describing how to invoke the autofix. -* **Manual fix required:** We could not resolve the conflicts automatically. We left a comment describing how to ask for help if needed. \ No newline at end of file +* **Skipped because it has no conflicts:**: We do not try to autofix the PR because it doesn't have any conflicts. We add a label as a guard against further processing. +* **Skipped as a WIP:**: We do not try to autofix the PR because it has the `WORK IN PROGRESS` label. We leave a comment describing how to queue the PR for autofix processing if desired. +* **Autofixed PR:** We fix the conflicts automatically, close the PR, and link a new PR for the author's review. +* **Stale PR:** We do not act on the PR, because it appears to be stale. We leave a comment describing how to queue the PR for autofix processing if desired. +* **Manual fix required:** We cannot resolve the conflicts automatically. We leave a comment describing why the conflicts are occurring, and how to ask for help if needed. \ No newline at end of file From 05a59d4c4c871dc406eef0a118be07dbb3cc49c7 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 13:29:22 -0500 Subject: [PATCH 55/70] Update name of mock reorged master branch --- astro_reorg/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index 7be03e5a3dc..c389c32d267 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -133,7 +133,7 @@ ignore: test: # Stand-in for post-reorg master: the branch that has the reorg applied. Test # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. - mock_reorged_master_branch: mock-reorged-master-7-14-235 + mock_reorged_master_branch: mock-reorged-master-7-16-128 # Frozen snapshot of master that test PRs branch off, so each PR's diff stays # small (a PR shows every commit in its head that isn't in the base branch). branch_from: jen.gilbert/astro-reorg-scripts From 2ad9af96510b9b7cf3c38cd20476da1f535c7162 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 14:18:34 -0500 Subject: [PATCH 56/70] Tweak scripts --- .gitignore | 3 + astro_reorg/create_test_prs.py | 100 +++++++++++++++++++++++++++- astro_reorg/resolve_pr_conflicts.py | 43 ++++++++---- 3 files changed, 129 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index f9bc4010d69..500a34f95e9 100644 --- a/.gitignore +++ b/.gitignore @@ -338,3 +338,6 @@ playwright-report/ .claude/*local* .superpowers/ data/site_region_usage.json + +# Reorg tooling +test_pr_list.md diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index bd927ff8907..59208ffb190 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -40,6 +40,7 @@ import sys import uuid import webbrowser +from datetime import datetime, timedelta, timezone from pathlib import Path try: @@ -114,6 +115,12 @@ "non-material wording change in the getting started page description.\n\n" "Do not merge." ), + "expected_outcome": ( + "**Auto-fixed.** The conflict is purely reorg-caused (file moved from " + "`content/` to `hugo/content/`). The resolver opens a new PR with the " + "commit replayed at the post-reorg path, closes this PR, and labels it " + "`astro-reorg-autofixed`." + ), }, { # Auto-fixable, but marked WIP — resolve_pr_conflicts.py should skip it @@ -131,6 +138,12 @@ "Do not merge." ), "extra_labels": ["WORK IN PROGRESS"], + "expected_outcome": ( + "**Skipped (WIP).** The conflict is reorg-caused and would otherwise be " + "auto-fixable, but the PR carries the `WORK IN PROGRESS` label. The resolver " + "posts a comment explaining the situation, applies `astro-reorg-skip`, and " + "leaves the PR open for the author to act on." + ), }, { # Unresolvable (Case 2): reorg-classified, but the base changed the same @@ -153,6 +166,13 @@ "old": "combined into a customized solution", "new": "combined into a tailored solution", }, + "expected_outcome": ( + "**Manual review (unresolvable reorg conflict).** The conflict is classified " + "as reorg-caused, but the base branch also edited the same line at the " + "post-reorg path (`hugo/content/en/getting_started/_index.md`), so " + "`git am --3way` cannot replay the patch. The resolver labels this PR " + "`astro-reorg-manual-review` and posts a comment." + ), }, { # Unresolvable (Case 1): the conflict is on a top-level file the reorg @@ -175,6 +195,12 @@ "old": "how to write and edit content", "new": "how to author and edit content", }, + "expected_outcome": ( + "**Manual review (non-reorg conflict).** The conflict is in `CONTRIBUTING.md`, " + "a top-level file the reorg never moves. The resolver classifies it as an " + "unrelated conflict and immediately labels this PR `astro-reorg-manual-review` " + "without attempting an auto-fix." + ), }, { # Auto-fixable: a new page plus a nav-menu link to it, where nobody else @@ -210,6 +236,13 @@ "menu line, so the auto-fix should replay both changes cleanly.\n\n" "Do not merge." ), + "expected_outcome": ( + "**Auto-fixed.** The PR adds a new page at a pre-reorg path (a " + "\"wrong-path addition\") and links it in the nav menu. Both changes are " + "reorg-caused and the base branch did not touch that menu line, so the " + "resolver replays both commits cleanly at the post-reorg paths, opens a " + "new PR, closes this one, and labels it `astro-reorg-autofixed`." + ), }, { # Unresolvable (Case 3): a new page plus a nav-menu link to it, where @@ -252,6 +285,13 @@ "old": " - name: Essentials", "new": " - name: Essential Features", }, + "expected_outcome": ( + "**Manual review (unresolvable reorg conflict).** The PR adds a new page " + "at a pre-reorg path and inserts a nav menu entry. The conflict is classified " + "as reorg-caused, but the base branch also renamed the same menu heading, so " + "`git am --3way` cannot replay the nav edit. The resolver labels this PR " + "`astro-reorg-manual-review` and posts a comment." + ), }, ] @@ -453,13 +493,61 @@ def create_pr(spec: dict, base: str) -> str | None: return url +def write_pr_list( + pr_pairs: list[tuple[dict, str]], + base: str, + conflicting_base: str | None, + safe_after: datetime, +) -> None: + """Write test_pr_list.md at the repo root listing each PR and its expected outcome.""" + base_flag = f"--base-branch {conflicting_base}" if conflicting_base else "--live" + n = len(pr_pairs) + safe_after_str = safe_after.strftime("%Y-%m-%d %H:%M %Z") + lines = [ + "# Test PR list", + "", + "Generated by `astro_reorg/create_test_prs.py`. Do not edit by hand.", + "", + f"**Base branch:** `{base}` ", + f"**Branch PRs are cut from:** `{BRANCH_FROM}` ", + f"**Safe to resolve after:** {safe_after_str} (GitHub needs ~5 min to compute mergeability)", + "", + "## Expected outcomes", + "", + "Run the resolver against these PRs to verify each outcome:", + "", + f"```", + f"# Dry run (no changes):", + f"python3 astro_reorg/resolve_pr_conflicts.py {base_flag} --limit {n}", + f"", + f"# Live run (applies fixes and labels):", + f"python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run {base_flag} --limit {n}", + f"```", + "", + "## PRs", + "", + ] + for spec, url in pr_pairs: + lines += [ + f"### [{spec['title']}]({url})", + "", + spec["expected_outcome"], + "", + ] + + output_path = REPO_ROOT / "test_pr_list.md" + output_path.write_text("\n".join(lines)) + print(f"\nWrote {output_path}") + + def main() -> None: preflight() ensure_label() original_branch = current_branch() - urls: list[str] = [] + pr_pairs: list[tuple[dict, str]] = [] conflicting_base: str | None = None + base: str = MOCK_BASE_BRANCH try: # If any spec needs the base to diverge, build one throwaway conflicting # base and point every PR at it, so a single resolve_pr_conflicts.py run @@ -473,19 +561,25 @@ def main() -> None: for spec in TEST_PRS: url = create_pr(spec, base) if url: - urls.append(url) + pr_pairs.append((spec, url)) finally: # Return to wherever we started, whatever happened. git("checkout", original_branch) - if not urls: + if not pr_pairs: print("\nNo PRs created.") return + urls = [url for _, url in pr_pairs] print(f"\nCreated {len(urls)} PR(s):") for url in urls: print(f" {url}") + from zoneinfo import ZoneInfo + central = ZoneInfo("America/Chicago") + safe_after = datetime.now(central) + timedelta(minutes=5) + write_pr_list(pr_pairs, base, conflicting_base, safe_after) + base_flag = f"--base-branch {conflicting_base}" if conflicting_base else "--live" n = len(urls) print( diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 955b4cdf705..383416c8df7 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -5,7 +5,7 @@ that would be made. Usage: - python3 astro_reorg/resolve_pr_conflicts.py [--no-dry-run] [--live | --base-branch BRANCH] [--pr NUMBER ...] [--limit N] + python3 astro_reorg/resolve_pr_conflicts.py --limit N [--no-dry-run] [--live | --base-branch BRANCH] [--pr NUMBER ...] Flags: --no-dry-run Actually apply fixes and labels instead of just reporting what would be done. @@ -110,8 +110,9 @@ TOP_LEVEL: set[str] = set(_config.get("top_level", [])) # Shared with create_test_prs.py: the mock base branch used for test runs. When -# invoked with --test, the script treats this branch as the post-reorg base so -# it operates on the same branch the test PRs were opened against. +# invoked without --live or --base-branch, the script defaults to this branch +# as the post-reorg base so it operates on the same branch the test PRs were +# opened against. _TEST_CONFIG = _config.get("test", {}) TEST_BASE_BRANCH: str | None = _TEST_CONFIG.get("mock_reorged_master_branch") @@ -566,6 +567,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: """ pr_number = pr["number"] head_ref = pr["headRefName"] + pr_base = pr["baseRefName"] is_fork = pr.get("isCrossRepository", False) if is_fork: @@ -579,10 +581,11 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: print(f" fetch failed: {fetch.stderr.strip()[:120]}", file=sys.stderr) return False - # The merge base is the last common ancestor of the PR branch and - # BASE_BRANCH — the point where the PR diverged from master before the - # reorg commit landed. - merge_base = git("merge-base", f"origin/{BASE_BRANCH}", pr_remote_ref) + # The merge base is the last common ancestor of the PR branch and the PR's + # actual base branch — the point where the PR diverged from master before + # the reorg commit landed. In production pr_base == BASE_BRANCH (master); + # in test mode it may be a per-run conflicting-base branch. + merge_base = git("merge-base", f"origin/{pr_base}", pr_remote_ref) if merge_base.returncode != 0: print(f" could not find merge base: {merge_base.stderr.strip()[:80]}", file=sys.stderr) return False @@ -623,7 +626,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: fix_branch = f"reorg-fix/pr-{pr_number}" tmpdir = tempfile.mkdtemp(prefix=f"reorg_fix_{pr_number}_") try: - add_wt = git("worktree", "add", "-b", fix_branch, tmpdir, f"origin/{BASE_BRANCH}") + add_wt = git("worktree", "add", "-b", fix_branch, tmpdir, f"origin/{pr_base}") if add_wt.returncode != 0: # Creating the branch failed — most likely a reorg-fix/pr- branch # from a prior run already exists. We do NOT reset and reuse it: @@ -667,7 +670,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: "pr", "create", "--repo", REPO, "--head", fix_branch, - "--base", BASE_BRANCH, + "--base", pr_base, "--title", new_pr_title, "--body", new_pr_body, ) @@ -701,10 +704,10 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: # --------------------------------------------------------------------------- def analyze_pr(pr: dict, dry_run: bool) -> bool: - """Process one PR. Return True if we acted on it (auto-fixed or labeled), - False if it needed no action (mergeable, mergeability not yet computed, or - no conflicts found locally). The return value drives --limit: - only acted-on PRs count toward it. + """Process one PR. Return True if we acted on it (auto-fixed or labeled in + a way that counts toward --limit), False otherwise (mergeability not yet + computed, no conflicts found locally, or MERGEABLE — which gets a label but + doesn't consume --limit). The return value drives --limit. """ pr_number = pr["number"] title = pr["title"] @@ -735,9 +738,21 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: print(f" fetch failed: {fetch.stderr.strip()[:120]}", file=sys.stderr) return False + # Use the PR's actual base branch for the merge test so the local result + # matches what GitHub computes. In production this is always BASE_BRANCH + # (master); in test mode the conflicting-base branch may differ because + # create_test_prs.py builds a per-run branch with extra base_edits baked in. + pr_base = pr["baseRefName"] + if pr_base != BASE_BRANCH: + fetch_base = git("fetch", "origin", pr_base) + if fetch_base.returncode != 0: + print(f" fetch of PR base {pr_base!r} failed: " + f"{fetch_base.stderr.strip()[:120]}", file=sys.stderr) + return False + tmpdir = tempfile.mkdtemp(prefix=f"reorg_check_{pr_number}_") try: - add_wt = git("worktree", "add", "--detach", tmpdir, f"origin/{BASE_BRANCH}") + add_wt = git("worktree", "add", "--detach", tmpdir, f"origin/{pr_base}") if add_wt.returncode != 0: print(f" worktree add failed: {add_wt.stderr.strip()[:120]}", file=sys.stderr) return False From 9551a9856d8a7eea918a04f30104b442cedfd574 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 15:26:17 -0500 Subject: [PATCH 57/70] Add reorg steps --- astro_reorg/{ => docs}/pr_handling_details.md | 0 astro_reorg/docs/reorg_execution_steps.md | 136 ++++++++++++++++++ astro_reorg/{ => docs}/test_setup.md | 0 astro_reorg/resolve_pr_conflicts.py | 13 +- 4 files changed, 147 insertions(+), 2 deletions(-) rename astro_reorg/{ => docs}/pr_handling_details.md (100%) create mode 100644 astro_reorg/docs/reorg_execution_steps.md rename astro_reorg/{ => docs}/test_setup.md (100%) diff --git a/astro_reorg/pr_handling_details.md b/astro_reorg/docs/pr_handling_details.md similarity index 100% rename from astro_reorg/pr_handling_details.md rename to astro_reorg/docs/pr_handling_details.md diff --git a/astro_reorg/docs/reorg_execution_steps.md b/astro_reorg/docs/reorg_execution_steps.md new file mode 100644 index 00000000000..c898b0a5ae1 --- /dev/null +++ b/astro_reorg/docs/reorg_execution_steps.md @@ -0,0 +1,136 @@ +# Reorg execution steps + +## When a date has been set + +### 1. Update the reorg README + +- [ ] Add the target date to the reorg README. + +### 2. Publish the README + +- [ ] Publish [REPO_REORG.md](../../REPO_REORG.md) on master, so you can link to it. +- [ ] Update the [`REPO_REORG_README_LINK` constant](../resolve_pr_conflicts.py#L121) in `resolve_pr_conflicts.py`, commit, and push. + +### 3. Announce the reorg + +Announce the reorg: + +- [ ] To the #documentation channel () +- [ ] To the #docs-backroom channel () +- [ ] In a banner in the `documentation` README + +## The day before + +### 1. Bump the announcement in the #documentation channel + + +### 2. Remind the docs on-call of the reorg + +- [ ] Post the message below in #docs-backroom, tagging the current on-call folks, and put the URL here: + +``` +Hi , + +Tomorrow at about , I'll declare a code freeze in order to reorganize the docs repo. These are the steps I'll take: + +1. Start the code freeze. +2. Reorg the repo and verify the build. +3. Make any necessary last-minute fixes to GitHub actions, etc. broken by the reorg. +4. Merge the reorg to master, and verify the build. +5. Run a script that processes every open PR in the docs repo: + - PRs with no conflicts: Label them to avoid reprocessing them. + - PRs marked `WORK IN PROGRESS`: Skip, but leave a comment on them describing how to opt in to the auto fix. + - PRs with no recent activity (1 month): Skip, but leave a comment on them describing how to opt in to the auto fix. + - PRs with conflicts that can be automatically fixed: Close the original PR and link a new autofix PR. + - PRs with conflicts that CANNOT be automatically fixed: Put a label on it to prevent repeat processing, and comment on the PR to delegate conflict resolution to the author (with an escalation path if they need it). +6. Lift the code freeze. + +Auto-fixable PRs are closed, manual-intervention PRs are automatically labeled `WORK IN PROGRESS`, and auto-created PRs are labeled `WORK IN PROGRESS` so their authors can review them. This means that your review queue will be strangely quiet at first, and should not ever contain weird noise from the reorg. + +Once an author has reviewed an auto-created PR and removed the `WORK IN PROGRESS` label, you can treat it as any other PR. + +If you have questions, please reach out in #docs-repo-reorg-support. +``` + +## The day of + +NOTE: This file will get deleted from your reorg branch, so don't edit it there. Edit it here instead. + +### 1. Bump the announcement in the #documentation channel + +### 2. Bump the detailed message in #docs-backroom + +### 3. Declare a code freeze + +See the [Confluence page](https://datadoghq.atlassian.net/wiki/spaces/WEB/pages/5286757291/Code+Freeze+Workflows). + +Announce the freeze in `#documentation` with a link to the reorg README. + +### 4. Create a reorged `master` branch + +- [ ] Merge master into the scripts branch: + ```bash + git checkout jen.gilbert/astro-reorg-scripts + git merge master + ``` +- [ ] Cut a reorg branch from the reorg scripts branch: + ```bash + git checkout -b jen.gilbert/hugo-site-folder + ``` +- [ ] Run the reorg script: + ```bash + python3 astro_reorg/execute_reorg.py + ``` +- [ ] Delete the `astro_reorg` folder: + ```bash + rm -rf astro_reorg/ + ``` +- [ ] Stage, commit, and push the result: + ```bash + git add -A -- ':!astro/' + git commit -m "Create hugo site folder" + ``` + +### 5. Push and open a PR against master + +```bash +git push -u origin jen.gilbert/hugo-site-folder && +gh pr create --base master --title "Create Hugo site folder" +``` + +### 5. Verify that the GitHub actions etc. are working, and make any necessary fixes + +### 6. Merge to master and verify the build + +### 7. Run the PR resolution script + +Run the script in batches, building confidence before scaling up. The script defaults to a dry run against the mock base branch — pass `--live` to target real master, and `--no-dry-run` to apply changes. + +- [ ] Start with a single PR as a dry run to confirm the output looks right: + ```bash + python3 astro_reorg/resolve_pr_conflicts.py --live --limit 1 + ``` +- [ ] If that looks good, run one PR for real: + ```bash + python3 astro_reorg/resolve_pr_conflicts.py --live --no-dry-run --limit 1 + ``` +- [ ] Review the result on GitHub. If it looks good, increase the limit and repeat until all PRs are processed: + ```bash + python3 astro_reorg/resolve_pr_conflicts.py --live --no-dry-run --limit 10 + python3 astro_reorg/resolve_pr_conflicts.py --live --no-dry-run --limit 50 + # ... and so on until all PRs are processed + ``` + +You can view [all processed PRs here](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-processed), regardless of their outcome. + +### 8. End the code freeze + +### 9. Post in #documentation and #docs-backroom + +### 10. Monitor the queues + +- [All processed PRs](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-processed) +- [Fixed-and-closed PRs](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-autofixed) +- [Auto-generated PRs](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-auto-pr) +- [Manual intervention PRs](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-manual-review) +- [PRs escalated with a help request](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-help-requested) diff --git a/astro_reorg/test_setup.md b/astro_reorg/docs/test_setup.md similarity index 100% rename from astro_reorg/test_setup.md rename to astro_reorg/docs/test_setup.md diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 383416c8df7..f2e0690fced 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -134,6 +134,11 @@ LABEL_WIP = "WORK IN PROGRESS" LABEL_COLOR = "e4e669" LABEL_DESCRIPTION = "Needs manual conflict resolution after replatforming reorg" +# Applied to every PR the script acts on, regardless of outcome (auto-fixed, +# manual review, WIP, stale, ...). Purely a visibility aid so all affected PRs +# can be found in GitHub with one label filter — it is NOT used for idempotency +# and is deliberately absent from the get_open_prs query. +LABEL_PROCESSED = "astro-reorg-processed" # PRs with no activity in this many days are treated as stale and receive a # comment + label instead of an auto-fix attempt. @@ -862,8 +867,8 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: print(" All conflicts are reorg-caused — attempting auto-fix.") success = attempt_fix(pr, dry_run) if success: - # attempt_fix closed the PR and labeled it LABEL_AUTOFIXED — that's - # the only label an autofixed PR carries. + # attempt_fix closed the PR and labeled it LABEL_AUTOFIXED. The + # caller adds LABEL_PROCESSED on top for the affected-PR filter. return True # Auto-fix failed (fork, apply error, etc.) — fall back to labeling. print(" Auto-fix failed or not applicable — labeling for manual review.") @@ -990,6 +995,7 @@ def main() -> None: ensure_label_exists(LABEL_AUTO_PR, args.dry_run) ensure_label_exists(LABEL_NO_CONFLICTS, args.dry_run) ensure_label_exists(LABEL_SKIP, args.dry_run) + ensure_label_exists(LABEL_PROCESSED, args.dry_run) existing_labels = gh_json("label", "list", "--repo", REPO, "--search", LABEL_WIP, "--json", "name") if not any(l["name"] == LABEL_WIP for l in existing_labels): # type: ignore[index] @@ -1012,6 +1018,9 @@ def main() -> None: # manual review rather than clobbering it. try: if analyze_pr(pr, args.dry_run): + # Mark every acted-on PR (however it was handled) so all + # reorg-affected PRs are findable with one label filter. + add_label(pr["number"], LABEL_PROCESSED, args.dry_run) acted += 1 except Exception as exc: print(f"\nERROR processing PR #{pr.get('number', '?')}: {exc}", From 4c55673b4c2b4815bff09c59abc82890b17b7b86 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 15:28:59 -0500 Subject: [PATCH 58/70] Finish initial execution steps --- astro_reorg/docs/reorg_execution_steps.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/astro_reorg/docs/reorg_execution_steps.md b/astro_reorg/docs/reorg_execution_steps.md index c898b0a5ae1..9acdabc0820 100644 --- a/astro_reorg/docs/reorg_execution_steps.md +++ b/astro_reorg/docs/reorg_execution_steps.md @@ -23,6 +23,7 @@ Announce the reorg: ### 1. Bump the announcement in the #documentation channel +See link you saved above. ### 2. Remind the docs on-call of the reorg @@ -54,12 +55,16 @@ If you have questions, please reach out in #docs-repo-reorg-support. ## The day of -NOTE: This file will get deleted from your reorg branch, so don't edit it there. Edit it here instead. +NOTE: This file will get deleted from your reorg branch, so don't edit it there. You can push what you have, then check it off [here](https://github.com/DataDog/documentation/blob/jen.gilbert/astro-reorg-scripts/astro_reorg/docs/reorg_execution_steps.md) instead. ### 1. Bump the announcement in the #documentation channel +See link you saved above. + ### 2. Bump the detailed message in #docs-backroom +See link you saved above. + ### 3. Declare a code freeze See the [Confluence page](https://datadoghq.atlassian.net/wiki/spaces/WEB/pages/5286757291/Code+Freeze+Workflows). From 99915de38d45d153f041b1d8e9a48767a15ad1da Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 15:50:47 -0500 Subject: [PATCH 59/70] Resolve inconsistencies --- astro_reorg/CLAUDE.md | 2 +- astro_reorg/docs/pr_handling_details.md | 24 ++- astro_reorg/docs/reorg_execution_steps.md | 16 +- astro_reorg/docs/test_setup.md | 35 ++-- astro_reorg/resolve_pr_conflicts.py | 242 ++++++++++++++++------ 5 files changed, 233 insertions(+), 86 deletions(-) diff --git a/astro_reorg/CLAUDE.md b/astro_reorg/CLAUDE.md index f824f735aca..fbb91b895a3 100644 --- a/astro_reorg/CLAUDE.md +++ b/astro_reorg/CLAUDE.md @@ -6,7 +6,7 @@ This repo is currently a Hugo site. We instead want it to contain a `hugo` and ` `astro_reorg/execute_reorg.py` implements the file and folder path changes, and updates any dependencies on those paths, such as GitHub actions, CODEOWNERS, and Husky workflows. -`astro_reorg/helpers.py` contains shared utilities used by the other scripts (path manipulation, git/shell helpers, YAML config loading). +`astro_reorg/helpers.py` contains utilities (path manipulation, git/shell helpers, YAML config loading) used by `validate_reorg.py`. The other scripts predate it and each define their own small helpers inline. `astro_reorg/local_rollback.py` functions as an "undo" action for the reorg: removes `hugo/` and restores `.gitignore`, `.github/`, and `.husky/` from git. diff --git a/astro_reorg/docs/pr_handling_details.md b/astro_reorg/docs/pr_handling_details.md index 0b0ceb996fe..0a10052066f 100644 --- a/astro_reorg/docs/pr_handling_details.md +++ b/astro_reorg/docs/pr_handling_details.md @@ -7,13 +7,14 @@ How the conflict-resolution script decides what to do with each open PR after th | Label | Applied to | Meaning | |-------|-----------|---------| | `LABEL_NO_CONFLICTS` | original PR | PR merges cleanly after the reorg; ignored from then on. | -| `LABEL_MANUAL_REVIEW` | original PR | Conflicts that can't be fixed automatically; queued for a person. Ignored from then on. | +| `LABEL_MANUAL_REVIEW` | original PR | Conflicts that can't be fixed automatically; queued for a person. Ignored from then on. Applied together with `LABEL_WIP` and a comment (see [Sending a PR to manual review](#sending-a-pr-to-manual-review)). | | `LABEL_STALE` | original PR | PR had no activity for more than `STALE_DAYS`; skipped instead of auto-fixed. | | `LABEL_AUTOFIXED` | original PR | PR was auto-fixed; the original is closed and points to a new fix PR. | | `LABEL_AUTO_PR` | new fix PR | Marks the automatically created fix PR. | | `LABEL_HELP_REQUESTED` | original PR (added by the author) | The author is asking a person to step in on an already-processed PR they're still stuck on. The script leaves these PRs alone. | | `LABEL_WIP` | original PR (added by the author) | Author marked the PR as not ready; it gets a comment and is then skipped. | | `LABEL_SKIP` | original PR | Applied by the script after commenting on a work-in-progress PR, so later runs don't pick it up again. | +| `LABEL_PROCESSED` | every PR the script acts on | Visibility aid so all reorg-affected PRs are findable with one filter, regardless of outcome. Not used for idempotency and deliberately absent from the query. | | `DO_NOT_MERGE_LABEL` | new fix PR (test runs only) | Keeps auto-created fix PRs out of other teams' review queues. | Key thresholds: @@ -45,12 +46,12 @@ The script does a trial merge against the post-reorg base and sorts each conflic What happens next depends on what it found: #### 3a. No conflicts could actually be pinned down -- If the trial merge failed but the script couldn't identify which files conflicted (an unusual case), it plays it safe and labels the PR `LABEL_MANUAL_REVIEW` rather than guessing. +- If the trial merge failed but the script couldn't identify which files conflicted (an unusual case), it plays it safe and [sends the PR to manual review](#sending-a-pr-to-manual-review) rather than guessing. - If the trial merge was clean after all, GitHub's earlier "conflicting" status was probably out of date; the PR is skipped so a later run can reassess. #### 3b. Some conflicts are unrelated to the reorg -The script never touches conflicts it didn't cause. It labels the PR `LABEL_MANUAL_REVIEW` and leaves it for a person. +The script never touches conflicts it didn't cause. It [sends the PR to manual review](#sending-a-pr-to-manual-review) and leaves it for a person. #### 3c. All conflicts were caused by the reorg @@ -62,8 +63,21 @@ Before fixing anything, the PR has to be "ready." These checks run in order. **Auto-fix** — If the PR is ready and every conflict came from the reorg, the script fixes it: - It replays the PR's commits onto a new fix branch with all file paths moved to the new `hugo/` layout, keeping the original authorship and commit messages. -- If that succeeds, it opens a new fix PR (labeled `LABEL_AUTO_PR` and `LABEL_WIP`, plus `DO_NOT_MERGE_LABEL` on test runs), then closes the original PR with a comment linking to the fix and labels the original `LABEL_AUTOFIXED`. An auto-fixed PR carries only that one label. -- If the fix can't be applied — for example the PR comes from a fork the script can't push to, or the replay fails — it labels the PR `LABEL_MANUAL_REVIEW` and posts a comment saying so. +- If that succeeds, it opens a new fix PR (labeled `LABEL_AUTO_PR` and `LABEL_WIP`, plus `DO_NOT_MERGE_LABEL` on test runs), then closes the original PR with a comment linking to the fix and labels the original `LABEL_AUTOFIXED`. Every PR the script acts on — including this one — also receives `LABEL_PROCESSED`. The fix PR's description **@mentions the original author**, so they're notified the moment it's opened (which is before the original is closed — see [Resuming an interrupted fix](#resuming-an-interrupted-fix)). +- If the fix can't be applied — for example the PR comes from a fork the script can't push to, or the replay fails — it [sends the PR to manual review](#sending-a-pr-to-manual-review). + +### Resuming an interrupted fix + +The auto-fix happens in stages: push the fix branch, open the fix PR, then close the original. If a run dies in between (a network blip, an interrupted process), a later run detects the partial state and **finishes the job** instead of bailing: + +- **Fix PR already open:** the script reuses it — re-applies its labels, then closes the original pointing at it and labels the original `LABEL_AUTOFIXED`. +- **Fix branch pushed but no PR:** the script opens the PR from the existing branch, then finishes as above. + +It never rebuilds or force-pushes an existing fix branch, so a fix already in review is never clobbered. If it truly can't build the branch (for instance it's checked out in a lingering worktree), it falls back to [manual review](#sending-a-pr-to-manual-review). Because the fix PR's description @mentions the author, they learn about the fix even if the original close step never ran. + +### Sending a PR to manual review + +Every non-auto-fixable outcome (a non-reorg conflict, an unclassifiable merge failure, or a failed replay) is handled the same way: the script applies `LABEL_MANUAL_REVIEW` (which excludes the PR from all future runs), applies `LABEL_WIP` (which keeps it out of the docs team's review queue until the author acts), and posts a single comment explaining the situation. The comment tells the author to resolve the conflicts and then remove the `WORK IN PROGRESS` label, or to add `LABEL_HELP_REQUESTED` if they'd rather a person take over. ## Stale handling diff --git a/astro_reorg/docs/reorg_execution_steps.md b/astro_reorg/docs/reorg_execution_steps.md index 9acdabc0820..ceb5b675df1 100644 --- a/astro_reorg/docs/reorg_execution_steps.md +++ b/astro_reorg/docs/reorg_execution_steps.md @@ -9,7 +9,7 @@ ### 2. Publish the README - [ ] Publish [REPO_REORG.md](../../REPO_REORG.md) on master, so you can link to it. -- [ ] Update the [`REPO_REORG_README_LINK` constant](../resolve_pr_conflicts.py#L121) in `resolve_pr_conflicts.py`, commit, and push. +- [ ] Set the `REORG_README_BRANCH` constant in `resolve_pr_conflicts.py` to `"master"` (it defaults to the scripts branch), commit, and push. Leave `CONFIG_LINK_BRANCH` pointing at the scripts branch — the reorg deletes `astro_reorg/` from master, so that link must stay on a branch that retains it. ### 3. Announce the reorg @@ -90,7 +90,7 @@ Announce the freeze in `#documentation` with a link to the reorg README. ```bash rm -rf astro_reorg/ ``` -- [ ] Stage, commit, and push the result: +- [ ] Stage and commit the result (the push happens in the next step): ```bash git add -A -- ':!astro/' git commit -m "Create hugo site folder" @@ -103,11 +103,11 @@ git push -u origin jen.gilbert/hugo-site-folder && gh pr create --base master --title "Create Hugo site folder" ``` -### 5. Verify that the GitHub actions etc. are working, and make any necessary fixes +### 6. Verify that the GitHub actions etc. are working, and make any necessary fixes -### 6. Merge to master and verify the build +### 7. Merge to master and verify the build -### 7. Run the PR resolution script +### 8. Run the PR resolution script Run the script in batches, building confidence before scaling up. The script defaults to a dry run against the mock base branch — pass `--live` to target real master, and `--no-dry-run` to apply changes. @@ -128,11 +128,11 @@ Run the script in batches, building confidence before scaling up. The script def You can view [all processed PRs here](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-processed), regardless of their outcome. -### 8. End the code freeze +### 9. End the code freeze -### 9. Post in #documentation and #docs-backroom +### 10. Post in #documentation and #docs-backroom -### 10. Monitor the queues +### 11. Monitor the queues - [All processed PRs](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-processed) - [Fixed-and-closed PRs](https://github.com/DataDog/documentation/pulls?q=is%3Apr+label%3Aastro-reorg-autofixed) diff --git a/astro_reorg/docs/test_setup.md b/astro_reorg/docs/test_setup.md index 9ddee8fc7b0..aba0fcf3b66 100644 --- a/astro_reorg/docs/test_setup.md +++ b/astro_reorg/docs/test_setup.md @@ -2,10 +2,10 @@ ## Prerequisites -Two branches must exist on the remote (configured in `config.yaml` under `test:`): +Two branches must exist on the remote, named in `config.yaml` under `test:`: -- A non-reorged mock master branch for the mock contributors to branch from. -- A reorged mock master branch that acts as a base for the PR. This usually doesn't need to be created, since we can just use `jen.gilbert/astro-reorg-scripts` as the base. +- `branch_from` — a non-reorged snapshot of master that the mock contributor PRs are cut from. The scripts branch (`jen.gilbert/astro-reorg-scripts`) works and already exists, so you usually don't need to create this one. +- `mock_reorged_master_branch` — a reorged stand-in for post-reorg master that the test PRs target and the resolver treats as the base. You create this below. It must have the reorg applied, so the scripts branch can't stand in for it. ## Setup steps @@ -50,24 +50,35 @@ python3 astro_reorg/local_rollback.py ### 1. Create the test PRs -Create the test PRs: - ```bash python3 astro_reorg/create_test_prs.py ``` -This script will output the commands you should use to execute a dry run and a real run of the automatic conflict resolution script. +This opens one PR per spec in `TEST_PRS`. Because some specs carry a `base_edit`, the script also builds a throwaway conflicting base branch (unique per run, so no force-push) and points every PR at it, so a single resolver run exercises both the clean auto-fixes and the manual-review fallbacks together. On completion it: + +- writes `test_pr_list.md` at the repo root, listing every PR and its expected outcome, and +- prints the exact dry-run and real-run commands to use next, already filled in with the right `--base-branch` and `--limit`. ### 2. Execute a dry run -python3 astro_reorg/resolve_pr_conflicts.py \ - --base-branch +Run the dry-run command printed by `create_test_prs.py`. It looks like: + +```bash +python3 astro_reorg/resolve_pr_conflicts.py --base-branch --limit +``` + +`--limit` is required, so don't drop it. ### 3. Execute a real run -python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run \ - --base-branch +Run the real-run command printed by `create_test_prs.py`. It looks like: + +```bash +python3 astro_reorg/resolve_pr_conflicts.py --no-dry-run --base-branch --limit ``` -Expected outcomes: the wording-tweak PR gets an auto-fix; the other three get -the `astro-reorg-manual-review` label. +Expected outcomes (`test_pr_list.md` is the authoritative list): + +- **Auto-fixed** — original closed and replaced with a `[reorg fix]` PR (`astro-reorg-autofixed`): the wording-tweak PR and the new-page-with-nav PR. +- **Skipped as WIP** — `astro-reorg-skip` label plus a comment: the `WORK IN PROGRESS` PR. +- **Manual review** — `astro-reorg-manual-review` and `WORK IN PROGRESS` labels plus a comment: the non-reorg-conflict PR and the two unresolvable-reorg-conflict PRs. diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index f2e0690fced..817793b88ad 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -70,10 +70,16 @@ `git am --3way`. --3way falls back to a per-patch 3-way merge when context lines have drifted because master made unrelated edits between the PR's base and today. - 5. Push as `reorg-fix/pr-`, open a new PR for it, close the original + 5. Push as `reorg-fix/pr-`, open a new PR for it (whose body @mentions + the original author so they're notified immediately), close the original PR with a comment pointing to the fix PR, and label the original astro-reorg-autofixed. + If a run dies partway through step 5 (branch pushed and/or fix PR opened, but + the original never closed), a later run detects the existing branch/PR and + finishes the job rather than bailing — it never rebuilds or force-pushes an + existing fix branch, so a fix already in review is safe. + PRs from forks cannot be auto-fixed (we don't have push access to the fork). They receive the astro-reorg-manual-review label. """ @@ -117,8 +123,19 @@ TEST_BASE_BRANCH: str | None = _TEST_CONFIG.get("mock_reorged_master_branch") REPO = "DataDog/documentation" -MASTER_BRANCH_NAME = "jen.gilbert/astro-reorg-scripts" -REPO_REORG_README_LINK = f"https://github.com/DataDog/documentation/blob/{MASTER_BRANCH_NAME}/REPO_REORG.md" + +# The author-facing comments link to two files that live on DIFFERENT branches +# after the reorg, so they need separate branch constants — do not merge them. +# +# REORG_README_BRANCH: branch hosting REPO_REORG.md. Before the reorg lands this +# is the scripts branch; once REPO_REORG.md is published on master (see +# docs/reorg_execution_steps.md), set this to "master". +# CONFIG_LINK_BRANCH: branch hosting astro_reorg/config.yaml. The reorg DELETES +# astro_reorg/ from master, so this link must point at a branch that RETAINS it +# (the scripts branch) — it can never be "master". +REORG_README_BRANCH = "jen.gilbert/astro-reorg-scripts" +CONFIG_LINK_BRANCH = "jen.gilbert/astro-reorg-scripts" +REPO_REORG_README_LINK = f"https://github.com/DataDog/documentation/blob/{REORG_README_BRANCH}/REPO_REORG.md" LABEL_MANUAL_REVIEW = "astro-reorg-manual-review" LABEL_STALE = "astro-reorg-stale" LABEL_AUTOFIXED = "astro-reorg-autofixed" @@ -161,7 +178,7 @@ def build_manual_review_comment() -> str: "If you feel comfortable resolving the conflicts yourself:\n\n" "1. Resolve the conflicts. For a full list of repo files and folders and their updated location, " "see [the configuration file for the reorg script]" - f"(https://github.com/DataDog/documentation/blob/{MASTER_BRANCH_NAME}/astro_reorg/config.yaml).\n" + f"(https://github.com/DataDog/documentation/blob/{CONFIG_LINK_BRANCH}/astro_reorg/config.yaml).\n" f"2. When your PR is ready for merge, remove the `{LABEL_WIP}` label.\n" "3. Wait for the standard docs team approval before merging. " "Optionally, you can check the 'ready for merge' checkbox in the PR description " @@ -203,14 +220,31 @@ def build_autofix_close_comment(new_pr_number: int | str) -> str: ) -def build_autofix_pr_body(original_pr_number: int | str, original_body: str) -> str: - """Body of the auto-generated fix PR.""" +def build_autofix_pr_body( + original_pr_number: int | str, + original_body: str, + author_login: str | None = None, +) -> str: + """Body of the auto-generated fix PR. + + When author_login is given, the body opens with an @mention so the original + author is notified the moment this PR is created. This matters because the + fix PR is opened before the original is closed: if a run is interrupted in + between, the @mention is the author's only signal that the fix exists. + """ + mention = ( + f"@{author_login} — the docs repo reorg created merge conflicts in your " + f"PR #{original_pr_number}. This is an auto-generated replacement with the " + f"file paths fixed; please use it instead of the original.\n\n" + if author_login else "" + ) return ( f"🤖 Auto-generated fix for #{original_pr_number}.\n\n" + f"{mention}" f"This PR replays the commits from #{original_pr_number} with file paths " f"translated to the post-reorg `hugo/` layout. The original commits " f"are preserved — same messages and authorship.\n\n" - f"The original PR (#{original_pr_number}) has been closed.\n\n" + f"The original PR (#{original_pr_number}) will be closed in favor of this one.\n\n" f"**Next steps:**\n\n" f"1. Verify that this PR looks correct in the browser.\n" f"2. Remove the `{LABEL_WIP}` label from this PR.\n" @@ -536,10 +570,96 @@ def post_comment(pr_number: int, body: str, dry_run: bool) -> None: print(f" Posted comment on PR #{pr_number}") +def send_to_manual_review(pr_number: int, dry_run: bool) -> None: + """Route a PR to manual review, consistently, wherever we can't auto-fix. + + Applies both the durable manual-review label (which excludes the PR from all + future runs) and WORK IN PROGRESS (which keeps it out of the docs team's + review queue until the author has resolved the conflicts), then comments once + explaining what to do. build_manual_review_comment() already tells the author + to remove the WORK IN PROGRESS label once the conflicts are resolved. + + Used for every non-auto-fixable outcome: non-reorg conflicts, unclassifiable + merge failures, and failed replays. + """ + add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + add_label(pr_number, LABEL_WIP, dry_run) + post_comment(pr_number, build_manual_review_comment(), dry_run) + + # --------------------------------------------------------------------------- # Auto-fix # --------------------------------------------------------------------------- +def find_open_fix_pr(fix_branch: str) -> dict | None: + """Return an open PR opened from fix_branch on a prior run, or None. + + A run that dies after opening the fix PR but before closing the original + leaves this PR behind. Finding it lets a later run finish the job instead of + bailing to manual review. + """ + prs = gh_json("pr", "list", "--repo", REPO, "--state", "open", + "--head", fix_branch, "--json", "number,url") + return prs[0] if prs else None # type: ignore[index] + + +def fix_branch_on_origin(fix_branch: str) -> bool: + """True if fix_branch already exists on origin (pushed by a prior run).""" + return bool(git("ls-remote", "--heads", "origin", fix_branch).stdout.strip()) + + +def finalize_autofix(pr: dict, fix_branch: str, existing_fix_pr: dict | None) -> bool: + """Finish an auto-fix once the fix branch exists on origin: ensure the fix PR + is open and labeled, then close the original PR pointing at it and label the + original autofixed. + + Kept separate from the branch-building steps so it can complete a run that + was interrupted partway through. It is idempotent: label adds are no-ops when + the label is already present, and the original PR is only ever closed once + (it is still open whenever we reach here, since a closed PR drops out of the + open-PR query). `existing_fix_pr` is a prior run's fix PR when one was already + opened; otherwise it is None and we open the PR now. + """ + pr_number = pr["number"] + + if existing_fix_pr: + new_pr_number = existing_fix_pr["number"] + print(f" Reusing existing fix PR #{new_pr_number}: {existing_fix_pr['url']}") + else: + original_body = pr.get("body") or "" + author_login = (pr.get("author") or {}).get("login") + new_pr_body = build_autofix_pr_body(pr_number, original_body, author_login) + new_pr_title = f"[reorg fix] {pr['title']}" + pr_create = gh_run( + "pr", "create", + "--repo", REPO, + "--head", fix_branch, + "--base", pr["baseRefName"], + "--title", new_pr_title, + "--body", new_pr_body, + ) + new_pr_url = pr_create.strip() + new_pr_number = int(new_pr_url.rstrip("/").split("/")[-1]) + print(f" Opened fix PR: {new_pr_url}") + + # Labels are a set on GitHub's side, so re-adding an existing one is a + # harmless no-op — safe when finishing a run that applied some already. + add_label(int(new_pr_number), LABEL_WIP, dry_run=False) + add_label(int(new_pr_number), LABEL_AUTO_PR, dry_run=False) + # In test mode, keep the auto-created PR out of other teams' review queues by + # marking it "Do Not Merge". Never do this on real master. + if IS_TEST_MODE: + add_label(int(new_pr_number), DO_NOT_MERGE_LABEL, dry_run=False) + + gh_run( + "pr", "close", str(pr_number), "--repo", REPO, + "--comment", build_autofix_close_comment(new_pr_number), + ) + print(f" Closed PR #{pr_number} with comment pointing to fix PR #{new_pr_number}") + add_label(pr_number, LABEL_AUTOFIXED, dry_run=False) + return True + + def attempt_fix(pr: dict, dry_run: bool) -> bool: """ Attempt to auto-fix a reorg-conflict PR by re-applying its commits at the @@ -610,11 +730,30 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: transformed = transform_diff_paths(patches) + fix_branch = f"reorg-fix/pr-{pr_number}" + + # A prior run may have died partway through: the fix branch was pushed and/or + # the fix PR was opened, but the original PR was never closed. Detect that so + # we FINISH the interrupted job rather than bail to manual review — which + # would post a misleading "resolve it yourself" comment on a PR that already + # has a working fix. Both checks are read-only, so they're safe in dry-run. + existing_fix_pr = find_open_fix_pr(fix_branch) + branch_pushed = existing_fix_pr is not None or fix_branch_on_origin(fix_branch) + if dry_run: + if existing_fix_pr: + print(f" [dry-run] fix PR #{existing_fix_pr['number']} already exists — " + f"would finish the interrupted auto-fix: label it, close " + f"#{pr_number} pointing to it, and label it {LABEL_AUTOFIXED!r}.") + return True + if branch_pushed: + print(f" [dry-run] fix branch {fix_branch!r} exists on origin with no open " + f"PR — would open a PR from it and finish the auto-fix.") + return True # Count patches and show per-patch file summaries. subjects = [l[len("Subject: "):] for l in transformed.splitlines() if l.startswith("Subject: ")] - print(f" [dry-run] would apply {len(subjects)} commit(s) to reorg-fix/pr-{pr_number}:") + print(f" [dry-run] would apply {len(subjects)} commit(s) to {fix_branch}:") for s in subjects[:10]: print(f" {s}") if len(subjects) > 10: @@ -628,17 +767,34 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: f"and label it {LABEL_AUTOFIXED!r}") return True - fix_branch = f"reorg-fix/pr-{pr_number}" + # Finish an interrupted run without rebuilding (and never force-pushing) the + # branch, so a fix already in review is never clobbered. + if existing_fix_pr: + print(f" Fix PR #{existing_fix_pr['number']} already exists — " + f"finishing the interrupted auto-fix.") + return finalize_autofix(pr, fix_branch, existing_fix_pr) + if branch_pushed: + print(f" Fix branch {fix_branch!r} exists on origin but has no open PR — " + f"opening it and finishing the interrupted auto-fix.") + return finalize_autofix(pr, fix_branch, None) + tmpdir = tempfile.mkdtemp(prefix=f"reorg_fix_{pr_number}_") try: + # A stale LOCAL branch can linger when a prior run's worktree was cleaned + # up but its branch wasn't (worktree removal doesn't delete the branch). + # We've already confirmed there's no fix PR and no origin branch, so any + # local reorg-fix/pr- is a safe-to-drop leftover. Prune first so the + # delete isn't blocked by a registration for an already-deleted worktree. + git("worktree", "prune") + git("branch", "-D", fix_branch) # no-op if it doesn't exist + add_wt = git("worktree", "add", "-b", fix_branch, tmpdir, f"origin/{pr_base}") if add_wt.returncode != 0: - # Creating the branch failed — most likely a reorg-fix/pr- branch - # from a prior run already exists. We do NOT reset and reuse it: - # that could clobber a fix that's already in review. Bail and let - # the PR fall back to manual review. + # Still couldn't create the branch (e.g. it's checked out in a + # lingering worktree we couldn't prune). Bail to manual review rather + # than risk clobbering anything. print(f" could not create fix branch {fix_branch!r} " - f"(may already exist) — leaving for manual review: " + f"— leaving for manual review: " f"{add_wt.stderr.strip()[:120]}", file=sys.stderr) return False worktree = Path(tmpdir) @@ -655,49 +811,16 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: push = run(["git", "push", "origin", fix_branch], cwd=worktree) if push.returncode != 0: - # A reorg-fix/pr- branch from a prior run already exists on - # origin. We don't force-push (repo policy), and a fix PR for it - # most likely already exists, so bail to manual review rather than - # clobber it. - print(f" push failed (fix branch may already exist): " - f"{push.stderr.strip()[:120]}", file=sys.stderr) - return False + # The branch appeared on origin since our earlier check (a concurrent + # run, most likely). We don't force-push; finish via whatever is + # already there rather than clobber it. + print(f" push failed (fix branch appeared on origin): " + f"{push.stderr.strip()[:120]} — finishing via the existing branch.", + file=sys.stderr) + return finalize_autofix(pr, fix_branch, find_open_fix_pr(fix_branch)) print(f" Pushed fix to branch {fix_branch!r}") - - # Open a new PR for the fix branch so the author can preview, review, - # and merge it directly — then close the original conflicting PR. - original_body = pr.get("body") or "" - new_pr_body = build_autofix_pr_body(pr_number, original_body) - # We only reach this point on a real run; dry-run returned earlier. - new_pr_title = f"[reorg fix] {pr['title']}" - pr_create = gh_run( - "pr", "create", - "--repo", REPO, - "--head", fix_branch, - "--base", pr_base, - "--title", new_pr_title, - "--body", new_pr_body, - ) - new_pr_url = pr_create.strip() - new_pr_number = new_pr_url.rstrip("/").split("/")[-1] - print(f" Opened fix PR: {new_pr_url}") - - add_label(int(new_pr_number), LABEL_WIP, dry_run=False) - add_label(int(new_pr_number), LABEL_AUTO_PR, dry_run=False) - - # In test mode, keep the auto-created PR out of other teams' review - # queues by marking it "Do Not Merge". Never do this on real master. - if IS_TEST_MODE: - add_label(int(new_pr_number), DO_NOT_MERGE_LABEL, dry_run=False) - - gh_run( - "pr", "close", str(pr_number), "--repo", REPO, - "--comment", build_autofix_close_comment(new_pr_number), - ) - print(f" Closed PR #{pr_number} with comment pointing to fix PR #{new_pr_number}") - add_label(pr_number, LABEL_AUTOFIXED, dry_run) - return True + return finalize_autofix(pr, fix_branch, None) finally: git("worktree", "remove", "--force", tmpdir) @@ -807,7 +930,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: # real conflict. print(" Merge failed but no conflicts could be classified " "— labeling for manual review.") - add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + send_to_manual_review(pr_number, dry_run) return True print(" No conflicts found locally (GitHub mergeability may be stale).") return False @@ -816,7 +939,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: # The PR has conflicts that are NOT from the reorg. We must not # touch it — just label it so a human can resolve it manually. print(" Non-reorg conflicts present — labeling for manual review.") - add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) + send_to_manual_review(pr_number, dry_run) return True # All conflicts are reorg-caused. Before attempting an auto-fix, skip @@ -872,8 +995,7 @@ def analyze_pr(pr: dict, dry_run: bool) -> bool: return True # Auto-fix failed (fork, apply error, etc.) — fall back to labeling. print(" Auto-fix failed or not applicable — labeling for manual review.") - add_label(pr_number, LABEL_MANUAL_REVIEW, dry_run) - post_comment(pr_number, build_manual_review_comment(), dry_run) + send_to_manual_review(pr_number, dry_run) return True finally: @@ -897,7 +1019,7 @@ def get_open_prs(only: list[int] | None = None, limit: int = 50) -> list[dict]: # Only fields actually consumed below. Note baseRefOid/headRefOid are NOT # requested: `gh pr list` doesn't support them (only `gh pr view` does), and # nothing here uses them. - fields = ("number,title,body,labels,headRefName," + fields = ("number,title,body,author,labels,headRefName," "baseRefName,isCrossRepository,mergeable,updatedAt") if only: # Explicitly named PRs are an intentional override, but still guard From a17dd112f7c3f36a0aa0d218d5fa7ed6dcf74284 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 16:00:38 -0500 Subject: [PATCH 60/70] Tweak comments --- astro_reorg/resolve_pr_conflicts.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 817793b88ad..bec6dda1a5e 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -167,6 +167,12 @@ # any skew so our own labeling never counts as fresh activity. STALE_REACTIVATE_GRACE = timedelta(seconds=60) +# Appended to every author-facing comment so readers know how to follow up. +AUTOMATED_COMMENT_FOOTER = ( + "\n\nThis is an automated comment, but if you have a question, you can mention me in this PR " + "(external contributors) or reach out in #docs-repo-reorg-support on Slack (internal contributors)." +) + # --------------------------------------------------------------------------- # Comment builders — one function per user-facing message # --------------------------------------------------------------------------- @@ -185,6 +191,7 @@ def build_manual_review_comment() -> str: "if you would like the docs team to merge it for you.\n\n" f"If you prefer that we resolve your conflicts, add the label `{LABEL_HELP_REQUESTED}` to your PR. " "This will add it to our support queue, and we will reach out to you as soon as possible." + + AUTOMATED_COMMENT_FOOTER ) @@ -194,7 +201,8 @@ def build_stale_comment() -> str: f"This PR has conflicts created by the [docs repo reorg project]({REPO_REORG_README_LINK}). " f"Because this PR is stale (more than {STALE_DAYS} days old), no attempt was made to auto-resolve the conflicts. " "If you still intend to use this PR, remove the label " - f"`{LABEL_STALE}`. Your PR will be processed in the next batch of attempted auto-fixes. " + f"`{LABEL_STALE}`. Your PR will be processed in the next batch of attempted auto-fixes." + + AUTOMATED_COMMENT_FOOTER ) @@ -205,6 +213,7 @@ def build_wip_comment() -> str: "Because this PR is marked as a work in progress, no attempt was made to auto-resolve the conflicts. " f"When your PR is ready, remove the `{LABEL_WIP}` label and the `{LABEL_SKIP}` label. " "Your PR will be processed in the next batch of attempted auto-fixes." + + AUTOMATED_COMMENT_FOOTER ) @@ -216,7 +225,8 @@ def build_autofix_close_comment(new_pr_number: int | str) -> str: f"(files moved from the repo root into `hugo/`). " f"A new PR with your commits translated to the correct paths " f"has been opened: #{new_pr_number}\n\n" - f"If the new PR looks correct, merge it." + f"Please follow the instructions in the PR description." + + AUTOMATED_COMMENT_FOOTER ) @@ -233,7 +243,7 @@ def build_autofix_pr_body( between, the @mention is the author's only signal that the fix exists. """ mention = ( - f"@{author_login} — the docs repo reorg created merge conflicts in your " + f"@{author_login} — the [docs repo reorg]({REPO_REORG_README_LINK}) created merge conflicts in your " f"PR #{original_pr_number}. This is an auto-generated replacement with the " f"file paths fixed; please use it instead of the original.\n\n" if author_login else "" From 78aec3f03b21bb19cb93d163e3854c86f88c7bea Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 16:18:04 -0500 Subject: [PATCH 61/70] Add mermaid diagram --- astro_reorg/docs/pr_handling_details.md | 77 +++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/astro_reorg/docs/pr_handling_details.md b/astro_reorg/docs/pr_handling_details.md index 0a10052066f..d444caf6b3d 100644 --- a/astro_reorg/docs/pr_handling_details.md +++ b/astro_reorg/docs/pr_handling_details.md @@ -102,3 +102,80 @@ To instead ask a person to step in rather than have the script retry, the author ## Limiting a run and dry runs - A run can be capped to act on only a set number of PRs. A PR counts against that cap when it's auto-fixed or gets a review/stale label and comment. PRs that need no action — clean PRs, ones GitHub hasn't evaluated, or ones that turn out clean on the trial merge — don't count. - By default the script only reports what it *would* do, without changing anything, and runs against a mock base branch for safety. There are flags to apply changes for real and to run against the real `master`. + +## State machine diagram + +```mermaid +flowchart TD + START([Query open PRs targeting base branch]) --> FILTER + + FILTER{Already has
NO_CONFLICTS label,
MANUAL_REVIEW label,
HELP_REQUESTED label,
or SKIP label?} + FILTER -- Yes --> IGNORE([⏭️  Ignore — skip permanently]) + FILTER -- No --> CONFLICT_STATUS + + CONFLICT_STATUS{Conflict
status?} + CONFLICT_STATUS -- No conflicts --> CLEAN[Add NO_CONFLICTS label] + CONFLICT_STATUS -- Unknown / not yet computed --> SKIP_PENDING([⏭️  Skip — no label
reassess next run]) + CONFLICT_STATUS -- Has conflicts --> TRIAL_MERGE + + CLEAN --> DONE_CLEAN([✅  Done — doesn't count against cap]) + + TRIAL_MERGE[Trial merge
against post-reorg base] + TRIAL_MERGE --> TRIAL_RESULT + + TRIAL_RESULT{Trial merge
result?} + TRIAL_RESULT -- "Can't identify conflicting files" --> MANUAL_REVIEW + TRIAL_RESULT -- "Trial merge clean
(stale GitHub status)" --> SKIP_STALE_STATUS([⏭️  Skip — no label
reassess next run]) + TRIAL_RESULT -- "Non-reorg conflicts
exist" --> MANUAL_REVIEW + TRIAL_RESULT -- "All conflicts are
reorg-caused" --> CHECK_WIP + + CHECK_WIP{PR has
WIP label?} + CHECK_WIP -- Yes --> WIP_ACTION["Comment: how to ask for help
Add SKIP label"] + CHECK_WIP -- No --> CHECK_STALE + + WIP_ACTION --> DONE_WIP([✅  Done — counts against cap]) + + CHECK_STALE{Is PR
stale?} + CHECK_STALE -- No --> AUTOFIX + CHECK_STALE -- "Yes, not yet
labeled stale" --> STALE_LABEL["Comment: PR is stale
Add STALE label"] + CHECK_STALE -- "Yes, already
has STALE label" --> CHECK_REACTIVATED + + STALE_LABEL --> DONE_STALE([✅  Done — counts against cap]) + + CHECK_REACTIVATED{Reactivated since
label was applied?} + CHECK_REACTIVATED -- "Can't read
activity history" --> SKIP_STALE([⏭️  Skip quietly — no label
no count against cap]) + CHECK_REACTIVATED -- Not reactivated --> SKIP_STALE + CHECK_REACTIVATED -- Reactivated --> REMOVE_STALE["Remove STALE label"] + + REMOVE_STALE --> AUTOFIX + + AUTOFIX{Partial fix
already in progress?} + AUTOFIX -- "Fix PR already open" --> REUSE["Reuse fix PR
Re-apply labels"] + AUTOFIX -- "Fix branch pushed
but no PR" --> OPEN_PR_FROM_BRANCH["Open fix PR
from existing branch"] + AUTOFIX -- "No partial state" --> REPLAY + + REPLAY[Replay commits onto
new fix branch with
updated hugo/ paths] + REPLAY -- "Replay fails
(fork, locked branch,
or other error)" --> MANUAL_REVIEW + REPLAY -- Success --> PUSH_AND_OPEN["Push fix branch
Open fix PR
(AUTO_PR + WIP labels;
DO_NOT_MERGE label in test runs)
@mention original author"] + + PUSH_AND_OPEN --> CLOSE_ORIGINAL + REUSE --> CLOSE_ORIGINAL + OPEN_PR_FROM_BRANCH --> CLOSE_ORIGINAL + + CLOSE_ORIGINAL["Close original PR
with link to fix PR
Add AUTOFIXED label to original
Add PROCESSED label to both"] + CLOSE_ORIGINAL --> DONE_FIXED([✅  Done — counts against cap]) + + MANUAL_REVIEW["🔴  Add MANUAL_REVIEW + WIP labels
Post comment:
resolve conflicts or add
HELP_REQUESTED label
(excluded from all future runs)"] + MANUAL_REVIEW --> DONE_MANUAL([✅  Done — counts against cap]) + + style MANUAL_REVIEW fill:#fff,stroke:#000,color:#000 + style DONE_FIXED fill:#fff,stroke:#000,color:#000 + style DONE_CLEAN fill:#fff,stroke:#000,color:#000 + style IGNORE fill:#fff,stroke:#000,color:#000 + style SKIP_PENDING fill:#fff,stroke:#000,color:#000 + style SKIP_STALE fill:#fff,stroke:#000,color:#000 + style SKIP_STALE_STATUS fill:#fff,stroke:#000,color:#000 + style DONE_WIP fill:#fff,stroke:#000,color:#000 + style DONE_STALE fill:#fff,stroke:#000,color:#000 + style DONE_MANUAL fill:#fff,stroke:#000,color:#000 +``` From 6ab7cce45a4350b783bb3d9f1159ee7371212b91 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Thu, 16 Jul 2026 16:33:36 -0500 Subject: [PATCH 62/70] Update mock master reorg branch name --- astro_reorg/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index c389c32d267..8459affc3d2 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -133,7 +133,7 @@ ignore: test: # Stand-in for post-reorg master: the branch that has the reorg applied. Test # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. - mock_reorged_master_branch: mock-reorged-master-7-16-128 + mock_reorged_master_branch: mock-reorged-master-7-16-433 # Frozen snapshot of master that test PRs branch off, so each PR's diff stays # small (a PR shows every commit in its head that isn't in the base branch). branch_from: jen.gilbert/astro-reorg-scripts From 21c5beb233b54e1159d5fe979a7bcf7f8d07f6a7 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 17 Jul 2026 08:43:51 -0500 Subject: [PATCH 63/70] Update mock master branch name --- astro_reorg/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index 8459affc3d2..e779587e889 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -133,7 +133,7 @@ ignore: test: # Stand-in for post-reorg master: the branch that has the reorg applied. Test # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. - mock_reorged_master_branch: mock-reorged-master-7-16-433 + mock_reorged_master_branch: mock-reorged-master-7-17-843 # Frozen snapshot of master that test PRs branch off, so each PR's diff stays # small (a PR shows every commit in its head that isn't in the base branch). branch_from: jen.gilbert/astro-reorg-scripts From fe65d492b44aa61e6e1b6c62735c70a289b6b179 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Fri, 17 Jul 2026 13:05:12 -0500 Subject: [PATCH 64/70] Fix broken ignore negation on build config files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 500a34f95e9..f78a8aae142 100644 --- a/.gitignore +++ b/.gitignore @@ -301,6 +301,7 @@ assets/jsconfig.json # local build files .backup_path local/bin/py/build/* +!local/bin/py/build/configurations/ !local/bin/py/build/configurations/pull_config_preview.yaml !local/bin/py/build/configurations/pull_config.yaml !local/bin/py/build/configurations/integration_merge.yaml From c66cb838fb1103f39ef75765c180699223a258f1 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Mon, 20 Jul 2026 10:55:53 -0500 Subject: [PATCH 65/70] Update reorg execution steps --- astro_reorg/docs/reorg_execution_steps.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/astro_reorg/docs/reorg_execution_steps.md b/astro_reorg/docs/reorg_execution_steps.md index ceb5b675df1..710d48eeef6 100644 --- a/astro_reorg/docs/reorg_execution_steps.md +++ b/astro_reorg/docs/reorg_execution_steps.md @@ -50,7 +50,9 @@ Auto-fixable PRs are closed, manual-intervention PRs are automatically labeled ` Once an author has reviewed an auto-created PR and removed the `WORK IN PROGRESS` label, you can treat it as any other PR. -If you have questions, please reach out in #docs-repo-reorg-support. +**You are not expected to provide support for any issues arising from the reorg.** I'll be here to help all day, and for longer than that if needed. If someone raises an issue in #documentation, you can just tag me or cross-post it to #docs-repo-reorg-support. + +If you have questions, please ask them on this thread or reach out in #docs-repo-reorg-support. ``` ## The day of @@ -105,6 +107,18 @@ gh pr create --base master --title "Create Hugo site folder" ### 6. Verify that the GitHub actions etc. are working, and make any necessary fixes +- [ ] Verify that GitHub actions aren't erring out from bad paths. +- [ ] While the preview is building, run some local checks: + - [ ] The Cdocs e2e tests are passing. + - [ ] Integrations pages are working correctly. + - [ ] API pages are working correctly. + - [ ] Single sourced content is working correctly. + +### 7. Verify the preview build + +- [ ] The GitLab artifacts for the preview build should be similar to those of the last build on `master`: same number of HTML files in `public`, for example. +- [ ] + ### 7. Merge to master and verify the build ### 8. Run the PR resolution script From e39c9cb14cbf7176ae5ea032b50beae2ecc6ab5a Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Mon, 20 Jul 2026 11:00:48 -0500 Subject: [PATCH 66/70] Tweak PR comment verbiage --- astro_reorg/resolve_pr_conflicts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index bec6dda1a5e..435d5505a54 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -189,7 +189,7 @@ def build_manual_review_comment() -> str: "3. Wait for the standard docs team approval before merging. " "Optionally, you can check the 'ready for merge' checkbox in the PR description " "if you would like the docs team to merge it for you.\n\n" - f"If you prefer that we resolve your conflicts, add the label `{LABEL_HELP_REQUESTED}` to your PR. " + f"If you need assistance resolving your conflicts, add the label `{LABEL_HELP_REQUESTED}` to your PR. " "This will add it to our support queue, and we will reach out to you as soon as possible." + AUTOMATED_COMMENT_FOOTER ) @@ -211,14 +211,14 @@ def build_wip_comment() -> str: return ( f"This PR has merge conflicts created by the [docs repo reorg project]({REPO_REORG_README_LINK}). " "Because this PR is marked as a work in progress, no attempt was made to auto-resolve the conflicts. " - f"When your PR is ready, remove the `{LABEL_WIP}` label and the `{LABEL_SKIP}` label. " + f"When your PR is finished, you can queue your PR for an auto-fix by removing the `{LABEL_WIP}` label and the `{LABEL_SKIP}` label. " "Your PR will be processed in the next batch of attempted auto-fixes." + AUTOMATED_COMMENT_FOOTER ) def build_autofix_close_comment(new_pr_number: int | str) -> str: - """Posted on the original PR when it is closed in favour of a fix PR.""" + """Posted on the original PR when it is closed in favor of a fix PR.""" return ( f"🤖 **Reorg conflict auto-fix:**\n\n" f"This PR has merge conflicts caused by the [recent docs repo reorg]({REPO_REORG_README_LINK}) " From 6258c1744cd8f61bbc68e2f0185d1a24d5815151 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Mon, 20 Jul 2026 11:03:16 -0500 Subject: [PATCH 67/70] Move generated PR list to a more appropriate folder --- .gitignore | 2 -- astro_reorg/create_test_prs.py | 2 +- astro_reorg/docs/.gitignore | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 astro_reorg/docs/.gitignore diff --git a/.gitignore b/.gitignore index f78a8aae142..ad550fc4359 100644 --- a/.gitignore +++ b/.gitignore @@ -340,5 +340,3 @@ playwright-report/ .superpowers/ data/site_region_usage.json -# Reorg tooling -test_pr_list.md diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index 59208ffb190..caebbded76b 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -535,7 +535,7 @@ def write_pr_list( "", ] - output_path = REPO_ROOT / "test_pr_list.md" + output_path = REPO_ROOT / "astro_reorg" / "docs" / "test_pr_list.md" output_path.write_text("\n".join(lines)) print(f"\nWrote {output_path}") diff --git a/astro_reorg/docs/.gitignore b/astro_reorg/docs/.gitignore new file mode 100644 index 00000000000..cc822d70466 --- /dev/null +++ b/astro_reorg/docs/.gitignore @@ -0,0 +1 @@ +test_pr_list.md From ce3cb3fbd90dc6f44c1b5cda0d07389cd9a752e1 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Mon, 20 Jul 2026 11:41:43 -0500 Subject: [PATCH 68/70] Code review --- astro_reorg/create_test_prs.py | 3 --- astro_reorg/execute_reorg.py | 28 ++++++++++++++-------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/astro_reorg/create_test_prs.py b/astro_reorg/create_test_prs.py index caebbded76b..52e69d78e83 100644 --- a/astro_reorg/create_test_prs.py +++ b/astro_reorg/create_test_prs.py @@ -7,9 +7,6 @@ 3. commits and pushes the branch, 4. opens a PR against the base branch (see below). -On completion it opens each new PR in the browser (falling back to printing -the URLs if a browser can't be launched). - BRANCH_FROM and MOCK_BASE_BRANCH are read from the `test:` section of config.yaml (shared with resolve_pr_conflicts.py, which defaults to this same mock base branch). Both must already exist on the remote before running. diff --git a/astro_reorg/execute_reorg.py b/astro_reorg/execute_reorg.py index 8ab62868f03..ce2470d411a 100644 --- a/astro_reorg/execute_reorg.py +++ b/astro_reorg/execute_reorg.py @@ -16,14 +16,14 @@ with config_path.open() as f: config = yaml.safe_load(f) -top_level = set(config.get("top_level", [])) -moves_to_hugo = set(config.get("moves_to_hugo", [])) +repo_level_files = set(config.get("top_level", [])) +hugo_level_files = set(config.get("moves_to_hugo", [])) ignore = set(config.get("ignore", [])) # Sanity-check the config itself for conflicts. -conflicts = top_level & moves_to_hugo -if conflicts: - for name in sorted(conflicts): +config_inconsistencies = repo_level_files & hugo_level_files +if config_inconsistencies: + for name in sorted(config_inconsistencies): print(f"ERROR: '{name}' appears in both top_level and moves_to_hugo", file=sys.stderr) sys.exit(1) @@ -33,9 +33,9 @@ if name in ignore: continue - if name in top_level: + if name in repo_level_files: pass # keep in place - elif name in moves_to_hugo: + elif name in hugo_level_files: pass # will move below else: errors.append(name) @@ -52,7 +52,7 @@ moved = 0 deleted = 0 for name in sorted(os.listdir(repo_root)): - if name in ignore or name in top_level or name == "hugo": + if name in ignore or name in repo_level_files or name == "hugo": continue src = repo_root / name dst = hugo_dir / name @@ -99,22 +99,22 @@ def route_gitignore_segment(line): print("\nSplitting .gitignore between root and hugo/...") gitignore = repo_root / ".gitignore" -gi_lines = gitignore.read_text().splitlines(keepends=True) +original_gitignore_lines = gitignore.read_text().splitlines(keepends=True) root_lines = [] hugo_lines = [] hugo_only_segments = set() # routed off root into hugo/ (for the summary) both_segments = set() # in neither config list -> kept in both (surfaced) -for raw in gi_lines: +for raw in original_gitignore_lines: segment = route_gitignore_segment(raw) if segment is None: # comment / blank -> keep in both root_lines.append(raw) hugo_lines.append(raw) - elif segment in moves_to_hugo: + elif segment in hugo_level_files: hugo_lines.append(raw) hugo_only_segments.add(segment) - elif segment in top_level: + elif segment in repo_level_files: root_lines.append(raw) else: root_lines.append(raw) @@ -212,7 +212,7 @@ def route_codeowners_pattern(pattern): body = "local" + body[len(".local"):] segment = "local" - if segment not in moves_to_hugo: + if segment not in hugo_level_files: return segment, None new_body = "hugo/" + body @@ -236,7 +236,7 @@ def route_codeowners_pattern(pattern): rest = stripped[len(pattern):] # original spacing + owners, verbatim segment, new_pattern = route_codeowners_pattern(pattern) if new_pattern is None: - if segment is not None and segment not in top_level: + if segment is not None and segment not in repo_level_files: left_alone.add(segment) continue lines[i] = indent + new_pattern + rest + newline From a8fbd35b25b8d7963299b87571096f3b5c0aeefb Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Mon, 20 Jul 2026 12:01:40 -0500 Subject: [PATCH 69/70] Code review --- astro_reorg/resolve_pr_conflicts.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/astro_reorg/resolve_pr_conflicts.py b/astro_reorg/resolve_pr_conflicts.py index 435d5505a54..9378b3a4405 100644 --- a/astro_reorg/resolve_pr_conflicts.py +++ b/astro_reorg/resolve_pr_conflicts.py @@ -27,7 +27,7 @@ Background: The reorg moves every entry in `moves_to_hugo` (astro_reorg/config.yaml) from the repo root into hugo/. For example, content/ → hugo/content/, - layouts/ → hugo/layouts/, etc. PRs opened before the reorg was merged to + layouts/ → hugo/layouts/, etc. PRs opened before the reorg was merged to master will have their branches pointing at the old paths. When github tries to compute mergeability, those PRs show as CONFLICTING. @@ -112,8 +112,8 @@ with CONFIG_PATH.open() as f: _config = yaml.safe_load(f) -MOVES_TO_HUGO: set[str] = set(_config.get("moves_to_hugo", [])) -TOP_LEVEL: set[str] = set(_config.get("top_level", [])) +HUGO_FOLDER_FILES: set[str] = set(_config.get("moves_to_hugo", [])) +TOP_LEVEL_FILES: set[str] = set(_config.get("top_level", [])) # Shared with create_test_prs.py: the mock base branch used for test runs. When # invoked without --live or --base-branch, the script defaults to this branch @@ -148,7 +148,6 @@ # Applied by the script to a work-in-progress PR after it comments once. Excluded # from the query so the PR isn't picked up (and re-commented on) on later runs. LABEL_SKIP = "astro-reorg-skip" -LABEL_WIP = "WORK IN PROGRESS" LABEL_COLOR = "e4e669" LABEL_DESCRIPTION = "Needs manual conflict resolution after replatforming reorg" # Applied to every PR the script acts on, regardless of outcome (auto-fixed, @@ -266,9 +265,10 @@ def build_autofix_pr_body( f"**Original PR description:**\n\n{original_body}" ) -# Existing repo label applied to auto-created fix PRs in test mode so they stay -# out of other teams' review queues. Assumed to already exist in the repo. -DO_NOT_MERGE_LABEL = "Do Not Merge" +# Existing repo labels assumed to already exist; the script checks for them but +# does not create them. +LABEL_WIP = "WORK IN PROGRESS" +LABEL_DO_NOT_MERGE = "Do Not Merge" # test mode only — keeps fix PRs out of review queues # Set in main() from --base-branch; everything else reads this. BASE_BRANCH = "master" @@ -365,9 +365,9 @@ def is_reorg_path(file_path: str) -> bool: parts = Path(file_path).parts if not parts: return False - if parts[0] in MOVES_TO_HUGO: + if parts[0] in HUGO_FOLDER_FILES: return True - if parts[0] == "hugo" and len(parts) > 1 and parts[1] in MOVES_TO_HUGO: + if parts[0] == "hugo" and len(parts) > 1 and parts[1] in HUGO_FOLDER_FILES: return True return False @@ -385,7 +385,7 @@ def to_post_reorg_path(file_path: str) -> str: return file_path if parts[0] == "hugo": return file_path - if parts[0] in MOVES_TO_HUGO: + if parts[0] in HUGO_FOLDER_FILES: return "hugo/" + file_path return file_path @@ -398,7 +398,7 @@ def to_pre_reorg_path(file_path: str) -> str | None: hugo/content/en/foo.md → content/en/foo.md """ parts = Path(file_path).parts - if len(parts) > 1 and parts[0] == "hugo" and parts[1] in MOVES_TO_HUGO: + if len(parts) > 1 and parts[0] == "hugo" and parts[1] in HUGO_FOLDER_FILES: return "/".join(parts[1:]) return None @@ -475,7 +475,7 @@ def get_wrong_path_additions(worktree: Path) -> list[str]: if len(parts) != 2: continue path = parts[1].strip() - if path_first_segment(path) in MOVES_TO_HUGO: + if path_first_segment(path) in HUGO_FOLDER_FILES: wrong.append(path) return wrong @@ -659,7 +659,7 @@ def finalize_autofix(pr: dict, fix_branch: str, existing_fix_pr: dict | None) -> # In test mode, keep the auto-created PR out of other teams' review queues by # marking it "Do Not Merge". Never do this on real master. if IS_TEST_MODE: - add_label(int(new_pr_number), DO_NOT_MERGE_LABEL, dry_run=False) + add_label(int(new_pr_number), LABEL_DO_NOT_MERGE, dry_run=False) gh_run( "pr", "close", str(pr_number), "--repo", REPO, @@ -772,7 +772,7 @@ def attempt_fix(pr: dict, dry_run: bool) -> bool: print(f" [dry-run] would open PR: {would_be_title!r}") print(f" [dry-run] would label fix PR {LABEL_WIP!r}, {LABEL_AUTO_PR!r}") if IS_TEST_MODE: - print(f" [dry-run] would label fix PR {DO_NOT_MERGE_LABEL!r} (test mode)") + print(f" [dry-run] would label fix PR {LABEL_DO_NOT_MERGE!r} (test mode)") print(f" [dry-run] would close PR #{pr_number} with comment pointing to fix PR, " f"and label it {LABEL_AUTOFIXED!r}") return True From 1171ea68f8e8cba36068a7fdc0cfbdd341618040 Mon Sep 17 00:00:00 2001 From: Jen Gilbert Date: Mon, 20 Jul 2026 12:31:42 -0500 Subject: [PATCH 70/70] Update the name of the mock master branch --- astro_reorg/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro_reorg/config.yaml b/astro_reorg/config.yaml index e779587e889..bb46a5eae47 100644 --- a/astro_reorg/config.yaml +++ b/astro_reorg/config.yaml @@ -133,7 +133,7 @@ ignore: test: # Stand-in for post-reorg master: the branch that has the reorg applied. Test # PRs target it, and resolve_pr_conflicts.py treats it as the base by default. - mock_reorged_master_branch: mock-reorged-master-7-17-843 + mock_reorged_master_branch: mock-reorged-master-7-20-1231 # Frozen snapshot of master that test PRs branch off, so each PR's diff stays # small (a PR shows every commit in its head that isn't in the base branch). branch_from: jen.gilbert/astro-reorg-scripts