diff --git a/src/whygraph/cli/commands/scan.py b/src/whygraph/cli/commands/scan.py index adfc5df..ac28be3 100644 --- a/src/whygraph/cli/commands/scan.py +++ b/src/whygraph/cli/commands/scan.py @@ -3,12 +3,20 @@ from __future__ import annotations import os +import time from pathlib import Path from typing import TYPE_CHECKING, TypeVar import click from rich.panel import Panel -from rich.progress import Progress +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, +) from rich.table import Table from rich.text import Text @@ -31,6 +39,13 @@ _T = TypeVar("_T") +# Per-phase icons for the live headers and the closing results panel. +# Kept in one place so the whole set is trivially swappable (plan ยง10.5). +_ICON_STRUCTURAL = "๐Ÿ”Ž" +_ICON_PR_ORIGINS = "๐Ÿ”—" +_ICON_LLM = "๐Ÿง " +_ICON_CODEGRAPH = "๐Ÿ•ธ" + @click.command(name="scan") @click.option( @@ -39,8 +54,9 @@ is_flag=True, default=False, help=( - "Skip Phase 2 (per-commit LLM descriptions). The git and GitHub " - "crawlers still run. The MCP tools `whygraph_evidence_for` and " + "Skip the final LLM-descriptions phase (per-commit descriptions). " + "The git and GitHub crawlers still run. The MCP tools " + "`whygraph_evidence_for` and " "`whygraph_rationale_brief` lazily backfill descriptions on demand, " "and a later `whygraph scan` (without this flag) backfills the rest." ), @@ -144,12 +160,34 @@ def scan_cmd( pr_origins_enabled=enrich_pr_origins and github_client is not None, ) + # Which optional phases have work โ€” decided up front so the phase + # headers can be numbered against the count of phases that actually run. + run_pr_origins = enrich_pr_origins and github_client is not None + run_analyze = descriptor is not None + phase_total = 1 + int(run_pr_origins) + int(run_analyze) # Phase 1 always runs + scan_log_path = db_path.parent / "scan.log" - with scan_log_redirect(scan_log_path), Progress() as progress: - # CodeGraph refresh โ€” runs concurrently as its own crawler. It - # writes .codegraph/ and has no data dependency on the WhyGraph DB, - # so it overlaps the entire crawl (started with phase 1, joined - # last). Best-effort: failures land on .warning, not .error. + phase_timings: dict[str, float] = {} + ran: list[Crawler] = [] + scan_t0 = time.monotonic() + # Share the stderr `console` with Progress so the phase headers render + # above the live bars on one stream, and add an M-of-N + elapsed column + # so the slow LLM phase reports "12/45 ยท 0:00:31". + with ( + scan_log_redirect(scan_log_path), + Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TimeElapsedColumn(), + console=console, + ) as progress, + ): + # CodeGraph refresh โ€” a background crawler. It writes .codegraph/ + # and has no data dependency on the WhyGraph DB, so it overlaps the + # entire crawl (started before Phase 1, joined last). Best-effort: + # failures land on .warning, not .error. codegraph_crawler = ( CodeGraphCrawler( progress, project_root=repository.root, image=codegraph_image @@ -157,60 +195,99 @@ def scan_cmd( if refresh_codegraph else None ) + if codegraph_crawler is not None: + codegraph_crawler.start() - # Phase 1 โ€” source crawlers, run concurrently. + n = 0 + + # โ”€โ”€ Phase 1 ยท Structural crawl โ€” git + GitHub, concurrent. โ”€โ”€ + n += 1 + console.rule( + f"{_ICON_STRUCTURAL} Phase {n}/{phase_total} ยท Structural crawl", + style="cyan", + ) + t0 = time.monotonic() phase1: list[Crawler] = [GitCrawler(progress, repository=repository)] if github_client is not None: phase1.append(GitHubCrawler(progress, client=github_client)) - - # Phase 2 โ€” started only once phase 1 has joined (it reads the - # commits + PRs phase 1 persisted). The analyzer and the PR-origin - # enricher run concurrently: analyze only touches main-walk commits, - # the enricher only inserts new on_default_branch=0 rows, so they - # never contend over the same commit row. - phase2: list[Crawler] = [] - if descriptor is not None: - phase2.append( - AnalyzeCrawler( - progress, - repository=repository, - descriptor=descriptor, - max_workers=config.scan_max_workers, - large_commit_file_count=config.analyze.large_commit_file_count, - ) - ) - # The enricher needs PR rows (the GitHub crawler ran) and the - # network for its fetch โ€” so it is gated on a resolved client, which - # is itself None under --no-remote. - if enrich_pr_origins and github_client is not None: - phase2.append( - PROriginEnricher( - progress, - repository=repository, - min_commits=config.analyze.pr_origin_min_commits, - large_commit_file_count=config.analyze.large_commit_file_count, - ) - ) - - if codegraph_crawler is not None: - codegraph_crawler.start() + ran += phase1 for c in phase1: c.start() for c in phase1: c.join() - for c in phase2: - c.start() - for c in phase2: - c.join() + phase_timings["Structural crawl"] = time.monotonic() - t0 + _print_phase_done( + "Structural crawl", + phase_timings["Structural crawl"], + ok=all(c.error is None for c in phase1), + ) + + # โ”€โ”€ Phase 2 ยท PR-origin recovery โ€” needs Phase 1's git + PR rows. + # Gated on a resolved client, which is None under --no-remote. โ”€โ”€ + if run_pr_origins: + n += 1 + console.rule( + f"{_ICON_PR_ORIGINS} Phase {n}/{phase_total} ยท PR-origin recovery", + style="cyan", + ) + t0 = time.monotonic() + enricher = PROriginEnricher( + progress, + repository=repository, + min_commits=config.analyze.pr_origin_min_commits, + large_commit_file_count=config.analyze.large_commit_file_count, + ) + ran.append(enricher) + enricher.start() + enricher.join() + phase_timings["PR-origin recovery"] = time.monotonic() - t0 + _print_phase_done( + "PR-origin recovery", + phase_timings["PR-origin recovery"], + ok=enricher.error is None, + ) + + # โ”€โ”€ Phase 3 ยท LLM descriptions โ€” the slow, token-heavy long pole, + # run strictly last and alone. Only ever describes main-walk + # commits, so the recovered on_default_branch=0 rows stay lazy. โ”€โ”€ + if run_analyze: + n += 1 + console.rule( + f"{_ICON_LLM} Phase {n}/{phase_total} ยท LLM descriptions", + style="cyan", + ) + t0 = time.monotonic() + analyzer = AnalyzeCrawler( + progress, + repository=repository, + descriptor=descriptor, + max_workers=config.scan_max_workers, + large_commit_file_count=config.analyze.large_commit_file_count, + ) + ran.append(analyzer) + analyzer.start() + analyzer.join() + phase_timings["LLM descriptions"] = time.monotonic() - t0 + _print_phase_done( + "LLM descriptions", + phase_timings["LLM descriptions"], + ok=analyzer.error is None, + ) + if codegraph_crawler is not None: codegraph_crawler.join() - console.print(f"Scan log: {scan_log_path}") - - if codegraph_crawler is not None and codegraph_crawler.warning is not None: - console.print(Text(codegraph_crawler.warning, style="yellow")) + total_elapsed = time.monotonic() - scan_t0 + _render_results_panel( + ran=ran, + codegraph_crawler=codegraph_crawler, + db_path=db_path, + scan_log_path=scan_log_path, + phase_timings=phase_timings, + total_elapsed=total_elapsed, + ) - crawlers = phase1 + phase2 + crawlers = list(ran) if codegraph_crawler is not None: crawlers.append(codegraph_crawler) failed = [c for c in crawlers if c.error is not None] @@ -220,6 +297,138 @@ def scan_cmd( raise click.exceptions.Exit(1) +def _fmt_elapsed(seconds: float) -> str: + """Format an elapsed duration as ``"4.1s"`` or ``"2m 08s"``.""" + if seconds < 60: + return f"{seconds:.1f}s" + minutes, secs = divmod(int(round(seconds)), 60) + return f"{minutes}m {secs:02d}s" + + +def _print_phase_done(title: str, seconds: float, *, ok: bool) -> None: + """Print the dim per-phase completion line under that phase's bars. + + ``ok`` reflects whether every crawler in the phase finished without an + error; a failed phase gets a red ``โœ—`` (the real error is also surfaced + by the closing failure sweep and results panel). + """ + glyph = "โœ“" if ok else "โœ—" + console.print( + f" {glyph} {title} ยท {_fmt_elapsed(seconds)}", + style="dim" if ok else "red", + ) + + +def _status_glyph(*, ok: bool, warn: bool = False) -> Text: + """Return the results-panel status cell: ``โœ“`` / ``โš `` / ``โœ—``.""" + if not ok: + return Text("โœ—", style="red") + if warn: + return Text("โš ", style="yellow") + return Text("โœ“", style="green") + + +def _optional_phase_cells( + crawler: "Crawler | None", timing: str +) -> tuple[object, object, str]: + """Return the (status, summary, timing) cells for an optional phase. + + A crawler that never ran (phase skipped via ``--no-remote`` / + ``--no-llm-descriptions``) renders a dim ``โ€” skipped`` status with no + summary or timing. + """ + if crawler is None: + return Text("โ€” skipped", style="dim"), "", "" + return _status_glyph(ok=crawler.error is None), crawler.summary or "โ€”", timing + + +def _render_results_panel( + *, + ran: "list[Crawler]", + codegraph_crawler: "Crawler | None", + db_path: Path, + scan_log_path: Path, + phase_timings: "dict[str, float]", + total_elapsed: float, +) -> None: + """Print the closing results panel โ€” a bookend to the pre-scan panel. + + One row per phase (git + GitHub merge into the structural row), each + carrying a status glyph, the crawler's own one-line summary, and the + phase's elapsed time; then the database / scan-log paths. CodeGraph is + a background task, so it gets a row but no per-phase timing. Pure + formatting over data already in hand โ€” no DB or network access โ€” so it + can never turn a successful crawl into a crash. + """ + by_name = {c.name: c for c in ran} + git = by_name.get("git") + github = by_name.get("github") + enricher = by_name.get("pr-origins") + analyzer = by_name.get("analyze") + + def _timing(title: str) -> str: + seconds = phase_timings.get(title) + return _fmt_elapsed(seconds) if seconds is not None else "" + + grid = Table.grid(padding=(0, 2)) + grid.add_column(no_wrap=True) # icon + grid.add_column(style="bold cyan", no_wrap=True) # label + grid.add_column(justify="center", no_wrap=True) # status + grid.add_column(overflow="fold") # summary + grid.add_column(justify="right", no_wrap=True) # timing + + # Structural row โ€” git + GitHub combined into one phase row. + structural = [c for c in (git, github) if c is not None] + structural_summary = " ยท ".join(c.summary for c in structural if c.summary) or "โ€”" + grid.add_row( + _ICON_STRUCTURAL, + "Structural crawl", + _status_glyph(ok=all(c.error is None for c in structural)), + structural_summary, + _timing("Structural crawl"), + ) + grid.add_row( + _ICON_PR_ORIGINS, + "PR-origin recovery", + *_optional_phase_cells(enricher, _timing("PR-origin recovery")), + ) + grid.add_row( + _ICON_LLM, + "LLM descriptions", + *_optional_phase_cells(analyzer, _timing("LLM descriptions")), + ) + + # CodeGraph โ€” background task; no per-phase timing. + if codegraph_crawler is None: + grid.add_row( + _ICON_CODEGRAPH, "CodeGraph", Text("โ€” skipped", style="dim"), "", "" + ) + else: + warning = getattr(codegraph_crawler, "warning", None) + grid.add_row( + _ICON_CODEGRAPH, + "CodeGraph", + _status_glyph(ok=codegraph_crawler.error is None, warn=warning is not None), + codegraph_crawler.summary or warning or "โ€”", + "", + ) + + grid.add_row("", "", "", "", "") + grid.add_row("", "Database", "", str(db_path), "") + grid.add_row("", "Scan log", "", str(scan_log_path), "") + + console.print( + Panel( + grid, + title=f"whygraph scan ยท done in {_fmt_elapsed(total_elapsed)}", + title_align="left", + border_style="cyan", + padding=(1, 2), + ) + ) + console.print() + + def _apply_github_token(config: "Config") -> None: """Export the configured GitHub token so every ``gh`` subprocess sees it. diff --git a/src/whygraph/scan/__init__.py b/src/whygraph/scan/__init__.py index 8860960..c5cecc9 100644 --- a/src/whygraph/scan/__init__.py +++ b/src/whygraph/scan/__init__.py @@ -1,14 +1,20 @@ -"""Scan subsystem โ€” concurrent crawlers that populate the WhyGraph DB. +"""Scan subsystem โ€” phased crawlers that populate the WhyGraph DB. Exposes :class:`Crawler` (the threaded base class) and the concrete crawlers โ€” :class:`GitCrawler` for local git history, :class:`GitHubCrawler` for GitHub pull requests and issues, :class:`CodeGraphCrawler` which refreshes the CodeGraph index, -:class:`AnalyzeCrawler` which describes each commit's diff with an LLM -(run after :class:`GitCrawler`), and :class:`PROriginEnricher` which -recovers a squash-merged PR's original commits. The CLI runs the source -crawlers (and CodeGraph) concurrently, then the analyzer and enricher, -against the shared SQLite database. +:class:`AnalyzeCrawler` which describes each commit's diff with an LLM, +and :class:`PROriginEnricher` which recovers a squash-merged PR's original +commits. + +The CLI drives them in three ordered phases against the shared SQLite +database: **Phase 1** runs :class:`GitCrawler` and :class:`GitHubCrawler` +concurrently; **Phase 2** runs :class:`PROriginEnricher` (which needs the +git + PR rows Phase 1 persisted); **Phase 3** runs :class:`AnalyzeCrawler` +last and alone (the slow, token-heavy LLM pass). :class:`CodeGraphCrawler` +is a best-effort background task started before Phase 1 and joined after +Phase 3 โ€” it has no data dependency on the DB, so it spans the whole scan. """ from whygraph.scan.analyze_crawler import AnalyzeCrawler diff --git a/src/whygraph/scan/analyze_crawler.py b/src/whygraph/scan/analyze_crawler.py index 9aa7367..1bcb68f 100644 --- a/src/whygraph/scan/analyze_crawler.py +++ b/src/whygraph/scan/analyze_crawler.py @@ -95,9 +95,11 @@ def work(self) -> None: todo = [c for c in commits if c.sha in pending] self.set_total(len(todo)) if not todo: + self.summary = "up to date" return failures: list[tuple[str, BaseException]] = [] + described = bulk = 0 with ThreadPoolExecutor(max_workers=self._max_workers) as pool: futures = {pool.submit(self._describe_commit, c): c.sha for c in todo} for future in as_completed(futures): @@ -105,6 +107,15 @@ def work(self) -> None: exc = future.exception() if exc is not None: failures.append((futures[future], exc)) + elif future.result() == "bulk": + bulk += 1 + elif future.result() == "described": + described += 1 + + parts = [f"{described} described"] + if bulk: + parts.append(f"{bulk} bulk-stubbed") + self.summary = " ยท ".join(parts) if failures: sha, exc = failures[0] @@ -113,7 +124,7 @@ def work(self) -> None: f"first failure {sha[:12]}: {exc}" ) - def _describe_commit(self, commit: CommitDC) -> None: + def _describe_commit(self, commit: CommitDC) -> str: """Describe one commit and persist it. Runs in a worker thread. Opens its own DB session โ€” sessions are not shareable across @@ -123,6 +134,13 @@ def _describe_commit(self, commit: CommitDC) -> None: in lazily on read. Commits with an empty diff are skipped silently; any other failure propagates and is collected by :meth:`work`. + + Returns + ------- + str + The outcome kind โ€” ``"bulk"`` (stub written), ``"empty"`` + (empty diff, skipped), or ``"described"`` (LLM description + written) โ€” tallied by :meth:`work` for its summary. """ if commit.stats.files_changed > self._large_commit_file_count: with get_session() as session: @@ -133,10 +151,10 @@ def _describe_commit(self, commit: CommitDC) -> None: # lazy read path uses to know a real per-file # description still needs generating. row.llm_description = bulk_commit_stub(commit.stats.files_changed) - return + return "bulk" diff = self._repository.diff(commit) if not diff.strip(): - return + return "empty" description = self._descriptor.describe(diff) with get_session() as session: row = session.get(CommitRow, commit.sha) @@ -145,3 +163,4 @@ def _describe_commit(self, commit: CommitDC) -> None: row.llm_description_model = ( f"{description.provider}:{description.model}" ) + return "described" diff --git a/src/whygraph/scan/codegraph_crawler.py b/src/whygraph/scan/codegraph_crawler.py index 9027c12..7f3e168 100644 --- a/src/whygraph/scan/codegraph_crawler.py +++ b/src/whygraph/scan/codegraph_crawler.py @@ -82,3 +82,4 @@ def work(self) -> None: # on a clean "complete" once the refresh returns. self.set_total(1) self.advance(1) + self.summary = "synced" if verb == "sync" else "indexed" diff --git a/src/whygraph/scan/crawler.py b/src/whygraph/scan/crawler.py index d09f418..855e8a1 100644 --- a/src/whygraph/scan/crawler.py +++ b/src/whygraph/scan/crawler.py @@ -50,6 +50,12 @@ class handles thread plumbing, progress-task registration, and Exception raised by :meth:`work`, captured so callers can inspect it after :meth:`threading.Thread.join`. ``None`` on success. + summary : str or None + One-line outcome delta set by :meth:`work` at the end of a + successful crawl (e.g. ``"45 commits"``), for the orchestrator's + closing results panel. Written from the worker thread and read + only after :meth:`threading.Thread.join`, mirroring the safety + model of :attr:`error`. ``None`` until set. """ def __init__( @@ -63,6 +69,7 @@ def __init__( self._progress = progress self._task_id: TaskID = progress.add_task(name, total=total) self.error: BaseException | None = None + self.summary: str | None = None # --- subclass hooks ------------------------------------------------ diff --git a/src/whygraph/scan/git_crawler.py b/src/whygraph/scan/git_crawler.py index 7df6652..8d23cbd 100644 --- a/src/whygraph/scan/git_crawler.py +++ b/src/whygraph/scan/git_crawler.py @@ -62,6 +62,7 @@ def work(self) -> None: session.exec(select(CommitFileChange.commit_sha).distinct()).all() ) scanned_at = datetime.now(timezone.utc).isoformat() + inserted = 0 for dc in commits: if dc.sha not in existing_file_changes: file_changes = self._repository.commit_file_changes(dc) @@ -78,6 +79,7 @@ def work(self) -> None: session.add( _to_row(dc, scanned_at=scanned_at, refactor_score=score) ) + inserted += 1 elif file_changes: # Existing commit row but we just computed its file # changes for the first time โ€” backfill the score so @@ -89,6 +91,8 @@ def work(self) -> None: session.add(existing) self.advance(1) + self.summary = f"{len(commits)} commits ({inserted} new)" + def _to_row(dc: CommitDC, *, scanned_at: str, refactor_score: int = 0) -> CommitRow: return CommitRow( diff --git a/src/whygraph/scan/github_crawler.py b/src/whygraph/scan/github_crawler.py index 159f295..f9caeaf 100644 --- a/src/whygraph/scan/github_crawler.py +++ b/src/whygraph/scan/github_crawler.py @@ -88,6 +88,8 @@ def work(self) -> None: session.add(_issue_to_row(issue, fetched_at=fetched_at)) self.advance(1) + self.summary = f"{len(prs)} PRs ยท {len(issues)} issues" + def _pr_to_row(dc: PullRequestDC, *, fetched_at: str) -> PullRequestRow: return PullRequestRow( diff --git a/src/whygraph/scan/pr_origin_enricher.py b/src/whygraph/scan/pr_origin_enricher.py index 37c31e2..fa245c1 100644 --- a/src/whygraph/scan/pr_origin_enricher.py +++ b/src/whygraph/scan/pr_origin_enricher.py @@ -214,6 +214,7 @@ def work(self) -> None: ) self.set_total(len(candidates)) if not candidates: + self.summary = "no squash candidates" return # ONE batched fetch โ€” only the gated candidates' refs, never the @@ -228,6 +229,7 @@ def work(self) -> None: len(candidates), exc, ) + self.summary = "fetch skipped" return scanned_at = datetime.now(timezone.utc).isoformat() @@ -246,3 +248,5 @@ def work(self) -> None: session.add(_to_origin_row(dc, scanned_at=scanned_at)) inserted.add(oid) self.advance(1) + + self.summary = f"{len(inserted)} commits recovered" diff --git a/tests/test_cli_scan_phases.py b/tests/test_cli_scan_phases.py new file mode 100644 index 0000000..08cc320 --- /dev/null +++ b/tests/test_cli_scan_phases.py @@ -0,0 +1,251 @@ +"""Tests for ``whygraph scan``'s three-phase orchestration and output. + +Pins the phase *sequencing* โ€” Phase 1 (git + GitHub, concurrent) โ†’ Phase 2 +(pr-origins) โ†’ Phase 3 (analyze, the LLM long pole, last and alone) โ€” plus +the numbered phase headers and the closing results panel. The crawlers are +stubbed with recording stand-ins so no git / GitHub / LLM / CodeGraph work +actually runs; only the orchestrator's ordering and rendering is exercised. +""" + +from __future__ import annotations + +import io +import subprocess +from pathlib import Path +from typing import Iterator + +import pytest +from click.testing import CliRunner +from rich.console import Console + +from whygraph import core +from whygraph.cli import main as whygraph_main +from whygraph.cli.commands import scan as scan_mod +from whygraph.core.config import Config +from whygraph.db import ensure_initialized +from whygraph.db import engine as db_engine + + +def _git(cwd: Path, *args: str) -> None: + subprocess.run(["git", "-C", str(cwd), *args], check=True, capture_output=True) + + +def _make_repo(root: Path) -> Path: + _git(root, "init", "-q", "-b", "main") + _git(root, "config", "user.email", "test@example.com") + _git(root, "config", "user.name", "Test User") + _git(root, "config", "commit.gpgsign", "false") + (root / "a.txt").write_text("hello\n") + _git(root, "add", "a.txt") + _git(root, "commit", "-q", "-m", "first") + return root + + +@pytest.fixture(autouse=True) +def _no_logging_side_effects(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("whygraph.cli.configure_logging", lambda *a, **kw: None) + + +@pytest.fixture +def repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + root = _make_repo(tmp_path) + monkeypatch.chdir(root) + return root + + +@pytest.fixture +def isolated_db(repo: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: + db_path = repo / ".whygraph" / "whygraph.db" + monkeypatch.setattr(core, "_config", Config(whygraph_db=db_path)) + db_engine._reset_engine() + ensure_initialized() + try: + yield db_path + finally: + db_engine._reset_engine() + core._reset_config() + + +class _DummyClient: + """A resolved GitHub client so Phase 1's GitHub + Phase 2 pr-origins run.""" + + owner = "acme" + name = "widgets" + pull_requests: tuple = () + issues: tuple = () + + +class _DummyDescriptor: + """Non-None descriptor so the LLM phase runs (probe not exercised).""" + + @classmethod + def from_config(cls, _cfg: object) -> "_DummyDescriptor": + return cls() + + +def _stub(name: str, order: list[tuple[str, str]]) -> type: + """A recording crawler class bound to ``name``, logging start/join order.""" + + class _Stub: + constructed = 0 + + def __init__(self, _progress: object, **_kwargs: object) -> None: + type(self).constructed += 1 + self.name = name + self.error = None + self.warning = None + self.summary = f"{name} ok" + + def start(self) -> None: + order.append(("start", name)) + + def join(self, timeout: float | None = None) -> None: + order.append(("join", name)) + + return _Stub + + +def _patch_crawlers( + monkeypatch: pytest.MonkeyPatch, order: list[tuple[str, str]] +) -> dict[str, type]: + """Replace every crawler class the CLI touches with a recording stub.""" + stubs = { + n: _stub(n, order) + for n in ("git", "github", "pr-origins", "analyze", "codegraph") + } + monkeypatch.setattr(scan_mod, "GitCrawler", stubs["git"]) + monkeypatch.setattr(scan_mod, "GitHubCrawler", stubs["github"]) + monkeypatch.setattr(scan_mod, "PROriginEnricher", stubs["pr-origins"]) + monkeypatch.setattr(scan_mod, "CodeGraphCrawler", stubs["codegraph"]) + # AnalyzeCrawler is imported lazily inside scan_cmd โ€” patch its source. + monkeypatch.setattr("whygraph.scan.AnalyzeCrawler", stubs["analyze"]) + return stubs + + +def _idx(order: list[tuple[str, str]], event: tuple[str, str]) -> int: + return order.index(event) + + +def test_three_phases_run_in_order_with_llm_last( + isolated_db: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + order: list[tuple[str, str]] = [] + stubs = _patch_crawlers(monkeypatch, order) + monkeypatch.setattr( + scan_mod, "_select_github_client", lambda *a, **k: _DummyClient() + ) + monkeypatch.setattr("whygraph.analyze.LlmDescriptor", _DummyDescriptor) + + result = CliRunner().invoke(whygraph_main, ["scan"]) + + assert result.exit_code == 0, result.output + # All five crawlers constructed exactly once. + for name in ("git", "github", "pr-origins", "analyze", "codegraph"): + assert stubs[name].constructed == 1, name + + # Numbered headers for all three phases. + assert "Phase 1/3 ยท Structural crawl" in result.output + assert "Phase 2/3 ยท PR-origin recovery" in result.output + assert "Phase 3/3 ยท LLM descriptions" in result.output + + # CodeGraph is a background task: started first, joined last. + assert order[0] == ("start", "codegraph") + assert order[-1] == ("join", "codegraph") + + # Phase 1 (git + github) both start before Phase 2 (pr-origins). + assert _idx(order, ("start", "git")) < _idx(order, ("start", "pr-origins")) + assert _idx(order, ("start", "github")) < _idx(order, ("start", "pr-origins")) + # Phase 1 both joined before Phase 2 starts. + assert _idx(order, ("join", "git")) < _idx(order, ("start", "pr-origins")) + assert _idx(order, ("join", "github")) < _idx(order, ("start", "pr-origins")) + # LLM is strictly last and alone: analyze starts only after pr-origins joins. + assert _idx(order, ("join", "pr-origins")) < _idx(order, ("start", "analyze")) + + # Closing results panel is present. + assert "done in" in result.output + + +def test_no_llm_descriptions_drops_the_llm_phase( + isolated_db: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + order: list[tuple[str, str]] = [] + stubs = _patch_crawlers(monkeypatch, order) + monkeypatch.setattr( + scan_mod, "_select_github_client", lambda *a, **k: _DummyClient() + ) + + result = CliRunner().invoke(whygraph_main, ["scan", "--no-llm-descriptions"]) + + assert result.exit_code == 0, result.output + # Two phases: structural + pr-origin recovery; no LLM phase. + assert "Phase 1/2 ยท Structural crawl" in result.output + assert "Phase 2/2 ยท PR-origin recovery" in result.output + assert "ยท LLM descriptions" not in result.output + assert stubs["analyze"].constructed == 0 + + +def test_single_phase_when_remote_and_llm_disabled( + isolated_db: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + order: list[tuple[str, str]] = [] + stubs = _patch_crawlers(monkeypatch, order) + + result = CliRunner().invoke( + whygraph_main, ["scan", "--no-remote", "--no-llm-descriptions"] + ) + + assert result.exit_code == 0, result.output + assert "Phase 1/1 ยท Structural crawl" in result.output + assert "Phase 2" not in result.output + assert stubs["github"].constructed == 0 + assert stubs["pr-origins"].constructed == 0 + assert stubs["analyze"].constructed == 0 + + +class _Fake: + """A minimal crawler stand-in for the pure results-panel unit test.""" + + def __init__( + self, + name: str, + *, + error: BaseException | None = None, + warning: str | None = None, + summary: str | None = None, + ) -> None: + self.name = name + self.error = error + self.warning = warning + self.summary = summary + + +def test_results_panel_is_defensive_and_total( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """It renders failed / warned / skipped rows without raising (R10/R11).""" + buf = io.StringIO() + monkeypatch.setattr(scan_mod, "console", Console(file=buf, width=120)) + + ran = [ + _Fake("git", summary="5 commits (5 new)"), + _Fake("github", summary="2 PRs ยท 1 issues"), + _Fake("analyze", error=RuntimeError("boom")), # failed โ†’ โœ— + # pr-origins absent from `ran` โ†’ "โ€” skipped" row + ] + codegraph = _Fake("codegraph", warning="CodeGraph refresh skipped โ€” no binary") + + scan_mod._render_results_panel( + ran=ran, + codegraph_crawler=codegraph, + db_path=Path("/repo/.whygraph/whygraph.db"), + scan_log_path=Path("/repo/.whygraph/scan.log"), + phase_timings={"Structural crawl": 4.1, "LLM descriptions": 128.0}, + total_elapsed=140.0, + ) + + out = buf.getvalue() + assert "โœ—" in out # failed analyze + assert "โš " in out # codegraph warning + assert "skipped" in out # absent pr-origins + assert "Scan log" in out # R11: path row retained + assert "done in" in out # total elapsed in the title diff --git a/tests/test_git_crawler.py b/tests/test_git_crawler.py index 638c751..24cb602 100644 --- a/tests/test_git_crawler.py +++ b/tests/test_git_crawler.py @@ -77,6 +77,19 @@ def _count_commits() -> int: return session.exec(select(func.count(CommitRow.sha))).one() +def test_summary_reports_commit_counts(repo_root: Path) -> None: + repo = Repository(repo_root) + + crawler = GitCrawler(Progress(), repository=repo) + crawler.run() + assert crawler.summary == "3 commits (3 new)" + + # A rescan sees the same commits, none new. + rescan = GitCrawler(Progress(), repository=repo) + rescan.run() + assert rescan.summary == "3 commits (0 new)" + + def test_first_scan_persists_all_commits(repo_root: Path) -> None: repo = Repository(repo_root) expected_shas = {c.sha for c in repo.commits} diff --git a/tests/test_scan_analyze_crawler.py b/tests/test_scan_analyze_crawler.py index c6ba48a..f465987 100644 --- a/tests/test_scan_analyze_crawler.py +++ b/tests/test_scan_analyze_crawler.py @@ -226,6 +226,84 @@ def test_skips_commit_with_empty_diff(isolated_db: Path, repo_path: Path) -> Non assert len(described) == 3 # the three real commits +def test_summary_reports_described_count(isolated_db: Path, repo_path: Path) -> None: + commits = _commits(repo_path) + _insert(commits) + + crawler = _run(repo_path, _StubDescriptor()) + + assert crawler.error is None + assert crawler.summary == "3 described" + + +def test_summary_reports_bulk_stubbed(isolated_db: Path, repo_path: Path) -> None: + commits = _commits(repo_path) + _insert(commits) + + # large_commit_file_count=0 makes every commit "bulk" โ€” none described. + crawler = _run(repo_path, _StubDescriptor(), large_commit_file_count=0) + + assert crawler.error is None + assert crawler.summary == "0 described ยท 3 bulk-stubbed" + + +def test_summary_up_to_date_when_nothing_pending( + isolated_db: Path, repo_path: Path +) -> None: + commits = _commits(repo_path) + _insert(commits, described=tuple(c.sha for c in commits)) + + crawler = _run(repo_path, _StubDescriptor()) + + assert crawler.error is None + assert crawler.summary == "up to date" + + +def test_origin_commits_stay_lazy(isolated_db: Path, repo_path: Path) -> None: + """A recovered on_default_branch=0 commit is never described. + + ``AnalyzeCrawler`` bounds its work to ``repository.commits`` (the main + walk), so PR-origin rows โ€” off the default branch โ€” stay NULL for the + lazy on-read backfill, regardless of when the analyze phase runs. Pins + the ยง2 invariant that makes sequencing analyze last behavior-preserving. + """ + commits = _commits(repo_path) + _insert(commits) # main-walk commits, all NULL descriptions + + origin_sha = "0" * 40 # not on `main`, so not in repository.commits + with get_session() as session: + session.add( + CommitRow( + sha=origin_sha, + parent_shas="", + author_name="Origin Author", + author_email="origin@example.com", + authored_at="2026-01-01T00:00:00+00:00", + committed_at="2026-01-01T00:00:00+00:00", + subject="recovered origin commit", + body="", + files_changed=0, + insertions=0, + deletions=0, + scanned_at="2026-01-01T00:00:00+00:00", + llm_description=None, + on_default_branch=0, + ) + ) + db_engine._reset_engine() + + crawler = _run(repo_path, _StubDescriptor()) + + assert crawler.error is None + descs = _descriptions() + # Origin commit untouched โ€” stays lazy. + assert descs[origin_sha] == (None, None) + # Every main-walk commit still described. + for c in commits: + assert descs[c.sha][0] == "DESCRIPTION" + assert crawler.summary == "3 described" + + def test_one_failing_commit_does_not_block_the_rest( isolated_db: Path, repo_path: Path ) -> None: