docs_crawler.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Generic documentation crawler.

What it does:
1) Crawl docs pages from a configurable docs site and save markdown files.
2) Build a local HTML viewer with left navigation + right markdown panel.

No product-specific rewriting or brand-coupled behavior.
"""

from __future__ import annotations

import argparse
import datetime as dt
import html
import http.server
import json
import os
import re
import shutil
import socket
import socketserver
import sys
import threading
import time
import traceback
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Dict, List, Optional, Sequence, Tuple

DEFAULT_CONFIG_PATH = "crawler_config.json"


# -----------------------------
# basic io helpers
# -----------------------------
def now_iso() -> str:
    return dt.datetime.now().astimezone().isoformat()


def now_hms() -> str:
    return dt.datetime.now().strftime("%H:%M:%S")


def log(message: str) -> None:
    text = f"[{now_hms()}] {message}"
    try:
        print(text, flush=True)
    except UnicodeEncodeError:
        enc = getattr(sys.stdout, "encoding", None) or "utf-8"
        safe = text.encode(enc, errors="replace").decode(enc, errors="replace")
        print(safe, flush=True)


def ensure_dir(path: Path) -> None:
    path.mkdir(parents=True, exist_ok=True)


def load_json(path: Path, default):
    if not path.exists():
        return default
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except Exception:
        return default


def save_json(path: Path, data) -> None:
    ensure_dir(path.parent)
    path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")


def write_text(path: Path, text: str) -> None:
    ensure_dir(path.parent)
    path.write_text(text, encoding="utf-8")


def copy_file(src: Path, dst: Path) -> bool:
    if not src.exists():
        return False
    ensure_dir(dst.parent)
    shutil.copy2(src, dst)
    return True


class ProgressHeartbeat:
    """
    Periodically prints progress heartbeat so users can see the program is alive.
    """

    def __init__(self, name: str, interval_seconds: int = 8) -> None:
        self.name = name
        self.interval_seconds = max(3, int(interval_seconds))
        self._lock = threading.Lock()
        self._stop = threading.Event()
        self._thread: Optional[threading.Thread] = None
        self._state = {
            "phase": "初始化",
            "step": "-",
            "url": "-",
            "done": 0,
            "total": 0,
            "last_touch": time.time(),
            "started_at": time.time(),
        }

    def start(self) -> "ProgressHeartbeat":
        self._thread = threading.Thread(target=self._run, name=f"heartbeat-{self.name}", daemon=True)
        self._thread.start()
        return self

    def stop(self) -> None:
        self._stop.set()
        if self._thread and self._thread.is_alive():
            self._thread.join(timeout=1.5)

    def update(
        self,
        *,
        phase: Optional[str] = None,
        step: Optional[str] = None,
        url: Optional[str] = None,
        done: Optional[int] = None,
        total: Optional[int] = None,
    ) -> None:
        with self._lock:
            if phase is not None:
                self._state["phase"] = phase
            if step is not None:
                self._state["step"] = step
            if url is not None:
                self._state["url"] = url
            if done is not None:
                self._state["done"] = int(done)
            if total is not None:
                self._state["total"] = int(total)
            self._state["last_touch"] = time.time()

    def _run(self) -> None:
        while not self._stop.wait(self.interval_seconds):
            with self._lock:
                phase = self._state["phase"]
                step = self._state["step"]
                url = self._state["url"]
                done = int(self._state["done"])
                total = int(self._state["total"])
                idle = int(time.time() - float(self._state["last_touch"]))
                started_at = float(self._state["started_at"])
            percent = 0.0 if total <= 0 else (done / total) * 100
            elapsed = max(0.001, time.time() - started_at)
            speed = done / elapsed  # docs/sec
            speed_min = speed * 60
            remain = max(0, total - done)
            eta_seconds = int(remain / speed) if speed > 0 else -1
            eta_text = "-" if eta_seconds < 0 else f"{eta_seconds}s"
            log(
                f"[保活] 阶段={phase} 进度={done}/{total} ({percent:.1f}%) "
                f"速度={speed_min:.2f} docs/min ETA={eta_text} "
                f"当前={step} 空闲={idle}s URL={url}"
            )


def progress_bar(done: int, total: int, width: int = 28) -> str:
    if total <= 0:
        return "[----------------------------] 0.0%"
    ratio = max(0.0, min(1.0, done / total))
    filled = int(ratio * width)
    bar = "█" * filled + "-" * (width - filled)
    return f"[{bar}] {ratio * 100:5.1f}%"


def log_progress_line(prefix: str, done: int, total: int, started_at: float) -> None:
    elapsed = max(0.001, time.time() - started_at)
    speed = done / elapsed
    speed_min = speed * 60
    remain = max(0, total - done)
    eta_seconds = int(remain / speed) if speed > 0 else -1
    eta_text = "-" if eta_seconds < 0 else f"{eta_seconds}s"
    bar = progress_bar(done, total)
    log(f"{prefix} {bar} {done}/{total} 速度={speed_min:.2f} docs/min ETA={eta_text}")


def sanitize_segment(text: str, fallback: str = "uncategorized") -> str:
    name = (text or "").strip()
    if not name:
        return fallback
    name = re.sub(r'[\\/:*?"<>|]+', "_", name)
    name = re.sub(r"\s+", "", name)
    return name.strip("._") or fallback


def slug_to_source_filename(slug: str) -> str:
    safe = re.sub(r"[^a-zA-Z0-9_\-/\.]+", "-", slug.strip("/"))
    safe = safe.replace("/", "__")
    if not safe:
        safe = "index"
    return f"{safe}.source.md"


def build_doc_source_url(base_url: str, docs_root: str, slug: str) -> str:
    root = "/" + (docs_root or "/docs").strip("/")
    s = (slug or "").strip("/")
    if s in {"", "index"}:
        return urllib.parse.urljoin(base_url, root.rstrip("/") + "/")
    return urllib.parse.urljoin(base_url, f"{root.rstrip('/')}/{s}")


# -----------------------------
# http / parse helpers
# -----------------------------
def build_doc_source_url_with_meta(base_url: str, docs_root: str, slug: str, meta: Optional[dict] = None) -> str:
    m = meta or {}
    source_path = (m.get("source_path") or "").strip()
    if source_path:
        return urllib.parse.urljoin(base_url, source_path)
    source_root = (m.get("source_root") or docs_root or "/docs").strip() or "/docs"
    return build_doc_source_url(base_url, source_root, slug)


def make_source_path(docs_root: str, rel_slug: str) -> str:
    root = "/" + (docs_root or "/docs").strip("/")
    rel = (rel_slug or "").strip("/")
    if rel in {"", "index"}:
        return root.rstrip("/") + "/"
    return f"{root.rstrip('/')}/{rel}"


def make_compound_slug(docs_root: str, rel_slug: str, multi_root: bool) -> str:
    rel = (rel_slug or "").strip("/") or "index"
    if not multi_root:
        return rel
    root_tag = (docs_root or "/docs").strip("/") or "docs"
    return f"{root_tag}/{rel}"


def make_route_candidates(rel_slug: str, slug: str) -> List[str]:
    candidates: List[str] = []
    for raw in [rel_slug or "", slug or ""]:
        s = raw.strip("/").lower()
        if not s:
            continue
        for x in [s, s.rstrip("/"), s.removesuffix(".html")]:
            if x and x not in candidates:
                candidates.append(x)
    if "index" not in candidates:
        candidates.append("index")
    return candidates


def http_get_text(
    url: str,
    timeout: int = 30,
    retries: int = 3,
    backoff_seconds: float = 1.5,
) -> str:
    req = urllib.request.Request(url, headers={"User-Agent": "DocsCrawler/1.0"})
    retries = max(1, int(retries))
    last_error: Optional[Exception] = None
    for attempt in range(1, retries + 1):
        t0 = time.time()
        log(f"[HTTP] 请求开始 attempt={attempt}/{retries} timeout={timeout}s url={url}")
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                raw = resp.read()
                elapsed = time.time() - t0
                log(f"[HTTP] 请求成功 status={getattr(resp, 'status', '?')} bytes={len(raw)} 耗时={elapsed:.2f}s url={url}")
                return raw.decode("utf-8", errors="replace")
        except (urllib.error.URLError, TimeoutError, socket.timeout) as e:
            last_error = e
            if attempt >= retries:
                break
            sleep_seconds = float(backoff_seconds) * attempt
            elapsed = time.time() - t0
            log(f"[HTTP][WARN] 请求失败 attempt={attempt}/{retries} 耗时={elapsed:.2f}s url={url} err={e}")
            log(f"[HTTP][WARN] 等待重试 sleep={sleep_seconds:.1f}s")
            time.sleep(sleep_seconds)
        except Exception:
            raise
    raise RuntimeError(f"request failed after {retries} retries: {url} | {last_error}")


def infer_category_number(text: str) -> Optional[int]:
    m = re.match(r"^\s*(\d+)(?:\.\d+)?", text or "")
    if not m:
        return None
    try:
        return int(m.group(1))
    except Exception:
        return None


def pick_attr_value(m: Optional[re.Match], start_idx: int = 1) -> str:
    if not m:
        return ""
    end = (m.lastindex or 0) + 1
    for i in range(start_idx, end):
        val = m.group(i)
        if val:
            return val
    return ""


def strip_tags(text: str) -> str:
    return html.unescape(re.sub(r"<[^>]+>", "", text or "")).strip()


def normalize_docs_href(href: str, docs_root: str = "/docs") -> str:
    docs_root = "/" + docs_root.strip("/")
    p = urllib.parse.urlparse(href or "")
    path = p.path or ""
    if not path.startswith(docs_root + "/"):
        return ""
    return path[len(docs_root) + 1 :].strip("/")


def _looks_like_docs_html(page_html: str, docs_root: str) -> bool:
    text = page_html or ""
    if not text:
        return False
    markers = [
        "theme-doc-sidebar-item",
        "VPSidebar",
        "VPDoc",
        "vitepress",
        "docusaurus",
    ]
    if any(m.lower() in text.lower() for m in markers):
        return True
    docs_root = "/" + (docs_root or "/docs").strip("/")
    if re.search(rf'href=(?:"|\')({re.escape(docs_root)}/)', text, re.I):
        return True
    return False


def _candidate_docs_roots_from_seed_html(seed_html: str, configured_docs_root: str) -> List[str]:
    candidates: List[str] = []
    seen: set[str] = set()

    def _add(path: str) -> None:
        p = "/" + (path or "").strip("/")
        if not p or p == "//":
            return
        if p in seen:
            return
        seen.add(p)
        candidates.append(p)

    _add(configured_docs_root or "/docs")
    for p in ["/docs", "/guide", "/manual", "/reference", "/api", "/help"]:
        _add(p)

    for m in re.finditer(r'href=(?:"([^"]+)"|\'([^\']+)\'|([^\s>]+))', seed_html or "", re.I):
        href = (m.group(1) or m.group(2) or m.group(3) or "").strip()
        if not href:
            continue
        pu = urllib.parse.urlparse(href)
        path = (pu.path or "").strip()
        if not path.startswith("/"):
            continue
        if path.startswith("/assets/") or path.startswith("/@") or path.startswith("/_"):
            continue
        segs = [x for x in path.split("/") if x]
        if not segs:
            continue
        if "." in segs[0]:
            continue
        first = "/" + segs[0]
        _add(first)

    return candidates




def _candidate_docs_root_samples_from_seed_html(seed_html: str) -> Dict[str, List[str]]:
    """
    Build sample doc paths by first path segment, for cases where root index
    (e.g. /components/) is 404 but leaf docs (e.g. /components/xxx.html) exist.
    """
    out: Dict[str, List[str]] = {}
    for m in re.finditer(r'href=(?:"([^"]+)"|\'([^\']+)\'|([^\s>]+))', seed_html or "", re.I):
        href = (m.group(1) or m.group(2) or m.group(3) or "").strip()
        if not href:
            continue
        pu = urllib.parse.urlparse(href)
        path = (pu.path or "").strip()
        if not path.startswith("/"):
            continue
        if path.startswith("/assets/") or path.startswith("/@") or path.startswith("/_"):
            continue
        segs = [x for x in path.split("/") if x]
        if not segs:
            continue
        if "." in segs[0]:
            continue
        root = "/" + segs[0]
        arr = out.setdefault(root, [])
        if path not in arr:
            arr.append(path)
    return out
def resolve_docs_root(
    *,
    base_url: str,
    configured_docs_root: str,
    seed_html: str,
    timeout: int,
    retries: int,
    backoff: float,
) -> Tuple[str, str]:
    """
    Resolve a valid docs_root automatically.
    Returns: (resolved_docs_root, docs_home_html)
    """
    last_error: Optional[Exception] = None
    candidates = _candidate_docs_roots_from_seed_html(seed_html, configured_docs_root)
    sample_map = _candidate_docs_root_samples_from_seed_html(seed_html)
    probe_retries = 1 if retries > 1 else retries
    probe_backoff = 0.0 if retries > 1 else backoff

    for root in candidates:
        url = urllib.parse.urljoin(base_url, root if root.endswith("/") else root + "/")
        try:
            html_text = http_get_text(
                url,
                timeout=timeout,
                retries=probe_retries,
                backoff_seconds=probe_backoff,
            )
            if _looks_like_docs_html(html_text, root):
                if root != ("/" + configured_docs_root.strip("/")):
                    log(f"[!] 自动修正 docs_root: {configured_docs_root} -> {root}")
                return root, html_text
        except Exception as e:
            last_error = e

        for sample_path in (sample_map.get(root) or [])[:5]:
            try:
                sample_url = urllib.parse.urljoin(base_url, sample_path)
                sample_html = http_get_text(
                    sample_url,
                    timeout=timeout,
                    retries=probe_retries,
                    backoff_seconds=probe_backoff,
                )
                if _looks_like_docs_html(sample_html, root):
                    if root != ("/" + configured_docs_root.strip("/")):
                        log(f"[!] 自动修正 docs_root: {configured_docs_root} -> {root}")
                    return root, sample_html
            except Exception as e:
                last_error = e
                continue

    cfg = "/" + configured_docs_root.strip("/")
    if last_error:
        log(f"[!] docs_root 自动探测失败，保留配置值：{cfg} ({last_error})")
    return cfg, ""
def resolve_docs_roots(
    *,
    base_url: str,
    configured_docs_root: str,
    seed_html: str,
    timeout: int,
    retries: int,
    backoff: float,
) -> Tuple[List[str], Dict[str, str]]:
    """
    Resolve docs roots.
    - If configured_docs_root is provided, keep single-root behavior.
    - If configured_docs_root is empty, auto detect all valid roots from seed links.
    Returns: (roots, root_home_html_map)
    """
    configured = (configured_docs_root or "").strip()
    if configured:
        root, html_text = resolve_docs_root(
            base_url=base_url,
            configured_docs_root=configured,
            seed_html=seed_html,
            timeout=timeout,
            retries=retries,
            backoff=backoff,
        )
        return [root], {root: html_text or ""}

    candidates = _candidate_docs_roots_from_seed_html(seed_html, "/docs")
    sample_map = _candidate_docs_root_samples_from_seed_html(seed_html)
    probe_retries = 1 if retries > 1 else retries
    probe_backoff = 0.0 if retries > 1 else backoff
    roots: List[str] = []
    root_html_map: Dict[str, str] = {}

    for root in candidates:
        url = urllib.parse.urljoin(base_url, root if root.endswith("/") else root + "/")
        resolved = False
        try:
            html_text = http_get_text(
                url,
                timeout=timeout,
                retries=probe_retries,
                backoff_seconds=probe_backoff,
            )
            if _looks_like_docs_html(html_text, root):
                roots.append(root)
                root_html_map[root] = html_text
                resolved = True
        except Exception:
            pass

        if resolved:
            continue

        for sample_path in (sample_map.get(root) or [])[:5]:
            try:
                sample_url = urllib.parse.urljoin(base_url, sample_path)
                sample_html = http_get_text(
                    sample_url,
                    timeout=timeout,
                    retries=probe_retries,
                    backoff_seconds=probe_backoff,
                )
                if not _looks_like_docs_html(sample_html, root):
                    continue
                roots.append(root)
                root_html_map[root] = sample_html
                resolved = True
                break
            except Exception:
                continue

    if not roots:
        fallback_root, fallback_html = resolve_docs_root(
            base_url=base_url,
            configured_docs_root="/docs",
            seed_html=seed_html,
            timeout=timeout,
            retries=retries,
            backoff=backoff,
        )
        roots = [fallback_root]
        root_html_map[fallback_root] = fallback_html or ""

    log(f"[*] auto docs roots: {', '.join(roots)}")
    return roots, root_html_map
def _extract_sidebar_structure_vitepress(seed_html: str, docs_root: str) -> Dict[str, dict]:
    docs_root = "/" + docs_root.strip("/")
    categories: Dict[str, dict] = {}
    slug_to_category: Dict[str, str] = {}
    slug_titles: Dict[str, str] = {}
    ordered_keys: List[str] = []

    aside_match = re.search(r"<aside[^>]*class=\"[^\"]*VPSidebar[^\"]*\"[^>]*>(.*?)</aside>", seed_html or "", re.I | re.S)
    container = aside_match.group(1) if aside_match else (seed_html or "")

    level0_re = re.compile(
        r'<section[^>]*class="[^"]*VPSidebarItem[^"]*level-0[^"]*"[^>]*>(.*?)</section>',
        re.I | re.S,
    )
    level0_blocks = level0_re.findall(container) or [container]

    for idx, block in enumerate(level0_blocks):
        title_match = re.search(r"<h2[^>]*class=\"[^\"]*text[^\"]*\"[^>]*>(.*?)</h2>", block, re.I | re.S)
        cat_label = strip_tags(title_match.group(1)) if title_match else ""
        if not cat_label:
            cat_label = f"Section {idx + 1}"
        cat_key = f"cat:{sanitize_segment(cat_label, fallback=f'section{idx + 1}')}"
        if cat_key not in categories:
            categories[cat_key] = {
                "key": cat_key,
                "label": cat_label,
                "order": idx,
                "number": None,
                "folder": sanitize_segment(cat_label),
                "kind": "group",
            }
            ordered_keys.append(cat_key)

        for m in re.finditer(
            r'<a[^>]*class="[^"]*VPLink[^"]*"[^>]*href=(?:"([^"]+)"|\'([^\']+)\'|([^\s>]+))[^>]*>(.*?)</a>',
            block,
            re.I | re.S,
        ):
            href = (m.group(1) or m.group(2) or m.group(3) or "").strip()
            if not href:
                continue
            p = urllib.parse.urlparse(href)
            path = (p.path or "").rstrip("/")
            if path == docs_root.rstrip("/"):
                rel = "index"
            else:
                rel = normalize_docs_href(href, docs_root=docs_root)
            if not rel:
                continue
            title = strip_tags(m.group(4) or "") or rel
            slug_to_category.setdefault(rel, cat_key)
            if title:
                slug_titles.setdefault(rel, title)

    return {
        "categories": categories,
        "slug_to_category": slug_to_category,
        "slug_titles": slug_titles,
        "ordered_keys": ordered_keys,
    }


def extract_sidebar_structure(seed_html: str, docs_root: str = "/docs") -> Dict[str, dict]:
    """
    Extract level-1 sidebar entries from docusaurus HTML.
    - category items (/docs/category/*)
    - direct doc links (/docs/*)
    """
    docs_root = "/" + docs_root.strip("/")
    item_re = re.compile(
        r'<li class="(?P<class>[^"]*theme-doc-sidebar-item-(?:category|link)[^"]*level-1[^"]*)"[^>]*>'
        r"(?P<body>.*?)"
        r'(?=(?:<li class="[^"]*theme-doc-sidebar-item-(?:category|link)[^"]*level-1[^"]*"|</ul>\s*</nav>|</ul>\s*</div>\s*</nav>|$))',
        re.S,
    )

    categories: Dict[str, dict] = {}
    slug_to_category: Dict[str, str] = {}
    slug_titles: Dict[str, str] = {}
    ordered_keys: List[str] = []

    for idx, m in enumerate(item_re.finditer(seed_html or "")):
        cls = m.group("class") or ""
        body = m.group("body") or ""

        a_match = re.search(
            r"<a[^>]*href=(?:\"([^\"]+)\"|'([^']+)'|([^\s>]+))[^>]*>(?P<a_body>.*?)</a>",
            body,
            re.S,
        )
        href = pick_attr_value(a_match, 1).strip()
        if not href:
            continue

        a_body = a_match.group("a_body") if a_match else body
        title_match = re.search(r"<span[^>]*title=(?:\"([^\"]+)\"|'([^']+)'|([^\s>]+))", a_body, re.S)
        title = html.unescape(pick_attr_value(title_match, 1)).strip() or strip_tags(a_body)
        if not title:
            continue

        rel = normalize_docs_href(href, docs_root=docs_root)
        if not rel:
            continue

        number = infer_category_number(title)
        order = number if number is not None else (1000 + idx)

        if "theme-doc-sidebar-item-category-level-1" in cls and rel.startswith("category/"):
            key = f"cat:{rel[len('category/'):]}"
            categories[key] = {
                "key": key,
                "label": title,
                "order": order,
                "number": number,
                "folder": sanitize_segment(title),
                "kind": "category",
            }
            ordered_keys.append(key)
            continue

        if "theme-doc-sidebar-item-link-level-1" in cls and not rel.startswith("category/"):
            slug = rel
            key = f"doc:{slug}"
            categories[key] = {
                "key": key,
                "label": title,
                "order": order,
                "number": number,
                "folder": sanitize_segment(title),
                "kind": "doc",
            }
            ordered_keys.append(key)
            slug_to_category[slug] = key
            slug_titles[slug] = title

    # unique keep order
    seen = set()
    ordered: List[str] = []
    for k in ordered_keys:
        if k in seen:
            continue
        seen.add(k)
        ordered.append(k)

    result = {
        "categories": categories,
        "slug_to_category": slug_to_category,
        "slug_titles": slug_titles,
        "ordered_keys": ordered,
    }
    if categories or slug_to_category:
        return result

    # fallback: VitePress sidebar
    vitepress = _extract_sidebar_structure_vitepress(seed_html, docs_root=docs_root)
    if (vitepress.get("categories") or {}) or (vitepress.get("slug_to_category") or {}):
        return vitepress
    return result


def parse_doc_entries_from_category_page(page_html: str, docs_root: str = "/docs") -> List[Tuple[str, str]]:
    """
    Parse second-level sidebar entries from category page.

    Important: do NOT parse all <a> links in page body, otherwise article internal links
    will pollute navigation and make level-2 menu differ from source site.
    """
    out: List[Tuple[str, str]] = []
    seen = set()
    docs_root_norm = "/" + docs_root.strip("/")

    # Prefer explicit sidebar level-2 link items.
    sidebar_re = re.compile(
        r'theme-doc-sidebar-item-link-level-2[^>]*>\s*<a[^>]*href=(?:"([^"]+)"|\'([^\']+)\'|([^\s>]+))[^>]*>'
        r'(?:\s*<span[^>]*title=(?:"([^"]+)"|\'([^\']+)\'|([^\s>]+))[^>]*>)?',
        re.S | re.I,
    )
    for m in sidebar_re.finditer(page_html or ""):
        href = (m.group(1) or m.group(2) or m.group(3) or "").strip()
        if not href:
            continue
        p = urllib.parse.urlparse(href)
        path = (p.path or "").rstrip("/")
        if path == docs_root_norm.rstrip("/"):
            rel = "index"
        else:
            rel = normalize_docs_href(href, docs_root=docs_root)
        if not rel or rel.startswith("category/"):
            continue
        label = html.unescape((m.group(4) or m.group(5) or m.group(6) or "").strip())
        if not label:
            label = rel
        key = (rel, label)
        if key in seen:
            continue
        seen.add(key)
        out.append((rel, label))

    if out:
        return out

    # Fallback for non-Docusaurus or unexpected structures.
    a_re = re.compile(
        r"<a[^>]*href=(?:\"([^\"]+)\"|'([^']+)'|([^\s>]+))[^>]*>(?P<body>.*?)</a>",
        re.S | re.I,
    )
    for m in a_re.finditer(page_html or ""):
        href = (m.group(1) or m.group(2) or m.group(3) or "").strip()
        if not href:
            continue
        p = urllib.parse.urlparse(href)
        path = (p.path or "").rstrip("/")
        if path == docs_root_norm.rstrip("/"):
            rel = "index"
        else:
            rel = normalize_docs_href(href, docs_root=docs_root)
        if not rel or rel.startswith("category/"):
            continue
        label = strip_tags(m.group("body") or "")
        if not label:
            continue
        key = (rel, label)
        if key in seen:
            continue
        seen.add(key)
        out.append((rel, label))
    return out


def parse_doc_slugs_from_sitemap(xml_text: str, docs_root: str = "/docs") -> List[str]:
    out: List[str] = []
    seen = set()
    docs_root = "/" + docs_root.strip("/")
    for url in re.findall(r"<loc>(.*?)</loc>", xml_text or "", re.I | re.S):
        u = url.strip()
        if not u:
            continue
        p = urllib.parse.urlparse(u)
        path = (p.path or "").rstrip("/")
        if not path.startswith(docs_root):
            continue
        rel = path[len(docs_root) :].strip("/")
        if not rel or rel.startswith("category/"):
            continue
        if rel in seen:
            continue
        seen.add(rel)
        out.append(rel)
    return out


def html_to_markdown(article_html: str) -> str:
    text = article_html or ""

    # remove script/style
    text = re.sub(r"<(script|style)\b[^>]*>.*?</\1>", "", text, flags=re.I | re.S)

    # code blocks first
    code_blocks: List[str] = []

    def _extract_code(m: re.Match) -> str:
        raw = m.group(1) or ""
        raw = re.sub(r"^\s*<code[^>]*>|</code>\s*$", "", raw, flags=re.I | re.S)
        raw = re.sub(r"<[^>]+>", "", raw)
        raw = html.unescape(raw)
        raw = raw.strip("\n")
        code_blocks.append(raw)
        return f"\n@@CODEBLOCK_{len(code_blocks)-1}@@\n"

    text = re.sub(r"<pre[^>]*>(.*?)</pre>", _extract_code, text, flags=re.I | re.S)

    # links
    text = re.sub(
        r"<a[^>]*href=(?:\"([^\"]+)\"|'([^']+)'|([^\s>]+))[^>]*>(.*?)</a>",
        lambda m: f"[{strip_tags(m.group(4) or '')}]({(m.group(1) or m.group(2) or m.group(3) or '').strip()})",
        text,
        flags=re.I | re.S,
    )

    # headings
    for n in range(6, 0, -1):
        text = re.sub(
            rf"<h{n}[^>]*>(.*?)</h{n}>",
            lambda m, lv=n: "\n" + ("#" * lv) + " " + strip_tags(m.group(1) or "") + "\n",
            text,
            flags=re.I | re.S,
        )

    # simple blocks
    text = re.sub(r"<blockquote[^>]*>(.*?)</blockquote>", lambda m: "\n> " + strip_tags(m.group(1) or "") + "\n", text, flags=re.I | re.S)
    text = re.sub(r"<li[^>]*>(.*?)</li>", lambda m: "\n- " + strip_tags(m.group(1) or ""), text, flags=re.I | re.S)
    text = re.sub(r"<(p|div|section|article|main)[^>]*>", "\n", text, flags=re.I)
    text = re.sub(r"</(p|div|section|article|main)>", "\n", text, flags=re.I)
    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)

    # inline emphasis
    text = re.sub(r"<(strong|b)[^>]*>(.*?)</\1>", lambda m: f"**{strip_tags(m.group(2) or '')}**", text, flags=re.I | re.S)
    text = re.sub(r"<(em|i)[^>]*>(.*?)</\1>", lambda m: f"*{strip_tags(m.group(2) or '')}*", text, flags=re.I | re.S)
    text = re.sub(r"<code[^>]*>(.*?)</code>", lambda m: f"`{strip_tags(m.group(1) or '')}`", text, flags=re.I | re.S)

    # strip leftovers
    text = strip_tags(text)

    # restore code blocks
    for idx, code in enumerate(code_blocks):
        marker = f"@@CODEBLOCK_{idx}@@"
        text = text.replace(marker, f"```\n{code}\n```")

    # cleanup
    lines = [ln.rstrip() for ln in text.replace("\r\n", "\n").split("\n")]
    out: List[str] = []
    blank = False
    for ln in lines:
        if not ln.strip():
            if not blank:
                out.append("")
                blank = True
            continue
        out.append(ln)
        blank = False
    return "\n".join(out).strip() + "\n"


def extract_doc_from_html(page_html: str, slug: str) -> Tuple[str, str]:
    article_match = re.search(r"<article\b[^>]*>(.*?)</article>", page_html or "", re.I | re.S)
    article_html = article_match.group(1) if article_match else ""

    title = ""
    if article_html:
        h1 = re.search(r"<h1\b[^>]*>(.*?)</h1>", article_html, re.I | re.S)
        if h1:
            title = strip_tags(h1.group(1))

    if not title:
        doc_title = re.search(r"<title\b[^>]*>(.*?)</title>", page_html or "", re.I | re.S)
        title = strip_tags(doc_title.group(1)) if doc_title else ""

    if not title:
        title = slug

    body_md = html_to_markdown(article_html or page_html or "")
    return title, body_md


def build_source_markdown(title: str, source_url: str, body_markdown: str) -> str:
    lines = [
        f"# {title}",
        "",
        f"> Source: `{source_url}`",
        "",
        body_markdown.strip(),
        "",
    ]
    return "\n".join(lines)


def parse_script_src_urls(page_html: str, base_url: str) -> List[str]:
    out: List[str] = []
    seen: set[str] = set()
    if not page_html:
        return out
    for m in re.finditer(r"<script\b[^>]*\bsrc\s*=\s*(\"([^\"]+)\"|'([^']+)'|([^\s>]+))", page_html, re.I | re.S):
        src = (m.group(2) or m.group(3) or m.group(4) or "").strip()
        if not src:
            continue
        full = urllib.parse.urljoin(base_url, src)
        if full in seen:
            continue
        seen.add(full)
        out.append(full)
    return out


def is_thin_body_markdown(body_markdown: str) -> bool:
    text = body_markdown or ""
    cleaned = re.sub(r"\s+", "", text)
    non_empty_lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
    heading_count = len(re.findall(r"^\s*#{1,6}\s+", text, re.M))
    list_count = len(re.findall(r"^\s*[-*]\s+", text, re.M))

    if len(cleaned) < 280:
        return True
    if len(non_empty_lines) <= 8 and any(token in text for token in ["本页总览", "官方广告"]):
        return True
    if heading_count <= 1 and list_count == 0 and len(cleaned) < 800:
        return True
    return False


def decode_js_string(raw: str) -> str:
    s = raw or ""
    try:
        return _normalize_unicode_brace_escapes(json.loads(f"\"{s}\""))
    except Exception:
        if "\\" not in s:
            return _normalize_unicode_brace_escapes(s)
        try:
            return _normalize_unicode_brace_escapes(bytes(s, "utf-8").decode("unicode_escape"))
        except Exception:
            return _normalize_unicode_brace_escapes(s)


def _normalize_unicode_brace_escapes(text: str) -> str:
    s = text or ""
    return re.sub(r"\\u\{([0-9a-fA-F]{1,6})\}", lambda m: chr(int(m.group(1), 16)), s)


def _scan_js_value(text: str, start: int) -> Tuple[str, int]:
    i = start
    depth_paren = depth_brace = depth_bracket = 0
    in_str = False
    quote = ""
    escaped = False

    while i < len(text):
        ch = text[i]
        if in_str:
            if escaped:
                escaped = False
            elif ch == "\\":
                escaped = True
            elif ch == quote:
                in_str = False
            i += 1
            continue

        if ch in ("'", '"'):
            in_str = True
            quote = ch
            i += 1
            continue

        if ch == "(":
            depth_paren += 1
        elif ch == ")":
            if depth_paren > 0:
                depth_paren -= 1
        elif ch == "{":
            depth_brace += 1
        elif ch == "}":
            if depth_brace > 0:
                depth_brace -= 1
        elif ch == "[":
            depth_bracket += 1
        elif ch == "]":
            if depth_bracket > 0:
                depth_bracket -= 1
        elif ch == "," and depth_paren == 0 and depth_brace == 0 and depth_bracket == 0:
            return text[start:i].strip(), i
        elif ch in "}" and depth_paren == 0 and depth_brace == 0 and depth_bracket == 0:
            return text[start:i].strip(), i
        i += 1
    return text[start:].strip(), len(text)


def _find_balanced(text: str, start: int, opener: str, closer: str) -> Tuple[str, int]:
    if start < 0 or start >= len(text) or text[start] != opener:
        return "", -1
    i = start + 1
    depth = 1
    in_str = False
    quote = ""
    escaped = False
    while i < len(text):
        ch = text[i]
        if in_str:
            if escaped:
                escaped = False
            elif ch == "\\":
                escaped = True
            elif ch == quote:
                in_str = False
            i += 1
            continue
        if ch in ("'", '"'):
            in_str = True
            quote = ch
            i += 1
            continue
        if ch == opener:
            depth += 1
        elif ch == closer:
            depth -= 1
            if depth == 0:
                return text[start : i + 1], i
        i += 1
    return "", -1


def _split_top_level_commas(text: str) -> List[str]:
    out: List[str] = []
    i = 0
    start = 0
    depth_paren = depth_brace = depth_bracket = 0
    in_str = False
    quote = ""
    escaped = False
    while i < len(text):
        ch = text[i]
        if in_str:
            if escaped:
                escaped = False
            elif ch == "\\":
                escaped = True
            elif ch == quote:
                in_str = False
            i += 1
            continue
        if ch in ("'", '"'):
            in_str = True
            quote = ch
            i += 1
            continue
        if ch == "(":
            depth_paren += 1
        elif ch == ")":
            depth_paren = max(0, depth_paren - 1)
        elif ch == "{":
            depth_brace += 1
        elif ch == "}":
            depth_brace = max(0, depth_brace - 1)
        elif ch == "[":
            depth_bracket += 1
        elif ch == "]":
            depth_bracket = max(0, depth_bracket - 1)
        elif ch == "," and depth_paren == 0 and depth_brace == 0 and depth_bracket == 0:
            part = text[start:i].strip()
            if part:
                out.append(part)
            start = i + 1
        i += 1
    tail = text[start:].strip()
    if tail:
        out.append(tail)
    return out


def _extract_prop_expr(obj_text: str, prop_name: str) -> str:
    marker = prop_name + ":"
    idx = obj_text.find(marker)
    if idx < 0:
        return ""
    start = idx + len(marker)
    val, _ = _scan_js_value(obj_text, start)
    return val.strip()


def _strip_js_quote(expr: str) -> str:
    s = (expr or "").strip()
    if len(s) >= 2 and ((s[0] == '"' and s[-1] == '"') or (s[0] == "'" and s[-1] == "'")):
        return decode_js_string(s[1:-1])
    return s


def _parse_js_template_literal(expr: str) -> str:
    s = (expr or "").strip()
    if not (len(s) >= 2 and s[0] == "`" and s[-1] == "`"):
        return s
    body = s[1:-1]
    body = body.replace("\\`", "`")
    body = body.replace("\\${", "${")
    body = body.replace("\\r\\n", "\n").replace("\\n", "\n").replace("\\r", "\n")
    try:
        decoded = bytes(body, "utf-8").decode("unicode_escape")
        return _normalize_unicode_brace_escapes(decoded)
    except Exception:
        return _normalize_unicode_brace_escapes(body)


def _parse_mdx_inline(expr: str) -> str:
    e = (expr or "").strip()
    if not e:
        return ""
    if e.startswith('"') and e.endswith('"'):
        return decode_js_string(e[1:-1])
    if e.startswith("'") and e.endswith("'"):
        return decode_js_string(e[1:-1])
    if e.startswith("`") and e.endswith("`"):
        return _parse_js_template_literal(e)
    if e.startswith("[") and e.endswith("]"):
        inner = e[1:-1].strip()
        parts = _split_top_level_commas(inner)
        return "".join(_parse_mdx_inline(p) for p in parts)
    comp_m = re.search(r"\(([A-Za-z_$][A-Za-z0-9_$]*)\.([A-Za-z0-9_]+)\s*,\{", e)
    if not comp_m:
        return ""
    comp = comp_m.group(2)
    obj_start = e.find("{", comp_m.end() - 1)
    obj_text, _ = _find_balanced(e, obj_start, "{", "}")
    if not obj_text:
        return ""
    body = obj_text[1:-1]
    child_expr = _extract_prop_expr(body, "children")
    child_text = _parse_mdx_inline(child_expr) if child_expr else ""

    if comp == "strong":
        return f"**{child_text}**"
    if comp == "del":
        return f"~~{child_text}~~"
    if comp == "code":
        return f"`{child_text}`"
    if comp == "a":
        href = _extract_prop_expr(body, "href").strip().strip('"').strip("'")
        if href:
            label = child_text or href
            return f"[{label}]({href})"
        return child_text
    if comp == "img":
        src = _extract_prop_expr(body, "src").strip().strip('"').strip("'")
        alt = _extract_prop_expr(body, "alt").strip().strip('"').strip("'")
        if src:
            return f"![{alt}]({src})"
        return alt
    return child_text


def _parse_mdx_code_expr(expr: str) -> Tuple[str, str]:
    e = (expr or "").strip()
    comp_m = re.search(r"\(([A-Za-z_$][A-Za-z0-9_$]*)\.code\s*,\{", e)
    if not comp_m:
        return "", ""
    obj_start = e.find("{", comp_m.end() - 1)
    obj_text, _ = _find_balanced(e, obj_start, "{", "}")
    if not obj_text:
        return "", ""
    body = obj_text[1:-1]
    class_expr = _extract_prop_expr(body, "className")
    child_expr = _extract_prop_expr(body, "children")
    lang = ""
    class_text = _strip_js_quote(class_expr)
    m_lang = re.search(r"language-([a-zA-Z0-9_+\-#]+)", class_text)
    if m_lang:
        lang = m_lang.group(1)

    code = _parse_mdx_inline(child_expr) if child_expr else ""
    code = (code or "").replace("\r\n", "\n").replace("\r", "\n").rstrip("\n")
    return lang, code


def _mdx_component_and_body(expr: str) -> Tuple[str, str]:
    e = (expr or "").strip()
    comp_m = re.search(r"\(([A-Za-z_$][A-Za-z0-9_$]*)\.([A-Za-z0-9_]+)\s*,\{", e)
    if not comp_m:
        return "", ""
    comp = comp_m.group(2)
    obj_start = e.find("{", comp_m.end() - 1)
    obj_text, _ = _find_balanced(e, obj_start, "{", "}")
    if not obj_text:
        return "", ""
    return comp, obj_text[1:-1]


def _parse_table_cells(expr: str) -> List[str]:
    e = (expr or "").strip()
    if not e:
        return []
    if e.startswith("[") and e.endswith("]"):
        out: List[str] = []
        inner = e[1:-1].strip()
        for part in _split_top_level_commas(inner):
            out.extend(_parse_table_cells(part))
        return out

    comp, body = _mdx_component_and_body(e)
    if not comp:
        txt = _parse_mdx_inline(e).strip()
        return [txt] if txt else []
    child_expr = _extract_prop_expr(body, "children")
    if comp in {"th", "td"}:
        return [_parse_mdx_inline(child_expr).strip()]
    if comp in {"tr", "thead", "tbody", "table"}:
        return _parse_table_cells(child_expr)
    return []


def _parse_table_rows(expr: str) -> List[List[str]]:
    e = (expr or "").strip()
    if not e:
        return []
    if e.startswith("[") and e.endswith("]"):
        rows: List[List[str]] = []
        inner = e[1:-1].strip()
        for part in _split_top_level_commas(inner):
            rows.extend(_parse_table_rows(part))
        return rows

    comp, body = _mdx_component_and_body(e)
    if not comp:
        return []
    child_expr = _extract_prop_expr(body, "children")
    if comp == "tr":
        cells = _parse_table_cells(child_expr)
        return [cells] if cells else []
    if comp in {"thead", "tbody", "table"}:
        return _parse_table_rows(child_expr)
    return []


def _parse_mdx_block_expr(expr: str, out_lines: List[str]) -> None:
    e = (expr or "").strip()
    if not e:
        return
    if e.startswith('"') and e.endswith('"'):
        text = decode_js_string(e[1:-1]).strip()
        if text:
            out_lines.append(text)
        return
    if e.startswith("'") and e.endswith("'"):
        text = decode_js_string(e[1:-1]).strip()
        if text:
            out_lines.append(text)
        return
    if e.startswith("[") and e.endswith("]"):
        inner = e[1:-1].strip()
        for part in _split_top_level_commas(inner):
            _parse_mdx_block_expr(part, out_lines)
        return

    comp_m = re.search(r"\(([A-Za-z_$][A-Za-z0-9_$]*)\.([A-Za-z0-9_]+)\s*,\{", e)
    if not comp_m:
        return
    comp = comp_m.group(2)
    obj_start = e.find("{", comp_m.end() - 1)
    obj_text, _ = _find_balanced(e, obj_start, "{", "}")
    if not obj_text:
        return
    body = obj_text[1:-1]
    child_expr = _extract_prop_expr(body, "children")
    title_expr = _extract_prop_expr(body, "title")
    title = _parse_mdx_inline(title_expr) if title_expr else ""

    if comp in {"h1", "h2", "h3", "h4", "h5", "h6"}:
        lv = int(comp[1])
        head = _parse_mdx_inline(child_expr)
        if head:
            out_lines.append(f"{'#' * lv} {head}")
            out_lines.append("")
        return
    if comp == "p":
        text = _parse_mdx_inline(child_expr)
        if text:
            out_lines.append(text)
            out_lines.append("")
        return
    if comp == "blockquote":
        text = _parse_mdx_inline(child_expr)
        if text:
            out_lines.append(f"> {text}")
            out_lines.append("")
        return
    if comp == "pre":
        lang, code = _parse_mdx_code_expr(child_expr)
        if code:
            out_lines.append(f"```{lang}".rstrip())
            out_lines.extend(code.split("\n"))
            out_lines.append("```")
            out_lines.append("")
        return
    if comp == "li":
        text = _parse_mdx_inline(child_expr)
        if text:
            out_lines.append(f"- {text}")
        return
    if comp in {"ul", "ol"}:
        _parse_mdx_block_expr(child_expr, out_lines)
        out_lines.append("")
        return
    if comp == "table":
        rows = _parse_table_rows(child_expr)
        rows = [r for r in rows if any((c or "").strip() for c in r)]
        if rows:
            col_count = max(len(r) for r in rows)
            normalized = [r + [""] * (col_count - len(r)) for r in rows]
            header = normalized[0]
            out_lines.append("| " + " | ".join((c or "").strip() for c in header) + " |")
            out_lines.append("| " + " | ".join(["---"] * col_count) + " |")
            for row in normalized[1:]:
                out_lines.append("| " + " | ".join((c or "").strip() for c in row) + " |")
            out_lines.append("")
        return
    if comp == "admonition":
        body_text = _parse_mdx_inline(child_expr)
        if title:
            out_lines.append(f"> [!TIP] {title}")
        if body_text:
            out_lines.append(f"> {body_text}")
        out_lines.append("")
        return

    # fallback
    text = _parse_mdx_inline(child_expr)
    if text:
        out_lines.append(text)
        out_lines.append("")


def parse_chunk_to_markdown(chunk_js: str, source_url: str, fallback_title: str) -> Tuple[str, str]:
    title = fallback_title
    slug_hint = ""
    module_scan_text = chunk_js or ""
    meta_match = re.search(r"JSON\.parse\('([^']+)'\)", chunk_js)
    if meta_match:
        raw = meta_match.group(1)
        meta_json = decode_js_string(raw)
        try:
            meta = json.loads(meta_json)
            if isinstance(meta, dict) and meta.get("title"):
                title = str(meta.get("title"))
            if isinstance(meta, dict) and isinstance(meta.get("id"), str):
                slug_hint = str(meta.get("id"))
        except Exception:
            pass

        meta_pos = meta_match.start()
        module_match = None
        for m in re.finditer(r"(\d+)\([^)]*\)\{", chunk_js or ""):
            if m.start() < meta_pos:
                module_match = m
            else:
                break
        if module_match:
            brace_idx = (chunk_js or "").find("{", module_match.end() - 1)
            module_body, _ = _find_balanced(chunk_js or "", brace_idx, "{", "}")
            if module_body:
                module_scan_text = module_body[1:-1]

    def _extract_from_function_block(fn_block: str) -> List[str]:
        out: List[str] = []
        children_anchor = fn_block.find("children:[")
        if children_anchor < 0:
            return out
        arr_start = fn_block.find("[", children_anchor)
        if arr_start < 0:
            return out
        arr_text, _ = _find_balanced(fn_block, arr_start, "[", "]")
        if not arr_text:
            return out
        items = _split_top_level_commas(arr_text[1:-1])
        for item in items:
            _parse_mdx_block_expr(item, out)
        return out

    def _find_function_blocks(js_text: str) -> List[str]:
        blocks: List[str] = []
        for m in re.finditer(r"function\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*\(", js_text):
            brace_idx = js_text.find("{", m.end())
            if brace_idx < 0:
                continue
            body, _ = _find_balanced(js_text, brace_idx, "{", "}")
            if not body:
                continue
            blocks.append(js_text[m.start() : brace_idx] + body)
        return blocks

    candidate_lines: List[List[str]] = []
    default_fn_name = ""
    m_default = re.search(r"default:\(\)=>\s*([A-Za-z_$][A-Za-z0-9_$]*)", module_scan_text or "")
    if m_default:
        default_fn_name = m_default.group(1)

    for fn_block in _find_function_blocks(module_scan_text or ""):
        if "children:[" not in fn_block:
            continue
        lines = _extract_from_function_block(fn_block)
        if lines:
            score = len("\n".join(lines))
            if default_fn_name and re.search(rf"function\s+{re.escape(default_fn_name)}\s*\(", fn_block):
                score += 300
            if slug_hint and slug_hint in fn_block:
                score += 120
            candidate_lines.append((score, lines))

    body_lines: List[str] = []
    if candidate_lines:
        candidate_lines.sort(key=lambda x: x[0], reverse=True)
        body_lines = candidate_lines[0][1]

    merged: List[str] = []
    prev_blank = True
    for ln in body_lines:
        s = (ln or "").strip()
        if not s:
            if not prev_blank:
                merged.append("")
            prev_blank = True
            continue
        merged.append(s)
        prev_blank = False

    body_markdown = "\n".join(merged).strip()
    if not body_markdown:
        body_markdown = "(无法从页面静态 HTML 提取到完整正文。)"
    return title, build_source_markdown(title=title, source_url=source_url, body_markdown=body_markdown)


def parse_main_route_chunk_map(main_js: str) -> Dict[str, str]:
    """
    Return slug -> chunk id map (numeric webpack chunk id).
    """
    route_map: Dict[str, str] = {}
    pat = re.compile(
        r'(?:\"?[a-f0-9]+\"?)\s*:\s*\[\(\)=>(?P<loader>.*?)\s*,\s*"@site/docs/(?P<doc>[^"]+?)\.(?:mdx|md)"\s*,',
        re.I | re.S,
    )
    for m in pat.finditer(main_js or ""):
        loader = m.group("loader") or ""
        ids = re.findall(r'n\.e\("([0-9]+)"\)', loader)
        if not ids:
            continue
        chunk_id = ids[-1]
        doc_path = (m.group("doc") or "").strip("/")
        if not doc_path:
            continue
        slug = doc_path.lower()
        if slug == "index":
            route_map["index"] = chunk_id
        elif slug.endswith("/index"):
            route_map[slug[:-6]] = chunk_id
            route_map[slug] = chunk_id
        else:
            route_map[slug] = chunk_id
    return route_map


def parse_runtime_chunk_url_map(runtime_js: str, base_url: str) -> Dict[str, str]:
    """
    Return chunk_id -> full chunk url.
    """
    out: Dict[str, str] = {}
    text = runtime_js or ""
    idx = text.find("n.u=")
    if idx < 0:
        return out

    first_obj_start = text.find("{", idx)
    if first_obj_start < 0:
        return out
    first_obj_text, first_obj_end = _find_balanced(text, first_obj_start, "{", "}")
    if not first_obj_text:
        return out
    second_obj_start = text.find("{", first_obj_end + 1)
    if second_obj_start < 0:
        return out
    second_obj_text, _ = _find_balanced(text, second_obj_start, "{", "}")
    if not second_obj_text:
        return out

    names = dict(re.findall(r'([0-9]+)\s*:\s*["\']([^"\']+)["\']', first_obj_text))
    hashes = dict(re.findall(r'([0-9]+)\s*:\s*["\']([^"\']+)["\']', second_obj_text))
    for chunk_id, h in hashes.items():
        if not h:
            continue
        name = names.get(chunk_id) or chunk_id
        out[chunk_id] = urllib.parse.urljoin(base_url, f"/assets/js/{name}.{h}.js")
    return out


# -----------------------------
# workspace / config
# -----------------------------
def default_config() -> dict:
    return {
        "target": {
            "base_url": "https://docs.example.com",
            "seed_path": "",
            "docs_root": "",
        },
        "workspace": ".site_sync",
        "crawler": {
            "timeout_seconds": 30,
            "http_retries": 3,
            "retry_backoff_seconds": 1.5,
            "continue_on_doc_error": True,
            "max_docs": 0,
        },
        "site": {
            "output_dir": "source_site",
            "menu_mode": "nav"
        }
    }


def workspace_paths(config: dict) -> dict:
    ws = Path(config.get("workspace") or ".site_sync").resolve()
    return {
        "ws": ws,
        "raw": ws / "raw",
        "source_md": ws / "source_md",
        "manifests": ws / "manifests",
        "reports": ws / "reports",
        "state": ws / "state" / "state.json",
        "source_manifest": ws / "manifests" / "source_manifest.json",
        "source_site": ws / (config.get("site", {}).get("output_dir") or "source_site"),
    }


def ensure_workspace(paths: dict) -> None:
    for key in ["ws", "raw", "source_md", "manifests", "reports", "state"]:
        p = paths[key]
        ensure_dir(p if key != "state" else p.parent)


# -----------------------------
# crawl
# -----------------------------
def _collect_sidebar_meta_for_root(
    *,
    base_url: str,
    docs_root: str,
    seed_html: str,
    timeout: int,
    retries: int,
    backoff: float,
) -> dict:
    sidebar = extract_sidebar_structure(seed_html, docs_root=docs_root)
    categories: Dict[str, dict] = dict(sidebar.get("categories") or {})
    ordered_keys: List[str] = list(sidebar.get("ordered_keys") or [])
    slug_titles: Dict[str, str] = dict(sidebar.get("slug_titles") or {})
    slug_meta: Dict[str, dict] = {}

    for slug, key in (sidebar.get("slug_to_category") or {}).items():
        c = categories.get(key) or {}
        slug_meta[slug] = {
            "title": slug_titles.get(slug) or c.get("label") or slug,
            "category_key": key,
            "category_order": c.get("order", 9999),
            "category_label": c.get("label") or "Uncategorized",
            "category_folder": c.get("folder") or sanitize_segment(c.get("label") or "Uncategorized"),
            "doc_order": c.get("order", 9999),
        }

    for key in ordered_keys:
        c = categories.get(key) or {}
        if c.get("kind") != "category":
            continue
        cat_slug = key[len("cat:") :] if key.startswith("cat:") else ""
        if not cat_slug:
            continue
        cat_url = urllib.parse.urljoin(base_url, f"{docs_root.rstrip('/')}/category/{cat_slug}")
        try:
            page_html = http_get_text(cat_url, timeout=timeout, retries=retries, backoff_seconds=backoff)
        except Exception:
            continue
        entries = parse_doc_entries_from_category_page(page_html, docs_root=docs_root)
        for idx, (slug, label) in enumerate(entries):
            slug_meta[slug] = {
                "title": label or slug,
                "category_key": key,
                "category_order": c.get("order", 9999),
                "category_label": c.get("label") or "Uncategorized",
                "category_folder": c.get("folder") or sanitize_segment(c.get("label") or "Uncategorized"),
                "doc_order": idx,
            }

    try:
        sitemap_url = urllib.parse.urljoin(base_url, "/sitemap.xml")
        sitemap_xml = http_get_text(sitemap_url, timeout=timeout, retries=1, backoff_seconds=0)
        for slug in parse_doc_slugs_from_sitemap(sitemap_xml, docs_root=docs_root):
            slug_meta.setdefault(
                slug,
                {
                    "title": slug,
                    "category_key": "uncategorized",
                    "category_order": 9999,
                    "category_label": "Uncategorized",
                    "category_folder": "uncategorized",
                    "doc_order": 10**9,
                },
            )
    except Exception:
        pass

    return {
        "categories": categories,
        "ordered_keys": ordered_keys,
        "slug_meta": slug_meta,
    }


def collect_sidebar_meta(config: dict) -> dict:
    target = config.get("target", {}) if isinstance(config.get("target"), dict) else {}
    crawler = config.get("crawler", {}) if isinstance(config.get("crawler"), dict) else {}
    base_url = (target.get("base_url") or "").strip()
    configured_docs_root = (target.get("docs_root") or "").strip()
    seed_path = target.get("seed_path", "")
    timeout = int(crawler.get("timeout_seconds", 30) or 30)
    retries = int(crawler.get("http_retries", 3) or 3)
    backoff = float(crawler.get("retry_backoff_seconds", 1.5) or 1.5)

    if not base_url:
        raise ValueError("target.base_url is required")

    seed_url = urllib.parse.urljoin(base_url, seed_path)
    seed_html = ""
    try:
        seed_html = http_get_text(seed_url, timeout=timeout, retries=retries, backoff_seconds=backoff)
    except Exception:
        seed_html = ""

    docs_roots, root_home_html_map = resolve_docs_roots(
        base_url=base_url,
        configured_docs_root=configured_docs_root,
        seed_html=seed_html,
        timeout=timeout,
        retries=retries,
        backoff=backoff,
    )

    multi_root_mode = (not configured_docs_root) and len(docs_roots) > 1
    categories: Dict[str, dict] = {}
    ordered_keys: List[str] = []
    slug_meta: Dict[str, dict] = {}

    for root_index, docs_root in enumerate(docs_roots):
        root_html = root_home_html_map.get(docs_root) or seed_html
        root_data = _collect_sidebar_meta_for_root(
            base_url=base_url,
            docs_root=docs_root,
            seed_html=root_html,
            timeout=timeout,
            retries=retries,
            backoff=backoff,
        )

        root_tag = sanitize_segment(docs_root.strip("/") or f"docs{root_index + 1}", fallback=f"docs{root_index + 1}")
        category_key_map: Dict[str, str] = {}

        for old_key, category in (root_data.get("categories") or {}).items():
            new_key = old_key if not multi_root_mode else f"root:{docs_root.strip('/')}:{old_key}"
            category_key_map[old_key] = new_key
            new_category = dict(category or {})
            new_category["key"] = new_key
            if multi_root_mode:
                new_category["folder"] = sanitize_segment(f"{root_tag}_{new_category.get('folder') or new_category.get('label') or 'uncategorized'}")
            categories[new_key] = new_category

        for old_key in (root_data.get("ordered_keys") or []):
            mapped = category_key_map.get(old_key) or old_key
            if mapped not in ordered_keys:
                ordered_keys.append(mapped)

        for rel_slug, meta in (root_data.get("slug_meta") or {}).items():
            compound_slug = make_compound_slug(docs_root, rel_slug, multi_root_mode)
            old_category_key = (meta or {}).get("category_key") or "uncategorized"
            mapped_category_key = category_key_map.get(old_category_key, old_category_key)
            mapped_category = categories.get(mapped_category_key) or {}
            source_path = make_source_path(docs_root, rel_slug)
            slug_meta[compound_slug] = {
                "title": (meta or {}).get("title") or rel_slug,
                "category_key": mapped_category_key,
                "category_order": (meta or {}).get("category_order", 9999),
                "category_label": (meta or {}).get("category_label") or mapped_category.get("label") or "Uncategorized",
                "category_folder": mapped_category.get("folder") or (meta or {}).get("category_folder") or "uncategorized",
                "doc_order": (meta or {}).get("doc_order", 10**9),
                "source_root": docs_root,
                "source_path": source_path,
                "route_keys": make_route_candidates(rel_slug, compound_slug),
            }

    primary_docs_root = docs_roots[0] if docs_roots else ("/" + (configured_docs_root or "docs").strip("/"))
    return {
        "base_url": base_url,
        "docs_root": primary_docs_root,
        "docs_roots": docs_roots,
        "multi_root_mode": multi_root_mode,
        "categories": categories,
        "ordered_keys": ordered_keys,
        "slug_meta": slug_meta,
    }

def crawl_once(config: dict, max_docs: int = 0) -> dict:
    paths = workspace_paths(config)
    ensure_workspace(paths)

    target = config.get("target", {}) if isinstance(config.get("target"), dict) else {}
    crawler = config.get("crawler", {}) if isinstance(config.get("crawler"), dict) else {}
    base_url = (target.get("base_url") or "").strip()
    docs_root = (target.get("docs_root") or "").strip()
    timeout = int(crawler.get("timeout_seconds", 30) or 30)
    retries = int(crawler.get("http_retries", 3) or 3)
    backoff = float(crawler.get("retry_backoff_seconds", 1.5) or 1.5)
    continue_on_doc_error = bool(crawler.get("continue_on_doc_error", True))

    if not base_url:
        raise ValueError("target.base_url is required")

    heartbeat = ProgressHeartbeat("crawl", interval_seconds=8).start()
    heartbeat.update(phase="读取源站导航", step="collect_sidebar_meta", url=base_url)
    sidebar = collect_sidebar_meta(config)
    docs_root = (sidebar.get("docs_root") or docs_root).strip() or docs_root
    slug_meta: Dict[str, dict] = sidebar.get("slug_meta", {})

    route_chunk_map: Dict[str, str] = {}
    chunk_url_map: Dict[str, str] = {}
    chunk_text_cache: Dict[str, str] = {}
    try:
        heartbeat.update(phase="分析页面脚本", step="discover_bundles", url=urllib.parse.urljoin(base_url, docs_root))
        discover_url = urllib.parse.urljoin(base_url, docs_root)
        discover_html = http_get_text(discover_url, timeout=timeout, retries=retries, backoff_seconds=backoff)
        script_urls = parse_script_src_urls(discover_html, base_url)
        runtime_js_url = next((u for u in script_urls if "/assets/js/runtime~main." in u), "")
        main_js_url = next((u for u in script_urls if "/assets/js/main." in u), "")
        log(f"[*] 脚本发现：runtime={runtime_js_url or '-'}")
        log(f"[*] 脚本发现：main={main_js_url or '-'}")

        if main_js_url:
            heartbeat.update(step="load_main_bundle", url=main_js_url)
            main_js = http_get_text(main_js_url, timeout=timeout, retries=retries, backoff_seconds=backoff)
            route_chunk_map = parse_main_route_chunk_map(main_js)
            log(f"[*] 路由块映射数量：{len(route_chunk_map)}")
        else:
            log("[!] 未找到 main bundle，文档正文回退能力将受限。")

        if runtime_js_url:
            heartbeat.update(step="load_runtime_bundle", url=runtime_js_url)
            runtime_js = http_get_text(runtime_js_url, timeout=timeout, retries=retries, backoff_seconds=backoff)
            chunk_url_map = parse_runtime_chunk_url_map(runtime_js, base_url)
            log(f"[*] 资源块映射数量：{len(chunk_url_map)}")
        else:
            log("[!] 未找到 runtime bundle，文档正文回退能力将受限。")
    except Exception as bundle_error:
        log(f"[!] 分析页面脚本失败，继续使用 HTML 抽取：{bundle_error}")

    slugs = list(slug_meta.keys())
    slugs.sort(
        key=lambda s: (
            int((slug_meta.get(s) or {}).get("category_order", 9999)),
            int((slug_meta.get(s) or {}).get("doc_order", 10**9)),
            s,
        )
    )

    requested_max = int(max_docs or crawler.get("max_docs", 0) or 0)
    if requested_max > 0:
        slugs = slugs[:requested_max]

    log(f"[*] 开始抓取：target={base_url} docs_root={docs_root}")
    log(f"[*] 待处理文档数：{len(slugs)}")

    prev_manifest = load_json(paths["source_manifest"], default={})
    prev_docs = {x.get("slug"): x for x in (prev_manifest.get("docs") or []) if x.get("slug")}

    docs_payload: List[dict] = []
    changed_slugs: List[str] = []
    failed: List[str] = []

    total = len(slugs)
    crawl_started_at = time.time()
    for idx, slug in enumerate(slugs, start=1):
        m = slug_meta.get(slug) or {}
        source_url = build_doc_source_url_with_meta(base_url, docs_root, slug, m)
        heartbeat.update(phase="抓取文档", step=slug, url=source_url, done=idx - 1, total=total)
        log(f"[{idx}/{total}] 开始抓取 slug={slug}")
        log(f"          URL={source_url}")
        doc_t0 = time.time()
        try:
            page_html = http_get_text(source_url, timeout=timeout, retries=retries, backoff_seconds=backoff)
            title, body_md = extract_doc_from_html(page_html, slug)
            if not title or title.strip() == slug:
                title = m.get("title") or slug

            markdown = build_source_markdown(title=title, source_url=source_url, body_markdown=body_md)

            if is_thin_body_markdown(body_md):
                route_keys = m.get("route_keys") if isinstance(m.get("route_keys"), list) else []
                if not route_keys:
                    slug_key = (slug or "").strip("/").lower() or "index"
                    route_keys = [slug_key, slug_key.rstrip("/"), slug_key.removesuffix(".html")]
                chunk_id = ""
                for rk in route_keys:
                    rk_key = (rk or "").strip("/").lower()
                    if not rk_key:
                        continue
                    chunk_id = route_chunk_map.get(rk_key) or route_chunk_map.get(rk_key.rstrip("/"))
                    if chunk_id:
                        break
                if chunk_id:
                    chunk_url = chunk_url_map.get(chunk_id)
                    if chunk_url:
                        log(f"    [回退] HTML 内容偏薄，尝试 JS 块提取 slug={slug} chunk_id={chunk_id}")
                        chunk_js = chunk_text_cache.get(chunk_url)
                        if chunk_js is None:
                            chunk_js = http_get_text(chunk_url, timeout=timeout, retries=retries, backoff_seconds=backoff)
                            chunk_text_cache[chunk_url] = chunk_js
                        chunk_title, chunk_markdown = parse_chunk_to_markdown(chunk_js, source_url, title)
                        if len(chunk_markdown) > int(len(markdown) * 1.20):
                            markdown = chunk_markdown
                            title = chunk_title or title
                            log(f"    [回退成功] slug={slug} 使用 JS 块正文，长度 {len(markdown)}")
                        else:
                            log(f"    [回退跳过] slug={slug} JS 块内容未明显提升，保留 HTML 抽取结果")
                    else:
                        log(f"    [回退跳过] slug={slug} 找到 chunk_id={chunk_id}，但无 chunk URL 映射")
                else:
                    log(f"    [回退跳过] slug={slug} 未找到对应 chunk_id")

            source_file = slug_to_source_filename(slug)
            write_text(paths["source_md"] / source_file, markdown)

            source_hash = str(abs(hash(markdown)))
            prev_hash = str((prev_docs.get(slug) or {}).get("source_hash", ""))
            if source_hash != prev_hash:
                changed_slugs.append(slug)

            docs_payload.append(
                {
                    "slug": slug,
                    "title": title,
                    "category_key": m.get("category_key") or "uncategorized",
                    "category_label": m.get("category_label") or "Uncategorized",
                    "category_order": m.get("category_order", 9999),
                    "category_folder": m.get("category_folder") or "uncategorized",
                    "source_file": source_file,
                    "source_url": source_url,
                    "source_hash": source_hash,
                    "updated_at": now_iso(),
                }
            )
            heartbeat.update(done=idx, total=total)
            elapsed = time.time() - doc_t0
            log(f"    [完成] slug={slug} title={title} 耗时={elapsed:.2f}s")
            if idx % 5 == 0 or idx == total:
                log_progress_line("    [抓取进度]", idx, total, crawl_started_at)
                log(f"    [统计] 成功={len(docs_payload)} 失败={len(failed)}")
        except Exception as e:
            failed.append(f"{slug}: {e}")
            heartbeat.update(done=idx, total=total)
            elapsed = time.time() - doc_t0
            log(f"    [失败] slug={slug} 耗时={elapsed:.2f}s err={e}")
            if idx % 5 == 0 or idx == total:
                log_progress_line("    [抓取进度]", idx, total, crawl_started_at)
                log(f"    [统计] 成功={len(docs_payload)} 失败={len(failed)}")
            if continue_on_doc_error:
                continue
            raise

    heartbeat.update(phase="写入清单与报告", step="source_manifest", url="-", done=total, total=total)

    now_slugs = {d["slug"] for d in docs_payload}
    removed_docs = [
        {
            "slug": d.get("slug"),
            "title": d.get("title") or d.get("slug"),
            "category_label": d.get("category_label") or "Uncategorized",
            "source_file": d.get("source_file") or "",
        }
        for d in (prev_manifest.get("docs") or [])
        if (d.get("slug") and d.get("slug") not in now_slugs)
    ]

    manifest = {
        "generated_at": now_iso(),
        "seed_url": urllib.parse.urljoin(base_url, target.get("seed_path", "")),
        "counts": {
            "total_docs": len(docs_payload),
            "changed_docs": len(changed_slugs),
            "removed_docs": len(removed_docs),
            "failed_docs": len(failed),
        },
        "changed_docs": changed_slugs,
        "removed_docs": removed_docs,
        "docs": docs_payload,
    }
    save_json(paths["source_manifest"], manifest)

    state = {
        "version": "1.0",
        "last_run": now_iso(),
        "pipeline": "crawl-only",
        "target": {
            "base_url": base_url,
            "docs_root": docs_root,
            "docs_roots": sidebar.get("docs_roots") or [docs_root],
        },
        "counts": manifest["counts"],
    }
    save_json(paths["state"], state)

    report_lines = [
        "# Crawl Report",
        "",
        f"- generated_at: `{manifest['generated_at']}`",
        f"- target: `{base_url}`",
        f"- docs_root: `{docs_root}`",
        f"- total_docs: `{manifest['counts']['total_docs']}`",
        f"- changed_docs: `{manifest['counts']['changed_docs']}`",
        f"- removed_docs: `{manifest['counts']['removed_docs']}`",
        f"- failed_docs: `{manifest['counts']['failed_docs']}`",
        "",
    ]
    if failed:
        report_lines += ["## Failed", ""] + [f"- {x}" for x in failed]
    write_text(paths["reports"] / "crawl-report.md", "\n".join(report_lines).rstrip() + "\n")

    result = {
        "processed": len(slugs),
        "changed": len(changed_slugs),
        "removed": len(removed_docs),
        "failed": len(failed),
        "source_manifest": str(paths["source_manifest"]),
        "source_dir": str(paths["source_md"]),
        "failed_docs": failed,
    }
    log(f"[*] 抓取阶段结束 processed={result['processed']} changed={result['changed']} removed={result['removed']} failed={result['failed']}")
    heartbeat.stop()
    return result


# -----------------------------
# site build
# -----------------------------
def title_sort_key(title: str) -> tuple:
    text = (title or "").strip()
    m = re.match(r"^(\d+(?:\.\d+)*)", text)
    if not m:
        return (1, [10**9], text.lower())
    nums = [int(x) for x in m.group(1).split(".")]
    return (0, nums, text.lower())


def build_site_data(docs: Sequence[dict], sidebar_meta: dict, strict_nav: bool = True) -> List[dict]:
    grouped: Dict[str, dict] = {}
    categories_meta: Dict[str, dict] = sidebar_meta.get("categories") or {}
    slug_meta: Dict[str, dict] = sidebar_meta.get("slug_meta") or {}

    for item in docs:
        slug = item.get("slug") or ""
        if not slug:
            continue

        doc_meta = slug_meta.get(slug) or {}
        if strict_nav and not doc_meta:
            continue

        category_key = doc_meta.get("category_key") or item.get("category_key") or "uncategorized"
        category = categories_meta.get(category_key) or {}
        label = category.get("label") or doc_meta.get("category_label") or item.get("category_label") or "Uncategorized"
        folder = sanitize_segment(category.get("folder") or doc_meta.get("category_folder") or item.get("category_folder") or "uncategorized")
        order = doc_meta.get("category_order") if isinstance(doc_meta.get("category_order"), int) else item.get("category_order")

        if category_key not in grouped:
            grouped[category_key] = {
                "key": category_key,
                "label": label,
                "folder": folder,
                "order": order if isinstance(order, int) else 9999,
                "docs": [],
            }

        rel_path = f"docs/{folder}/{slug}.md"
        grouped[category_key]["docs"].append(
            {
                "slug": slug,
                "title": doc_meta.get("title") or item.get("title") or slug,
                "source_url": item.get("source_url") or "",
                "source_file": item.get("source_file") or "",
                "rel_path": rel_path,
                "doc_order": doc_meta.get("doc_order") if isinstance(doc_meta.get("doc_order"), int) else 10**9,
            }
        )

    if strict_nav:
        for key, c in categories_meta.items():
            if c.get("kind") != "category":
                continue
            if key in grouped:
                continue
            label = c.get("label") or "Uncategorized"
            grouped[key] = {
                "key": key,
                "label": label,
                "folder": sanitize_segment(c.get("folder") or label),
                "order": c.get("order") if isinstance(c.get("order"), int) else 9999,
                "docs": [],
            }

    categories = list(grouped.values())
    categories.sort(key=lambda x: (x.get("order", 9999), str(x.get("label") or "")))
    for cat in categories:
        cat["docs"].sort(key=lambda d: (d.get("doc_order", 10**9), title_sort_key(d.get("title") or d.get("slug") or "")))
    return categories


def build_index_html(site_payload: dict) -> str:
    payload_json = json.dumps(site_payload, ensure_ascii=False)
    template = """<!doctype html>
<html lang="zh-CN">
<head>
  <meta charset="utf-8" />
  <meta name="viewport" content="width=device-width, initial-scale=1" />
  <title>Docs Viewer</title>
  <style>
    * { box-sizing: border-box; }
    body { margin:0; font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,'Microsoft YaHei',sans-serif; color:#111827; background:#f8fafc; }
    .top { position:sticky; top:0; z-index:20; padding:10px 14px; background:#111827; color:#f9fafb; border-bottom:1px solid #0b1220; }
    .top .title { font-size:16px; font-weight:600; }
    .top .meta { margin-top:4px; font-size:12px; opacity:.92; }
    .layout { display:grid; grid-template-columns:320px 1fr; height:calc(100vh - 62px); min-height:560px; }
    .sidebar { border-right:1px solid #e5e7eb; background:#fff; overflow:auto; padding:10px; }
    .content { padding:16px; overflow:auto; }
    .search { position:sticky; top:0; z-index:5; background:#fff; padding-bottom:10px; }
    .search input { width:100%; border:1px solid #d1d5db; border-radius:8px; padding:8px 10px; font-size:13px; outline:none; }
    .search input:focus { border-color:#2563eb; box-shadow:0 0 0 2px rgba(37,99,235,.12); }
    details { border:1px solid #eef2f7; border-radius:8px; margin-bottom:10px; background:#fff; }
    summary { cursor:pointer; font-weight:600; font-size:13px; padding:8px 10px; background:#f9fafb; border-radius:8px; }
    .doc-list { padding:6px 4px 8px 4px; }
    .doc-link { display:block; text-decoration:none; color:#1f2937; font-size:13px; line-height:1.45; border-radius:6px; padding:6px 8px; margin:2px 0; }
    .doc-link:hover { background:#eef2ff; }
    .doc-link.active { background:#dbeafe; color:#1e40af; font-weight:600; }
    .article { max-width:1100px; margin:0 auto; background:#fff; border:1px solid #e5e7eb; border-radius:10px; overflow:hidden; box-shadow:0 2px 10px rgba(15,23,42,.04); }
    .article-top { padding:14px 16px; border-bottom:1px solid #e5e7eb; background:#f9fafb; }
    .article-title { margin:0; font-size:20px; line-height:1.35; }
    .article-meta { margin-top:6px; font-size:12px; color:#4b5563; }
    .article-meta a { color:#2563eb; text-decoration:none; }
    .article-meta a:hover { text-decoration:underline; }
    .markdown { padding:16px; white-space:normal; line-height:1.72; color:#111827; }
    .markdown h1,.markdown h2,.markdown h3,.markdown h4 { margin:1.2em 0 .6em; }
    .markdown p { margin:.7em 0; }
    .markdown ul { padding-left:1.4em; }
    .markdown blockquote { margin:.8em 0; border-left:4px solid #cbd5e1; background:#f8fafc; color:#334155; padding:.6em .9em; }
    .markdown pre { background:#0f172a; color:#e5e7eb; border-radius:8px; padding:12px; overflow:auto; font-size:13px; line-height:1.5; }
    .markdown code { background:#f3f4f6; border:1px solid #e5e7eb; border-radius:4px; padding:1px 5px; font-size:.92em; }
    .markdown pre code { background:transparent; border:none; padding:0; color:inherit; }
    .empty { max-width:1100px; margin:0 auto; background:#fff; border:1px dashed #cbd5e1; border-radius:10px; padding:24px; color:#64748b; text-align:center; }
    @media (max-width:980px) { .layout { grid-template-columns:1fr; height:auto; } .sidebar { height:48vh; border-right:none; border-bottom:1px solid #e5e7eb; } .content { min-height:52vh; } }
  </style>
</head>
<body>
  <div class="top"><div class="title">Docs Viewer</div><div class="meta" id="topMeta">loading...</div></div>
  <div class="layout">
    <aside class="sidebar"><div class="search"><input id="q" placeholder="Search title / slug..." /></div><div id="nav"></div></aside>
    <main class="content" id="content"><div class="empty">Select a document from the left panel.</div></main>
  </div>

  <script id="siteData" type="application/json">__SITE_DATA_JSON__</script>
  <script>
    const SITE_DATA = JSON.parse(document.getElementById("siteData").textContent);
    const navEl = document.getElementById("nav");
    const qEl = document.getElementById("q");
    const contentEl = document.getElementById("content");
    const topMetaEl = document.getElementById("topMeta");
    const categories = SITE_DATA.categories || [];

    topMetaEl.textContent = `docs ${SITE_DATA.counts?.docs || 0} | categories ${SITE_DATA.counts?.categories || 0} | generated ${SITE_DATA.generated_at || '-'}`;

    function esc(s) {
      return (s || "").replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
    }

    function inlineMd(text) {
      let out = esc(text);
      out = out.replace(/`([^`]+)`/g, "<code>$1</code>");
      out = out.replace(/\\*\\*([^*]+)\\*\\*/g, "<strong>$1</strong>");
      out = out.replace(/\\*([^*]+)\\*/g, "<em>$1</em>");
      out = out.replace(/\\[([^\\]]+)\\]\\(([^\\)]+)\\)/g, '<a href="$2" target="_blank" rel="noreferrer">$1</a>');
      return out;
    }

    function renderMarkdown(mdText) {
      const lines = (mdText || "").replace(/\\r\\n/g, "\\n").split("\\n");
      const html = [];
      let inCode = false;
      let codeBuf = [];
      let inList = false;
      const flushList = () => { if (inList) { html.push("</ul>"); inList = false; } };
      const flushCode = () => { if (inCode) { html.push("<pre><code>" + esc(codeBuf.join("\\n")) + "</code></pre>"); inCode = false; codeBuf = []; } };
      for (const rawLine of lines) {
        const line = rawLine ?? "";
        if (line.trim().startsWith("```")) {
          if (!inCode) { flushList(); inCode = true; codeBuf = []; } else { flushCode(); }
          continue;
        }
        if (inCode) { codeBuf.push(line); continue; }
        if (!line.trim()) { flushList(); html.push(""); continue; }
        const h = line.match(/^(#{1,6})\\s+(.*)$/);
        if (h) { flushList(); const lv = h[1].length; html.push(`<h${lv}>${inlineMd(h[2])}</h${lv}>`); continue; }
        if (line.startsWith(">")) { flushList(); html.push(`<blockquote>${inlineMd(line.replace(/^>\\s?/, ""))}</blockquote>`); continue; }
        const ul = line.match(/^\\s*[-*]\\s+(.*)$/);
        if (ul) { if (!inList) { html.push("<ul>"); inList = true; } html.push(`<li>${inlineMd(ul[1])}</li>`); continue; }
        flushList();
        html.push(`<p>${inlineMd(line)}</p>`);
      }
      flushList();
      flushCode();
      return html.join("\\n");
    }

    function allDocs() { return categories.flatMap(cat => (cat.docs || []).map(doc => ({...doc, category_label: cat.label}))); }

    function renderNav(keyword = "") {
      const kw = (keyword || "").trim().toLowerCase();
      navEl.innerHTML = "";
      for (const cat of categories) {
        const docs = (cat.docs || []).filter(doc => {
          if (!kw) return true;
          return (doc.title || "").toLowerCase().includes(kw) || (doc.slug || "").toLowerCase().includes(kw);
        });
        if (kw && !docs.length) continue;
        const box = document.createElement("details");
        box.open = true;
        const sum = document.createElement("summary");
        sum.textContent = `${cat.label} (${docs.length})`;
        box.appendChild(sum);
        const list = document.createElement("div");
        list.className = "doc-list";
        for (const doc of docs) {
          const a = document.createElement("a");
          a.href = "#";
          a.className = "doc-link";
          a.dataset.slug = doc.slug;
          a.textContent = doc.title || doc.slug;
          a.onclick = (e) => { e.preventDefault(); loadDoc(doc, a); };
          list.appendChild(a);
        }
        box.appendChild(list);
        navEl.appendChild(box);
      }
    }

    async function loadDoc(doc, anchorEl) {
      if (!doc || !doc.rel_path) return;
      document.querySelectorAll(".doc-link.active").forEach(el => el.classList.remove("active"));
      if (anchorEl) anchorEl.classList.add("active");
      try {
        const resp = await fetch(doc.rel_path + `?t=${Date.now()}`);
        if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
        const md = await resp.text();
        const body = renderMarkdown(md);
        const sourceUrl = doc.source_url ? `<a href="${doc.source_url}" target="_blank" rel="noreferrer">${doc.source_url}</a>` : "-";
        contentEl.innerHTML = `<article class="article"><div class="article-top"><h1 class="article-title">${esc(doc.title || doc.slug)}</h1><div class="article-meta">slug: <code>${esc(doc.slug || "")}</code> | category: ${esc(doc.category_label || "")} | source: ${sourceUrl}</div></div><div class="markdown">${body}</div></article>`;
      } catch (err) {
        contentEl.innerHTML = `<div class="empty">load failed: ${esc(String(err))}</div>`;
      }
    }

    qEl.addEventListener("input", () => renderNav(qEl.value));
    renderNav("");
    const first = allDocs()[0];
    if (first) {
      let firstAnchor = null;
      const links = document.querySelectorAll(".doc-link[data-slug]");
      for (const link of links) {
        if ((link.dataset.slug || "") === (first.slug || "")) {
          firstAnchor = link;
          break;
        }
      }
      loadDoc(first, firstAnchor || null);
    }
  </script>
</body>
</html>
"""
    return template.replace("__SITE_DATA_JSON__", payload_json)


def build_site(config: dict, menu_mode: Optional[str] = None) -> dict:
    paths = workspace_paths(config)
    ensure_workspace(paths)

    manifest = load_json(paths["source_manifest"], default={})
    docs = manifest.get("docs") or []
    if not docs:
        return {"ok": False, "message": f"empty manifest: {paths['source_manifest']}"}

    desired_mode = (menu_mode or config.get("site", {}).get("menu_mode") or "nav").strip().lower()
    strict_nav = desired_mode != "all"
    log(f"[*] 开始构建站点：menu_mode={'nav(仅导航)' if strict_nav else 'all(全部)'}")
    heartbeat = ProgressHeartbeat("build_site", interval_seconds=8).start()
    heartbeat.update(phase="构建目录树", step="build_site_data", url="-")

    sidebar = collect_sidebar_meta(config)
    categories = build_site_data(docs=docs, sidebar_meta=sidebar, strict_nav=strict_nav)

    site_root = paths["source_site"]
    docs_root = site_root / "docs"
    ensure_dir(docs_root)

    copied = 0
    missing: List[str] = []
    total_docs = sum(len(cat.get("docs") or []) for cat in categories)
    log(f"[*] 站点目录统计：分类={len(categories)} 文档={total_docs}")
    build_started_at = time.time()
    copied_attempt = 0
    for cat in categories:
        folder = sanitize_segment(cat.get("folder") or cat.get("label") or "uncategorized")
        for doc in cat.get("docs") or []:
            copied_attempt += 1
            heartbeat.update(
                phase="复制文档到站点目录",
                step=doc.get("slug") or "-",
                url=doc.get("source_url") or "-",
                done=copied_attempt - 1,
                total=total_docs,
            )
            source_file = doc.get("source_file") or ""
            if not source_file:
                missing.append(doc.get("slug") or "")
                continue
            src = paths["source_md"] / source_file
            dst = docs_root / folder / f"{doc.get('slug')}.md"
            if copy_file(src, dst):
                copied += 1
            else:
                missing.append(doc.get("slug") or "")
            heartbeat.update(done=copied_attempt, total=total_docs)
            if copied_attempt % 10 == 0 or copied_attempt == total_docs:
                log_progress_line("    [构建进度]", copied_attempt, total_docs, build_started_at)
                log(f"    [统计] 复制成功={copied} 缺失={len(missing)}")

    manifest_slugs = {d.get("slug") for d in docs if d.get("slug")}
    included_slugs = {d.get("slug") for c in categories for d in (c.get("docs") or []) if d.get("slug")}
    excluded_slugs = sorted([s for s in manifest_slugs if s not in included_slugs])

    payload = {
        "generated_at": now_iso(),
        "menu_mode": "nav" if strict_nav else "all",
        "counts": {
            "manifest_docs": len(manifest_slugs),
            "categories": len(categories),
            "docs": sum(len(c.get("docs") or []) for c in categories),
            "excluded_docs": len(excluded_slugs),
            "copied_docs": copied,
            "missing_docs": len(missing),
        },
        "categories": categories,
    }

    heartbeat.update(phase="写入站点文件", step="site_manifest + index.html", url="-", done=total_docs, total=total_docs)
    save_json(site_root / "site_manifest.json", payload)
    write_text(site_root / "index.html", build_index_html(payload))

    lines = [
        "# Site Build Report",
        "",
        f"- generated_at: `{payload['generated_at']}`",
        f"- output_dir: `{site_root}`",
        f"- menu_mode: `{payload['menu_mode']}`",
        f"- manifest_docs: `{payload['counts']['manifest_docs']}`",
        f"- menu_docs: `{payload['counts']['docs']}`",
        f"- hidden_docs: `{payload['counts']['excluded_docs']}`",
        f"- copied_docs: `{payload['counts']['copied_docs']}`",
        f"- missing_docs: `{payload['counts']['missing_docs']}`",
        "",
        f"- entry: `{site_root / 'index.html'}`",
    ]
    if excluded_slugs:
        lines += ["", "## Hidden Slugs", ""] + [f"- `{s}`" for s in excluded_slugs[:300]]
    if missing:
        lines += ["", "## Missing Slugs", ""] + [f"- `{s}`" for s in missing[:300]]
    write_text(paths["reports"] / "site-build-report.md", "\n".join(lines).rstrip() + "\n")
    heartbeat.stop()
    log(
        f"[*] 构建阶段结束 categories={payload['counts']['categories']} "
        f"menu_docs={payload['counts']['docs']} hidden={payload['counts']['excluded_docs']} missing={payload['counts']['missing_docs']}"
    )

    return {
        "ok": True,
        "site_root": str(site_root),
        "index_html": str(site_root / "index.html"),
        "site_manifest": str(site_root / "site_manifest.json"),
        "counts": payload["counts"],
        "menu_mode": payload["menu_mode"],
    }


# -----------------------------
# command handlers
# -----------------------------
def cmd_init(args) -> int:
    cfg_path = Path(args.config)
    if cfg_path.exists() and not args.force:
        print(f"[!] config exists: {cfg_path}. use --force to overwrite")
        return 1

    save_json(cfg_path, default_config())
    cfg = load_json(cfg_path, default={})
    paths = workspace_paths(cfg)
    ensure_workspace(paths)
    print(f"[+] initialized: {cfg_path}")
    print(f"- workspace: {paths['ws']}")
    return 0


def cmd_crawl(args) -> int:
    cfg = load_json(Path(args.config), default={})
    if not cfg:
        print(f"[!] invalid config: {args.config}")
        return 1
    result = crawl_once(cfg, max_docs=int(args.max_docs or 0))
    log("[+] 抓取命令完成")
    for k in ["processed", "changed", "removed", "failed", "source_dir", "source_manifest"]:
        log(f"- {k}: {result.get(k)}")
    if result.get("failed_docs"):
        log("- failed_docs:")
        for x in result["failed_docs"][:50]:
            log(f"  - {x}")
    return 0 if (result.get("failed", 0) == 0) else 2


def cmd_build_site(args) -> int:
    cfg = load_json(Path(args.config), default={})
    if not cfg:
        print(f"[!] invalid config: {args.config}")
        return 1
    result = build_site(cfg, menu_mode=args.menu_mode)
    if not result.get("ok"):
        print(f"[!] {result.get('message')}")
        return 1
    log("[+] 站点构建命令完成")
    log(f"- site_root: {result['site_root']}")
    log(f"- index_html: {result['index_html']}")
    log(f"- site_manifest: {result['site_manifest']}")
    log(f"- menu_mode: {result['menu_mode']}")
    log(f"- counts: {result['counts']}")
    return 0


def cmd_run(args) -> int:
    cfg = load_json(Path(args.config), default={})
    if not cfg:
        print(f"[!] invalid config: {args.config}")
        return 1

    log("[*] 阶段 1/2：抓取文档")
    crawl_result = crawl_once(cfg, max_docs=int(args.max_docs or 0))
    log("[*] 阶段 2/2：构建本地站点")
    site_result = build_site(cfg, menu_mode=args.menu_mode)

    log("[+] 任务执行完成")
    log(f"- crawl: processed={crawl_result.get('processed')} changed={crawl_result.get('changed')} failed={crawl_result.get('failed')}")
    if not site_result.get("ok"):
        log(f"- site: failed: {site_result.get('message')}")
        return 2
    log(f"- site: index={site_result.get('index_html')} menu_mode={site_result.get('menu_mode')}")
    return 0 if crawl_result.get("failed", 0) == 0 else 2


def cmd_status(args) -> int:
    cfg = load_json(Path(args.config), default={})
    if not cfg:
        print(f"[!] invalid config: {args.config}")
        return 1
    paths = workspace_paths(cfg)
    state = load_json(paths["state"], default={})
    manifest = load_json(paths["source_manifest"], default={})

    print("Docs Crawler Status")
    print(f"- last_run: {state.get('last_run')}")
    print(f"- target: {((state.get('target') or {}).get('base_url'))}")
    print(f"- docs_root: {((state.get('target') or {}).get('docs_root'))}")
    print(f"- total_docs: {(manifest.get('counts') or {}).get('total_docs', 0)}")
    print(f"- changed_docs: {(manifest.get('counts') or {}).get('changed_docs', 0)}")
    print(f"- removed_docs: {(manifest.get('counts') or {}).get('removed_docs', 0)}")
    print(f"- workspace: {paths['ws']}")
    print(f"- source_dir: {paths['source_md']}")
    print(f"- source_manifest: {paths['source_manifest']}")
    print(f"- site_dir: {paths['source_site']}")
    return 0


def cmd_preview(args) -> int:
    cfg = load_json(Path(args.config), default={})
    if not cfg:
        print(f"[!] invalid config: {args.config}")
        return 1
    paths = workspace_paths(cfg)
    site_dir = paths["source_site"]
    index_file = site_dir / "index.html"
    if not index_file.exists():
        print(f"[!] site not built yet: {index_file}")
        return 1

    port = int(args.port or 8777)
    print(f"[*] preview: http://127.0.0.1:{port}/index.html")
    os.chdir(site_dir)
    handler = http.server.SimpleHTTPRequestHandler
    with socketserver.TCPServer(("127.0.0.1", port), handler) as httpd:
        httpd.serve_forever()


def build_parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(description="Generic docs crawler and local site builder")
    p.add_argument("--config", default=DEFAULT_CONFIG_PATH, help="config file path")

    sub = p.add_subparsers(dest="command", required=True)

    sp_init = sub.add_parser("init", help="initialize config and workspace")
    sp_init.add_argument("--force", action="store_true", help="overwrite existing config")
    sp_init.set_defaults(func=cmd_init)

    sp_crawl = sub.add_parser("crawl", help="crawl docs and write markdown files")
    sp_crawl.add_argument("--max-docs", type=int, help="process only first N docs")
    sp_crawl.set_defaults(func=cmd_crawl)

    sp_build = sub.add_parser("build-site", help="build local HTML viewer from markdown files")
    sp_build.add_argument("--menu-mode", choices=["nav", "all"], help="nav=sidebar docs only, all=all crawled docs")
    sp_build.set_defaults(func=cmd_build_site)

    sp_run = sub.add_parser("run", help="crawl and then build site")
    sp_run.add_argument("--max-docs", type=int, help="process only first N docs")
    sp_run.add_argument("--menu-mode", choices=["nav", "all"], help="nav=sidebar docs only, all=all crawled docs")
    sp_run.set_defaults(func=cmd_run)

    sp_status = sub.add_parser("status", help="show status")
    sp_status.set_defaults(func=cmd_status)

    sp_preview = sub.add_parser("preview", help="serve local site")
    sp_preview.add_argument("--port", type=int, default=8777)
    sp_preview.set_defaults(func=cmd_preview)

    return p


def main(argv: Optional[List[str]] = None) -> int:
    parser = build_parser()
    args = parser.parse_args(argv)
    try:
        return int(args.func(args) or 0)
    except Exception as e:
        print(f"[!] {e}")
        traceback.print_exc()
        return 2


if __name__ == "__main__":
    raise SystemExit(main())